commit b239504cd67083eef55faefa359943c1378498a6
author Hung-ying Tyan <tyanh@chromium.org> Wed Oct 29 15:55:43 2014 +0800
committer chrome-internal-fetch <chrome-internal-fetch@google.com> Tue Nov 04 07:03:55 2014 +0000
tree ffd2c6c2f657ff652e24d130b877a5f2ccbed7e8
parent 37bcc951d328d064255426a10ec6fd855c0eaeb4

Add @retry to flaky calls in chameleon_test

DUT may be slow in discovering newly connected external display. Function
calls (e.g., get_external_connector_name()) to retrieve information of the
display may fail during the discovery. So far, the following errors (wrapped
in xmlrpclib.Fault) are observed in running various tests that make those
calls:
<type 'exceptions.KeyError'>:...
<type 'exceptions.IndexError'>:list index out of range
<class \'autotest_lib.client.common_lib.error.CmdError\'>:Command <DISPLAY=:0 XAUTHORITY=/home/chronos/.Xauthority xrandr > failed, rc=1, Command returned non-zero exit status

This patch adds @retry to these calls to make them robust.

The patch also fixed a bug in display_facade_adapter.wait_for_output() where
it doesn't return the result from the RPC call.

BUG=chromium:408376
TEST=run the following ChameleonTest continuously on nyan_big and squawks and
see errors being caught and the calls being re-tried:

class SimpleTest(chameleon_test.ChameleonTest):
    def run_once(self, host):
        self.reconnect_output()
        self.set_mirrored(True)

Change-Id: Ia4966f45177fd9b562088837a052a5cbf38aeb2f
Reviewed-on: https://chromium-review.googlesource.com/226750
Tested-by: Hung-ying Tyan <tyanh@chromium.org>
Commit-Queue: Hung-ying Tyan <tyanh@chromium.org>
Reviewed-by: Wai-Hong Tam <waihong@chromium.org>

server/cros/chameleon/chameleon_test.py

diff --git a/server/cros/chameleon/chameleon_test.py b/server/cros/chameleon/chameleon_test.py
index 9ca0d2b..8b81d1d 100644
--- a/server/cros/chameleon/chameleon_test.py
+++ b/server/cros/chameleon/chameleon_test.py
@@ -11,6 +11,7 @@
 from PIL import ImageChops
 
 from autotest_lib.client.common_lib import error
+from autotest_lib.client.common_lib.cros import retry
 from autotest_lib.client.cros.chameleon import edid
 from autotest_lib.server import test
 from autotest_lib.server.cros.chameleon import remote_facade_factory
@@ -44,6 +45,9 @@
     _PIXEL_DIFF_VALUE_MARGIN_FOR_ANALOG_SIGNAL = 5
     _PIXEL_DIFF_VALUE_MARGIN_FOR_DIGITAL_SIGNAL = 1
 
+    _FLAKY_CALL_RETRY_TIME_OUT_SEC = 20
+    _FLAKY_CALL_RETRY_DELAY_SEC = 1
+
     def initialize(self, host):
         """Initializes.
 
@@ -200,6 +204,9 @@
         return self.display_facade.is_mirrored_enabled()
 
 
+    @retry.retry(xmlrpclib.Fault,
+                 timeout_min=_FLAKY_CALL_RETRY_TIME_OUT_SEC / 60.0,
+                 delay_sec=_FLAKY_CALL_RETRY_DELAY_SEC)
     def set_mirrored(self, test_mirrored):
         """Sets the external display is in mirrored mode or extended mode
 
@@ -292,17 +299,30 @@
            self.display_facade.connect()
 
 
+    @retry.retry(xmlrpclib.Fault,
+                 timeout_min=_FLAKY_CALL_RETRY_TIME_OUT_SEC / 60.0,
+                 delay_sec=_FLAKY_CALL_RETRY_DELAY_SEC)
+    def wait_for_output(self, output):
+        """Waits for the specified output to be connected.
+
+        @param output: name of the output in a string.
+        @raise error.TestFail if output fails to get connected.
+        """
+        if not self.display_facade.wait_for_output(output):
+            raise error.TestFail('Fail to get %s connected' % output)
+
+
     def reconnect_output(self, unplug_duration_sec=5):
         """Reconnects the output with an unplug followed by a plug.
 
         @param unplug_duration_sec: duration of unplug in second.
         """
         logging.info('Reconnect output...')
-        output = self.display_facade.get_external_connector_name()
+        output = self.get_dut_display_connector()
         self.chameleon_port.unplug()
         time.sleep(unplug_duration_sec)
         self.chameleon_port.plug()
-        self.display_facade.wait_for_output(output)
+        self.wait_for_output(output)
 
 
     def cleanup(self):
@@ -354,6 +374,9 @@
         return None
 
 
+    @retry.retry(xmlrpclib.Fault,
+                 timeout_min=_FLAKY_CALL_RETRY_TIME_OUT_SEC / 60.0,
+                 delay_sec=_FLAKY_CALL_RETRY_DELAY_SEC)
     def get_dut_display_connector(self):
         """Gets the name of the connected display connector of DUT.
 
@@ -371,7 +394,7 @@
         @param timeout: Duration in second to retry checking the connector.
         @raise error.TestFail if the check does not pass.
         """
-        current_connector = self.display_facade.get_external_connector_name()
+        current_connector = self.get_dut_display_connector()
         now = time.time()
         end_time = now + timeout
         while expected_connector != current_connector and now < end_time:

server/cros/chameleon/display_facade_adapter.py

diff --git a/server/cros/chameleon/display_facade_adapter.py b/server/cros/chameleon/display_facade_adapter.py
index a6970dd..9e6ddf5 100644
--- a/server/cros/chameleon/display_facade_adapter.py
+++ b/server/cros/chameleon/display_facade_adapter.py
@@ -116,8 +116,9 @@
         """Waits for the specified output to be connected.
 
         @param output: The output name as a string.
+        @return: True if output is connected; False otherwise.
         """
-        self._display_proxy.wait_output_connected(output)
+        return self._display_proxy.wait_output_connected(output)
 
 
     def hide_cursor(self):