chromite: Move Timeout* functions to chromite.lib.timeout_util.
BUG=chromium:323675
CQ-DEPEND=CL:178733
TEST=cbuildbot --buildbot --debug --remote runs on x86-alex-paladin,
x86-mario-paladin, x86-mario-release
TEST=repo grep -E \
'cros_build_lib\.(Timeout|FatalTimeout|TimeoutDecorator|'\
'WaitForCondition|WaitForReturnValue|WaitForTreeStatus|'\
'IsTreeOpen|GetTreeStatus|SubCommandTimeout|Timeout|'\
'WaitForCondition|TreeOpen)'
TEST=all unit tests
Change-Id: Ida5b3b5be04a3b891da0f0c06db832d3d01be972
Reviewed-on: https://chromium-review.googlesource.com/178734
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
Tested-by: David James <davidjames@chromium.org>
Commit-Queue: David James <davidjames@chromium.org>
diff --git a/buildbot/cbuildbot_commands.py b/buildbot/cbuildbot_commands.py
index 723ecba..fc9caaa 100644
--- a/buildbot/cbuildbot_commands.py
+++ b/buildbot/cbuildbot_commands.py
@@ -26,6 +26,7 @@
from chromite.lib import locking
from chromite.lib import osutils
from chromite.lib import parallel
+from chromite.lib import timeout_util
from chromite.scripts import upload_symbols
@@ -1218,10 +1219,10 @@
gs_context = gs.GSContext(acl=acl, dry_run=debug)
try:
- with cros_build_lib.Timeout(timeout):
+ with timeout_util.Timeout(timeout):
gs_context.CopyInto(local_path, upload_url)
- except cros_build_lib.TimeoutError:
- raise cros_build_lib.TimeoutError('Timed out uploading %s' % filename)
+ except timeout_util.TimeoutError:
+ raise timeout_util.TimeoutError('Timed out uploading %s' % filename)
else:
# Update the list of uploaded files.
if update_list:
diff --git a/buildbot/cbuildbot_stages.py b/buildbot/cbuildbot_stages.py
index bed4330..2692b5b 100644
--- a/buildbot/cbuildbot_stages.py
+++ b/buildbot/cbuildbot_stages.py
@@ -47,6 +47,7 @@
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import patch as cros_patch
+from chromite.lib import timeout_util
_FULL_BINHOST = 'FULL_BINHOST'
_PORTAGE_BINHOST = 'PORTAGE_BINHOST'
@@ -273,7 +274,7 @@
commands.UploadArchivedFile(
self.archive_path, self.upload_url, filename, self.debug,
update_list=True, acl=self.acl)
- except (cros_build_lib.RunCommandError, cros_build_lib.TimeoutError) as e:
+ except (cros_build_lib.RunCommandError, timeout_util.TimeoutError) as e:
cros_build_lib.PrintBuildbotStepText('Upload failed')
if strict:
raise
@@ -1571,7 +1572,7 @@
verified by either the Pre-CQ or CQ.
"""
# Submit non-manifest changes if we can.
- if cros_build_lib.IsTreeOpen(
+ if timeout_util.IsTreeOpen(
validation_pool.ValidationPool.STATUS_URL):
pool.SubmitNonManifestChanges(check_tree_open=False)
@@ -2387,7 +2388,7 @@
def PerformStage(self):
if not self.archive_stage.WaitForRecoveryImage():
raise InvalidTestConditionException('Missing recovery image.')
- with cros_build_lib.Timeout(self.SIGNER_TEST_TIMEOUT):
+ with timeout_util.Timeout(self.SIGNER_TEST_TIMEOUT):
commands.RunSignerTests(self._build_root, self._current_board)
@@ -2408,7 +2409,7 @@
extra_env = {}
if self._build_config['useflags']:
extra_env['USE'] = ' '.join(self._build_config['useflags'])
- with cros_build_lib.Timeout(self.UNIT_TEST_TIMEOUT):
+ with timeout_util.Timeout(self.UNIT_TEST_TIMEOUT):
commands.RunUnitTests(self._build_root,
self._current_board,
full=(not self._build_config['quick_unit']),
@@ -2545,7 +2546,7 @@
"""Override and don't set status to FAIL but FORGIVEN instead."""
# Deal with timeout errors specially.
- if isinstance(exception, cros_build_lib.TimeoutError):
+ if isinstance(exception, timeout_util.TimeoutError):
return self._HandleStageTimeoutException(exception)
# 2 for warnings returned by run_suite.py, or CLIENT_HTTP_CODE error
@@ -2581,7 +2582,7 @@
else:
debug = self._options.debug
lab_status.CheckLabStatus(self._current_board)
- with cros_build_lib.Timeout(
+ with timeout_util.Timeout(
self.suite_config.timeout + constants.HWTEST_TIMEOUT_EXTENSION):
commands.RunHWTestSuite(build,
self.suite_config.suite,
diff --git a/buildbot/cbuildbot_stages_unittest.py b/buildbot/cbuildbot_stages_unittest.py
index 99f3ee7..e3209e4 100755
--- a/buildbot/cbuildbot_stages_unittest.py
+++ b/buildbot/cbuildbot_stages_unittest.py
@@ -46,6 +46,7 @@
from chromite.lib import parallel
from chromite.lib import parallel_unittest
from chromite.lib import partial_mock
+from chromite.lib import timeout_util
from chromite.scripts import cbuildbot
# TODO(build): Finish test wrapper (http://crosbug.com/37517).
@@ -725,7 +726,7 @@
# Raise an exception if the user wanted the command to fail.
if timeout:
- m.AndRaise(cros_build_lib.TimeoutError('Timed out'))
+ m.AndRaise(timeout_util.TimeoutError('Timed out'))
cros_build_lib.PrintBuildbotStepFailure()
cros_build_lib.Error(mox.IgnoreArg())
elif returncode != 0:
@@ -1778,7 +1779,7 @@
return_value=committed, autospec=True)
self.PatchObject(gerrit.GerritOnBorgHelper, 'Query',
return_value=my_patches, autospec=True)
- self.PatchObject(cros_build_lib, 'IsTreeOpen', return_value=tree_open,
+ self.PatchObject(timeout_util, 'IsTreeOpen', return_value=tree_open,
autospec=True)
exit_it = itertools.chain([False] * runs, itertools.repeat(True))
self.PatchObject(validation_pool.ValidationPool, 'ShouldExitEarly',
diff --git a/buildbot/cbuildbot_unittest.py b/buildbot/cbuildbot_unittest.py
index 60b4c47..3491b4e 100755
--- a/buildbot/cbuildbot_unittest.py
+++ b/buildbot/cbuildbot_unittest.py
@@ -22,6 +22,7 @@
from chromite.lib import cros_test_lib
from chromite.lib import osutils
from chromite.lib import parallel_unittest
+from chromite.lib import timeout_util
from chromite.scripts import cbuildbot
# TODO(build): Finish test wrapper (http://crosbug.com/37517).
@@ -277,7 +278,7 @@
# Let this test run for a max of 30s; if it takes longer, then it's
# likely that there is an exec loop in the pathways.
- @cros_build_lib.TimeoutDecorator(30)
+ @timeout_util.TimeoutDecorator(30)
def testDepotTools(self):
"""Test that the entry point used by depot_tools works."""
path = os.path.join(constants.SOURCE_ROOT, 'chromite', 'buildbot',
diff --git a/buildbot/validation_pool.py b/buildbot/validation_pool.py
index 4dff12e..5069074 100644
--- a/buildbot/validation_pool.py
+++ b/buildbot/validation_pool.py
@@ -31,6 +31,7 @@
from chromite.lib import gob_util
from chromite.lib import gs
from chromite.lib import patch as cros_patch
+from chromite.lib import timeout_util
# Third-party libraries bundled with chromite need to be listed after the
# first chromite import.
@@ -1052,7 +1053,7 @@
# TimeoutErrors are often flaky.
exc = self.tracebacks[0].exception
if (isinstance(exc, results_lib.StepFailure) and exc.possibly_flaky or
- isinstance(exc, cros_build_lib.TimeoutError)):
+ isinstance(exc, timeout_util.TimeoutError)):
flaky = True
return flaky
@@ -1419,7 +1420,7 @@
time_left = end_time - time.time()
# Wait until the tree opens.
- if check_tree_open and not cros_build_lib.IsTreeOpen(
+ if check_tree_open and not timeout_util.IsTreeOpen(
cls.STATUS_URL, cls.SLEEP_TIMEOUT, timeout=time_left,
throttled_ok=throttled_ok):
raise TreeIsClosedException(closed_or_throttled=not throttled_ok)
@@ -1792,9 +1793,9 @@
dry_run=self.dryrun)
if (check_tree_open and not self.dryrun and not
- cros_build_lib.IsTreeOpen(self.STATUS_URL, self.SLEEP_TIMEOUT,
- timeout=self.MAX_TIMEOUT,
- throttled_ok=throttled_ok)):
+ timeout_util.IsTreeOpen(self.STATUS_URL, self.SLEEP_TIMEOUT,
+ timeout=self.MAX_TIMEOUT,
+ throttled_ok=throttled_ok)):
raise TreeIsClosedException(close_or_throttled=not throttled_ok)
# First, reload all of the changes from the Gerrit server so that we have a
diff --git a/buildbot/validation_pool_unittest.py b/buildbot/validation_pool_unittest.py
index c4a2721..e6e65db 100755
--- a/buildbot/validation_pool_unittest.py
+++ b/buildbot/validation_pool_unittest.py
@@ -34,6 +34,7 @@
from chromite.lib import partial_mock
from chromite.lib import patch as cros_patch
from chromite.lib import patch_unittest
+from chromite.lib import timeout_util
import mock
@@ -90,7 +91,7 @@
self.build_root = 'fakebuildroot'
self.PatchObject(gob_util, 'CreateHttpConn',
side_effect=AssertionError('Test should not contact GoB'))
- self.PatchObject(cros_build_lib, 'IsTreeOpen', return_value=True)
+ self.PatchObject(timeout_util, 'IsTreeOpen', return_value=True)
def MockPatch(self, change_id=None, patch_number=None, is_merged=False,
project='chromiumos/chromite', remote=constants.EXTERNAL_REMOTE,
diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py
index 55dcb48..0d46aec 100644
--- a/lib/cros_build_lib.py
+++ b/lib/cros_build_lib.py
@@ -9,7 +9,6 @@
from email.utils import formatdate
import errno
import functools
-import json
import logging
import os
import re
@@ -19,7 +18,6 @@
import sys
import tempfile
import time
-import urllib
# TODO(build): Fix this.
# This should be absolute import, but that requires fixing all
@@ -1011,137 +1009,6 @@
return NoOpContextManager()
-class TimeoutError(Exception):
- """Raises when code within Timeout has been run too long."""
-
-
-@contextlib.contextmanager
-def Timeout(max_run_time):
- """ContextManager that alarms if code is ran for too long.
-
- Timeout can run nested and raises a TimeoutException if the timeout
- is reached. Timeout can also nest underneath FatalTimeout.
-
- Args:
- max_run_time: Number (integer) of seconds to wait before sending SIGALRM.
- """
- max_run_time = int(max_run_time)
- if max_run_time <= 0:
- raise ValueError("max_run_time must be greater than zero")
-
- # pylint: disable=W0613
- def kill_us(sig_num, frame):
- raise TimeoutError("Timeout occurred- waited %s seconds." % max_run_time)
-
- original_handler = signal.signal(signal.SIGALRM, kill_us)
- previous_time = int(time.time())
-
- # Signal the min in case the leftover time was smaller than this timeout.
- remaining_timeout = signal.alarm(0)
- if remaining_timeout:
- signal.alarm(min(remaining_timeout, max_run_time))
- else:
- signal.alarm(max_run_time)
-
- try:
- yield
- finally:
- # Cancel the alarm request and restore the original handler.
- signal.alarm(0)
- signal.signal(signal.SIGALRM, original_handler)
-
- # Ensure the previous handler will fire if it was meant to.
- if remaining_timeout > 0:
- # Signal the previous handler if it would have already passed.
- time_left = remaining_timeout - (int(time.time()) - previous_time)
- if time_left <= 0:
- signals.RelaySignal(original_handler, signal.SIGALRM, None)
- else:
- signal.alarm(time_left)
-
-
-@contextlib.contextmanager
-def FatalTimeout(max_run_time):
- """ContextManager that exits the program if code is run for too long.
-
- This implementation is fairly simple, thus multiple timeouts
- cannot be active at the same time.
-
- Additionally, if the timeout has elapsed, it'll trigger a SystemExit
- exception within the invoking code, ultimately propagating that past
- itself. If the underlying code tries to suppress the SystemExit, once
- a minute it'll retrigger SystemExit until control is returned to this
- manager.
-
- Args:
- max_run_time: a positive integer.
- """
- max_run_time = int(max_run_time)
- if max_run_time <= 0:
- raise ValueError("max_run_time must be greater than zero")
-
- # pylint: disable=W0613
- def kill_us(sig_num, frame):
- # While this SystemExit *should* crash it's way back up the
- # stack to our exit handler, we do have live/production code
- # that uses blanket except statements which could suppress this.
- # As such, keep scheduling alarms until our exit handler runs.
- # Note that there is a potential conflict via this code, and
- # RunCommand's kill_timeout; thus we set the alarming interval
- # fairly high.
- signal.alarm(60)
- raise SystemExit("Timeout occurred- waited %i seconds, failing."
- % max_run_time)
-
- original_handler = signal.signal(signal.SIGALRM, kill_us)
- remaining_timeout = signal.alarm(max_run_time)
- if remaining_timeout:
- # Restore things to the way they were.
- signal.signal(signal.SIGALRM, original_handler)
- signal.alarm(remaining_timeout)
- # ... and now complain. Unfortunately we can't easily detect this
- # upfront, thus the reset dance above.
- raise Exception("_Timeout cannot be used in parallel to other alarm "
- "handling code; failing")
- try:
- yield
- finally:
- # Cancel the alarm request and restore the original handler.
- signal.alarm(0)
- signal.signal(signal.SIGALRM, original_handler)
-
-
-def TimeoutDecorator(max_time):
- """Decorator used to ensure a func is interrupted if it's running too long."""
- # Save off the built-in versions of time.time, signal.signal, and
- # signal.alarm, in case they get mocked out later. We want to ensure that
- # tests don't accidentally mock out the functions used by Timeout.
- def _Save():
- return time.time, signal.signal, signal.alarm
- def _Restore(values):
- (time.time, signal.signal, signal.alarm) = values
- builtins = _Save()
-
- def NestedTimeoutDecorator(func):
- @functools.wraps(func)
- def TimeoutWrapper(*args, **kwargs):
- new = _Save()
- try:
- _Restore(builtins)
- with Timeout(max_time):
- _Restore(new)
- try:
- func(*args, **kwargs)
- finally:
- _Restore(builtins)
- finally:
- _Restore(new)
-
- return TimeoutWrapper
-
- return NestedTimeoutDecorator
-
-
class ContextManagerStack(object):
"""Context manager that is designed to safely allow nesting and stacking.
@@ -1220,64 +1087,6 @@
raise exc_type, exc, traceback
-def WaitForCondition(*args, **kwargs):
- """Periodically run a function, waiting in between runs.
-
- Continues to run until the function returns True.
-
- Args:
- See WaitForReturnValue([True], ...)
-
- Raises:
- TimeoutError when the timeout is exceeded.
- """
- WaitForReturnValue([True], *args, **kwargs)
-
-
-def WaitForReturnValue(values, func, timeout, period=1, side_effect_func=None,
- func_args=None, func_kwargs=None):
- """Periodically run a function, waiting in between runs.
-
- Continues to run until the function return value is in the list
- of accepted |values|.
-
- Args:
- values: A list or set of acceptable return values from |func|.
- func: The function to run to test for a value.
- timeout: The maximum amount of time to wait, in integer seconds. Minimum
- value: 1.
- period: Integer number of seconds between calls to |func|. Default: 1
- side_effect_func: Optional function to be called between polls of func,
- typically to output logging messages.
- func_args: Optional list of positional arguments to be passed to |func|.
- func_kwargs: Optional dictionary of keyword arguments to be passed to
- |func|.
-
- Returns:
- The value most recently returned by |func|.
-
- Raises:
- TimeoutError when the timeout is exceeded.
- """
- assert period >= 0
- # Timeout requires timeout >= 1.
- assert timeout >= 1
- func_args = func_args or []
- func_kwargs = func_kwargs or {}
- with Timeout(timeout):
- while True:
- timestamp = time.time()
- value = func(*func_args, **func_kwargs)
- if value in values:
- return value
-
- time_remaining = period - int(time.time() - timestamp)
- if time_remaining > 0:
- if side_effect_func:
- side_effect_func()
- time.sleep(time_remaining)
-
-
def RunCurl(args, **kwargs):
"""Runs curl and wraps around all necessary hacks."""
cmd = ['curl']
@@ -1437,114 +1246,6 @@
return trues, falses
-def _GetStatus(status_url):
- """Polls |status_url| and returns the retrieved tree status.
-
- This function gets a JSON response from |status_url|, and returns the
- value associated with the 'general_state' key, if one exists and the
- http request was successful.
-
- Returns:
- The tree status, as a string, if it was successfully retrieved. Otherwise
- None.
- """
- try:
- # Check for successful response code.
- response = urllib.urlopen(status_url)
- if response.getcode() == 200:
- data = json.load(response)
- if data.has_key('general_state'):
- return data['general_state']
- # We remain robust against IOError's.
- except IOError as e:
- logging.error('Could not reach %s: %r', status_url, e)
-
-
-def WaitForTreeStatus(status_url, period=1, timeout=1, throttled_ok=False):
- """Wait for tree status to be open (or throttled, if |throttled_ok|).
-
- Args:
- status_url: The status url to check i.e.
- 'https://status.appspot.com/current?format=json'
- polling_period: Time in seconds to wait between polling
-
- Returns:
- The most recent tree status, either constants.TREE_OPEN or
- constants.TREE_THROTTLED (if |throttled_ok|)
-
- Raises:
- TimeoutError if timeout expired before tree reached acceptable status.
- """
- acceptable_states = set([constants.TREE_OPEN])
- verb = 'open'
- if throttled_ok:
- acceptable_states.add(constants.TREE_THROTTLED)
- verb = 'not be closed'
-
- timeout = max(timeout, 1)
-
- end_time = time.time() + timeout
-
- def _LogMessage():
- time_left = end_time - time.time()
- Info('Waiting for the tree to %s (%d minutes left)...', verb,
- time_left / 60)
-
- def _get_status():
- return _GetStatus(status_url)
-
- return WaitForReturnValue(acceptable_states, _get_status, timeout=timeout,
- period=period, side_effect_func=_LogMessage)
-
-
-def IsTreeOpen(status_url, period=1, timeout=1, throttled_ok=False):
- """Wait for tree status to be open (or throttled, if |throttled_ok|).
-
- Args:
- status_url: The status url to check i.e.
- 'https://status.appspot.com/current?format=json'
- polling_period: Time in seconds to wait between polling
-
- Returns:
- True if the tree is open (or throttled, if |throttled_ok|). False if
- timeout expired before tree reached acceptable status.
- """
- try:
- WaitForTreeStatus(status_url, period, timeout, throttled_ok)
- except TimeoutError:
- return False
- return True
-
-
-def GetTreeStatus(status_url, polling_period=0, timeout=0):
- """Returns the current tree status as fetched from |status_url|.
-
- This function returns the tree status as a string, either
- constants.TREE_OPEN, constants.TREE_THROTTLED, or constants.TREE_CLOSED.
-
- Args:
- status_url: The status url to check i.e.
- 'https://status.appspot.com/current?format=json'
- polling_period: Time to wait in seconds between polling attempts.
- timeout: Maximum time in seconds to wait for status.
-
- Returns:
- constants.TREE_OPEN, constants.TREE_THROTTLED, or constants.TREE_CLOSED
-
- Raises:
- TimeoutError if the timeout expired before the status could be successfully
- fetched.
- """
- acceptable_states = set([constants.TREE_OPEN, constants.TREE_THROTTLED,
- constants.TREE_CLOSED])
-
- def _get_status():
- return _GetStatus(status_url)
-
- return WaitForReturnValue(acceptable_states, _get_status, timeout,
- polling_period)
-
-
@contextlib.contextmanager
def _Open(input):
"""Convenience ctx that accepts a file path or an already open file object."""
diff --git a/lib/cros_build_lib_unittest.py b/lib/cros_build_lib_unittest.py
index 8d49f5d..2a391e4 100755
--- a/lib/cros_build_lib_unittest.py
+++ b/lib/cros_build_lib_unittest.py
@@ -19,7 +19,6 @@
import signal
import StringIO
import time
-import urllib
import __builtin__
from chromite.buildbot import constants
@@ -735,33 +734,6 @@
self.assertFalse(cros_build_lib.BooleanShellValue(v, False))
-class TestTimeouts(cros_test_lib.TestCase):
-
- def testTimeout(self):
- """Tests that we can nest Timeout correctly."""
- self.assertFalse('mock' in str(time.sleep).lower())
- with cros_build_lib.Timeout(30):
- with cros_build_lib.Timeout(20):
- with cros_build_lib.Timeout(1):
- self.assertRaises(cros_build_lib.TimeoutError, time.sleep, 10)
-
- # Should not raise a timeout exception as 20 > 2.
- time.sleep(1)
-
- def testTimeoutNested(self):
- """Tests that we still re-raise an alarm if both are reached."""
- with cros_build_lib.Timeout(1):
- try:
- with cros_build_lib.Timeout(2):
- self.assertRaises(cros_build_lib.TimeoutError, time.sleep, 1)
-
- # Craziness to catch nested timeouts.
- except cros_build_lib.TimeoutError:
- pass
- else:
- self.assertTrue(False, 'Should have thrown an exception')
-
-
class TestContextManagerStack(cros_test_lib.TestCase):
def test(self):
@@ -915,136 +887,6 @@
self.assertEqual(branches, ['refs/remotes/origin/release-R23-2913.B'])
-# pylint: disable=W0212,R0904
-class TestTreeStatus(cros_test_lib.MoxTestCase):
- """Tests TreeStatus method in cros_build_lib."""
-
- status_url = 'https://chromiumos-status.appspot.com/current?format=json'
-
- def setUp(self):
- pass
-
- def _TreeStatusFile(self, message, general_state):
- """Returns a file-like object with the status message writtin in it."""
- my_response = self.mox.CreateMockAnything()
- my_response.json = '{"message": "%s", "general_state": "%s"}' % (
- message, general_state)
- return my_response
-
- def _SetupMockTreeStatusResponses(self, status_url,
- final_tree_status='Tree is open.',
- final_general_state=constants.TREE_OPEN,
- rejected_tree_status='Tree is closed.',
- rejected_general_state=
- constants.TREE_CLOSED,
- rejected_status_count=0,
- retries_500=0,
- output_final_status=True):
- """Mocks out urllib.urlopen commands to simulate a given tree status.
-
- Args:
-
- status_url: The status url that status will be fetched from.
- final_tree_status: The final value of tree status that will be returned
- by urlopen.
- final_general_state: The final value of 'general_state' that will be
- returned by urlopen.
- rejected_tree_status: An intermediate value of tree status that will be
- returned by urlopen and retried upon.
- rejected_general_state: An intermediate value of 'general_state' that
- will be returned by urlopen and retried upon.
- rejected_status_count: The number of times urlopen will return the
- rejected state.
- retries_500: The number of times urlopen will fail with a 500 code.
- output_final_status: If True, the status given by final_tree_status and
- final_general_state will be the last status returned by urlopen. If
- False, final_tree_status will never be returned, and instead an
- unlimited number of times rejected_response will be returned.
- """
-
- final_response = self._TreeStatusFile(final_tree_status,
- final_general_state)
- rejected_response = self._TreeStatusFile(rejected_tree_status,
- rejected_general_state)
- error_500_response = self.mox.CreateMockAnything()
- self.mox.StubOutWithMock(urllib, 'urlopen')
-
- for _ in range(retries_500):
- urllib.urlopen(status_url).AndReturn(error_500_response)
- error_500_response.getcode().AndReturn(500)
-
- if output_final_status:
- for _ in range(rejected_status_count):
- urllib.urlopen(status_url).AndReturn(rejected_response)
- rejected_response.getcode().AndReturn(200)
- rejected_response.read().AndReturn(rejected_response.json)
-
- urllib.urlopen(status_url).AndReturn(final_response)
- final_response.getcode().AndReturn(200)
- final_response.read().AndReturn(final_response.json)
- else:
- urllib.urlopen(status_url).MultipleTimes().AndReturn(rejected_response)
- rejected_response.getcode().MultipleTimes().AndReturn(200)
- rejected_response.read().MultipleTimes().AndReturn(
- rejected_response.json)
-
- self.mox.ReplayAll()
-
- def testTreeIsOpen(self):
- """Tests that we return True is the tree is open."""
- self._SetupMockTreeStatusResponses(self.status_url,
- rejected_status_count=5,
- retries_500=5)
- self.assertTrue(cros_build_lib.IsTreeOpen(self.status_url,
- period=0))
-
- def testTreeIsClosed(self):
- """Tests that we return false is the tree is closed."""
- self._SetupMockTreeStatusResponses(self.status_url,
- output_final_status=False)
- self.assertFalse(cros_build_lib.IsTreeOpen(self.status_url,
- period=0.1))
-
- def testTreeIsThrottled(self):
- """Tests that we return True if the tree is throttled."""
- self._SetupMockTreeStatusResponses(self.status_url,
- 'Tree is throttled (flaky bug on flaky builder)',
- constants.TREE_THROTTLED)
- self.assertTrue(cros_build_lib.IsTreeOpen(self.status_url,
- throttled_ok=True))
-
- def testTreeIsThrottledNotOk(self):
- """Tests that we respect throttled_ok"""
- self._SetupMockTreeStatusResponses(self.status_url,
- rejected_tree_status='Tree is throttled (flaky bug on flaky builder)',
- rejected_general_state=constants.TREE_THROTTLED,
- output_final_status=False)
- self.assertFalse(cros_build_lib.IsTreeOpen(self.status_url,
- period=0.1))
-
- def testWaitForStatusOpen(self):
- """Tests that we can wait for a tree open response."""
- self._SetupMockTreeStatusResponses(self.status_url)
- self.assertEqual(cros_build_lib.WaitForTreeStatus(self.status_url),
- constants.TREE_OPEN)
-
-
- def testWaitForStatusThrottled(self):
- """Tests that we can wait for a tree open response."""
- self._SetupMockTreeStatusResponses(self.status_url,
- final_general_state=constants.TREE_THROTTLED)
- self.assertEqual(cros_build_lib.WaitForTreeStatus(self.status_url,
- throttled_ok=True),
- constants.TREE_THROTTLED)
-
- def testWaitForStatusFailure(self):
- """Tests that we can wait for a tree open response."""
- self._SetupMockTreeStatusResponses(self.status_url,
- output_final_status=False)
- self.assertRaises(cros_build_lib.TimeoutError,
- cros_build_lib.WaitForTreeStatus, self.status_url,
- period=0.1)
-
class Test_iflatten_instance(cros_test_lib.TestCase):
def test_it(self):
diff --git a/lib/cros_test_lib.py b/lib/cros_test_lib.py
index 729e0bb..11ba983 100644
--- a/lib/cros_test_lib.py
+++ b/lib/cros_test_lib.py
@@ -34,6 +34,7 @@
import gob_util
import osutils
import terminal
+import timeout_util
if 'chromite' not in sys.modules:
# TODO(build): Finish test wrapper (http://crosbug.com/37517).
@@ -218,7 +219,7 @@
if timeout is not None:
for name, func in scope.iteritems():
if name.startswith('test') and hasattr(func, '__call__'):
- wrapper = cros_build_lib.TimeoutDecorator(timeout)
+ wrapper = timeout_util.TimeoutDecorator(timeout)
scope[name] = wrapper(func)
return type.__new__(mcs, name, bases, scope)
diff --git a/lib/cros_test_lib_unittest.py b/lib/cros_test_lib_unittest.py
index c5ff28f..093c063 100755
--- a/lib/cros_test_lib_unittest.py
+++ b/lib/cros_test_lib_unittest.py
@@ -12,10 +12,10 @@
sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..', '..'))
-from chromite.lib import cros_build_lib
from chromite.lib import cros_test_lib
from chromite.lib import cros_build_lib_unittest
from chromite.lib import partial_mock
+from chromite.lib import timeout_util
# TODO(build): Finish test wrapper (http://crosbug.com/37517).
# Until then, this has to be after the chromite imports.
@@ -196,7 +196,7 @@
# Run the test case, verifying it raises a TimeoutError.
test = TimeoutTestCase(methodName='testSleeping')
- self.assertRaises(cros_build_lib.TimeoutError, test.testSleeping)
+ self.assertRaises(timeout_util.TimeoutError, test.testSleeping)
if __name__ == '__main__':
diff --git a/lib/parallel.py b/lib/parallel.py
index e4160c0..075f04a 100644
--- a/lib/parallel.py
+++ b/lib/parallel.py
@@ -27,6 +27,7 @@
from chromite.buildbot import cbuildbot_results as results_lib
from chromite.lib import cros_build_lib
from chromite.lib import osutils
+from chromite.lib import timeout_util
_BUFSIZE = 1024
@@ -304,7 +305,7 @@
error = str(ex)
possibly_flaky = ex.possibly_flaky
except BaseException as ex:
- possibly_flaky = isinstance(ex, cros_build_lib.TimeoutError)
+ possibly_flaky = isinstance(ex, timeout_util.TimeoutError)
error = traceback.format_exc()
if self._killing.is_set():
traceback.print_exc()
diff --git a/lib/remote_access.py b/lib/remote_access.py
index 8ffd748..fc232ff 100644
--- a/lib/remote_access.py
+++ b/lib/remote_access.py
@@ -12,6 +12,7 @@
import time
from chromite.lib import cros_build_lib
+from chromite.lib import timeout_util
_path = os.path.dirname(os.path.realpath(__file__))
@@ -174,9 +175,9 @@
self.RemoteSh('touch %s && reboot' % REBOOT_MARKER)
time.sleep(CHECK_INTERVAL)
try:
- cros_build_lib.WaitForCondition(self._CheckIfRebooted, CHECK_INTERVAL,
- REBOOT_MAX_WAIT)
- except cros_build_lib.TimeoutError:
+ timeout_util.WaitForCondition(self._CheckIfRebooted, CHECK_INTERVAL,
+ REBOOT_MAX_WAIT)
+ except timeout_util.TimeoutError:
cros_build_lib.Die('Reboot has not completed after %s seconds; giving up.'
% (REBOOT_MAX_WAIT,))
diff --git a/lib/stats.py b/lib/stats.py
index ad2d2fc..63a5f4f 100644
--- a/lib/stats.py
+++ b/lib/stats.py
@@ -16,6 +16,7 @@
from chromite.lib import cros_build_lib
from chromite.lib import git
from chromite.lib import osutils
+from chromite.lib import timeout_util
class Stats(object):
@@ -158,11 +159,11 @@
if not cls._UploadConditionsMet(stats):
return
- with cros_build_lib.Timeout(timeout):
+ with timeout_util.Timeout(timeout):
try:
cls._Upload(stats, url)
# Stats upload errors are silenced, for the sake of user experience.
- except cros_build_lib.TimeoutError:
+ except timeout_util.TimeoutError:
logging.debug(cls.TIMEOUT_ERROR, timeout)
except urllib2.HTTPError as e:
# HTTPError has a geturl() method, but it relies on self.url, which
diff --git a/lib/stats_unittest.py b/lib/stats_unittest.py
index 5df8cfb..40d0b6f 100755
--- a/lib/stats_unittest.py
+++ b/lib/stats_unittest.py
@@ -11,12 +11,12 @@
import urllib2
sys.path.insert(0, os.path.abspath('%s/../..' % os.path.dirname(__file__)))
-from chromite.lib import cros_build_lib
from chromite.lib import cros_test_lib
from chromite.lib import parallel
from chromite.lib import parallel_unittest
from chromite.lib import partial_mock
from chromite.lib import stats
+from chromite.lib import timeout_util
# pylint: disable=W0212
@@ -161,7 +161,7 @@
def testUploadTimeoutIgnore(self):
"""We don't propagate timeouts during upload."""
self.CheckSuppressException(
- cros_build_lib.TimeoutError(),
+ timeout_util.TimeoutError(),
stats.StatsUploader.TIMEOUT_ERROR
% (stats.StatsUploader.UPLOAD_TIMEOUT,))
diff --git a/lib/timeout_util.py b/lib/timeout_util.py
new file mode 100644
index 0000000..d044251
--- /dev/null
+++ b/lib/timeout_util.py
@@ -0,0 +1,315 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Functions for implementing timeouts."""
+
+
+import contextlib
+import functools
+import logging
+import json
+import signal
+import time
+import urllib
+
+from chromite.buildbot import constants
+from chromite.lib import signals
+
+
+class TimeoutError(Exception):
+ """Raises when code within Timeout has been run too long."""
+
+
+@contextlib.contextmanager
+def Timeout(max_run_time):
+ """ContextManager that alarms if code is ran for too long.
+
+ Timeout can run nested and raises a TimeoutException if the timeout
+ is reached. Timeout can also nest underneath FatalTimeout.
+
+ Args:
+ max_run_time: Number (integer) of seconds to wait before sending SIGALRM.
+ """
+ max_run_time = int(max_run_time)
+ if max_run_time <= 0:
+ raise ValueError("max_run_time must be greater than zero")
+
+ # pylint: disable=W0613
+ def kill_us(sig_num, frame):
+ raise TimeoutError("Timeout occurred- waited %s seconds." % max_run_time)
+
+ original_handler = signal.signal(signal.SIGALRM, kill_us)
+ previous_time = int(time.time())
+
+ # Signal the min in case the leftover time was smaller than this timeout.
+ remaining_timeout = signal.alarm(0)
+ if remaining_timeout:
+ signal.alarm(min(remaining_timeout, max_run_time))
+ else:
+ signal.alarm(max_run_time)
+
+ try:
+ yield
+ finally:
+ # Cancel the alarm request and restore the original handler.
+ signal.alarm(0)
+ signal.signal(signal.SIGALRM, original_handler)
+
+ # Ensure the previous handler will fire if it was meant to.
+ if remaining_timeout > 0:
+ # Signal the previous handler if it would have already passed.
+ time_left = remaining_timeout - (int(time.time()) - previous_time)
+ if time_left <= 0:
+ signals.RelaySignal(original_handler, signal.SIGALRM, None)
+ else:
+ signal.alarm(time_left)
+
+
+@contextlib.contextmanager
+def FatalTimeout(max_run_time):
+ """ContextManager that exits the program if code is run for too long.
+
+ This implementation is fairly simple, thus multiple timeouts
+ cannot be active at the same time.
+
+ Additionally, if the timeout has elapsed, it'll trigger a SystemExit
+ exception within the invoking code, ultimately propagating that past
+ itself. If the underlying code tries to suppress the SystemExit, once
+ a minute it'll retrigger SystemExit until control is returned to this
+ manager.
+
+ Args:
+ max_run_time: a positive integer.
+ """
+ max_run_time = int(max_run_time)
+ if max_run_time <= 0:
+ raise ValueError("max_run_time must be greater than zero")
+
+ # pylint: disable=W0613
+ def kill_us(sig_num, frame):
+ # While this SystemExit *should* crash it's way back up the
+ # stack to our exit handler, we do have live/production code
+ # that uses blanket except statements which could suppress this.
+ # As such, keep scheduling alarms until our exit handler runs.
+ # Note that there is a potential conflict via this code, and
+ # RunCommand's kill_timeout; thus we set the alarming interval
+ # fairly high.
+ signal.alarm(60)
+ raise SystemExit("Timeout occurred- waited %i seconds, failing."
+ % max_run_time)
+
+ original_handler = signal.signal(signal.SIGALRM, kill_us)
+ remaining_timeout = signal.alarm(max_run_time)
+ if remaining_timeout:
+ # Restore things to the way they were.
+ signal.signal(signal.SIGALRM, original_handler)
+ signal.alarm(remaining_timeout)
+ # ... and now complain. Unfortunately we can't easily detect this
+ # upfront, thus the reset dance above.
+ raise Exception("_Timeout cannot be used in parallel to other alarm "
+ "handling code; failing")
+ try:
+ yield
+ finally:
+ # Cancel the alarm request and restore the original handler.
+ signal.alarm(0)
+ signal.signal(signal.SIGALRM, original_handler)
+
+
+def TimeoutDecorator(max_time):
+ """Decorator used to ensure a func is interrupted if it's running too long."""
+ # Save off the built-in versions of time.time, signal.signal, and
+ # signal.alarm, in case they get mocked out later. We want to ensure that
+ # tests don't accidentally mock out the functions used by Timeout.
+ def _Save():
+ return time.time, signal.signal, signal.alarm
+ def _Restore(values):
+ (time.time, signal.signal, signal.alarm) = values
+ builtins = _Save()
+
+ def NestedTimeoutDecorator(func):
+ @functools.wraps(func)
+ def TimeoutWrapper(*args, **kwargs):
+ new = _Save()
+ try:
+ _Restore(builtins)
+ with Timeout(max_time):
+ _Restore(new)
+ try:
+ func(*args, **kwargs)
+ finally:
+ _Restore(builtins)
+ finally:
+ _Restore(new)
+
+ return TimeoutWrapper
+
+ return NestedTimeoutDecorator
+
+
+def WaitForCondition(*args, **kwargs):
+ """Periodically run a function, waiting in between runs.
+
+ Continues to run until the function returns True.
+
+ Args:
+ See WaitForReturnValue([True], ...)
+
+ Raises:
+ TimeoutError when the timeout is exceeded.
+ """
+ WaitForReturnValue([True], *args, **kwargs)
+
+
+def WaitForReturnValue(values, func, timeout, period=1, side_effect_func=None,
+ func_args=None, func_kwargs=None):
+ """Periodically run a function, waiting in between runs.
+
+ Continues to run until the function return value is in the list
+ of accepted |values|.
+
+ Args:
+ values: A list or set of acceptable return values from |func|.
+ func: The function to run to test for a value.
+ timeout: The maximum amount of time to wait, in integer seconds. Minimum
+ value: 1.
+ period: Integer number of seconds between calls to |func|. Default: 1
+ side_effect_func: Optional function to be called between polls of func,
+ typically to output logging messages.
+ func_args: Optional list of positional arguments to be passed to |func|.
+ func_kwargs: Optional dictionary of keyword arguments to be passed to
+ |func|.
+
+ Returns:
+ The value most recently returned by |func|.
+
+ Raises:
+ TimeoutError when the timeout is exceeded.
+ """
+ assert period >= 0
+ # Timeout requires timeout >= 1.
+ assert timeout >= 1
+ func_args = func_args or []
+ func_kwargs = func_kwargs or {}
+ with Timeout(timeout):
+ while True:
+ timestamp = time.time()
+ value = func(*func_args, **func_kwargs)
+ if value in values:
+ return value
+
+ time_remaining = period - int(time.time() - timestamp)
+ if time_remaining > 0:
+ if side_effect_func:
+ side_effect_func()
+ time.sleep(time_remaining)
+
+
+def _GetStatus(status_url):
+ """Polls |status_url| and returns the retrieved tree status.
+
+ This function gets a JSON response from |status_url|, and returns the
+ value associated with the 'general_state' key, if one exists and the
+ http request was successful.
+
+ Returns:
+ The tree status, as a string, if it was successfully retrieved. Otherwise
+ None.
+ """
+ try:
+ # Check for successful response code.
+ response = urllib.urlopen(status_url)
+ if response.getcode() == 200:
+ data = json.load(response)
+ if data.has_key('general_state'):
+ return data['general_state']
+ # We remain robust against IOError's.
+ except IOError as e:
+ logging.error('Could not reach %s: %r', status_url, e)
+
+
+def WaitForTreeStatus(status_url, period=1, timeout=1, throttled_ok=False):
+ """Wait for tree status to be open (or throttled, if |throttled_ok|).
+
+ Args:
+ status_url: The status url to check i.e.
+ 'https://status.appspot.com/current?format=json'
+ polling_period: Time in seconds to wait between polling
+
+ Returns:
+ The most recent tree status, either constants.TREE_OPEN or
+ constants.TREE_THROTTLED (if |throttled_ok|)
+
+ Raises:
+ TimeoutError if timeout expired before tree reached acceptable status.
+ """
+ acceptable_states = set([constants.TREE_OPEN])
+ verb = 'open'
+ if throttled_ok:
+ acceptable_states.add(constants.TREE_THROTTLED)
+ verb = 'not be closed'
+
+ timeout = max(timeout, 1)
+
+ end_time = time.time() + timeout
+
+ def _LogMessage():
+ time_left = end_time - time.time()
+ logging.info('Waiting for the tree to %s (%d minutes left)...', verb,
+ time_left / 60)
+
+ def _get_status():
+ return _GetStatus(status_url)
+
+ return WaitForReturnValue(acceptable_states, _get_status, timeout=timeout,
+ period=period, side_effect_func=_LogMessage)
+
+
+
+def IsTreeOpen(status_url, period=1, timeout=1, throttled_ok=False):
+ """Wait for tree status to be open (or throttled, if |throttled_ok|).
+
+ Args:
+ status_url: The status url to check i.e.
+ 'https://status.appspot.com/current?format=json'
+ polling_period: Time in seconds to wait between polling
+
+ Returns:
+ True if the tree is open (or throttled, if |throttled_ok|). False if
+ timeout expired before tree reached acceptable status.
+ """
+ try:
+ WaitForTreeStatus(status_url, period, timeout, throttled_ok)
+ except TimeoutError:
+ return False
+ return True
+
+
+def GetTreeStatus(status_url, polling_period=0, timeout=0):
+ """Returns the current tree status as fetched from |status_url|.
+
+ This function returns the tree status as a string, either
+ constants.TREE_OPEN, constants.TREE_THROTTLED, or constants.TREE_CLOSED.
+
+ Args:
+ status_url: The status url to check i.e.
+ 'https://status.appspot.com/current?format=json'
+ polling_period: Time to wait in seconds between polling attempts.
+ timeout: Maximum time in seconds to wait for status.
+
+ Returns:
+ constants.TREE_OPEN, constants.TREE_THROTTLED, or constants.TREE_CLOSED
+
+ Raises:
+ TimeoutError if the timeout expired before the status could be successfully
+ fetched.
+ """
+ acceptable_states = set([constants.TREE_OPEN, constants.TREE_THROTTLED,
+ constants.TREE_CLOSED])
+
+ def _get_status():
+ return _GetStatus(status_url)
+
+ return WaitForReturnValue(acceptable_states, _get_status, timeout,
+ polling_period)
diff --git a/lib/timeout_util_unittest.py b/lib/timeout_util_unittest.py
new file mode 100755
index 0000000..e5e0888
--- /dev/null
+++ b/lib/timeout_util_unittest.py
@@ -0,0 +1,182 @@
+#!/usr/bin/python
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Test suite for timeout_util.py"""
+
+import os
+import sys
+import time
+import urllib
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
+ os.path.abspath(__file__)))))
+
+from chromite.buildbot import constants
+from chromite.lib import cros_test_lib
+from chromite.lib import timeout_util
+
+
+# pylint: disable=W0212,R0904
+
+
+class TestTimeouts(cros_test_lib.TestCase):
+
+ def testTimeout(self):
+ """Tests that we can nest Timeout correctly."""
+ self.assertFalse('mock' in str(time.sleep).lower())
+ with timeout_util.Timeout(30):
+ with timeout_util.Timeout(20):
+ with timeout_util.Timeout(1):
+ self.assertRaises(timeout_util.TimeoutError, time.sleep, 10)
+
+ # Should not raise a timeout exception as 20 > 2.
+ time.sleep(1)
+
+ def testTimeoutNested(self):
+ """Tests that we still re-raise an alarm if both are reached."""
+ with timeout_util.Timeout(1):
+ try:
+ with timeout_util.Timeout(2):
+ self.assertRaises(timeout_util.TimeoutError, time.sleep, 1)
+
+ # Craziness to catch nested timeouts.
+ except timeout_util.TimeoutError:
+ pass
+ else:
+ self.assertTrue(False, 'Should have thrown an exception')
+
+
+class TestTreeStatus(cros_test_lib.MoxTestCase):
+ """Tests TreeStatus method in cros_build_lib."""
+
+ status_url = 'https://chromiumos-status.appspot.com/current?format=json'
+
+ def setUp(self):
+ pass
+
+ def _TreeStatusFile(self, message, general_state):
+ """Returns a file-like object with the status message writtin in it."""
+ my_response = self.mox.CreateMockAnything()
+ my_response.json = '{"message": "%s", "general_state": "%s"}' % (
+ message, general_state)
+ return my_response
+
+ def _SetupMockTreeStatusResponses(self, status_url,
+ final_tree_status='Tree is open.',
+ final_general_state=constants.TREE_OPEN,
+ rejected_tree_status='Tree is closed.',
+ rejected_general_state=
+ constants.TREE_CLOSED,
+ rejected_status_count=0,
+ retries_500=0,
+ output_final_status=True):
+ """Mocks out urllib.urlopen commands to simulate a given tree status.
+
+ Args:
+
+ status_url: The status url that status will be fetched from.
+ final_tree_status: The final value of tree status that will be returned
+ by urlopen.
+ final_general_state: The final value of 'general_state' that will be
+ returned by urlopen.
+ rejected_tree_status: An intermediate value of tree status that will be
+ returned by urlopen and retried upon.
+ rejected_general_state: An intermediate value of 'general_state' that
+ will be returned by urlopen and retried upon.
+ rejected_status_count: The number of times urlopen will return the
+ rejected state.
+ retries_500: The number of times urlopen will fail with a 500 code.
+ output_final_status: If True, the status given by final_tree_status and
+ final_general_state will be the last status returned by urlopen. If
+ False, final_tree_status will never be returned, and instead an
+ unlimited number of times rejected_response will be returned.
+ """
+
+ final_response = self._TreeStatusFile(final_tree_status,
+ final_general_state)
+ rejected_response = self._TreeStatusFile(rejected_tree_status,
+ rejected_general_state)
+ error_500_response = self.mox.CreateMockAnything()
+ self.mox.StubOutWithMock(urllib, 'urlopen')
+
+ for _ in range(retries_500):
+ urllib.urlopen(status_url).AndReturn(error_500_response)
+ error_500_response.getcode().AndReturn(500)
+
+ if output_final_status:
+ for _ in range(rejected_status_count):
+ urllib.urlopen(status_url).AndReturn(rejected_response)
+ rejected_response.getcode().AndReturn(200)
+ rejected_response.read().AndReturn(rejected_response.json)
+
+ urllib.urlopen(status_url).AndReturn(final_response)
+ final_response.getcode().AndReturn(200)
+ final_response.read().AndReturn(final_response.json)
+ else:
+ urllib.urlopen(status_url).MultipleTimes().AndReturn(rejected_response)
+ rejected_response.getcode().MultipleTimes().AndReturn(200)
+ rejected_response.read().MultipleTimes().AndReturn(
+ rejected_response.json)
+
+ self.mox.ReplayAll()
+
+ def testTreeIsOpen(self):
+ """Tests that we return True is the tree is open."""
+ self._SetupMockTreeStatusResponses(self.status_url,
+ rejected_status_count=5,
+ retries_500=5)
+ self.assertTrue(timeout_util.IsTreeOpen(self.status_url,
+ period=0))
+
+ def testTreeIsClosed(self):
+ """Tests that we return false is the tree is closed."""
+ self._SetupMockTreeStatusResponses(self.status_url,
+ output_final_status=False)
+ self.assertFalse(timeout_util.IsTreeOpen(self.status_url,
+ period=0.1))
+
+ def testTreeIsThrottled(self):
+ """Tests that we return True if the tree is throttled."""
+ self._SetupMockTreeStatusResponses(self.status_url,
+ 'Tree is throttled (flaky bug on flaky builder)',
+ constants.TREE_THROTTLED)
+ self.assertTrue(timeout_util.IsTreeOpen(self.status_url,
+ throttled_ok=True))
+
+ def testTreeIsThrottledNotOk(self):
+ """Tests that we respect throttled_ok"""
+ self._SetupMockTreeStatusResponses(self.status_url,
+ rejected_tree_status='Tree is throttled (flaky bug on flaky builder)',
+ rejected_general_state=constants.TREE_THROTTLED,
+ output_final_status=False)
+ self.assertFalse(timeout_util.IsTreeOpen(self.status_url,
+ period=0.1))
+
+ def testWaitForStatusOpen(self):
+ """Tests that we can wait for a tree open response."""
+ self._SetupMockTreeStatusResponses(self.status_url)
+ self.assertEqual(timeout_util.WaitForTreeStatus(self.status_url),
+ constants.TREE_OPEN)
+
+
+ def testWaitForStatusThrottled(self):
+ """Tests that we can wait for a tree open response."""
+ self._SetupMockTreeStatusResponses(self.status_url,
+ final_general_state=constants.TREE_THROTTLED)
+ self.assertEqual(timeout_util.WaitForTreeStatus(self.status_url,
+ throttled_ok=True),
+ constants.TREE_THROTTLED)
+
+ def testWaitForStatusFailure(self):
+ """Tests that we can wait for a tree open response."""
+ self._SetupMockTreeStatusResponses(self.status_url,
+ output_final_status=False)
+ self.assertRaises(timeout_util.TimeoutError,
+ timeout_util.WaitForTreeStatus, self.status_url,
+ period=0.1)
+
+
+if __name__ == '__main__':
+ cros_test_lib.main()
diff --git a/scripts/cbuildbot.py b/scripts/cbuildbot.py
index a3e60be..c08ce38 100644
--- a/scripts/cbuildbot.py
+++ b/scripts/cbuildbot.py
@@ -43,6 +43,7 @@
from chromite.lib import patch as cros_patch
from chromite.lib import parallel
from chromite.lib import sudo
+from chromite.lib import timeout_util
_DEFAULT_LOG_DIR = 'cbuildbot_logs'
@@ -1534,7 +1535,7 @@
stack.Add(critical_section.ForkWatchdog)
if options.timeout > 0:
- stack.Add(cros_build_lib.FatalTimeout, options.timeout)
+ stack.Add(timeout_util.FatalTimeout, options.timeout)
if not options.buildbot:
build_config = cbuildbot_config.OverrideConfigForTrybot(
diff --git a/scripts/cros_best_revision.py b/scripts/cros_best_revision.py
index 2dc2b67..af60d71 100644
--- a/scripts/cros_best_revision.py
+++ b/scripts/cros_best_revision.py
@@ -23,6 +23,7 @@
from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import parallel
+from chromite.lib import timeout_util
class LKGMNotFound(Exception):
@@ -136,7 +137,7 @@
commit_cmd = ['svn', 'commit', '--message',
self. _COMMIT_MSG % dict(version=self._lkgm)]
- if not cros_build_lib.IsTreeOpen(gclient.STATUS_URL,
+ if not timeout_util.IsTreeOpen(gclient.STATUS_URL,
self._SLEEP_TIMEOUT, timeout=self._TREE_TIMEOUT):
raise LKGMNotCommitted('Chromium Tree is closed')
diff --git a/scripts/cros_best_revision_unittest.py b/scripts/cros_best_revision_unittest.py
index 13758ee..9a45617 100755
--- a/scripts/cros_best_revision_unittest.py
+++ b/scripts/cros_best_revision_unittest.py
@@ -15,13 +15,13 @@
from chromite.buildbot import cbuildbot_config
from chromite.buildbot import constants
from chromite.buildbot import manifest_version
-from chromite.lib import cros_build_lib
from chromite.lib import cros_build_lib_unittest
from chromite.lib import cros_test_lib
from chromite.lib import gclient
from chromite.lib import gs_unittest
from chromite.lib import osutils
from chromite.lib import partial_mock
+from chromite.lib import timeout_util
from chromite.scripts import cros_best_revision
@@ -116,7 +116,7 @@
def testCommitNewLKGM(self):
"""Tests that we can commit a new LKGM file."""
self.committer._lkgm = '4.0.0'
- self.PatchObject(cros_build_lib, 'IsTreeOpen', return_value=True)
+ self.PatchObject(timeout_util, 'IsTreeOpen', return_value=True)
self.committer.CommitNewLKGM()
# Check the file was actually written out correctly.
diff --git a/scripts/deploy_chrome.py b/scripts/deploy_chrome.py
index 232d63a..9101d70 100644
--- a/scripts/deploy_chrome.py
+++ b/scripts/deploy_chrome.py
@@ -40,6 +40,7 @@
from chromite.lib import parallel
from chromite.lib import remote_access as remote
from chromite.lib import stats
+from chromite.lib import timeout_util
from chromite.scripts import lddtree
@@ -164,7 +165,7 @@
# Developers sometimes run session_manager manually, in which case we'll
# need to help shut the chrome processes down.
try:
- with cros_build_lib.Timeout(KILL_PROC_MAX_WAIT):
+ with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
while self._ChromeFileInUse():
logging.warning('The chrome binary on the device is in use.')
logging.warning('Killing chrome and session_manager processes...\n')
@@ -174,7 +175,7 @@
# Wait for processes to actually terminate
time.sleep(POST_KILL_WAIT)
logging.info('Rechecking the chrome binary...')
- except cros_build_lib.TimeoutError:
+ except timeout_util.TimeoutError:
msg = ('Could not kill processes after %s seconds. Please exit any '
'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
raise DeployFailure(msg)
