blob: 5aa1f81ff9c8c7f9734fee79c2f8485ef139b358 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import logging, random, signal, sys, time
from autotest_lib.client.common_lib import error
from autotest_lib.frontend.afe.json_rpc import proxy
class TimeoutException(Exception):
Exception to be raised for when alarm is triggered.
def handler(signum, frame):
Register a handler for the timeout.
raise TimeoutException('Call is timed out.')
def timeout(func, args=(), kwargs={}, timeout_sec=60.0, default_result=None):
This function run the given function using the args, kwargs and
return the given default value if the timeout_sec is exceeded.
@param func: function to be called.
@param args: arguments for function to be called.
@param kwargs: keyword arguments for function to be called.
@param timeout_sec: timeout setting for call to exit, in seconds.
@param default_result: default return value for the function call.
@return 1: is_timeout 2: result of the function call. If
is_timeout is True, the call is timed out. If the
value is False, the call is finished on time.
old_handler = signal.signal(signal.SIGALRM, handler)
timeout_sec_n = int(timeout_sec)
# In case the timeout is rounded to 0, force to set it to default value.
if timeout_sec_n == 0:
timeout_sec_n = 60
old_alarm_sec = signal.alarm(timeout_sec_n)
if old_alarm_sec > 0:
old_timeout_time = time.time() + old_alarm_sec
default_result = func(*args, **kwargs)
return False, default_result
except TimeoutException:
return True, default_result
# Cancel the timer if the function returned before timeout or
# exception being thrown.
# Restore previous Signal handler and alarm
if old_handler:
signal.signal(signal.SIGALRM, old_handler)
if old_alarm_sec > 0:
old_alarm_sec = int(old_timeout_time - time.time())
if old_alarm_sec <= 0:
old_alarm_sec = 1;
def retry(ExceptionToCheck, timeout_min=1.0, delay_sec=3):
"""Retry calling the decorated function using a delay with jitter.
Will raise RPC ValidationError exceptions from the decorated
function without retrying; a malformed RPC isn't going to
magically become good.
original from:
@param ExceptionToCheck: the exception to check. May be a tuple of
exceptions to check.
@param timeout_min: timeout in minutes until giving up.
@param delay_sec: pre-jittered delay between retries in seconds. Actual
delays will be centered around this value, ranging up to
50% off this midpoint.
def deco_retry(func):
def delay():
'Jitter' the delay, up to 50% in either direction.
random_delay = random.uniform(.5 * delay_sec, 1.5 * delay_sec)
logging.warning('Retrying in %f seconds...', random_delay)
def func_retry(*args, **kwargs):
deadline = time.time() + timeout_min * 60 # convert to seconds.
# Used to cache exception to be raised later.
exc_info = None
delayed_enabled = False
while time.time() < deadline:
if delayed_enabled:
delayed_enabled = True
# Clear the cache
exc_info = None
is_timeout, result = timeout(func, args, kwargs,
if not is_timeout:
return result
except (error.CrosDynamicSuiteException,
except ExceptionToCheck as e:
logging.warning('%s(%s)', e.__class__, e)
# Cache the exception to be raised later.
exc_info = sys.exc_info()
# The call must have timed out or raised ExceptionToCheck.
if not exc_info:
raise TimeoutException('Call is timed out.')
# Raise the cached exception with original backtrace.
raise exc_info[0], exc_info[1], exc_info[2]
return func_retry # true decorator
return deco_retry