blob: 5bcc88fc7991e0cdb5decb468d00328b73c1d581 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import logging
import re
import sys
import urllib2
import urlparse
from autotest_lib.client.bin import utils
from autotest_lib.client.common_lib import error, global_config
from autotest_lib.client.common_lib.cros import dev_server
from autotest_lib.client.common_lib.cros import kernel_utils
from autotest_lib.server import autotest
from autotest_lib.server import utils as server_utils
from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
from autotest_lib.server.cros.dynamic_suite import tools
from chromite.lib import retry_util
from chromite.lib import metrics
except ImportError:
metrics = utils.metrics_mock
def _metric_name(base_name):
return 'chromeos/autotest/provision/' + base_name
# Local stateful update path is relative to the CrOS source directory.
# A list of update engine client states that occur after an update is triggered.
_STATEFUL_UPDATE_SCRIPT = 'stateful_update'
_QUICK_PROVISION_SCRIPT = 'quick-provision'
_UPDATER_BIN = '/usr/bin/update_engine_client'
_UPDATER_LOGS = ['/var/log/messages', '/var/log/update_engine']
# PROVISION_FAILED - A flag file to indicate provision failures. The
# file is created at the start of any AU procedure (see
# `ChromiumOSUpdater._prepare_host()`). The file's location in
# stateful means that on successul update it will be removed. Thus, if
# this file exists, it indicates that we've tried and failed in a
# previous attempt to update.
PROVISION_FAILED = '/var/tmp/provision_failed'
# A flag file used to enable special handling in lab DUTs. Some
# parts of the system in Chromium OS test images will behave in ways
# convenient to the test lab when this file is present. Generally,
# we create this immediately after any update completes.
_LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
# _TARGET_VERSION - A file containing the new version to which we plan
# to update. This file is used by the CrOS shutdown code to detect and
# handle certain version downgrade cases. Specifically: Downgrading
# may trigger an unwanted powerwash in the target build when the
# following conditions are met:
# * Source build is a v4.4 kernel with R69-10756.0.0 or later.
# * Target build predates the R69-10756.0.0 cutoff.
# When this file is present and indicates a downgrade, the OS shutdown
# code on the DUT knows how to prevent the powerwash.
_TARGET_VERSION = '/run/update_target_version'
# _REBOOT_FAILURE_MESSAGE - This is the standard message text returned
# when the Host.reboot() method fails. The source of this text comes
# from `wait_for_restart()` in client/common_lib/hosts/
_REBOOT_FAILURE_MESSAGE = 'Host did not return from reboot'
GS_CACHE_PORT = '8888'
class RootFSUpdateError(error.TestFail):
"""Raised when the RootFS fails to update."""
class StatefulUpdateError(error.TestFail):
"""Raised when the stateful partition fails to update."""
class _AttributedUpdateError(error.TestFail):
"""Update failure with an attributed cause."""
def __init__(self, attribution, msg):
super(_AttributedUpdateError, self).__init__(
'%s: %s' % (attribution, msg))
self._message = msg
def _classify(self):
for err_pattern, classification in self._CLASSIFIERS:
if re.match(err_pattern, self._message):
return classification
return None
def failure_summary(self):
"""Summarize this error for metrics reporting."""
classification = self._classify()
if classification:
return '%s: %s' % (self._SUMMARY, classification)
return self._SUMMARY
class HostUpdateError(_AttributedUpdateError):
"""Failure updating a DUT attributable to the DUT.
This class of exception should be raised when the most likely cause
of failure was a condition existing on the DUT prior to the update,
such as a hardware problem, or a bug in the software on the DUT.
DUT_DOWN = 'No answer to ssh'
_SUMMARY = 'DUT failed prior to update'
(_REBOOT_FAILURE_MESSAGE, 'Reboot failed'),
def __init__(self, hostname, msg):
super(HostUpdateError, self).__init__(
'Error on %s prior to update' % hostname, msg)
class DevServerError(_AttributedUpdateError):
"""Failure updating a DUT attributable to the devserver.
This class of exception should be raised when the most likely cause
of failure was the devserver serving the target image for update.
_SUMMARY = 'Devserver failed prior to update'
def __init__(self, devserver, msg):
super(DevServerError, self).__init__(
'Devserver error on %s' % devserver, msg)
class ImageInstallError(_AttributedUpdateError):
"""Failure updating a DUT when installing from the devserver.
This class of exception should be raised when the target DUT fails
to download and install the target image from the devserver, and
either the devserver or the DUT might be at fault.
_SUMMARY = 'Image failed to download and install'
def __init__(self, hostname, devserver, msg):
super(ImageInstallError, self).__init__(
'Download and install failed from %s onto %s'
% (devserver, hostname), msg)
class NewBuildUpdateError(_AttributedUpdateError):
"""Failure updating a DUT attributable to the target build.
This class of exception should be raised when updating to a new
build fails, and the most likely cause of the failure is a bug in
the newly installed target build.
CHROME_FAILURE = 'Chrome failed to reach login screen'
UPDATE_ENGINE_FAILURE = ('update-engine failed to call '
ROLLBACK_FAILURE = 'System rolled back to previous build'
_SUMMARY = 'New build failed'
(CHROME_FAILURE, 'Chrome did not start'),
(UPDATE_ENGINE_FAILURE, 'update-engine did not start'),
def __init__(self, update_version, msg):
super(NewBuildUpdateError, self).__init__(
'Failure in build %s' % update_version, msg)
def failure_summary(self):
#pylint: disable=missing-docstring
return 'Build failed to work after installing'
def _url_to_version(update_url):
"""Return the version based on update_url.
@param update_url: url to the image to update to.
# The Chrome OS version is generally the last element in the URL. The only
# exception is delta update URLs, which are rooted under the version; e.g.,
# http://.../update/.../0.14.755.0/au/0.14.754.0. In this case we want to
# strip off the au section of the path before reading the version.
return re.sub('/au/.*', '',
def url_to_image_name(update_url):
"""Return the image name based on update_url.
From a URL like:
return lumpy-release/R27-3837.0.0
@param update_url: url to the image to update to.
@returns a string representing the image name in the update_url.
return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:])
def get_update_failure_reason(exception):
"""Convert an exception into a failure reason for metrics.
The passed in `exception` should be one raised by failure of
`ChromiumOSUpdater.run_update`. The returned string will describe
the failure. If the input exception value is not a truish value
the return value will be `None`.
The number of possible return strings is restricted to a limited
enumeration of values so that the string may be safely used in
Monarch metrics without worrying about cardinality of the range of
string values.
@param exception Exception to be converted to a failure reason.
@return A string suitable for use in Monarch metrics, or `None`.
if exception:
if isinstance(exception, _AttributedUpdateError):
return exception.failure_summary
return 'Unknown Error: %s' % type(exception).__name__
return None
def _get_devserver_build_from_update_url(update_url):
"""Get the devserver and build from the update url.
@param update_url: The url for update.
Eg: http://devserver:port/update/build.
@return: A tuple of (devserver url, build) or None if the update_url
doesn't match the expected pattern.
@raises ValueError: If the update_url doesn't match the expected pattern.
@raises ValueError: If no global_config was found, or it doesn't contain an
pattern = global_config.global_config.get_config_value(
'CROS', 'image_url_pattern', type=str, default='')
if not pattern:
raise ValueError('Cannot parse update_url, the global config needs '
'an image_url_pattern.')
re_pattern = pattern.replace('%s', '(\S+)')
parts =, update_url)
if not parts or len(parts.groups()) < 2:
raise ValueError('%s is not an update url' % update_url)
return parts.groups()
def _list_image_dir_contents(update_url):
"""Lists the contents of the devserver for a given build/update_url.
@param update_url: An update url. Eg: http://devserver:port/update/build.
if not update_url:
logging.warning('Need update_url to list contents of the devserver.')
error_msg = 'Cannot check contents of devserver, update url %s' % update_url
devserver_url, build = _get_devserver_build_from_update_url(update_url)
except ValueError as e:
logging.warning('%s: %s', error_msg, e)
devserver = dev_server.ImageServer(devserver_url)
# The devserver will retry on URLError to avoid flaky connections, but will
# eventually raise the URLError if it persists. All HTTPErrors get
# converted to DevServerExceptions.
except (dev_server.DevServerException, urllib2.URLError) as e:
logging.warning('%s: %s', error_msg, e)
def _get_metric_fields(update_url):
"""Return a dict of metric fields.
This is used for sending autoupdate metrics for the given update URL.
@param update_url Metrics fields will be calculated from this URL.
build_name = url_to_image_name(update_url)
board, build_type, milestone, _ = server_utils.ParseBuildName(
except server_utils.ParseBuildNameException:
logging.warning('Unable to parse build name %s for metrics. '
'Continuing anyway.', build_name)
board, build_type, milestone = ('', '', '')
return {
'dev_server': dev_server.get_resolved_hostname(update_url),
'board': board,
'build_type': build_type,
'milestone': milestone,
class ChromiumOSUpdater(object):
"""Chromium OS specific DUT update functionality."""
def __init__(self, update_url, host=None, interactive=True,
"""Initializes the object.
@param update_url: The URL we want the update to use.
@param host: A client.common_lib.hosts.Host implementation.
@param interactive: Bool whether we are doing an interactive update.
@param use_quick_provision: Whether we should attempt to perform
the update using the quick-provision script.
self.update_url = update_url = host
self.interactive = interactive
self.update_version = _url_to_version(update_url)
self._use_quick_provision = use_quick_provision
def _run(self, cmd, *args, **kwargs):
"""Abbreviated form of"""
return, *args, **kwargs)
def check_update_status(self):
"""Returns the current update engine state.
We use the `update_engine_client -status' command and parse the line
indicating the update state, e.g. "CURRENT_OP=UPDATE_STATUS_IDLE".
update_status ='%s -status | grep CURRENT_OP' %
return update_status.stdout.strip().split('=')[-1]
def _rootdev(self, options=''):
"""Returns the stripped output of rootdev <options>.
@param options: options to run rootdev.
return self._run('rootdev %s' % options).stdout.strip()
def _get_last_update_error(self):
"""Get the last autoupdate error code."""
command_result = self._run(
'%s --last_attempt_error' % _UPDATER_BIN)
return command_result.stdout.strip().replace('\n', ', ')
def _base_update_handler_no_retry(self, run_args):
"""Base function to handle a remote update ssh call.
@param run_args: Dictionary of args passed to function.
@throws: intercepts and re-throws all exceptions
except Exception as e:
logging.debug('exception in update handler: %s', e)
raise e
def _base_update_handler(self, run_args, err_msg_prefix=None):
"""Handle a remote update ssh call, possibly with retries.
@param run_args: Dictionary of args passed to function.
@param err_msg_prefix: Prefix of the exception error message.
def exception_handler(e):
"""Examines exceptions and returns True if the update handler
should be retried.
@param e: the exception intercepted by the retry util.
return (isinstance(e, error.AutoservSSHTimeout) or
(isinstance(e, error.GenericHostRunError) and
hasattr(e, 'description') and
('ERROR_CODE=37', e.description) or'generic error .255.', e.description))))
# Try the update twice (arg 2 is max_retry, not including the first
# call). Some exceptions may be caught by the retry handler.
retry_util.GenericRetry(exception_handler, 1,
except Exception as e:
message = err_msg_prefix + ': ' + str(e)
raise RootFSUpdateError(message)
def _wait_for_update_service(self):
"""Ensure that the update engine daemon is running, possibly
by waiting for it a bit in case the DUT just rebooted and the
service hasn't started yet.
def handler(e):
"""Retry exception handler.
Assumes that the error is due to the update service not having
started yet.
@param e: the exception intercepted by the retry util.
if isinstance(e, error.AutoservRunError):
logging.debug('update service check exception: %s\n'
'retrying...', e)
return True
return False
# Retry at most three times, every 5s.
status = retry_util.GenericRetry(handler, 3,
# Expect the update engine to be idle.
if status != UPDATER_IDLE:
raise RootFSUpdateError(
'Update engine status is %s (%s was expected).'
% (status, UPDATER_IDLE))
def _reset_update_engine(self):
"""Resets the host to prepare for a clean update regardless of state."""
self._run('stop ui || true')
self._run('stop update-engine || true')
self._run('start update-engine')
def _reset_stateful_partition(self):
"""Clear any pending stateful update request."""
self._run('%s --stateful_change=reset 2>&1'
% self._get_stateful_update_script())
self._run('rm -f %s' % _TARGET_VERSION)
def _set_target_version(self):
"""Set the "target version" for the update."""
version_number = self.update_version.split('-')[1]
self._run('echo %s > %s' % (version_number, _TARGET_VERSION))
def _revert_boot_partition(self):
"""Revert the boot partition."""
part = self._rootdev('-s')
logging.warning('Reverting update; Boot partition will be %s', part)
return self._run('/postinst %s 2>&1' % part)
def _verify_update_completed(self):
"""Verifies that an update has completed.
@raise RootFSUpdateError if the DUT doesn't indicate that
download is complete and the DUT is ready for reboot.
status = self.check_update_status()
error_msg = ''
if status == UPDATER_IDLE:
error_msg = 'Update error: %s' % self._get_last_update_error()
raise RootFSUpdateError(
'Update engine status is %s (%s was expected). %s'
% (status, UPDATER_NEED_REBOOT, error_msg))
return kernel_utils.verify_kernel_state_after_update(
def trigger_update(self):
"""Triggers a background update."""
# If this function is called immediately after reboot (which it
# can be), there is no guarantee that the update engine is up
# and running yet, so wait for it.
autoupdate_cmd = ('%s --check_for_update --omaha_url=%s' %
(_UPDATER_BIN, self.update_url))
run_args = {'command': autoupdate_cmd}
err_prefix = 'Failed to trigger an update on %s. ' %'Triggering update via: %s', autoupdate_cmd)
metric_fields = {'success': False}
self._base_update_handler(run_args, err_prefix)
metric_fields['success'] = True
c = metrics.Counter('chromeos/autotest/autoupdater/trigger')
def update_image(self):
"""Updates the device root FS and kernel and verifies success."""
autoupdate_cmd = ('%s --update --omaha_url=%s' %
(_UPDATER_BIN, self.update_url))
if not self.interactive:
autoupdate_cmd = '%s --interactive=false' % autoupdate_cmd
run_args = {'command': autoupdate_cmd, 'timeout': 3600}
err_prefix = ('Failed to install device image using payload at %s '
'on %s. ' % (self.update_url,'Updating image via: %s', autoupdate_cmd)
metric_fields = {'success': False}
self._base_update_handler(run_args, err_prefix)
metric_fields['success'] = True
c = metrics.Counter('chromeos/autotest/autoupdater/update')
return self._verify_update_completed()
def _get_remote_script(self, script_name):
"""Ensure that `script_name` is present on the DUT.
The given script (e.g. `stateful_update`) may be present in the
stateful partition under /usr/local/bin, or we may have to
download it from the devserver.
Determine whether the script is present or must be downloaded
and download if necessary. Then, return a command fragment
sufficient to run the script from whereever it now lives on the
@param script_name The name of the script as expected in
/usr/local/bin and on the devserver.
@return A string with the command (minus arguments) that will
run the target script.
remote_script = '/usr/local/bin/%s' % script_name
return remote_script'mkdir -p -m 1777 /usr/local/tmp')
remote_tmp_script = '/usr/local/tmp/%s' % script_name
server_name = urlparse.urlparse(self.update_url)[1]
script_url = 'http://%s/static/%s' % (server_name, script_name)
fetch_script = 'curl -Ss -o %s %s && head -1 %s' % (
remote_tmp_script, script_url, remote_tmp_script)
first_line = self._run(fetch_script).stdout.strip()
if first_line and first_line.startswith('#!'):
script_interpreter = first_line.lstrip('#!')
if script_interpreter:
return '%s %s' % (script_interpreter, remote_tmp_script)
return None
def _get_stateful_update_script(self):
"""Returns a command to run the stateful update script.
Find `stateful_update` on the target or install it, as
necessary. If installation fails, raise an exception.
@raise StatefulUpdateError if the script can't be found or
@return A string that can be joined with arguments to run the
`stateful_update` command on the DUT.
script_command = self._get_remote_script(_STATEFUL_UPDATE_SCRIPT)
if not script_command:
raise StatefulUpdateError('Could not install %s on DUT'
return script_command
def update_stateful(self, clobber=True):
"""Updates the stateful partition.
@param clobber: If True, a clean stateful installation.
@raise StatefulUpdateError if the update script fails to
complete successfully.
"""'Updating stateful partition...')
statefuldev_url = self.update_url.replace('update', 'static')
# Attempt stateful partition update; this must succeed so that the newly
# installed host is testable after update.
statefuldev_cmd = [self._get_stateful_update_script(), statefuldev_url]
if clobber:
self._run(' '.join(statefuldev_cmd), timeout=1200)
except error.AutoservRunError:
raise StatefulUpdateError(
'Failed to perform stateful update on %s' %
def _prepare_host(self):
"""Make sure the target DUT is working and ready for update.
Initially, the target DUT's state is unknown. The DUT is
expected to be online, but we strive to be forgiving if Chrome
and/or the update engine aren't fully functional.
# Summary of work, and the rationale:
# 1. Reboot, because it's a good way to clear out problems.
# 2. Touch the PROVISION_FAILED file, to allow repair to detect
# failure later.
# 3. Run the hook for host class specific preparation.
# 4. Stop Chrome, because the system is designed to eventually
# reboot if Chrome is stuck in a crash loop.
# 5. Force `update-engine` to start, because if Chrome failed
# to start properly, the status of the `update-engine` job
# will be uncertain.
if not
raise HostUpdateError(,
self._run('touch %s' % PROVISION_FAILED)
self._reset_update_engine()'Updating from version %s to %s.',,
def _install_via_update_engine(self):
"""Install an updating using the production AU flow.
This uses the standard AU flow and the `stateful_update` script
to download and install a root FS, kernel and stateful
filesystem content.
@return The kernel expected to be booted next.
"""'Installing image using update_engine.')
expected_kernel = self.update_image()
return expected_kernel
def _quick_provision_with_gs_cache(self, provision_command, devserver_name,
"""Run quick_provision using GsCache server.
@param provision_command: The path of quick_provision command.
@param devserver_name: The devserver name and port (optional).
@param image_name: The image to be installed.
"""'Try quick provision with gs_cache.')
# If enabled, GsCache server listion on different port on the
# devserver.
gs_cache_server = devserver_name.replace(DEVSERVER_PORT, GS_CACHE_PORT)
gs_cache_url = ('http://%s/download/chromeos-image-archive'
% gs_cache_server)
# Check if GS_Cache server is enabled on the server.
self._run('curl -s -o /dev/null %s' % gs_cache_url)
command = '%s --noreboot %s %s' % (provision_command, image_name,
fields={'devserver': devserver_name, 'gs_cache': True})
def _quick_provision_with_devserver(self, provision_command,
devserver_name, image_name):
"""Run quick_provision using legacy devserver.
@param provision_command: The path of quick_provision command.
@param devserver_name: The devserver name and port (optional).
@param image_name: The image to be installed.
"""'Try quick provision with devserver.')
ds = dev_server.ImageServer('http://%s' % devserver_name)
ds.stage_artifacts(image_name, ['quick_provision', 'stateful'])
except dev_server.DevServerException as e:
raise error.TestFail, str(e), sys.exc_info()[2]
static_url = 'http://%s/static' % devserver_name
command = '%s --noreboot %s %s' % (provision_command, image_name,
fields={'devserver': devserver_name, 'gs_cache': False})
def _install_via_quick_provision(self):
"""Install an updating using the `quick-provision` script.
This uses the `quick-provision` script to download and install
a root FS, kernel and stateful filesystem content.
@return The kernel expected to be booted next.
if not self._use_quick_provision:
return None
image_name = url_to_image_name(self.update_url)'Installing image using quick-provision.')
provision_command = self._get_remote_script(_QUICK_PROVISION_SCRIPT)
server_name = urlparse.urlparse(self.update_url)[1]
server_name, image_name)
except Exception:
server_name, image_name)
return kernel_utils.verify_kernel_state_after_update(
except Exception:
# N.B. We handle only `Exception` here. Non-Exception
# classes (such as KeyboardInterrupt) are handled by our
# caller.
logging.exception('quick-provision script failed; '
'will fall back to update_engine.')
return None
def _install_update(self):
"""Install the requested image on the DUT, but don't start it.
This downloads and installs a root FS, kernel and stateful
filesystem content. This does not reboot the DUT, so the update
is merely pending when the method returns.
@return The kernel expected to be booted next.
"""'Installing image at %s onto %s',
return (self._install_via_quick_provision()
or self._install_via_update_engine())
# N.B. This handling code includes non-Exception classes such
# as KeyboardInterrupt. We need to clean up, but we also must
# re-raise.
# Collect update engine logs in the event of failure.
if'Collecting update engine logs due to failure...')
def _complete_update(self, expected_kernel):
"""Finish the update, and confirm that it succeeded.
Initial condition is that the target build has been downloaded
and installed on the DUT, but has not yet been booted. This
function is responsible for rebooting the DUT, and checking that
the new build is running successfully.
@param expected_kernel: kernel expected to be active after reboot.
# Regarding the 'crossystem' command below: In some cases,
# the update flow puts the TPM into a state such that it
# fails verification. We don't know why. However, this
# call papers over the problem by clearing the TPM during
# the reboot.
# We ignore failures from 'crossystem'. Although failure
# here is unexpected, and could signal a bug, the point of
# the exercise is to paper over problems; allowing this to
# fail would defeat the purpose.
self._run('crossystem clear_tpm_owner_request=1',
# Touch the lab machine file to leave a marker that
# distinguishes this image from other test images.
# Afterwards, we must re-run the autoreboot script because
# it depends on the _LAB_MACHINE_FILE.
autoreboot_cmd = ('FILE="%s" ; [ -f "$FILE" ] || '
'( touch "$FILE" ; start autoreboot )')
self._run(autoreboot_cmd % _LAB_MACHINE_FILE)
expected_kernel, NewBuildUpdateError.ROLLBACK_FAILURE,
logging.debug('Cleaning up old autotest directories.')
installed_autodir = autotest.Autotest.get_installed_autodir(
self._run('rm -rf ' + installed_autodir)
except autotest.AutodirNotFoundError:
logging.debug('No autotest installed directory found.')
def run_update(self):
"""Perform a full update of a DUT in the test lab.
This downloads and installs the root FS and stateful partition
content needed for the update specified in `` and
`self.update_url`. The update is performed according to the
requirements for provisioning a DUT for testing the requested
At the end of the procedure, metrics are reported describing the
outcome of the operation.
@returns A tuple of the form `(image_name, attributes)`, where
`image_name` is the name of the image installed, and
`attributes` is new attributes to be applied to the DUT.
server_name = dev_server.get_resolved_hostname(self.update_url)
fields={'devserver': server_name})
except _AttributedUpdateError:
except Exception as e:
logging.exception('Failure preparing host prior to update.')
raise HostUpdateError(, str(e))
expected_kernel = self._install_update()
except _AttributedUpdateError:
except Exception as e:
logging.exception('Failure during download and install.')
raise ImageInstallError(, server_name, str(e))
except _AttributedUpdateError:
except Exception as e:
logging.exception('Failure from build after update.')
raise NewBuildUpdateError(self.update_version, str(e))
image_name = url_to_image_name(self.update_url)
# update_url is different from devserver url needed to stage autotest
# packages, therefore, resolve a new devserver url here.
devserver_url = dev_server.ImageServer.resolve(
repo_url = tools.get_package_url(devserver_url, image_name)
return image_name, {ds_constants.JOB_REPO_URL: repo_url}