blob: b4d9cbe8c4bc5a834b00be9cc2a2110fecccb027 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import glob
import logging
import os
import re
import urllib2
import urlparse
from autotest_lib.client.bin import utils
from autotest_lib.client.common_lib import error, global_config
from autotest_lib.client.common_lib.cros import dev_server
from autotest_lib.server import autotest
from autotest_lib.server import utils as server_utils
from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
from autotest_lib.server.cros.dynamic_suite import tools
from chromite.lib import retry_util
try:
from chromite.lib import metrics
except ImportError:
metrics = utils.metrics_mock
def _metric_name(base_name):
return 'chromeos/autotest/provision/' + base_name
# Local stateful update path is relative to the CrOS source directory.
UPDATER_IDLE = 'UPDATE_STATUS_IDLE'
UPDATER_NEED_REBOOT = 'UPDATE_STATUS_UPDATED_NEED_REBOOT'
# A list of update engine client states that occur after an update is triggered.
UPDATER_PROCESSING_UPDATE = ['UPDATE_STATUS_CHECKING_FORUPDATE',
'UPDATE_STATUS_UPDATE_AVAILABLE',
'UPDATE_STATUS_DOWNLOADING',
'UPDATE_STATUS_FINALIZING']
_STATEFUL_UPDATE_SCRIPT = 'stateful_update'
_QUICK_PROVISION_SCRIPT = 'quick-provision'
_UPDATER_BIN = '/usr/bin/update_engine_client'
_UPDATER_LOGS = ['/var/log/messages', '/var/log/update_engine']
_KERNEL_A = {'name': 'KERN-A', 'kernel': 2, 'root': 3}
_KERNEL_B = {'name': 'KERN-B', 'kernel': 4, 'root': 5}
# Time to wait for new kernel to be marked successful after
# auto update.
_KERNEL_UPDATE_TIMEOUT = 120
# PROVISION_FAILED - A flag file to indicate provision failures. The
# file is created at the start of any AU procedure (see
# `ChromiumOSUpdater._prepare_host()`). The file's location in
# stateful means that on successul update it will be removed. Thus, if
# this file exists, it indicates that we've tried and failed in a
# previous attempt to update.
PROVISION_FAILED = '/var/tmp/provision_failed'
# A flag file used to enable special handling in lab DUTs. Some
# parts of the system in Chromium OS test images will behave in ways
# convenient to the test lab when this file is present. Generally,
# we create this immediately after any update completes.
_LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
# _TARGET_VERSION - A file containing the new version to which we plan
# to update. This file is used by the CrOS shutdown code to detect and
# handle certain version downgrade cases. Specifically: Downgrading
# may trigger an unwanted powerwash in the target build when the
# following conditions are met:
# * Source build is a v4.4 kernel with R69-10756.0.0 or later.
# * Target build predates the R69-10756.0.0 cutoff.
# When this file is present and indicates a downgrade, the OS shutdown
# code on the DUT knows how to prevent the powerwash.
_TARGET_VERSION = '/run/update_target_version'
# _REBOOT_FAILURE_MESSAGE - This is the standard message text returned
# when the Host.reboot() method fails. The source of this text comes
# from `wait_for_restart()` in client/common_lib/hosts/base_classes.py.
_REBOOT_FAILURE_MESSAGE = 'Host did not return from reboot'
class RootFSUpdateError(error.TestFail):
"""Raised when the RootFS fails to update."""
class StatefulUpdateError(error.TestFail):
"""Raised when the stateful partition fails to update."""
class _AttributedUpdateError(error.TestFail):
"""Update failure with an attributed cause."""
def __init__(self, attribution, msg):
super(_AttributedUpdateError, self).__init__(
'%s: %s' % (attribution, msg))
self._message = msg
def _classify(self):
for err_pattern, classification in self._CLASSIFIERS:
if re.match(err_pattern, self._message):
return classification
return None
@property
def failure_summary(self):
"""Summarize this error for metrics reporting."""
classification = self._classify()
if classification:
return '%s: %s' % (self._SUMMARY, classification)
else:
return self._SUMMARY
class HostUpdateError(_AttributedUpdateError):
"""Failure updating a DUT attributable to the DUT.
This class of exception should be raised when the most likely cause
of failure was a condition existing on the DUT prior to the update,
such as a hardware problem, or a bug in the software on the DUT.
"""
DUT_DOWN = 'No answer to ssh'
_SUMMARY = 'DUT failed prior to update'
_CLASSIFIERS = [
(DUT_DOWN, DUT_DOWN),
(_REBOOT_FAILURE_MESSAGE, 'Reboot failed'),
]
def __init__(self, hostname, msg):
super(HostUpdateError, self).__init__(
'Error on %s prior to update' % hostname, msg)
class DevServerError(_AttributedUpdateError):
"""Failure updating a DUT attributable to the devserver.
This class of exception should be raised when the most likely cause
of failure was the devserver serving the target image for update.
"""
_SUMMARY = 'Devserver failed prior to update'
_CLASSIFIERS = []
def __init__(self, devserver, msg):
super(DevServerError, self).__init__(
'Devserver error on %s' % devserver, msg)
class ImageInstallError(_AttributedUpdateError):
"""Failure updating a DUT when installing from the devserver.
This class of exception should be raised when the target DUT fails
to download and install the target image from the devserver, and
either the devserver or the DUT might be at fault.
"""
_SUMMARY = 'Image failed to download and install'
_CLASSIFIERS = []
def __init__(self, hostname, devserver, msg):
super(ImageInstallError, self).__init__(
'Download and install failed from %s onto %s'
% (devserver, hostname), msg)
class NewBuildUpdateError(_AttributedUpdateError):
"""Failure updating a DUT attributable to the target build.
This class of exception should be raised when updating to a new
build fails, and the most likely cause of the failure is a bug in
the newly installed target build.
"""
CHROME_FAILURE = 'Chrome failed to reach login screen'
UPDATE_ENGINE_FAILURE = ('update-engine failed to call '
'chromeos-setgoodkernel')
ROLLBACK_FAILURE = 'System rolled back to previous build'
_SUMMARY = 'New build failed'
_CLASSIFIERS = [
(CHROME_FAILURE, 'Chrome did not start'),
(UPDATE_ENGINE_FAILURE, 'update-engine did not start'),
(ROLLBACK_FAILURE, ROLLBACK_FAILURE),
]
def __init__(self, update_version, msg):
super(NewBuildUpdateError, self).__init__(
'Failure in build %s' % update_version, msg)
@property
def failure_summary(self):
#pylint: disable=missing-docstring
return 'Build failed to work after installing'
def _url_to_version(update_url):
"""Return the version based on update_url.
@param update_url: url to the image to update to.
"""
# The Chrome OS version is generally the last element in the URL. The only
# exception is delta update URLs, which are rooted under the version; e.g.,
# http://.../update/.../0.14.755.0/au/0.14.754.0. In this case we want to
# strip off the au section of the path before reading the version.
return re.sub('/au/.*', '',
urlparse.urlparse(update_url).path).split('/')[-1].strip()
def url_to_image_name(update_url):
"""Return the image name based on update_url.
From a URL like:
http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
return lumpy-release/R27-3837.0.0
@param update_url: url to the image to update to.
@returns a string representing the image name in the update_url.
"""
return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:])
def get_update_failure_reason(exception):
"""Convert an exception into a failure reason for metrics.
The passed in `exception` should be one raised by failure of
`ChromiumOSUpdater.run_update`. The returned string will describe
the failure. If the input exception value is not a truish value
the return value will be `None`.
The number of possible return strings is restricted to a limited
enumeration of values so that the string may be safely used in
Monarch metrics without worrying about cardinality of the range of
string values.
@param exception Exception to be converted to a failure reason.
@return A string suitable for use in Monarch metrics, or `None`.
"""
if exception:
if isinstance(exception, _AttributedUpdateError):
return exception.failure_summary
else:
return 'Unknown Error: %s' % type(exception).__name__
return None
def _get_devserver_build_from_update_url(update_url):
"""Get the devserver and build from the update url.
@param update_url: The url for update.
Eg: http://devserver:port/update/build.
@return: A tuple of (devserver url, build) or None if the update_url
doesn't match the expected pattern.
@raises ValueError: If the update_url doesn't match the expected pattern.
@raises ValueError: If no global_config was found, or it doesn't contain an
image_url_pattern.
"""
pattern = global_config.global_config.get_config_value(
'CROS', 'image_url_pattern', type=str, default='')
if not pattern:
raise ValueError('Cannot parse update_url, the global config needs '
'an image_url_pattern.')
re_pattern = pattern.replace('%s', '(\S+)')
parts = re.search(re_pattern, update_url)
if not parts or len(parts.groups()) < 2:
raise ValueError('%s is not an update url' % update_url)
return parts.groups()
def _list_image_dir_contents(update_url):
"""Lists the contents of the devserver for a given build/update_url.
@param update_url: An update url. Eg: http://devserver:port/update/build.
"""
if not update_url:
logging.warning('Need update_url to list contents of the devserver.')
return
error_msg = 'Cannot check contents of devserver, update url %s' % update_url
try:
devserver_url, build = _get_devserver_build_from_update_url(update_url)
except ValueError as e:
logging.warning('%s: %s', error_msg, e)
return
devserver = dev_server.ImageServer(devserver_url)
try:
devserver.list_image_dir(build)
# The devserver will retry on URLError to avoid flaky connections, but will
# eventually raise the URLError if it persists. All HTTPErrors get
# converted to DevServerExceptions.
except (dev_server.DevServerException, urllib2.URLError) as e:
logging.warning('%s: %s', error_msg, e)
def _get_metric_fields(update_url):
"""Return a dict of metric fields.
This is used for sending autoupdate metrics for the given update URL.
@param update_url Metrics fields will be calculated from this URL.
"""
build_name = url_to_image_name(update_url)
try:
board, build_type, milestone, _ = server_utils.ParseBuildName(
build_name)
except server_utils.ParseBuildNameException:
logging.warning('Unable to parse build name %s for metrics. '
'Continuing anyway.', build_name)
board, build_type, milestone = ('', '', '')
return {
'dev_server': dev_server.get_resolved_hostname(update_url),
'board': board,
'build_type': build_type,
'milestone': milestone,
}
# TODO(garnold) This implements shared updater functionality needed for
# supporting the autoupdate_EndToEnd server-side test. We should probably
# migrate more of the existing ChromiumOSUpdater functionality to it as we
# expand non-CrOS support in other tests.
class ChromiumOSUpdater(object):
"""Chromium OS specific DUT update functionality."""
def __init__(self, update_url, host=None, interactive=True,
use_quick_provision=False):
"""Initializes the object.
@param update_url: The URL we want the update to use.
@param host: A client.common_lib.hosts.Host implementation.
@param interactive: Bool whether we are doing an interactive update.
@param use_quick_provision: Whether we should attempt to perform
the update using the quick-provision script.
"""
self.update_url = update_url
self.host = host
self.interactive = interactive
self.update_version = _url_to_version(update_url)
self._use_quick_provision = use_quick_provision
def _run(self, cmd, *args, **kwargs):
"""Abbreviated form of self.host.run(...)"""
return self.host.run(cmd, *args, **kwargs)
def check_update_status(self):
"""Returns the current update engine state.
We use the `update_engine_client -status' command and parse the line
indicating the update state, e.g. "CURRENT_OP=UPDATE_STATUS_IDLE".
"""
update_status = self.host.run(command='%s -status | grep CURRENT_OP' %
_UPDATER_BIN)
return update_status.stdout.strip().split('=')[-1]
def _rootdev(self, options=''):
"""Returns the stripped output of rootdev <options>.
@param options: options to run rootdev.
"""
return self._run('rootdev %s' % options).stdout.strip()
def get_kernel_state(self):
"""Returns the (<active>, <inactive>) kernel state as a pair.
@raise RootFSUpdateError if the DUT reports a root partition
number that isn't one of the known valid values.
"""
active_root = int(re.findall('\d+\Z', self._rootdev('-s'))[0])
if active_root == _KERNEL_A['root']:
return _KERNEL_A, _KERNEL_B
elif active_root == _KERNEL_B['root']:
return _KERNEL_B, _KERNEL_A
else:
raise RootFSUpdateError(
'Encountered unknown root partition: %s' % active_root)
def _cgpt(self, flag, kernel):
"""Return numeric cgpt value for the specified flag, kernel, device."""
return int(self._run('cgpt show -n -i %d %s $(rootdev -s -d)' % (
kernel['kernel'], flag)).stdout.strip())
def _get_next_kernel(self):
"""Return the kernel that has priority for the next boot."""
priority_a = self._cgpt('-P', _KERNEL_A)
priority_b = self._cgpt('-P', _KERNEL_B)
if priority_a > priority_b:
return _KERNEL_A
else:
return _KERNEL_B
def _get_kernel_success(self, kernel):
"""Return boolean success flag for the specified kernel.
@param kernel: information of the given kernel, either _KERNEL_A
or _KERNEL_B.
"""
return self._cgpt('-S', kernel) != 0
def _get_kernel_tries(self, kernel):
"""Return tries count for the specified kernel.
@param kernel: information of the given kernel, either _KERNEL_A
or _KERNEL_B.
"""
return self._cgpt('-T', kernel)
def _get_last_update_error(self):
"""Get the last autoupdate error code."""
command_result = self._run(
'%s --last_attempt_error' % _UPDATER_BIN)
return command_result.stdout.strip().replace('\n', ', ')
def _base_update_handler_no_retry(self, run_args):
"""Base function to handle a remote update ssh call.
@param run_args: Dictionary of args passed to ssh_host.run function.
@throws: intercepts and re-throws all exceptions
"""
try:
self.host.run(**run_args)
except Exception as e:
logging.debug('exception in update handler: %s', e)
raise e
def _base_update_handler(self, run_args, err_msg_prefix=None):
"""Handle a remote update ssh call, possibly with retries.
@param run_args: Dictionary of args passed to ssh_host.run function.
@param err_msg_prefix: Prefix of the exception error message.
"""
def exception_handler(e):
"""Examines exceptions and returns True if the update handler
should be retried.
@param e: the exception intercepted by the retry util.
"""
return (isinstance(e, error.AutoservSSHTimeout) or
(isinstance(e, error.GenericHostRunError) and
hasattr(e, 'description') and
(re.search('ERROR_CODE=37', e.description) or
re.search('generic error .255.', e.description))))
try:
# Try the update twice (arg 2 is max_retry, not including the first
# call). Some exceptions may be caught by the retry handler.
retry_util.GenericRetry(exception_handler, 1,
self._base_update_handler_no_retry,
run_args)
except Exception as e:
message = err_msg_prefix + ': ' + str(e)
raise RootFSUpdateError(message)
def _wait_for_update_service(self):
"""Ensure that the update engine daemon is running, possibly
by waiting for it a bit in case the DUT just rebooted and the
service hasn't started yet.
"""
def handler(e):
"""Retry exception handler.
Assumes that the error is due to the update service not having
started yet.
@param e: the exception intercepted by the retry util.
"""
if isinstance(e, error.AutoservRunError):
logging.debug('update service check exception: %s\n'
'retrying...', e)
return True
else:
return False
# Retry at most three times, every 5s.
status = retry_util.GenericRetry(handler, 3,
self.check_update_status,
sleep=5)
# Expect the update engine to be idle.
if status != UPDATER_IDLE:
raise RootFSUpdateError(
'Update engine status is %s (%s was expected).'
% (status, UPDATER_IDLE))
def _reset_update_engine(self):
"""Resets the host to prepare for a clean update regardless of state."""
self._run('stop ui || true')
self._run('stop update-engine || true')
self._run('start update-engine')
self._wait_for_update_service()
def _reset_stateful_partition(self):
"""Clear any pending stateful update request."""
self._run('%s --stateful_change=reset 2>&1'
% self._get_stateful_update_script())
self._run('rm -f %s' % _TARGET_VERSION)
def _set_target_version(self):
"""Set the "target version" for the update."""
version_number = self.update_version.split('-')[1]
self._run('echo %s > %s' % (version_number, _TARGET_VERSION))
def _revert_boot_partition(self):
"""Revert the boot partition."""
part = self._rootdev('-s')
logging.warning('Reverting update; Boot partition will be %s', part)
return self._run('/postinst %s 2>&1' % part)
def _verify_kernel_state(self):
"""Verify that the next kernel to boot is correct for update.
This tests that the kernel state is correct for a successfully
downloaded and installed update. That is, the next kernel to
boot must be the currently inactive kernel.
@raise RootFSUpdateError if the DUT next kernel isn't the
expected next kernel.
"""
inactive_kernel = self.get_kernel_state()[1]
next_kernel = self._get_next_kernel()
if next_kernel != inactive_kernel:
raise RootFSUpdateError(
'Update failed. The kernel for next boot is %s, '
'but %s was expected.'
% (next_kernel['name'], inactive_kernel['name']))
return inactive_kernel
def _verify_update_completed(self):
"""Verifies that an update has completed.
@raise RootFSUpdateError if the DUT doesn't indicate that
download is complete and the DUT is ready for reboot.
"""
status = self.check_update_status()
if status != UPDATER_NEED_REBOOT:
error_msg = ''
if status == UPDATER_IDLE:
error_msg = 'Update error: %s' % self._get_last_update_error()
raise RootFSUpdateError(
'Update engine status is %s (%s was expected). %s'
% (status, UPDATER_NEED_REBOOT, error_msg))
return self._verify_kernel_state()
def trigger_update(self):
"""Triggers a background update."""
# If this function is called immediately after reboot (which it
# can be), there is no guarantee that the update engine is up
# and running yet, so wait for it.
self._wait_for_update_service()
autoupdate_cmd = ('%s --check_for_update --omaha_url=%s' %
(_UPDATER_BIN, self.update_url))
run_args = {'command': autoupdate_cmd}
err_prefix = 'Failed to trigger an update on %s. ' % self.host.hostname
logging.info('Triggering update via: %s', autoupdate_cmd)
metric_fields = {'success': False}
try:
self._base_update_handler(run_args, err_prefix)
metric_fields['success'] = True
finally:
c = metrics.Counter('chromeos/autotest/autoupdater/trigger')
metric_fields.update(_get_metric_fields(self.update_url))
c.increment(fields=metric_fields)
def update_image(self):
"""Updates the device root FS and kernel and verifies success."""
autoupdate_cmd = ('%s --update --omaha_url=%s' %
(_UPDATER_BIN, self.update_url))
if not self.interactive:
autoupdate_cmd = '%s --interactive=false' % autoupdate_cmd
run_args = {'command': autoupdate_cmd, 'timeout': 3600}
err_prefix = ('Failed to install device image using payload at %s '
'on %s. ' % (self.update_url, self.host.hostname))
logging.info('Updating image via: %s', autoupdate_cmd)
metric_fields = {'success': False}
try:
self._base_update_handler(run_args, err_prefix)
metric_fields['success'] = True
finally:
c = metrics.Counter('chromeos/autotest/autoupdater/update')
metric_fields.update(_get_metric_fields(self.update_url))
c.increment(fields=metric_fields)
return self._verify_update_completed()
def _get_remote_script(self, script_name):
"""Ensure that `script_name` is present on the DUT.
The given script (e.g. `stateful_update`) may be present in the
stateful partition under /usr/local/bin, or we may have to
download it from the devserver.
Determine whether the script is present or must be downloaded
and download if necessary. Then, return a command fragment
sufficient to run the script from whereever it now lives on the
DUT.
@param script_name The name of the script as expected in
/usr/local/bin and on the devserver.
@return A string with the command (minus arguments) that will
run the target script.
"""
remote_script = '/usr/local/bin/%s' % script_name
if self.host.path_exists(remote_script):
return remote_script
remote_tmp_script = '/tmp/%s' % script_name
server_name = urlparse.urlparse(self.update_url)[1]
script_url = 'http://%s/static/%s' % (server_name, script_name)
fetch_script = (
'curl -o %s %s && head -1 %s | grep "^#!" | sed "s/#!//"') % (
remote_tmp_script, script_url, remote_tmp_script)
script_interpreter = self._run(fetch_script,
ignore_status=True).stdout.strip()
if not script_interpreter:
return None
return '%s %s' % (script_interpreter, remote_tmp_script)
def _get_stateful_update_script(self):
"""Returns a command to run the stateful update script.
Find `stateful_update` on the target or install it, as
necessary. If installation fails, raise an exception.
@raise StatefulUpdateError if the script can't be found or
installed.
@return A string that can be joined with arguments to run the
`stateful_update` command on the DUT.
"""
script_command = self._get_remote_script(_STATEFUL_UPDATE_SCRIPT)
if not script_command:
raise StatefulUpdateError('Could not install %s on DUT'
% _STATEFUL_UPDATE_SCRIPT)
return script_command
def rollback_rootfs(self, powerwash):
"""Triggers rollback and waits for it to complete.
@param powerwash: If true, powerwash as part of rollback.
@raise RootFSUpdateError if anything went wrong.
"""
version = self.host.get_release_version()
# Introduced can_rollback in M36 (build 5772). # etc/lsb-release matches
# X.Y.Z. This version split just pulls the first part out.
try:
build_number = int(version.split('.')[0])
except ValueError:
logging.error('Could not parse build number.')
build_number = 0
if build_number >= 5772:
can_rollback_cmd = '%s --can_rollback' % _UPDATER_BIN
logging.info('Checking for rollback.')
try:
self._run(can_rollback_cmd)
except error.AutoservRunError as e:
raise RootFSUpdateError("Rollback isn't possible on %s: %s" %
(self.host.hostname, str(e)))
rollback_cmd = '%s --rollback --follow' % _UPDATER_BIN
if not powerwash:
rollback_cmd += ' --nopowerwash'
logging.info('Performing rollback.')
try:
self._run(rollback_cmd)
except error.AutoservRunError as e:
raise RootFSUpdateError('Rollback failed on %s: %s' %
(self.host.hostname, str(e)))
self._verify_update_completed()
def update_stateful(self, clobber=True):
"""Updates the stateful partition.
@param clobber: If True, a clean stateful installation.
@raise StatefulUpdateError if the update script fails to
complete successfully.
"""
logging.info('Updating stateful partition...')
statefuldev_url = self.update_url.replace('update', 'static')
# Attempt stateful partition update; this must succeed so that the newly
# installed host is testable after update.
statefuldev_cmd = [self._get_stateful_update_script(), statefuldev_url]
if clobber:
statefuldev_cmd.append('--stateful_change=clean')
statefuldev_cmd.append('2>&1')
try:
self._run(' '.join(statefuldev_cmd), timeout=1200)
except error.AutoservRunError:
raise StatefulUpdateError(
'Failed to perform stateful update on %s' %
self.host.hostname)
def verify_boot_expectations(self, expected_kernel, rollback_message):
"""Verifies that we fully booted given expected kernel state.
This method both verifies that we booted using the correct kernel
state and that the OS has marked the kernel as good.
@param expected_kernel: kernel that we are verifying with,
i.e. I expect to be booted onto partition 4 etc. See output of
get_kernel_state.
@param rollback_message: string include in except message text
if we booted with the wrong partition.
@raise NewBuildUpdateError if any of the various checks fail.
"""
# Figure out the newly active kernel.
active_kernel = self.get_kernel_state()[0]
# Check for rollback due to a bad build.
if active_kernel != expected_kernel:
# Kernel crash reports should be wiped between test runs, but
# may persist from earlier parts of the test, or from problems
# with provisioning.
#
# Kernel crash reports will NOT be present if the crash happened
# before encrypted stateful is mounted.
#
# TODO(dgarrett): Integrate with server/crashcollect.py at some
# point.
kernel_crashes = glob.glob('/var/spool/crash/kernel.*.kcrash')
if kernel_crashes:
rollback_message += ': kernel_crash'
logging.debug('Found %d kernel crash reports:',
len(kernel_crashes))
# The crash names contain timestamps that may be useful:
# kernel.20131207.005945.0.kcrash
for crash in kernel_crashes:
logging.debug(' %s', os.path.basename(crash))
# Print out some information to make it easier to debug
# the rollback.
logging.debug('Dumping partition table.')
self._run('cgpt show $(rootdev -s -d)')
logging.debug('Dumping crossystem for firmware debugging.')
self._run('crossystem --all')
raise NewBuildUpdateError(self.update_version, rollback_message)
# Make sure chromeos-setgoodkernel runs.
try:
utils.poll_for_condition(
lambda: (self._get_kernel_tries(active_kernel) == 0
and self._get_kernel_success(active_kernel)),
exception=RootFSUpdateError(),
timeout=_KERNEL_UPDATE_TIMEOUT, sleep_interval=5)
except RootFSUpdateError:
services_status = self._run('status system-services').stdout
if services_status != 'system-services start/running\n':
event = NewBuildUpdateError.CHROME_FAILURE
else:
event = NewBuildUpdateError.UPDATE_ENGINE_FAILURE
raise NewBuildUpdateError(self.update_version, event)
def _prepare_host(self):
"""Make sure the target DUT is working and ready for update.
Initially, the target DUT's state is unknown. The DUT is
expected to be online, but we strive to be forgiving if Chrome
and/or the update engine aren't fully functional.
"""
# Summary of work, and the rationale:
# 1. Reboot, because it's a good way to clear out problems.
# 2. Touch the PROVISION_FAILED file, to allow repair to detect
# failure later.
# 3. Run the hook for host class specific preparation.
# 4. Stop Chrome, because the system is designed to eventually
# reboot if Chrome is stuck in a crash loop.
# 5. Force `update-engine` to start, because if Chrome failed
# to start properly, the status of the `update-engine` job
# will be uncertain.
if not self.host.is_up():
raise HostUpdateError(self.host.hostname,
HostUpdateError.DUT_DOWN)
self._reset_stateful_partition()
self.host.reboot(timeout=self.host.REBOOT_TIMEOUT)
self._run('touch %s' % PROVISION_FAILED)
self.host.prepare_for_update()
self._reset_update_engine()
logging.info('Updating from version %s to %s.',
self.host.get_release_version(),
self.update_version)
def _verify_devserver(self):
"""Check that our chosen devserver is still working.
@raise DevServerError if the devserver fails any sanity check.
"""
server = 'http://%s' % urlparse.urlparse(self.update_url)[1]
try:
if not dev_server.ImageServer.devserver_healthy(server):
raise DevServerError(
server, 'Devserver is not healthy')
except Exception as e:
raise DevServerError(
server, 'Devserver is not up and available')
def _install_via_update_engine(self):
"""Install an updating using the production AU flow.
This uses the standard AU flow and the `stateful_update` script
to download and install a root FS, kernel and stateful
filesystem content.
@return The kernel expected to be booted next.
"""
logging.info('Installing image using update_engine.')
expected_kernel = self.update_image()
self.update_stateful()
self._set_target_version()
return expected_kernel
def _install_via_quick_provision(self):
"""Install an updating using the `quick-provision` script.
This uses the `quick-provision` script to download and install
a root FS, kernel and stateful filesystem content.
@return The kernel expected to be booted next.
"""
if not self._use_quick_provision:
return None
build_re = global_config.global_config.get_config_value(
'CROS', 'quick_provision_build_regex', type=str, default='')
image_name = url_to_image_name(self.update_url)
if not build_re or re.match(build_re, image_name) is None:
logging.info('Not eligible for quick-provision.')
return None
logging.info('Installing image using quick-provision.')
provision_command = self._get_remote_script(_QUICK_PROVISION_SCRIPT)
server_name = urlparse.urlparse(self.update_url)[1]
static_url = 'http://%s/static' % server_name
command = '%s --noreboot %s %s' % (
provision_command, image_name, static_url)
try:
self._run(command)
self._set_target_version()
return self._verify_kernel_state()
except Exception:
# N.B. We handle only `Exception` here. Non-Exception
# classes (such as KeyboardInterrupt) are handled by our
# caller.
logging.exception('quick-provision script failed; '
'will fall back to update_engine.')
self._revert_boot_partition()
self._reset_stateful_partition()
self._reset_update_engine()
return None
def _install_update(self):
"""Install the requested image on the DUT, but don't start it.
This downloads and installs a root FS, kernel and stateful
filesystem content. This does not reboot the DUT, so the update
is merely pending when the method returns.
@return The kernel expected to be booted next.
"""
logging.info('Installing image at %s onto %s',
self.update_url, self.host.hostname)
try:
return (self._install_via_quick_provision()
or self._install_via_update_engine())
except:
# N.B. This handling code includes non-Exception classes such
# as KeyboardInterrupt. We need to clean up, but we also must
# re-raise.
self._revert_boot_partition()
self._reset_stateful_partition()
self._reset_update_engine()
# Collect update engine logs in the event of failure.
if self.host.job:
logging.info('Collecting update engine logs due to failure...')
self.host.get_file(
_UPDATER_LOGS, self.host.job.sysinfo.sysinfodir,
preserve_perm=False)
_list_image_dir_contents(self.update_url)
raise
def _complete_update(self, expected_kernel):
"""Finish the update, and confirm that it succeeded.
Initial condition is that the target build has been downloaded
and installed on the DUT, but has not yet been booted. This
function is responsible for rebooting the DUT, and checking that
the new build is running successfully.
@param expected_kernel: kernel expected to be active after reboot.
"""
# Regarding the 'crossystem' command below: In some cases,
# the update flow puts the TPM into a state such that it
# fails verification. We don't know why. However, this
# call papers over the problem by clearing the TPM during
# the reboot.
#
# We ignore failures from 'crossystem'. Although failure
# here is unexpected, and could signal a bug, the point of
# the exercise is to paper over problems; allowing this to
# fail would defeat the purpose.
self._run('crossystem clear_tpm_owner_request=1',
ignore_status=True)
self.host.reboot(timeout=self.host.REBOOT_TIMEOUT)
# Touch the lab machine file to leave a marker that
# distinguishes this image from other test images.
# Afterwards, we must re-run the autoreboot script because
# it depends on the _LAB_MACHINE_FILE.
autoreboot_cmd = ('FILE="%s" ; [ -f "$FILE" ] || '
'( touch "$FILE" ; start autoreboot )')
self._run(autoreboot_cmd % _LAB_MACHINE_FILE)
self.verify_boot_expectations(
expected_kernel, NewBuildUpdateError.ROLLBACK_FAILURE)
logging.debug('Cleaning up old autotest directories.')
try:
installed_autodir = autotest.Autotest.get_installed_autodir(
self.host)
self._run('rm -rf ' + installed_autodir)
except autotest.AutodirNotFoundError:
logging.debug('No autotest installed directory found.')
def run_update(self):
"""Perform a full update of a DUT in the test lab.
This downloads and installs the root FS and stateful partition
content needed for the update specified in `self.host` and
`self.update_url`. The update is performed according to the
requirements for provisioning a DUT for testing the requested
build.
At the end of the procedure, metrics are reported describing the
outcome of the operation.
@returns A tuple of the form `(image_name, attributes)`, where
`image_name` is the name of the image installed, and
`attributes` is new attributes to be applied to the DUT.
"""
server_name = dev_server.get_resolved_hostname(self.update_url)
metrics.Counter(_metric_name('install')).increment(
fields={'devserver': server_name})
self._verify_devserver()
try:
self._prepare_host()
except _AttributedUpdateError:
raise
except Exception as e:
logging.exception('Failure preparing host prior to update.')
raise HostUpdateError(self.host.hostname, str(e))
try:
expected_kernel = self._install_update()
except _AttributedUpdateError:
raise
except Exception as e:
logging.exception('Failure during download and install.')
server_name = dev_server.get_resolved_hostname(self.update_url)
raise ImageInstallError(self.host.hostname, server_name, str(e))
try:
self._complete_update(expected_kernel)
except _AttributedUpdateError:
raise
except Exception as e:
logging.exception('Failure from build after update.')
raise NewBuildUpdateError(self.update_version, str(e))
image_name = url_to_image_name(self.update_url)
# update_url is different from devserver url needed to stage autotest
# packages, therefore, resolve a new devserver url here.
devserver_url = dev_server.ImageServer.resolve(
image_name, self.host.hostname).url()
repo_url = tools.get_package_url(devserver_url, image_name)
return image_name, {ds_constants.JOB_REPO_URL: repo_url}