blob: a2ba95d9f95a532f426580be55f23a0fe5d08ade [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import functools
import httplib
import logging
import os
import re
import socket
import subprocess
import time
import xmlrpclib
from autotest_lib.client.bin import utils
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import global_config
from autotest_lib.client.common_lib.cros import autoupdater
from autotest_lib.client.common_lib.cros import dev_server
from autotest_lib.client.common_lib.cros import retry
from autotest_lib.client.cros import constants
from autotest_lib.server import autoserv_parser
from autotest_lib.server import autotest
from autotest_lib.server import site_host_attributes
from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
from autotest_lib.server.cros.dynamic_suite import tools, frontend_wrappers
from autotest_lib.server.cros.servo import servo
from autotest_lib.server.hosts import remote
from autotest_lib.site_utils.rpm_control_system import rpm_client
def _make_servo_hostname(hostname):
host_parts = hostname.split('.')
host_parts[0] = host_parts[0] + '-servo'
return '.'.join(host_parts)
def _get_lab_servo(target_hostname):
"""Instantiate a Servo for |target_hostname| in the lab.
Assuming that |target_hostname| is a device in the CrOS test
lab, create and return a Servo object pointed at the servo
attached to that DUT. The servo in the test lab is assumed
to already have servod up and running on it.
@param target_hostname: device whose servo we want to target.
@return an appropriately configured Servo instance.
servo_host = _make_servo_hostname(target_hostname)
if utils.host_is_in_lab_zone(servo_host):
return servo.Servo(servo_host=servo_host)
except: # pylint: disable=W0702
# TODO(jrbarnette): Long-term, if we can't get to
# a servo in the lab, we want to fail, so we should
# pass any exceptions along. Short-term, we're not
# ready to rely on servo, so we ignore failures.
return None
def make_ssh_command(user='root', port=22, opts='', hosts_file=None,
connect_timeout=None, alive_interval=None):
"""Override default make_ssh_command to use options tuned for Chrome OS.
Tuning changes:
- ConnectTimeout=30; maximum of 30 seconds allowed for an SSH connection
failure. Consistency with
- ServerAliveInterval=180; which causes SSH to ping connection every
180 seconds. In conjunction with ServerAliveCountMax ensures that if the
connection dies, Autotest will bail out quickly. Originally tried 60 secs,
but saw frequent job ABORTS where the test completed successfully.
- ServerAliveCountMax=3; consistency with
- ConnectAttempts=4; reduce flakiness in connection errors; consistency
- UserKnownHostsFile=/dev/null; we don't care about the keys. Host keys
change with every new installation, don't waste memory/space saving them.
- SSH protocol forced to 2; needed for ServerAliveInterval.
@param user User name to use for the ssh connection.
@param port Port on the target host to use for ssh connection.
@param opts Additional options to the ssh command.
@param hosts_file Ignored.
@param connect_timeout Ignored.
@param alive_interval Ignored.
base_command = ('/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no'
' -o UserKnownHostsFile=/dev/null -o BatchMode=yes'
' -o ConnectTimeout=30 -o ServerAliveInterval=180'
' -o ServerAliveCountMax=3 -o ConnectionAttempts=4'
' -o Protocol=2 -l %s -p %d')
return base_command % (opts, user, port)
def add_label_detector(label_function_list, label_list=None, label=None):
"""Decorator used to group functions together into the provided list.
@param label_function_list: List of label detecting functions to add
decorated function to.
@param label_list: List of detectable labels to add detectable labels to.
(Default: None)
@param label: Label string that is detectable by this detection function
(Default: None)
def add_func(func):
@param func: The function to be added as a detector.
if label and label_list is not None:
return func
return add_func
class SiteHost(remote.RemoteHost):
"""Chromium OS specific subclass of Host."""
_parser = autoserv_parser.autoserv_parser
_AFE = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10)
# Time to wait for new kernel to be marked successful after
# auto update.
# Timeout values (in seconds) associated with various Chrome OS
# state changes.
# In general, a good rule of thumb is that the timeout can be up
# to twice the typical measured value on the slowest platform.
# The times here have not necessarily been empirically tested to
# meet this criterion.
# SLEEP_TIMEOUT: Time to allow for suspend to memory.
# RESUME_TIMEOUT: Time to allow for resume after suspend, plus
# time to restart the netwowrk.
# BOOT_TIMEOUT: Time to allow for boot from power off. Among
# other things, this must account for the 30 second dev-mode
# screen delay and time to start the network.
# USB_BOOT_TIMEOUT: Time to allow for boot from a USB device,
# including the 30 second dev-mode delay and time to start the
# network.
# SHUTDOWN_TIMEOUT: Time to allow for shut down.
# REBOOT_TIMEOUT: Combination of shutdown and reboot times.
# _UPDATE_REBOOT_TIMEOUT: Time to allow for reboot after AU; this
# time provides no allowance for the 30 second dev-mode delay,
# but is deliberately generous to avoid try-job failures.
# _INSTALL_TIMEOUT: Time to allow for chromeos-install.
# TODO(jrbarnette) - temporarily set this value to 2 min to allow
# for Reset to 1 minute once that bug
# is fixed.
# _USB_POWER_TIMEOUT: Time to allow for USB to power toggle ON and OFF.
# _POWER_CYCLE_TIMEOUT: Time to allow for manual power cycle.
_RPM_RECOVERY_BOARDS = global_config.global_config.get_config_value('CROS',
'rpm_recovery_boards', type=str).split(',')
_LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
_RPM_HOSTNAME_REGEX = ('chromeos[0-9]+(-row[0-9]+)?-rack[0-9]+[a-z]*-'
_LIGHTSENSOR_FILES = ['in_illuminance0_input',
_LIGHTSENSOR_SEARCH_DIR = '/sys/bus/iio/devices'
label_decorator = functools.partial(add_label_detector, _LABEL_FUNCTIONS,
# Constants used in ping_wait_up() and ping_wait_down().
# _PING_WAIT_COUNT is the approximate number of polling
# cycles to use when waiting for a host state change.
# _PING_STATUS_DOWN and _PING_STATUS_UP are names used
# for arguments to the internal _ping_wait_for_status()
# method.
# Allowed values for the power_method argument.
# POWER_CONTROL_RPM: Passed as default arg for power_off/on/cycle() methods.
# POWER_CONTROL_SERVO: Used in set_power() and power_cycle() methods.
# POWER_CONTROL_MANUAL: Used in set_power() and power_cycle() methods.
_RPM_OUTLET_CHANGED = 'outlet_changed'
def get_servo_arguments(args_dict):
"""Extract servo options from `args_dict` and return the result.
Take the provided dictionary of argument options and return
a subset that represent standard arguments needed to
construct a servo object for a host. The intent is to
provide standard argument processing from run_remote_tests
for tests that require a servo to operate.
Recommended usage:
args_dict = utils.args_to_dict(args)
servo_args = hosts.SiteHost.get_servo_arguments(args_dict)
host = hosts.create_host(machine, servo_args=servo_args)
@param args_dict Dictionary from which to extract the servo
servo_args = {}
for arg in ('servo_host', 'servo_port'):
if arg in args_dict:
servo_args[arg] = args_dict[arg]
return servo_args
def _initialize(self, hostname, servo_args=None, *args, **dargs):
"""Initialize superclasses, and |self.servo|.
For creating the host servo object, there are three
possibilities: First, if the host is a lab system known to
have a servo board, we connect to that servo unconditionally.
Second, if we're called from a control file that requires
servo features for testing, it will pass settings for
`servo_host`, `servo_port`, or both. If neither of these
cases apply, `self.servo` will be `None`.
super(SiteHost, self)._initialize(hostname=hostname,
*args, **dargs)
# self.env is a dictionary of environment variable settings
# to be exported for commands run on the host.
# LIBC_FATAL_STDERR_ can be useful for diagnosing certain
# errors that might happen.
self.env['LIBC_FATAL_STDERR_'] = '1'
self._xmlrpc_proxy_map = {}
self.servo = _get_lab_servo(hostname)
if not self.servo and servo_args is not None:
self.servo = servo.Servo(**servo_args)
def get_repair_image_name(self):
"""Generate a image_name from variables in the global config.
@returns a str of $board-version/$BUILD.
stable_version = global_config.global_config.get_config_value(
'CROS', 'stable_cros_version')
build_pattern = global_config.global_config.get_config_value(
'CROS', 'stable_build_pattern')
board = self._get_board_from_afe()
if board is None:
raise error.AutoservError('DUT has no board attribute, '
'cannot be repaired.')
return build_pattern % (board, stable_version)
def _host_in_AFE(self):
"""Check if the host is an object the AFE knows.
@returns the host object.
return self._AFE.get_hosts(hostname=self.hostname)
def clear_cros_version_labels_and_job_repo_url(self):
"""Clear cros_version labels and host attribute job_repo_url."""
if not self._host_in_AFE():
host_list = [self.hostname]
labels = self._AFE.get_labels(
for label in labels:
self._AFE.set_host_attribute('job_repo_url', None,
def add_cros_version_labels_and_job_repo_url(self, image_name):
"""Add cros_version labels and host attribute job_repo_url.
@param image_name: The name of the image e.g.
if not self._host_in_AFE():
cros_label = '%s%s' % (ds_constants.VERSION_PREFIX, image_name)
devserver_url = dev_server.ImageServer.resolve(image_name).url()
labels = self._AFE.get_labels(name=cros_label)
if labels:
label = labels[0]
label = self._AFE.create_label(name=cros_label)
repo_url = tools.get_package_url(devserver_url, image_name)
self._AFE.set_host_attribute('job_repo_url', repo_url,
def _try_stateful_update(self, update_url, force_update, updater):
"""Try to use stateful update to initialize DUT.
When DUT is already running the same version that machine_install
tries to install, stateful update is a much faster way to clean up
the DUT for testing, compared to a full reimage. It is implemeted
by calling autoupdater.run_update, but skipping updating root, as
updating the kernel is time consuming and not necessary.
@param update_url: url of the image.
@param force_update: Set to True to update the image even if the DUT
is running the same version.
@param updater: ChromiumOSUpdater instance used to update the DUT.
@returns: True if the DUT was updated with stateful update.
if not updater.check_version():
return False
if not force_update:'Canceling stateful update because the new and '
'old versions are the same.')
return False
# Following folders should be rebuilt after stateful update.
# A test file is used to confirm each folder gets rebuilt after
# the stateful update.
folders_to_check = ['/var', '/home', '/mnt/stateful_partition']
test_file = '.test_file_to_be_deleted'
for folder in folders_to_check:
touch_path = os.path.join(folder, test_file)'touch %s' % touch_path)
if not updater.run_update(force_update=True, update_root=False):
return False
# Reboot to complete stateful update.
self.reboot(timeout=self._UPDATE_REBOOT_TIMEOUT, wait=True)
check_file_cmd = 'test -f %s; echo $?'
for folder in folders_to_check:
test_file_path = os.path.join(folder, test_file)
result = % test_file_path,
if result.exit_status == 1:
return False
return True
def _post_update_processing(self, updater, inactive_kernel=None):
"""After the DUT is updated, confirm machine_install succeeded.
@param updater: ChromiumOSUpdater instance used to update the DUT.
@param inactive_kernel: kernel state of inactive kernel before reboot.
# Touch the lab machine file to leave a marker that distinguishes
# this image from other test images.'touch %s' % self._LAB_MACHINE_FILE)
# Kick off the autoreboot script as the _LAB_MACHINE_FILE was
# missing on the first boot.'start autoreboot')
# Following the reboot, verify the correct version.
if not updater.check_version_to_confirm_install():
# Print out crossystem to make it easier to debug the rollback.
logging.debug('Dumping partition table.')'cgpt show $(rootdev -s -d)')
logging.debug('Dumping crossystem for firmware debugging.')'crossystem --all')
logging.error('Expected Chromium OS version: %s. '
'Found Chromium OS %s',
updater.update_version, updater.get_build_id())
raise autoupdater.ChromiumOSError('Updater failed on host %s' %
# Figure out newly active kernel.
new_active_kernel, _ = updater.get_kernel_state()
# Ensure that previously inactive kernel is now the active kernel.
if inactive_kernel and new_active_kernel != inactive_kernel:
raise autoupdater.ChromiumOSError(
'Update failed. New kernel partition is not active after'
' boot.')
host_attributes = None
if self._host_in_AFE():
host_attributes = site_host_attributes.HostAttributes(self.hostname)
if host_attributes and host_attributes.has_chromeos_firmware:
# Wait until tries == 0 and success, or until timeout.
lambda: (updater.get_kernel_tries(new_active_kernel) == 0
and updater.get_kernel_success(new_active_kernel)),
'Update failed. Timed out waiting for system to mark'
' new kernel as successful.'),
timeout=self._KERNEL_UPDATE_TIMEOUT, sleep_interval=5)
def _stage_build_and_return_update_url(self, image_name):
"""Stage a build on a devserver and return the update_url.
@param image_name: a name like lumpy-release/R27-3837.0.0
@returns an update URL like:
"""'Staging requested build: %s', image_name)
devserver = dev_server.ImageServer.resolve(image_name)
devserver.trigger_download(image_name, synchronous=False)
return tools.image_url_pattern() % (devserver.url(), image_name)
def machine_install(self, update_url=None, force_update=False,
local_devserver=False, repair=False):
"""Install the DUT.
Use stateful update if the DUT is already running the same build.
Stateful update does not update kernel and tends to run much faster
than a full reimage. If the DUT is running a different build, or it
failed to do a stateful update, full update, including kernel update,
will be applied to the DUT.
Once a host enters machine_install its cros_version label will be
removed as well as its host attribute job_repo_url (used for
package install).
@param update_url: The url to use for the update
pattern: http://$devserver:###/update/$build
If update_url is None and repair is True we will install the
stable image listed in global_config under
@param force_update: Force an update even if the version installed
is the same. Default:False
@param local_devserver: Used by run_remote_test to allow people to
use their local devserver. Default: False
@param repair: Whether or not we are in repair mode. This adds special
cases for repairing a machine like starting update_engine.
Setting repair to True sets force_update to True as well.
default: False
@raises autoupdater.ChromiumOSError
if not update_url and self._parser.options.image:
requested_build = self._parser.options.image
if requested_build.startswith('http://'):
update_url = requested_build
# Try to stage any build that does not start with http:// on
# the devservers defined in global_config.ini.
update_url = self._stage_build_and_return_update_url(
elif not update_url and not repair:
raise autoupdater.ChromiumOSError(
'Update failed. No update URL provided.')
elif not update_url and repair:
update_url = self._stage_build_and_return_update_url(
if repair:
# In case the system is in a bad state, we always reboot the machine
# before machine_install.
self.reboot(timeout=self._UPDATE_REBOOT_TIMEOUT, wait=True)'stop update-engine; start update-engine')
force_update = True
updater = autoupdater.ChromiumOSUpdater(update_url, host=self,
updated = False
# Remove cros-version and job_repo_url host attribute from host.
# If the DUT is already running the same build, try stateful update
# first. Stateful update does not update kernel and tends to run much
# faster than a full reimage.
updated = self._try_stateful_update(update_url, force_update,
if updated:'DUT is updated with stateful update.')
except Exception as e:
logging.warn('Failed to stateful update DUT, force to update.')
inactive_kernel = None
# Do a full update if stateful update is not applicable or failed.
if not updated:
# In case the system is in a bad state, we always reboot the
# machine before machine_install.
self.reboot(timeout=self._UPDATE_REBOOT_TIMEOUT, wait=True)
# TODO(sosa): Remove temporary hack to get rid of bricked machines
# that can't update due to a corrupted policy.'rm -rf /var/lib/whitelist')'touch /var/lib/whitelist')'chmod -w /var/lib/whitelist')'stop update-engine; start update-engine')
if updater.run_update(force_update):
updated = True
# Figure out active and inactive kernel.
active_kernel, inactive_kernel = updater.get_kernel_state()
# Ensure inactive kernel has higher priority than active.
if (updater.get_kernel_priority(inactive_kernel)
< updater.get_kernel_priority(active_kernel)):
raise autoupdater.ChromiumOSError(
'Update failed. The priority of the inactive kernel'
' partition is less than that of the active kernel'
' partition.')
update_engine_log = '/var/log/update_engine.log''Dumping %s', update_engine_log)'cat %s' % update_engine_log)
# Updater has returned successfully; reboot the host.
if updated:
self._post_update_processing(updater, inactive_kernel)
image_name = autoupdater.url_to_image_name(update_url)
# Clean up any old autotest directories which may be lying around.
for path in global_config.global_config.get_config_value(
'AUTOSERV', 'client_autodir_paths', type=list):'rm -rf ' + path)
def _get_label_from_afe(self, label_prefix):
"""Retrieve a host's specific label from the AFE.
Looks for a host label that has the form <label_prefix>:<value>
and returns the "<value>" part of the label. None is returned
if there is not a label matching the pattern
@returns the label that matches the prefix or 'None'
labels = self._AFE.get_labels(name__startswith=label_prefix,
if labels and len(labels) == 1:
return labels[0].name.split(label_prefix, 1)[1]
def _get_board_from_afe(self):
"""Retrieve this host's board from its labels in the AFE.
Looks for a host label of the form "board:<board>", and
returns the "<board>" part of the label. `None` is returned
if there is not a single, unique label matching the pattern.
@returns board from label, or `None`.
return self._get_label_from_afe(ds_constants.BOARD_PREFIX)
def get_build(self):
"""Retrieve the current build for this Host from the AFE.
Looks through this host's labels in the AFE to determine its build.
@returns The current build or None if it could not find it or if there
were multiple build labels assigned to this host.
return self._get_label_from_afe(ds_constants.VERSION_PREFIX)
def _install_repair(self):
"""Attempt to repair this host using upate-engine.
If the host is up, try installing the DUT with a stable
"repair" version of Chrome OS as defined in the global_config
under CROS.stable_cros_version.
@raises AutoservRepairMethodNA if the DUT is not reachable.
@raises ChromiumOSError if the install failed for some reason.
if not self.is_up():
raise error.AutoservRepairMethodNA('DUT unreachable for install.')'Attempting to reimage machine to repair image.')
except autoupdater.ChromiumOSError as e:
logging.exception(e)'Repair via install failed.')
def servo_install(self, image_url=None):
Re-install the OS on the DUT by:
1) installing a test image on a USB storage device attached to the Servo
2) booting that image in recovery mode, and then
3) installing the image with chromeos-install.
@param image_url: If specified use as the url to install on the DUT.
otherwise boot the currently staged image on the USB stick.
@raises AutoservError if the image fails to boot.
if not self.wait_up(timeout=self.USB_BOOT_TIMEOUT):
raise error.AutoservRepairFailure(
'DUT failed to boot from USB after %d seconds' %
self.USB_BOOT_TIMEOUT)'chromeos-install --yes', timeout=self._INSTALL_TIMEOUT)
if not self.wait_up(timeout=self.BOOT_TIMEOUT):
raise error.AutoservError('DUT failed to reboot installed '
'test image after %d seconds' %
def servo_repair_reinstall(self, image_url=None):
"""Reinstall the DUT utilizing servo and a test image.
Re-install the OS on the DUT by:
1) installing a test image on a USB storage device attached to the Servo
2) booting that image in recovery mode, and then
3) installing the image with chromeos-install.
@param image_url: If specified use as the url to install on the DUT.
otherwise use the latest image staged on the devserver for the
particular board the DUT has.
@raises AutoservRepairMethodNA if the device does not have servo
if not self.servo:
raise error.AutoservRepairMethodNA('Repair Reinstall NA: '
'DUT has no servo support.')'Attempting to recovery servo enabled device with '
board = self._get_board_from_afe()
if board is None:
raise error.AutoservRepairMethodNA('DUT has no board attribute, '
'cannot be repaired.')
if not image_url:
image_url = dev_server.ImageServer.devserver_url_for_servo(board)
def _servo_repair_power(self):
"""Attempt to repair DUT using an attached Servo.
Attempt to power on the DUT via power_long_press.
@raises AutoservRepairMethodNA if the device does not have servo
@raises AutoservRepairFailure if the repair fails for any reason.
if not self.servo:
raise error.AutoservRepairMethodNA('Repair Power NA: '
'DUT has no servo support.')'Attempting to recover servo enabled device by '
'powering it off and on.')
if self.wait_up(self.BOOT_TIMEOUT):
raise error.AutoservRepairFailure('DUT did not boot after long_press.')
def _powercycle_to_repair(self):
"""Utilize the RPM Infrastructure to bring the host back up.
If the host is not up/repaired after the first powercycle we utilize
auto fallback to the last good install by powercycling and rebooting the
host 6 times.
@raises AutoservRepairMethodNA if the device does not support remote
@raises AutoservRepairFailure if the repair fails for any reason.
if not self.has_power():
raise error.AutoservRepairMethodNA('Device does not support power.')'Attempting repair via RPM powercycle.')
failed_cycles = 0
while not self.wait_up(timeout=self.BOOT_TIMEOUT):
failed_cycles += 1
if failed_cycles >= self._MAX_POWER_CYCLE_ATTEMPTS:
raise error.AutoservRepairFailure(
'Powercycled host %s %d times; device did not come back'
' online.' % (self.hostname, failed_cycles))
if failed_cycles == 0:'Powercycling was successful first time.')
else:'Powercycling was successful after %d failures.',
def repair_full(self):
"""Repair a host for repair level NO_PROTECTION.
This overrides the base class function for repair; it does
not call back to the parent class, but instead offers a
simplified implementation based on the capabilities in the
Chrome OS test lab.
If `self.verify()` fails, the following procedures are
1. Try to re-install to a known stable image using
2. If there's a servo for the DUT, try to power the DUT off and
3. If there's a servo for the DUT, try to re-install via
the servo.
4. If the DUT can be power-cycled via RPM, try to repair
by power-cycling.
As with the parent method, the last operation performed on
the DUT must be to call `self.verify()`; if that call fails,
the exception it raises is passed back to the caller.
@raises AutoservRepairTotalFailure if the repair process fails to
fix the DUT.
# TODO(scottz): This should use something similar to label_decorator,
# but needs to be populated in order so DUTs are repaired with the
# least amount of effort.
repair_funcs = [self._install_repair, self._servo_repair_power,
errors = []
for repair_func in repair_funcs:
except Exception as e:
logging.warn('Failed to repair device: %s', e)
raise error.AutoservRepairTotalFailure(
'All attempts at repairing the device failed:\n%s' %
def close(self):
super(SiteHost, self).close()
def _cleanup_poweron(self):
"""Special cleanup method to make sure hosts always get power back."""
afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10)
hosts = afe.get_hosts(hostname=self.hostname)
if not hosts or not (self._RPM_OUTLET_CHANGED in
logging.debug('This host has recently interacted with the RPM'
' Infrastructure. Ensuring power is on.')
except rpm_client.RemotePowerException:
# If cleanup has completed but there was an issue with the RPM
# Infrastructure, log an error message rather than fail cleanup
logging.error('Failed to turn Power On for this host after '
'cleanup through the RPM Infrastructure.')
afe.set_host_attribute(self._RPM_OUTLET_CHANGED, None,
def cleanup(self):
client_at = autotest.Autotest(self)'rm -f %s' % constants.CLEANUP_LOGS_PAUSED_FILE)
'_clear_login_prompt_state')'restart ui')
except (error.AutotestRunError, error.AutoservRunError):
logging.warn('Unable to restart ui, rebooting device.')
# Since restarting the UI fails fall back to normal Autotest
# cleanup routines, i.e. reboot the machine.
super(SiteHost, self).cleanup()
# Check if the rpm outlet was manipulated.
if self.has_power():
def reboot(self, **dargs):
This function reboots the site host. The more generic
RemoteHost.reboot() performs sync and sleeps for 5
seconds. This is not necessary for Chrome OS devices as the
sync should be finished in a short time during the reboot
if 'reboot_cmd' not in dargs:
dargs['reboot_cmd'] = ('((reboot & sleep 10; reboot -f &)'
' </dev/null >/dev/null 2>&1 &)')
# Enable fastsync to avoid running extra sync commands before reboot.
if 'fastsync' not in dargs:
dargs['fastsync'] = True
super(SiteHost, self).reboot(**dargs)
def verify_software(self):
"""Verify working software on a Chrome OS system.
Tests for the following conditions:
1. All conditions tested by the parent version of this
2. Sufficient space in /mnt/stateful_partition.
3. Sufficient space in /mnt/stateful_partition/encrypted.
4. update_engine answers a simple status request over DBus.
super(SiteHost, self).verify_software()
'SERVER', 'gb_diskspace_required', type=float,
'SERVER', 'gb_encrypted_diskspace_required', type=float,
default=0.1))'update_engine_client --status')
# Makes sure python is present, loads and can use built in functions.
# We have seen cases where importing cPickle fails with undefined
# symbols in'python -c "import cPickle"')
def xmlrpc_connect(self, command, port, command_name=None,
ready_test_name=None, timeout_seconds=10):
"""Connect to an XMLRPC server on the host.
The `command` argument should be a simple shell command that
starts an XMLRPC server on the given `port`. The command
must not daemonize, and must terminate cleanly on SIGTERM.
The command is started in the background on the host, and a
local XMLRPC client for the server is created and returned
to the caller.
Note that the process of creating an XMLRPC client makes no
attempt to connect to the remote server; the caller is
responsible for determining whether the server is running
correctly, and is ready to serve requests.
Optionally, the caller can pass ready_test_name, a string
containing the name of a method to call on the proxy. This
method should take no parameters and return successfully only
when the server is ready to process client requests. When
ready_test_name is set, xmlrpc_connect will block until the
proxy is ready, and throw a TestError if the server isn't
ready by timeout_seconds.
@param command Shell command to start the server.
@param port Port number on which the server is expected to
be serving.
@param command_name String to use as input to `pkill` to
terminate the XMLRPC server on the host.
@param ready_test_name String containing the name of a
method defined on the XMLRPC server.
@param timeout_seconds Number of seconds to wait
for the server to become 'ready.' Will throw a
TestFail error if server is not ready in time.
# Chrome OS on the target closes down most external ports
# for security. We could open the port, but doing that
# would conflict with security tests that check that only
# expected ports are open. So, to get to the port on the
# target we use an ssh tunnel.
local_port = utils.get_unused_port()
tunnel_options = '-n -N -q -L %d:localhost:%d' % (local_port, port)
ssh_cmd = make_ssh_command(opts=tunnel_options)
tunnel_cmd = '%s %s' % (ssh_cmd, self.hostname)
logging.debug('Full tunnel command: %s', tunnel_cmd)
tunnel_proc = subprocess.Popen(tunnel_cmd, shell=True, close_fds=True)
logging.debug('Started XMLRPC tunnel, local = %d'
' remote = %d, pid = %d',
local_port, port,
# Start the server on the host. Redirection in the command
# below is necessary, because 'ssh' won't terminate until
# background child processes close stdin, stdout, and
# stderr.
remote_cmd = '( %s ) </dev/null >/dev/null 2>&1 & echo $!' % command
remote_pid ='\n')
logging.debug('Started XMLRPC server on host %s, pid = %s',
self.hostname, remote_pid)
self._xmlrpc_proxy_map[port] = (command_name, tunnel_proc)
rpc_url = 'http://localhost:%d' % local_port
proxy = xmlrpclib.ServerProxy(rpc_url, allow_none=True)
if ready_test_name is not None:
# retry.retry logs each attempt; calculate delay_sec to
# keep log spam to a dull roar.
delay_sec=min(max(timeout_seconds/20.0, 0.1), 1))
def ready_test():
""" Call proxy.ready_test_name(). """
getattr(proxy, ready_test_name)()
successful = False
try:'Waiting %d seconds for XMLRPC server '
'to start.', timeout_seconds)
successful = True
except retry.TimeoutException:
raise error.TestError('Unable to start XMLRPC server after '
'%d seconds.' % timeout_seconds)
if not successful:
logging.error('Failed to start XMLRPC server.')
self.xmlrpc_disconnect(port)'XMLRPC server started successfully.')
return proxy
def xmlrpc_disconnect(self, port):
"""Disconnect from an XMLRPC server on the host.
Terminates the remote XMLRPC server previously started for
the given `port`. Also closes the local ssh tunnel created
for the connection to the host. This function does not
directly alter the state of a previously returned XMLRPC
client object; however disconnection will cause all
subsequent calls to methods on the object to fail.
This function does nothing if requested to disconnect a port
that was not previously connected via `self.xmlrpc_connect()`
@param port Port number passed to a previous call to
if port not in self._xmlrpc_proxy_map:
entry = self._xmlrpc_proxy_map[port]
remote_name = entry[0]
tunnel_proc = entry[1]
if remote_name:
# We use 'pkill' to find our target process rather than
# a PID, because the host may have rebooted since
# connecting, and we don't want to kill an innocent
# process with the same PID.
# 'pkill' helpfully exits with status 1 if no target
# process is found, for which run() will throw an
# exception. We don't want that, so we the ignore
# status."pkill -f '%s'" % remote_name, ignore_status=True)
if tunnel_proc.poll() is None:
logging.debug('Terminated tunnel, pid %d',
logging.debug('Tunnel pid %d terminated early, status %d',, tunnel_proc.returncode)
del self._xmlrpc_proxy_map[port]
def xmlrpc_disconnect_all(self):
"""Disconnect all known XMLRPC proxy ports."""
for port in self._xmlrpc_proxy_map.keys():
def _ping_check_status(self, status):
"""Ping the host once, and return whether it has a given status.
@param status Check the ping status against this value.
@return True iff `status` and the result of ping are the same
(i.e. both True or both False).
ping_val =, tries=1, deadline=1)
return not (status ^ (ping_val == 0))
def _ping_wait_for_status(self, status, timeout):
"""Wait for the host to have a given status (UP or DOWN).
Status is checked by polling. Polling will not last longer
than the number of seconds in `timeout`. The polling
interval will be long enough that only approximately
_PING_WAIT_COUNT polling cycles will be executed, subject
to a maximum interval of about one minute.
@param status Waiting will stop immediately if `ping` of the
host returns this status.
@param timeout Poll for at most this many seconds.
@return True iff the host status from `ping` matched the
requested status at the time of return.
# _ping_check_status() takes about 1 second, hence the
# "- 1" in the formula below.
poll_interval = min(int(timeout / self._PING_WAIT_COUNT), 60) - 1
end_time = time.time() + timeout
while time.time() <= end_time:
if self._ping_check_status(status):
return True
if poll_interval > 0:
# The last thing we did was sleep(poll_interval), so it may
# have been too long since the last `ping`. Check one more
# time, just to be sure.
return self._ping_check_status(status)
def ping_wait_up(self, timeout):
"""Wait for the host to respond to `ping`.
N.B. This method is not a reliable substitute for
`wait_up()`, because a host that responds to ping will not
necessarily respond to ssh. This method should only be used
if the target DUT can be considered functional even if it
can't be reached via ssh.
@param timeout Minimum time to allow before declaring the
host to be non-responsive.
@return True iff the host answered to ping before the timeout.
return self._ping_wait_for_status(self._PING_STATUS_UP, timeout)
def ping_wait_down(self, timeout):
"""Wait until the host no longer responds to `ping`.
This function can be used as a slightly faster version of
`wait_down()`, by avoiding potentially long ssh timeouts.
@param timeout Minimum time to allow for the host to become
@return True iff the host quit answering ping before the
return self._ping_wait_for_status(self._PING_STATUS_DOWN, timeout)
def test_wait_for_sleep(self):
"""Wait for the client to enter low-power sleep mode.
The test for "is asleep" can't distinguish a system that is
powered off; to confirm that the unit was asleep, it is
necessary to force resume, and then call
This function is expected to be called from a test as part
of a sequence like the following:
boot_id = host.get_boot_id()
# trigger sleep on the host
# trigger resume on the host
@exception TestFail The host did not go to sleep within
the allowed time.
if not self.ping_wait_down(timeout=self.SLEEP_TIMEOUT):
raise error.TestFail(
'client failed to sleep after %d seconds' %
def test_wait_for_resume(self, old_boot_id):
"""Wait for the client to resume from low-power sleep mode.
The `old_boot_id` parameter should be the value from
`get_boot_id()` obtained prior to entering sleep mode. A
`TestFail` exception is raised if the boot id changes.
See @ref test_wait_for_sleep for more on this function's
@param old_boot_id A boot id value obtained before the
target host went to sleep.
@exception TestFail The host did not respond within the
allowed time.
@exception TestFail The host responded, but the boot id test
indicated a reboot rather than a sleep
if not self.wait_up(timeout=self.RESUME_TIMEOUT):
raise error.TestFail(
'client failed to resume from sleep after %d seconds' %
new_boot_id = self.get_boot_id()
if new_boot_id != old_boot_id:
raise error.TestFail(
'client rebooted, but sleep was expected'
' (old boot %s, new boot %s)'
% (old_boot_id, new_boot_id))
def test_wait_for_shutdown(self):
"""Wait for the client to shut down.
The test for "has shut down" can't distinguish a system that
is merely asleep; to confirm that the unit was down, it is
necessary to force boot, and then call test_wait_for_boot().
This function is expected to be called from a test as part
of a sequence like the following:
boot_id = host.get_boot_id()
# trigger shutdown on the host
# trigger boot on the host
@exception TestFail The host did not shut down within the
allowed time.
if not self.ping_wait_down(timeout=self.SHUTDOWN_TIMEOUT):
raise error.TestFail(
'client failed to shut down after %d seconds' %
def test_wait_for_boot(self, old_boot_id=None):
"""Wait for the client to boot from cold power.
The `old_boot_id` parameter should be the value from
`get_boot_id()` obtained prior to shutting down. A
`TestFail` exception is raised if the boot id does not
change. The boot id test is omitted if `old_boot_id` is not
See @ref test_wait_for_shutdown for more on this function's
@param old_boot_id A boot id value obtained before the
shut down.
@exception TestFail The host did not respond within the
allowed time.
@exception TestFail The host responded, but the boot id test
indicated that there was no reboot.
if not self.wait_up(timeout=self.REBOOT_TIMEOUT):
raise error.TestFail(
'client failed to reboot after %d seconds' %
elif old_boot_id:
if self.get_boot_id() == old_boot_id:
raise error.TestFail(
'client is back up, but did not reboot'
' (boot %s)' % old_boot_id)
def check_for_rpm_support(hostname):
"""For a given hostname, return whether or not it is powered by an RPM.
@return None if this host does not follows the defined naming format
for RPM powered DUT's in the lab. If it does follow the format,
it returns a regular expression MatchObject instead.
return re.match(SiteHost._RPM_HOSTNAME_REGEX, hostname)
def has_power(self):
"""For this host, return whether or not it is powered by an RPM.
@return True if this host is in the CROS lab and follows the defined
naming format.
return SiteHost.check_for_rpm_support(self.hostname)
def _set_power(self, state, power_method):
"""Sets the power to the host via RPM, Servo or manual.
@param state Specifies which power state to set to DUT
@param power_method Specifies which method of power control to
use. By default "RPM" will be used. Valid values
are the strings "RPM", "manual", "servoj10".
if state.upper() not in ACCEPTABLE_STATES:
raise error.TestError('State must be one of: %s.'
if power_method == self.POWER_CONTROL_SERVO:'Setting servo port J10 to %s', state)
self.servo.set('prtctl3_pwren', state.lower())
elif power_method == self.POWER_CONTROL_MANUAL:'You have %d seconds to set the AC power to %s.',
if not self.has_power():
raise error.TestFail('DUT does not have RPM connected.')
afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10)
afe.set_host_attribute(self._RPM_OUTLET_CHANGED, True,
rpm_client.set_power(self.hostname, state.upper())
def power_off(self, power_method=POWER_CONTROL_RPM):
"""Turn off power to this host via RPM, Servo or manual.
@param power_method Specifies which method of power control to
use. By default "RPM" will be used. Valid values
are the strings "RPM", "manual", "servoj10".
self._set_power('OFF', power_method)
def power_on(self, power_method=POWER_CONTROL_RPM):
"""Turn on power to this host via RPM, Servo or manual.
@param power_method Specifies which method of power control to
use. By default "RPM" will be used. Valid values
are the strings "RPM", "manual", "servoj10".
self._set_power('ON', power_method)
def power_cycle(self, power_method=POWER_CONTROL_RPM):
"""Cycle power to this host by turning it OFF, then ON.
@param power_method Specifies which method of power control to
use. By default "RPM" will be used. Valid values
are the strings "RPM", "manual", "servoj10".
if power_method in (self.POWER_CONTROL_SERVO,
rpm_client.set_power(self.hostname, 'CYCLE')
def get_platform(self):
"""Determine the correct platform label for this host.
@returns a string representing this host's platform.
crossystem = utils.Crossystem(self)
# Extract fwid value and use the leading part as the platform id.
# fwid generally follow the format of {platform}.{firmware version}
# Example: Alex.X.YYY.Z or Google_Alex.X.YYY.Z
platform = crossystem.fwid().split('.')[0].lower()
# Newer platforms start with 'Google_' while the older ones do not.
return platform.replace('google_', '')
def get_board(self):
"""Determine the correct board label for this host.
@returns a string representing this host's board.
release_info = utils.parse_cmd_output('cat /etc/lsb-release',
board = release_info['CHROMEOS_RELEASE_BOARD']
# Devices in the lab generally have the correct board name but our own
# development devices have {board_name}-signed-{key_type}. The board
# name may also begin with 'x86-' which we need to keep.
board_format_string = ds_constants.BOARD_PREFIX + '%s'
if 'x86' not in board:
return board_format_string % board.split('-')[0]
return board_format_string % '-'.join(board.split('-')[0:2])
def has_lightsensor(self):
"""Determine the correct board label for this host.
@returns the string 'lightsensor' if this host has a lightsensor or
None if it does not.
search_cmd = "find -L %s -maxdepth 4 | egrep '%s'" % (
# Run the search cmd following the symlinks. Stderr_tee is set to
# None as there can be a symlink loop, but the command will still
# execute correctly with a few messages printed to stderr., stdout_tee=None, stderr_tee=None)
return 'lightsensor'
except error.AutoservRunError:
# egrep exited with a return code of 1 meaning none of the possible
# lightsensor files existed.
return None
def has_bluetooth(self):
"""Determine the correct board label for this host.
@returns the string 'bluetooth' if this host has bluetooth or
None if it does not.
try:'test -d /sys/class/bluetooth/hci0')
# test exited with a return code of 0.
return 'bluetooth'
except error.AutoservRunError:
# test exited with a return code 1 meaning the directory did not
# exist.
return None
def get_labels(self):
"""Return a list of labels for this given host.
This is the main way to retrieve all the automatic labels for a host
as it will run through all the currently implemented label functions.
labels = []
for label_function in self._LABEL_FUNCTIONS:
label = label_function(self)
if label:
return labels