blob: 3f75fd4625227de801142a5fc02801f8aba23e55 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Install an initial test image on a set of DUTs.
The target DUTs are newly deployed, and are normally in a slightly
anomalous state:
* The DUTs are running a production base image, not a test image.
By extension, the DUTs aren't reachable over SSH.
* The DUTs are not necessarily in the AFE database. DUTs that
_are_ in the database should be locked. Either way, the DUTs
cannot be scheduled to run tests.
* The servos for the DUTs need not be configured with the proper
board.
The script imposes these preconditions:
* Every DUT has a properly connected servo.
* Every DUT and servo has proper DHCP and DNS configurations.
* Every servo host is up and running, and accessible via SSH.
* There is a known, working test image that can be staged and
installed on the target DUTs via servo.
* Every DUT has the same board.
Installation is done using the standard servo repair process (that
is, boot and install a test image from USB). The implementation
uses the `multiprocessing` module to run all installations in
parallel, separate processes.
"""
import functools
import logging
import multiprocessing
import os
import shutil
import subprocess
import sys
import tempfile
import time
import common
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import time_utils
from autotest_lib.server import frontend
from autotest_lib.server import hosts
from autotest_lib.server.cros.dynamic_suite.constants import VERSION_PREFIX
from autotest_lib.site_utils.deployment import commandline
from autotest_lib.site_utils.suite_scheduler.constants import Labels
_LOG_FORMAT = '%(asctime)s | %(levelname)-10s | %(message)s'
_DEFAULT_POOL = Labels.POOL_PREFIX + 'suites'
_DIVIDER = '\n============\n'
def _create_host(hostname, board):
"""Create a CrosHost object for a DUT to be installed.
@param hostname Hostname of the target DUT.
@param board Board name of the target DUT.
"""
host = hosts.create_host(hostname, try_lab_servo=True)
# Monkey patch our host object to think there's a board label
# in the AFE. The horror! The horror!
#
# TODO(jrbarnette): This is wrong; we patch the method because
# CrosHost._servo_repair_reinstall() calls it, but that means
# we're coupled to the implementation of CrosHost. Alas, it's
# hard to do better without either 1) copying large chunks of
# _servo_repair_reinstall(), or 2) extensively refactoring
# CrosHost.
host._get_board_from_afe = lambda: board
return host
def _check_servo(host):
"""Check that servo for the given host is working.
Perform these steps:
* Confirm that the servo host is reachable via SSH.
* Stop `servod` on the servo host if it's running, and restart
it with the host's designated board. We deliberately ignore
any prior configuration.
* Re-verify that the servo service on the servo host is
working correctly.
* Re-initialize the DUT host object with the correct servo
object, since this won't have been done in the case that
`servod` was down.
* Re-initialize the servo settings, since restarting `servod`
can change the actual settings from the expected defaults.
(In particular, restarting `servod` leaves the USB stick
plugged in to the servo host.)
@param host CrosHost object with the servo to be initialized.
"""
if not host._servo_host:
raise Exception('No answer to ping from Servo host')
if not host._servo_host.is_up():
raise Exception('No answer to ssh from Servo host')
# Stop servod, ignoring failures, then restart with the proper
# board.
#
# There's a lag between when `start servod` completes and when
# servod is actually up and serving. The call to time.sleep()
# below gives time to make sure that the verify() call won't
# fail.
host._servo_host.run('stop servod || :')
host._servo_host.run('start servod BOARD=%s' %
host._get_board_from_afe())
time.sleep(4)
logging.debug('Starting servo host verification')
host._servo_host.verify()
host.servo = host._servo_host.get_servo()
host.servo.initialize_dut()
if not host.servo.probe_host_usb_dev():
raise Exception('No USB stick detected on Servo host')
def _configure_install_logging():
"""Configure the logging module for `_install_dut()`."""
handler = logging.StreamHandler(sys.stderr)
formatter = logging.Formatter(_LOG_FORMAT, time_utils.TIME_FMT)
handler.setFormatter(formatter)
root_logger = logging.getLogger()
for h in root_logger.handlers:
root_logger.removeHandler(h)
root_logger.addHandler(handler)
def _try_lock_host(afe_host):
"""Lock a host in the AFE, and report whether it succeeded.
The lock action is logged regardless of success; failures are
logged if they occur.
@param afe_host AFE Host instance to be locked.
@return `True` on success, or `False` on failure.
"""
try:
logging.warning('Locking host now.')
afe_host.modify(locked=True,
lock_reason='Running deployment_test')
except Exception as e:
logging.exception('Failed to lock: %s', e)
return False
return True
def _try_unlock_host(afe_host):
"""Unlock a host in the AFE, and report whether it succeeded.
The unlock action is logged regardless of success; failures are
logged if they occur.
@param afe_host AFE Host instance to be unlocked.
@return `True` on success, or `False` on failure.
"""
try:
logging.warning('Unlocking host.')
afe_host.modify(locked=False, lock_reason='')
except Exception as e:
logging.exception('Failed to unlock: %s', e)
return False
return True
def _install_dut(arguments, hostname):
"""Install the initial test image on one DUT using servo.
Implementation note: This function is expected to run in a
subprocess created by a multiprocessing Pool object. As such,
it can't (shouldn't) write to shared files like `sys.stdout`.
@param hostname Host name of the DUT to install on.
@param arguments Parsed results from
ArgumentParser.parse_args().
@return On success, return `None`. On failure, return a string
with an error message.
"""
# In some cases, autotest code that we're calling below may put
# stuff onto stdout with 'print' statements. Most notably, the
# AFE frontend may print 'FAILED RPC CALL' (boo, hiss). We want
# nothing from this subprocess going to the output we inherited
# from our parent, so redirect stdout and stderr here, before
# we make any AFE calls. Note that this does what we want only
# because we're in a subprocess.
log_name = os.path.join(arguments.dir, hostname + '.log')
sys.stdout = open(log_name, 'w')
sys.stderr = sys.stdout
_configure_install_logging()
afe = frontend.AFE(server=arguments.web)
hostlist = afe.get_hosts([hostname])
unlock_on_failure = False
if hostlist:
afe_host = hostlist[0]
if not afe_host.locked:
if _try_lock_host(afe_host):
unlock_on_failure = True
else:
return 'Failed to lock host'
if (afe_host.status != 'Ready' and
afe_host.status != 'Repair Failed'):
if unlock_on_failure and not _try_unlock_host(afe_host):
return 'Host is in use, and failed to unlock it'
return 'Host is in use by Autotest'
else:
afe_host = None
try:
host = _create_host(hostname, arguments.board)
_check_servo(host)
if not arguments.noinstall:
host._servo_repair_reinstall()
except error.AutoservRunError as re:
logging.exception('Failed to install: %s', re)
if unlock_on_failure and not _try_unlock_host(afe_host):
logging.error('Failed to unlock host!')
return 'chromeos-install failed'
except Exception as e:
logging.exception('Failed to install: %s', e)
if unlock_on_failure and not _try_unlock_host(afe_host):
logging.error('Failed to unlock host!')
return str(e)
finally:
host.close()
if afe_host is not None:
if not _try_unlock_host(afe_host):
return 'Failed to unlock after successful install'
else:
logging.debug('Creating host in AFE.')
atest_path = os.path.join(
os.path.dirname(os.path.abspath(sys.argv[0])),
'atest')
status = subprocess.call(
[atest_path, 'host', 'create', hostname])
if status != 0:
logging.error('Host creation failed, status = %d', status)
return 'Failed to add host to AFE'
# Must re-query to get state changes, especially label changes.
afe_host = afe.get_hosts([hostname])[0]
have_board = any([label.startswith(Labels.BOARD_PREFIX)
for label in afe_host.labels])
if not have_board:
afe_host.delete()
return 'Failed to add labels to host'
version = [label for label in afe_host.labels
if label.startswith(VERSION_PREFIX)]
if version:
afe_host.remove_labels(version)
return None
def _report_hosts(heading, host_results_list):
"""Report results for a list of hosts.
To improve visibility, results are preceded by a header line,
followed by a divider line. Then results are printed, one host
per line.
@param heading The header string to be printed before
results.
@param host_results_list A list of (hostname, message) tuples
to be printed one per line.
"""
if not host_results_list:
return
sys.stdout.write(heading)
sys.stdout.write(_DIVIDER)
for t in host_results_list:
sys.stdout.write('%-30s %s\n' % t)
sys.stdout.write('\n')
def _report_results(afe, hostnames, results):
"""Gather and report a summary of results from installation.
Segregate results into successes and failures, reporting
each separately. At the end, report the total of successes
and failures.
@param afe AFE object for RPC calls.
@param hostnames List of the hostnames that were tested.
@param results List of error messages, in the same order
as the hostnames. `None` means the
corresponding host succeeded.
"""
success_hosts = []
success_reports = []
failure_reports = []
for r, h in zip(results, hostnames):
if r is None:
success_hosts.append(h)
else:
failure_reports.append((h, r))
if success_hosts:
afe_host_list = afe.get_hosts(hostnames=success_hosts)
afe.reverify_hosts(hostnames=success_hosts)
for h in afe.get_hosts(hostnames=success_hosts):
for label in h.labels:
if label.startswith(Labels.POOL_PREFIX):
success_reports.append(
(h.hostname, 'Host already in %s' % label))
break
else:
h.add_labels([_DEFAULT_POOL])
success_reports.append(
(h.hostname, 'Host added to %s' % _DEFAULT_POOL))
sys.stdout.write(_DIVIDER)
_report_hosts('Successes', success_reports)
_report_hosts('Failures', failure_reports)
sys.stdout.write('Installation complete: '
'%d successes, %d failures.\n' %
(len(success_reports), len(failure_reports)))
def main(argv):
"""Standard main routine.
@param argv Command line arguments including `sys.argv[0]`.
"""
# Override tempfile.tempdir. Some of the autotest code we call
# will create temporary files that don't get cleaned up. So, we
# put the temp files in our results directory, so that we can
# clean up everything in one fell swoop.
tempfile.tempdir = tempfile.mkdtemp()
arguments = commandline.parse_command(argv)
if not arguments:
sys.exit(1)
sys.stderr.write('Installation output logs in %s\n' % arguments.dir)
afe = frontend.AFE(server=arguments.web)
if arguments.build:
afe.run('set_stable_version',
version=arguments.build,
board=arguments.board)
install_pool = multiprocessing.Pool(len(arguments.hostnames))
install_function = functools.partial(_install_dut, arguments)
results_list = install_pool.map(install_function,
arguments.hostnames)
current_build = afe.run('get_stable_version',
board=arguments.board)
sys.stderr.write('\nRepair version for board %s is now %s.\n' %
(arguments.board, current_build))
_report_results(afe, arguments.hostnames, results_list)
# MacDuff:
# [ ... ]
# Did you say all? O hell-kite! All?
# What, all my pretty chickens and their dam
# At one fell swoop?
shutil.rmtree(tempfile.tempdir)
if __name__ == '__main__':
try:
main(sys.argv)
except KeyboardInterrupt:
pass
except EnvironmentError as e:
sys.stderr.write('Unexpected OS error:\n %s\n' % e)
except Exception as e:
sys.stderr.write('Unexpected exception:\n %s\n' % e)