blob: 9de3c50c2c3b13d1b320d2ae397cf7fda32dfd07 [file] [log] [blame]
#!/usr/bin/python
#
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script to run an Autotest job on machines in a remote lab.
Takes an image. Verifies the image to be a test image (we need SSH access). Then
splits the image into update.gz and stateful.tgz components. Finally, uploads
the components to a Dev Server in the lab.
Once everything is in the necessary places a job is scheduled using the Autotest
command line tools on /home/build/static and URL returned to the user.
"""
__author__ = 'dalecurtis@google.com (Dale Curtis)'
__version__ = 'v1.3'
import json
import logging
import optparse
import os
import shutil
import sys
import tempfile
import chromeos_test_common
from chromeos_test import autotest_util
from chromeos_test import build_util
from chromeos_test import common_util
from chromeos_test import test_config
from chromeos_test.colors import Colors
from chromeos_test.dev_server import DevServer
# Autotest directory relative to CrOS root.
DEFAULT_AUTOTEST_DIR = 'src/third_party/autotest/files'
# Location of default board file.
DEFAULT_BOARD_FILE = 'src/scripts/.default_board'
# Root of Chrome OS checkout should be up a few directories relative to us.
DEFAULT_CROS_DIR = chromeos_test_common.CROS_DIR
# Root of the default build directory relative to CrOS root.
DEFAULT_IMAGE_DIR = 'src/build/images'
# Tag prefix for Dev builds.
DEV_BUILD_PREFIX = 'dev'
LAB_TEST_CONFIG = os.path.join(chromeos_test_common.CURRENT_DIR,
'lab_test.json')
# Path to ChromeOS testing key in CrOS checkout.
CROS_TEST_KEYS_DIR = 'src/scripts/mod_for_test_scripts/ssh_keys/'
CROS_TEST_KEY_PRIV = os.path.join(CROS_TEST_KEYS_DIR, 'testing_rsa')
CROS_TEST_KEY_PUB = os.path.join(CROS_TEST_KEYS_DIR, 'testing_rsa.pub')
# Exit code to use on error.
ERROR_EXIT_CODE = 1
# URL base for viewing a job.
JOB_URL_BASE = 'http://cautotest/afe/#tab_id=view_job&object_id='
def KerberosExceptionHandler(f):
"""Decorator which provides additional information for Kerberos exceptions."""
def _Wrapped():
try:
return f()
except common_util.ChromeOSTestError, e:
if 'Kerberos' in e[-1]:
LogErrorAndExit(
'There appears to be a problem with your credentials. Please run'
' kinit and try again.')
else:
raise
return _Wrapped
def FindTest(autotest_dir, test_regex):
"""Uses a test name regex to find the proper control file in Autotest dirs."""
search_paths = 'client/tests client/site_tests server/tests server/site_tests'
cmd = ('find %s -maxdepth 2 -type f \\( -name control.* -or -name control \\)'
'| egrep -v "~$" | egrep "%s"' % (search_paths, test_regex))
return common_util.RunCommand(cmd=cmd, cwd=autotest_dir, output=True)
def FindAutotestDir(options):
"""Determine whether to use cros_workon or emerged Autotests. Returns path."""
if options.autotest_dir:
if not os.path.exists(options.autotest_dir):
LogErrorAndExit('Could not find the specified Autotest directory.')
else:
logging.info('As requested, using the specified Autotest directory '
'at %s.', Colors.Color(Colors.BOLD_BLUE,
options.autotest_dir))
return options.autotest_dir
autotest_dir = os.path.join(options.cros_dir, DEFAULT_AUTOTEST_DIR)
if options.use_emerged:
autotest_dir = os.path.join(
options.cros_dir, 'chroot/build', options.board, 'usr/local/autotest')
if not os.path.exists(autotest_dir):
LogErrorAndExit('Could not find pre-installed autotest, you need to '
'emerge-%s autotest autotest-tests.', options.board)
logging.info('As requested, using emerged autotests already installed at '
'%s.', Colors.Color(Colors.BOLD_BLUE, autotest_dir))
elif not os.path.exists(autotest_dir):
LogErrorAndExit('Could not find Autotest, run "cros_workon start autotest" '
'and "repo sync" to continue.')
else:
logging.info('Detected cros_workon autotests. Using autotests from %s. To '
'use emerged autotest, pass --use_emerged.',
Colors.Color(Colors.BOLD_BLUE, autotest_dir))
return autotest_dir
def VerifyImageAndGetId(cros_dir, image_path, install_shim=False):
"""Verifies image is a test image and returns tuple of version, hash.
Args:
cros_dir: Location of Chrome OS code base.
image_path: Path to image to verify and convert.
install_shim: True to verify an install shim instead of a test image.
Returns:
Tuple of (build_version, build_hash).
"""
tempdir = tempfile.mkdtemp()
build_util.MountImage(cros_dir, tempdir,
image_file=os.path.basename(image_path),
image_dir=os.path.dirname(image_path))
try:
cmd = 'cat etc/lsb-release | grep CHROMEOS_RELEASE_DESCRIPTION'
msg = 'Failed to read /etc/lsb-release from mounted image!'
version = common_util.RunCommand(
cmd=cmd, cwd=os.path.join(tempdir, build_util.ROOTFS_MOUNT_DIR),
error_msg=msg, output=True)
if install_shim:
cmd = 'file root/.factory_installer'
msg = ('The specified image is not an install shim! Only install shims '
'allowed.')
else:
cmd = ('diff root/.ssh/authorized_keys %s'
% os.path.join(cros_dir, CROS_TEST_KEY_PUB))
msg = 'The specified image is not a test image! Only test images allowed.'
common_util.RunCommand(
cmd=cmd, cwd=os.path.join(tempdir, build_util.ROOTFS_MOUNT_DIR),
error_msg=msg)
finally:
build_util.UnmountImage(cros_dir, tempdir)
shutil.rmtree(tempdir, ignore_errors=True)
# String looks like '<tag>=<version> (Test Build <hash> ...' After =, we want
# the first and third elements. TODO(dalecurtis): verify what we're parsing.
return version.split('=')[1].split(' ')[0:4:3]
def ProcessLocalBuild(cros_dir, dev, image_path, force=False):
"""Process a local build. Verifies and converts a test image into updates.
Args:
cros_dir: Location of Chrome OS code base.
dev: Instantiated Dev Server Class.
image_path: Path to test image to verify and convert.
force: Force creation of updates even if build already exists on server.
Returns:
Tuple of (build_tag, image_dir, remote_build_dir).
build_tag: Unique identifier for this build.
image_dir: Path on local disk
"""
logging.info('Verifying the specified image is a test image.')
build_version, build_hash = VerifyImageAndGetId(cros_dir, image_path)
build_tag = '%s-%s-%s' % (os.environ['USER'], build_version, build_hash)
logging.info(
'Processing build %s.', Colors.Color(Colors.BOLD_BLUE, build_tag))
if force:
logging.info('Forcing upload of new build components due to --force.')
# Prepare the Dev Server for this build.
remote_build_dir, exists = dev.PrepareDevServer(
'/'.join([DEV_BUILD_PREFIX, build_tag]), force=force)
image_dir = os.path.dirname(image_path)
image_file = os.path.basename(image_path)
try:
# Create update zips if they don't exist.
if not exists:
logging.info('Generating update.')
build_util.CreateUpdateZip(
cros_dir, image_dir, image_file=image_file)
# Create stateful update zip.
logging.info('Generating stateful update.')
build_util.CreateStatefulZip(cros_dir, image_dir, image_file=image_file)
else:
logging.info(Colors.Color(
Colors.BOLD_BLUE, 'Using existing build found on Dev Server.'))
except:
if remote_build_dir:
dev.RemoteCommand('rmdir ' + remote_build_dir)
raise
return build_tag, image_dir, remote_build_dir, exists
def LogErrorAndExit(msg, *args, **kwargs):
"""Simple log error and exit method."""
logging.error(Colors.Color(Colors.BOLD_RED, msg), *args, **kwargs)
sys.exit(ERROR_EXIT_CODE)
@KerberosExceptionHandler
def GetPlatformDict():
"""Return a list of Autotest platform labels accessible to current user."""
platform_dict = autotest_util.GetPlatformDict()
if not platform_dict:
LogErrorAndExit('There are no platforms ACL accessible by you. Please'
' contact the ChromeOS Autotest team'
' (chromeos-lab-infrastructure@google.com).')
return platform_dict
@KerberosExceptionHandler
def PrintMachineList():
"""Display the output of atest host list."""
cmd = '%s host list --user $USER' % autotest_util.ATEST_PATH
msg = 'Failed to retrieve host list from Autotest.'
print common_util.RunCommand(cmd, error_msg=msg, output=True)
def ParseOptions():
"""Parse and verify command line options.
Returns:
Tuple of options dictionary, relative path to test control file, the path to
Autotest, and the lab test JSON config.
"""
parser = optparse.OptionParser(
'usage: %prog [options] <test name>\n'
'\n'
'The test name can be a regular expression so long as it only'
' matches a single test. For example:\n'
'\n'
' %prog -i test.bin --board x86-generic BootPerfServer')
parser.add_option('--autotest_dir', help='Skip autodetection of autotest and '
'use the specified location.')
parser.add_option('--board', dest='board',
help=('The board for which you are building autotest. Will '
'attempt to read default from <cros_dir>/%s'
% DEFAULT_BOARD_FILE))
parser.add_option('--build', dest='build',
help=('Instead of using a local build, use an official '
'build already on the server; e.g. 0.13.507.0 or '
'latest to use the most recent build.'))
parser.add_option('-c', '--cros', dest='cros_dir',
default=chromeos_test_common.CROS_DIR,
help=('Location of Chrome OS code base. Defaults to '
'"%default".'))
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
default=False, help='Enable debugging/verbose output.')
parser.add_option('-d', '--deps', dest='deps', default=None,
help='Comma deliminated list of dependencies.')
parser.add_option('-f', '--force', dest='force', action='store_true',
default=False,
help='Force upload even if build already exists on server.')
parser.add_option('-i', '--image', dest='image_path',
help=('Path to test image to deploy for testing. If no'
' image is specified, the script attempts to use'
' <cros_dir>/%s/<board>/latest/%s'
% (DEFAULT_IMAGE_DIR, build_util.TEST_IMAGE)))
parser.add_option('--list_machines', dest='list_machines',
action='store_true',
help=('Display the list of available machines as well as'
' their current status.'))
parser.add_option('-l', '--list_platforms', dest='list_platforms',
action='store_true',
help=('Display the list of valid platforms for use with'
' --platforms.'))
parser.add_option('-m', '--mail', dest='mail',
help=('A comma seperated list of email addresses to notify'
' upon job completion.'))
parser.add_option('-o', '--override', dest='override', action='store_true',
default=False,
help=('Override board and platform safety checks.'
' Experienced users only! Please don\'t brick our'
' machines :)'))
parser.add_option('-p', '--platforms', dest='platforms',
help=('Comma separated list of platforms to use for'
' testing. Use the --list_platforms option to see the'
' list of valid platforms. Multiple tests on the same'
' platform can be run by using the * character; e.g.,'
' 2*<platform> would use two machines of type'
' <platform>.'))
parser.add_option('-t', '--tests', dest='tests', action='store_true',
default=False,
help=('Package tests with stateful partition. Will cause'
' the stateful partition to be reuploaded to the'
' server even if it already exists. If tests aren\'t'
' packaged, the versions on the Autotest server will'
' be used.'))
parser.add_option('-x', '--priority', dest='priority', default='urgent',
help='The priority of the job. default: [%default].')
parser.add_option('--use_emerged', dest='use_emerged', action='store_true',
default=False,
help='Force use of emerged autotest packages')
options, args = parser.parse_args()
if options.verbose:
logging.getLogger().setLevel(logging.DEBUG)
# Make sure we're outside the chroot.
if os.path.isfile('/etc/debian_chroot'):
LogErrorAndExit(
'LabTest must be run outside the chroot to access corp resources.')
if options.list_machines:
parser.print_help()
print Colors.Color(
Colors.BOLD_BLUE,
'\nGenerating list of machines (this may take a few seconds):')
PrintMachineList()
sys.exit(0)
if options.list_platforms:
parser.print_help()
print Colors.Color(
Colors.BOLD_BLUE,
'\nGenerating list of valid platforms (this may take a few seconds):')
format = '%-40s %-20s'
print format % ('Platform', 'Dependencies')
for platform, deps in GetPlatformDict().iteritems():
print format % (platform, ' '.join(deps))
sys.exit(0)
logging.info('Verifying command line options.')
if not args:
LogErrorAndExit('A test name must be specified.')
# Make sure CrOS checkout directory exists.
if not os.path.exists(options.cros_dir):
LogErrorAndExit('Could not find Chrome OS checkout, please specify the path'
' with -c.')
# Convert paths to abs path.
for item in ('autotest_dir', 'cros_dir', 'image_path'):
if getattr(options, item):
abs_path = os.path.normpath(os.path.join(os.getcwd(),
getattr(options, item)))
setattr(options, item, abs_path)
# Attempt to load LabTest config.
with open(LAB_TEST_CONFIG) as config_file:
config = json.load(config_file)
# Attempt to determine the default board.
default_board_file = os.path.join(options.cros_dir, DEFAULT_BOARD_FILE)
if not options.board:
logging.info('No board specified, attempting to load the default.')
if not os.path.isfile(default_board_file):
LogErrorAndExit('The default board could not be read. Please specify the '
'board type with --board.')
with open(default_board_file, 'r') as f:
options.board = f.read().strip()
logging.info('Using default board "%s"',
Colors.Color(Colors.BOLD_BLUE, options.board))
# Convert boards with multiple names into a single format.
if options.board in config['preferred_board_fixups']:
options.board = config['preferred_board_fixups'][options.board]
if not options.platforms:
if options.board in config['board_platform_map']:
# If the platform exists in the map, override any further checks.
options.override = True
options.platforms = config['board_platform_map'][options.board]
logging.info(
'No platform specified, using the default platform for this board '
'"%s"', Colors.Color(Colors.BOLD_BLUE, options.platforms))
else:
LogErrorAndExit(
'An unknown board has been specified, please specify the platform '
'type with --platform.')
# Make sure the specified image actually exists...
if options.image_path:
if not os.path.isfile(options.image_path):
LogErrorAndExit('The specified test image does not exist.')
elif not options.build:
logging.info('No image specified, attempting to find the latest image.')
options.image_path = os.path.join(
options.cros_dir, DEFAULT_IMAGE_DIR, options.board, 'latest',
build_util.TEST_IMAGE)
if not os.path.isfile(options.image_path):
LogErrorAndExit(
'No test image specified and the default could not be found.')
logging.info(
'Default image found, using %s',
Colors.Color(Colors.BOLD_BLUE, options.image_path))
# Figure out the Autotest directory based on command line options.
autotest_dir = FindAutotestDir(options)
# Identify the desired test case. Limit to only one test for now.
test_pattern = ' '.join(args)
try:
matched_test = FindTest(autotest_dir, test_pattern).strip()
except common_util.ChromeOSTestError:
LogErrorAndExit('Cannot find a match for test name "%s"' % test_pattern)
if len(matched_test.split('\n')) > 1:
logging.error('The given test pattern is ambiguous. Disambiguate by '
'passing one of these patterns instead:')
for test in matched_test.split('\n'):
logging.error(' ^%s$', test)
sys.exit(ERROR_EXIT_CODE)
# Verify the requested platforms.
platform_dict = GetPlatformDict()
# Strip out any multipliers from the platform list.
platform_split = options.platforms.split(',')
platform_names = set(p.lstrip('0123456789* ') for p in platform_split)
bad_platforms = platform_names - set(platform_dict.keys())
if bad_platforms:
LogErrorAndExit('The following platforms are invalid: %s',
', '.join(bad_platforms))
# Add 1* for any platforms without a count.
for i in xrange(0, len(platform_split)):
if not platform_split[i][0].isdigit():
platform_split[i] = '1*' + platform_split[i]
options.platforms = ','.join(platform_split)
# Verify specified platforms match the provided board.
if not options.override and options.board != 'x86-generic':
# Only allow board, platform pairs we have configured for testing.
cros_config = test_config.TestConfig(
os.path.join(chromeos_test_common.CRON_DIR,
test_config.DEFAULT_CONFIG_FILE))
valid_platforms = cros_config.ParseConfigGroups(board_re=options.board)[2]
for p in platform_names:
if not p in valid_platforms:
LogErrorAndExit('The specified platform (%s) is not valid for the '
'specified board (%s). Valid platforms for this board '
'are: %s.', p, options.board,
', '.join(valid_platforms))
return options, matched_test, autotest_dir, config
def main():
# Setup logging.
logging.basicConfig(format=' - %(levelname)s: %(message)s')
logging.getLogger().setLevel(logging.INFO)
print '-' * 80
print ('LabTest! A script to run Autotest jobs on machines in a remote lab.'
' (%s)' % __version__)
print '-' * 80
options = local_build_dir = remote_build_dir = None
try:
# Parse options and find the requested control file.
options, control_file, autotest_dir, config = ParseOptions()
start_str = 'Running %s on the following platforms: %s' % (
Colors.Color(Colors.BOLD_GREEN, control_file),
Colors.Color(Colors.BOLD_GREEN, options.platforms))
if options.deps:
start_str += ' with deps: %s' % Colors.Color(Colors.BOLD_GREEN,
options.deps)
logging.info(start_str)
# Load Dev Server configuration.
dev_config = config['dev_server']
remote_host = dev_config.get('remote_host', None)
# Initialize Dev Server.
dev = DevServer(
dev_config['dev_host'], dev_config['dir'], dev_config['user'],
private_key=os.path.join(options.cros_dir, CROS_TEST_KEY_PRIV),
remote_host=remote_host)
# Determine if we have any tests to upload.
tests_to_upload = options.tests
# If the user hasn't specified an official build, process their local build.
if not options.build:
build_tag, local_build_dir, remote_build_dir, exists = ProcessLocalBuild(
options.cros_dir, dev, options.image_path, force=options.force)
else:
# Scan the Dev Server for using the partial board, build information we
# have. Afterward, update the options values with the full ids.
options.board, options.build = dev.FindDevServerBuild(
options.board, options.build)
build_tag = '%s-%s' % (os.environ['USER'], options.build)
logging.info(
'Official build requested, using build %s for testing.',
Colors.Color(Colors.BOLD_GREEN, options.build))
if tests_to_upload:
# Create a temporary directory to hold Autotest packages.
local_build_dir = tempfile.mkdtemp()
# Make a copy of the official build so we don't corrupt it.
remote_build_dir = dev.CloneDevServerBuild(
options.board, options.build,
'/'.join([DEV_BUILD_PREFIX, build_tag]), force=options.force)
# Extract test name from path and prepare Autotest packages for upload.
test_name = os.path.basename(os.path.dirname(control_file))
if tests_to_upload:
logging.info('Preparing Autotest packages for upload to Dev Server.')
build_util.PrepareAutotestPkgs(
autotest_dir, local_build_dir, test_name=test_name)
# If we've processed a build, upload all build components.
if remote_build_dir and not options.build and not exists:
logging.info('Uploading build components to Dev Server.')
dev.UploadBuildComponents(remote_build_dir, local_build_dir)
elif tests_to_upload:
# Otherwise, just upload Autotest packages if there are any.
logging.info('Uploading Autotest packages to Dev Server.')
dev.UploadAutotestPackages(remote_build_dir, local_build_dir)
# If official build and no modified tests, use an existing build URL.
if options.build and not tests_to_upload:
update_url = dev.GetUpdateUrl(options.board, options.build)
else:
# Otherwise determine the update URL for the processed build.
update_url = dev.GetUpdateUrl(DEV_BUILD_PREFIX, build_tag)
# Hackish, but the only way we have to determine server versus client jobs.
server = control_file.startswith('server/')
# Special case to fix up job names in the suites directory. These files are
# all of the format suites/control.<name>.
if test_name.lower() == 'suites':
test_name = os.path.basename(control_file).split('.')[-1]
# Now that all components are uploaded, start the Autotest job.
job_name = '%s_%s' % (build_tag, test_name)
logging.info('Creating job %s.', Colors.Color(Colors.BOLD_BLUE, job_name))
job_id = autotest_util.CreateJob(
name=job_name, control=os.path.join(autotest_dir, control_file),
platforms=options.platforms, update_url=update_url, server=server,
mail=options.mail, labels=options.deps, priority=options.priority)
logging.info(
Colors.Color(Colors.BOLD_GREEN, 'Job created successfully, URL: %s%s'),
JOB_URL_BASE, job_id)
except Exception, e:
if remote_build_dir:
dev.RemoteCommand('rm -rf ' + remote_build_dir)
if isinstance(e, common_util.ChromeOSTestError):
logging.error(Colors.Color(Colors.BOLD_RED, e[0]))
if not options or options.verbose:
logging.exception(e)
else:
raise
finally:
# When --build is used, local_build_dir contains only tmp files, so cleanup.
if options and options.build and local_build_dir:
common_util.RunCommand('rm -rf ' + local_build_dir)
if __name__ == '__main__':
main()