blob: fcc221daec768c63dc7445d46970698138502711 [file] [log] [blame]
#!/usr/bin/python
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Runs on autotest servers from a cron job to self update them.
This script is designed to run on all autotest servers to allow them to
automatically self-update based on the manifests used to create their (existing)
repos.
"""
from __future__ import print_function
import ConfigParser
import argparse
import os
import re
import subprocess
import socket
import sys
import time
import common
from autotest_lib.client.common_lib import global_config
from autotest_lib.server import utils as server_utils
from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
# How long after restarting a service do we watch it to see if it's stable.
SERVICE_STABILITY_TIMER = 60
# A dict to map update_commands defined in config file to repos or files that
# decide whether need to update these commands. E.g. if no changes under
# frontend repo, no need to update afe.
COMMANDS_TO_REPOS_DICT = {'afe': 'frontend/client/',
'tko': 'frontend/client/'}
BUILD_EXTERNALS_COMMAND = 'build_externals'
_RESTART_SERVICES_FILE = os.path.join(os.environ['HOME'],
'push_restart_services')
AFE = frontend_wrappers.RetryingAFE(
server=server_utils.get_global_afe_hostname(), timeout_min=5,
delay_sec=10)
HOSTNAME = socket.gethostname()
class DirtyTreeException(Exception):
"""Raised when the tree has been modified in an unexpected way."""
class UnknownCommandException(Exception):
"""Raised when we try to run a command name with no associated command."""
class UnstableServices(Exception):
"""Raised if a service appears unstable after restart."""
def strip_terminal_codes(text):
"""This function removes all terminal formatting codes from a string.
@param text: String of text to cleanup.
@returns String with format codes removed.
"""
ESC = '\x1b'
return re.sub(ESC+r'\[[^m]*m', '', text)
def _clean_pyc_files():
print('Removing .pyc files')
try:
subprocess.check_output([
'find', '.',
'(',
# These are ignored to reduce IO load (crbug.com/759780).
'-path', './site-packages',
'-o', '-path', './containers',
'-o', '-path', './logs',
'-o', '-path', './results',
')',
'-prune',
'-o', '-name', '*.pyc',
'-exec', 'rm', '-f', '{}', '+'])
except Exception as e:
print('Warning: fail to remove .pyc! %s' % e)
def verify_repo_clean():
"""This function cleans the current repo then verifies that it is valid.
@raises DirtyTreeException if the repo is still not clean.
@raises subprocess.CalledProcessError on a repo command failure.
"""
subprocess.check_output(['git', 'stash', '-u'])
subprocess.check_output(['git', 'stash', 'clear'])
out = subprocess.check_output(['repo', 'status'], stderr=subprocess.STDOUT)
out = strip_terminal_codes(out).strip()
if not 'working directory clean' in out and not 'working tree clean' in out:
raise DirtyTreeException('%s repo not clean: %s' % (HOSTNAME, out))
def _clean_externals():
"""Clean untracked files within ExternalSource and site-packages/
@raises subprocess.CalledProcessError on a git command failure.
"""
dirs_to_clean = ['site-packages/', 'ExternalSource/']
cmd = ['git', 'clean', '-fxd'] + dirs_to_clean
subprocess.check_output(cmd)
def repo_versions():
"""This function collects the versions of all git repos in the general repo.
@returns A dictionary mapping project names to git hashes for HEAD.
@raises subprocess.CalledProcessError on a repo command failure.
"""
cmd = ['repo', 'forall', '-p', '-c', 'pwd && git log -1 --format=%h']
output = strip_terminal_codes(subprocess.check_output(cmd))
# The expected output format is:
# project chrome_build/
# /dir/holding/chrome_build
# 73dee9d
#
# project chrome_release/
# /dir/holding/chrome_release
# 9f3a5d8
lines = output.splitlines()
PROJECT_PREFIX = 'project '
project_heads = {}
for n in range(0, len(lines), 4):
project_line = lines[n]
project_dir = lines[n+1]
project_hash = lines[n+2]
# lines[n+3] is a blank line, but doesn't exist for the final block.
# Convert 'project chrome_build/' -> 'chrome_build'
assert project_line.startswith(PROJECT_PREFIX)
name = project_line[len(PROJECT_PREFIX):].rstrip('/')
project_heads[name] = (project_dir, project_hash)
return project_heads
def repo_versions_to_decide_whether_run_cmd_update():
"""Collect versions of repos/files defined in COMMANDS_TO_REPOS_DICT.
For the update_commands defined in config files, no need to run the command
every time. Only run it when the repos/files related to the commands have
been changed.
@returns A set of tuples: {(cmd, repo_version), ()...}
"""
results = set()
for cmd, repo in COMMANDS_TO_REPOS_DICT.iteritems():
version = subprocess.check_output(
['git', 'log', '-1', '--pretty=tformat:%h',
'%s/%s' % (common.autotest_dir, repo)])
results.add((cmd, version.strip()))
return results
def repo_sync(update_push_servers=False):
"""Perform a repo sync.
@param update_push_servers: If True, then update test_push servers to ToT.
Otherwise, update server to prod branch.
@raises subprocess.CalledProcessError on a repo command failure.
"""
subprocess.check_output(['repo', 'sync', '--force-sync'])
if update_push_servers:
print('Updating push servers, checkout cros/master')
subprocess.check_output(['git', 'checkout', 'cros/master'],
stderr=subprocess.STDOUT)
else:
print('Updating server to prod branch')
subprocess.check_output(['git', 'checkout', 'cros/prod'],
stderr=subprocess.STDOUT)
_clean_pyc_files()
def discover_update_commands():
"""Lookup the commands to run on this server.
These commonly come from shadow_config.ini, since they vary by server type.
@returns List of command names in string format.
"""
try:
return global_config.global_config.get_config_value(
'UPDATE', 'commands', type=list)
except (ConfigParser.NoSectionError, global_config.ConfigError):
return []
def get_restart_services():
"""Find the services that need restarting on the current server.
These commonly come from shadow_config.ini, since they vary by server type.
@returns Iterable of service names in string format.
"""
with open(_RESTART_SERVICES_FILE) as f:
for line in f:
yield line.rstrip()
def update_command(cmd_tag, dryrun=False, use_chromite_master=False):
"""Restart a command.
The command name is looked up in global_config.ini to find the full command
to run, then it's executed.
@param cmd_tag: Which command to restart.
@param dryrun: If true print the command that would have been run.
@param use_chromite_master: True if updating chromite to master, rather
than prod.
@raises UnknownCommandException If cmd_tag can't be looked up.
@raises subprocess.CalledProcessError on a command failure.
"""
# Lookup the list of commands to consider. They are intended to be
# in global_config.ini so that they can be shared everywhere.
cmds = dict(global_config.global_config.config.items(
'UPDATE_COMMANDS'))
if cmd_tag not in cmds:
raise UnknownCommandException(cmd_tag, cmds)
command = cmds[cmd_tag]
# When updating push servers, pass an arg to build_externals to update
# chromite to master branch for testing
if use_chromite_master and cmd_tag == BUILD_EXTERNALS_COMMAND:
command += ' --use_chromite_master'
print('Running: %s: %s' % (cmd_tag, command))
if dryrun:
print('Skip: %s' % command)
else:
try:
subprocess.check_output(command, shell=True,
cwd=common.autotest_dir,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print('FAILED %s :' % HOSTNAME)
print(e.output)
raise
def restart_service(service_name, dryrun=False):
"""Restart a service.
Restarts the standard service with "service <name> restart".
@param service_name: The name of the service to restart.
@param dryrun: Don't really run anything, just print out the command.
@raises subprocess.CalledProcessError on a command failure.
"""
cmd = ['sudo', 'service', service_name, 'restart']
print('Restarting: %s' % service_name)
if dryrun:
print('Skip: %s' % ' '.join(cmd))
else:
subprocess.check_call(cmd, stderr=subprocess.STDOUT)
def service_status(service_name):
"""Return the results "status <name>" for a given service.
This string is expected to contain the pid, and so to change is the service
is shutdown or restarted for any reason.
@param service_name: The name of the service to check on.
@returns The output of the external command.
Ex: autofs start/running, process 1931
@raises subprocess.CalledProcessError on a command failure.
"""
return subprocess.check_output(['sudo', 'service', service_name, 'status'])
def restart_services(service_names, dryrun=False, skip_service_status=False):
"""Restart services as needed for the current server type.
Restart the listed set of services, and watch to see if they are stable for
at least SERVICE_STABILITY_TIMER. It restarts all services quickly,
waits for that delay, then verifies the status of all of them.
@param service_names: The list of service to restart and monitor.
@param dryrun: Don't really restart the service, just print out the command.
@param skip_service_status: Set to True to skip service status check.
Default is False.
@raises subprocess.CalledProcessError on a command failure.
@raises UnstableServices if any services are unstable after restart.
"""
service_statuses = {}
if dryrun:
for name in service_names:
restart_service(name, dryrun=True)
return
# Restart each, and record the status (including pid).
for name in service_names:
restart_service(name)
# Skip service status check if --skip-service-status is specified. Used for
# servers in backup status.
if skip_service_status:
print('--skip-service-status is specified, skip checking services.')
return
# Wait for a while to let the services settle.
time.sleep(SERVICE_STABILITY_TIMER)
service_statuses = {name: service_status(name) for name in service_names}
time.sleep(SERVICE_STABILITY_TIMER)
# Look for any services that changed status.
unstable_services = [n for n in service_names
if service_status(n) != service_statuses[n]]
# Report any services having issues.
if unstable_services:
raise UnstableServices('%s service restart failed: %s' %
(HOSTNAME, unstable_services))
def run_deploy_actions(cmds_skip=set(), dryrun=False,
skip_service_status=False, use_chromite_master=False):
"""Run arbitrary update commands specified in global.ini.
@param cmds_skip: cmds no need to run since the corresponding repo/file
does not change.
@param dryrun: Don't really restart the service, just print out the command.
@param skip_service_status: Set to True to skip service status check.
Default is False.
@param use_chromite_master: True if updating chromite to master, rather
than prod.
@raises subprocess.CalledProcessError on a command failure.
@raises UnstableServices if any services are unstable after restart.
"""
defined_cmds = set(discover_update_commands())
cmds = defined_cmds - cmds_skip
if cmds:
print('Running update commands:', ', '.join(cmds))
for cmd in cmds:
update_command(cmd, dryrun=dryrun,
use_chromite_master=use_chromite_master)
services = list(get_restart_services())
if services:
print('Restarting Services:', ', '.join(services))
restart_services(services, dryrun=dryrun,
skip_service_status=skip_service_status)
def report_changes(versions_before, versions_after):
"""Produce a report describing what changed in all repos.
@param versions_before: Results of repo_versions() from before the update.
@param versions_after: Results of repo_versions() from after the update.
@returns string containing a human friendly changes report.
"""
result = []
if versions_after:
for project in sorted(set(versions_before.keys() + versions_after.keys())):
result.append('%s:' % project)
_, before_hash = versions_before.get(project, (None, None))
after_dir, after_hash = versions_after.get(project, (None, None))
if project not in versions_before:
result.append('Added.')
elif project not in versions_after:
result.append('Removed.')
elif before_hash == after_hash:
result.append('No Change.')
else:
hashes = '%s..%s' % (before_hash, after_hash)
cmd = ['git', 'log', hashes, '--oneline']
out = subprocess.check_output(cmd, cwd=after_dir,
stderr=subprocess.STDOUT)
result.append(out.strip())
result.append('')
else:
for project in sorted(versions_before.keys()):
_, before_hash = versions_before[project]
result.append('%s: %s' % (project, before_hash))
result.append('')
return '\n'.join(result)
def parse_arguments(args):
"""Parse command line arguments.
@param args: The command line arguments to parse. (ususally sys.argsv[1:])
@returns An argparse.Namespace populated with argument values.
"""
parser = argparse.ArgumentParser(
description='Command to update an autotest server.')
parser.add_argument('--skip-verify', action='store_false',
dest='verify', default=True,
help='Disable verification of a clean repository.')
parser.add_argument('--skip-update', action='store_false',
dest='update', default=True,
help='Skip the repository source code update.')
parser.add_argument('--skip-actions', action='store_false',
dest='actions', default=True,
help='Skip the post update actions.')
parser.add_argument('--skip-report', action='store_false',
dest='report', default=True,
help='Skip the git version report.')
parser.add_argument('--actions-only', action='store_true',
help='Run the post update actions (restart services).')
parser.add_argument('--dryrun', action='store_true',
help='Don\'t actually run any commands, just log.')
parser.add_argument('--skip-service-status', action='store_true',
help='Skip checking the service status.')
parser.add_argument('--update_push_servers', action='store_true',
help='Indicate to update test_push server. If not '
'specify, then update server to production.')
parser.add_argument('--force-clean-externals', action='store_true',
default=False,
help='Force a cleanup of all untracked files within '
'site-packages/ and ExternalSource/, so that '
'build_externals will build from scratch.')
parser.add_argument('--force_update', action='store_true',
help='Force to run the update commands for afe, tko '
'and build_externals')
results = parser.parse_args(args)
if results.actions_only:
results.verify = False
results.update = False
results.report = False
# TODO(dgarrett): Make these behaviors support dryrun.
if results.dryrun:
results.verify = False
results.update = False
results.force_clean_externals = False
if not results.update_push_servers:
print('Will skip service check for pushing servers in prod.')
results.skip_service_status = True
return results
class ChangeDir(object):
"""Context manager for changing to a directory temporarily."""
def __init__(self, dir):
self.new_dir = dir
self.old_dir = None
def __enter__(self):
self.old_dir = os.getcwd()
os.chdir(self.new_dir)
def __exit__(self, exc_type, exc_val, exc_tb):
os.chdir(self.old_dir)
def _sync_chromiumos_repo():
"""Update ~chromeos-test/chromiumos repo."""
print('Updating ~chromeos-test/chromiumos')
with ChangeDir(os.path.expanduser('~chromeos-test/chromiumos')):
ret = subprocess.call(['repo', 'sync', '--force-sync'],
stderr=subprocess.STDOUT)
_clean_pyc_files()
if ret != 0:
print('Update failed, exited with status: %d' % ret)
def main(args):
"""Main method."""
# Be careful before you change this call to `os.chdir()`:
# We make several calls to `subprocess.check_output()` and
# friends that depend on this directory, most notably calls to
# the 'repo' command from `verify_repo_clean()`.
os.chdir(common.autotest_dir)
global_config.global_config.parse_config_file()
behaviors = parse_arguments(args)
print('Updating server: %s' % HOSTNAME)
if behaviors.verify:
print('Checking tree status:')
verify_repo_clean()
print('Tree status: clean')
if behaviors.force_clean_externals:
print('Cleaning all external packages and their cache...')
_clean_externals()
print('...done.')
versions_before = repo_versions()
versions_after = set()
cmd_versions_before = repo_versions_to_decide_whether_run_cmd_update()
cmd_versions_after = set()
if behaviors.update:
print('Updating Repo.')
repo_sync(behaviors.update_push_servers)
versions_after = repo_versions()
cmd_versions_after = repo_versions_to_decide_whether_run_cmd_update()
_sync_chromiumos_repo()
if behaviors.actions:
# If the corresponding repo/file not change, no need to run the cmd.
cmds_skip = (set() if behaviors.force_update else
{t[0] for t in cmd_versions_before & cmd_versions_after})
run_deploy_actions(
cmds_skip, behaviors.dryrun, behaviors.skip_service_status,
use_chromite_master=behaviors.update_push_servers)
if behaviors.report:
print('Changes:')
print(report_changes(versions_before, versions_after))
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))