blob: 9ca87f79dfceaf70af1d73797441ff82700a6253 [file] [log] [blame]
# Lint as: python2, python3
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import random
import re
import six
import common
from autotest_lib.client.common_lib import global_config
_CONFIG = global_config.global_config
# comments injected into the control file.
_INJECT_BEGIN = '# INJECT_BEGIN - DO NOT DELETE THIS LINE'
_INJECT_END = '# INJECT_END - DO NOT DELETE LINE'
# The regex for an injected line in the control file with the format:
# varable_name=varable_value
_INJECT_VAR_RE = re.compile('^[_A-Za-z]\w*=.+$')
def image_url_pattern():
"""Returns image_url_pattern from global_config."""
return _CONFIG.get_config_value('CROS', 'image_url_pattern', type=str)
def firmware_url_pattern():
"""Returns firmware_url_pattern from global_config."""
return _CONFIG.get_config_value('CROS', 'firmware_url_pattern', type=str)
def factory_image_url_pattern():
"""Returns path to factory image after it's been staged."""
return _CONFIG.get_config_value('CROS', 'factory_image_url_pattern',
type=str)
def sharding_factor():
"""Returns sharding_factor from global_config."""
return _CONFIG.get_config_value('CROS', 'sharding_factor', type=int)
def infrastructure_user():
"""Returns infrastructure_user from global_config."""
return _CONFIG.get_config_value('CROS', 'infrastructure_user', type=str)
def package_url_pattern(is_launch_control_build=False):
"""Returns package_url_pattern from global_config.
@param is_launch_control_build: True if the package url is for Launch
Control build. Default is False.
"""
if is_launch_control_build:
return _CONFIG.get_config_value('ANDROID', 'package_url_pattern',
type=str)
else:
return _CONFIG.get_config_value('CROS', 'package_url_pattern', type=str)
def try_job_timeout_mins():
"""Returns try_job_timeout_mins from global_config."""
return _CONFIG.get_config_value('SCHEDULER', 'try_job_timeout_mins',
type=int, default=4*60)
def get_package_url(devserver_url, build):
"""Returns the package url from the |devserver_url| and |build|.
@param devserver_url: a string specifying the host to contact e.g.
http://my_host:9090.
@param build: the build/image string to use e.g. mario-release/R19-123.0.1.
@return the url where you can find the packages for the build.
"""
return package_url_pattern() % (devserver_url, build)
def get_devserver_build_from_package_url(package_url,
is_launch_control_build=False):
"""The inverse method of get_package_url.
@param package_url: a string specifying the package url.
@param is_launch_control_build: True if the package url is for Launch
Control build. Default is False.
@return tuple containing the devserver_url, build.
"""
pattern = package_url_pattern(is_launch_control_build)
re_pattern = pattern.replace('%s', '(\S+)')
devserver_build_tuple = re.search(re_pattern, package_url).groups()
# TODO(beeps): This is a temporary hack around the fact that all
# job_repo_urls in the database currently contain 'archive'. Remove
# when all hosts have been reimaged at least once. Ref: crbug.com/214373.
return (devserver_build_tuple[0],
devserver_build_tuple[1].replace('archive/', ''))
def get_build_from_image(image):
"""Get the build name from the image string.
@param image: A string of image, can be the build name or a url to the
build, e.g.,
http://devserver/update/alex-release/R27-3837.0.0
@return: Name of the build. Return None if fail to parse build name.
"""
if not image.startswith('http://'):
return image
else:
match = re.match('.*/([^/]+/R\d+-[^/]+)', image)
if match:
return match.group(1)
def get_random_best_host(afe, host_list, require_usable_hosts=True):
"""
Randomly choose the 'best' host from host_list, using fresh status.
Hit the AFE to get latest status for the listed hosts. Then apply
the following heuristic to pick the 'best' set:
Remove unusable hosts (not tools.is_usable()), then
'Ready' > 'Running, Cleaning, Verifying, etc'
If any 'Ready' hosts exist, return a random choice. If not, randomly
choose from the next tier. If there are none of those either, None.
@param afe: autotest front end that holds the hosts being managed.
@param host_list: an iterable of Host objects, per server/frontend.py
@param require_usable_hosts: only return hosts currently in a usable
state.
@return a Host object, or None if no appropriate host is found.
"""
if not host_list:
return None
hostnames = [host.hostname for host in host_list]
updated_hosts = afe.get_hosts(hostnames=hostnames)
usable_hosts = [host for host in updated_hosts if is_usable(host)]
ready_hosts = [host for host in usable_hosts if host.status == 'Ready']
unusable_hosts = [h for h in updated_hosts if not is_usable(h)]
if ready_hosts:
return random.choice(ready_hosts)
if usable_hosts:
return random.choice(usable_hosts)
if not require_usable_hosts and unusable_hosts:
return random.choice(unusable_hosts)
return None
def remove_legacy_injection(control_file_in):
"""
Removes the legacy injection part from a control file.
@param control_file_in: the contents of a control file to munge.
@return The modified control file string.
"""
if not control_file_in:
return control_file_in
new_lines = []
lines = control_file_in.strip().splitlines()
remove_done = False
for line in lines:
if remove_done:
new_lines.append(line)
else:
if not _INJECT_VAR_RE.match(line):
remove_done = True
new_lines.append(line)
return '\n'.join(new_lines)
def remove_injection(control_file_in):
"""
Removes the injection part from a control file.
@param control_file_in: the contents of a control file to munge.
@return The modified control file string.
"""
if not control_file_in:
return control_file_in
start = control_file_in.find(_INJECT_BEGIN)
if start >=0:
end = control_file_in.find(_INJECT_END, start)
if start < 0 or end < 0:
return remove_legacy_injection(control_file_in)
end += len(_INJECT_END)
ch = control_file_in[end]
total_length = len(control_file_in)
while end <= total_length and (
ch == '\n' or ch == ' ' or ch == '\t'):
end += 1
if end < total_length:
ch = control_file_in[end]
return control_file_in[:start] + control_file_in[end:]
def inject_vars(vars, control_file_in):
"""
Inject the contents of |vars| into |control_file_in|.
@param vars: a dict to shoehorn into the provided control file string.
@param control_file_in: the contents of a control file to munge.
@return the modified control file string.
"""
control_file = ''
control_file += _INJECT_BEGIN + '\n'
for key, value in six.iteritems(vars):
# None gets injected as 'None' without this check; same for digits.
if isinstance(value, str):
control_file += "%s=%s\n" % (key, repr(value))
else:
control_file += "%s=%r\n" % (key, value)
args_dict_str = "%s=%s\n" % ('args_dict', repr(vars))
return control_file + args_dict_str + _INJECT_END + '\n' + control_file_in
def is_usable(host):
"""
Given a host, determine if the host is usable right now.
@param host: Host instance (as in server/frontend.py)
@return True if host is alive and not incorrectly locked. Else, False.
"""
return alive(host) and not incorrectly_locked(host)
def alive(host):
"""
Given a host, determine if the host is alive.
@param host: Host instance (as in server/frontend.py)
@return True if host is not under, or in need of, repair. Else, False.
"""
return host.status not in ['Repair Failed', 'Repairing']
def incorrectly_locked(host):
"""
Given a host, determine if the host is locked by some user.
If the host is unlocked, or locked by the test infrastructure,
this will return False. There is only one system user defined as part
of the test infrastructure and is listed in global_config.ini under the
[CROS] section in the 'infrastructure_user' field.
@param host: Host instance (as in server/frontend.py)
@return False if the host is not locked, or locked by the infra.
True if the host is locked by the infra user.
"""
return (host.locked and host.locked_by != infrastructure_user())
def _testname_to_keyval_key(testname):
"""Make a test name acceptable as a keyval key.
@param testname Test name that must be converted.
@return A string with selected bad characters replaced
with allowable characters.
"""
# Characters for keys in autotest keyvals are restricted; in
# particular, '/' isn't allowed. Alas, in the case of an
# aborted job, the test name will be a path that includes '/'
# characters. We want to file bugs for aborted jobs, so we
# apply a transform here to avoid trouble.
return testname.replace('/', '_')
_BUG_ID_KEYVAL = '-Bug_Id'
_BUG_COUNT_KEYVAL = '-Bug_Count'
def create_bug_keyvals(job_id, testname, bug_info):
"""Create keyvals to record a bug filed against a test failure.
@param testname Name of the test for which to record a bug.
@param bug_info Pair with the id of the bug and the count of
the number of times the bug has been seen.
@param job_id The afe job id of job which the test is associated to.
job_id will be a part of the key.
@return Keyvals to be recorded for the given test.
"""
testname = _testname_to_keyval_key(testname)
keyval_base = '%s_%s' % (job_id, testname) if job_id else testname
return {
keyval_base + _BUG_ID_KEYVAL: bug_info[0],
keyval_base + _BUG_COUNT_KEYVAL: bug_info[1]
}
def get_test_failure_bug_info(keyvals, job_id, testname):
"""Extract information about a bug filed against a test failure.
This method tries to extract bug_id and bug_count from the keyvals
of a suite. If for some reason it cannot retrieve the bug_id it will
return (None, None) and there will be no link to the bug filed. We will
instead link directly to the logs of the failed test.
If it cannot retrieve the bug_count, it will return (int(bug_id), None)
and this will result in a link to the bug filed, with an inline message
saying we weren't able to determine how many times the bug occured.
If it retrieved both the bug_id and bug_count, we return a tuple of 2
integers and link to the bug filed, as well as mention how many times
the bug has occured in the buildbot stages.
@param keyvals Keyvals associated with a suite job.
@param job_id The afe job id of the job that runs the test.
@param testname Name of a test from the suite.
@return None if there is no bug info, or a pair with the
id of the bug, and the count of the number of
times the bug has been seen.
"""
testname = _testname_to_keyval_key(testname)
keyval_base = '%s_%s' % (job_id, testname) if job_id else testname
bug_id = keyvals.get(keyval_base + _BUG_ID_KEYVAL)
if not bug_id:
return None, None
bug_id = int(bug_id)
bug_count = keyvals.get(keyval_base + _BUG_COUNT_KEYVAL)
bug_count = int(bug_count) if bug_count else None
return bug_id, bug_count
def create_job_name(build, suite, test_name):
"""Create the name of a test job based on given build, suite, and test_name.
@param build: name of the build, e.g., lumpy-release/R31-1234.0.0.
@param suite: name of the suite, e.g., bvt.
@param test_name: name of the test, e.g., dummy_Pass.
@return: the test job's name, e.g.,
lumpy-release/R31-1234.0.0/bvt/dummy_Pass.
"""
return '/'.join([build, suite, test_name])
def get_test_name(build, suite, test_job_name):
"""Get the test name from test job name.
Name of test job may contain information like build and suite. This method
strips these information and return only the test name.
@param build: name of the build, e.g., lumpy-release/R31-1234.0.0.
@param suite: name of the suite, e.g., bvt.
@param test_job_name: name of the test job, e.g.,
lumpy-release/R31-1234.0.0/bvt/dummy_Pass_SERVER_JOB.
@return: the test name, e.g., dummy_Pass_SERVER_JOB.
"""
# Do not change this naming convention without updating
# site_utils.parse_job_name.
return test_job_name.replace('%s/%s/' % (build, suite), '')