blob: bb06aaa9d46e328eadc71f46d7a1a4fb259d4a1d [file] [log] [blame]
# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Chromite main test runner.
Run the specified tests. If none are specified, we'll scan the
tree looking for tests to run and then only run the semi-fast ones.
You can add a .testignore file to a dir to disable scanning it.
"""
from __future__ import print_function
import errno
import logging
import multiprocessing
import os
import signal
import stat
import sys
import tempfile
from chromite.cbuildbot import constants
from chromite.lib import cgroups
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import namespaces
from chromite.lib import osutils
from chromite.lib import timeout_util
# How long (in minutes) to let a test run before we kill it.
TEST_TIMEOUT = 20
# How long (in minutes) before we send SIGKILL after the timeout above.
TEST_SIG_TIMEOUT = 5
# How long (in seconds) to let tests clean up after CTRL+C is sent.
SIGINT_TIMEOUT = 5
# How long (in seconds) to let all children clean up after CTRL+C is sent.
CTRL_C_TIMEOUT = SIGINT_TIMEOUT + 5
# Test has to run inside the chroot.
INSIDE = 'inside'
# Test has to run outside the chroot.
OUTSIDE = 'outside'
# Don't run this test (please add a comment as to why).
SKIP = 'skip'
# List all exceptions, with a token describing what's odd here.
SPECIAL_TESTS = {
# Tests that need to run inside the chroot.
'cbuildbot/stages/test_stages_unittest.py': INSIDE,
'cros/commands/cros_build_unittest.py': INSIDE,
# TODO: This should be INSIDE; needs upgraded pylint first.
'cros/commands/lint_unittest.py': SKIP,
'lib/filetype_unittest.py': INSIDE,
'lib/upgrade_table_unittest.py': INSIDE,
'scripts/cros_list_modified_packages_unittest.py': INSIDE,
'scripts/cros_mark_as_stable_unittest.py': INSIDE,
'scripts/cros_mark_chrome_as_stable_unittest.py': INSIDE,
'scripts/cros_mark_mojo_as_stable_unittest.py': INSIDE,
'scripts/sync_package_status_unittest.py': INSIDE,
'scripts/cros_portage_upgrade_unittest.py': INSIDE,
'scripts/dep_tracker_unittest.py': INSIDE,
'scripts/test_image_unittest.py': INSIDE,
'scripts/upload_package_status_unittest.py': INSIDE,
# Tests that need to run outside the chroot.
'lib/cgroups_unittest.py': OUTSIDE,
# Tests that take >2 minutes to run. All the slow tests are
# disabled atm though ...
#'scripts/cros_portage_upgrade_unittest.py': SKIP,
}
SLOW_TESTS = {
# Tests that require network can be really slow.
'buildbot/manifest_version_unittest.py': SKIP,
'buildbot/repository_unittest.py': SKIP,
'buildbot/remote_try_unittest.py': SKIP,
'lib/cros_build_lib_unittest.py': SKIP,
'lib/gerrit_unittest.py': SKIP,
'lib/patch_unittest.py': SKIP,
# cgroups_unittest runs cros_sdk a lot, so is slow.
'lib/cgroups_unittest.py': SKIP,
}
def RunTest(test, cmd, tmpfile, finished, total):
"""Run |test| with the |cmd| line and save output to |tmpfile|.
Args:
test: The human readable name for this test.
cmd: The full command line to run the test.
tmpfile: File to write test output to.
finished: Counter to update when this test finishes running.
total: Total number of tests to run.
Returns:
The exit code of the test.
"""
cros_build_lib.Info('Starting %s', test)
def _Finished(_log_level, _log_msg, result, delta):
with finished.get_lock():
finished.value += 1
if result.returncode:
func = cros_build_lib.Error
msg = 'Failed'
else:
func = cros_build_lib.Info
msg = 'Finished'
func('%s [%i/%i] %s (%s)', msg, finished.value, total, test, delta)
ret = cros_build_lib.TimedCommand(
cros_build_lib.RunCommand, cmd, capture_output=True, error_code_ok=True,
combine_stdout_stderr=True, debug_level=logging.DEBUG,
int_timeout=SIGINT_TIMEOUT, timed_log_callback=_Finished)
if ret.returncode:
tmpfile.write(ret.output)
if not ret.output:
tmpfile.write('<no output>\n')
tmpfile.close()
return ret.returncode
def BuildTestSets(tests, chroot_available, network):
"""Build the tests to execute.
Take care of special test handling like whether it needs to be inside or
outside of the sdk, whether the test should be skipped, etc...
Args:
tests: List of tests to execute.
chroot_available: Whether we can execute tests inside the sdk.
network: Whether to execute network tests.
Returns:
List of tests to execute and their full command line.
"""
testsets = []
for test in tests:
cmd = [test]
# See if this test requires special consideration.
status = SPECIAL_TESTS.get(test)
if status is SKIP:
cros_build_lib.Info('Skipping %s', test)
continue
elif status is INSIDE:
if not cros_build_lib.IsInsideChroot():
if not chroot_available:
cros_build_lib.Info('Skipping %s: chroot not available', test)
continue
cmd = ['cros_sdk', '--', os.path.join('..', '..', 'chromite', test)]
elif status is OUTSIDE:
if cros_build_lib.IsInsideChroot():
cros_build_lib.Info('Skipping %s: must be outside the chroot', test)
continue
else:
mode = os.stat(test).st_mode
if stat.S_ISREG(mode):
if not mode & 0o111:
cros_build_lib.Debug('Skipping %s: not executable', test)
continue
else:
cros_build_lib.Debug('Skipping %s: not a regular file', test)
continue
# Build up the final test command.
cmd.append('--verbose')
if network:
cmd.append('--network')
cmd = ['timeout', '--preserve-status', '-k', '%sm' % TEST_SIG_TIMEOUT,
'%sm' % TEST_TIMEOUT] + cmd
testsets.append((test, cmd, tempfile.TemporaryFile()))
return testsets
def RunTests(tests, jobs=1, chroot_available=True, network=False, dryrun=False):
"""Execute |paths| with |jobs| in parallel (including |network| tests).
Args:
tests: The tests to run.
jobs: How many tests to run in parallel.
chroot_available: Whether we can run tests inside the sdk.
network: Whether to run network based tests.
dryrun: Do everything but execute the test.
Returns:
True if all tests pass, else False.
"""
finished = multiprocessing.Value('i')
testsets = []
pids = []
failed = aborted = False
def WaitOne():
(pid, status) = os.wait()
pids.remove(pid)
return status
# Launch all the tests!
try:
# Build up the testsets.
testsets = BuildTestSets(tests, chroot_available, network)
# Fork each test and add it to the list.
for test, cmd, tmpfile in testsets:
if len(pids) >= jobs:
if WaitOne():
failed = True
pid = os.fork()
if pid == 0:
ret = 1
try:
if dryrun:
cros_build_lib.Info('Would have run: %s',
cros_build_lib.CmdToStr(cmd))
ret = 0
else:
ret = RunTest(test, cmd, tmpfile, finished, len(testsets))
except KeyboardInterrupt:
pass
except BaseException:
cros_build_lib.Error('%s failed', test, exc_info=True)
# We cannot run clean up hooks in the child because it'll break down
# things like tempdir context managers.
os._exit(ret) # pylint: disable=protected-access
pids.append(pid)
# Wait for all of them to get cleaned up.
while pids:
if WaitOne():
failed = True
except KeyboardInterrupt:
# If the user wants to stop, reap all the pending children.
cros_build_lib.Warning('CTRL+C received; cleaning up tests')
aborted = True
CleanupChildren(pids)
# Walk through the results.
failed_tests = []
for test, cmd, tmpfile in testsets:
tmpfile.seek(0)
output = tmpfile.read()
if output:
failed_tests.append(test)
print()
cros_build_lib.Error('### LOG: %s', test)
print(output.rstrip())
print()
if failed_tests:
cros_build_lib.Error('The following %i tests failed:\n %s',
len(failed_tests), '\n '.join(sorted(failed_tests)))
return False
elif aborted or failed:
return False
return True
def CleanupChildren(pids):
"""Clean up all the children in |pids|."""
# Note: SIGINT was already sent due to the CTRL+C via the kernel itself.
# So this func is just waiting for them to clean up.
handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
def _CheckWaitpid(ret):
(pid, _status) = ret
if pid:
try:
pids.remove(pid)
except ValueError:
# We might have reaped a grandchild -- be robust.
pass
return len(pids)
def _Waitpid():
try:
return os.waitpid(-1, os.WNOHANG)
except OSError as e:
if e.errno == errno.ECHILD:
# All our children went away!
pids[:] = []
return (0, 0)
else:
raise
def _RemainingTime(remaining):
print('\rwaiting %s for %i tests to exit ... ' % (remaining, len(pids)),
file=sys.stderr, end='')
try:
timeout_util.WaitForSuccess(_CheckWaitpid, _Waitpid,
timeout=CTRL_C_TIMEOUT, period=0.1,
side_effect_func=_RemainingTime)
print('All tests cleaned up!')
return
except timeout_util.TimeoutError:
# Let's kill them hard now.
print('Hard killing %i tests' % len(pids))
for pid in pids:
try:
os.kill(pid, signal.SIGKILL)
except OSError as e:
if e.errno != errno.ESRCH:
raise
finally:
signal.signal(signal.SIGINT, handler)
def FindTests(search_paths=('.',)):
"""Find all the tests available in |search_paths|."""
for search_path in search_paths:
for root, dirs, files in os.walk(search_path):
if os.path.exists(os.path.join(root, '.testignore')):
# Delete the dir list in place.
dirs[:] = []
continue
dirs[:] = [x for x in dirs if x[0] != '.']
for path in files:
test = os.path.join(os.path.relpath(root, search_path), path)
if test.endswith('_unittest.py'):
yield test
def ChrootAvailable():
"""See if `cros_sdk` will work at all.
If we try to run unittests in the buildtools group, we won't be able to
create one.
"""
ret = cros_build_lib.RunCommand(
['repo', 'list'], capture_output=True, error_code_ok=True,
combine_stdout_stderr=True, debug_level=logging.DEBUG)
return 'chromiumos-overlay' in ret.output
def _ReExecuteIfNeeded(argv, network):
"""Re-execute as root so we can unshare resources."""
if os.geteuid() != 0:
cmd = ['sudo', '-E', 'HOME=%s' % os.environ['HOME'],
'PATH=%s' % os.environ['PATH'], '--'] + argv
os.execvp(cmd[0], cmd)
else:
cgroups.Cgroup.InitSystem()
# Using a pid ns would be good, but CTRL+C does not play well yet.
namespaces.SimpleUnshare(net=not network, pid=False)
# We got our namespaces, so switch back to the user to run the tests.
gid = int(os.environ.pop('SUDO_GID'))
uid = int(os.environ.pop('SUDO_UID'))
os.setresgid(gid, gid, gid)
os.setresuid(uid, uid, uid)
os.environ['USER'] = os.environ.pop('SUDO_USER')
def GetParser():
parser = commandline.ArgumentParser(description=__doc__)
parser.add_argument('-q', '--quick', default=False, action='store_true',
help='Only run the really quick tests')
parser.add_argument('-n', '--dry-run', default=False, action='store_true',
dest='dryrun',
help='Do everything but actually run the test')
parser.add_argument('-l', '--list', default=False, action='store_true',
help='List all the available tests')
parser.add_argument('-j', '--jobs', type=int,
help='Number of tests to run in parallel at a time')
parser.add_argument('--network', default=False, action='store_true',
help='Run tests that depend on good network connectivity')
parser.add_argument('tests', nargs='*', default=None, help='Tests to run')
return parser
def main(argv):
parser = GetParser()
opts = parser.parse_args(argv)
opts.Freeze()
# Process list output quickly as it takes no privileges.
if opts.list:
print('\n'.join(sorted(opts.tests or FindTests())))
return
# Now let's run some tests.
_ReExecuteIfNeeded([sys.argv[0]] + argv, opts.network)
os.chdir(constants.CHROMITE_DIR)
tests = opts.tests or FindTests()
if opts.quick:
SPECIAL_TESTS.update(SLOW_TESTS)
jobs = opts.jobs or multiprocessing.cpu_count()
with cros_build_lib.ContextManagerStack() as stack:
# If we're running outside the chroot, try to contain ourselves.
if cgroups.Cgroup.IsSupported() and not cros_build_lib.IsInsideChroot():
stack.Add(cgroups.SimpleContainChildren, 'run_tests')
# Throw all the tests into a custom tempdir so that if we do CTRL+C, we can
# quickly clean up all the files they might have left behind.
stack.Add(osutils.TempDir, prefix='chromite.run_tests.', set_global=True,
sudo_rm=True)
if RunTests(tests, jobs=jobs, chroot_available=ChrootAvailable(),
network=opts.network, dryrun=opts.dryrun):
cros_build_lib.Info('All tests succeeded!')
else:
return 1
if not opts.network:
cros_build_lib.Warning('Network tests skipped; use --network to run them')