blob: dca18f0def372cd75c8dc274f93a91b55cf027de [file] [log] [blame]
# Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Main builder code for Chromium OS.
Used by Chromium OS buildbot configuration for all Chromium OS builds including
full and pre-flight-queue builds.
import distutils.version
import glob
import multiprocessing
import optparse
import os
import pprint
import sys
from chromite.buildbot import builderstage as bs
from chromite.buildbot import cbuildbot_background as background
from chromite.buildbot import cbuildbot_config
from chromite.buildbot import cbuildbot_stages as stages
from chromite.buildbot import cbuildbot_results as results_lib
from chromite.buildbot import constants
from chromite.buildbot import gerrit_helper
from chromite.buildbot import patch as cros_patch
from chromite.buildbot import remote_try
from chromite.buildbot import repository
from chromite.buildbot import tee
from chromite.lib import cgroups
from chromite.lib import cleanup
from chromite.lib import cros_build_lib as cros_lib
from chromite.lib import sudo
cros_lib.STRICT_SUDO = True
_DEFAULT_LOG_DIR = 'cbuildbot_logs'
_BUILDBOT_LOG_FILE = 'cbuildbot.log'
_DEFAULT_INT_BUILDROOT = 'trybot-internal'
_PATH_TO_CBUILDBOT = 'chromite/bin/cbuildbot'
constants.CANARY_TYPE, constants.CHROME_PFQ_TYPE,
def _PrintValidConfigs(trybot_only=True):
"""Print a list of valid buildbot configs.
trybot_only: Only print selected trybot configs, as specified by the
'trybot_list' config setting.
print 'config'.ljust(COLUMN_WIDTH), 'description'
print '------'.ljust(COLUMN_WIDTH), '-----------'
config_names = cbuildbot_config.config.keys()
for name in config_names:
if not trybot_only or cbuildbot_config.config[name]['trybot_list']:
desc = ''
if cbuildbot_config.config[name]['description']:
desc = cbuildbot_config.config[name]['description']
print name.ljust(COLUMN_WIDTH), desc
def _GetConfig(config_name):
"""Gets the configuration for the build"""
if not cbuildbot_config.config.has_key(config_name):
print 'Non-existent configuration %s specified.' % config_name
print 'Please specify one of:'
result = cbuildbot_config.config[config_name]
return result
def _GetChromiteTrackingBranch():
"""Returns the remote branch associated with chromite."""
cwd = os.path.dirname(os.path.realpath(__file__))
branch = cros_lib.GetCurrentBranch(cwd)
if branch:
tracking_branch = cros_lib.GetTrackingBranch(branch, cwd)[1]
if tracking_branch.startswith('refs/heads/'):
return tracking_branch.replace('refs/heads/', '')
# If we are not on a branch, or if the tracking branch is a revision,
# use the push branch. For repo repositories, this will be the manifest
# branch configured for this project. For other repositories, we'll just
# guess 'master', since there's no easy way to find out what branch
# we're on.
return cros_lib.GetPushBranch(cwd)[1]
def _PreProcessPatches(gerrit_patches, local_patches):
"""Validate patches ASAP to catch user errors. Also generate patch info.
gerrit_patches: List of gerrit CL ID's passed in by user.
local_patches: List of local project branches to generate patches from.
A tuple containing a list of cros_patch.GerritPatch and a list of
cros_patch.LocalPatch objects.
gerrit_patch_info = []
local_patch_info = []
if gerrit_patches:
gerrit_patch_info = gerrit_helper.GetGerritPatchInfo(gerrit_patches)
for patch in gerrit_patch_info:
if patch.IsAlreadyMerged():
cros_lib.Warning('Patch %s has already been merged.' % str(patch))
except gerrit_helper.GerritException as e:
if local_patches:
local_patch_info = cros_patch.PrepareLocalPatches(
except cros_patch.PatchException as e:
return gerrit_patch_info, local_patch_info
class Builder(object):
"""Parent class for all builder types.
This class functions as a parent class for various build types. It's intended
use is builder_instance.Run().
build_config: The configuration dictionary from cbuildbot_config.
options: The options provided from optparse in main().
completed_stages_file: Where we store resume state.
archive_url: Where our artifacts for this builder will be archived.
tracking_branch: The tracking branch for this build.
release_tag: The associated "chrome os version" of this build.
gerrit_patches: Gerrit patches to be included in build.
local_patches: Local patches to be included in build.
def __init__(self, options, build_config):
"""Initializes instance variables. Must be called by all subclasses."""
self.build_config = build_config
self.options = options
# TODO, Remove here and in config after bug chromium-os:14649 is fixed.
if self.build_config['chromeos_official']:
os.environ['CHROMEOS_OFFICIAL'] = '1'
self.completed_stages_file = os.path.join(options.buildroot,
self.archive_stages = {}
self.archive_urls = {}
self.release_tag = None
self.tracking_branch = _GetChromiteTrackingBranch()
self.gerrit_patches = None
self.local_patches = None
def Initialize(self):
"""Runs through the initialization steps of an actual build."""
if self.options.resume and os.path.exists(self.completed_stages_file):
with open(self.completed_stages_file, 'r') as load_file:
# We only want to do this if we need to patch changes.
if not results_lib.Results.GetPrevious().get(
self._GetStageInstance(stages.PatchChangesStage, None, None).name):
self.gerrit_patches, self.local_patches = _PreProcessPatches(
self.options.gerrit_patches, self.options.local_patches)
def _GetStageInstance(self, stage, *args, **kwargs):
"""Helper function to get an instance given the args.
Useful as almost all stages just take in options and build_config.
config = kwargs.pop('config', self.build_config)
return stage(self.options, config, *args, **kwargs)
def _SetReleaseTag(self):
"""Sets the release tag from the manifest_manager.
Must be run after sync stage as syncing enables us to have a release tag.
# Extract version we have decided to build into self.release_tag.
manifest_manager = stages.ManifestVersionedSyncStage.manifest_manager
if manifest_manager:
self.release_tag = manifest_manager.current_version
def _RunStage(self, stage, *args, **kwargs):
"""Wrapper to run a stage."""
stage_instance = self._GetStageInstance(stage, *args, **kwargs)
return stage_instance.Run()
def GetSyncInstance(self):
"""Returns an instance of a SyncStage that should be run.
Subclasses must override this method.
raise NotImplementedError()
def RunStages(self):
"""Subclasses must override this method. Runs the appropriate code."""
raise NotImplementedError()
def _WriteCheckpoint(self):
"""Drops a completed stages file with current state."""
with open(self.completed_stages_file, 'w+') as save_file:
def _ShouldReExecuteInBuildRoot(self):
"""Returns True if this build should be re-executed in the buildroot."""
abs_buildroot = os.path.abspath(self.options.buildroot)
return not os.path.abspath(__file__).startswith(abs_buildroot)
def _ReExecuteInBuildroot(self, sync_instance):
"""Reexecutes self in buildroot and returns True if build succeeds.
This allows the buildbot code to test itself when changes are patched for
buildbot-related code. This is a no-op if the buildroot == buildroot
of the running chromite checkout.
sync_instance: Instance of the sync stage that was run to sync.
True if the Build succeeded.
# If we are resuming, use last checkpoint.
if not self.options.resume:
# Re-write paths to use absolute paths.
# Suppress any timeout options given from the commandline in the
# invoked cbuildbot; our timeout will enforce it instead.
args_to_append = ['--resume', '--timeout', '0', '--buildroot',
if self.options.chrome_root:
args_to_append += ['--chrome_root',
if stages.ManifestVersionedSyncStage.manifest_manager:
ver = stages.ManifestVersionedSyncStage.manifest_manager.current_version
args_to_append += ['--version', ver]
if isinstance(sync_instance, stages.CommitQueueSyncStage):
vp_file = sync_instance.SaveValidationPool()
args_to_append += ['--validation_pool', vp_file]
# Re-run the command in the buildroot.
# Finally, be generous and give the invoked cbuildbot 30s to shutdown
# when something occurs. It should exit quicker, but the sigterm may
# hit while the system is particularly busy.
return_obj = cros_lib.RunCommand(
[_PATH_TO_CBUILDBOT] + sys.argv[1:] + args_to_append,
cwd=self.options.buildroot, error_code_ok=True, kill_timeout=30)
return return_obj.returncode == 0
def Run(self):
"""Main runner for this builder class. Runs build and prints summary."""
print_report = True
success = True
sync_instance = self.GetSyncInstance()
if (self.gerrit_patches or self.local_patches
or self.options.remote_patches):
self.gerrit_patches, self.local_patches)
if self._ShouldReExecuteInBuildRoot():
print_report = False
success = self._ReExecuteInBuildroot(sync_instance)
if print_report:
print '\n\n\n@@@BUILD_STEP Report@@@\n'
results_lib.Results.Report(sys.stdout, self.archive_urls,
success = results_lib.Results.BuildSucceededSoFar()
return success
class SimpleBuilder(Builder):
"""Builder that performs basic vetting operations."""
def GetSyncInstance(self):
"""Sync to lkgm or TOT as necessary.
Returns: the instance of the sync stage that was run.
if self.options.lkgm or self.build_config['use_lkgm']:
sync_stage = self._GetStageInstance(stages.LKGMSyncStage)
sync_stage = self._GetStageInstance(stages.SyncStage)
return sync_stage
def _RunBackgroundStagesForBoard(self, board):
"""Run background board-specific stages for the specified board."""
archive_stage = self.archive_stages[board]
configs = self.build_config['board_specific_configs']
config = configs.get(board, self.build_config)
stage_list = [[stages.VMTestStage, board, archive_stage],
[stages.ChromeTestStage, board, archive_stage],
[stages.UnitTestStage, board],
[stages.UploadPrebuiltsStage, board]]
# We can not run hw tests without archiving the payloads.
if self.options.archive:
for suite in config['hw_tests']:
stage_list.append([stages.HWTestStage, board, archive_stage, suite])
steps = [self._GetStageInstance(*x, config=config).Run for x in stage_list]
background.RunParallelSteps(steps + [archive_stage.Run])
def RunStages(self):
"""Runs through build process."""
# TODO(sosa): Split these out into classes.
if self.build_config['build_type'] == constants.CHROOT_BUILDER_TYPE:
elif self.build_config['build_type'] == constants.REFRESH_PACKAGES_TYPE:
configs = self.build_config['board_specific_configs']
for board in self.build_config['boards']:
config = configs.get(board, self.build_config)
archive_stage = self._GetStageInstance(stages.ArchiveStage, board,
self.archive_stages[board] = archive_stage
# Set up a process pool to run test/archive stages in the background.
# This process runs task(board) for each board added to the queue.
queue = multiprocessing.Queue()
task = self._RunBackgroundStagesForBoard
with background.BackgroundTaskRunner(queue, task):
for board in self.build_config['boards']:
# Run BuildTarget in the foreground.
archive_stage = self.archive_stages[board]
config = configs.get(board, self.build_config)
self._RunStage(stages.BuildTargetStage, board, archive_stage,
self.release_tag, config=config)
self.archive_urls[board] = archive_stage.GetDownloadUrl()
# Kick off task(board) in the background.
class DistributedBuilder(SimpleBuilder):
"""Build class that has special logic to handle distributed builds.
These builds sync using git/manifest logic in manifest_versions. In general
they use a non-distributed builder code for the bulk of the work.
def __init__(self, options, build_config):
"""Initializes a buildbot builder.
Extra variables:
completion_stage_class: Stage used to complete a build. Set in the Sync
super(DistributedBuilder, self).__init__(options, build_config)
self.completion_stage_class = None
def GetSyncInstance(self):
"""Syncs the tree using one of the distributed sync logic paths.
Returns: the instance of the sync stage that was run.
# Determine sync class to use. CQ overrides PFQ bits so should check it
# first.
if cbuildbot_config.IsCQType(self.build_config['build_type']):
sync_stage = self._GetStageInstance(stages.CommitQueueSyncStage)
self.completion_stage_class = stages.CommitQueueCompletionStage
elif cbuildbot_config.IsPFQType(self.build_config['build_type']):
sync_stage = self._GetStageInstance(stages.LKGMCandidateSyncStage)
self.completion_stage_class = stages.LKGMCandidateSyncCompletionStage
sync_stage = self._GetStageInstance(stages.ManifestVersionedSyncStage)
self.completion_stage_class = stages.ManifestVersionedSyncCompletionStage
return sync_stage
def Publish(self, was_build_successful):
"""Completes build by publishing any required information."""
completion_stage = self._GetStageInstance(self.completion_stage_class,
name =
if not results_lib.Results.WasStageSuccessful(name):
should_publish_changes = False
should_publish_changes = (self.build_config['master'] and
if should_publish_changes:
def RunStages(self):
"""Runs simple builder logic and publishes information to overlays."""
was_build_successful = False
super(DistributedBuilder, self).RunStages()
was_build_successful = results_lib.Results.BuildSucceededSoFar()
except SystemExit as ex:
# If a stage calls sys.exit(0), it's exiting with success, so that means
# we should mark ourselves as successful.
if ex.code == 0:
was_build_successful = True
def _ConfirmBuildRoot(buildroot):
"""Confirm with user the inferred buildroot, and mark it as confirmed."""
warning = 'Using default directory %s as buildroot' % buildroot
response = cros_lib.YesNoPrompt(default=cros_lib.NO, warning=warning,
if response == cros_lib.NO:
print('Please specify a buildroot with the --buildroot option.')
if not os.path.exists(buildroot):
def _DetermineDefaultBuildRoot(internal_build):
"""Default buildroot to be under the directory that contains current checkout.
internal_build: Whether the build is an internal build
repo_dir = cros_lib.FindRepoDir()
if not repo_dir:
cros_lib.Die('Could not find root of local checkout. Please specify'
'using --buildroot option.')
# Place trybot buildroot under the directory containing current checkout.
top_level = os.path.dirname(os.path.realpath(os.path.dirname(repo_dir)))
if internal_build:
buildroot = os.path.join(top_level, _DEFAULT_INT_BUILDROOT)
buildroot = os.path.join(top_level, _DEFAULT_EXT_BUILDROOT)
return buildroot
def _BackupPreviousLog(log_file, backup_limit=25):
"""Rename previous log.
log_file: The absolute path to the previous log.
if os.path.exists(log_file):
old_logs = sorted(glob.glob(log_file + '.*'),
if len(old_logs) >= backup_limit:
last = 0
if old_logs:
last = int(old_logs.pop().rpartition('.')[2])
os.rename(log_file, log_file + '.' + str(last + 1))
def _RunBuildStagesWrapper(options, build_config):
"""Helper function that wraps RunBuildStages()."""
def IsDistributedBuilder():
"""Determines whether the build_config should be a DistributedBuilder."""
if not options.buildbot:
return False
elif build_config['build_type'] in _DISTRIBUTED_TYPES:
chrome_rev = build_config['chrome_rev']
if options.chrome_rev: chrome_rev = options.chrome_rev
# We don't do distributed logic to TOT Chrome PFQ's, nor local
# chrome roots (e.g. chrome try bots)
if chrome_rev not in [constants.CHROME_REV_TOT,
return True
return False
# Start tee-ing output to file.
log_file = None
if options.tee:
default_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)
dirname = options.log_dir or default_dir
log_file = os.path.join(dirname, _BUILDBOT_LOG_FILE)
with cros_lib.AllowDisabling(options.tee, tee.Tee, log_file):
cros_lib.Info("cbuildbot executed with args %s"
% ' '.join(map(repr, sys.argv)))
if IsDistributedBuilder():
buildbot = DistributedBuilder(options, build_config)
buildbot = SimpleBuilder(options, build_config)
if not buildbot.Run():
if options.tee:
cros_lib.Info('Output should be saved to %s' % log_file)
# Parser related functions
def _CheckLocalPatches(local_patches):
"""Do an early quick check of the passed-in patches.
If the branch of a project is not specified we append the current branch the
project is on.
verified_patches = []
for patch in local_patches:
components = patch.split(':')
if len(components) > 2:
msg = 'Specify local patches in project[:branch] format.'
raise optparse.OptionValueError(msg)
# validate project
project = components[0]
if not cros_lib.DoesProjectExist('.', project):
raise optparse.OptionValueError('Project %s does not exist.' % project)
project_dir = cros_lib.GetProjectDir('.', project)
# If no branch was specified, we use the project's current branch.
if len(components) == 1:
branch = cros_lib.GetCurrentBranch(project_dir)
if not branch:
raise optparse.OptionValueError('project %s is not on a branch!'
% project)
# Append branch information to patch
patch = '%s:%s' % (project, branch)
branch = components[1]
if not cros_lib.DoesLocalBranchExist(project_dir, branch):
raise optparse.OptionValueError('Project %s does not have branch %s'
% (project, branch))
return verified_patches
def _CheckBuildRootOption(_option, _opt_str, value, parser):
"""Validate and convert buildroot to full-path form."""
value = value.strip()
if not value or value == '/':
raise optparse.OptionValueError('Invalid buildroot specified')
parser.values.buildroot = os.path.realpath(os.path.expanduser(value))
def _CheckLogDirOption(_option, _opt_str, value, parser):
"""Validate and convert buildroot to full-path form."""
parser.values.log_dir = os.path.abspath(os.path.expanduser(value))
def _CheckChromeVersionOption(_option, _opt_str, value, parser):
"""Upgrade other options based on chrome_version being passed."""
value = value.strip()
if parser.values.chrome_rev is None and value:
parser.values.chrome_rev = constants.CHROME_REV_SPEC
parser.values.chrome_version = value
def _CheckChromeRootOption(_option, _opt_str, value, parser):
"""Validate and convert chrome_root to full-path form."""
value = value.strip()
if not value or value == '/':
raise optparse.OptionValueError('Invalid chrome_root specified')
if parser.values.chrome_rev is None:
parser.values.chrome_rev = constants.CHROME_REV_LOCAL
parser.values.chrome_root = os.path.realpath(os.path.expanduser(value))
def _CheckChromeRevOption(_option, _opt_str, value, parser):
"""Validate the chrome_rev option."""
value = value.strip()
if value not in constants.VALID_CHROME_REVISIONS:
raise optparse.OptionValueError('Invalid chrome rev specified')
parser.values.chrome_rev = value
def _CreateParser():
"""Generate and return the parser with all the options."""
# Parse options
usage = "usage: %prog [options] buildbot_config"
parser = optparse.OptionParser(usage=usage)
# Main options
parser.add_option('-a', '--all', action='store_true', dest='print_all',
help=('List all of the buildbot configs available. Use '
'with the --list option'))
parser.add_option('-r', '--buildroot', action='callback', dest='buildroot',
type='string', callback=_CheckBuildRootOption,
help=('Root directory where source is checked out to, and '
'where the build occurs. For external build configs, '
"defaults to 'trybot' directory at top level of your "
'repo-managed checkout.'))
parser.add_option('--chrome_rev', default=None, type='string',
action='callback', dest='chrome_rev',
help=('Revision of Chrome to use, of type '
'[%s]' % '|'.join(constants.VALID_CHROME_REVISIONS)))
parser.add_option('-g', '--gerrit-patches', action='append', default=[],
type='string', metavar="'Id1 *int_Id2...IdN'",
help=("Space-separated list of short-form Gerrit "
"Change-Id's or change numbers to patch. Please "
"prepend '*' to internal Change-Id's"))
parser.add_option('-l', '--list', action='store_true', dest='list',
help=('List the suggested trybot configs to use. Use '
'--all to list all of the available configs.'))
parser.add_option('-p', '--local-patches', action='append', default=[],
help=('Space-separated list of project branches with '
'patches to apply. Projects are specified by name. '
'If no branch is specified the current branch of the '
'project will be used.'))
parser.add_option('--profile', default=None, type='string', action='store',
help=('Name of profile to sub-specify board variant.'))
parser.add_option('--remote', default=False, action='store_true',
help=('Specifies that this tryjob should be run remotely.'))
# Advanced options
group = optparse.OptionGroup(
'Advanced Options',
'Caution: use these options at your own risk.')
group.add_option('--buildbot', dest='buildbot', action='store_true',
default=False, help='This is running on a buildbot')
help='build number', type='int', default=0)
group.add_option('--chrome_root', default=None, type='string',
action='callback', dest='chrome_root',
help='Local checkout of Chrome to use.')
group.add_option('--chrome_version', default=None, type='string',
action='callback', dest='chrome_version',
help='Used with SPEC logic to force a particular SVN '
'revision of chrome rather than the latest.')
group.add_option('--clobber', action='store_true', dest='clobber',
help='Clears an old checkout before syncing')
group.add_option('--lkgm', action='store_true', dest='lkgm', default=False,
help='Sync to last known good manifest blessed by PFQ')
parser.add_option('--log_dir', action='callback', dest='log_dir',
type='string', callback=_CheckLogDirOption,
help=('Directory where logs are stored.'))
group.add_option('--maxarchives', dest='max_archive_builds',
default=3, type='int',
help="Change the local saved build count limit.")
group.add_option('--noarchive', action='store_false', dest='archive',
help="Don't run archive stage.")
group.add_option('--nobuild', action='store_false', dest='build',
help="Don't actually build (for cbuildbot dev")
group.add_option('--noclean', action='store_false', dest='clean',
help="Don't clean the buildroot")
group.add_option('--noprebuilts', action='store_false', dest='prebuilts',
help="Don't upload prebuilts.")
group.add_option('--nosync', action='store_false', dest='sync',
help="Don't sync before building.")
group.add_option('--nocgroups', action='store_false', dest='cgroups',
help='Disable cbuildbots usage of cgroups.')
group.add_option('--notests', action='store_false', dest='tests',
help='Override values from buildconfig and run no tests.')
group.add_option('--nouprev', action='store_false', dest='uprev',
help='Override values from buildconfig and never uprev.')
group.add_option('--reference-repo', action='store', default=None,
help='Reuse git data stored in an existing repo '
'checkout. This can drastically reduce the network '
'time spent setting up the trybot checkout. By '
"default, if this option isn't given but cbuildbot "
'is invoked from a repo checkout, cbuildbot will '
'use the repo root.')
# Indicates this is running on a remote trybot machine. '
group.add_option('--remote-trybot', dest='remote_trybot', action='store_true',
default=False, help=optparse.SUPPRESS_HELP)
# Patches uploaded by trybot client when run using the -p option.
group.add_option('--remote-patches', action='append', default=[],
group.add_option('--resume', action='store_true', default=False,
help='Skip stages already successfully completed.')
group.add_option('--timeout', action='store', type='int', default=0,
help="Specify the maximum amount of time this job can run "
"for, at which point the build will be aborted. If "
"set to zero, then there is no timeout")
group.add_option('--validation_pool', default=None,
help='Path to a pickled validation pool. Intended for use '
'only with the commit queue.')
group.add_option('--version', dest='force_version', default=None,
help='Used with manifest logic. Forces use of this version '
'rather than create or get latest.')
# Added for --remote compatibility against ToT; no-op option.
group.add_option('--remote-version', help=optparse.SUPPRESS_HELP)
# Dummy arguments for compatibility with higher branches.
group.add_option('--sourceroot', default=None, type='string',
group.add_option('--nobootstrap', dest='bootstrap', default=True,
action='store_false', help=optparse.SUPPRESS_HELP)
group.add_option('-b', '--branch', default=None, type='string',
group.add_option('--local', default=False, action='store_true',
# Debug options
group = optparse.OptionGroup(parser, "Debug Options")
group.add_option('--debug', action='store_true', default=None,
help='Override some options to run as a developer.')
group.add_option('--dump_config', action='store_true', dest='dump_config',
help='Dump out build config options, and exit.')
group.add_option('--notee', action='store_false', dest='tee', default=True,
help="Disable logging and internal tee process. Primarily "
"used for debugging cbuildbot itself.")
return parser
def _FinishParsing(options, args):
"""Perform some parsing tasks that need to take place after optparse.
This function needs to be easily testable! Keep it free of
environment-dependent code. Put more detailed usage validation in
options, args: The options/args object returned by optparse
if options.chrome_root:
if options.chrome_rev != constants.CHROME_REV_LOCAL:
cros_lib.Die('Chrome rev must be %s if chrome_root is set.' %
if options.chrome_rev == constants.CHROME_REV_LOCAL:
cros_lib.Die('Chrome root must be set if chrome_rev is %s.' %
if options.chrome_version:
if options.chrome_rev != constants.CHROME_REV_SPEC:
cros_lib.Die('Chrome rev must be %s if chrome_version is set.' %
if options.chrome_rev == constants.CHROME_REV_SPEC:
cros_lib.Die('Chrome rev must not be %s if chrome_version is not set.' %
if options.remote and not (options.gerrit_patches or options.local_patches):
cros_lib.Die('Must provide patches when running with --remote.')
if len(args) > 1 and not options.remote:
cros_lib.Die('Multiple configs not supported if not running with --remote.')
if options.buildbot and options.remote_trybot:
cros_lib.Die('--buildbot and --remote-trybot cannot be used together.')
# Record whether --debug was set explicitly vs. it was inferred.
options.debug_forced = False
if options.debug:
options.debug_forced = True
# We don't set debug by default for
# 1. --buildbot invocations.
# 2. --remote invocations, because it needs to push changes to the tryjob
# repo.
options.debug = not options.buildbot and not options.remote
def _SplitAndFlatten(appended_items):
"""Given a list of space-separated items, split into flattened list.
Given ['abc def', 'hij'] return ['abc', 'def', 'hij'].
appended_items: List of delimiter-separated items.
Returns: Flattened list.
new_list = []
for item in appended_items:
return new_list
def _PostParseCheck(options, args):
"""Perform some usage validation after we've parsed the arguments
options/args: The options/args object returned by optparse
if not options.resume:
options.gerrit_patches = _SplitAndFlatten(options.gerrit_patches)
options.remote_patches = _SplitAndFlatten(options.remote_patches)
# TODO(rcui): Split this into two stages, one that parses, another that
# validates. Parsing step will be called by _FinishParsing().
options.local_patches = _CheckLocalPatches(
except optparse.OptionValueError as e:
def _ParseCommandLine(parser, argv):
"""Completely parse the commandline arguments"""
(options, args) = parser.parse_args(argv)
# Strip out null arguments.
# TODO(rcui): Remove when buildbot is fixed
args = [arg for arg in args if arg]
_FinishParsing(options, args)
return options, args
def main(argv):
# Set umask to 022 so files created by buildbot are readable.
if cros_lib.IsInsideChroot():
cros_lib.Die('Please run cbuildbot from outside the chroot.')
parser = _CreateParser()
(options, args) = _ParseCommandLine(parser, argv)
if options.list:
_PrintValidConfigs(not options.print_all)
_PostParseCheck(options, args)
if options.remote:
# Verify configs are valid.
for bot in args:
# Verify gerrit patches are valid.
print 'Verifying patches...'
_, local_patches = _PreProcessPatches(options.gerrit_patches,
print 'Submitting tryjob...'
tryjob = remote_try.RemoteTryJob(options, args, local_patches)
print 'Tryjob submitted!'
print ('Go to %s to view the status of your job.'
% tryjob.GetTrybotConsoleLink())
if args:
# Only expecting one config
bot_id = args[-1]
build_config = _GetConfig(bot_id)
parser.error('Invalid usage. Use -h to see usage.')
if options.reference_repo is None:
repo_path = os.path.join(constants.SOURCE_ROOT, '.repo')
# If we're being run from a repo checkout, reuse the repo's git pool to
# cut down on sync time.
if os.path.exists(repo_path):
options.reference_repo = constants.SOURCE_ROOT
elif options.reference_repo:
if not os.path.exists(options.reference_repo):
parser.error('Reference path %s does not exist'
% (options.reference_repo,))
elif not os.path.exists(os.path.join(options.reference_repo, '.repo')):
parser.error('Reference path %s does not look to be the base of a '
'repo checkout; no .repo exists in the root.'
% (options.reference_repo,))
if options.buildbot:
if not options.cgroups:
parser.error('Options --buildbot and --nocgroups cannot be used '
'together. Cgroup support is required for buildbot mode.')
if not cgroups.Cgroup.CgroupsSupported():
parser.error('Option --buildbot was given, but this system does not '
'support cgroups. Failing.')
missing = []
ret = cros_lib.RunCommand('which %s' % program, shell=True,
redirect_stderr=True, redirect_stdout=True,
error_code_ok=True, print_cmd=False)
if ret.returncode != 0:
if missing:
parser.error("Option --buildbot requires the following binaries which "
"couldn't be found in $PATH: %s"
% (', '.join(missing)))
if options.reference_repo:
options.reference_repo = os.path.abspath(options.reference_repo)
if options.dump_config:
# This works, but option ordering is bad...
print 'Configuration %s:' % bot_id
pretty_printer = pprint.PrettyPrinter(indent=2)
if not options.buildroot:
if options.buildbot:
parser.error('Please specify a buildroot with the --buildroot option.')
options.buildroot = _DetermineDefaultBuildRoot(build_config['internal'])
# We use a marker file in the buildroot to indicate the user has
# consented to using this directory.
if not os.path.exists(repository.GetTrybotMarkerPath(options.buildroot)):
# Sanity check of buildroot- specifically that it's not pointing into the
# midst of an existing repo since git-repo doesn't support nesting.
if (not repository.IsARepoRoot(options.buildroot) and
parser.error('Configured buildroot %s points into a repository checkout, '
'rather than the root of it. This is not supported.'
% options.buildroot)
with cleanup.EnforcedCleanupSection() as critical_section:
with sudo.SudoKeepAlive():
with cros_lib.AllowDisabling(options.cgroups,
cgroups.SimpleContainChildren, 'cbuildbot'):
# Mark everything between EnforcedCleanupSection and here as having to
# be rolled back via the contextmanager cleanup handlers. This ensures
# that sudo bits cannot outlive cbuildbot, that anything cgroups
# would kill gets killed, etc.
with cros_lib.AllowDisabling(options.timeout > 0,
cros_lib.Timeout, options.timeout):
if not options.buildbot:
build_config = cbuildbot_config.OverrideConfigForTrybot(
_RunBuildStagesWrapper(options, build_config)