blob: e67006e020a3a9e8521cb7cf37e2ed5cb77854c5 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Code related to Remote tryjobs."""
from __future__ import print_function
import constants
import getpass
import json
import os
import sys
import time
if __name__ == '__main__':
sys.path.insert(0, constants.SOURCE_ROOT)
from chromite.cbuildbot import repository
from chromite.cbuildbot import manifest_version
from chromite.lib import cros_build_lib
from chromite.lib import cache
from chromite.lib import git
class ChromiteUpgradeNeeded(Exception):
"""Exception thrown when it's detected that we need to upgrade chromite."""
def __init__(self, version=None):
self.version = version
self.args = (version,)
def __str__(self):
version_str = ''
if self.version:
version_str = " Need format version %r support." % (self.version,)
return (
"Your version of cbuildbot is too old; please resync it, "
"and then retry your submission.%s" % (version_str,))
class ValidationError(Exception):
"""Thrown when tryjob validation fails."""
class RemoteTryJob(object):
"""Remote Tryjob that is submitted through a Git repo."""
EXTERNAL_URL = os.path.join(constants.EXTERNAL_GOB_URL,
INTERNAL_URL = os.path.join(constants.INTERNAL_GOB_URL,
# In version 3, remote patches have an extra field.
# In version 4, cherry-picking is the norm, thus multiple patches are
# generated.
TRYJOB_FORMAT_FILE = '.tryjob_minimal_format_version'
# Constants for controlling the length of JSON fields sent to buildbot.
# - The trybot description is shown when the run starts, and helps users
# distinguish between their various runs. If no trybot description is
# specified, the list of patches is used as the description. The buildbot
# database limits this field to MAX_DESCRIPTION_LENGTH characters.
# - When checking the trybot description length, we also add some PADDING
# to give buildbot room to add extra formatting around the fields used in
# the description.
# - We limit the number of patches listed in the description to
# MAX_PATCHES_IN_DESCRIPTION. This is for readability only.
# - Every individual field that is stored in a buildset is limited to
# MAX_PROPERTY_LENGTH. We use this to ensure that our serialized list of
# arguments fits within that limit.
def __init__(self, options, bots, local_patches):
"""Construct the object.
options: The parsed options passed into cbuildbot.
bots: A list of configs to run tryjobs for.
local_patches: A list of LocalPatch objects.
self.options = options
self.user = getpass.getuser()
self.repo_cache = cache.DiskCache(self.options.cache_dir)
cwd = os.path.dirname(os.path.realpath(__file__))
self.user_email = git.GetProjectUserEmail(cwd)
cros_build_lib.Info('Using email:%s', self.user_email)
# Name of the job that appears on the waterfall.
patch_list = options.gerrit_patches + options.local_patches = options.remote_description
if is None: = ''
if options.branch != 'master': = '[%s] ' % options.branch += ','.join(patch_list[:self.MAX_PATCHES_IN_DESCRIPTION])
if len(patch_list) > self.MAX_PATCHES_IN_DESCRIPTION:
remaining_patches = len(patch_list) - self.MAX_PATCHES_IN_DESCRIPTION += '... (%d more CLs)' % (remaining_patches,)
self.bots = bots[:]
self.slaves_request = options.slaves
self.description = ('name: %s\n patches: %s\nbots: %s' %
(, patch_list, self.bots))
self.extra_args = options.pass_through_args
if '--buildbot' not in self.extra_args:
self.local_patches = local_patches
self.repo_url = self.EXTERNAL_URL
self.cache_key = ('trybot',)
self.manifest = None
if repository.IsARepoRoot(options.sourceroot):
self.manifest = git.ManifestCheckout.Cached(options.sourceroot)
if repository.IsInternalRepoCheckout(options.sourceroot):
self.repo_url = self.INTERNAL_URL
self.cache_key = ('trybot-internal',)
def values(self):
return {
'bot' : self.bots,
'email' : [self.user_email],
'extra_args' : self.extra_args,
'name' :,
'slaves_request' : self.slaves_request,
'user' : self.user,
'version' : self.TRYJOB_FORMAT_VERSION,
def _VerifyForBuildbot(self):
"""Early validation, to ensure the job can be processed by buildbot."""
# Buildbot stores the trybot description in a property with a 256
# character limit. Validate that our description is well under the limit.
if (len(self.user) + len( + self.PADDING >
'remote tryjob description is too long, truncating it') =[:self.MAX_DESCRIPTION_LENGTH - self.PADDING] + '...'
# Buildbot will set extra_args as a buildset 'property'. It will store
# the property in its database in JSON form. The limit of the database
# field is 1023 characters.
if len(json.dumps(self.extra_args)) > self.MAX_PROPERTY_LENGTH:
raise ValidationError(
'The number of extra arguments passed to cbuildbot has exceeded the '
'limit. If you have a lot of local patches, upload them and use the '
'-g flag instead.')
def _Submit(self, workdir, testjob, dryrun):
"""Internal submission function. See Submit() for arg description."""
# TODO(rcui): convert to shallow clone when that's available.
current_time = str(int(time.time()))
ref_base = os.path.join('refs/tryjobs', self.user_email, current_time)
for patch in self.local_patches:
# Isolate the name; if it's a tag or a remote, let through.
# Else if it's a branch, get the full branch name minus refs/heads.
local_branch = git.StripRefsHeads(patch.ref, False)
ref_final = os.path.join(ref_base, local_branch, patch.sha1)
checkout = patch.GetCheckout(self.manifest)
print('Uploading patch %s' % patch)
patch.Upload(checkout['push_url'], ref_final, dryrun=dryrun)
# TODO(rcui): Pass in the remote instead of tag.
tag = constants.EXTERNAL_PATCH_TAG
if checkout['remote'] == constants.INTERNAL_REMOTE:
tag = constants.INTERNAL_PATCH_TAG
% (patch.project, local_branch, ref_final,
patch.tracking_branch, tag))
repository.UpdateGitRepo(workdir, self.repo_url)
version_path = os.path.join(workdir,
with open(version_path, 'r') as f:
val = int(
except ValueError:
raise ChromiteUpgradeNeeded()
raise ChromiteUpgradeNeeded(val)
push_branch = manifest_version.PUSH_BRANCH
remote_branch = ('origin', 'refs/remotes/origin/test') if testjob else None
git.CreatePushBranch(push_branch, workdir, sync=False,
file_name = '%s.%s' % (self.user,
user_dir = os.path.join(workdir, self.user)
if not os.path.isdir(user_dir):
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
git.RunGit(workdir, ['add', fullpath])
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work.
'GIT_COMMITTER_EMAIL' : self.user_email,
'GIT_AUTHOR_EMAIL' : self.user_email,
git.RunGit(workdir, ['commit', '-m', self.description],
git.PushWithRetry(push_branch, workdir, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
def Submit(self, workdir=None, testjob=False, dryrun=False):
"""Submit the tryjob through Git.
workdir: The directory to clone tryjob repo into. If you pass this
in, you are responsible for deleting the directory. Used for
testjob: Submit job to the test branch of the tryjob repo. The tryjob
will be ignored by production master.
dryrun: Setting to true will run everything except the final submit step.
if workdir is None:
with self.repo_cache.Lookup(self.cache_key) as ref:
self._Submit(ref.path, testjob, dryrun)
self._Submit(workdir, testjob, dryrun)
def GetTrybotConsoleLink(self):
"""Get link to the console for the user."""
return ('%s/console?name=%s' % (constants.TRYBOT_DASHBOARD,
def GetTrybotWaterfallLink(self):
"""Get link to the waterfall for the user."""
# Note that this will only show the jobs submitted by the user in the last
# 24 hours.
return '%s/waterfall?committer=%s&builder=%s' % (