blob: c820d4408cd96b84c58d0bb0b6ea39f539f429a4 [file] [log] [blame]
# -*- coding: utf-8 -*-
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Code related to Remote tryjobs."""
from __future__ import print_function
import getpass
import json
import os
import time
from chromite.lib import auth
from chromite.lib import buildbucket_lib
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import cros_logging as logging
from chromite.lib import git
site_config = config_lib.GetConfig()
# URL to open a build details page.
class ValidationError(Exception):
"""Thrown when tryjob validation fails."""
class RemoteRequestFailure(Exception):
"""Thrown when requesting a tryjob fails."""
def DefaultDescription(description_branch='master', patches=None):
"""Calculate the default description for a tryjob.
description_branch: String name of branch to build.
patches: List of strings describing all patches includes.
Usually based on raw command line values.
result = ''
if description_branch != 'master':
result += '[%s] ' % description_branch
if patches:
result += ','.join(patches[:RemoteTryJob.MAX_PATCHES_IN_DESCRIPTION])
if len(patches) > RemoteTryJob.MAX_PATCHES_IN_DESCRIPTION:
remaining_patches = len(patches) - RemoteTryJob.MAX_PATCHES_IN_DESCRIPTION
result += '... (%d more CLs)' % (remaining_patches,)
return result
class RemoteTryJob(object):
"""Remote Tryjob that is submitted through a Git repo."""
# Constants for controlling the length of JSON fields sent to buildbot.
# - The trybot description is shown when the run starts, and helps users
# distinguish between their various runs. If no trybot description is
# specified, the list of patches is used as the description. The buildbot
# database limits this field to MAX_DESCRIPTION_LENGTH characters.
# - When checking the trybot description length, we also add some PADDING
# to give buildbot room to add extra formatting around the fields used in
# the description.
# - We limit the number of patches listed in the description to
# MAX_PATCHES_IN_DESCRIPTION. This is for readability only.
# - Every individual field that is stored in a buildset is limited to
# MAX_PROPERTY_LENGTH. We use this to ensure that our serialized list of
# arguments fits within that limit.
# Buildbucket_put response must contain 'buildbucket_bucket:bucket]',
# '[config:config_name] and '[buildbucket_id:id]'.
BUILDBUCKET_PUT_RESP_FORMAT = ('Successfully sent PUT request to '
'[buildbucket_bucket:%s] '
'with [config:%s] [buildbucket_id:%s].')
def __init__(self,
"""Construct the object.
build_configs: A list of configs to run tryjobs for.
build_group: String describing how build group on waterfall.
remote_description: Requested tryjob description.
branch: Name of branch to build for.
pass_through_args: Command line arguments to pass to cbuildbot in job.
local_patches: A list of LocalPatch objects.
committer_email: Email address of person requesting job, or None.
swarming: Boolean, do we use a swarming build?
master_buildbucket_id: String with buildbucket id of scheduling builder.
self.user = getpass.getuser()
if committer_email is not None:
self.user_email = committer_email
cwd = os.path.dirname(os.path.realpath(__file__))
self.user_email = git.GetProjectUserEmail(cwd)'Using email:%s', self.user_email)
# Name of the job that appears on the waterfall.
self.build_configs = build_configs[:]
self.build_group = build_group
self.extra_args = pass_through_args = remote_description
self.branch = branch
self.local_patches = local_patches
self.swarming = swarming
self.master_buildbucket_id = master_buildbucket_id
# Needed for handling local patches.
self.manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
# List of buildbucket_ids for submitted jobs.
self.buildbucket_ids = []
def _VerifyForBuildbot(self):
"""Early validation, to ensure the job can be processed by buildbot."""
# TODO: Delete this after all tryjobs are on swarming. This restriction
# will have been lifted.
# Buildbot stores the trybot description in a property with a 256
# character limit. Validate that our description is well under the limit.
if (len(self.user) + len( + self.PADDING >
logging.warning('remote tryjob description is too long, truncating it') =[:self.MAX_DESCRIPTION_LENGTH - self.PADDING] + '...'
# Buildbot will set extra_args as a buildset 'property'. It will store
# the property in its database in JSON form. The limit of the database
# field is 1023 characters.
if len(json.dumps(self.extra_args)) > self.MAX_PROPERTY_LENGTH:
raise ValidationError(
'The number of extra arguments passed to cbuildbot has exceeded the '
'limit. If you have a lot of local patches, upload them and use the '
'-g flag instead.')
def Submit(self, testjob=False, dryrun=False):
"""Submit the tryjob through Git.
testjob: Submit job to the test branch of the tryjob repo. The tryjob
will be ignored by production master.
dryrun: Setting to true will run everything except the final submit step.
# TODO(rcui): convert to shallow clone when that's available.
current_time = str(int(time.time()))
ref_base = os.path.join('refs/tryjobs', self.user_email, current_time)
for patch in self.local_patches:
# Isolate the name; if it's a tag or a remote, let through.
# Else if it's a branch, get the full branch name minus refs/heads.
local_branch = git.StripRefsHeads(patch.ref, False)
ref_final = os.path.join(ref_base, local_branch, patch.sha1)
checkout = patch.GetCheckout(self.manifest)
print('Uploading patch %s' % patch)
patch.Upload(checkout['push_url'], ref_final, dryrun=dryrun)
# TODO(rcui): Pass in the remote instead of tag.
tag = constants.EXTERNAL_PATCH_TAG
if checkout['remote'] == site_config.params.INTERNAL_REMOTE:
tag = constants.INTERNAL_PATCH_TAG
% (patch.project, local_branch, ref_final,
patch.tracking_branch, tag))
self._PostConfigsToBuildBucket(testjob, dryrun)
def _GetBuilder(self, bot):
"""Find and return the builder for bot."""
if self.swarming:
return 'Generic'
if bot in site_config and site_config[bot]['_template']:
return site_config[bot]['_template']
# Default to etc builder.
return 'etc'
def _GetRequestBody(self, bot):
"""Generate the request body for a swarming buildbucket request.
bot: The bot config to put.
buildbucket request properties as a python dict.
if self.swarming:
return {
'bucket': bucket,
'parameters_json': json.dumps({
'builder_name': 'Generic',
'properties': {
'bot' : self.build_configs,
'email' : [self.user_email],
'extra_args' : self.extra_args,
'name' :,
'user' : self.user,
'cbb_config': bot,
'cbb_extra_args': self.extra_args,
'owners': [self.user_email],
# These tags are indexed and searchable in buildbucket.
'tags': [
'cbb_build_group:%s' % self.build_group,
'cbb_branch:%s' % self.branch,
'cbb_config:%s' % bot,
'cbb_master_build_id:%s' % self.master_buildbucket_id,
'cbb_email:%s' % self.user_email,
def _PutConfigToBuildBucket(self, buildbucket_client, bot, dryrun):
"""Put the tryjob request to buildbucket.
buildbucket_client: The buildbucket client instance.
bot: The bot config to put.
dryrun: Whether a dryrun.
request_body = self._GetRequestBody(bot)
content = buildbucket_client.PutBuildRequest(
json.dumps(request_body), dryrun)
if buildbucket_lib.GetNestedAttr(content, ['error']):
raise RemoteRequestFailure(
'buildbucket error.\nReason: %s\n Message: %s' %
buildbucket_id = buildbucket_lib.GetBuildId(content)
(constants.TRYSERVER_BUILDBUCKET_BUCKET, bot, buildbucket_id))
def _PostConfigsToBuildBucket(self, testjob=False, dryrun=False):
"""Posts the tryjob configs to buildbucket.
dryrun: Whether to skip the request to buildbucket.
testjob: Whether to use the test instance of the buildbucket server.
host = (buildbucket_lib.BUILDBUCKET_TEST_HOST if testjob
else buildbucket_lib.BUILDBUCKET_HOST)
buildbucket_client = buildbucket_lib.BuildbucketClient(
auth.GetAccessToken, host,
for bot in self.build_configs:
self._PutConfigToBuildBucket(buildbucket_client, bot, dryrun)
def GetTrybotWaterfallLinks(self):
"""Get link to the waterfall for the user.
List of URLs to view submitted tryjobs.
# TODO: Use GE in all cases, after support is in production.
if self.swarming:
return [BUILD_DETAILS_PATTERN % {'buildbucket_id': b}
for b in self.buildbucket_ids]
# The builders on the trybot waterfall are named after the templates.
builders = set(self._GetBuilder(bot) for bot in self.build_configs)
# Note that this will only show the jobs submitted by the user in the last
# 24 hours.
return ['%s/waterfall?committer=%s&%s' % (
constants.TRYBOT_DASHBOARD, self.user_email,
'&'.join('builder=%s' % b for b in sorted(builders)))]