blob: 9e63fc8127d819981793a27a57013e65df72f8cf [file] [log] [blame]
# -*- coding: utf-8 -*-
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Code related to Remote tryjobs."""
from __future__ import print_function
import collections
import json
import sys
from chromite.lib import auth
from chromite.lib import buildbucket_lib
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import cros_logging as logging
from chromite.lib import pformat
from chromite.lib import uri_lib
assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
class RemoteRequestFailure(Exception):
"""Thrown when requesting a tryjob fails."""
# Contains the results of a single scheduled build.
ScheduledBuild = collections.namedtuple(
('bucket', 'buildbucket_id', 'build_config', 'url', 'created_ts'))
def ChildBuildSet(parent_buildbucket_id):
"""Compute the buildset id for all slaves of a master builder.
parent_buildbucket_id: The buildbucket id of the master build.
A string to use as a buildset for the slave builders, or None.
if not parent_buildbucket_id:
return None
return 'cros/parent_buildbucket_id/%s' % parent_buildbucket_id
class RequestBuild(object):
"""Request a builder via buildbucket."""
# Buildbucket_put response must contain 'buildbucket_bucket:bucket]',
# '[config:config_name] and '[buildbucket_id:id]'.
'Successfully sent PUT request to '
'[buildbucket_bucket:%(bucket)s] '
'with [config:%(build_config)s] [buildbucket_id:%(buildbucket_id)s].')
def __init__(self,
"""Construct the object.
build_config: A build config name to schedule.
luci_builder: Name of builder to execute the build, or None.
For waterfall builds, this is the name of the build column.
For swarming builds, this is the LUCI builder name.
display_label: String describing how build group on waterfall, or None.
branch: Name of branch to build for.
extra_args: Command line arguments to pass to cbuildbot in job.
extra_properties: Additional input properties to add to the request.
user_email: Email address of person requesting job, or None.
email_template: Name of the luci-notify template to use. None for
default. Ignored if user_email is not set.
master_cidb_id: CIDB id of scheduling builder, or None.
master_buildbucket_id: buildbucket id of scheduling builder, or None.
bucket: Which bucket do we request the build in?
requested_bot: Name of bot to prefer (for performance), or None.
self.bucket = bucket
self.extra_properties = extra_properties or {}
site_config = config_lib.GetConfig()
if build_config in site_config:
# Extract from build_config, if possible.
self.luci_builder = site_config[build_config].luci_builder
self.display_label = site_config[build_config].display_label
self.workspace_branch = site_config[build_config].workspace_branch
self.goma_client_type = site_config[build_config].goma_client_type
# Use generic defaults if needed (lowest priority)
self.luci_builder = config_lib.LUCI_BUILDER_TRY
self.display_label = config_lib.DISPLAY_LABEL_TRYJOB
self.workspace_branch = None
self.goma_client_type = None
# But allow an explicit overrides.
if luci_builder:
self.luci_builder = luci_builder
if display_label:
self.display_label = display_label
self.build_config = build_config
self.branch = branch
self.extra_args = extra_args
self.user_email = user_email
self.email_template = email_template or 'default'
self.master_cidb_id = master_cidb_id
self.master_buildbucket_id = master_buildbucket_id
self.requested_bot = requested_bot
def _GetRequestBody(self):
"""Generate the request body for a swarming buildbucket request.
buildbucket request properties as a python dict.
tags = {
# buildset identifies a group of related builders.
'buildset': ChildBuildSet(self.master_buildbucket_id),
'cbb_display_label': self.display_label,
'cbb_branch': self.branch,
'cbb_config': self.build_config,
'cbb_email': self.user_email,
'cbb_master_build_id': self.master_cidb_id,
'cbb_master_buildbucket_id': self.master_buildbucket_id,
'cbb_workspace_branch': self.workspace_branch,
'cbb_goma_client_type': self.goma_client_type,
if self.master_cidb_id or self.master_buildbucket_id:
# Used by Legoland as part of grouping slave builds. Set to False for
# slave builds, not set otherwise.
tags['master'] = 'False'
# Include the extra_properties we might have passed into the tags.
# Don't include tags with no value, there is no point.
# Convert tag values to strings.
# Note that cbb_master_build_id must be a string (not a number) in
# properties because JSON does not distnguish integers and floats, so
# nothing guarantees that 0 won't turn into 0.0.
# Recipe expects it to be a string anyway.
tags = {k: str(v) for k, v in tags.items() if v}
# All tags should also be listed as properties.
properties = tags.copy()
properties['cbb_extra_args'] = self.extra_args
parameters = {
'builder_name': self.luci_builder,
'properties': properties,
if self.user_email:
parameters['email_notify'] = [{
'email': self.user_email,
'template': self.email_template,
# If a specific bot was requested, pass along the request with a
# 240 second (4 minute) timeout. If the bot isn't available, we
# will fall back to the general builder restrictions (probably
# based on role).
if self.requested_bot:
parameters['swarming'] = {
'override_builder_cfg': {
'dimensions': [
'240:id:%s' % self.requested_bot,
return {
'bucket': self.bucket,
'parameters_json': pformat.json(parameters, compact=True),
# These tags are indexed and searchable in buildbucket.
'tags': ['%s:%s' % (k, tags[k]) for k in sorted(tags.keys())],
def _PutConfigToBuildBucket(self, buildbucket_client, dryrun):
"""Put the tryjob request to buildbucket.
buildbucket_client: The buildbucket client instance.
dryrun: bool controlling dryrun behavior.
ScheduledBuild describing the scheduled build.
request_body = self._GetRequestBody()
content = buildbucket_client.PutBuildRequest(
json.dumps(request_body), dryrun)
if buildbucket_lib.GetNestedAttr(content, ['error']):
raise RemoteRequestFailure(
'buildbucket error.\nReason: %s\n Message: %s' %
buildbucket_id = buildbucket_lib.GetBuildId(content)
url = uri_lib.ConstructMiloBuildUri(buildbucket_id)
created_ts = buildbucket_lib.GetBuildCreated_ts(content)
result = ScheduledBuild(
self.bucket, buildbucket_id, self.build_config, url, created_ts), result._asdict())
return result
def Submit(self, testjob=False, dryrun=False):
"""Submit the tryjob through Git.
testjob: Submit job to the test branch of the tryjob repo. The tryjob
will be ignored by production master.
dryrun: Setting to true will run everything except the final submit step.
A ScheduledBuild instance.
host = (buildbucket_lib.BUILDBUCKET_TEST_HOST if testjob
else buildbucket_lib.BUILDBUCKET_HOST)
buildbucket_client = buildbucket_lib.BuildbucketClient(
auth.GetAccessToken, host,
return self._PutConfigToBuildBucket(buildbucket_client, dryrun)