lib/request_build.py - third_party/chromite - Git at Google

 # Copyright 2012 The ChromiumOS Authors
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """Code related to Remote tryjobs."""

 import collections
 import uuid

 from chromite.third_party.google.protobuf import duration_pb2
 from chromite.third_party.google.protobuf.struct_pb2 import Struct
 from chromite.third_party.infra_libs.buildbucket.proto import (
     build_pb2,
     builder_common_pb2,
     common_pb2,
 )

 from chromite.lib import buildbucket_v2
 from chromite.lib import config_lib


 # Buildbucket buckets.
 INTERNAL_SWARMING_BUILDBUCKET_BUCKET = "general"


 class RemoteRequestFailure(Exception):
     """Thrown when requesting a tryjob fails."""


 # Contains the results of a single scheduled build.
 ScheduledBuild = collections.namedtuple(
     "ScheduledBuild",
     ("bucket", "buildbucket_id", "build_config", "url", "created_ts"),
 )


 def ChildBuildSet(parent_buildbucket_id):
     """Compute the buildset id for all slaves of a master builder.

     Args:
         parent_buildbucket_id: The buildbucket id of the master build.

     Returns:
         A string to use as a buildset for the slave builders, or None.
     """
     if not parent_buildbucket_id:
         return None

     return "cros/parent_buildbucket_id/%s" % parent_buildbucket_id


 class RequestBuild:
     """Request a builder via buildbucket."""

     # Buildbucket_put response must contain 'buildbucket_bucket:bucket]',
     # '[config:config_name] and '[buildbucket_id:id]'.
     BUILDBUCKET_PUT_RESP_FORMAT = (
         "Successfully sent PUT request to "
         "[buildbucket_bucket:%(bucket)s] "
         "with [config:%(build_config)s] [buildbucket_id:%(buildbucket_id)s]."
     )

     def __init__(
         self,
         build_config,
         luci_builder=None,
         display_label=None,
         branch="master",
         extra_args=None,
         extra_properties=None,
         user_email=None,
         email_template=None,
         master_cidb_id=None,
         master_buildbucket_id=None,
         bucket=INTERNAL_SWARMING_BUILDBUCKET_BUCKET,
         requested_bot=None,
     ) -> None:
         """Construct the object.

         Args:
             build_config: A build config name to schedule.
             luci_builder: Name of builder to execute the build, or None.
                 For waterfall builds, this is the name of the build column.
                 For swarming builds, this is the LUCI builder name.
             display_label: String describing how build group on waterfall, or
                 None.
             branch: Name of branch to build for.
             extra_args: Command line arguments to pass to cbuildbot in job.
             extra_properties: Additional input properties to add to the request.
             user_email: Email address of person requesting job, or None.
             email_template: Name of the luci-notify template to use. None for
                 default. Ignored if user_email is not set.
             master_cidb_id: CIDB id of scheduling builder, or None.
             master_buildbucket_id: buildbucket id of scheduling builder, or
                 None.
             bucket: Which bucket do we request the build in?
             requested_bot: Name of bot to prefer (for performance), or None.
         """
         self.bucket = bucket
         self.extra_properties = extra_properties or {}

         site_config = config_lib.GetConfig()
         if build_config in site_config:
             # Extract from build_config, if possible.
             self.luci_builder = site_config[build_config].luci_builder
             self.display_label = site_config[build_config].display_label
             self.workspace_branch = site_config[build_config].workspace_branch
         else:
             # Use generic defaults if needed (lowest priority).
             self.luci_builder = config_lib.LUCI_BUILDER_TRY
             self.display_label = config_lib.DISPLAY_LABEL_TRYJOB
             self.workspace_branch = None

         # But allow an explicit overrides.
         if luci_builder:
             self.luci_builder = luci_builder

         if display_label:
             self.display_label = display_label

         self.build_config = build_config
         self.branch = branch
         self.extra_args = extra_args
         self.user_email = user_email
         self.email_template = email_template or "default"
         self.master_cidb_id = master_cidb_id
         self.master_buildbucket_id = master_buildbucket_id
         self.requested_bot = requested_bot

     def CreateBuildRequest(self):
         """Generate the details for Buildbucket V2 request.

         Returns:
             Parameters for V2 ScheduleBuild.
         """
         tags = {
             # buildset identifies a group of related builders.
             "buildset": ChildBuildSet(self.master_buildbucket_id),
             "cbb_display_label": self.display_label,
             "cbb_branch": self.branch,
             "cbb_config": self.build_config,
             "cbb_email": self.user_email,
             "cbb_master_build_id": self.master_cidb_id,
             "cbb_master_buildbucket_id": self.master_buildbucket_id,
             "cbb_workspace_branch": self.workspace_branch,
             "cbb_goma_client_type": None,
         }

         if self.master_cidb_id or self.master_buildbucket_id:
             # Used by dashboards as part of grouping slave builds. Set to False
             # for slave builds, not set otherwise.
             tags["master"] = "False"

         # Include the extra_properties we might have passed into the tags.
         tags.update(self.extra_properties)

         # Don't include tags with no value, there is no point.
         # Convert tag values to strings.
         #
         # Note that cbb_master_build_id must be a string (not a number) in
         # properties because JSON does not distinguish integers and floats, so
         # nothing guarantees that 0 won't turn into 0.0.
         # Recipe expects it to be a string anyway.
         tags = {k: str(v) for k, v in tags.items() if v}

         properties = Struct()
         properties.update({k: str(v) for k, v in tags.items() if v})
         properties.update({"cbb_extra_args": self.extra_args})
         if self.user_email:
             properties.update(
                 {
                     "email_notify": [
                         {
                             "email": self.user_email,
                             "template": self.email_template,
                         }
                     ]
                 }
             )

         tags_proto = []
         for k, v in sorted(tags.items()):
             if v:
                 tags_proto.append(common_pb2.StringPair(key=k, value=v))
         dimensions = []

         # If a specific bot was requested, pass along the request with a
         # 240 second (4 minute) timeout. If the bot isn't available, we
         # will fall back to the general builder restrictions (probably
         # based on role).
         if self.requested_bot:
             dimensions = [
                 common_pb2.RequestedDimension(
                     key="id",
                     value=self.requested_bot,
                     expiration=duration_pb2.Duration(seconds=240),
                 )
             ]
         return {
             "request_id": uuid.uuid1(),
             "builder": builder_common_pb2.BuilderID(
                 project="chromeos",
                 bucket=self.bucket,
                 builder=self.luci_builder,
             ),
             "properties": properties,
             "tags": tags_proto,
             "dimensions": dimensions if dimensions else None,
         }

     def Submit(self, dryrun=False):
         """Submit the tryjob through Git.

         Args:
             dryrun: Setting to true will run everything except the final submit
                 step.

         Returns:
             A ScheduledBuild instance.
         """
         buildbucket_client = buildbucket_v2.BuildbucketV2()
         request = self.CreateBuildRequest()
         if dryrun:
             return build_pb2.Build(id="1")
         return buildbucket_client.ScheduleBuild(
             request_id=str(request["request_id"]),
             builder=request["builder"],
             properties=request["properties"],
             tags=request["tags"],
             dimensions=request["dimensions"],
         )
	# Copyright 2012 The ChromiumOS Authors
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""Code related to Remote tryjobs."""

	import collections
	import uuid

	from chromite.third_party.google.protobuf import duration_pb2
	from chromite.third_party.google.protobuf.struct_pb2 import Struct
	from chromite.third_party.infra_libs.buildbucket.proto import (
	build_pb2,
	builder_common_pb2,
	common_pb2,
	)

	from chromite.lib import buildbucket_v2
	from chromite.lib import config_lib


	# Buildbucket buckets.
	INTERNAL_SWARMING_BUILDBUCKET_BUCKET = "general"


	class RemoteRequestFailure(Exception):
	"""Thrown when requesting a tryjob fails."""


	# Contains the results of a single scheduled build.
	ScheduledBuild = collections.namedtuple(
	"ScheduledBuild",
	("bucket", "buildbucket_id", "build_config", "url", "created_ts"),
	)


	def ChildBuildSet(parent_buildbucket_id):
	"""Compute the buildset id for all slaves of a master builder.

	Args:
	parent_buildbucket_id: The buildbucket id of the master build.

	Returns:
	A string to use as a buildset for the slave builders, or None.
	"""
	if not parent_buildbucket_id:
	return None

	return "cros/parent_buildbucket_id/%s" % parent_buildbucket_id


	class RequestBuild:
	"""Request a builder via buildbucket."""

	# Buildbucket_put response must contain 'buildbucket_bucket:bucket]',
	# '[config:config_name] and '[buildbucket_id:id]'.
	BUILDBUCKET_PUT_RESP_FORMAT = (
	"Successfully sent PUT request to "
	"[buildbucket_bucket:%(bucket)s] "
	"with [config:%(build_config)s] [buildbucket_id:%(buildbucket_id)s]."
	)

	def __init__(
	self,
	build_config,
	luci_builder=None,
	display_label=None,
	branch="master",
	extra_args=None,
	extra_properties=None,
	user_email=None,
	email_template=None,
	master_cidb_id=None,
	master_buildbucket_id=None,
	bucket=INTERNAL_SWARMING_BUILDBUCKET_BUCKET,
	requested_bot=None,
	) -> None:
	"""Construct the object.

	Args:
	build_config: A build config name to schedule.
	luci_builder: Name of builder to execute the build, or None.
	For waterfall builds, this is the name of the build column.
	For swarming builds, this is the LUCI builder name.
	display_label: String describing how build group on waterfall, or
	None.
	branch: Name of branch to build for.
	extra_args: Command line arguments to pass to cbuildbot in job.
	extra_properties: Additional input properties to add to the request.
	user_email: Email address of person requesting job, or None.
	email_template: Name of the luci-notify template to use. None for
	default. Ignored if user_email is not set.
	master_cidb_id: CIDB id of scheduling builder, or None.
	master_buildbucket_id: buildbucket id of scheduling builder, or
	None.
	bucket: Which bucket do we request the build in?
	requested_bot: Name of bot to prefer (for performance), or None.
	"""
	self.bucket = bucket
	self.extra_properties = extra_properties or {}

	site_config = config_lib.GetConfig()
	if build_config in site_config:
	# Extract from build_config, if possible.
	self.luci_builder = site_config[build_config].luci_builder
	self.display_label = site_config[build_config].display_label
	self.workspace_branch = site_config[build_config].workspace_branch
	else:
	# Use generic defaults if needed (lowest priority).
	self.luci_builder = config_lib.LUCI_BUILDER_TRY
	self.display_label = config_lib.DISPLAY_LABEL_TRYJOB
	self.workspace_branch = None

	# But allow an explicit overrides.
	if luci_builder:
	self.luci_builder = luci_builder

	if display_label:
	self.display_label = display_label

	self.build_config = build_config
	self.branch = branch
	self.extra_args = extra_args
	self.user_email = user_email
	self.email_template = email_template or "default"
	self.master_cidb_id = master_cidb_id
	self.master_buildbucket_id = master_buildbucket_id
	self.requested_bot = requested_bot

	def CreateBuildRequest(self):
	"""Generate the details for Buildbucket V2 request.

	Returns:
	Parameters for V2 ScheduleBuild.
	"""
	tags = {
	# buildset identifies a group of related builders.
	"buildset": ChildBuildSet(self.master_buildbucket_id),
	"cbb_display_label": self.display_label,
	"cbb_branch": self.branch,
	"cbb_config": self.build_config,
	"cbb_email": self.user_email,
	"cbb_master_build_id": self.master_cidb_id,
	"cbb_master_buildbucket_id": self.master_buildbucket_id,
	"cbb_workspace_branch": self.workspace_branch,
	"cbb_goma_client_type": None,
	}

	if self.master_cidb_id or self.master_buildbucket_id:
	# Used by dashboards as part of grouping slave builds. Set to False
	# for slave builds, not set otherwise.
	tags["master"] = "False"

	# Include the extra_properties we might have passed into the tags.
	tags.update(self.extra_properties)

	# Don't include tags with no value, there is no point.
	# Convert tag values to strings.
	#
	# Note that cbb_master_build_id must be a string (not a number) in
	# properties because JSON does not distinguish integers and floats, so
	# nothing guarantees that 0 won't turn into 0.0.
	# Recipe expects it to be a string anyway.
	tags = {k: str(v) for k, v in tags.items() if v}

	properties = Struct()
	properties.update({k: str(v) for k, v in tags.items() if v})
	properties.update({"cbb_extra_args": self.extra_args})
	if self.user_email:
	properties.update(
	{
	"email_notify": [
	{
	"email": self.user_email,
	"template": self.email_template,
	}
	]
	}
	)

	tags_proto = []
	for k, v in sorted(tags.items()):
	if v:
	tags_proto.append(common_pb2.StringPair(key=k, value=v))
	dimensions = []

	# If a specific bot was requested, pass along the request with a
	# 240 second (4 minute) timeout. If the bot isn't available, we
	# will fall back to the general builder restrictions (probably
	# based on role).
	if self.requested_bot:
	dimensions = [
	common_pb2.RequestedDimension(
	key="id",
	value=self.requested_bot,
	expiration=duration_pb2.Duration(seconds=240),
	)
	]
	return {
	"request_id": uuid.uuid1(),
	"builder": builder_common_pb2.BuilderID(
	project="chromeos",
	bucket=self.bucket,
	builder=self.luci_builder,
	),
	"properties": properties,
	"tags": tags_proto,
	"dimensions": dimensions if dimensions else None,
	}

	def Submit(self, dryrun=False):
	"""Submit the tryjob through Git.

	Args:
	dryrun: Setting to true will run everything except the final submit
	step.

	Returns:
	A ScheduledBuild instance.
	"""
	buildbucket_client = buildbucket_v2.BuildbucketV2()
	request = self.CreateBuildRequest()
	if dryrun:
	return build_pb2.Build(id="1")
	return buildbucket_client.ScheduleBuild(
	request_id=str(request["request_id"]),
	builder=request["builder"],
	properties=request["properties"],
	tags=request["tags"],
	dimensions=request["dimensions"],
	)