blob: b31c2fc4e258a387f997157044521a06fd1d393c [file] [log] [blame]
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Module containing the report stages."""
from __future__ import print_function
import logging
import os
import sys
from chromite.cbuildbot import commands
from chromite.cbuildbot import constants
from chromite.cbuildbot import failures_lib
from chromite.cbuildbot import metadata_lib
from chromite.cbuildbot import results_lib
from chromite.cbuildbot import tree_status
from chromite.cbuildbot.stages import completion_stages
from chromite.cbuildbot.stages import generic_stages
from chromite.cbuildbot.stages import sync_stages
from chromite.lib import alerts
from chromite.lib import cidb
from chromite.lib import cros_build_lib
from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import portage_util
from chromite.lib import retry_stats
from chromite.lib import toolchain
def WriteBasicMetadata(builder_run):
"""Writes basic metadata that should be known at start of execution.
This method writes to |build_run|'s metadata instance the basic metadata
values that should be known at the beginning of the first cbuildbot
execution, prior to any reexecutions.
In particular, this method does not write any metadata values that depend
on the builder config, as the config may be modified by patches that are
applied before the final reexectuion.
This method is safe to run more than once (for instance, once per cbuildbot
execution) because it will write the same data each time.
builder_run: The BuilderRun instance for this build.
start_time = results_lib.Results.start_time
start_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=start_time)
metadata = {
# Data for this build.
'bot-hostname': cros_build_lib.GetHostName(fully_qualified=True),
'build-number': builder_run.buildnumber,
'builder-name': os.environ.get('BUILDBOT_BUILDERNAME', ''),
'buildbot-url': os.environ.get('BUILDBOT_BUILDBOTURL', ''),
os.environ.get('BUILDBOT_MASTERNAME', ''),
'bot-config': builder_run.config['name'],
'time': {
'start': start_time_stamp,
'master_build_id': builder_run.options.master_build_id,
def GetChildConfigListMetadata(child_configs, config_status_map):
"""Creates a list for the child configs metadata.
This creates a list of child config dictionaries from the given child
configs, optionally adding the final status if the success map is
child_configs: The list of child configs for this build.
config_status_map: The map of config name to final build status.
List of child config dictionaries, with optional final status
child_config_list = []
for c in child_configs:
pass_fail_status = None
if config_status_map:
if config_status_map[c['name']]:
pass_fail_status = constants.FINAL_STATUS_PASSED
pass_fail_status = constants.FINAL_STATUS_FAILED
child_config_list.append({'name': c['name'],
'boards': c['boards'],
'status': pass_fail_status})
return child_config_list
class BuildStartStage(generic_stages.BuilderStage):
"""The first stage to run.
This stage writes a few basic metadata values that are known at the start of
build, and inserts the build into the database, if appropriate.
def PerformStage(self):
d = self._run.attrs.metadata.GetDict()
# BuildStartStage should only run once per build. But just in case it
# is somehow running a second time, we do not want to insert an additional
# database entry. Detect if a database entry has been inserted already
# and if so quit the stage.
if 'build_id' in d:'Already have build_id %s, not inserting an entry.',
# Note: In other build stages we use self._run.GetCIDBHandle to fetch
# a cidb handle. However, since we don't yet have a build_id, we can't
# do that here.
if cidb.CIDBConnectionFactory.IsCIDBSetup():
db_type = cidb.CIDBConnectionFactory.GetCIDBConnectionType()
db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
if db:
waterfall = d['buildbot-master-name']
assert waterfall in constants.CIDB_KNOWN_WATERFALLS
build_id = db.InsertBuild(
self._run.attrs.metadata.UpdateWithDict({'build_id': build_id,
'db_type': db_type})'Inserted build_id %s into cidb database type %s.',
build_id, db_type)
def HandleSkip(self):
"""Ensure that re-executions use the same db instance as initial db."""
metadata_dict = self._run.attrs.metadata.GetDict()
if 'build_id' in metadata_dict:
db_type = cidb.CIDBConnectionFactory.GetCIDBConnectionType()
if not 'db_type' in metadata_dict:
# This will only execute while this CL is in the commit queue. After
# this CL lands, this block can be removed.
self._run.attrs.metadata.UpdateWithDict({'db_type': db_type})
if db_type != metadata_dict['db_type']:
raise AssertionError('Invalid attempt to switch from database %s to '
'%s.' % (metadata_dict['db_type'], db_type))
class BuildReexecutionFinishedStage(generic_stages.BuilderStage,
"""The first stage to run after the final cbuildbot reexecution.
This stage is the first stage run after the final cbuildbot
bootstrap/reexecution. By the time this stage is run, the sync stages
are complete and version numbers of chromeos are known (though chrome
version may not be known until SyncChrome).
This stage writes metadata values that are first known after the final
reexecution (such as those that come from the config). This stage also
updates the build's cidb entry if appropriate.
Where possible, metadata that is already known at this time should be
written at this time rather than in ReportStage.
def PerformStage(self):
config = self._run.config
build_root = self._build_root
# Flat list of all child config boards. Since child configs
# are not allowed to have children, it is not necessary to search
# deeper than one generation.
child_configs = GetChildConfigListMetadata(
child_configs=config['child_configs'], config_status_map=None)
sdk_verinfo = cros_build_lib.LoadKeyValueFile(
os.path.join(build_root, constants.SDK_VERSION_FILE),
verinfo = self._run.GetVersionInfo(build_root)
platform_tag = getattr(self._run.attrs, 'release_tag')
if not platform_tag:
platform_tag = verinfo.VersionString()
version = {
'full': self._run.GetVersion(),
'milestone': verinfo.chrome_branch,
'platform': platform_tag,
metadata = {
# Version of the metadata format.
'metadata-version': '2',
'boards': config['boards'],
'child-configs': child_configs,
'build_type': config['build_type'],
# Data for the toolchain used.
'sdk-version': sdk_verinfo.get('SDK_LATEST_VERSION', '<unknown>'),
'toolchain-url': sdk_verinfo.get('TC_PATH', '<unknown>'),
if len(config['boards']) == 1:
toolchains = toolchain.GetToolchainsForBoard(config['boards'][0],
metadata['toolchain-tuple'] = (
toolchain.FilterToolchains(toolchains, 'default', True).keys() +
toolchain.FilterToolchains(toolchains, 'default', False).keys())'Metadata being written: %s', metadata)
# Update 'version' separately to avoid overwriting the existing
# entries in it (e.g. PFQ builders may have written the Chrome
# version to uprev)."Metadata 'version' being written: %s", version)
self._run.attrs.metadata.UpdateKeyDictWithDict('version', version)
# Ensure that all boards and child config boards have a per-board
# metadata subdict.
for b in config['boards']:
self._run.attrs.metadata.UpdateBoardDictWithDict(b, {})
for cc in child_configs:
for b in cc['boards']:
self._run.attrs.metadata.UpdateBoardDictWithDict(b, {})
# Upload build metadata (and write it to database if necessary)
# Write child-per-build and board-per-build rows to database
build_id, db = self._run.GetCIDBHandle()
if db:
# TODO(akeshet): replace this with a GetValue call once
# is resolved
per_board_dict = self._run.attrs.metadata.GetDict()['board-metadata']
for board, board_metadata in per_board_dict.items():
db.InsertBoardPerBuild(build_id, board)
if board_metadata:
db.UpdateBoardPerBuildMetadata(build_id, board, board_metadata)
for child_config in self._run.attrs.metadata.GetValue('child-configs'):
db.InsertChildConfigPerBuild(build_id, child_config['name'])
class ReportStage(generic_stages.BuilderStage,
"""Summarize all the builds."""
_HTML_HEAD = """<html>
<title>Archive Index: %(board)s / %(version)s</title>
<h2>Artifacts Index: %(board)s / %(version)s (%(config)s config)</h2>"""
def __init__(self, builder_run, sync_instance, completion_instance, **kwargs):
super(ReportStage, self).__init__(builder_run, **kwargs)
# TODO(mtennant): All these should be retrieved from builder_run instead.
# Or, more correctly, the info currently retrieved from these stages should
# be stored and retrieved from builder_run instead.
self._sync_instance = sync_instance
self._completion_instance = completion_instance
def _UpdateRunStreak(self, builder_run, final_status):
"""Update the streak counter for this builder, if applicable, and notify.
Update the pass/fail streak counter for the builder. If the new
streak should trigger a notification email then send it now.
builder_run: BuilderRun for this run.
final_status: Final status string for this run.
# Exclude tryjobs from streak counting.
if not builder_run.options.remote_trybot and not builder_run.options.local:
streak_value = self._UpdateStreakCounter(
verb = 'passed' if streak_value > 0 else 'failed'
cros_build_lib.Info('Builder %s has %s %s time(s) in a row.',, verb, abs(streak_value))
# See if updated streak should trigger a notification email.
if (builder_run.config.health_alert_recipients and
builder_run.config.health_threshold > 0 and
streak_value <= -builder_run.config.health_threshold):
'Builder failed %i consecutive times, sending health alert email '
'to %s.',
if not self._run.debug:
alerts.SendEmail('%s health alert' %,
extra_fields={'X-cbuildbot-alert': 'cq-health'})
def _UpdateStreakCounter(self, final_status, counter_name,
"""Update the given streak counter based on the final status of build.
A streak counter counts the number of consecutive passes or failures of
a particular builder. Consecutive passes are indicated by a positive value,
consecutive failures by a negative value.
final_status: String indicating final status of build,
constants.FINAL_STATUS_PASSED indicating success.
counter_name: Name of counter to increment, typically the name of the
build config.
dry_run: Pretend to update counter only. Default: False.
The new value of the streak counter.
gs_ctx = gs.GSContext(dry_run=dry_run)
counter_url = os.path.join(constants.MANIFEST_VERSIONS_GS_URL,
gs_counter = gs.GSCounter(gs_ctx, counter_url)
if final_status == constants.FINAL_STATUS_PASSED:
streak_value = gs_counter.StreakIncrement()
streak_value = gs_counter.StreakDecrement()
return streak_value
def _HealthAlertMessage(self, fail_count):
"""Returns the body of a health alert email message."""
return 'The builder named %s has failed %i consecutive times. See %s' % (
self._run.config['name'], fail_count, self.ConstructDashboardURL())
def _UploadMetadataForRun(self, final_status):
"""Upload metadata.json for this entire run.
final_status: Final status string for this run.
def _UploadArchiveIndex(self, builder_run):
"""Upload an HTML index for the artifacts at remote archive location.
If there are no artifacts in the archive then do nothing.
builder_run: BuilderRun object for this run.
If an index file is uploaded then a dict is returned where each value
is the same (the URL for the uploaded HTML index) and the keys are
the boards it applies to, including None if applicable. If no index
file is uploaded then this returns None.
archive = builder_run.GetArchive()
archive_path = archive.archive_path
config = builder_run.config
boards = config.boards
if boards:
board_names = ' '.join(boards)
boards = [None]
board_names = '<no board>'
# See if there are any artifacts found for this run.
uploaded = os.path.join(archive_path, commands.UPLOADED_LIST_FILENAME)
if not os.path.exists(uploaded):
# UPLOADED doesn't exist. Normal if Archive stage never ran, which
# is possibly normal. Regardless, no archive index is needed.'No archived artifacts found for %s run (%s)',, board_names)
# Prepare html head.
head_data = {
'board': board_names,
'version': builder_run.GetVersion(),
head = self._HTML_HEAD % head_data
files = osutils.ReadFile(uploaded).splitlines() + [
'.|Google Storage Index',
index = os.path.join(archive_path, 'index.html')
# TODO (sbasi) Rework the way we do uploading to
# multiple buckets. Currently this can only be done in the Archive Stage
# therefore index.html will only end up in the normal Chrome OS bucket.
commands.GenerateHtmlIndex(index, files, url_base=archive.download_url,
archive_path, [archive.upload_url], os.path.basename(index),
debug=self._run.debug, acl=self.acl)
return dict((b, archive.download_url + '/index.html') for b in boards)
def GetReportMetadata(self, config=None, stage=None, final_status=None,
sync_instance=None, completion_instance=None):
"""Generate ReportStage metadata.
config: The build config for this run. Defaults to self._run.config.
stage: The stage name that this metadata file is being uploaded for.
final_status: Whether the build passed or failed. If None, the build
will be treated as still running.
sync_instance: The stage instance that was used for syncing the source
code. This should be a derivative of SyncStage. If None, the list of
commit queue patches will not be included in the metadata.
completion_instance: The stage instance that was used to wait for slave
completion. Used to add slave build information to master builder's
metadata. If None, no such status information will be included. It not
None, this should be a derivative of MasterSlaveSyncCompletionStage.
A JSON-able dictionary representation of the metadata object.
builder_run = self._run
config = config or builder_run.config
commit_queue_stages = (sync_stages.CommitQueueSyncStage,
get_changes_from_pool = (sync_instance and
isinstance(sync_instance, commit_queue_stages) and
get_statuses_from_slaves = (
config['master'] and
completion_instance and
child_configs_list = GetChildConfigListMetadata(
return metadata_lib.CBuildbotMetadata.GetReportMetadataDict(
builder_run, get_changes_from_pool,
get_statuses_from_slaves, config, stage, final_status, sync_instance,
completion_instance, child_configs_list)
def ArchiveResults(self, final_status):
"""Archive our build results.
final_status: constants.FINAL_STATUS_PASSED or
A dictionary with the aggregated _UploadArchiveIndex results.
# Make sure local archive directory is prepared, if it was not already.
if not os.path.exists(self.archive_path):
# Upload metadata, and update the pass/fail streak counter for the main
# run only. These aren't needed for the child builder runs.
self._UpdateRunStreak(self._run, final_status)
# Iterate through each builder run, whether there is just the main one
# or multiple child builder runs.
archive_urls = {}
for builder_run in self._run.GetUngroupedBuilderRuns():
# Generate an index for archived artifacts if there are any. All the
# archived artifacts for one run/config are in one location, so the index
# is only specific to each run/config. In theory multiple boards could
# share that archive, but in practice it is usually one board. A
# run/config without a board will also usually not have artifacts to
# archive, but that restriction is not assumed here.
run_archive_urls = self._UploadArchiveIndex(builder_run)
if run_archive_urls:
# Check if the builder_run is tied to any boards and if so get all
# upload urls.
if final_status == constants.FINAL_STATUS_PASSED:
# Update the LATEST files if the build passed.
upload_urls = self._GetUploadUrls(
'LATEST-*', builder_run=builder_run)
except portage_util.MissingOverlayException as e:
# If the build failed prematurely, some overlays might be
# missing. Ignore them in this stage.
archive = builder_run.GetArchive()
return archive_urls
def PerformStage(self):
"""Perform the actual work for this stage.
This includes final metadata archival, and update CIDB with our final status
as well as producting a logged build result summary.
if results_lib.Results.BuildSucceededSoFar():
final_status = constants.FINAL_STATUS_PASSED
final_status = constants.FINAL_STATUS_FAILED
if not hasattr(self._run.attrs, 'release_tag'):
# If, for some reason, sync stage was not completed and
# release_tag was not set. Set it to None here because
# ArchiveResults() depends the existence of this attr.
self._run.attrs.release_tag = None
archive_urls = self.ArchiveResults(final_status)
metadata_url = os.path.join(self.upload_url, constants.METADATA_JSON)
sys.stdout, archive_urls=archive_urls,
current_version=(self._run.attrs.release_tag or ''))
build_id, db = self._run.GetCIDBHandle()
if db:
# TODO(akeshet): Eliminate this status string translate once
# these differing status strings are merged,
translateStatus = lambda s: (constants.BUILDER_STATUS_PASSED
if s == constants.FINAL_STATUS_PASSED
status_for_db = translateStatus(final_status)
child_metadatas = self._run.attrs.metadata.GetDict().get(
'child-configs', [])
for child_metadata in child_metadatas:
db.FinishChildConfig(build_id, child_metadata['name'],
# TODO(pprabhu): After BuildData and CBuildbotMetdata are merged, remove
# this extra temporary object creation.
# XXX:HACK We're creating a BuildData with an empty URL. Don't try to
# MarkGathered this object.
build_data = metadata_lib.BuildData("",
# TODO(akeshet): Find a clearer way to get the "primary upload url" for
# the metadata.json file. One alternative is _GetUploadUrls(...)[0].
# Today it seems that element 0 of its return list is the primary upload
# url, but there is no guarantee or unit test coverage of that.
db.FinishBuild(build_id, status=status_for_db,
class RefreshPackageStatusStage(generic_stages.BuilderStage):
"""Stage for refreshing Portage package status in online spreadsheet."""
def PerformStage(self):