blob: 8d011bf1ee203ac028187f1f23461d026dcbdeff [file] [log] [blame]
#!/usr/bin/python
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A library to generate and store the manifests for cros builders to use."""
import logging
import os
import re
import tempfile
from xml.dom import minidom
from chromite.cbuildbot import cbuildbot_config
from chromite.cbuildbot import constants
from chromite.cbuildbot import manifest_version
from chromite.lib import cros_build_lib
from chromite.lib import git
from chromite.lib import timeout_util
# Paladin constants for manifest names.
PALADIN_COMMIT_ELEMENT = 'pending_commit'
PALADIN_REMOTE_ATTR = 'remote'
PALADIN_GERRIT_NUMBER_ATTR = 'gerrit_number'
PALADIN_PROJECT_ATTR = 'project'
PALADIN_BRANCH_ATTR = 'branch'
PALADIN_PROJECT_URL_ATTR = 'project_url'
PALADIN_REF_ATTR = 'ref'
PALADIN_CHANGE_ID_ATTR = 'change_id'
PALADIN_COMMIT_ATTR = 'commit'
PALADIN_PATCH_NUMBER_ATTR = 'patch_number'
PALADIN_OWNER_EMAIL_ATTR = 'owner_email'
PALADIN_FAIL_COUNT_ATTR = 'fail_count'
PALADIN_PASS_COUNT_ATTR = 'pass_count'
PALADIN_TOTAL_FAIL_COUNT_ATTR = 'total_fail_count'
CHROME_ELEMENT = 'chrome'
CHROME_VERSION_ATTR = 'version'
MANIFEST_ELEMENT = 'manifest'
DEFAULT_ELEMENT = 'default'
PROJECT_ELEMENT = 'project'
PROJECT_NAME_ATTR = 'name'
PROJECT_REMOTE_ATTR = 'remote'
class PromoteCandidateException(Exception):
"""Exception thrown for failure to promote manifest candidate."""
class FilterManifestException(Exception):
"""Exception thrown when failing to filter the internal manifest."""
class _LKGMCandidateInfo(manifest_version.VersionInfo):
"""Class to encapsualte the chrome os lkgm candidate info
You can instantiate this class in two ways.
1)using a version file, specifically chromeos_version.sh,
which contains the version information.
2) just passing in the 4 version components (major, minor, sp, patch and
revision number),
Args:
You can instantiate this class in two ways.
1)using a version file, specifically chromeos_version.sh,
which contains the version information.
2) passing in a string with the 3 version components + revision e.g. 41.0.0-r1
Args:
version_string: Optional 3 component version string to parse. Contains:
build_number: release build number.
branch_build_number: current build number on a branch.
patch_number: patch number.
revision_number: version revision
chrome_branch: If version_string specified, specify chrome_branch i.e. 13.
version_file: version file location.
"""
LKGM_RE = r'(\d+\.\d+\.\d+)(?:-rc(\d+))?'
def __init__(self, version_string=None, chrome_branch=None, incr_type=None,
version_file=None):
self.revision_number = 1
if version_string:
match = re.search(self.LKGM_RE, version_string)
assert match, 'LKGM did not re %s' % self.LKGM_RE
super(_LKGMCandidateInfo, self).__init__(match.group(1), chrome_branch,
incr_type=incr_type)
if match.group(2):
self.revision_number = int(match.group(2))
else:
super(_LKGMCandidateInfo, self).__init__(version_file=version_file,
incr_type=incr_type)
def VersionString(self):
"""returns the full version string of the lkgm candidate"""
return '%s.%s.%s-rc%s' % (self.build_number, self.branch_build_number,
self.patch_number, self.revision_number)
@classmethod
def VersionCompare(cls, version_string):
"""Useful method to return a comparable version of a LKGM string."""
lkgm = cls(version_string)
return map(int, [lkgm.build_number, lkgm.branch_build_number,
lkgm.patch_number, lkgm.revision_number])
def IncrementVersion(self):
"""Increments the version by incrementing the revision #."""
self.revision_number += 1
return self.VersionString()
def UpdateVersionFile(self, *args, **kwargs):
"""Update the version file on disk.
For LKGMCandidateInfo there is no version file so this function is a no-op.
"""
class LKGMManager(manifest_version.BuildSpecsManager):
"""A Class to manage lkgm candidates and their states.
Vars:
lkgm_subdir: Subdirectory within manifest repo to store candidates.
"""
# Sub-directories for LKGM and Chrome LKGM's.
LKGM_SUBDIR = 'LKGM-candidates'
CHROME_PFQ_SUBDIR = 'chrome-LKGM-candidates'
COMMIT_QUEUE_SUBDIR = 'paladin'
# Set path in repository to keep latest approved LKGM manifest.
LKGM_PATH = 'LKGM/lkgm.xml'
def __init__(self, source_repo, manifest_repo, build_names, build_type,
incr_type, force, branch, manifest=constants.DEFAULT_MANIFEST,
dry_run=True, master=False):
"""Initialize an LKGM Manager.
Args:
source_repo: Repository object for the source code.
manifest_repo: Manifest repository for manifest versions/buildspecs.
build_names: Identifiers for the build. Must match cbuildbot_config
entries. If multiple identifiers are provided, the first item in the
list must be an identifier for the group.
build_type: Type of build. Must be a pfq type.
incr_type: How we should increment this version - build|branch|patch
force: Create a new manifest even if there are no changes.
branch: Branch this builder is running on.
manifest: Manifest to use for checkout. E.g. 'full' or 'buildtools'.
dry_run: Whether we actually commit changes we make or not.
master: Whether we are the master builder.
"""
super(LKGMManager, self).__init__(
source_repo=source_repo, manifest_repo=manifest_repo,
manifest=manifest, build_names=build_names, incr_type=incr_type,
force=force, branch=branch, dry_run=dry_run, master=master)
self.lkgm_path = os.path.join(self.manifest_dir, self.LKGM_PATH)
self.compare_versions_fn = _LKGMCandidateInfo.VersionCompare
self.build_type = build_type
# Chrome PFQ and PFQ's exist at the same time and version separately so they
# must have separate subdirs in the manifest-versions repository.
if self.build_type == constants.CHROME_PFQ_TYPE:
self.rel_working_dir = self.CHROME_PFQ_SUBDIR
elif cbuildbot_config.IsCQType(self.build_type):
self.rel_working_dir = self.COMMIT_QUEUE_SUBDIR
else:
assert cbuildbot_config.IsPFQType(self.build_type)
self.rel_working_dir = self.LKGM_SUBDIR
def GetCurrentVersionInfo(self):
"""Returns the lkgm version info from the version file."""
version_info = super(LKGMManager, self).GetCurrentVersionInfo()
return _LKGMCandidateInfo(version_info.VersionString(),
chrome_branch=version_info.chrome_branch,
incr_type=self.incr_type)
def _AddChromeVersionToManifest(self, manifest, chrome_version):
"""Adds the chrome element with version |chrome_version| to |manifest|.
The manifest file should contain the Chrome version to build for
PFQ slaves.
Args:
manifest: Path to the manifest
chrome_version: A string representing the version of Chrome
(e.g. 35.0.1863.0).
"""
manifest_dom = minidom.parse(manifest)
chrome = manifest_dom.createElement(CHROME_ELEMENT)
chrome.setAttribute(CHROME_VERSION_ATTR, chrome_version)
manifest_dom.documentElement.appendChild(chrome)
with open(manifest, 'w+') as manifest_file:
manifest_dom.writexml(manifest_file)
def _AddPatchesToManifest(self, manifest, patches):
"""Adds list of |patches| to given |manifest|.
The manifest should have sufficient information for the slave
builders to fetch the patches from Gerrit and to print the CL link
(see cros_patch.GerritFetchOnlyPatch).
Args:
manifest: Path to the manifest.
patches: A list of cros_patch.GerritPatch objects.
"""
manifest_dom = minidom.parse(manifest)
for patch in patches:
pending_commit = manifest_dom.createElement(PALADIN_COMMIT_ELEMENT)
pending_commit.setAttribute(PALADIN_REMOTE_ATTR, patch.remote)
pending_commit.setAttribute(
PALADIN_GERRIT_NUMBER_ATTR, patch.gerrit_number)
pending_commit.setAttribute(PALADIN_PROJECT_ATTR, patch.project)
pending_commit.setAttribute(PALADIN_PROJECT_URL_ATTR, patch.project_url)
pending_commit.setAttribute(PALADIN_REF_ATTR, patch.ref)
pending_commit.setAttribute(PALADIN_BRANCH_ATTR, patch.tracking_branch)
pending_commit.setAttribute(PALADIN_CHANGE_ID_ATTR, patch.change_id)
pending_commit.setAttribute(PALADIN_COMMIT_ATTR, patch.commit)
pending_commit.setAttribute(PALADIN_PATCH_NUMBER_ATTR, patch.patch_number)
pending_commit.setAttribute(PALADIN_OWNER_EMAIL_ATTR, patch.owner_email)
pending_commit.setAttribute(PALADIN_FAIL_COUNT_ATTR,
str(patch.fail_count))
pending_commit.setAttribute(PALADIN_PASS_COUNT_ATTR,
str(patch.pass_count))
pending_commit.setAttribute(PALADIN_TOTAL_FAIL_COUNT_ATTR,
str(patch.total_fail_count))
manifest_dom.documentElement.appendChild(pending_commit)
with open(manifest, 'w+') as manifest_file:
manifest_dom.writexml(manifest_file)
@staticmethod
def _GetDefaultRemote(manifest_dom):
"""Returns the default remote in a manifest (if any).
Args:
manifest_dom: DOM Document object representing the manifest.
Returns:
Default remote if one exists, None otherwise.
"""
default_nodes = manifest_dom.getElementsByTagName(DEFAULT_ELEMENT)
if default_nodes:
if len(default_nodes) > 1:
raise FilterManifestException(
'More than one <default> element found in manifest')
return default_nodes[0].getAttribute(PROJECT_REMOTE_ATTR)
return None
@staticmethod
def _FilterCrosInternalProjectsFromManifest(
manifest, whitelisted_remotes=constants.EXTERNAL_REMOTES):
"""Returns a path to a new manifest with internal repositories stripped.
Args:
manifest: Path to an existing manifest that may have internal
repositories.
whitelisted_remotes: Tuple of remotes to allow in the external manifest.
Only projects with those remotes will be included in the external
manifest.
Returns:
Path to a new manifest that is a copy of the original without internal
repositories or pending commits.
"""
temp_fd, new_path = tempfile.mkstemp('external_manifest')
manifest_dom = minidom.parse(manifest)
manifest_node = manifest_dom.getElementsByTagName(MANIFEST_ELEMENT)[0]
projects = manifest_dom.getElementsByTagName(PROJECT_ELEMENT)
pending_commits = manifest_dom.getElementsByTagName(PALADIN_COMMIT_ELEMENT)
default_remote = LKGMManager._GetDefaultRemote(manifest_dom)
internal_projects = set()
for project_element in projects:
project_remote = project_element.getAttribute(PROJECT_REMOTE_ATTR)
project = project_element.getAttribute(PROJECT_NAME_ATTR)
if not project_remote:
if not default_remote:
# This should not happen for a valid manifest. Either each
# project must have a remote specified or there should
# be manifest default we could use.
raise FilterManifestException(
'Project %s has unspecified remote with no default' % project)
project_remote = default_remote
if project_remote not in whitelisted_remotes:
internal_projects.add(project)
manifest_node.removeChild(project_element)
for commit_element in pending_commits:
if commit_element.getAttribute(
PALADIN_PROJECT_ATTR) in internal_projects:
manifest_node.removeChild(commit_element)
with os.fdopen(temp_fd, 'w') as manifest_file:
# Filter out empty lines.
filtered_manifest_noempty = filter(
str.strip, manifest_dom.toxml('utf-8').splitlines())
manifest_file.write(os.linesep.join(filtered_manifest_noempty))
return new_path
def CreateNewCandidate(self, validation_pool=None,
chrome_version=None,
retries=manifest_version.NUM_RETRIES,
build_id=None):
"""Creates, syncs to, and returns the next candidate manifest.
Args:
validation_pool: Validation pool to apply to the manifest before
publishing.
chrome_version: The Chrome version to write in the manifest. Defaults
to None, in which case no version is written.
retries: Number of retries for updating the status. Defaults to
manifest_version.NUM_RETRIES.
build_id: Optional integer cidb id of the build that is creating
this candidate.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
"""
self.CheckoutSourceCode()
# Refresh manifest logic from manifest_versions repository to grab the
# LKGM to generate the blamelist.
version_info = self.GetCurrentVersionInfo()
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
self._GenerateBlameListSinceLKGM()
new_manifest = self.CreateManifest()
# For Chrome PFQ, add the version of Chrome to use.
if chrome_version:
self._AddChromeVersionToManifest(new_manifest, chrome_version)
# For the Commit Queue, apply the validation pool as part of checkout.
if validation_pool:
# If we have nothing that could apply from the validation pool and
# we're not also a pfq type, we got nothing to do.
assert self.cros_source.directory == validation_pool.build_root
if (not validation_pool.ApplyPoolIntoRepo() and
not cbuildbot_config.IsPFQType(self.build_type)):
return None
self._AddPatchesToManifest(new_manifest, validation_pool.changes)
last_error = None
for attempt in range(0, retries + 1):
try:
# Refresh manifest logic from manifest_versions repository.
# Note we don't need to do this on our first attempt as we needed to
# have done it to get the LKGM.
if attempt != 0:
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
# If we don't have any valid changes to test, make sure the checkout
# is at least different.
if ((not validation_pool or not validation_pool.changes) and
not self.force and self.HasCheckoutBeenBuilt()):
return None
# Check whether the latest spec available in manifest-versions is
# newer than our current version number. If so, use it as the base
# version number. Otherwise, we default to 'rc1'.
if self.latest:
latest = max(self.latest, version_info.VersionString(),
key=self.compare_versions_fn)
version_info = _LKGMCandidateInfo(
latest, chrome_branch=version_info.chrome_branch,
incr_type=self.incr_type)
git.CreatePushBranch(manifest_version.PUSH_BRANCH, self.manifest_dir,
sync=False)
version = self.GetNextVersion(version_info)
self.PublishManifest(new_manifest, version, build_id=build_id)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
logging.error(err_msg)
last_error = err_msg
else:
raise manifest_version.GenerateBuildSpecException(last_error)
def CreateFromManifest(self, manifest, retries=manifest_version.NUM_RETRIES,
dashboard_url=None, build_id=None):
"""Sets up an lkgm_manager from the given manifest.
This method sets up an LKGM manager and publishes a new manifest to the
manifest versions repo based on the passed in manifest but filtering
internal repositories and changes out of it.
Args:
manifest: A manifest that possibly contains private changes/projects. It
is named with the given version we want to create a new manifest from
i.e R20-1920.0.1-rc7.xml where R20-1920.0.1-rc7 is the version.
retries: Number of retries for updating the status.
dashboard_url: Optional url linking to builder dashboard for this build.
build_id: Optional integer cidb build id of the build publishing the
manifest.
Raises:
GenerateBuildSpecException in case of failure to check-in the new
manifest because of a git error or the manifest is already checked-in.
"""
last_error = None
new_manifest = self._FilterCrosInternalProjectsFromManifest(manifest)
version_info = self.GetCurrentVersionInfo()
for _attempt in range(0, retries + 1):
try:
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
git.CreatePushBranch(manifest_version.PUSH_BRANCH, self.manifest_dir,
sync=False)
version = os.path.splitext(os.path.basename(manifest))[0]
logging.info('Publishing filtered build spec')
self.PublishManifest(new_manifest, version, build_id=build_id)
self.SetInFlight(version, dashboard_url=dashboard_url)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
logging.error(err_msg)
last_error = err_msg
else:
raise manifest_version.GenerateBuildSpecException(last_error)
def GetLatestCandidate(self, dashboard_url=None, timeout=3 * 60):
"""Gets and syncs to the next candiate manifest.
Args:
retries: Number of retries for updating the status
dashboard_url: Optional url linking to builder dashboard for this build.
timeout: The timeout in seconds.
Returns:
Local path to manifest to build or None in case of no need to build.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
"""
def _AttemptToGetLatestCandidate():
"""Attempts to acquire latest candidate using manifest repo."""
self.RefreshManifestCheckout()
self.InitializeManifestVariables(self.GetCurrentVersionInfo())
if self.latest_unprocessed:
return self.latest_unprocessed
elif self.dry_run and self.latest:
return self.latest
def _PrintRemainingTime(minutes_left):
logging.info('Found nothing new to build, will keep trying for %d more'
' minutes.', minutes_left)
logging.info('If this is a PFQ, then you should have forced the master'
', which runs cbuildbot_master')
# TODO(sosa): We only really need the overlay for the version info but we
# do a full checkout here because we have no way of refining it currently.
self.CheckoutSourceCode()
try:
version_to_build = timeout_util.WaitForSuccess(
lambda x: x is None,
_AttemptToGetLatestCandidate,
timeout,
period=self.SLEEP_TIMEOUT,
side_effect_func=_PrintRemainingTime)
except timeout_util.TimeoutError:
version_to_build = None
if version_to_build:
logging.info('Starting build spec: %s', version_to_build)
self.SetInFlight(version_to_build, dashboard_url=dashboard_url)
self.current_version = version_to_build
# Actually perform the sync.
manifest = self.GetLocalManifest(version_to_build)
self.cros_source.Sync(manifest)
self._GenerateBlameListSinceLKGM()
return manifest
else:
return None
def PromoteCandidate(self, retries=manifest_version.NUM_RETRIES):
"""Promotes the current LKGM candidate to be a real versioned LKGM."""
assert self.current_version, 'No current manifest exists.'
last_error = None
path_to_candidate = self.GetLocalManifest(self.current_version)
assert os.path.exists(path_to_candidate), 'Candidate not found locally.'
# This may potentially fail for not being at TOT while pushing.
for attempt in range(0, retries + 1):
try:
if attempt > 0:
self.RefreshManifestCheckout()
git.CreatePushBranch(manifest_version.PUSH_BRANCH,
self.manifest_dir, sync=False)
manifest_version.CreateSymlink(path_to_candidate, self.lkgm_path)
git.RunGit(self.manifest_dir, ['add', self.LKGM_PATH])
self.PushSpecChanges(
'Automatic: %s promoting %s to LKGM' % (self.build_names[0],
self.current_version))
return
except cros_build_lib.RunCommandError as e:
last_error = 'Failed to promote manifest. error: %s' % e
logging.error(last_error)
logging.error('Retrying to promote manifest: Retry %d/%d', attempt + 1,
retries)
else:
raise PromoteCandidateException(last_error)
def _ShouldGenerateBlameListSinceLKGM(self):
"""Returns True if we should generate the blamelist."""
# We want to generate the blamelist only for valid pfq types and if we are
# building on the master branch i.e. revving the build number.
return (self.incr_type == 'build' and
cbuildbot_config.IsPFQType(self.build_type) and
self.build_type != constants.CHROME_PFQ_TYPE)
def _GenerateBlameListSinceLKGM(self):
"""Prints out links to all CL's that have been committed since LKGM.
Add buildbot trappings to print <a href='url'>text</a> in the waterfall for
each CL committed since we last had a passing build.
"""
if not self._ShouldGenerateBlameListSinceLKGM():
logging.info('Not generating blamelist for lkgm as it is not appropriate '
'for this build type.')
return
# Suppress re-printing changes we tried ourselves on paladin
# builders since they are redundant.
only_print_chumps = self.build_type == constants.PALADIN_TYPE
GenerateBlameList(self.cros_source, self.lkgm_path,
only_print_chumps=only_print_chumps)
def GetLatestPassingSpec(self):
"""Get the last spec file that passed in the current branch."""
raise NotImplementedError()
def GenerateBlameList(source_repo, lkgm_path, only_print_chumps=False):
"""Generate the blamelist since the specified manifest.
Args:
source_repo: Repository object for the source code.
lkgm_path: Path to LKGM manifest.
only_print_chumps: If True, only print changes that were chumped.
"""
handler = git.Manifest(lkgm_path)
reviewed_on_re = re.compile(r'\s*Reviewed-on:\s*(\S+)')
author_re = re.compile(r'\s*Author:.*<(\S+)@\S+>\s*')
committer_re = re.compile(r'\s*Commit:.*<(\S+)@\S+>\s*')
for rel_src_path, checkout in handler.checkouts_by_path.iteritems():
project = checkout['name']
# Additional case in case the repo has been removed from the manifest.
src_path = source_repo.GetRelativePath(rel_src_path)
if not os.path.exists(src_path):
cros_build_lib.Info('Detected repo removed from manifest %s' % project)
continue
revision = checkout['revision']
cmd = ['log', '--pretty=full', '%s..HEAD' % revision]
try:
result = git.RunGit(src_path, cmd)
except cros_build_lib.RunCommandError as ex:
# Git returns 128 when the revision does not exist.
if ex.result.returncode != 128:
raise
cros_build_lib.Warning('Detected branch removed from local checkout.')
cros_build_lib.PrintBuildbotStepWarnings()
return
current_author = None
current_committer = None
for line in unicode(result.output, 'ascii', 'ignore').splitlines():
author_match = author_re.match(line)
if author_match:
current_author = author_match.group(1)
committer_match = committer_re.match(line)
if committer_match:
current_committer = committer_match.group(1)
review_match = reviewed_on_re.match(line)
if review_match:
review = review_match.group(1)
_, _, change_number = review.rpartition('/')
items = [
os.path.basename(project),
current_author,
change_number,
]
if current_committer not in ('chrome-bot', 'chrome-internal-fetch',
'chromeos-commit-bot'):
items.insert(0, 'CHUMP')
elif only_print_chumps:
continue
cros_build_lib.PrintBuildbotLink(' | '.join(items), review)