blob: af60d7168a0acbae12d3a77a570d1b9305882a76 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This program examines success/fail history for Chrome/ium OS builds and
checks in a LKGM version for Chrome OS for other consumers.
import distutils.version
import logging
import os
import shutil
import tempfile
from chromite.buildbot import cbuildbot_config
from chromite.buildbot import cbuildbot_stages
from chromite.buildbot import constants
from chromite.buildbot import manifest_version
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import gclient
from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import timeout_util
class LKGMNotFound(Exception):
"""Raised if a newer valid LKGM could not be found."""
class LKGMNotCommitted(Exception):
"""Raised if we could not submit a new LKGM."""
class ChromeCommitter(object):
"""Committer object responsible for obtaining a new LKGM and committing it."""
_COMMIT_MSG = ('Automated Commit: Committing new LKGM version %(version)s '
'for chromeos.')
def __init__(self, checkout_dir, dryrun):
self._checkout_dir = checkout_dir
self._dryrun = dryrun
self._lkgm = None
self._old_lkgm = None
def CheckoutChromeLKGM(self):
"""Checkout chromeos LKGM file for chrome into tmp checkout dir."""
# We function only on an empty directory.
lkgm_dir = '/'.join([gclient.CHROME_COMMITTER_URL,
# We checkout the bare necessities to get the .svn/file needed to commit.
cros_build_lib.RunCommand(['svn', 'checkout', '--depth=empty', lkgm_dir,
cros_build_lib.RunCommand(['svn', 'update', constants.CHROME_LKGM_FILE],
self._old_lkgm = osutils.ReadFile(
os.path.join(self._checkout_dir, constants.CHROME_LKGM_FILE))
def _GetLatestCanaryVersions(self):
"""Returns the latest CANDIDATES_TO_CONSIDER canary versions."""
gs_handle = gs.GSContext()
version_paths = gs_handle.LS(
# Strip gs://<path> prefix and trailing /'s.
versions = [os.path.basename(v.rstrip('/')) for v in version_paths]
lv = distutils.version.LooseVersion
# We only care about canary versions which always end in 0.0.
canary_versions = [v for v in versions if v.endswith('.0.0')]
new_canary_versions = [v for v in canary_versions
if lv(v) > lv(self._old_lkgm)]
return sorted(new_canary_versions, key=lv,
def FindNewLKGM(self):
"""Finds a new LKGM for chrome from previous chromeos releases."""
versions = self._GetLatestCanaryVersions()
if not versions:
raise LKGMNotFound('No valid LKGM found newer than the old LKGM.')
canaries = cbuildbot_config.GetCanariesForChromeLKGM()'Considering the following versions: %s', ' '.join(versions))'Using scores from the following canaries: %s',
' '.join(canaries))
# Scores are based on passing builders.
version_scores = {}
for version in versions:
for builder in canaries:
status = manifest_version.BuildSpecsManager.GetBuildStatus(
builder, version, retries=0)
if status:
if status.Passed():
version_scores[version] = version_scores.get(version, 0) + 1
elif status.Failed():
# We don't consider builds with any reporting failures.
version_scores[version] = 0
break'Version %s had score %d', version,
version_scores.get(version, 0))
# We want to get the version with the highest score. In case of a tie, we
# want to choose the highest version.
lkgm = max((v, k) for k, v in version_scores.iteritems())[1]
if not version_scores[lkgm] > 0:
raise LKGMNotFound('No valid LKGM found. Scores are too low.')
self._lkgm = lkgm
def CommitNewLKGM(self):
"""Commits the new LKGM file using our template commit message."""
lv = distutils.version.LooseVersion
if not self._lkgm and not lv(self._lkgm) < lv(self._old_lkgm):
raise LKGMNotFound('No valid LKGM found. Did you run FindNewLKGM?')
# Add the new versioned file.
os.path.join(self._checkout_dir, constants.CHROME_LKGM_FILE),
add_cmd = ['svn', 'add', constants.CHROME_LKGM_FILE]
cros_build_lib.RunCommand(add_cmd, cwd=self._checkout_dir)
# Commit it!
commit_cmd = ['svn', 'commit', '--message',
self. _COMMIT_MSG % dict(version=self._lkgm)]
if not timeout_util.IsTreeOpen(gclient.STATUS_URL,
self._SLEEP_TIMEOUT, timeout=self._TREE_TIMEOUT):
raise LKGMNotCommitted('Chromium Tree is closed')
# Sadly svn commit does not have a dryrun option.
if not self._dryrun:
cros_build_lib.RunCommand(commit_cmd, cwd=self._checkout_dir)
except cros_build_lib.RunCommandError as e:
raise LKGMNotCommitted('Could not submit LKGM: %r' % e)
else:'Would have run: %s', ' '.join(commit_cmd))
def UpdateLatestFilesForBot(self, config, versions):
"""Update the LATEST files, for a given bot, in Google Storage.
config: The builder config to update.
versions: Versions of ChromeOS to look at, sorted in descending order.
base_url = cbuildbot_stages.ArchivingStage.GetBaseUploadURL(config)
acl = cbuildbot_stages.ArchivingStage.GetUploadACL(config)
latest_url = None
# gs.GSContext skips over all commands (including read-only checks)
# when dry_run is True, so we have to create two context objects.
# TODO(davidjames): Fix this.
gs_ctx = gs.GSContext()
copy_ctx = gs.GSContext(dry_run=self._dryrun)
for version in reversed(versions):
url = os.path.join(base_url, 'LATEST-%s' % version)
found = gs_ctx.Exists(url, print_cmd=False)
if not found and latest_url:
copy_ctx.Copy(latest_url, url, version=0, acl=acl)
cros_build_lib.Info('Copied %s -> %s', latest_url, url)
except gs.GSContextPreconditionFailed:
found = True
if found:
cros_build_lib.Info('Found %s', url)
latest_url = url
def UpdateLatestFiles(self):
"""Update the LATEST files since LKGM, in Google Storage."""
ext_cfgs, int_cfgs = cbuildbot_config.FindFullConfigsForBoard(board=None)
versions = self._GetLatestCanaryVersions() + [self._old_lkgm]
tasks = [[cfg, versions] for cfg in ext_cfgs + int_cfgs]
parallel.RunTasksInProcessPool(self.UpdateLatestFilesForBot, tasks,
def _GetParser():
"""Returns the parser to use for this module."""
parser = commandline.ArgumentParser(usage=__doc__, caching=True)
parser.add_argument('--dryrun', action='store_true', default=False,
help="Find the next LKGM but don't commit it.")
return parser
def main(argv):
parser = _GetParser()
args = parser.parse_args(argv)
checkout_dir = tempfile.mkdtemp(prefix='cbr_chrome_checkout')
committer = ChromeCommitter(checkout_dir, dryrun=args.dryrun)
return 0