Convert ebuild to pull chrome sources from git.
BUG=chromium:376027
TEST=release builders pull from git and build successfully.
Change-Id: I251d86cd928fcfff516b0e5e955bab05068aa3f6
Reviewed-on: https://chromium-review.googlesource.com/214120
Reviewed-by: Stefan Zager <szager@chromium.org>
Tested-by: Stefan Zager <szager@chromium.org>
diff --git a/cbuildbot/commands.py b/cbuildbot/commands.py
index 1ff241a..1f1e00e 100644
--- a/cbuildbot/commands.py
+++ b/cbuildbot/commands.py
@@ -4,6 +4,7 @@
"""Module containing the various individual commands a builder can run."""
+import base64
import fnmatch
import glob
import logging
@@ -19,8 +20,8 @@
from chromite.cbuildbot import constants
from chromite.cros.tests import cros_vm_test
from chromite.lib import cros_build_lib
-from chromite.lib import gclient
from chromite.lib import git
+from chromite.lib import gob_util
from chromite.lib import gs
from chromite.lib import locking
from chromite.lib import osutils
@@ -1757,15 +1758,13 @@
os.path.join(archive_dir, STATEFUL_FILE))
-def GetChromeLKGM(svn_revision):
- """Returns the ChromeOS LKGM from Chrome given the SVN revision."""
- svn_url = '/'.join([gclient.GetBaseURLs()[0], constants.SVN_CHROME_LKGM])
- svn_revision_args = []
- if svn_revision:
- svn_revision_args = ['-r', str(svn_revision)]
-
- svn_cmd = ['svn', 'cat', svn_url] + svn_revision_args
- return cros_build_lib.RunCommand(svn_cmd, capture_output=True).output.strip()
+def GetChromeLKGM(revision):
+ """Returns the ChromeOS LKGM from Chrome given the git revision."""
+ revision = revision or 'refs/heads/master'
+ lkgm_url_path = '%s/+/%s/%s?format=text' % (
+ constants.CHROMIUM_SRC_PROJECT, revision, constants.PATH_TO_CHROME_LKGM)
+ contents_b64 = gob_util.FetchUrl(constants.EXTERNAL_GOB_HOST, lkgm_url_path)
+ return base64.b64decode(contents_b64.read()).strip()
def SyncChrome(build_root, chrome_root, useflags, tag=None, revision=None):
diff --git a/cbuildbot/commands_unittest.py b/cbuildbot/commands_unittest.py
index fcc5c6d..5e0faa2 100755
--- a/cbuildbot/commands_unittest.py
+++ b/cbuildbot/commands_unittest.py
@@ -6,9 +6,12 @@
"""Unittests for commands."""
+import base64
import os
import sys
+from StringIO import StringIO
+
import constants
sys.path.insert(0, constants.SOURCE_ROOT)
from chromite.cbuildbot import commands
@@ -17,8 +20,8 @@
from chromite.lib import cros_test_lib
from chromite.lib import gs
from chromite.lib import git
+from chromite.lib import gob_util
from chromite.lib import osutils
-from chromite.lib import partial_mock
from chromite.scripts import pushimage
# TODO(build): Finish test wrapper (http://crosbug.com/37517).
@@ -319,15 +322,17 @@
self.assertCommandContains(['./build_image'])
def _TestChromeLKGM(self, chrome_revision):
- """Helper method for testing the GetChromeLKGM method.
-
- Args:
- chrome_revision: either a number or None.
- """
+ """Helper method for testing the GetChromeLKGM method."""
chrome_lkgm = '3322.0.0'
- output = '\n\n%s\n' % chrome_lkgm
- self.rc.AddCmdResult(partial_mock.In('svn'), output=output)
- self.assertEqual(chrome_lkgm, commands.GetChromeLKGM(chrome_revision))
+ url = '%s/+/%s/%s?format=text' % (
+ constants.CHROMIUM_SRC_PROJECT,
+ chrome_revision or 'refs/heads/master',
+ constants.PATH_TO_CHROME_LKGM)
+ with mock.patch.object(
+ gob_util, 'FetchUrl',
+ return_value=StringIO(base64.b64encode(chrome_lkgm))) as patcher:
+ self.assertEqual(chrome_lkgm, commands.GetChromeLKGM(chrome_revision))
+ patcher.assert_called_with(constants.EXTERNAL_GOB_HOST, url)
def testChromeLKGM(self):
"""Verifies that we can get the chrome lkgm without a chrome revision."""
@@ -335,8 +340,7 @@
def testChromeLKGMWithRevision(self):
"""Verifies that we can get the chrome lkgm with a chrome revision."""
- self._TestChromeLKGM(1234)
- self.assertCommandContains(['svn', 'cat', '-r', '1234'])
+ self._TestChromeLKGM('deadbeef' * 5)
def testAbortCQHWTests(self):
commands.AbortCQHWTests('my-version', debug=False)
diff --git a/cbuildbot/constants.py b/cbuildbot/constants.py
index 5a2f621..f80361a 100644
--- a/cbuildbot/constants.py
+++ b/cbuildbot/constants.py
@@ -54,6 +54,7 @@
CORP_DOMAIN = 'corp.google.com'
GOLO_DOMAIN = 'golo.chromium.org'
+CHROME_DOMAIN = 'chrome.' + CORP_DOMAIN
GOB_HOST = '%s.googlesource.com'
@@ -78,6 +79,9 @@
CHROMITE_URL = '%s/%s' % (EXTERNAL_GOB_URL, CHROMITE_PROJECT)
CHROMIUM_SRC_PROJECT = 'chromium/src'
CHROMIUM_GOB_URL = '%s/%s.git' % (EXTERNAL_GOB_URL, CHROMIUM_SRC_PROJECT)
+CHROME_INTERNAL_PROJECT = 'chrome/src-internal'
+CHROME_INTERNAL_GOB_URL = '%s/%s.git' % (
+ INTERNAL_GOB_URL, CHROME_INTERNAL_PROJECT)
MANIFEST_PROJECT = 'chromiumos/manifest'
MANIFEST_INT_PROJECT = 'chromeos/manifest-internal'
diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py
index 15e4322..c6260af 100644
--- a/cbuildbot/stages/sync_stages.py
+++ b/cbuildbot/stages/sync_stages.py
@@ -27,6 +27,7 @@
from chromite.lib import git
from chromite.lib import osutils
from chromite.lib import patch as cros_patch
+from chromite.scripts import cros_mark_chrome_as_stable
PRE_CQ = validation_pool.PRE_CQ
@@ -513,6 +514,7 @@
# lkgm_manager deals with making sure we're synced to whatever manifest
# we get back in GetNextManifest so syncing again is redundant.
self.skip_sync = True
+ self._chrome_version = None
def _GetInitializedManager(self, internal):
"""Returns an initialized lkgm manager.
@@ -569,6 +571,22 @@
return self.manifest_manager.GetLatestCandidate(
dashboard_url=self.ConstructDashboardURL())
+ def GetLatestChromeVersion(self):
+ """Returns the version of Chrome to uprev."""
+ return cros_mark_chrome_as_stable.GetLatestRelease(
+ constants.CHROMIUM_GOB_URL)
+
+ @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
+ def PerformStage(self):
+ """Performs the stage."""
+ if (self._chrome_rev == constants.CHROME_REV_LATEST and
+ self._run.config.master):
+ # PFQ master needs to determine what version of Chrome to build
+ # for all slaves.
+ self._chrome_version = self.GetLatestChromeVersion()
+
+ ManifestVersionedSyncStage.PerformStage(self)
+
class CommitQueueSyncStage(MasterSlaveSyncStage):
"""Commit Queue Sync stage that handles syncing and applying patches.
diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py
index 6e598c3..284ae90 100644
--- a/lib/cros_build_lib.py
+++ b/lib/cros_build_lib.py
@@ -728,6 +728,14 @@
return domain if domain else 'localdomain'
+def HostIsCIBuilder(fq_hostname=None):
+ """Return True iff a host is a continuous-integration builder."""
+ if not fq_hostname:
+ fq_hostname = GetHostName(fully_qualified=True)
+ return (fq_hostname.endswith('.' + constants.GOLO_DOMAIN) or
+ fq_hostname.endswith('.' + constants.CHROME_DOMAIN))
+
+
def TimedCommand(functor, *args, **kwargs):
"""Wrapper for simple log timing of other python functions.
diff --git a/lib/gclient.py b/lib/gclient.py
index 28c770e..1089e39 100644
--- a/lib/gclient.py
+++ b/lib/gclient.py
@@ -6,9 +6,10 @@
import os
import pprint
-import re
+from chromite.cbuildbot import constants
from chromite.lib import cros_build_lib
+from chromite.lib import git
from chromite.lib import osutils
CHROME_COMMITTER_URL = 'svn://svn.chromium.org/chrome'
@@ -34,68 +35,37 @@
return None
-def _UseGoloMirror():
- """Check whether to use the golo.chromium.org mirrors.
-
- This function returns whether or not we should use the mirrors from
- golo.chromium.org, which we presume are only accessible from within
- that subdomain, and a few other known exceptions.
- """
- host = cros_build_lib.GetHostName(fully_qualified=True)
- GOLO_SUFFIXES = [
- '.golo.chromium.org',
- '.chrome.corp.google.com',
- ]
- return any([host.endswith(s) for s in GOLO_SUFFIXES])
-
-
-def GetBaseURLs():
- """Get the base URLs for checking out Chromium and Chrome."""
- if _UseGoloMirror():
- external_url = '%s/chrome' % SVN_MIRROR_URL
- internal_url = '%s/chrome-internal' % SVN_MIRROR_URL
- else:
- external_url = 'http://src.chromium.org/svn'
- internal_url = 'svn://svn.chromium.org/chrome-internal'
-
- return external_url, internal_url
-
-
-def GetTipOfTrunkSvnRevision(svn_url):
- """Returns the current svn revision for the chrome tree."""
- cmd = ['svn', 'info', svn_url]
- svn_info = cros_build_lib.RunCommand(cmd, redirect_stdout=True).output
-
- revision_re = re.compile(r'^Revision:\s+(\d+)')
- for line in svn_info.splitlines():
- match = revision_re.match(line)
- if match:
- svn_revision = match.group(1)
- cros_build_lib.Info('Found SVN Revision %s' % svn_revision)
- return svn_revision
-
- raise Exception('Could not find revision information from %s' % svn_url)
-
-
def _GetGclientURLs(internal, rev):
- """Get the URLs to use in gclient file.
+ """Get the URLs and deps_file values to use in gclient file.
See WriteConfigFile below.
"""
results = []
- external_url, internal_url = GetBaseURLs()
- if rev is None or isinstance(rev, (int, long)):
- rev_str = '@%s' % rev if rev else ''
- results.append(('src', '%s/trunk/src%s' % (external_url, rev_str)))
+ if rev is None or git.IsSHA1(rev):
+ # Regular chromium checkout; src may float to origin/master or be pinned.
+ url = constants.CHROMIUM_GOB_URL
+ if rev:
+ url += ('@' + rev)
+ # TODO(szager): .DEPS.git will eventually be deprecated in favor of DEPS.
+ # When that happens, this could should continue to work, because gclient
+ # will fall back to DEPS if .DEPS.git doesn't exist. Eventually, this
+ # code should be cleaned up to stop referring to non-existent .DEPS.git.
+ results.append(('src', url, '.DEPS.git'))
if internal:
- results.append(('src-internal', '%s/trunk/src-internal' % internal_url))
+ results.append(
+ ('src-internal', constants.CHROME_INTERNAL_GOB_URL, '.DEPS.git'))
elif internal:
- # TODO(petermayo): Fall back to the archive directory if needed.
- primary_url = '%s/trunk/tools/buildspec/releases/%s' % (internal_url, rev)
- results.append(('CHROME_DEPS', primary_url))
+ # Internal buildspec: check out the buildspec repo and set deps_file to
+ # the path to the desired release spec.
+ url = constants.INTERNAL_GOB_URL + '/chrome/tools/buildspec.git'
+ results.append(('CHROME_DEPS', url, 'releases/%s/.DEPS.git' % rev))
else:
- results.append(('CHROMIUM_DEPS', '%s/releases/%s' % (external_url, rev)))
+ # External buildspec: use the main chromium src repository, pinned to the
+ # release tag, with deps_file set to .DEPS.git (which is created by
+ # publish_deps.py).
+ url = constants.CHROMIUM_GOB_URL + '@refs/tags/' + rev
+ results.append(('src', url, '.DEPS.git'))
return results
@@ -106,20 +76,17 @@
See WriteConfigFile below.
"""
urls = _GetGclientURLs(internal, rev)
- custom_deps, custom_vars = {}, {}
- if _UseGoloMirror():
- custom_vars.update({
- 'svn_url': SVN_MIRROR_URL,
- 'webkit_trunk': '%s/webkit-readonly/trunk' % SVN_MIRROR_URL,
- 'googlecode_url': SVN_MIRROR_URL + '/%s',
- 'gsutil': SVN_MIRROR_URL + '/gsutil',
- 'sourceforge_url': SVN_MIRROR_URL + '/%(repo)s'
- })
-
- solutions = [{'name': name,
- 'url': url,
- 'custom_deps': custom_deps,
- 'custom_vars': custom_vars} for (name, url) in urls]
+ solutions = []
+ for (name, url, deps_file) in urls:
+ solution = {
+ 'name': name,
+ 'url': url,
+ 'custom_deps': {},
+ 'custom_vars': {},
+ }
+ if deps_file:
+ solution['deps_file'] = deps_file
+ solutions.append(solution)
return solutions
@@ -129,8 +96,15 @@
See WriteConfigFile below.
"""
solutions = _GetGclientSolutions(internal=internal, rev=rev)
- return 'solutions = %s\n' % pprint.pformat(solutions)
+ result = 'solutions = %s\n' % pprint.pformat(solutions)
+ # Horrible hack, I will go to hell for this. The bots need to have a git
+ # cache set up; but how can we tell whether this code is running on a bot
+ # or a developer's machine?
+ if cros_build_lib.HostIsCIBuilder():
+ result += "cache_dir = '/b/git-cache'\n"
+
+ return result
def WriteConfigFile(gclient, cwd, internal, rev):
"""Initialize the specified directory as a gclient checkout.
@@ -142,9 +116,10 @@
gclient: Path to gclient.
cwd: Directory to sync.
internal: Whether you want an internal checkout.
- rev: Revision or tag to use. If None, use the latest from trunk. If this is
- a number, use the specified revision. If this is a string, use the
- specified tag.
+ rev: Revision or tag to use.
+ - If None, use the latest from trunk.
+ - If this is a sha1, use the specified revision.
+ - Otherwise, treat this as a chrome version string.
"""
spec = _GetGclientSpec(internal=internal, rev=rev)
cmd = [gclient, 'config', '--spec', spec]
@@ -170,7 +145,8 @@
reset: Reset to pristine version of the source code.
"""
cmd = [gclient, 'sync', '--verbose', '--nohooks', '--transitive',
- '--manually_grab_svn_rev']
+ '--with_branch_heads', '--with_tags']
+
if reset:
cmd += ['--reset', '--force', '--delete_unversioned_trees']
cros_build_lib.RunCommand(cmd, cwd=cwd)
diff --git a/lib/gob_util.py b/lib/gob_util.py
index 2e0ecff..5770f97 100644
--- a/lib/gob_util.py
+++ b/lib/gob_util.py
@@ -15,6 +15,7 @@
import os
import socket
import urllib
+import urlparse
from cStringIO import StringIO
from chromite.lib import retry_util
@@ -458,3 +459,19 @@
elif new_revision != revision:
raise GOBError(200, 'While resetting labels on change "%s", '
'a new patchset was uploaded.' % change)
+
+
+def GetTipOfTrunkRevision(git_url):
+ """Returns the current git revision on the master branch."""
+ parsed_url = urlparse.urlparse(git_url)
+ path = parsed_url[2].rstrip('/') + '/+log/master?n=1&format=JSON'
+ j = FetchUrlJson(parsed_url[1], path, ignore_404=False)
+ if not j:
+ raise GOBError(
+ 'Could not find revision information from %s' % git_url)
+ try:
+ return j['log'][0]['commit']
+ except (IndexError, KeyError, TypeError):
+ msg = ('The json returned by https://%s%s has an unfamiliar structure:\n'
+ '%s\n' % (parsed_url[1], path, j))
+ raise GOBError(msg)
diff --git a/scripts/cbuildbot.py b/scripts/cbuildbot.py
index 35fb4de..469637b 100644
--- a/scripts/cbuildbot.py
+++ b/scripts/cbuildbot.py
@@ -48,9 +48,9 @@
from chromite.lib import cleanup
from chromite.lib import commandline
from chromite.lib import cros_build_lib
-from chromite.lib import gclient
from chromite.lib import gerrit
from chromite.lib import git
+from chromite.lib import gob_util
from chromite.lib import osutils
from chromite.lib import patch as cros_patch
from chromite.lib import parallel
@@ -853,9 +853,8 @@
if options.chrome_rev:
chrome_rev = options.chrome_rev
if chrome_rev == constants.CHROME_REV_TOT:
- # Build the TOT Chrome revision.
- svn_url = gclient.GetBaseURLs()[0]
- options.chrome_version = gclient.GetTipOfTrunkSvnRevision(svn_url)
+ options.chrome_version = gob_util.GetTipOfTrunkRevision(
+ constants.CHROMIUM_GOB_URL)
options.chrome_rev = constants.CHROME_REV_SPEC
# If it's likely we'll need to build Chrome, fetch the source.
@@ -1170,7 +1169,7 @@
action='callback', dest='chrome_version',
callback=_CheckChromeVersionOption,
help=('Used with SPEC logic to force a particular '
- 'SVN revision of chrome rather than the '
+ 'git revision of chrome rather than the '
'latest.'))
group.add_remote_option('--clobber', action='store_true', dest='clobber',
default=False,
diff --git a/scripts/cros_mark_chrome_as_stable.py b/scripts/cros_mark_chrome_as_stable.py
index 6f98073..9d25c84 100644
--- a/scripts/cros_mark_chrome_as_stable.py
+++ b/scripts/cros_mark_chrome_as_stable.py
@@ -15,18 +15,21 @@
emerge-x86-generic =chrome-base/chromeos-chrome-8.0.552.0_alpha_r1
"""
+import base64
+import distutils.version
import filecmp
import optparse
import os
import re
import sys
-import time
+import urlparse
from chromite.cbuildbot import constants
from chromite.cbuildbot import portage_utilities
from chromite.lib import cros_build_lib
-from chromite.lib import gclient
from chromite.lib import git
+from chromite.lib import gob_util
+from chromite.lib import timeout_util
from chromite.scripts import cros_mark_as_stable
# Helper regex's for finding ebuilds.
@@ -48,14 +51,10 @@
_REV_TYPES_FOR_LINKS = [constants.CHROME_REV_LATEST,
constants.CHROME_REV_STICKY]
+# TODO(szager): This is inaccurate, but is it safe to change? I have no idea.
_CHROME_SVN_TAG = 'CROS_SVN_COMMIT'
-def _GetSvnUrl(base_url):
- """Returns the path to the svn url for the given chrome branch."""
- return os.path.join(base_url, 'trunk')
-
-
def _GetVersionContents(chrome_version_info):
"""Returns the current Chromium version, from the contents of a VERSION file.
@@ -69,40 +68,37 @@
return '.'.join(chrome_version_array)
-def _GetSpecificVersionUrl(base_url, revision, time_to_wait=600):
+def _GetSpecificVersionUrl(git_url, revision, time_to_wait=600):
"""Returns the Chromium version, from a repository URL and version.
Args:
- base_url: URL for the root of the chromium checkout.
- revision: the SVN revision we want to use.
+ git_url: Repository URL for chromium.
+ revision: the git revision we want to use.
time_to_wait: the minimum period before abandoning our wait for the
desired revision to be present.
"""
- svn_url = os.path.join(_GetSvnUrl(base_url), 'src', 'chrome', 'VERSION')
- if not revision or not (int(revision) > 0):
- raise Exception('Revision must be positive, got %s' % revision)
+ parsed_url = urlparse.urlparse(git_url)
+ host = parsed_url[1]
+ path = parsed_url[2].rstrip('/') + (
+ '/+/%s/chrome/VERSION?format=text' % revision)
- start = time.time()
- # Use the fact we are SVN, hence ordered.
- # Dodge the fact it will silently ignore the revision if it is not
- # yet known. (i.e. too high)
- repo_version = gclient.GetTipOfTrunkSvnRevision(base_url)
- while revision > repo_version:
- if time.time() - start > time_to_wait:
- raise Exception('Timeout Exceeeded')
+ # Allow for git repository replication lag with sleep/retry loop.
+ def _fetch():
+ fh = gob_util.FetchUrl(host, path, ignore_404=True)
+ return fh.read() if fh else None
- msg = 'Repository only has version %s, looking for %s. Sleeping...'
- cros_build_lib.Info(msg, repo_version, revision)
- time.sleep(30)
- repo_version = gclient.GetTipOfTrunkSvnRevision(base_url)
+ def _wait_msg(_remaining_minutes):
+ cros_build_lib.Info(
+ 'Repository does not yet have revision %s. Sleeping...',
+ revision)
- chrome_version_info = cros_build_lib.RunCommand(
- ['svn', 'cat', '-r', revision, svn_url],
- redirect_stdout=True,
- error_message='Could not read version file at %s revision %s.' %
- (svn_url, revision)).output
-
- return _GetVersionContents(chrome_version_info)
+ content = timeout_util.WaitForSuccess(
+ retry_check=lambda x: not bool(x),
+ func=_fetch,
+ timeout=time_to_wait,
+ period=30,
+ side_effect_func=_wait_msg)
+ return _GetVersionContents(base64.b64decode(content))
def _GetTipOfTrunkVersionFile(root):
@@ -120,7 +116,7 @@
return _GetVersionContents(chrome_version_info)
-def CheckIfChromeRightForOS(url):
+def CheckIfChromeRightForOS(deps_content):
"""Checks if DEPS is right for Chrome OS.
This function checks for a variable called 'buildspec_platforms' to
@@ -128,15 +124,12 @@
then it chooses that DEPS.
Args:
- url: url where DEPS file present.
+ deps_content: Content of release buildspec DEPS file.
Returns:
True if DEPS is the right Chrome for Chrome OS.
"""
- deps_contents = cros_build_lib.RunCommand(['svn', 'cat', url],
- redirect_stdout=True).output
-
- platforms_search = re.search(r'buildspec_platforms.*\s.*\s', deps_contents)
+ platforms_search = re.search(r'buildspec_platforms.*\s.*\s', deps_content)
if platforms_search:
platforms = platforms_search.group()
@@ -146,37 +139,40 @@
return False
-def _GetLatestRelease(base_url, branch=None):
- """Gets the latest release version from the buildspec_url for the branch.
+def GetLatestRelease(git_url, branch=None):
+ """Gets the latest release version from the release tags in the repository.
Args:
- base_url: Base URL for the SVN repository.
+ git_url: URL of git repository.
branch: If set, gets the latest release for branch, otherwise latest
release.
Returns:
Latest version string.
"""
- buildspec_url = os.path.join(base_url, 'releases')
- svn_ls = cros_build_lib.RunCommand(['svn', 'ls', buildspec_url],
- redirect_stdout=True).output
- sorted_ls = cros_build_lib.RunCommand(['sort', '--version-sort', '-r'],
- input=svn_ls,
- redirect_stdout=True).output
+ # TODO(szager): This only works for public release buildspecs in the chromium
+ # src repository. Internal buildspecs are tracked differently. At the time
+ # of writing, I can't find any callers that use this method to scan for
+ # internal buildspecs. But there may be something lurking...
+
+ parsed_url = urlparse.urlparse(git_url)
+ path = parsed_url[2].rstrip('/') + '/+refs/tags?format=JSON'
+ j = gob_util.FetchUrlJson(parsed_url[1], path, ignore_404=False)
if branch:
chrome_version_re = re.compile(r'^%s\.\d+.*' % branch)
else:
chrome_version_re = re.compile(r'^[0-9]+\..*')
-
- for chrome_version in sorted_ls.splitlines():
- if chrome_version_re.match(chrome_version):
- deps_url = os.path.join(buildspec_url, chrome_version, 'DEPS')
- deps_check = cros_build_lib.RunCommand(['svn', 'ls', deps_url],
- error_code_ok=True,
- redirect_stdout=True).output
- if deps_check == 'DEPS\n':
- if CheckIfChromeRightForOS(deps_url):
- return chrome_version.rstrip('/')
+ matching_versions = [key for key in j.keys() if chrome_version_re.match(key)]
+ matching_versions.sort(key=distutils.version.LooseVersion)
+ for chrome_version in reversed(matching_versions):
+ path = parsed_url[2].rstrip() + (
+ '/+/refs/tags/%s/DEPS?format=text' % chrome_version)
+ fh = gob_util.FetchUrl(parsed_url[1], path, ignore_404=False)
+ content = fh.read() if fh else None
+ if content:
+ deps_content = base64.b64decode(content)
+ if CheckIfChromeRightForOS(deps_content):
+ return chrome_version
return None
@@ -363,7 +359,7 @@
are release candidates for the next sticky version.
constants.CHROME_REV_STICKY - Revs the sticky version.
chrome_version: The \d.\d.\d.\d version of Chrome.
- commit: Used with constants.CHROME_REV_TOT. The svn revision of chrome.
+ commit: Used with constants.CHROME_REV_TOT. The git revision of chrome.
overlay_dir: Path to the chromeos-chrome package dir.
Returns:
@@ -431,24 +427,15 @@
return '%s-%s' % (new_ebuild.package, new_ebuild.version)
-def ParseMaxRevision(revision_list):
- """Returns the max revision from a list of url@revision string."""
- revision_re = re.compile(r'.*@(\d+)')
-
- def RevisionKey(revision):
- return revision_re.match(revision).group(1)
-
- max_revision = max(revision_list.split(), key=RevisionKey)
- return max_revision.rpartition('@')[2]
-
-
def main(_argv):
usage_options = '|'.join(constants.VALID_CHROME_REVISIONS)
usage = '%s OPTIONS [%s]' % (__file__, usage_options)
parser = optparse.OptionParser(usage)
parser.add_option('-b', '--boards', default='x86-generic')
- parser.add_option('-c', '--chrome_url', default=gclient.GetBaseURLs()[0])
- parser.add_option('-f', '--force_revision', default=None)
+ parser.add_option('-c', '--chrome_url',
+ default=constants.CHROMIUM_GOB_URL)
+ parser.add_option('-f', '--force_version', default=None,
+ help='Chrome version or git revision hash to use')
parser.add_option('-s', '--srcroot', default=os.path.join(os.environ['HOME'],
'trunk', 'src'),
help='Path to the src directory')
@@ -481,20 +468,20 @@
elif chrome_rev == constants.CHROME_REV_SPEC:
commit_to_use = options.force_revision
if '@' in commit_to_use:
- commit_to_use = ParseMaxRevision(commit_to_use)
+ commit_to_use = commit_to_use.rpartition('@')[2]
version_to_uprev = _GetSpecificVersionUrl(options.chrome_url,
commit_to_use)
elif chrome_rev == constants.CHROME_REV_TOT:
- commit_to_use = gclient.GetTipOfTrunkSvnRevision(options.chrome_url)
+ commit_to_use = gob_util.GetTipOfTrunkRevision(options.chrome_url)
version_to_uprev = _GetSpecificVersionUrl(options.chrome_url,
commit_to_use)
elif chrome_rev == constants.CHROME_REV_LATEST:
- version_to_uprev = _GetLatestRelease(options.chrome_url)
+ version_to_uprev = GetLatestRelease(options.chrome_url)
else:
sticky_ebuild = _GetStickyEBuild(stable_ebuilds)
sticky_version = sticky_ebuild.chrome_version
sticky_branch = sticky_version.rpartition('.')[0]
- version_to_uprev = _GetLatestRelease(options.chrome_url, sticky_branch)
+ version_to_uprev = GetLatestRelease(options.chrome_url, sticky_branch)
stable_candidate = FindChromeUprevCandidate(stable_ebuilds, chrome_rev,
sticky_branch)
diff --git a/scripts/cros_mark_chrome_as_stable_unittest.py b/scripts/cros_mark_chrome_as_stable_unittest.py
index 2a17d84..fff55cf 100755
--- a/scripts/cros_mark_chrome_as_stable_unittest.py
+++ b/scripts/cros_mark_chrome_as_stable_unittest.py
@@ -9,9 +9,12 @@
# run with:
# cros_sdk ../../chromite.cbuildbot/cros_mark_chrome_as_stable_unittest.py
+import base64
+import cStringIO
import mox
import os
import sys
+from textwrap import dedent
sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..', '..'))
@@ -19,8 +22,8 @@
from chromite.cbuildbot import portage_utilities
from chromite.lib import cros_build_lib
from chromite.lib import cros_test_lib
-from chromite.lib import gclient
from chromite.lib import git
+from chromite.lib import gob_util
from chromite.scripts import cros_mark_chrome_as_stable
# pylint: disable=W0212,R0904
@@ -135,127 +138,105 @@
self.assertEqual(candidate.ebuild_path, self.sticky_rc)
- def testGetTipOfTrunkSvnRevision(self):
+ def testGetTipOfTrunkRevision(self):
"""Tests if we can get the latest svn revision from TOT."""
A_URL = 'dorf://mink/delaane/forkat/sertiunu.ortg./desk'
- self.mox.StubOutWithMock(cros_build_lib, 'RunCommand')
- cros_build_lib.RunCommand(
- ['svn', 'info', A_URL],
- redirect_stdout=True).AndReturn(
- _StubCommandResult(
- 'Some Junk 2134\nRevision: %s\nOtherInfo: test_data' %
- fake_svn_rev))
+ self.mox.StubOutWithMock(gob_util, 'FetchUrlJson')
+ gob_util.FetchUrlJson(
+ 'mink',
+ '/delaane/forkat/sertiunu.ortg./desk/+log/master?n=1&format=JSON',
+ ignore_404=False).AndReturn(
+ { 'log': [ { 'commit': 'deadbeef' * 5 } ] })
self.mox.ReplayAll()
- revision = gclient.GetTipOfTrunkSvnRevision(A_URL)
+ revision = gob_util.GetTipOfTrunkRevision(A_URL)
self.mox.VerifyAll()
- self.assertEquals(revision, fake_svn_rev)
+ self.assertEquals(revision, 'deadbeef' * 5)
def testGetTipOfTrunkVersion(self):
"""Tests if we get the latest version from TOT."""
- ARBITRARY_URL = 'Pratooey'
- path = os.path.join(cros_mark_chrome_as_stable._GetSvnUrl(ARBITRARY_URL),
- 'src', 'chrome', 'VERSION')
- self.mox.StubOutWithMock(cros_build_lib, 'RunCommand')
- cros_build_lib.RunCommand(
- ['svn', 'info', ARBITRARY_URL],
- redirect_stdout=True).AndReturn(
- _StubCommandResult(
- 'Some Junk 2134\nRevision: %s\nOtherInfo: test_data' %
- fake_svn_rev))
- cros_build_lib.RunCommand(
- ['svn', 'cat', '-r', fake_svn_rev, path], redirect_stdout=True,
- error_message=mox.IsA(str)).AndReturn(
- _StubCommandResult('A=8\nB=0\nC=256\nD=0'))
-
+ TEST_URL = 'proto://host.org/path/to/repo'
+ TEST_VERSION_CONTENTS = dedent('''\
+ A=8
+ B=0
+ C=256
+ D=0''')
+ self.mox.StubOutWithMock(gob_util, 'FetchUrl')
+ gob_util.FetchUrl(
+ 'host.org',
+ '/path/to/repo/+/test-revision/chrome/VERSION?format=text',
+ ignore_404=True).AndReturn(
+ cStringIO.StringIO(base64.b64encode(TEST_VERSION_CONTENTS)))
self.mox.ReplayAll()
- version = cros_mark_chrome_as_stable._GetSpecificVersionUrl(ARBITRARY_URL,
- fake_svn_rev)
+ version = cros_mark_chrome_as_stable._GetSpecificVersionUrl(
+ TEST_URL, 'test-revision')
self.mox.VerifyAll()
self.assertEquals(version, '8.0.256.0')
def testCheckIfChromeRightForOS(self):
"""Tests if we can find the chromeos build from our mock DEPS."""
- ARBITRARY_URL = 'phthp://sores.chromium.org/tqs/7.0.224.1/DEPS'
test_data1 = "buildspec_platforms:\n 'chromeos,',\n"
test_data2 = "buildspec_platforms:\n 'android,',\n"
- self.mox.StubOutWithMock(cros_build_lib, 'RunCommand')
- cros_build_lib.RunCommand(
- ['svn', 'cat', ARBITRARY_URL],
- redirect_stdout=True).AndReturn(_StubCommandResult(test_data1))
- cros_build_lib.RunCommand(
- ['svn', 'cat', ARBITRARY_URL],
- redirect_stdout=True).AndReturn(_StubCommandResult(test_data2))
- self.mox.ReplayAll()
expected_deps = cros_mark_chrome_as_stable.CheckIfChromeRightForOS(
- ARBITRARY_URL)
+ test_data1)
unexpected_deps = cros_mark_chrome_as_stable.CheckIfChromeRightForOS(
- ARBITRARY_URL)
- self.mox.VerifyAll()
+ test_data2)
self.assertTrue(expected_deps)
self.assertFalse(unexpected_deps)
def testGetLatestRelease(self):
"""Tests if we can find the latest release from our mock url data."""
- ARBITRARY_URL = 'phthp://sores.chromium.org/tqs'
- input_data = ['7.0.224.1/', '7.0.224.2/', '8.0.365.5/', 'LATEST.txt']
- test_data = '\n'.join(input_data)
- sorted_data = '\n'.join(reversed(input_data))
- self.mox.StubOutWithMock(cros_build_lib, 'RunCommand')
- cros_build_lib.RunCommand(
- ['svn', 'ls', ARBITRARY_URL + '/releases'],
- redirect_stdout=True).AndReturn(_StubCommandResult(test_data))
- cros_build_lib.RunCommand(
- ['sort', '--version-sort', '-r'], input=test_data,
- redirect_stdout=True).AndReturn(_StubCommandResult(sorted_data))
- # pretend this one is missing to test the skipping logic.
- cros_build_lib.RunCommand(
- ['svn', 'ls', ARBITRARY_URL + '/releases/8.0.365.5/DEPS'],
- error_code_ok=True, redirect_stdout=True).AndReturn(
- _StubCommandResult('BAH BAH BAH'))
- cros_build_lib.RunCommand(
- ['svn', 'ls', ARBITRARY_URL + '/releases/7.0.224.2/DEPS'],
- error_code_ok=True, redirect_stdout=True).AndReturn(
- _StubCommandResult('DEPS\n'))
+ TEST_HOST = 'sores.chromium.org'
+ TEST_URL = 'phthp://%s/tqs' % TEST_HOST
+ TEST_TAGS = ['7.0.224.1', '7.0.224', '8.0.365.5', 'foo', 'bar-12.13.14.15']
+ TEST_REFS_JSON = dict((tag, None) for tag in TEST_TAGS)
+ TEST_BAD_DEPS_CONTENT = dedent('''\
+ buildspec_platforms: 'TRS-80,',
+ ''')
+ TEST_GOOD_DEPS_CONTENT = dedent('''\
+ buildspec_platforms: 'chromeos,',
+ ''')
+
+ self.mox.StubOutWithMock(gob_util, 'FetchUrl')
+ self.mox.StubOutWithMock(gob_util, 'FetchUrlJson')
+ gob_util.FetchUrlJson(
+ TEST_HOST, '/tqs/+refs/tags?format=JSON', ignore_404=False).AndReturn(
+ TEST_REFS_JSON)
+ gob_util.FetchUrl(
+ TEST_HOST, '/tqs/+/refs/tags/8.0.365.5/DEPS?format=text',
+ ignore_404=False).AndReturn(
+ cStringIO.StringIO(base64.b64encode(TEST_BAD_DEPS_CONTENT)))
+ gob_util.FetchUrl(
+ TEST_HOST, '/tqs/+/refs/tags/7.0.224.1/DEPS?format=text',
+ ignore_404=False).AndReturn(
+ cStringIO.StringIO(base64.b64encode(TEST_GOOD_DEPS_CONTENT)))
self.mox.ReplayAll()
- self.mox.StubOutWithMock(cros_mark_chrome_as_stable,
- 'CheckIfChromeRightForOS')
- cros_mark_chrome_as_stable.CheckIfChromeRightForOS(
- ARBITRARY_URL + '/releases/7.0.224.2/DEPS').AndReturn(
- _StubCommandResult('True'))
- self.mox.ReplayAll()
- release = cros_mark_chrome_as_stable._GetLatestRelease(ARBITRARY_URL)
+ release = cros_mark_chrome_as_stable.GetLatestRelease(TEST_URL)
self.mox.VerifyAll()
- self.assertEqual('7.0.224.2', release)
+ self.assertEqual('7.0.224.1', release)
def testGetLatestStickyRelease(self):
"""Tests if we can find the latest sticky release from our mock url data."""
- ARBITRARY_URL = 'http://src.chromium.org/svn'
- test_data = '\n'.join(['7.0.222.1/',
- '8.0.224.2/',
- '8.0.365.5/',
- 'LATEST.txt'])
- self.mox.StubOutWithMock(cros_build_lib, 'RunCommand')
- cros_build_lib.RunCommand(
- ['svn', 'ls', ARBITRARY_URL + '/releases'],
- redirect_stdout=True).AndReturn(_StubCommandResult('some_data'))
- cros_build_lib.RunCommand(
- ['sort', '--version-sort', '-r'], input='some_data',
- redirect_stdout=True).AndReturn(_StubCommandResult(test_data))
- cros_build_lib.RunCommand(
- ['svn', 'ls', ARBITRARY_URL + '/releases/8.0.224.2/DEPS'],
- error_code_ok=True, redirect_stdout=True).AndReturn(
- _StubCommandResult('DEPS\n'))
+ TEST_HOST = 'sores.chromium.org'
+ TEST_URL = 'phthp://%s/tqs' % TEST_HOST
+ TEST_TAGS = ['7.0.224.2', '7.0.224', '7.0.365.5', 'foo', 'bar-12.13.14.15']
+ TEST_REFS_JSON = dict((tag, None) for tag in TEST_TAGS)
+ TEST_DEPS_CONTENT = dedent('''\
+ buildspec_platforms: 'chromeos,',
+ ''')
+
+ self.mox.StubOutWithMock(gob_util, 'FetchUrl')
+ self.mox.StubOutWithMock(gob_util, 'FetchUrlJson')
+ gob_util.FetchUrlJson(
+ TEST_HOST, '/tqs/+refs/tags?format=JSON', ignore_404=False).AndReturn(
+ TEST_REFS_JSON)
+ gob_util.FetchUrl(
+ TEST_HOST, '/tqs/+/refs/tags/7.0.224.2/DEPS?format=text',
+ ignore_404=False).AndReturn(
+ cStringIO.StringIO(base64.b64encode(TEST_DEPS_CONTENT)))
self.mox.ReplayAll()
- self.mox.StubOutWithMock(cros_mark_chrome_as_stable,
- 'CheckIfChromeRightForOS')
- cros_mark_chrome_as_stable.CheckIfChromeRightForOS(
- ARBITRARY_URL + '/releases/8.0.224.2/DEPS').AndReturn(
- _StubCommandResult(True))
- self.mox.ReplayAll()
- release = cros_mark_chrome_as_stable._GetLatestRelease(ARBITRARY_URL,
- '8.0.224')
+ release = cros_mark_chrome_as_stable.GetLatestRelease(TEST_URL, '7.0.224')
self.mox.VerifyAll()
- self.assertEqual('8.0.224.2', release)
+ self.assertEqual('7.0.224.2', release)
def testLatestChromeRevisionListLink(self):
"""Tests link generation to rev lists.
diff --git a/scripts/sync_chrome.py b/scripts/sync_chrome.py
index bddea2d..a090fc0 100644
--- a/scripts/sync_chrome.py
+++ b/scripts/sync_chrome.py
@@ -19,8 +19,8 @@
version = parser.add_mutually_exclusive_group()
version.add_argument('--tag', help='Sync to specified Chrome release',
dest='version')
- version.add_argument('--revision', help='Sync to specified SVN revision',
- type=int, dest='version')
+ version.add_argument('--revision', help='Sync to specified git revision',
+ dest='version')
parser.add_argument('--internal', help='Sync internal version of Chrome',
action='store_true', default=False)