| # Copyright 2012 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """This module uprevs Chrome for cbuildbot. |
| |
| After calling, it prints outs CHROME_VERSION_ATOM=(version atom string). A |
| caller could then use this atom with emerge to build the newly uprevved version |
| of Chrome e.g. |
| |
| ./cros_mark_chrome_as_stable tot |
| Returns chrome-base/chromeos-chrome-8.0.552.0_alpha_r1 |
| |
| emerge-x86-generic =chrome-base/chromeos-chrome-8.0.552.0_alpha_r1 |
| """ |
| |
| import base64 |
| import distutils.version # pylint: disable=import-error,no-name-in-module |
| import filecmp |
| import logging |
| import os |
| import re |
| import urllib.parse |
| |
| from chromite.cbuildbot import cbuildbot_alerts |
| from chromite.lib import commandline |
| from chromite.lib import constants |
| from chromite.lib import cros_build_lib |
| from chromite.lib import git |
| from chromite.lib import gob_util |
| from chromite.lib import portage_util |
| from chromite.lib import timeout_util |
| from chromite.scripts import cros_mark_as_stable |
| |
| |
| # Helper regex's for finding ebuilds. |
| _CHROME_VERSION_REGEX = r"\d+\.\d+\.\d+\.\d+" |
| _NON_STICKY_REGEX = r"%s[(_rc.*)|(_alpha.*)]+" % _CHROME_VERSION_REGEX |
| |
| # Dir where all the action happens. |
| _OVERLAY_DIR = "%(srcroot)s/third_party/chromiumos-overlay/" |
| |
| _GIT_COMMIT_MESSAGE = ( |
| "Marking %(chrome_rev)s for %(chrome_pn)s ebuild " |
| "with version %(chrome_version)s as stable." |
| ) |
| |
| # URLs that print lists of chrome revisions between two versions of the browser. |
| _CHROME_VERSION_URL = ( |
| "http://omahaproxy.appspot.com/changelog?" |
| "old_version=%(old)s&new_version=%(new)s" |
| ) |
| |
| # Only print links when we rev these types. |
| _REV_TYPES_FOR_LINKS = [ |
| constants.CHROME_REV_LATEST, |
| constants.CHROME_REV_STICKY, |
| ] |
| |
| |
| def _GetVersionContents(chrome_version_info): |
| """Get the current Chromium version, from the contents of a VERSION file. |
| |
| Args: |
| chrome_version_info: The contents of a chromium VERSION file. |
| """ |
| chrome_version_array = [] |
| for line in chrome_version_info.splitlines(): |
| chrome_version_array.append(line.rpartition("=")[2]) |
| |
| return ".".join(chrome_version_array) |
| |
| |
| def _GetSpecificVersionUrl(git_url, revision, time_to_wait=600): |
| """Returns the Chromium version, from a repository URL and version. |
| |
| Args: |
| git_url: Repository URL for chromium. |
| revision: the git revision we want to use. |
| time_to_wait: the minimum period before abandoning our wait for the |
| desired revision to be present. |
| """ |
| parsed_url = urllib.parse.urlparse(git_url) |
| host = parsed_url[1] |
| path = parsed_url[2].rstrip("/") + ( |
| "/+/%s/chrome/VERSION?format=text" % revision |
| ) |
| |
| # Allow for git repository replication lag with sleep/retry loop. |
| def _fetch(): |
| return gob_util.FetchUrl(host, path, ignore_404=True) |
| |
| def _wait_msg(_remaining): |
| logging.info( |
| "Repository does not yet have revision %s. Sleeping...", revision |
| ) |
| |
| content = timeout_util.WaitForSuccess( |
| retry_check=lambda x: not bool(x), |
| func=_fetch, |
| timeout=time_to_wait, |
| period=30, |
| side_effect_func=_wait_msg, |
| ) |
| return _GetVersionContents(base64.b64decode(content).decode("utf-8")) |
| |
| |
| def _GetTipOfTrunkVersionFile(root): |
| """Returns the current Chromium version, from a file in a checkout. |
| |
| Args: |
| root: path to the root of the chromium checkout. |
| """ |
| version_file = os.path.join(root, "src", "chrome", "VERSION") |
| try: |
| chrome_version_info = cros_build_lib.run( |
| ["cat", version_file], stdout=True |
| ).stdout |
| except cros_build_lib.RunCommandError as e: |
| e.msg += "\nCould not read version file at %s." % version_file |
| raise e |
| |
| return _GetVersionContents(chrome_version_info) |
| |
| |
| def CheckIfChromeRightForOS(deps_content): |
| """Checks if DEPS is right for Chrome OS. |
| |
| This function checks for a variable called 'buildspec_platforms' to |
| find out if its 'chromeos' or 'all'. If any of those values, |
| then it chooses that DEPS. |
| |
| Args: |
| deps_content: Content of release buildspec DEPS file. |
| |
| Returns: |
| True if DEPS is the right Chrome for Chrome OS. |
| """ |
| platforms_search = re.search(r"buildspec_platforms.*\s.*\s", deps_content) |
| |
| if platforms_search: |
| platforms = platforms_search.group() |
| if "chromeos" in platforms or "all" in platforms: |
| return True |
| |
| return False |
| |
| |
| def GetLatestRelease(git_url, branch=None): |
| """Gets the latest release version from the release tags in the repository. |
| |
| Args: |
| git_url: URL of git repository. |
| branch: If set, gets the latest release for branch, otherwise latest |
| release. |
| |
| Returns: |
| Latest version string. |
| """ |
| # TODO(szager): This only works for public release buildspecs in the |
| # chromium src repository. Internal buildspecs are tracked differently. At |
| # the time of writing, I can't find any callers that use this method to scan |
| # for internal buildspecs. But there may be something lurking... |
| |
| parsed_url = urllib.parse.urlparse(git_url) |
| path = parsed_url[2].rstrip("/") + "/+refs/tags?format=JSON" |
| j = gob_util.FetchUrlJson(parsed_url[1], path, ignore_404=False) |
| if branch: |
| chrome_version_re = re.compile(r"^%s\.\d+.*" % branch) |
| else: |
| chrome_version_re = re.compile(r"^[0-9]+\..*") |
| matching_versions = [ |
| key for key in j.keys() if chrome_version_re.match(key) |
| ] |
| matching_versions.sort(key=distutils.version.LooseVersion) |
| for chrome_version in reversed(matching_versions): |
| path = parsed_url[2].rstrip() + ( |
| "/+/refs/tags/%s/DEPS?format=text" % chrome_version |
| ) |
| content = gob_util.FetchUrl(parsed_url[1], path, ignore_404=False) |
| if content: |
| deps_content = base64.b64decode(content).decode("utf-8") |
| if CheckIfChromeRightForOS(deps_content): |
| return chrome_version |
| |
| return None |
| |
| |
| def _GetStickyEBuild(stable_ebuilds): |
| """Returns the sticky ebuild.""" |
| sticky_ebuilds = [] |
| non_sticky_re = re.compile(_NON_STICKY_REGEX) |
| for ebuild in stable_ebuilds: |
| if not non_sticky_re.match(ebuild.version): |
| sticky_ebuilds.append(ebuild) |
| |
| if not sticky_ebuilds: |
| raise Exception("No sticky ebuilds found") |
| elif len(sticky_ebuilds) > 1: |
| logging.warning("More than one sticky ebuild found:") |
| for ebuild in sticky_ebuilds: |
| logging.warning(" %s", ebuild.ebuild_path) |
| logging.warning("Make sure to *not* create xxx.0-r1.ebuilds") |
| |
| return portage_util.BestEBuild(sticky_ebuilds) |
| |
| |
| class ChromeEBuild(portage_util.EBuild): |
| """Thin sub-class of EBuild that adds a chrome_version field.""" |
| |
| chrome_version_re = re.compile(r".*-(%s|9999).*" % (_CHROME_VERSION_REGEX)) |
| chrome_version = "" |
| |
| def __init__(self, path): |
| portage_util.EBuild.__init__(self, path) |
| re_match = self.chrome_version_re.match( |
| str(self.ebuild_path_no_revision) |
| ) |
| if re_match: |
| self.chrome_version = re_match.group(1) |
| |
| def __str__(self): |
| return str(self.ebuild_path) |
| |
| |
| def FindChromeCandidates(package_dir): |
| """Return a tuple of chrome's unstable ebuild and stable ebuilds. |
| |
| Args: |
| package_dir: The path to where the package ebuild is stored. |
| |
| Returns: |
| Tuple [unstable_ebuild, stable_ebuilds]. |
| |
| Raises: |
| Exception: if no unstable ebuild exists for Chrome. |
| """ |
| stable_ebuilds = [] |
| unstable_ebuilds = [] |
| for path in [ |
| os.path.join(package_dir, entry) for entry in os.listdir(package_dir) |
| ]: |
| if path.endswith(".ebuild"): |
| ebuild = ChromeEBuild(path) |
| if not ebuild.chrome_version: |
| logging.warning("Poorly formatted ebuild found at %s", path) |
| else: |
| if "9999" in ebuild.version: |
| unstable_ebuilds.append(ebuild) |
| else: |
| stable_ebuilds.append(ebuild) |
| |
| # Apply some confidence checks. |
| if not unstable_ebuilds: |
| raise Exception("Missing 9999 ebuild for %s" % package_dir) |
| if not stable_ebuilds: |
| logging.warning("Missing stable ebuild for %s", package_dir) |
| |
| return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds |
| |
| |
| def FindChromeUprevCandidate(stable_ebuilds, chrome_rev, sticky_branch): |
| """Finds the Chrome uprev candidate for the given chrome_rev. |
| |
| Using the pre-flight logic, this means the stable ebuild you are uprevving |
| from. The difference here is that the version could be different and in |
| that case we want to find it to delete it. |
| |
| Args: |
| stable_ebuilds: A list of stable ebuilds. |
| chrome_rev: The chrome_rev designating which candidate to find. |
| sticky_branch: The branch that is currently sticky with Major/Minor |
| components. For example: 9.0.553. Can be None but not if chrome_rev |
| is CHROME_REV_STICKY. |
| |
| Returns: |
| The EBuild, otherwise None if none found. |
| """ |
| candidates = [] |
| if chrome_rev in [ |
| constants.CHROME_REV_LOCAL, |
| constants.CHROME_REV_TOT, |
| constants.CHROME_REV_SPEC, |
| ]: |
| # These are labelled alpha, for historic reasons, |
| # not just for the fun of confusion. |
| chrome_branch_re = re.compile(r"%s.*_alpha.*" % _CHROME_VERSION_REGEX) |
| for ebuild in stable_ebuilds: |
| if chrome_branch_re.search(ebuild.version): |
| candidates.append(ebuild) |
| |
| elif chrome_rev == constants.CHROME_REV_STICKY: |
| assert sticky_branch is not None |
| chrome_branch_re = re.compile(r"%s\..*" % sticky_branch) |
| for ebuild in stable_ebuilds: |
| if chrome_branch_re.search(ebuild.version): |
| candidates.append(ebuild) |
| |
| else: |
| chrome_branch_re = re.compile(r"%s.*_rc.*" % _CHROME_VERSION_REGEX) |
| for ebuild in stable_ebuilds: |
| if chrome_branch_re.search(ebuild.version): |
| candidates.append(ebuild) |
| |
| if candidates: |
| return portage_util.BestEBuild(candidates) |
| else: |
| return None |
| |
| |
| def GetChromeRevisionLinkFromVersions(old_chrome_version, chrome_version): |
| """Return appropriately formatted link to revision info, given versions |
| |
| Given two Chrome version strings (e.g. 9.0.533.0), generate a link to a |
| page that prints the Chromium revisions between those two versions. |
| |
| Args: |
| old_chrome_version: version to diff from |
| chrome_version: version to which to diff |
| |
| Returns: |
| The desired URL. |
| """ |
| return _CHROME_VERSION_URL % { |
| "old": old_chrome_version, |
| "new": chrome_version, |
| } |
| |
| |
| def GetChromeRevisionListLink(old_chrome, new_chrome, chrome_rev): |
| """Returns a link to the list of revisions between two Chromium versions |
| |
| Given two ChromeEBuilds and the kind of rev we're doing, generate a |
| link to a page that prints the Chromium changes between those two |
| revisions, inclusive. |
| |
| Args: |
| old_chrome: ebuild for the version to diff from |
| new_chrome: ebuild for the version to which to diff |
| chrome_rev: one of constants.VALID_CHROME_REVISIONS |
| |
| Returns: |
| The desired URL. |
| """ |
| assert chrome_rev in _REV_TYPES_FOR_LINKS |
| return GetChromeRevisionLinkFromVersions( |
| old_chrome.chrome_version, new_chrome.chrome_version |
| ) |
| |
| |
| def MarkChromeEBuildAsStable( |
| stable_candidate, |
| unstable_ebuild, |
| chrome_pn, |
| chrome_rev, |
| chrome_version, |
| package_dir, |
| ): |
| r"""Uprevs the chrome ebuild specified by chrome_rev. |
| |
| This is the main function that uprevs the chrome_rev from a stable candidate |
| to its new version. |
| |
| Args: |
| stable_candidate: ebuild that corresponds to the stable ebuild we are |
| revving from. If None, builds a new ebuild given the version |
| and logic for chrome_rev type with revision set to 1. |
| unstable_ebuild: ebuild corresponding to the unstable ebuild for chrome. |
| chrome_pn: package name. |
| chrome_rev: one of constants.VALID_CHROME_REVISIONS or LOCAL |
| constants.CHROME_REV_SPEC - Requires commit value. Revs the ebuild |
| for the specified version and uses the portage suffix of _alpha. |
| constants.CHROME_REV_TOT - Requires commit value. Revs the ebuild |
| for the TOT version and uses the portage suffix of _alpha. |
| constants.CHROME_REV_LOCAL - Requires a chrome_root. Revs the ebuild |
| for the local version and uses the portage suffix of _alpha. |
| constants.CHROME_REV_LATEST - This uses the portage suffix of _rc as |
| they are release candidates for the next sticky version. |
| constants.CHROME_REV_STICKY - Revs the sticky version. |
| chrome_version: The \d.\d.\d.\d version of Chrome. |
| package_dir: Path to the chromeos-chrome package dir. |
| |
| Returns: |
| Full portage version atom (including rc's, etc) that was revved. |
| """ |
| |
| def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild): |
| """Returns True if the new ebuild is redundant. |
| |
| This is True if the current stable ebuild is an exact copy of the new |
| one. |
| """ |
| if not stable_ebuild: |
| return False |
| |
| if stable_candidate.chrome_version == new_ebuild.chrome_version: |
| return filecmp.cmp( |
| new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False |
| ) |
| |
| # Mark latest release and sticky branches as stable. |
| mark_stable = chrome_rev not in [ |
| constants.CHROME_REV_TOT, |
| constants.CHROME_REV_SPEC, |
| constants.CHROME_REV_LOCAL, |
| ] |
| |
| # Case where we have the last stable candidate with same version just rev. |
| if stable_candidate and stable_candidate.chrome_version == chrome_version: |
| new_ebuild_path = "%s-r%d.ebuild" % ( |
| stable_candidate.ebuild_path_no_revision, |
| stable_candidate.current_revision + 1, |
| ) |
| else: |
| suffix = "rc" if mark_stable else "alpha" |
| pf = "%s-%s_%s-r1" % (chrome_pn, chrome_version, suffix) |
| new_ebuild_path = os.path.join(package_dir, "%s.ebuild" % pf) |
| |
| portage_util.EBuild.MarkAsStable( |
| unstable_ebuild.ebuild_path, |
| new_ebuild_path, |
| {}, |
| make_stable=mark_stable, |
| ) |
| new_ebuild = ChromeEBuild(new_ebuild_path) |
| |
| # Determine whether this is ebuild is redundant. |
| if IsTheNewEBuildRedundant(new_ebuild, stable_candidate): |
| msg = "Previous ebuild with same version found and ebuild is redundant." |
| logging.info(msg) |
| os.unlink(new_ebuild_path) |
| return None |
| |
| if stable_candidate and chrome_rev in _REV_TYPES_FOR_LINKS: |
| cbuildbot_alerts.PrintBuildbotLink( |
| "Chromium revisions", |
| GetChromeRevisionListLink(stable_candidate, new_ebuild, chrome_rev), |
| ) |
| |
| git.RunGit(package_dir, ["add", new_ebuild_path]) |
| if stable_candidate and not stable_candidate.IsSticky(): |
| git.RunGit(package_dir, ["rm", str(stable_candidate.ebuild_path)]) |
| |
| portage_util.EBuild.CommitChange( |
| _GIT_COMMIT_MESSAGE |
| % { |
| "chrome_pn": chrome_pn, |
| "chrome_rev": chrome_rev, |
| "chrome_version": chrome_version, |
| }, |
| package_dir, |
| ) |
| |
| return "%s-%s" % (new_ebuild.package, new_ebuild.version) |
| |
| |
| def GetParser(): |
| """Return a command line parser.""" |
| parser = commandline.ArgumentParser(description=__doc__) |
| parser.add_argument("-b", "--boards") |
| parser.add_argument( |
| "-c", "--chrome_url", default=constants.CHROMIUM_GOB_URL |
| ) |
| parser.add_argument( |
| "-f", |
| "--force_version", |
| help="Chrome version or git revision hash to use", |
| ) |
| parser.add_argument( |
| "-s", |
| "--srcroot", |
| default=os.path.join(os.environ["HOME"], "trunk", "src"), |
| help="Path to the src directory", |
| ) |
| parser.add_argument( |
| "-t", |
| "--tracking_branch", |
| default="cros/master", |
| help="Branch we are tracking changes against", |
| ) |
| parser.add_argument("revision", choices=constants.VALID_CHROME_REVISIONS) |
| return parser |
| |
| |
| def main(argv): |
| parser = GetParser() |
| options = parser.parse_args(argv) |
| options.Freeze() |
| chrome_rev = options.revision |
| |
| if options.force_version and chrome_rev not in ( |
| constants.CHROME_REV_SPEC, |
| constants.CHROME_REV_LATEST, |
| ): |
| parser.error( |
| "--force_version is not compatible with the %r " |
| "option." % (chrome_rev,) |
| ) |
| |
| overlay_dir = os.path.abspath(_OVERLAY_DIR % {"srcroot": options.srcroot}) |
| chrome_package_dir = os.path.join(overlay_dir, constants.CHROME_CP) |
| version_to_uprev = None |
| sticky_branch = None |
| |
| (unstable_ebuild, stable_ebuilds) = FindChromeCandidates(chrome_package_dir) |
| |
| if chrome_rev == constants.CHROME_REV_LOCAL: |
| if "CHROME_ROOT" in os.environ: |
| chrome_root = os.environ["CHROME_ROOT"] |
| else: |
| chrome_root = os.path.join(os.environ["HOME"], "chrome_root") |
| |
| version_to_uprev = _GetTipOfTrunkVersionFile(chrome_root) |
| logging.info("Using local source, versioning is untrustworthy.") |
| elif chrome_rev == constants.CHROME_REV_SPEC: |
| if "." in options.force_version: |
| version_to_uprev = options.force_version |
| else: |
| commit_to_use = options.force_version |
| if "@" in commit_to_use: |
| commit_to_use = commit_to_use.rpartition("@")[2] |
| version_to_uprev = _GetSpecificVersionUrl( |
| options.chrome_url, commit_to_use |
| ) |
| elif chrome_rev == constants.CHROME_REV_TOT: |
| commit_to_use = gob_util.GetTipOfTrunkRevision(options.chrome_url) |
| version_to_uprev = _GetSpecificVersionUrl( |
| options.chrome_url, commit_to_use |
| ) |
| elif chrome_rev == constants.CHROME_REV_LATEST: |
| if options.force_version: |
| if "." not in options.force_version: |
| parser.error( |
| "%s only accepts released Chrome versions, not SVN or " |
| "Git revisions." % (chrome_rev,) |
| ) |
| version_to_uprev = options.force_version |
| else: |
| version_to_uprev = GetLatestRelease(options.chrome_url) |
| else: |
| sticky_ebuild = _GetStickyEBuild(stable_ebuilds) |
| sticky_version = sticky_ebuild.chrome_version |
| sticky_branch = sticky_version.rpartition(".")[0] |
| version_to_uprev = GetLatestRelease(options.chrome_url, sticky_branch) |
| |
| stable_candidate = FindChromeUprevCandidate( |
| stable_ebuilds, chrome_rev, sticky_branch |
| ) |
| |
| if stable_candidate: |
| logging.info("Stable candidate found %s", stable_candidate) |
| else: |
| logging.info("No stable candidate found.") |
| |
| tracking_branch = "remotes/m/%s" % os.path.basename(options.tracking_branch) |
| existing_branch = git.GetCurrentBranch(chrome_package_dir) |
| work_branch = cros_mark_as_stable.GitBranch( |
| constants.STABLE_EBUILD_BRANCH, tracking_branch, chrome_package_dir |
| ) |
| work_branch.CreateBranch() |
| |
| # In the case of uprevving overlays that have patches applied to them, |
| # include the patched changes in the stabilizing branch. |
| if existing_branch: |
| git.RunGit(chrome_package_dir, ["rebase", existing_branch]) |
| |
| chrome_version_atom = MarkChromeEBuildAsStable( |
| stable_candidate, |
| unstable_ebuild, |
| "chromeos-chrome", |
| chrome_rev, |
| version_to_uprev, |
| chrome_package_dir, |
| ) |
| if chrome_version_atom: |
| if options.boards: |
| cros_mark_as_stable.CleanStalePackages( |
| options.srcroot, |
| options.boards.split(":"), |
| [chrome_version_atom], |
| ) |
| |
| # If we did rev Chrome, now is a good time to uprev other packages. |
| for other_ebuild in constants.OTHER_CHROME_PACKAGES: |
| other_ebuild_name = os.path.basename(other_ebuild) |
| other_package_dir = os.path.join(overlay_dir, other_ebuild) |
| ( |
| other_unstable_ebuild, |
| other_stable_ebuilds, |
| ) = FindChromeCandidates(other_package_dir) |
| other_stable_candidate = FindChromeUprevCandidate( |
| other_stable_ebuilds, chrome_rev, sticky_branch |
| ) |
| revved_atom = MarkChromeEBuildAsStable( |
| other_stable_candidate, |
| other_unstable_ebuild, |
| other_ebuild_name, |
| chrome_rev, |
| version_to_uprev, |
| other_package_dir, |
| ) |
| if revved_atom and options.boards: |
| cros_mark_as_stable.CleanStalePackages( |
| options.srcroot, options.boards.split(":"), [revved_atom] |
| ) |
| |
| # Explicit print to communicate to caller. |
| if chrome_version_atom: |
| print("CHROME_VERSION_ATOM=%s" % chrome_version_atom) |