| #!/usr/bin/python2.6 |
| # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Perform various tasks related to updating Portage packages.""" |
| |
| import logging |
| import optparse |
| import os |
| import parallel_emerge |
| import portage |
| import re |
| import shutil |
| import sys |
| import tempfile |
| |
| sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) |
| import chromite.lib.cros_build_lib as cros_lib |
| import chromite.lib.operation as operation |
| import chromite.lib.upgrade_table as utable |
| import merge_package_status as mps |
| |
| oper = operation.Operation('cros_portage_upgrade') |
| oper.verbose = True # Without verbose Info messages don't show up. |
| |
| NOT_APPLICABLE = 'N/A' |
| WORLD_TARGET = 'world' |
| |
| # TODO(mtennant): Use this class to replace the 'info' dictionary used |
| # throughout. In the meantime, it simply serves as documentation for |
| # the values in that dictionary. |
| class PInfo(object): |
| """Class to accumulate package info during upgrade process. |
| |
| This class is basically a formalized dictionary.""" |
| |
| __slots__ = [ |
| 'category', # Package category only |
| # TODO(mtennant): Rename 'cpv' to 'curr_cpv' or similar. |
| 'cpv', # Current full cpv (revision included) |
| 'cpv_cmp_upstream', # 0 = current, >0 = outdated, <0 = futuristic |
| 'emerge_ok', # True if upgraded_cpv is emergeable |
| 'emerge_output', # Output from pretend emerge of upgraded_cpv |
| 'latest_upstream_cpv', # Latest (non-stable ok) upstream cpv |
| 'overlay', # Overlay package currently in |
| 'package', # category/package_name |
| 'package_name', # The 'p' in 'cpv' |
| 'package_ver', # The 'pv' in 'cpv' |
| 'slot', # Current package slot |
| 'stable_upstream_cpv', # Latest stable upstream cpv |
| 'state', # One of utable.UpgradeTable.STATE_* |
| 'upgraded_cpv', # If upgraded, it is to this cpv |
| 'upgraded_stable', # Boolean. If upgraded_cpv, indicates if stable. |
| 'upgraded_unmasked', # Boolean. If upgraded_cpv, indicates if unmasked. |
| 'upstream_cpv', # latest/stable upstream cpv according to request |
| 'user_arg', # Original user arg for this pkg, if applicable |
| 'version_rev', # Just revision (e.g. 'r1'). '' if no revision |
| ] |
| |
| class Upgrader(object): |
| """A class to perform various tasks related to updating Portage packages.""" |
| |
| PORTAGE_GIT_URL = ('ssh://gerrit.chromium.org:29418/' |
| 'chromiumos/overlays/portage.git') |
| ORIGIN_GENTOO = 'origin/gentoo' |
| UPSTREAM_TMP_REPO = '/tmp/cros_portage_upgrade-gentoo-portage' |
| UPSTREAM_OVERLAY_NAME = 'portage' |
| STABLE_OVERLAY_NAME = 'portage-stable' |
| CROS_OVERLAY_NAME = 'chromiumos-overlay' |
| HOST_BOARD = 'amd64-host' |
| OPT_SLOTS = ['amend', 'csv_file', 'no_upstream_cache', 'rdeps', |
| 'upgrade', 'upgrade_deep', 'upstream', 'unstable_ok', 'verbose'] |
| |
| EQUERY_CMD = 'equery' |
| EMERGE_CMD = 'emerge' |
| PORTAGEQ_CMD = 'portageq' |
| BOARD_CMDS = set([EQUERY_CMD, EMERGE_CMD, PORTAGEQ_CMD]) |
| |
| __slots__ = ['_amend', # Boolean to use --amend with upgrade commit |
| '_args', # Commandline arguments (all portage targets) |
| '_curr_arch', # Architecture for current board run |
| '_curr_board', # Board for current board run |
| '_curr_table', # Package status for current board run |
| '_cros_overlay', # Path to chromiumos-overlay repo |
| '_csv_file', # File path for writing csv output |
| '_deps_graph', # Dependency graph from portage |
| '_emptydir', # Path to temporary empty directory |
| '_equery_regexp',# Regexp mask bits from 'equery list' output |
| '_master_archs', # Set. Archs of tables merged into master_table |
| '_master_cnt', # Number of tables merged into master_table |
| '_master_table', # Merged table from all board runs |
| '_no_upstream_cache', # Boolean. Delete upstream cache when done |
| '_porttree', # Reference to portage porttree object |
| '_rdeps', # Boolean, if True pass --root-deps=rdeps |
| '_stable_repo', # Path to portage-stable |
| '_stable_repo_stashed', # True if portage-stable has a git stash |
| '_stable_repo_status', # git status report at start of run |
| '_targets', # Processed list of portage targets |
| '_upgrade', # Boolean indicating upgrade requested |
| '_upgrade_cnt', # Num pkg upgrades in this run (all boards) |
| '_upgrade_deep', # Boolean indicating upgrade_deep requested |
| '_upstream', # User-provided path to upstream repo |
| '_upstream_repo',# Path to upstream portage repo |
| '_unstable_ok', # Boolean to allow unstable upstream also |
| '_verbose', # Boolean |
| ] |
| |
| def __init__(self, options=None, args=None): |
| self._args = args |
| self._targets = mps.ProcessTargets(args) |
| |
| self._master_table = None |
| self._master_cnt = 0 |
| self._master_archs = set() |
| self._upgrade_cnt = 0 |
| |
| self._stable_repo = os.path.join(options.srcroot, 'third_party', |
| self.STABLE_OVERLAY_NAME) |
| self._upstream_repo = options.upstream |
| if not self._upstream_repo: |
| self._upstream_repo = self.UPSTREAM_TMP_REPO |
| self._cros_overlay = os.path.join(options.srcroot, 'third_party', |
| self.CROS_OVERLAY_NAME) |
| |
| # Save options needed later. |
| for opt in self.OPT_SLOTS: |
| setattr(self, '_' + opt, getattr(options, opt, None)) |
| |
| self._porttree = None |
| self._emptydir = None |
| self._deps_graph = None |
| |
| # Pre-compiled regexps for speed. |
| self._equery_regexp = re.compile(r'^\[...\]\s*\[(.)(.)\]\s+\S+$') |
| |
| def _GetPkgKeywordsFile(self): |
| """Return the path to the package.keywords file in chromiumos-overlay.""" |
| return '%s/profiles/targets/chromeos/package.keywords' % self._cros_overlay |
| |
| def _SaveStatusOnStableRepo(self): |
| """Get the 'git status' for everything in |self._stable_repo|. |
| |
| The results are saved in a dict at self._stable_repo_status where each key |
| is a file path rooted at |self._stable_repo|, and the value is the status |
| for that file as returned by 'git status -s'. (e.g. 'A' for 'Added'). |
| """ |
| result = self._RunGit(self._stable_repo, 'status -s', redirect_stdout=True) |
| if result.returncode == 0: |
| statuses = {} |
| for line in result.output.strip().split('\n'): |
| if not line: |
| continue |
| |
| linesplit = line.split() |
| (status, path) = linesplit[0], linesplit[1] |
| statuses[path] = status |
| |
| self._stable_repo_status = statuses |
| else: |
| self._stable_repo_status = None |
| |
| self._stable_repo_stashed = False |
| |
| def _CheckStableRepoOnBranch(self): |
| """Raise exception if |self._stable_repo| is not on a branch now.""" |
| result = self._RunGit(self._stable_repo, 'branch', redirect_stdout=True) |
| if result.returncode == 0: |
| for line in result.output.split('\n'): |
| match = re.search(r'^\*\s+(.+)$', line) |
| if match: |
| # Found current branch, see if it is a real branch. |
| branch = match.group(1) |
| if branch != '(no branch)': |
| return |
| raise RuntimeError("To perform upgrade, %s must be on a branch." % |
| self._stable_repo) |
| |
| raise RuntimeError("Unable to determine whether %s is on a branch." % |
| self._stable_repo) |
| |
| def _PkgUpgradeRequested(self, info): |
| """Return True if upgrade of pkg in |info| hash was requested by user.""" |
| if self._upgrade_deep: |
| return True |
| |
| if self._upgrade: |
| return bool('user_arg' in info) |
| |
| return False |
| |
| @staticmethod |
| def _FindBoardArch(board): |
| """Return the architecture for a given board name.""" |
| # Host is a special case |
| if board == Upgrader.HOST_BOARD: |
| return 'amd64' |
| |
| # Leverage Portage 'portageq' tool to do this. |
| cmd = ['portageq-%s' % board, 'envvar', 'ARCH'] |
| cmd_result = cros_lib.RunCommand(cmd, print_cmd=False, |
| redirect_stdout=True, exit_code=True) |
| if cmd_result.returncode == 0: |
| return cmd_result.output.strip() |
| else: |
| return None |
| |
| @staticmethod |
| def _GetPreOrderDepGraphPackage(deps_graph, package, pkglist, visited): |
| """Collect packages from |deps_graph| into |pkglist| in pre-order.""" |
| if package in visited: |
| return |
| visited.add(package) |
| for parent in deps_graph[package]['provides']: |
| Upgrader._GetPreOrderDepGraphPackage(deps_graph, parent, pkglist, visited) |
| pkglist.append(package) |
| |
| @staticmethod |
| def _GetPreOrderDepGraph(deps_graph): |
| """Return packages from |deps_graph| in pre-order.""" |
| pkglist = [] |
| visited = set() |
| for package in deps_graph: |
| Upgrader._GetPreOrderDepGraphPackage(deps_graph, package, pkglist, |
| visited) |
| return pkglist |
| |
| @staticmethod |
| def _CmpCpv(cpv1, cpv2): |
| """Returns standard cmp result between |cpv1| and |cpv2|. |
| |
| If one cpv is None then the other is greater. |
| """ |
| if cpv1 is None and cpv2 is None: |
| return 0 |
| if cpv1 is None: |
| return -1 |
| if cpv2 is None: |
| return 1 |
| return portage.versions.pkgcmp(portage.versions.pkgsplit(cpv1), |
| portage.versions.pkgsplit(cpv2)) |
| |
| @staticmethod |
| def _GetCatPkgFromCpv(cpv): |
| """Returns category/package_name from a full |cpv|. |
| |
| If |cpv| is incomplete, may return only the package_name. |
| |
| If package_name cannot be determined, return None. |
| """ |
| if not cpv: |
| return None |
| |
| # Result is None or (cat, pn, version, rev) |
| result = portage.versions.catpkgsplit(cpv) |
| if result: |
| # This appears to be a quirk of portage? Category string == 'null'. |
| if result[0] is None or result[0] == 'null': |
| return result[1] |
| return '%s/%s' % (result[0], result[1]) |
| |
| return None |
| |
| @staticmethod |
| def _GetVerRevFromCpv(cpv): |
| """Returns just the version-revision string from a full |cpv|.""" |
| if not cpv: |
| return None |
| |
| # Result is None or (cat, pn, version, rev) |
| result = portage.versions.catpkgsplit(cpv) |
| if result: |
| (version, rev) = result[2:4] |
| if rev != 'r0': |
| return '%s-%s' % (version, rev) |
| else: |
| return version |
| |
| return None |
| |
| def _RunGit(self, repo, command, redirect_stdout=False, |
| combine_stdout_stderr=False): |
| """Runs |command| in the git |repo|. |
| |
| This leverages the cros_build_lib.RunCommand function. The |
| |redirect_stdout| and |combine_stdout_stderr| arguments are |
| passed to that function. |
| |
| Returns a Result object as documented by cros_build_lib.RunCommand. |
| Most usefully, the result object has a .output attribute containing |
| the output from the command (if |redirect_stdout| was True). |
| """ |
| cmdline = ['/bin/sh', '-c', 'cd %s && git -c core.pager=" " %s' % |
| (repo, command)] |
| result = cros_lib.RunCommand(cmdline, exit_code=True, |
| print_cmd=self._verbose, |
| redirect_stdout=redirect_stdout, |
| combine_stdout_stderr=combine_stdout_stderr) |
| return result |
| |
| def _SplitEBuildPath(self, ebuild_path): |
| """Split a full ebuild path into (overlay, cat, pn, pv).""" |
| (ebuild_path, ebuild) = os.path.splitext(ebuild_path) |
| (ebuild_path, pv) = os.path.split(ebuild_path) |
| (ebuild_path, pn) = os.path.split(ebuild_path) |
| (ebuild_path, cat) = os.path.split(ebuild_path) |
| (ebuild_path, overlay) = os.path.split(ebuild_path) |
| return (overlay, cat, pn, pv) |
| |
| def _GenPortageEnvvars(self, arch, unstable_ok, portdir=None, |
| portage_configroot=None): |
| """Returns dictionary of envvars for running portage tools. |
| |
| If |arch| is set, then ACCEPT_KEYWORDS will be included and set |
| according to |unstable_ok|. |
| |
| PORTDIR is set to |portdir| value, if not None. |
| PORTAGE_CONFIGROOT is set to |portage_configroot| value, if not None. |
| """ |
| envvars = {} |
| if arch: |
| if unstable_ok: |
| envvars['ACCEPT_KEYWORDS'] = arch + ' ~' + arch |
| else: |
| envvars['ACCEPT_KEYWORDS'] = arch |
| |
| if portdir is not None: |
| envvars['PORTDIR'] = portdir |
| if portage_configroot is not None: |
| envvars['PORTAGE_CONFIGROOT'] = portage_configroot |
| |
| return envvars |
| |
| def _FindUpstreamCPV(self, pkg, unstable_ok=False): |
| """Returns latest cpv in |_upstream_repo| that matches |pkg|. |
| |
| The |pkg| argument can specify as much or as little of the full CPV |
| syntax as desired, exactly as accepted by the Portage 'equery' command. |
| To find whether an exact version exists upstream specify the full |
| CPV. To find the latest version specify just the category and package |
| name. |
| |
| Results are filtered by architecture keyword using |self._curr_arch|. |
| By default, only ebuilds stable on that arch will be accepted. To |
| accept unstable ebuilds, set |unstable_ok| to True. |
| |
| Returns upstream cpv, if found. |
| """ |
| envvars = self._GenPortageEnvvars(self._curr_arch, unstable_ok, |
| portdir=self._upstream_repo, |
| portage_configroot=self._emptydir) |
| |
| # Point equery to the upstream source to get latest version for keywords. |
| equery = ['equery', 'which', pkg ] |
| cmd_result = cros_lib.RunCommand(equery, extra_env=envvars, |
| print_cmd=self._verbose, |
| exit_code=True, error_ok=True, |
| redirect_stdout=True, |
| combine_stdout_stderr=True, |
| ) |
| if cmd_result.returncode == 0: |
| ebuild_path = cmd_result.output.strip() |
| (overlay, cat, pn, pv) = self._SplitEBuildPath(ebuild_path) |
| return os.path.join(cat, pv) |
| else: |
| return None |
| |
| def _GetBoardCmd(self, cmd): |
| """Return the board-specific version of |cmd|, if applicable.""" |
| if cmd in self.BOARD_CMDS: |
| # Host "board is a special case. |
| if self._curr_board != self.HOST_BOARD: |
| return '%s-%s' % (cmd, self._curr_board) |
| |
| return cmd |
| |
| def _IsEmergeable(self, cpv, stable_only): |
| """Indicate whether |cpv| can be emerged on current board. |
| |
| This essentially runs emerge with the --pretend option to verify |
| that all dependencies for this package version are satisfied. |
| |
| The |stable_only| argument determines whether an unstable version |
| is acceptable. |
| |
| Return tuple with two elements: |
| [0] True if |cpv| can be emerged. |
| [1] Output from the emerge command. |
| """ |
| envvars = self._GenPortageEnvvars(self._curr_arch, not stable_only) |
| |
| emerge = self._GetBoardCmd(self.EMERGE_CMD) |
| cmd = [emerge, '-p', '=' + cpv] |
| result = cros_lib.RunCommand(cmd, exit_code=True, error_ok=True, |
| extra_env=envvars, print_cmd=False, |
| redirect_stdout=True, |
| combine_stdout_stderr=True, |
| ) |
| |
| return (result.returncode == 0, ' '.join(cmd), result.output) |
| |
| def _FindCurrentCPV(self, pkg): |
| """Returns current cpv on |_curr_board| that matches |pkg|, or None.""" |
| envvars = self._GenPortageEnvvars(self._curr_arch, False) |
| |
| equery = self._GetBoardCmd(self.EQUERY_CMD) |
| cmd = [equery, 'which', pkg] |
| cmd_result = cros_lib.RunCommand(cmd, exit_code=True, error_ok=True, |
| extra_env=envvars, print_cmd=False, |
| redirect_stdout=True, |
| combine_stdout_stderr=True, |
| ) |
| |
| if cmd_result.returncode == 0: |
| ebuild_path = cmd_result.output.strip() |
| (overlay, cat, pn, pv) = self._SplitEBuildPath(ebuild_path) |
| return os.path.join(cat, pv) |
| else: |
| return None |
| |
| def _GetMaskBits(self, cpv): |
| """Return two-Boolean tuple (unmasked, stable) for |cpv| on current arch.""" |
| |
| envvars = self._GenPortageEnvvars(self._curr_arch, False) |
| |
| equery = self._GetBoardCmd('equery') |
| cmd = [equery, '--no-pipe', 'list', '-op', cpv] |
| result = cros_lib.RunCommand(cmd, exit_code=True, error_ok=True, |
| extra_env=envvars, print_cmd=False, |
| redirect_stdout=True, |
| combine_stdout_stderr=True, |
| ) |
| output = result.output.strip() |
| |
| # Expect output like one of these cases (~ == unstable, M == masked): |
| # [-P-] [ ~] sys-fs/fuse-2.7.3:0 |
| # [-P-] [ ] sys-fs/fuse-2.7.3:0 |
| # [-P-] [M ] sys-fs/fuse-2.7.3:0 |
| # [-P-] [M~] sys-fs/fuse-2.7.3:0 |
| for line in output.split('\n'): |
| match = self._equery_regexp.search(line) |
| if match: |
| return ('M' != match.group(1), '~' != match.group(2)) |
| |
| raise RuntimeError("Unable to determine whether %s is stable from equery " |
| "output:\n%s" % (cpv, output)) |
| |
| def _VerifyEbuildOverlay(self, cpv, overlay, stable_only): |
| """Raises exception if ebuild for |cpv| is not from |overlay|. |
| |
| Essentially, this verifies that the upgraded ebuild in portage-stable |
| is indeed the one being picked up, rather than some other ebuild with |
| the same version in another overlay. |
| """ |
| # Further explanation: this check should always pass, but might not |
| # if the copy/upgrade from upstream did not work. This is just a |
| # sanity check. |
| envvars = self._GenPortageEnvvars(self._curr_arch, not stable_only) |
| |
| equery = self._GetBoardCmd(self.EQUERY_CMD) |
| cmd = [equery, 'which', '--include-masked', cpv] |
| result = cros_lib.RunCommand(cmd, exit_code=True, error_ok=True, |
| extra_env=envvars, print_cmd=False, |
| redirect_stdout=True, |
| combine_stdout_stderr=True, |
| ) |
| ebuild_path = result.output.strip() |
| (ovrly, cat, pn, pv) = self._SplitEBuildPath(ebuild_path) |
| if ovrly != overlay: |
| raise RuntimeError('Somehow ebuild for %s is not coming from %s:\n %s\n' |
| 'Please show this error to the build team.' % |
| (cpv, overlay, ebuild_path)) |
| |
| def _GiveMaskedError(self, upgraded_cpv, emerge_output): |
| """Raise RuntimeError saying that |upgraded_cpv| is masked off. |
| |
| See if hint found in |emerge_output| to improve error emssage. |
| """ |
| |
| # Expecting emerge_output to have lines like this: |
| # The following mask changes are necessary to proceed: |
| # #required by =sys-fs/lvm2-2.02.73-r1 (argument) |
| # # /home/mtennant/trunk/src/third_party/chromiumos-overlay/profiles\ |
| # /targets/chromeos/package.mask: |
| package_mask = None |
| |
| regex_line1 = re.compile(r'#\s*required by =%s' % upgraded_cpv) |
| regex_line2 = re.compile(r'#\s*(\S+/package\.mask):') |
| |
| emerge_lines = emerge_output.split('\n') |
| for ix1 in range(len(emerge_lines)): |
| line = emerge_lines[ix1] |
| |
| # Look for first line. |
| if regex_line1.search(line): |
| # Look for second line within range_limit lines of first one. |
| range_limit = 3 |
| for ix2 in range(ix1 + 1, min(ix1 + range_limit, len(emerge_lines))): |
| line = emerge_lines[ix2] |
| match = regex_line2.search(line) |
| if match: |
| package_mask = match.group(1) |
| break |
| break |
| |
| oper.Error("Emerge output for %s on %s follows:" % |
| (upgraded_cpv, self._curr_arch)) |
| print emerge_output |
| |
| if package_mask: |
| raise RuntimeError("Upgraded package '%s' appears to be masked by a line " |
| "in\n'%s'\n" |
| "Full emerge output is above. Address mask issue, " |
| "then run this again." % |
| (upgraded_cpv, package_mask)) |
| else: |
| raise RuntimeError("Upgraded package '%s' is masked somehow (See full " |
| "emerge output above). Address that and then run this " |
| "again." % upgraded_cpv) |
| |
| def _PkgUpgradeStaged(self, upstream_cpv): |
| """Return True if package upgrade is already staged.""" |
| if not upstream_cpv: |
| return False |
| |
| (cat, pkgname, version, rev) = portage.versions.catpkgsplit(upstream_cpv) |
| ebuild = upstream_cpv.replace(cat + '/', '') + '.ebuild' |
| ebuild_relative_path = os.path.join(cat, pkgname, ebuild) |
| |
| status = self._stable_repo_status.get(ebuild_relative_path, None) |
| if status and 'A' == status: |
| return True |
| |
| return False |
| |
| def _AnyChangesStaged(self): |
| """Return True if any local changes are staged in stable repo.""" |
| return bool(len(self._stable_repo_status)) |
| |
| def _StashChanges(self): |
| """Run 'git stash save' on stable repo.""" |
| # Only one level of stashing expected/supported. |
| self._RunGit(self._stable_repo, 'stash save', |
| redirect_stdout=True, combine_stdout_stderr=True) |
| self._stable_repo_stashed = True |
| |
| def _UnstashAnyChanges(self): |
| """Unstash any changes in stable repo.""" |
| # Only one level of stashing expected/supported. |
| if self._stable_repo_stashed: |
| self._RunGit(self._stable_repo, 'stash pop', |
| redirect_stdout=True, combine_stdout_stderr=True) |
| self._stable_repo_stashed = False |
| |
| def _DropAnyStashedChanges(self): |
| """Drop any stashed changes in stable repo.""" |
| # Only one level of stashing expected/supported. |
| if self._stable_repo_stashed: |
| self._RunGit(self._stable_repo, 'stash drop', |
| redirect_stdout=True, combine_stdout_stderr=True) |
| self._stable_repo_stashed = False |
| |
| def _CopyUpstreamPackage(self, upstream_cpv): |
| """Upgrades package in |upstream_cpv| to the version in |upstream_cpv|. |
| |
| Returns: |
| The upstream_cpv if the package was upgraded, None otherwise. |
| """ |
| if not upstream_cpv: |
| return None |
| |
| (cat, pkgname, version, rev) = portage.versions.catpkgsplit(upstream_cpv) |
| ebuild = upstream_cpv.replace(cat + '/', '') + '.ebuild' |
| catpkgsubdir = os.path.join(cat, pkgname) |
| pkgdir = os.path.join(self._stable_repo, catpkgsubdir) |
| upstream_pkgdir = os.path.join(self._upstream_repo, cat, pkgname) |
| |
| # If pkgdir already exists, remove everything except ebuilds that |
| # correspond to ebuilds that are also upstream. |
| if os.path.exists(pkgdir): |
| items = os.listdir(pkgdir) |
| for item in items: |
| src = os.path.join(upstream_pkgdir, item) |
| if not item.endswith('.ebuild') or not os.path.exists(src): |
| self._RunGit(pkgdir, 'rm -rf ' + item, redirect_stdout=True) |
| else: |
| os.makedirs(pkgdir) |
| |
| # Grab all non-ebuilds from upstream plus the specific ebuild requested. |
| # TODO: Selectively exclude files under the 'files' directory that clearly |
| # apply to other package versions. For example, 'files/foo-1.2.3-bar.patch' |
| # when upgrading to foo-3.2.1. Must do this carefully. |
| if os.path.exists(upstream_pkgdir): |
| items = os.listdir(upstream_pkgdir) |
| for item in items: |
| src = os.path.join(upstream_pkgdir, item) |
| dst = os.path.join(pkgdir, item) |
| if not item.endswith('.ebuild') or item == ebuild: |
| if os.path.isdir(src): |
| shutil.copytree(src, dst) |
| else: |
| shutil.copy2(src, dst) |
| self._RunGit(self._stable_repo, 'add ' + catpkgsubdir) |
| |
| return upstream_cpv |
| |
| def _GetPackageUpgradeState(self, info): |
| """Return state value for package in |info|.""" |
| # See whether this specific cpv exists upstream. |
| cpv = info['cpv'] |
| cpv_exists_upstream = bool(self._FindUpstreamCPV(cpv, unstable_ok=True)) |
| |
| # The value in info['cpv_cmp_upstream'] represents a comparison of cpv |
| # version and the upstream version, where: |
| # 0 = current, >0 = outdated, <0 = futuristic! |
| |
| # Convention is that anything not in portage overlay has been altered. |
| overlay = info['overlay'] |
| locally_patched = (overlay != NOT_APPLICABLE and |
| overlay != self.UPSTREAM_OVERLAY_NAME and |
| overlay != self.STABLE_OVERLAY_NAME) |
| locally_duplicated = locally_patched and cpv_exists_upstream |
| |
| # Gather status details for this package |
| if info['cpv_cmp_upstream'] is None: |
| # No upstream cpv to compare to (although this might include a |
| # a restriction to only stable upstream versions). This is concerning |
| # if the package is coming from 'portage' or 'portage-stable' overlays. |
| if locally_patched and info['latest_upstream_cpv'] is None: |
| state = utable.UpgradeTable.STATE_LOCAL_ONLY |
| else: |
| state = utable.UpgradeTable.STATE_UNKNOWN |
| elif info['cpv_cmp_upstream'] > 0: |
| if locally_duplicated: |
| state = utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_DUPLICATED |
| elif locally_patched: |
| state = utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_PATCHED |
| else: |
| state = utable.UpgradeTable.STATE_NEEDS_UPGRADE |
| elif locally_duplicated: |
| state = utable.UpgradeTable.STATE_DUPLICATED |
| elif locally_patched: |
| state = utable.UpgradeTable.STATE_PATCHED |
| else: |
| state = utable.UpgradeTable.STATE_CURRENT |
| |
| return state |
| |
| # TODO(mtennant): Generate output from finished table instead. |
| def _PrintPackageLine(self, info): |
| """Print a brief one-line report of package status.""" |
| upstream_cpv = info['upstream_cpv'] |
| if info['upgraded_cpv']: |
| if info['emerge_ok']: |
| action_stat = ' (UPGRADED, EMERGE WORKS)' |
| else: |
| action_stat = ' (UPGRADED, BUT EMERGE FAILS)' |
| else: |
| action_stat = '' |
| |
| up_stat = {utable.UpgradeTable.STATE_UNKNOWN: ' no package found upstream!', |
| utable.UpgradeTable.STATE_LOCAL_ONLY: ' (exists locally only)', |
| utable.UpgradeTable.STATE_NEEDS_UPGRADE: ' -> %s' % upstream_cpv, |
| utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_PATCHED: |
| ' <-> %s' % upstream_cpv, |
| utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_DUPLICATED: |
| ' (locally duplicated) <-> %s' % upstream_cpv, |
| utable.UpgradeTable.STATE_PATCHED: ' <- %s' % upstream_cpv, |
| utable.UpgradeTable.STATE_DUPLICATED: ' (locally duplicated)', |
| utable.UpgradeTable.STATE_CURRENT: ' (current)', |
| }[info['state']] |
| |
| print '[%s] %s%s%s' % (info['overlay'], info['cpv'], |
| up_stat, action_stat) |
| |
| def _AppendPackageRow(self, info): |
| """Add a row to status table for the package in |info|.""" |
| cpv = info['cpv'] |
| upgraded_cpv = info['upgraded_cpv'] |
| |
| upgraded_ver = '' |
| if upgraded_cpv: |
| upgraded_ver = Upgrader._GetVerRevFromCpv(upgraded_cpv) |
| |
| # "~" Prefix means the upgraded version is not stable on this arch. |
| if not info['upgraded_stable']: |
| upgraded_ver = '~' + upgraded_ver |
| |
| if not info['emerge_ok']: |
| upgraded_ver = '(emerge fails)' + upgraded_ver |
| |
| # Assemble 'depends on' and 'required by' strings. |
| depsstr = NOT_APPLICABLE |
| usedstr = NOT_APPLICABLE |
| if cpv: |
| deps_entry = self._deps_graph[cpv] |
| depslist = sorted(deps_entry['needs'].keys()) # dependencies |
| depsstr = ' '.join(depslist) |
| usedset = deps_entry['provides'] # used by |
| usedlist = sorted([p for p in usedset]) |
| usedstr = ' '.join(usedlist) |
| |
| stable_up_ver = Upgrader._GetVerRevFromCpv(info['stable_upstream_cpv']) |
| if not stable_up_ver: |
| stable_up_ver = NOT_APPLICABLE |
| latest_up_ver = Upgrader._GetVerRevFromCpv(info['latest_upstream_cpv']) |
| if not latest_up_ver: |
| latest_up_ver = NOT_APPLICABLE |
| |
| row = {self._curr_table.COL_PACKAGE: info['package'], |
| self._curr_table.COL_SLOT: info['slot'], |
| self._curr_table.COL_OVERLAY: info['overlay'], |
| self._curr_table.COL_CURRENT_VER: info['version_rev'], |
| self._curr_table.COL_STABLE_UPSTREAM_VER: stable_up_ver, |
| self._curr_table.COL_LATEST_UPSTREAM_VER: latest_up_ver, |
| self._curr_table.COL_STATE: info['state'], |
| self._curr_table.COL_DEPENDS_ON: depsstr, |
| self._curr_table.COL_USED_BY: usedstr, |
| self._curr_table.COL_TARGET: ' '.join(self._targets), |
| } |
| |
| # Only include if upgrade was involved. Table may not have this column |
| # if upgrade was not requested. |
| if upgraded_ver: |
| row[self._curr_table.COL_UPGRADED] = upgraded_ver |
| |
| self._curr_table.AppendRow(row) |
| |
| def _UpgradePackage(self, info): |
| """Gathers upgrade status for pkg, performs upgrade if requested. |
| |
| The upgrade is performed only if the package is outdated and --upgrade |
| is specified. |
| |
| The |info| hash must have the following entries: |
| package, category, package_name |
| cpv must exist but can be None |
| |
| Regardless, the following entries in |info| dict are filled in: |
| stable_upstream_cpv |
| latest_upstream_cpv |
| upstream_cpv (one of the above, depending on --stable-only option) |
| upgrade_cpv (if upgrade performed) |
| """ |
| cpv = info['cpv'] |
| catpkg = info['package'] |
| info['stable_upstream_cpv'] = self._FindUpstreamCPV(catpkg) |
| info['latest_upstream_cpv'] = self._FindUpstreamCPV(catpkg, |
| unstable_ok=True) |
| |
| # The upstream version can be either latest stable or latest overall, |
| # or specified explicitly by the user at the command line. In the latter |
| # case, 'upstream_cpv' will already be set. |
| if not info['upstream_cpv']: |
| if not self._unstable_ok: |
| info['upstream_cpv'] = info['stable_upstream_cpv'] |
| else: |
| info['upstream_cpv'] = info['latest_upstream_cpv'] |
| |
| # Perform the actual upgrade, if requested. |
| info['cpv_cmp_upstream'] = None |
| info['upgraded_cpv'] = None |
| if info['upstream_cpv']: |
| # cpv_cmp_upstream values: 0 = current, >0 = outdated, <0 = futuristic! |
| info['cpv_cmp_upstream'] = Upgrader._CmpCpv(info['upstream_cpv'], cpv) |
| |
| # Determine whether upgrade of this package is requested. |
| if self._PkgUpgradeRequested(info): |
| if self._PkgUpgradeStaged(info['upstream_cpv']): |
| oper.Info('Determined that %s is already staged.' % |
| info['upstream_cpv']) |
| info['upgraded_cpv'] = info['upstream_cpv'] |
| elif info['cpv_cmp_upstream'] > 0: |
| oper.Info('Copying %s from upstream.' % info['upstream_cpv']) |
| info['upgraded_cpv'] = self._CopyUpstreamPackage(info['upstream_cpv']) |
| |
| return bool(info['upgraded_cpv']) |
| |
| def _VerifyPackageUpgrade(self, info): |
| """Verify that the upgraded package in |info| passes checks.""" |
| # Verify that upgraded package can be emerged and save results. |
| (unmasked, stable) = self._GetMaskBits(info['upgraded_cpv']) |
| info['upgraded_unmasked'] = unmasked |
| info['upgraded_stable'] = stable |
| |
| (em_ok, em_cmd, em_out) = self._IsEmergeable(info['upgraded_cpv'], |
| info['upgraded_stable']) |
| info['emerge_ok'] = em_ok |
| info['emerge_cmd'] = em_cmd |
| info['emerge_output'] = em_out |
| |
| if not info['upgraded_unmasked']: |
| # If the package is masked, then emerge should have failed. |
| if info['emerge_ok']: |
| oper.Error("Emerge passed for masked package! Something " |
| "fishy here. Emerge output follows:") |
| print info['emerge_output'] |
| raise RuntimeError("Show this to the build team.") |
| |
| self._GiveMaskedError(info['upgraded_cpv'], info['emerge_output']) |
| |
| self._VerifyEbuildOverlay(info['upstream_cpv'], self.STABLE_OVERLAY_NAME, |
| info['upgraded_stable']) |
| |
| def _PackageReport(self, info): |
| """Report on whatever was done with package in |info|.""" |
| |
| info['state'] = self._GetPackageUpgradeState(info) |
| |
| # Print a quick summary of package status. |
| self._PrintPackageLine(info) |
| |
| # Add a row to status table for this package |
| self._AppendPackageRow(info) |
| |
| def _CreateCommitMessage(self, upgrade_lines, bug_num=None): |
| """Create appropriate git commit message for upgrades in |upgrade_lines|.""" |
| message = '' |
| upgrade_count = len(upgrade_lines) |
| upgrade_str = '\n'.join(upgrade_lines) |
| if upgrade_count == 1: |
| message = 'Upgrade the following Portage package\n\n%s\n' % upgrade_str |
| else: |
| message = ('Upgrade the following %d Portage packages\n\n%s\n' % |
| (upgrade_count, upgrade_str)) |
| |
| # The space before <fill-in> (at least for TEST=) fails pre-submit check, |
| # which is the intention here. |
| if bug_num: |
| message += '\nBUG=chromium-os:%s' % bug_num |
| else: |
| message += '\nBUG= <fill-in>' |
| message += '\nTEST= <fill-in>' |
| |
| return message |
| |
| def _AmendCommitMessage(self, upgrade_lines): |
| """Create git commit message combining |upgrade_lines| with last commit.""" |
| # First get the body of the last commit message. |
| git_cmd = 'show --pretty=format:"__BEGIN BODY__%n%b%n__END BODY__"' |
| result = self._RunGit(self._stable_repo, git_cmd, redirect_stdout=True) |
| match = re.search(r'__BEGIN BODY__\n(.+)__END BODY__', |
| result.output, re.DOTALL) |
| if match: |
| # Extract the upgrade_lines of last commit. |
| body = match.group(1) |
| for line in body.split('\n'): |
| if line: |
| upgrade_lines.append(line) |
| else: |
| break |
| |
| return self._CreateCommitMessage(upgrade_lines) |
| |
| def _GiveEmergeResults(self, infolist): |
| """Summarize emerge checks, raise RuntimeError if there was a problem.""" |
| emerge_ok = True |
| for info in infolist: |
| if info['upgraded_cpv']: |
| if info['emerge_ok']: |
| oper.Info('Confirmed %s can be emerged on %s after upgrade.' % |
| (info['upgraded_cpv'], self._curr_board)) |
| else: |
| emerge_ok = False |
| oper.Error('Unable to emerge %s after upgrade.\n' |
| 'The output of "%s" follows:' % |
| (info['upgraded_cpv'], info['emerge_cmd'])) |
| print info['emerge_output'] |
| |
| if not emerge_ok: |
| raise RuntimeError('Failed to complete upgrades on %s (see above). ' |
| 'Address the emerge errors before continuing.' % |
| self._curr_board) |
| |
| def _UpgradePackages(self, infolist): |
| """Given a list of cpv info maps, adds the upstream cpv to the infos.""" |
| self._curr_table.Clear() |
| |
| try: |
| for info in infolist: |
| if self._UpgradePackage(info): |
| self._upgrade_cnt += 1 |
| |
| # The verification of upgrades needs to happen after upgrades are done. |
| # The reason is that it cannot be guaranteed that infolist is ordered such |
| # that dependencies are satisified after each individual upgrade, because |
| # one or more of the packages may only exist upstream. |
| for info in infolist: |
| if info['upgraded_cpv']: |
| self._VerifyPackageUpgrade(info) |
| |
| self._PackageReport(info) |
| |
| self._GiveEmergeResults(infolist) |
| except RuntimeError as ex: |
| oper.Error(str(ex)) |
| |
| raise RuntimeError('\nTo reset all changes in %s now:\n' |
| ' cd %s; git reset --hard; cd -' % |
| (self._stable_repo, self._stable_repo)) |
| # Allow the changes to stay staged so that the user can attempt to address |
| # the issue (perhaps an edit to package.mask is required, or another |
| # package must also be upgraded). |
| |
| def _GenParallelEmergeArgv(self, args): |
| """Creates an argv for parallel_emerge using current options and |args|.""" |
| argv = ['--emptytree', '--pretend'] |
| if self._curr_board and self._curr_board != self.HOST_BOARD: |
| argv.append('--board=%s' % self._curr_board) |
| if not self._verbose: |
| argv.append('--quiet') |
| if self._rdeps: |
| argv.append('--root-deps=rdeps') |
| argv.extend(args) |
| |
| return argv |
| |
| def _SetPortTree(self, settings, trees): |
| """Set self._porttree from portage |settings| and |trees|.""" |
| root = settings["ROOT"] |
| self._porttree = trees[root]['porttree'] |
| |
| def _GetPortageDBAPI(self): |
| """Retrieve the Portage dbapi object, if available.""" |
| try: |
| return self._porttree.dbapi |
| except AttributeError: |
| return None |
| |
| def _CreateInfoFromCPV(self, cpv, cpv_key=None): |
| """Return a basic info object created from |cpv|.""" |
| info = {} |
| self._FillInfoFromCPV(info, cpv, cpv_key) |
| return info |
| |
| def _FillInfoFromCPV(self, info, cpv, cpv_key=None): |
| """Flesh out |info| from |cpv|.""" |
| pkg = Upgrader._GetCatPkgFromCpv(cpv) |
| (cat, pn) = pkg.split('/') |
| |
| info['cpv'] = None |
| info['upstream_cpv'] = None |
| |
| info['package'] = pkg |
| info['package_name'] = pn |
| info['category'] = cat |
| |
| if cpv_key: |
| info[cpv_key] = cpv |
| |
| def _GetCurrentVersions(self, target_infolist): |
| """Returns a list of pkg infos of the current package dependencies. |
| |
| The dependencies are taken from giving the 'package' values in each |
| info of |target_infolist| to (parallel_)emerge. |
| |
| The returned list is ordered such that the dependencies of any mentioned |
| package occur earlier in the list.""" |
| emerge_args = [info['package'] for info in target_infolist] |
| argv = self._GenParallelEmergeArgv(emerge_args) |
| |
| deps = parallel_emerge.DepGraphGenerator() |
| deps.Initialize(argv) |
| |
| deps_tree, deps_info = deps.GenDependencyTree() |
| |
| self._SetPortTree(deps.emerge.settings, deps.emerge.trees) |
| |
| self._deps_graph = deps.GenDependencyGraph(deps_tree, deps_info) |
| cpvlist = Upgrader._GetPreOrderDepGraph(self._deps_graph) |
| cpvlist.reverse() |
| |
| infolist = [] |
| for cpv in cpvlist: |
| # See if this cpv was in target_infolist |
| is_target = False |
| for info in target_infolist: |
| if cpv == info['cpv']: |
| infolist.append(info) |
| is_target = True |
| break |
| if not is_target: |
| infolist.append(self._CreateInfoFromCPV(cpv, cpv_key='cpv')) |
| |
| return infolist |
| |
| def _FinalizeLocalInfolist(self, orig_infolist): |
| """Filters and fleshes out |orig_infolist|, returns new list. |
| |
| Each info object is assumed to have entries for: |
| 'cpv', 'package', 'package_name', 'category' |
| """ |
| infolist = [] |
| for info in orig_infolist: |
| # No need to report or try to upgrade chromeos-base packages. |
| if info['category'] == 'chromeos-base': continue |
| |
| dbapi = self._GetPortageDBAPI() |
| ebuild_path = dbapi.findname2(info['cpv'])[0] |
| (overlay, cat, pn, pv) = self._SplitEBuildPath(ebuild_path) |
| ver_rev = pv.replace(pn + '-', '') |
| slot, = dbapi.aux_get(info['cpv'], ['SLOT']) |
| |
| info['slot'] = slot |
| info['overlay'] = overlay |
| info['version_rev'] = ver_rev |
| info['package_ver'] = pv |
| |
| infolist.append(info) |
| |
| return infolist |
| |
| def _FinalizeUpstreamInfolist(self, infolist): |
| """Adds missing values in each upstream info in |infolist|, returns list.""" |
| |
| for info in infolist: |
| info['slot'] = NOT_APPLICABLE |
| info['overlay'] = NOT_APPLICABLE |
| info['version_rev'] = NOT_APPLICABLE |
| info['package_ver'] = NOT_APPLICABLE |
| |
| return infolist |
| |
| def _ResolveAndVerifyArgs(self, args, upgrade_mode): |
| """Resolve |args| to full pkgs, and check validity of each. |
| |
| Each argument will be resolved to a full category/packagename, if possible, |
| by looking in both the local overlays and the upstream overlay. Any |
| argument that cannot be resolved will raise a RuntimeError. |
| |
| Arguments that specify a specific version of a package are only |
| allowed when |upgrade_mode| is True. |
| |
| The 'world' target is handled as a local package. |
| |
| Any errors will raise a RuntimeError. |
| |
| Return list of package infos, one for each argument. Each will have: |
| 'user_arg' = Original command line argument package was resolved from |
| 'package' = Resolved category/package_name |
| 'package_name' = package_name |
| 'category' = category (None for 'world' target) |
| Packages found in local overlays will also have: |
| 'cpv' = Current cpv ('world' for 'world' target) |
| Packages found upstream will also have: |
| 'upstream_cpv' = Upstream cpv |
| """ |
| infolist = [] |
| |
| for arg in args: |
| info = {'user_arg': arg} |
| |
| if arg == WORLD_TARGET: |
| # The 'world' target is a special case. Consider it a valid target |
| # locally, but not an upstream package. |
| info['package'] = arg |
| info['package_name'] = arg |
| info['category'] = None |
| info['cpv'] = arg |
| else: |
| catpkg = Upgrader._GetCatPkgFromCpv(arg) |
| verrev = Upgrader._GetVerRevFromCpv(arg) |
| |
| if verrev and not upgrade_mode: |
| raise RuntimeError("Specifying specific versions is only allowed " |
| "in upgrade mode. Don't know what to do with " |
| "'%s'." % arg) |
| |
| # Local cpv search ignores version in argument, if any. If version is |
| # in argument, though, it *must* be found upstream. |
| local_arg = catpkg if catpkg else arg |
| |
| local_cpv = self._FindCurrentCPV(local_arg) |
| upstream_cpv = self._FindUpstreamCPV(arg, self._unstable_ok) |
| |
| if not upstream_cpv and verrev: |
| # See if --unstable-ok is required for this upstream version. |
| if not self._unstable_ok and self._FindUpstreamCPV(arg, True): |
| raise RuntimeError("Upstream '%s' is unstable on %s. Re-run with " |
| "--unstable-ok option?" % (arg, self._curr_arch)) |
| else: |
| raise RuntimeError("Unable to find '%s' upstream on %s." % |
| (arg, self._curr_arch)) |
| |
| any_cpv = local_cpv if local_cpv else upstream_cpv |
| if any_cpv: |
| self._FillInfoFromCPV(info, any_cpv) |
| |
| if local_cpv and upstream_cpv: |
| oper.Info("Resolved '%s' to '%s' (local) and '%s' (upstream)" % |
| (arg, local_cpv, upstream_cpv)) |
| info['cpv'] = local_cpv |
| info['upstream_cpv'] = upstream_cpv |
| elif local_cpv: |
| oper.Info("Resolved '%s' to '%s' (local)" % |
| (arg, local_cpv)) |
| info['cpv'] = local_cpv |
| elif upstream_cpv: |
| oper.Info("Resolved '%s' to '%s' (upstream)" % |
| (arg, upstream_cpv)) |
| info['upstream_cpv'] = upstream_cpv |
| else: |
| msg = ("Unable to resolve '%s' as a package either local or upstream." |
| % arg) |
| if arg.find('/') < 0: |
| msg = msg + " Try specifying the full category/package_name." |
| |
| raise RuntimeError(msg) |
| |
| infolist.append(info) |
| |
| return infolist |
| |
| def PrepareToRun(self): |
| """Checkout upstream gentoo if necessary, and any other prep steps.""" |
| if not self._upstream: |
| if os.path.exists(self._upstream_repo): |
| # Previously created upstream cache can be re-used. Just update it. |
| oper.Info('Updating previously created upstream cache at %s.' % |
| self._upstream_repo) |
| self._RunGit(self._upstream_repo, 'remote update') |
| else: |
| # Create local copy of upstream gentoo. |
| oper.Info('Cloning origin/gentoo at %s as upstream reference.' % |
| self._upstream_repo) |
| root = os.path.dirname(self._upstream_repo) |
| name = os.path.basename(self._upstream_repo) |
| self._RunGit(root, 'clone %s %s' % (self.PORTAGE_GIT_URL, name)) |
| |
| # Create a README file to explain its presence. |
| fh = open(self._upstream_repo + '-README', 'w') |
| fh.write('Directory at %s is local copy of upstream ' |
| 'Gentoo/Portage packages. Used by cros_portage_upgrade.\n' |
| 'Feel free to delete if you want the space back.\n' % |
| self._upstream_repo) |
| fh.close() |
| |
| self._RunGit(self._upstream_repo, 'checkout %s' % self.ORIGIN_GENTOO, |
| redirect_stdout=True, combine_stdout_stderr=True) |
| |
| # An empty directory is needed to trick equery later. |
| self._emptydir = tempfile.mkdtemp() |
| |
| def RunCompleted(self): |
| """Undo any checkout of upstream gentoo if requested.""" |
| if not self._upstream: |
| if self._no_upstream_cache: |
| oper.Info('Removing upstream cache at %s as requested.' |
| % self._upstream_repo) |
| shutil.rmtree(self._upstream_repo) |
| |
| # Remove the README file, too. |
| readmepath = self._upstream_repo + '-README' |
| if os.path.exists(readmepath): |
| os.remove(readmepath) |
| else: |
| oper.Info('Keeping upstream cache at %s.' % self._upstream_repo) |
| |
| os.rmdir(self._emptydir) |
| |
| def CommitIsStaged(self): |
| """Return True if upgrades are staged and ready for a commit.""" |
| return bool(self._upgrade_cnt) |
| |
| def Commit(self): |
| """Commit whatever has been prepared in the stable repo.""" |
| # Trying to create commit message body lines that look like these: |
| # Upgraded foo/bar-1.2.3 to version 1.2.4 |
| # Upgraded foo/bar-1.2.3 to version 1.2.4 (arm) AND version 1.2.4-r1 (x86) |
| # Upgraded foo/bar-1.2.3 to version 1.2.4 (but emerge fails) |
| |
| commit_lines = [] # Lines for the body of the commit message |
| key_lines = [] # Lines needed in package.keywords |
| pkg_overlays = {} # Overlays for upgraded packages in non-portage overlays. |
| |
| # Assemble hash of COL_UPGRADED column names by arch. |
| upgraded_cols = {} |
| for arch in self._master_archs: |
| tmp_col = utable.UpgradeTable.COL_UPGRADED |
| col = utable.UpgradeTable.GetColumnName(tmp_col, arch) |
| upgraded_cols[arch] = col |
| |
| table = self._master_table |
| for row in table: |
| pkg = row[table.COL_PACKAGE] |
| pkg_commit_line = None |
| pkg_keys = {} # key=cpv, value=set of arches that need keyword overwrite |
| |
| # First determine how many unique upgraded versions there are. |
| upgraded_versset = set() |
| upgraded_verslist = [] |
| for arch in self._master_archs: |
| upgraded_ver = row[upgraded_cols[arch]] |
| if upgraded_ver: |
| # Remove ~ prefix if found, and remember as unstable. |
| upgraded_stable = True |
| if upgraded_ver[0] == '~': |
| upgraded_stable = False |
| upgraded_ver = upgraded_ver[1:] |
| |
| # This package has been upgraded for this arch. |
| upgraded_versset.add(upgraded_ver) |
| upgraded_verslist.append(upgraded_ver) |
| |
| # Is this upgraded version unstable for this arch? If so, save |
| # arch under upgraded_cpv as one that will need a package.keywords |
| # entry. |
| if not upgraded_stable: |
| upgraded_cpv = pkg + '-' + upgraded_ver |
| cpv_key = pkg_keys.get(upgraded_cpv, set()) |
| cpv_key.add(arch) |
| pkg_keys[upgraded_cpv] = cpv_key |
| |
| # Save the overlay this package is originally from, if the overlay |
| # is not a Portage overlay (e.g. chromiumos-overlay). |
| ovrly_col = utable.UpgradeTable.COL_OVERLAY |
| ovrly_col = utable.UpgradeTable.GetColumnName(ovrly_col, arch) |
| ovrly = row[ovrly_col] |
| if (ovrly != NOT_APPLICABLE and |
| ovrly != self.UPSTREAM_OVERLAY_NAME and |
| ovrly != self.STABLE_OVERLAY_NAME): |
| pkg_overlays[pkg] = ovrly |
| |
| if len(upgraded_versset) == 1: |
| # Upgrade is the same across all archs. |
| upgraded_ver = upgraded_versset.pop() |
| arch_str = ', '.join(sorted(self._master_archs)) |
| pkg_commit_line = ('Upgraded %s to version %s on %s' % |
| (pkg, upgraded_ver, arch_str)) |
| |
| elif len(upgraded_versset) > 1: |
| # Iterate again, and specify arch for each upgraded version. |
| tokens = [] |
| for upgraded_ver in upgraded_verslist: |
| tokens.append('%s on %s' % (upgraded_ver, arch)) |
| pkg_commit_line = ('Upgraded %s to versions %s' % |
| (pkg, ' AND '.join(tokens))) |
| |
| if pkg_commit_line: |
| commit_lines.append(pkg_commit_line) |
| if len(pkg_keys): |
| for upgraded_cpv in pkg_keys: |
| cpv_key = pkg_keys[upgraded_cpv] |
| key_lines.append('=%s %s' % |
| (upgraded_cpv, |
| ' '.join(cpv_key))) |
| |
| if commit_lines: |
| if self._amend: |
| message = self._AmendCommitMessage(commit_lines) |
| self._RunGit(self._stable_repo, "commit --amend -am '%s'" % message) |
| else: |
| message = self._CreateCommitMessage(commit_lines) |
| self._RunGit(self._stable_repo, "commit -am '%s'" % message) |
| |
| oper.Warning('\n' |
| 'Upgrade changes committed (see above),' |
| ' but message needs edit BY YOU:\n' |
| ' cd %s; git commit --amend; cd -' % |
| self._stable_repo) |
| # See if any upgraded packages are in non-portage overlays now, meaning |
| # they probably require a patch and should not go into portage-stable. |
| if pkg_overlays: |
| lines = ['%s [%s]' % (p, pkg_overlays[p]) for p in pkg_overlays] |
| oper.Warning('\n' |
| 'The following packages were coming from a non-portage' |
| ' overlay, which means they were probably patched.\n' |
| 'You should consider whether the upgraded package' |
| ' needs the same patches applied now.\n' |
| 'If so, do not commit these changes in portage-stable.' |
| ' Instead, copy them to the applicable overlay dir.\n' |
| '%s' % |
| '\n'.join(lines)) |
| oper.Info('\n' |
| 'To remove any individual file above from commit do:\n' |
| ' cd %s; git reset HEAD~ <filepath>; rm <filepath>;' |
| ' git commit --amend; cd -' % |
| self._stable_repo) |
| |
| if key_lines: |
| oper.Warning('\n' |
| 'Because one or more unstable versions are involved' |
| ' you must add line(s) like the following to\n' |
| ' %s:\n%s\n' |
| 'This is needed before testing, and should be pushed' |
| ' in a separate changelist BEFORE the\n' |
| 'the changes in portage-stable.' % |
| (self._GetPkgKeywordsFile(), |
| '\n'.join(key_lines))) |
| |
| oper.Info('\n' |
| 'If you wish to undo all the changes to %s:\n' |
| ' cd %s; git reset --hard HEAD~; cd -' % |
| (self.STABLE_OVERLAY_NAME, self._stable_repo)) |
| |
| def PreRunChecks(self): |
| """Run any board-independent validation checks before Run is called.""" |
| # Upfront check(s) if upgrade is requested. |
| if self._upgrade or self._upgrade_deep: |
| # Stable source must be on branch. |
| self._CheckStableRepoOnBranch() |
| |
| def RunBoard(self, board): |
| """Runs the upgrader based on the supplied options and arguments. |
| |
| Currently just lists all package dependencies in pre-order along with |
| potential upgrades.""" |
| # Preserve status report for entire stable repo (output of 'git status -s'). |
| self._SaveStatusOnStableRepo() |
| |
| self._porttree = None |
| self._deps_graph = None |
| |
| self._curr_board = board |
| self._curr_arch = Upgrader._FindBoardArch(board) |
| upgrade_mode = self._upgrade or self._upgrade_deep |
| self._curr_table = utable.UpgradeTable(self._curr_arch, |
| upgrade=upgrade_mode, |
| name=board) |
| |
| if self._AnyChangesStaged(): |
| self._StashChanges() |
| |
| try: |
| target_infolist = self._ResolveAndVerifyArgs(self._args, upgrade_mode) |
| upstream_only_infolist = [i for i in target_infolist if not i['cpv']] |
| if not upgrade_mode and upstream_only_infolist: |
| # This means that not all arguments were found in local source, which is |
| # only allowed in upgrade mode. |
| msg = ("The following packages were not found in current overlays" |
| " (but they do exist upstream):\n%s" % |
| '\n'.join([info['arg'] for info in upstream_only_infolist])) |
| raise RuntimeError(msg) |
| |
| local_target_infolist = [i for i in target_infolist if i['cpv']] |
| full_infolist = self._GetCurrentVersions(local_target_infolist) |
| full_infolist = self._FinalizeLocalInfolist(full_infolist) |
| |
| # Append any command line targets that were not found in current overlays. |
| # The idea is that they will still be found upstream for upgrading. |
| if upgrade_mode: |
| tmp_list = self._FinalizeUpstreamInfolist(upstream_only_infolist) |
| full_infolist = full_infolist + tmp_list |
| |
| self._UnstashAnyChanges() |
| self._UpgradePackages(full_infolist) |
| |
| finally: |
| self._DropAnyStashedChanges() |
| |
| # Merge tables together after each run. |
| self._master_cnt += 1 |
| self._master_archs.add(self._curr_arch) |
| if self._master_table: |
| tables = [self._master_table, self._curr_table] |
| self._master_table = mps.MergeTables(tables) |
| else: |
| self._master_table = self._curr_table |
| self._master_table._arch = None |
| |
| def WriteTableFiles(self, csv=None): |
| """Write |self._master_table| to |csv| file, if requested.""" |
| |
| # Sort the table by package name, then slot |
| def PkgSlotSort(row): |
| return (row[self._master_table.COL_PACKAGE], |
| row[self._master_table.COL_SLOT]) |
| self._master_table.Sort(PkgSlotSort) |
| |
| if csv: |
| filehandle = open(csv, 'w') |
| oper.Info('Writing package status as csv to %s' % csv) |
| self._master_table.WriteCSV(filehandle) |
| filehandle.close() |
| |
| def _BoardIsSetUp(board): |
| """Return true if |board| has been setup.""" |
| return os.path.isdir('/build/%s' % board) |
| |
| def main(): |
| """Main function.""" |
| usage = 'Usage: %prog [options] packages...' |
| epilog = ('\n' |
| 'There are essentially two "modes": status report mode and ' |
| 'upgrade mode.\nStatus report mode is the default; upgrade ' |
| 'mode is enabled by either --upgrade or --upgrade-deep.\n' |
| '\n' |
| 'In either mode, packages can be specified in any manner ' |
| 'commonly accepted by Portage tools. For example:\n' |
| ' category/package_name\n' |
| ' package_name\n' |
| ' category/package_name-version (upgrade mode only)\n' |
| '\n' |
| 'Status report mode will report on the status of the specified ' |
| 'packages relative to upstream,\nwithout making any changes. ' |
| 'In this mode, the specified packages are often high-level\n' |
| 'targets such as "chromeos" or "chromeos-dev". ' |
| 'The --to-csv option is often used in this mode.\n' |
| 'The --unstable-ok option in this mode will make ' |
| 'the upstream comparison (e.g. "needs update") be\n' |
| 'relative to the latest upstream version, stable or not.\n' |
| '\n' |
| 'Upgrade mode will attempt to upgrade the specified ' |
| 'packages to one of the following versions:\n' |
| '1) The version specified in argument (e.g. foo/bar-1.2.3)\n' |
| '2) The latest stable version upstream (the default)\n' |
| '3) The latest overall version upstream (with --unstable-ok)\n' |
| '\n' |
| 'Unlike with --upgrade, if --upgrade-deep is specified, ' |
| 'then the package dependencies will also be upgraded.\n' |
| 'In upgrade mode, it is ok if the specified packages only ' |
| 'exist upstream.\n' |
| '\n' |
| 'Status report mode examples:\n' |
| '> cros_portage_upgrade --board=arm-generic:x86-generic ' |
| '--to-csv=cros-aebl.csv chromeos\n' |
| '> cros_portage_upgrade --unstable-ok --board=x86-mario ' |
| '--to-csv=cros_test-mario chromeos chromeos-dev chromeos-test\n' |
| 'Upgrade mode examples:\n' |
| '> cros_portage_upgrade --board=arm-generic:x86-generic ' |
| '--upgrade sys-devel/gdb virtual/yacc\n' |
| '> cros_portage_upgrade --unstable-ok --board=x86-mario ' |
| '--upgrade-deep gdata\n' |
| '> cros_portage_upgrade --board=x86-generic --upgrade ' |
| 'media-libs/libpng-1.2.45\n' |
| '\n' |
| ) |
| |
| class MyOptParser(optparse.OptionParser): |
| """Override default epilog formatter, which strips newlines.""" |
| def format_epilog(self, formatter): |
| return self.epilog |
| |
| parser = MyOptParser(usage=usage, epilog=epilog) |
| parser.add_option('--amend', dest='amend', action='store_true', |
| default=False, |
| help="Amend existing commit when doing upgrade.") |
| parser.add_option('--board', dest='board', type='string', action='store', |
| default=None, help="Target board(s), colon-separated") |
| parser.add_option('--host', dest='host', action='store_true', |
| default=False, |
| help="Host target pseudo-board") |
| parser.add_option('--no-upstream-cache', dest='no_upstream_cache', |
| action='store_true', default=False, |
| help="Do not preserve cached upstream for future runs") |
| parser.add_option('--rdeps', dest='rdeps', action='store_true', |
| default=False, |
| help="Use runtime dependencies only") |
| parser.add_option('--srcroot', dest='srcroot', type='string', action='store', |
| default='%s/trunk/src' % os.environ['HOME'], |
| help="Path to root src directory [default: '%default']") |
| parser.add_option('--to-csv', dest='csv_file', type='string', action='store', |
| default=None, help="File to write csv-formatted results to") |
| parser.add_option('--upgrade', dest='upgrade', |
| action='store_true', default=False, |
| help="Upgrade target package(s) only") |
| parser.add_option('--upgrade-deep', dest='upgrade_deep', action='store_true', |
| default=False, |
| help="Upgrade target package(s) and all dependencies") |
| parser.add_option('--upstream', dest='upstream', type='string', |
| action='store', default=None, |
| help="Latest upstream repo location [default: '%default']") |
| parser.add_option('--unstable-ok', dest='unstable_ok', action='store_true', |
| default=False, |
| help="Use latest upstream ebuild, stable or not") |
| parser.add_option('--verbose', dest='verbose', action='store_true', |
| default=False, |
| help="Enable verbose output (for debugging)") |
| |
| (options, args) = parser.parse_args() |
| |
| if (options.verbose): logging.basicConfig(level=logging.DEBUG) |
| |
| if not options.board and not options.host: |
| parser.print_help() |
| oper.Die('Board (or host) is required.') |
| |
| if not args: |
| parser.print_help() |
| oper.Die('No packages provided.') |
| |
| # The --upgrade and --upgrade-deep options are mutually exclusive. |
| if options.upgrade_deep and options.upgrade: |
| parser.print_help() |
| oper.Die('The --upgrade and --upgrade-deep options ' + |
| 'are mutually exclusive.') |
| |
| # The --upstream and --no-upstream-cache options are mutually exclusive. |
| if options.upstream and options.no_upstream_cache: |
| parser.print_help() |
| oper.Die('The --upstream and --no-upstream-cache options ' + |
| 'are mutually exclusive.') |
| |
| # If upstream portage is provided, verify that it is a valid directory. |
| if options.upstream and not os.path.isdir(options.upstream): |
| parser.print_help() |
| oper.Die('Argument to --upstream must be a valid directory.') |
| |
| # If --to-csv given verify file can be opened for write. |
| if options.csv_file: |
| try: |
| fh = open(options.csv_file, 'w') |
| fh.close() |
| except IOError as ex: |
| parser.print_help() |
| oper.Die('Unable to open %s for writing: %s' % (options.csv_file, |
| str(ex))) |
| |
| upgrader = Upgrader(options, args) |
| upgrader.PreRunChecks() |
| |
| # Automatically handle board 'host' as 'amd64-host'. |
| boards = [] |
| if options.board: |
| boards = options.board.split(':') |
| boards = [Upgrader.HOST_BOARD if b == 'host' else b for b in boards] |
| |
| # Make sure host pseudo-board is run first. |
| if options.host and Upgrader.HOST_BOARD not in boards: |
| boards.insert(0, Upgrader.HOST_BOARD) |
| elif Upgrader.HOST_BOARD in boards: |
| boards = [b for b in boards if b != Upgrader.HOST_BOARD] |
| boards.insert(0, Upgrader.HOST_BOARD) |
| |
| # Check that all boards have been setup first. |
| for board in boards: |
| if (board != Upgrader.HOST_BOARD and not _BoardIsSetUp(board)): |
| parser.print_help() |
| oper.Die('You must setup the %s board first.' % board) |
| |
| passed = True |
| try: |
| upgrader.PrepareToRun() |
| |
| for board in boards: |
| oper.Info('Running with board %s' % board) |
| upgrader.RunBoard(board) |
| except RuntimeError as ex: |
| passed = False |
| oper.Error(str(ex)) |
| |
| finally: |
| upgrader.RunCompleted() |
| |
| if not passed: |
| oper.Die("Failed with above errors.") |
| |
| if upgrader.CommitIsStaged(): |
| upgrader.Commit() |
| |
| # TODO(mtennant): Move stdout output to here, rather than as-we-go. That |
| # way it won't come out for each board. Base it on contents of final table. |
| # Make verbose-dependent? |
| |
| upgrader.WriteTableFiles(csv=options.csv_file) |
| |
| |
| if __name__ == '__main__': |
| main() |
