| #!/usr/bin/python |
| |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """This module uprevs a given package's ebuild to the next revision.""" |
| |
| import optparse |
| import os |
| import sys |
| |
| from chromite.buildbot import constants |
| from chromite.buildbot import portage_utilities |
| from chromite.lib import cros_build_lib |
| from chromite.lib import git |
| from chromite.lib import osutils |
| from chromite.lib import parallel |
| |
| |
| # Commit message for uprevving Portage packages. |
| _GIT_COMMIT_MESSAGE = 'Marking 9999 ebuild for %s as stable.' |
| |
| # Dictionary of valid commands with usage information. |
| COMMAND_DICTIONARY = { |
| 'commit': 'Marks given ebuilds as stable locally', |
| 'push': 'Pushes previous marking of ebuilds to remote repo', |
| } |
| |
| |
| # ======================= Global Helper Functions ======================== |
| |
| |
| def CleanStalePackages(boards, package_atoms): |
| """Cleans up stale package info from a previous build. |
| |
| Args: |
| boards: Boards to clean the packages from. |
| package_atoms: A list of package atoms to unmerge. |
| """ |
| if package_atoms: |
| cros_build_lib.Info('Cleaning up stale packages %s.' % package_atoms) |
| |
| # First unmerge all the packages for a board, then eclean it. |
| # We need these two steps to run in order (unmerge/eclean), |
| # but we can let all the boards run in parallel. |
| def _CleanStalePackages(board): |
| if board: |
| suffix = '-' + board |
| runcmd = cros_build_lib.RunCommand |
| else: |
| suffix = '' |
| runcmd = cros_build_lib.SudoRunCommand |
| |
| emerge, eclean = 'emerge' + suffix, 'eclean' + suffix |
| if not osutils.FindMissingBinaries([emerge, eclean]): |
| # If nothing was found to be unmerged, emerge will exit(1). |
| result = runcmd([emerge, '-q', '--unmerge'] + package_atoms, |
| extra_env={'CLEAN_DELAY': '0'}, error_code_ok=True) |
| if not result.returncode in (0, 1): |
| raise cros_build_lib.RunCommandError('unexpected error', result) |
| runcmd([eclean, '-d', 'packages'], |
| redirect_stdout=True, redirect_stderr=True) |
| |
| tasks = [] |
| for board in boards: |
| tasks.append([board]) |
| tasks.append([None]) |
| |
| parallel.RunTasksInProcessPool(_CleanStalePackages, tasks) |
| |
| |
| # TODO(build): This code needs to be gutted and rebased to cros_build_lib. |
| def _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd): |
| """Returns true if there are local commits.""" |
| current_branch = git.GetCurrentBranch(cwd) |
| |
| if current_branch != stable_branch: |
| return False |
| output = git.RunGit( |
| cwd, ['rev-parse', 'HEAD', tracking_branch]).output.split() |
| return output[0] != output[1] |
| |
| |
| def _CheckSaneArguments(command, options): |
| """Checks to make sure the flags are sane. Dies if arguments are not sane.""" |
| if not command in COMMAND_DICTIONARY.keys(): |
| _PrintUsageAndDie('%s is not a valid command' % command) |
| if not options.packages and command == 'commit' and not options.all: |
| _PrintUsageAndDie('Please specify at least one package') |
| if options.boards: |
| cros_build_lib.AssertInsideChroot() |
| if not os.path.isdir(options.srcroot): |
| _PrintUsageAndDie('srcroot is not a valid path') |
| options.srcroot = os.path.abspath(options.srcroot) |
| |
| |
| def _PrintUsageAndDie(error_message=''): |
| """Prints optional error_message the usage and returns an error exit code.""" |
| command_usage = 'Commands: \n' |
| # Add keys and usage information from dictionary. |
| commands = sorted(COMMAND_DICTIONARY.keys()) |
| for command in commands: |
| command_usage += ' %s: %s\n' % (command, COMMAND_DICTIONARY[command]) |
| commands_str = '|'.join(commands) |
| cros_build_lib.Warning('Usage: %s FLAGS [%s]\n\n%s' % ( |
| sys.argv[0], commands_str, command_usage)) |
| if error_message: |
| cros_build_lib.Die(error_message) |
| else: |
| sys.exit(1) |
| |
| |
| # ======================= End Global Helper Functions ======================== |
| |
| |
| def PushChange(stable_branch, tracking_branch, dryrun, cwd): |
| """Pushes commits in the stable_branch to the remote git repository. |
| |
| Pushes local commits from calls to CommitChange to the remote git |
| repository specified by current working directory. If changes are |
| found to commit, they will be merged to the merge branch and pushed. |
| In that case, the local repository will be left on the merge branch. |
| |
| Args: |
| stable_branch: The local branch with commits we want to push. |
| tracking_branch: The tracking branch of the local branch. |
| dryrun: Use git push --dryrun to emulate a push. |
| cwd: The directory to run commands in. |
| |
| Raises: |
| OSError: Error occurred while pushing. |
| """ |
| if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd): |
| cros_build_lib.Info('No work found to push in %s. Exiting', cwd) |
| return |
| |
| # For the commit queue, our local branch may contain commits that were |
| # just tested and pushed during the CommitQueueCompletion stage. Sync |
| # and rebase our local branch on top of the remote commits. |
| remote, push_branch = git.GetTrackingBranch(cwd, for_push=True) |
| git.SyncPushBranch(cwd, remote, push_branch) |
| |
| # Check whether any local changes remain after the sync. |
| if not _DoWeHaveLocalCommits(stable_branch, push_branch, cwd): |
| cros_build_lib.Info('All changes already pushed for %s. Exiting', cwd) |
| return |
| |
| # Add a failsafe check here. Only CLs from the 'chrome-bot' user should |
| # be involved here. If any other CLs are found then complain. |
| # In dryruns extra CLs are normal, though, and can be ignored. |
| bad_cl_cmd = ['log', '--format=short', '--perl-regexp', |
| '--author', '^(?!chrome-bot)', '%s..%s' % ( |
| push_branch, stable_branch)] |
| bad_cls = git.RunGit(cwd, bad_cl_cmd).output |
| if bad_cls.strip() and not dryrun: |
| cros_build_lib.Error('The Uprev stage found changes from users other' |
| ' than chrome-bot:\n\n%s', bad_cls) |
| raise AssertionError('Unexpected CLs found during uprev stage.') |
| |
| description = git.RunGit(cwd, |
| ['log', '--format=format:%s%n%n%b', '%s..%s' % ( |
| push_branch, stable_branch)]).output |
| description = 'Marking set of ebuilds as stable\n\n%s' % description |
| cros_build_lib.Info('For %s, using description %s', cwd, description) |
| git.CreatePushBranch(constants.MERGE_BRANCH, cwd) |
| git.RunGit(cwd, ['merge', '--squash', stable_branch]) |
| git.RunGit(cwd, ['commit', '-m', description]) |
| git.RunGit(cwd, ['config', 'push.default', 'tracking']) |
| git.PushWithRetry(constants.MERGE_BRANCH, cwd, dryrun=dryrun) |
| |
| |
| class GitBranch(object): |
| """Wrapper class for a git branch.""" |
| |
| def __init__(self, branch_name, tracking_branch, cwd): |
| """Sets up variables but does not create the branch. |
| |
| Args: |
| branch_name: The name of the branch. |
| tracking_branch: The associated tracking branch. |
| cwd: The git repository to work in. |
| """ |
| self.branch_name = branch_name |
| self.tracking_branch = tracking_branch |
| self.cwd = cwd |
| |
| def CreateBranch(self): |
| self.Checkout() |
| |
| def Checkout(self, branch=None): |
| """Function used to check out to another GitBranch.""" |
| if not branch: |
| branch = self.branch_name |
| if branch == self.tracking_branch or self.Exists(branch): |
| git_cmd = ['git', 'checkout', '-f', branch] |
| else: |
| git_cmd = ['repo', 'start', branch, '.'] |
| cros_build_lib.RunCommandCaptureOutput(git_cmd, print_cmd=False, |
| cwd=self.cwd) |
| |
| def Exists(self, branch=None): |
| """Returns True if the branch exists.""" |
| if not branch: |
| branch = self.branch_name |
| branches = git.RunGit(self.cwd, ['branch']).output |
| return branch in branches.split() |
| |
| |
| def main(_argv): |
| parser = optparse.OptionParser('cros_mark_as_stable OPTIONS packages') |
| parser.add_option('--all', action='store_true', |
| help='Mark all packages as stable.') |
| parser.add_option('-b', '--boards', default='', |
| help='Colon-separated list of boards') |
| parser.add_option('--drop_file', |
| help='File to list packages that were revved.') |
| parser.add_option('--dryrun', action='store_true', |
| help='Passes dry-run to git push if pushing a change.') |
| parser.add_option('-o', '--overlays', |
| help='Colon-separated list of overlays to modify.') |
| parser.add_option('-p', '--packages', |
| help='Colon separated list of packages to rev.') |
| parser.add_option('-r', '--srcroot', |
| default=os.path.join(constants.SOURCE_ROOT, 'src'), |
| help='Path to root src directory.') |
| parser.add_option('--verbose', action='store_true', |
| help='Prints out debug info.') |
| (options, args) = parser.parse_args() |
| |
| portage_utilities.EBuild.VERBOSE = options.verbose |
| |
| if len(args) != 1: |
| _PrintUsageAndDie('Must specify a valid command [commit, push]') |
| |
| command = args[0] |
| package_list = None |
| if options.packages: |
| package_list = options.packages.split(':') |
| |
| _CheckSaneArguments(command, options) |
| if options.overlays: |
| overlays = {} |
| for path in options.overlays.split(':'): |
| if not os.path.isdir(path): |
| cros_build_lib.Die('Cannot find overlay: %s' % path) |
| overlays[path] = [] |
| else: |
| cros_build_lib.Warning('Missing --overlays argument') |
| overlays = { |
| '%s/private-overlays/chromeos-overlay' % options.srcroot: [], |
| '%s/third_party/chromiumos-overlay' % options.srcroot: [] |
| } |
| |
| manifest = git.ManifestCheckout.Cached(options.srcroot) |
| |
| if command == 'commit': |
| portage_utilities.BuildEBuildDictionary(overlays, options.all, package_list) |
| |
| # Contains the array of packages we actually revved. |
| revved_packages = [] |
| new_package_atoms = [] |
| |
| # Slight optimization hack: process the chromiumos overlay before any other |
| # cros-workon overlay first so we can do background cache generation in it. |
| # A perfect solution would walk all the overlays, figure out any dependencies |
| # between them (with layout.conf), and then process them in dependency order. |
| # However, this operation isn't slow enough to warrant that level of |
| # complexity, so we'll just special case the main overlay. |
| # |
| # Similarly, generate the cache in the portage-stable tree asap. We know |
| # we won't have any cros-workon packages in there, so generating the cache |
| # is the only thing it'll be doing. The chromiumos overlay instead might |
| # have revbumping to do before it can generate the cache. |
| keys = overlays.keys() |
| for overlay in ('/third_party/chromiumos-overlay', |
| '/third_party/portage-stable'): |
| for k in keys: |
| if k.endswith(overlay): |
| keys.remove(k) |
| keys.insert(0, k) |
| break |
| |
| with parallel.BackgroundTaskRunner(portage_utilities.RegenCache) as queue: |
| for overlay in keys: |
| ebuilds = overlays[overlay] |
| if not os.path.isdir(overlay): |
| cros_build_lib.Warning('Skipping %s' % overlay) |
| continue |
| |
| # Note we intentionally work from the non push tracking branch; |
| # everything built thus far has been against it (meaning, http mirrors), |
| # thus we should honor that. During the actual push, the code switches |
| # to the correct urls, and does an appropriate rebasing. |
| tracking_branch = git.GetTrackingBranchViaManifest( |
| overlay, manifest=manifest)[1] |
| |
| if command == 'push': |
| PushChange(constants.STABLE_EBUILD_BRANCH, tracking_branch, |
| options.dryrun, cwd=overlay) |
| elif command == 'commit': |
| existing_commit = git.GetGitRepoRevision(overlay) |
| work_branch = GitBranch(constants.STABLE_EBUILD_BRANCH, tracking_branch, |
| cwd=overlay) |
| work_branch.CreateBranch() |
| if not work_branch.Exists(): |
| cros_build_lib.Die('Unable to create stabilizing branch in %s' % |
| overlay) |
| |
| # In the case of uprevving overlays that have patches applied to them, |
| # include the patched changes in the stabilizing branch. |
| git.RunGit(overlay, ['rebase', existing_commit]) |
| |
| messages = [] |
| for ebuild in ebuilds: |
| if options.verbose: |
| cros_build_lib.Info('Working on %s', ebuild.package) |
| try: |
| new_package = ebuild.RevWorkOnEBuild(options.srcroot, manifest) |
| if new_package: |
| revved_packages.append(ebuild.package) |
| new_package_atoms.append('=%s' % new_package) |
| messages.append(_GIT_COMMIT_MESSAGE % ebuild.package) |
| except (OSError, IOError): |
| cros_build_lib.Warning( |
| 'Cannot rev %s\n' |
| 'Note you will have to go into %s ' |
| 'and reset the git repo yourself.' % (ebuild.package, overlay)) |
| raise |
| |
| if messages: |
| portage_utilities.EBuild.CommitChange('\n\n'.join(messages), overlay) |
| |
| if cros_build_lib.IsInsideChroot(): |
| # Regenerate caches if need be. We do this all the time to |
| # catch when users make changes without updating cache files. |
| queue.put([overlay]) |
| |
| if command == 'commit': |
| if cros_build_lib.IsInsideChroot(): |
| CleanStalePackages(options.boards.split(':'), new_package_atoms) |
| if options.drop_file: |
| osutils.WriteFile(options.drop_file, ' '.join(revved_packages)) |
