| # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Gclient-specific SCM-specific operations.""" |
| |
| from __future__ import print_function |
| |
| import collections |
| import contextlib |
| import errno |
| import json |
| import logging |
| import os |
| import posixpath |
| import re |
| import sys |
| import tempfile |
| import threading |
| import traceback |
| |
| try: |
| import urlparse |
| except ImportError: # For Py3 compatibility |
| import urllib.parse as urlparse |
| |
| import gclient_utils |
| import git_cache |
| import scm |
| import shutil |
| import subprocess2 |
| |
| |
| THIS_FILE_PATH = os.path.abspath(__file__) |
| |
| GSUTIL_DEFAULT_PATH = os.path.join( |
| os.path.dirname(os.path.abspath(__file__)), 'gsutil.py') |
| |
| |
| class NoUsableRevError(gclient_utils.Error): |
| """Raised if requested revision isn't found in checkout.""" |
| |
| |
| class DiffFiltererWrapper(object): |
| """Simple base class which tracks which file is being diffed and |
| replaces instances of its file name in the original and |
| working copy lines of the git diff output.""" |
| index_string = None |
| original_prefix = "--- " |
| working_prefix = "+++ " |
| |
| def __init__(self, relpath, print_func): |
| # Note that we always use '/' as the path separator to be |
| # consistent with cygwin-style output on Windows |
| self._relpath = relpath.replace("\\", "/") |
| self._current_file = None |
| self._print_func = print_func |
| |
| def SetCurrentFile(self, current_file): |
| self._current_file = current_file |
| |
| @property |
| def _replacement_file(self): |
| return posixpath.join(self._relpath, self._current_file) |
| |
| def _Replace(self, line): |
| return line.replace(self._current_file, self._replacement_file) |
| |
| def Filter(self, line): |
| if (line.startswith(self.index_string)): |
| self.SetCurrentFile(line[len(self.index_string):]) |
| line = self._Replace(line) |
| else: |
| if (line.startswith(self.original_prefix) or |
| line.startswith(self.working_prefix)): |
| line = self._Replace(line) |
| self._print_func(line) |
| |
| |
| class GitDiffFilterer(DiffFiltererWrapper): |
| index_string = "diff --git " |
| |
| def SetCurrentFile(self, current_file): |
| # Get filename by parsing "a/<filename> b/<filename>" |
| self._current_file = current_file[:(len(current_file)/2)][2:] |
| |
| def _Replace(self, line): |
| return re.sub("[a|b]/" + self._current_file, self._replacement_file, line) |
| |
| |
| # SCMWrapper base class |
| |
| class SCMWrapper(object): |
| """Add necessary glue between all the supported SCM. |
| |
| This is the abstraction layer to bind to different SCM. |
| """ |
| def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None, |
| out_cb=None, print_outbuf=False): |
| self.url = url |
| self._root_dir = root_dir |
| if self._root_dir: |
| self._root_dir = self._root_dir.replace('/', os.sep) |
| self.relpath = relpath |
| if self.relpath: |
| self.relpath = self.relpath.replace('/', os.sep) |
| if self.relpath and self._root_dir: |
| self.checkout_path = os.path.join(self._root_dir, self.relpath) |
| if out_fh is None: |
| out_fh = sys.stdout |
| self.out_fh = out_fh |
| self.out_cb = out_cb |
| self.print_outbuf = print_outbuf |
| |
| def Print(self, *args, **kwargs): |
| kwargs.setdefault('file', self.out_fh) |
| if kwargs.pop('timestamp', True): |
| self.out_fh.write('[%s] ' % gclient_utils.Elapsed()) |
| print(*args, **kwargs) |
| |
| def RunCommand(self, command, options, args, file_list=None): |
| commands = ['update', 'updatesingle', 'revert', |
| 'revinfo', 'status', 'diff', 'pack', 'runhooks'] |
| |
| if not command in commands: |
| raise gclient_utils.Error('Unknown command %s' % command) |
| |
| if not command in dir(self): |
| raise gclient_utils.Error('Command %s not implemented in %s wrapper' % ( |
| command, self.__class__.__name__)) |
| |
| return getattr(self, command)(options, args, file_list) |
| |
| @staticmethod |
| def _get_first_remote_url(checkout_path): |
| log = scm.GIT.Capture( |
| ['config', '--local', '--get-regexp', r'remote.*.url'], |
| cwd=checkout_path) |
| # Get the second token of the first line of the log. |
| return log.splitlines()[0].split(' ', 1)[1] |
| |
| def GetCacheMirror(self): |
| if getattr(self, 'cache_dir', None): |
| url, _ = gclient_utils.SplitUrlRevision(self.url) |
| return git_cache.Mirror(url) |
| return None |
| |
| def GetActualRemoteURL(self, options): |
| """Attempt to determine the remote URL for this SCMWrapper.""" |
| # Git |
| if os.path.exists(os.path.join(self.checkout_path, '.git')): |
| actual_remote_url = self._get_first_remote_url(self.checkout_path) |
| |
| mirror = self.GetCacheMirror() |
| # If the cache is used, obtain the actual remote URL from there. |
| if (mirror and mirror.exists() and |
| mirror.mirror_path.replace('\\', '/') == |
| actual_remote_url.replace('\\', '/')): |
| actual_remote_url = self._get_first_remote_url(mirror.mirror_path) |
| return actual_remote_url |
| return None |
| |
| def DoesRemoteURLMatch(self, options): |
| """Determine whether the remote URL of this checkout is the expected URL.""" |
| if not os.path.exists(self.checkout_path): |
| # A checkout which doesn't exist can't be broken. |
| return True |
| |
| actual_remote_url = self.GetActualRemoteURL(options) |
| if actual_remote_url: |
| return (gclient_utils.SplitUrlRevision(actual_remote_url)[0].rstrip('/') |
| == gclient_utils.SplitUrlRevision(self.url)[0].rstrip('/')) |
| else: |
| # This may occur if the self.checkout_path exists but does not contain a |
| # valid git checkout. |
| return False |
| |
| def _DeleteOrMove(self, force): |
| """Delete the checkout directory or move it out of the way. |
| |
| Args: |
| force: bool; if True, delete the directory. Otherwise, just move it. |
| """ |
| if force and os.environ.get('CHROME_HEADLESS') == '1': |
| self.Print('_____ Conflicting directory found in %s. Removing.' |
| % self.checkout_path) |
| gclient_utils.AddWarning('Conflicting directory %s deleted.' |
| % self.checkout_path) |
| gclient_utils.rmtree(self.checkout_path) |
| else: |
| bad_scm_dir = os.path.join(self._root_dir, '_bad_scm', |
| os.path.dirname(self.relpath)) |
| |
| try: |
| os.makedirs(bad_scm_dir) |
| except OSError as e: |
| if e.errno != errno.EEXIST: |
| raise |
| |
| dest_path = tempfile.mkdtemp( |
| prefix=os.path.basename(self.relpath), |
| dir=bad_scm_dir) |
| self.Print('_____ Conflicting directory found in %s. Moving to %s.' |
| % (self.checkout_path, dest_path)) |
| gclient_utils.AddWarning('Conflicting directory %s moved to %s.' |
| % (self.checkout_path, dest_path)) |
| shutil.move(self.checkout_path, dest_path) |
| |
| |
| class GitWrapper(SCMWrapper): |
| """Wrapper for Git""" |
| name = 'git' |
| remote = 'origin' |
| |
| @property |
| def cache_dir(self): |
| try: |
| return git_cache.Mirror.GetCachePath() |
| except RuntimeError: |
| return None |
| |
| def __init__(self, url=None, *args, **kwargs): |
| """Removes 'git+' fake prefix from git URL.""" |
| if url and (url.startswith('git+http://') or |
| url.startswith('git+https://')): |
| url = url[4:] |
| SCMWrapper.__init__(self, url, *args, **kwargs) |
| filter_kwargs = { 'time_throttle': 1, 'out_fh': self.out_fh } |
| if self.out_cb: |
| filter_kwargs['predicate'] = self.out_cb |
| self.filter = gclient_utils.GitFilter(**filter_kwargs) |
| self._running_under_rosetta = None |
| |
| def GetCheckoutRoot(self): |
| return scm.GIT.GetCheckoutRoot(self.checkout_path) |
| |
| def GetRevisionDate(self, _revision): |
| """Returns the given revision's date in ISO-8601 format (which contains the |
| time zone).""" |
| # TODO(floitsch): get the time-stamp of the given revision and not just the |
| # time-stamp of the currently checked out revision. |
| return self._Capture(['log', '-n', '1', '--format=%ai']) |
| |
| def _GetDiffFilenames(self, base): |
| """Returns the names of files modified since base.""" |
| return self._Capture( |
| # Filter to remove base if it is None. |
| list(filter(bool, ['-c', 'core.quotePath=false', 'diff', '--name-only', |
| base]) |
| )).split() |
| |
| def diff(self, options, _args, _file_list): |
| _, revision = gclient_utils.SplitUrlRevision(self.url) |
| if not revision: |
| revision = 'refs/remotes/%s/master' % self.remote |
| self._Run(['-c', 'core.quotePath=false', 'diff', revision], options) |
| |
| def pack(self, _options, _args, _file_list): |
| """Generates a patch file which can be applied to the root of the |
| repository. |
| |
| The patch file is generated from a diff of the merge base of HEAD and |
| its upstream branch. |
| """ |
| try: |
| merge_base = [self._Capture(['merge-base', 'HEAD', self.remote])] |
| except subprocess2.CalledProcessError: |
| merge_base = [] |
| gclient_utils.CheckCallAndFilter( |
| ['git', 'diff'] + merge_base, |
| cwd=self.checkout_path, |
| filter_fn=GitDiffFilterer(self.relpath, print_func=self.Print).Filter) |
| |
| def _Scrub(self, target, options): |
| """Scrubs out all changes in the local repo, back to the state of target.""" |
| quiet = [] |
| if not options.verbose: |
| quiet = ['--quiet'] |
| self._Run(['reset', '--hard', target] + quiet, options) |
| if options.force and options.delete_unversioned_trees: |
| # where `target` is a commit that contains both upper and lower case |
| # versions of the same file on a case insensitive filesystem, we are |
| # actually in a broken state here. The index will have both 'a' and 'A', |
| # but only one of them will exist on the disk. To progress, we delete |
| # everything that status thinks is modified. |
| output = self._Capture([ |
| '-c', 'core.quotePath=false', 'status', '--porcelain'], strip=False) |
| for line in output.splitlines(): |
| # --porcelain (v1) looks like: |
| # XY filename |
| try: |
| filename = line[3:] |
| self.Print('_____ Deleting residual after reset: %r.' % filename) |
| gclient_utils.rm_file_or_tree( |
| os.path.join(self.checkout_path, filename)) |
| except OSError: |
| pass |
| |
| def _FetchAndReset(self, revision, file_list, options): |
| """Equivalent to git fetch; git reset.""" |
| self._SetFetchConfig(options) |
| |
| self._Fetch(options, prune=True, quiet=options.verbose) |
| self._Scrub(revision, options) |
| if file_list is not None: |
| files = self._Capture( |
| ['-c', 'core.quotePath=false', 'ls-files']).splitlines() |
| file_list.extend( |
| [os.path.join(self.checkout_path, f) for f in files]) |
| |
| def _DisableHooks(self): |
| hook_dir = os.path.join(self.checkout_path, '.git', 'hooks') |
| if not os.path.isdir(hook_dir): |
| return |
| for f in os.listdir(hook_dir): |
| if not f.endswith('.sample') and not f.endswith('.disabled'): |
| disabled_hook_path = os.path.join(hook_dir, f + '.disabled') |
| if os.path.exists(disabled_hook_path): |
| os.remove(disabled_hook_path) |
| os.rename(os.path.join(hook_dir, f), disabled_hook_path) |
| |
| def _maybe_break_locks(self, options): |
| """This removes all .lock files from this repo's .git directory, if the |
| user passed the --break_repo_locks command line flag. |
| |
| In particular, this will cleanup index.lock files, as well as ref lock |
| files. |
| """ |
| if options.break_repo_locks: |
| git_dir = os.path.join(self.checkout_path, '.git') |
| for path, _, filenames in os.walk(git_dir): |
| for filename in filenames: |
| if filename.endswith('.lock'): |
| to_break = os.path.join(path, filename) |
| self.Print('breaking lock: %s' % (to_break,)) |
| try: |
| os.remove(to_break) |
| except OSError as ex: |
| self.Print('FAILED to break lock: %s: %s' % (to_break, ex)) |
| raise |
| |
| def apply_patch_ref(self, patch_repo, patch_rev, target_rev, options, |
| file_list): |
| """Apply a patch on top of the revision we're synced at. |
| |
| The patch ref is given by |patch_repo|@|patch_rev|. |
| |target_rev| is usually the branch that the |patch_rev| was uploaded against |
| (e.g. 'refs/heads/main'), but this is not required. |
| |
| We cherry-pick all commits reachable from |patch_rev| on top of the curret |
| HEAD, excluding those reachable from |target_rev| |
| (i.e. git cherry-pick target_rev..patch_rev). |
| |
| Graphically, it looks like this: |
| |
| ... -> o -> [possibly already landed commits] -> target_rev |
| \ |
| -> [possibly not yet landed dependent CLs] -> patch_rev |
| |
| The final checkout state is then: |
| |
| ... -> HEAD -> [possibly not yet landed dependent CLs] -> patch_rev |
| |
| After application, if |options.reset_patch_ref| is specified, we soft reset |
| the cherry-picked changes, keeping them in git index only. |
| |
| Args: |
| patch_repo: The patch origin. |
| e.g. 'https://foo.googlesource.com/bar' |
| patch_rev: The revision to patch. |
| e.g. 'refs/changes/1234/34/1'. |
| target_rev: The revision to use when finding the merge base. |
| Typically, the branch that the patch was uploaded against. |
| e.g. 'refs/heads/main' or 'refs/heads/infra/config'. |
| options: The options passed to gclient. |
| file_list: A list where modified files will be appended. |
| """ |
| |
| # Abort any cherry-picks in progress. |
| try: |
| self._Capture(['cherry-pick', '--abort']) |
| except subprocess2.CalledProcessError: |
| pass |
| |
| base_rev = self._Capture(['rev-parse', 'HEAD']) |
| |
| if not target_rev: |
| raise gclient_utils.Error('A target revision for the patch must be given') |
| elif target_rev.startswith(('refs/heads/', 'refs/branch-heads')): |
| # If |target_rev| is in refs/heads/** or refs/branch-heads/**, try first |
| # to find the corresponding remote ref for it, since |target_rev| might |
| # point to a local ref which is not up to date with the corresponding |
| # remote ref. |
| remote_ref = ''.join(scm.GIT.RefToRemoteRef(target_rev, self.remote)) |
| self.Print('Trying the corresponding remote ref for %r: %r\n' % ( |
| target_rev, remote_ref)) |
| if scm.GIT.IsValidRevision(self.checkout_path, remote_ref): |
| target_rev = remote_ref |
| elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev): |
| # Fetch |target_rev| if it's not already available. |
| url, _ = gclient_utils.SplitUrlRevision(self.url) |
| mirror = self._GetMirror(url, options, target_rev) |
| if mirror: |
| rev_type = 'branch' if target_rev.startswith('refs/') else 'hash' |
| self._UpdateMirrorIfNotContains(mirror, options, rev_type, target_rev) |
| self._Fetch(options, refspec=target_rev) |
| |
| self.Print('===Applying patch===') |
| self.Print('Revision to patch is %r @ %r.' % (patch_repo, patch_rev)) |
| self.Print('Current dir is %r' % self.checkout_path) |
| self._Capture(['reset', '--hard']) |
| self._Capture(['fetch', '--no-tags', patch_repo, patch_rev]) |
| patch_rev = self._Capture(['rev-parse', 'FETCH_HEAD']) |
| |
| if not options.rebase_patch_ref: |
| self._Capture(['checkout', patch_rev]) |
| # Adjust base_rev to be the first parent of our checked out patch ref; |
| # This will allow us to correctly extend `file_list`, and will show the |
| # correct file-list to programs which do `git diff --cached` expecting to |
| # see the patch diff. |
| base_rev = self._Capture(['rev-parse', patch_rev+'~']) |
| |
| else: |
| self.Print('Will cherrypick %r .. %r on top of %r.' % ( |
| target_rev, patch_rev, base_rev)) |
| try: |
| if scm.GIT.IsAncestor(self.checkout_path, patch_rev, target_rev): |
| # If |patch_rev| is an ancestor of |target_rev|, check it out. |
| self._Capture(['checkout', patch_rev]) |
| else: |
| # If a change was uploaded on top of another change, which has already |
| # landed, one of the commits in the cherry-pick range will be |
| # redundant, since it has already landed and its changes incorporated |
| # in the tree. |
| # We pass '--keep-redundant-commits' to ignore those changes. |
| self._Capture(['cherry-pick', target_rev + '..' + patch_rev, |
| '--keep-redundant-commits']) |
| |
| except subprocess2.CalledProcessError as e: |
| self.Print('Failed to apply patch.') |
| self.Print('Revision to patch was %r @ %r.' % (patch_repo, patch_rev)) |
| self.Print('Tried to cherrypick %r .. %r on top of %r.' % ( |
| target_rev, patch_rev, base_rev)) |
| self.Print('Current dir is %r' % self.checkout_path) |
| self.Print('git returned non-zero exit status %s:\n%s' % ( |
| e.returncode, e.stderr.decode('utf-8'))) |
| # Print the current status so that developers know what changes caused |
| # the patch failure, since git cherry-pick doesn't show that |
| # information. |
| self.Print(self._Capture(['status'])) |
| try: |
| self._Capture(['cherry-pick', '--abort']) |
| except subprocess2.CalledProcessError: |
| pass |
| raise |
| |
| if file_list is not None: |
| file_list.extend(self._GetDiffFilenames(base_rev)) |
| |
| if options.reset_patch_ref: |
| self._Capture(['reset', '--soft', base_rev]) |
| |
| def update(self, options, args, file_list): |
| """Runs git to update or transparently checkout the working copy. |
| |
| All updated files will be appended to file_list. |
| |
| Raises: |
| Error: if can't get URL for relative path. |
| """ |
| if args: |
| raise gclient_utils.Error("Unsupported argument(s): %s" % ",".join(args)) |
| |
| self._CheckMinVersion("1.6.6") |
| |
| # If a dependency is not pinned, track the default remote branch. |
| default_rev = 'refs/remotes/%s/master' % self.remote |
| url, deps_revision = gclient_utils.SplitUrlRevision(self.url) |
| revision = deps_revision |
| managed = True |
| if options.revision: |
| # Override the revision number. |
| revision = str(options.revision) |
| if revision == 'unmanaged': |
| # Check again for a revision in case an initial ref was specified |
| # in the url, for example bla.git@refs/heads/custombranch |
| revision = deps_revision |
| managed = False |
| if not revision: |
| revision = default_rev |
| |
| if managed: |
| self._DisableHooks() |
| |
| printed_path = False |
| verbose = [] |
| if options.verbose: |
| self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False) |
| verbose = ['--verbose'] |
| printed_path = True |
| |
| revision_ref = revision |
| if ':' in revision: |
| revision_ref, _, revision = revision.partition(':') |
| |
| if revision_ref.startswith('refs/branch-heads'): |
| options.with_branch_heads = True |
| |
| mirror = self._GetMirror(url, options, revision_ref) |
| if mirror: |
| url = mirror.mirror_path |
| |
| remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) |
| if remote_ref: |
| # Rewrite remote refs to their local equivalents. |
| revision = ''.join(remote_ref) |
| rev_type = "branch" |
| elif revision.startswith('refs/'): |
| # Local branch? We probably don't want to support, since DEPS should |
| # always specify branches as they are in the upstream repo. |
| rev_type = "branch" |
| else: |
| # hash is also a tag, only make a distinction at checkout |
| rev_type = "hash" |
| |
| # If we are going to introduce a new project, there is a possibility that |
| # we are syncing back to a state where the project was originally a |
| # sub-project rolled by DEPS (realistic case: crossing the Blink merge point |
| # syncing backwards, when Blink was a DEPS entry and not part of src.git). |
| # In such case, we might have a backup of the former .git folder, which can |
| # be used to avoid re-fetching the entire repo again (useful for bisects). |
| backup_dir = self.GetGitBackupDirPath() |
| target_dir = os.path.join(self.checkout_path, '.git') |
| if os.path.exists(backup_dir) and not os.path.exists(target_dir): |
| gclient_utils.safe_makedirs(self.checkout_path) |
| os.rename(backup_dir, target_dir) |
| # Reset to a clean state |
| self._Scrub('HEAD', options) |
| |
| if (not os.path.exists(self.checkout_path) or |
| (os.path.isdir(self.checkout_path) and |
| not os.path.exists(os.path.join(self.checkout_path, '.git')))): |
| if mirror: |
| self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision) |
| try: |
| self._Clone(revision, url, options) |
| except subprocess2.CalledProcessError: |
| self._DeleteOrMove(options.force) |
| self._Clone(revision, url, options) |
| if file_list is not None: |
| files = self._Capture( |
| ['-c', 'core.quotePath=false', 'ls-files']).splitlines() |
| file_list.extend( |
| [os.path.join(self.checkout_path, f) for f in files]) |
| if mirror: |
| self._Capture( |
| ['remote', 'set-url', '--push', 'origin', mirror.url]) |
| if not verbose: |
| # Make the output a little prettier. It's nice to have some whitespace |
| # between projects when cloning. |
| self.Print('') |
| return self._Capture(['rev-parse', '--verify', 'HEAD']) |
| |
| if mirror: |
| self._Capture( |
| ['remote', 'set-url', '--push', 'origin', mirror.url]) |
| |
| if not managed: |
| self._SetFetchConfig(options) |
| self.Print('________ unmanaged solution; skipping %s' % self.relpath) |
| return self._Capture(['rev-parse', '--verify', 'HEAD']) |
| |
| self._maybe_break_locks(options) |
| |
| if mirror: |
| self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision) |
| |
| # See if the url has changed (the unittests use git://foo for the url, let |
| # that through). |
| current_url = self._Capture(['config', 'remote.%s.url' % self.remote]) |
| return_early = False |
| # TODO(maruel): Delete url != 'git://foo' since it's just to make the |
| # unit test pass. (and update the comment above) |
| # Skip url auto-correction if remote.origin.gclient-auto-fix-url is set. |
| # This allows devs to use experimental repos which have a different url |
| # but whose branch(s) are the same as official repos. |
| if (current_url.rstrip('/') != url.rstrip('/') and url != 'git://foo' and |
| subprocess2.capture( |
| ['git', 'config', 'remote.%s.gclient-auto-fix-url' % self.remote], |
| cwd=self.checkout_path).strip() != 'False'): |
| self.Print('_____ switching %s from %s to new upstream %s' % ( |
| self.relpath, current_url, url)) |
| if not (options.force or options.reset): |
| # Make sure it's clean |
| self._CheckClean(revision) |
| # Switch over to the new upstream |
| self._Run(['remote', 'set-url', self.remote, url], options) |
| if mirror: |
| with open(os.path.join( |
| self.checkout_path, '.git', 'objects', 'info', 'alternates'), |
| 'w') as fh: |
| fh.write(os.path.join(url, 'objects')) |
| self._EnsureValidHeadObjectOrCheckout(revision, options, url) |
| self._FetchAndReset(revision, file_list, options) |
| |
| return_early = True |
| else: |
| self._EnsureValidHeadObjectOrCheckout(revision, options, url) |
| |
| if return_early: |
| return self._Capture(['rev-parse', '--verify', 'HEAD']) |
| |
| cur_branch = self._GetCurrentBranch() |
| |
| # Cases: |
| # 0) HEAD is detached. Probably from our initial clone. |
| # - make sure HEAD is contained by a named ref, then update. |
| # Cases 1-4. HEAD is a branch. |
| # 1) current branch is not tracking a remote branch |
| # - try to rebase onto the new hash or branch |
| # 2) current branch is tracking a remote branch with local committed |
| # changes, but the DEPS file switched to point to a hash |
| # - rebase those changes on top of the hash |
| # 3) current branch is tracking a remote branch w/or w/out changes, and |
| # no DEPS switch |
| # - see if we can FF, if not, prompt the user for rebase, merge, or stop |
| # 4) current branch is tracking a remote branch, but DEPS switches to a |
| # different remote branch, and |
| # a) current branch has no local changes, and --force: |
| # - checkout new branch |
| # b) current branch has local changes, and --force and --reset: |
| # - checkout new branch |
| # c) otherwise exit |
| |
| # GetUpstreamBranch returns something like 'refs/remotes/origin/main' for |
| # a tracking branch |
| # or 'main' if not a tracking branch (it's based on a specific rev/hash) |
| # or it returns None if it couldn't find an upstream |
| if cur_branch is None: |
| upstream_branch = None |
| current_type = "detached" |
| logging.debug("Detached HEAD") |
| else: |
| upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path) |
| if not upstream_branch or not upstream_branch.startswith('refs/remotes'): |
| current_type = "hash" |
| logging.debug("Current branch is not tracking an upstream (remote)" |
| " branch.") |
| elif upstream_branch.startswith('refs/remotes'): |
| current_type = "branch" |
| else: |
| raise gclient_utils.Error('Invalid Upstream: %s' % upstream_branch) |
| |
| self._SetFetchConfig(options) |
| |
| # Fetch upstream if we don't already have |revision|. |
| if not scm.GIT.IsValidRevision(self.checkout_path, revision, sha_only=True): |
| self._Fetch(options, prune=options.force) |
| |
| if not scm.GIT.IsValidRevision(self.checkout_path, revision, |
| sha_only=True): |
| # Update the remotes first so we have all the refs. |
| remote_output = scm.GIT.Capture(['remote'] + verbose + ['update'], |
| cwd=self.checkout_path) |
| if verbose: |
| self.Print(remote_output) |
| |
| revision = self._AutoFetchRef(options, revision) |
| |
| # This is a big hammer, debatable if it should even be here... |
| if options.force or options.reset: |
| target = 'HEAD' |
| if options.upstream and upstream_branch: |
| target = upstream_branch |
| self._Scrub(target, options) |
| |
| if current_type == 'detached': |
| # case 0 |
| # We just did a Scrub, this is as clean as it's going to get. In |
| # particular if HEAD is a commit that contains two versions of the same |
| # file on a case-insensitive filesystem (e.g. 'a' and 'A'), there's no way |
| # to actually "Clean" the checkout; that commit is uncheckoutable on this |
| # system. The best we can do is carry forward to the checkout step. |
| if not (options.force or options.reset): |
| self._CheckClean(revision) |
| self._CheckDetachedHead(revision, options) |
| if self._Capture(['rev-list', '-n', '1', 'HEAD']) == revision: |
| self.Print('Up-to-date; skipping checkout.') |
| else: |
| # 'git checkout' may need to overwrite existing untracked files. Allow |
| # it only when nuclear options are enabled. |
| self._Checkout( |
| options, |
| revision, |
| force=(options.force and options.delete_unversioned_trees), |
| quiet=True, |
| ) |
| if not printed_path: |
| self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False) |
| elif current_type == 'hash': |
| # case 1 |
| # Can't find a merge-base since we don't know our upstream. That makes |
| # this command VERY likely to produce a rebase failure. For now we |
| # assume origin is our upstream since that's what the old behavior was. |
| upstream_branch = self.remote |
| if options.revision or deps_revision: |
| upstream_branch = revision |
| self._AttemptRebase(upstream_branch, file_list, options, |
| printed_path=printed_path, merge=options.merge) |
| printed_path = True |
| elif rev_type == 'hash': |
| # case 2 |
| self._AttemptRebase(upstream_branch, file_list, options, |
| newbase=revision, printed_path=printed_path, |
| merge=options.merge) |
| printed_path = True |
| elif remote_ref and ''.join(remote_ref) != upstream_branch: |
| # case 4 |
| new_base = ''.join(remote_ref) |
| if not printed_path: |
| self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False) |
| switch_error = ("Could not switch upstream branch from %s to %s\n" |
| % (upstream_branch, new_base) + |
| "Please use --force or merge or rebase manually:\n" + |
| "cd %s; git rebase %s\n" % (self.checkout_path, new_base) + |
| "OR git checkout -b <some new branch> %s" % new_base) |
| force_switch = False |
| if options.force: |
| try: |
| self._CheckClean(revision) |
| # case 4a |
| force_switch = True |
| except gclient_utils.Error as e: |
| if options.reset: |
| # case 4b |
| force_switch = True |
| else: |
| switch_error = '%s\n%s' % (e.message, switch_error) |
| if force_switch: |
| self.Print("Switching upstream branch from %s to %s" % |
| (upstream_branch, new_base)) |
| switch_branch = 'gclient_' + remote_ref[1] |
| self._Capture(['branch', '-f', switch_branch, new_base]) |
| self._Checkout(options, switch_branch, force=True, quiet=True) |
| else: |
| # case 4c |
| raise gclient_utils.Error(switch_error) |
| else: |
| # case 3 - the default case |
| rebase_files = self._GetDiffFilenames(upstream_branch) |
| if verbose: |
| self.Print('Trying fast-forward merge to branch : %s' % upstream_branch) |
| try: |
| merge_args = ['merge'] |
| if options.merge: |
| merge_args.append('--ff') |
| else: |
| merge_args.append('--ff-only') |
| merge_args.append(upstream_branch) |
| merge_output = self._Capture(merge_args) |
| except subprocess2.CalledProcessError as e: |
| rebase_files = [] |
| if re.match(b'fatal: Not possible to fast-forward, aborting.', |
| e.stderr): |
| if not printed_path: |
| self.Print('_____ %s at %s' % (self.relpath, revision), |
| timestamp=False) |
| printed_path = True |
| while True: |
| if not options.auto_rebase: |
| try: |
| action = self._AskForData( |
| 'Cannot %s, attempt to rebase? ' |
| '(y)es / (q)uit / (s)kip : ' % |
| ('merge' if options.merge else 'fast-forward merge'), |
| options) |
| except ValueError: |
| raise gclient_utils.Error('Invalid Character') |
| if options.auto_rebase or re.match(r'yes|y', action, re.I): |
| self._AttemptRebase(upstream_branch, rebase_files, options, |
| printed_path=printed_path, merge=False) |
| printed_path = True |
| break |
| elif re.match(r'quit|q', action, re.I): |
| raise gclient_utils.Error("Can't fast-forward, please merge or " |
| "rebase manually.\n" |
| "cd %s && git " % self.checkout_path |
| + "rebase %s" % upstream_branch) |
| elif re.match(r'skip|s', action, re.I): |
| self.Print('Skipping %s' % self.relpath) |
| return |
| else: |
| self.Print('Input not recognized') |
| elif re.match(b"error: Your local changes to '.*' would be " |
| b"overwritten by merge. Aborting.\nPlease, commit your " |
| b"changes or stash them before you can merge.\n", |
| e.stderr): |
| if not printed_path: |
| self.Print('_____ %s at %s' % (self.relpath, revision), |
| timestamp=False) |
| printed_path = True |
| raise gclient_utils.Error(e.stderr.decode('utf-8')) |
| else: |
| # Some other problem happened with the merge |
| logging.error("Error during fast-forward merge in %s!" % self.relpath) |
| self.Print(e.stderr.decode('utf-8')) |
| raise |
| else: |
| # Fast-forward merge was successful |
| if not re.match('Already up-to-date.', merge_output) or verbose: |
| if not printed_path: |
| self.Print('_____ %s at %s' % (self.relpath, revision), |
| timestamp=False) |
| printed_path = True |
| self.Print(merge_output.strip()) |
| if not verbose: |
| # Make the output a little prettier. It's nice to have some |
| # whitespace between projects when syncing. |
| self.Print('') |
| |
| if file_list is not None: |
| file_list.extend( |
| [os.path.join(self.checkout_path, f) for f in rebase_files]) |
| |
| # If the rebase generated a conflict, abort and ask user to fix |
| if self._IsRebasing(): |
| raise gclient_utils.Error('\n____ %s at %s\n' |
| '\nConflict while rebasing this branch.\n' |
| 'Fix the conflict and run gclient again.\n' |
| 'See man git-rebase for details.\n' |
| % (self.relpath, revision)) |
| |
| if verbose: |
| self.Print('Checked out revision %s' % self.revinfo(options, (), None), |
| timestamp=False) |
| |
| # If --reset and --delete_unversioned_trees are specified, remove any |
| # untracked directories. |
| if options.reset and options.delete_unversioned_trees: |
| # GIT.CaptureStatus() uses 'dit diff' to compare to a specific SHA1 (the |
| # merge-base by default), so doesn't include untracked files. So we use |
| # 'git ls-files --directory --others --exclude-standard' here directly. |
| paths = scm.GIT.Capture( |
| ['-c', 'core.quotePath=false', 'ls-files', |
| '--directory', '--others', '--exclude-standard'], |
| self.checkout_path) |
| for path in (p for p in paths.splitlines() if p.endswith('/')): |
| full_path = os.path.join(self.checkout_path, path) |
| if not os.path.islink(full_path): |
| self.Print('_____ removing unversioned directory %s' % path) |
| gclient_utils.rmtree(full_path) |
| |
| return self._Capture(['rev-parse', '--verify', 'HEAD']) |
| |
| def revert(self, options, _args, file_list): |
| """Reverts local modifications. |
| |
| All reverted files will be appended to file_list. |
| """ |
| if not os.path.isdir(self.checkout_path): |
| # revert won't work if the directory doesn't exist. It needs to |
| # checkout instead. |
| self.Print('_____ %s is missing, syncing instead' % self.relpath) |
| # Don't reuse the args. |
| return self.update(options, [], file_list) |
| |
| default_rev = "refs/heads/master" |
| if options.upstream: |
| if self._GetCurrentBranch(): |
| upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path) |
| default_rev = upstream_branch or default_rev |
| _, deps_revision = gclient_utils.SplitUrlRevision(self.url) |
| if not deps_revision: |
| deps_revision = default_rev |
| if deps_revision.startswith('refs/heads/'): |
| deps_revision = deps_revision.replace('refs/heads/', self.remote + '/') |
| try: |
| deps_revision = self.GetUsableRev(deps_revision, options) |
| except NoUsableRevError as e: |
| # If the DEPS entry's url and hash changed, try to update the origin. |
| # See also http://crbug.com/520067. |
| logging.warning( |
| "Couldn't find usable revision, will retrying to update instead: %s", |
| e.message) |
| return self.update(options, [], file_list) |
| |
| if file_list is not None: |
| files = self._GetDiffFilenames(deps_revision) |
| |
| self._Scrub(deps_revision, options) |
| self._Run(['clean', '-f', '-d'], options) |
| |
| if file_list is not None: |
| file_list.extend([os.path.join(self.checkout_path, f) for f in files]) |
| |
| def revinfo(self, _options, _args, _file_list): |
| """Returns revision""" |
| return self._Capture(['rev-parse', 'HEAD']) |
| |
| def runhooks(self, options, args, file_list): |
| self.status(options, args, file_list) |
| |
| def status(self, options, _args, file_list): |
| """Display status information.""" |
| if not os.path.isdir(self.checkout_path): |
| self.Print('________ couldn\'t run status in %s:\n' |
| 'The directory does not exist.' % self.checkout_path) |
| else: |
| merge_base = [] |
| if self.url: |
| _, base_rev = gclient_utils.SplitUrlRevision(self.url) |
| if base_rev: |
| merge_base = [base_rev] |
| self._Run( |
| ['-c', 'core.quotePath=false', 'diff', '--name-status'] + merge_base, |
| options, always_show_header=options.verbose) |
| if file_list is not None: |
| files = self._GetDiffFilenames(merge_base[0] if merge_base else None) |
| file_list.extend([os.path.join(self.checkout_path, f) for f in files]) |
| |
| def GetUsableRev(self, rev, options): |
| """Finds a useful revision for this repository.""" |
| sha1 = None |
| if not os.path.isdir(self.checkout_path): |
| raise NoUsableRevError( |
| 'This is not a git repo, so we cannot get a usable rev.') |
| |
| if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev): |
| sha1 = rev |
| else: |
| # May exist in origin, but we don't have it yet, so fetch and look |
| # again. |
| self._Fetch(options) |
| if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev): |
| sha1 = rev |
| |
| if not sha1: |
| raise NoUsableRevError( |
| 'Hash %s does not appear to be a valid hash in this repo.' % rev) |
| |
| return sha1 |
| |
| def GetGitBackupDirPath(self): |
| """Returns the path where the .git folder for the current project can be |
| staged/restored. Use case: subproject moved from DEPS <-> outer project.""" |
| return os.path.join(self._root_dir, |
| 'old_' + self.relpath.replace(os.sep, '_')) + '.git' |
| |
| def _GetMirror(self, url, options, revision_ref=None): |
| """Get a git_cache.Mirror object for the argument url.""" |
| if not self.cache_dir: |
| return None |
| mirror_kwargs = { |
| 'print_func': self.filter, |
| 'refs': [] |
| } |
| if hasattr(options, 'with_branch_heads') and options.with_branch_heads: |
| mirror_kwargs['refs'].append('refs/branch-heads/*') |
| elif revision_ref and revision_ref.startswith('refs/branch-heads/'): |
| mirror_kwargs['refs'].append(revision_ref) |
| if hasattr(options, 'with_tags') and options.with_tags: |
| mirror_kwargs['refs'].append('refs/tags/*') |
| elif revision_ref and revision_ref.startswith('refs/tags/'): |
| mirror_kwargs['refs'].append(revision_ref) |
| return git_cache.Mirror(url, **mirror_kwargs) |
| |
| def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision): |
| """Update a git mirror by fetching the latest commits from the remote, |
| unless mirror already contains revision whose type is sha1 hash. |
| """ |
| if rev_type == 'hash' and mirror.contains_revision(revision): |
| if options.verbose: |
| self.Print('skipping mirror update, it has rev=%s already' % revision, |
| timestamp=False) |
| return |
| |
| if getattr(options, 'shallow', False): |
| # HACK(hinoka): These repositories should be super shallow. |
| if 'flash' in mirror.url: |
| depth = 10 |
| else: |
| depth = 10000 |
| else: |
| depth = None |
| mirror.populate(verbose=options.verbose, |
| bootstrap=not getattr(options, 'no_bootstrap', False), |
| depth=depth, |
| lock_timeout=getattr(options, 'lock_timeout', 0)) |
| |
| def _Clone(self, revision, url, options): |
| """Clone a git repository from the given URL. |
| |
| Once we've cloned the repo, we checkout a working branch if the specified |
| revision is a branch head. If it is a tag or a specific commit, then we |
| leave HEAD detached as it makes future updates simpler -- in this case the |
| user should first create a new branch or switch to an existing branch before |
| making changes in the repo.""" |
| if not options.verbose: |
| # git clone doesn't seem to insert a newline properly before printing |
| # to stdout |
| self.Print('') |
| cfg = gclient_utils.DefaultIndexPackConfig(url) |
| clone_cmd = cfg + ['clone', '--no-checkout', '--progress'] |
| if self.cache_dir: |
| clone_cmd.append('--shared') |
| if options.verbose: |
| clone_cmd.append('--verbose') |
| clone_cmd.append(url) |
| # If the parent directory does not exist, Git clone on Windows will not |
| # create it, so we need to do it manually. |
| parent_dir = os.path.dirname(self.checkout_path) |
| gclient_utils.safe_makedirs(parent_dir) |
| |
| template_dir = None |
| if hasattr(options, 'no_history') and options.no_history: |
| if gclient_utils.IsGitSha(revision): |
| # In the case of a subproject, the pinned sha is not necessarily the |
| # head of the remote branch (so we can't just use --depth=N). Instead, |
| # we tell git to fetch all the remote objects from SHA..HEAD by means of |
| # a template git dir which has a 'shallow' file pointing to the sha. |
| template_dir = tempfile.mkdtemp( |
| prefix='_gclient_gittmp_%s' % os.path.basename(self.checkout_path), |
| dir=parent_dir) |
| self._Run(['init', '--bare', template_dir], options, cwd=self._root_dir) |
| with open(os.path.join(template_dir, 'shallow'), 'w') as template_file: |
| template_file.write(revision) |
| clone_cmd.append('--template=' + template_dir) |
| else: |
| # Otherwise, we're just interested in the HEAD. Just use --depth. |
| clone_cmd.append('--depth=1') |
| |
| tmp_dir = tempfile.mkdtemp( |
| prefix='_gclient_%s_' % os.path.basename(self.checkout_path), |
| dir=parent_dir) |
| try: |
| clone_cmd.append(tmp_dir) |
| if self.print_outbuf: |
| print_stdout = True |
| filter_fn = None |
| else: |
| print_stdout = False |
| filter_fn = self.filter |
| self._Run(clone_cmd, options, cwd=self._root_dir, retry=True, |
| print_stdout=print_stdout, filter_fn=filter_fn) |
| gclient_utils.safe_makedirs(self.checkout_path) |
| gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'), |
| os.path.join(self.checkout_path, '.git')) |
| # TODO(https://github.com/git-for-windows/git/issues/2569): Remove once |
| # fixed. |
| if sys.platform.startswith('win'): |
| try: |
| self._Run(['config', '--unset', 'core.worktree'], options, |
| cwd=self.checkout_path) |
| except subprocess2.CalledProcessError: |
| pass |
| except: |
| traceback.print_exc(file=self.out_fh) |
| raise |
| finally: |
| if os.listdir(tmp_dir): |
| self.Print('_____ removing non-empty tmp dir %s' % tmp_dir) |
| gclient_utils.rmtree(tmp_dir) |
| if template_dir: |
| gclient_utils.rmtree(template_dir) |
| self._SetFetchConfig(options) |
| self._Fetch(options, prune=options.force) |
| revision = self._AutoFetchRef(options, revision) |
| remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) |
| self._Checkout(options, ''.join(remote_ref or revision), quiet=True) |
| if self._GetCurrentBranch() is None: |
| # Squelch git's very verbose detached HEAD warning and use our own |
| self.Print( |
| ('Checked out %s to a detached HEAD. Before making any commits\n' |
| 'in this repo, you should use \'git checkout <branch>\' to switch to\n' |
| 'an existing branch or use \'git checkout %s -b <branch>\' to\n' |
| 'create a new branch for your work.') % (revision, self.remote)) |
| |
| def _AskForData(self, prompt, options): |
| if options.jobs > 1: |
| self.Print(prompt) |
| raise gclient_utils.Error("Background task requires input. Rerun " |
| "gclient with --jobs=1 so that\n" |
| "interaction is possible.") |
| return gclient_utils.AskForData(prompt) |
| |
| |
| def _AttemptRebase(self, upstream, files, options, newbase=None, |
| branch=None, printed_path=False, merge=False): |
| """Attempt to rebase onto either upstream or, if specified, newbase.""" |
| if files is not None: |
| files.extend(self._GetDiffFilenames(upstream)) |
| revision = upstream |
| if newbase: |
| revision = newbase |
| action = 'merge' if merge else 'rebase' |
| if not printed_path: |
| self.Print('_____ %s : Attempting %s onto %s...' % ( |
| self.relpath, action, revision)) |
| printed_path = True |
| else: |
| self.Print('Attempting %s onto %s...' % (action, revision)) |
| |
| if merge: |
| merge_output = self._Capture(['merge', revision]) |
| if options.verbose: |
| self.Print(merge_output) |
| return |
| |
| # Build the rebase command here using the args |
| # git rebase [options] [--onto <newbase>] <upstream> [<branch>] |
| rebase_cmd = ['rebase'] |
| if options.verbose: |
| rebase_cmd.append('--verbose') |
| if newbase: |
| rebase_cmd.extend(['--onto', newbase]) |
| rebase_cmd.append(upstream) |
| if branch: |
| rebase_cmd.append(branch) |
| |
| try: |
| rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path) |
| except subprocess2.CalledProcessError as e: |
| if (re.match(br'cannot rebase: you have unstaged changes', e.stderr) or |
| re.match(br'cannot rebase: your index contains uncommitted changes', |
| e.stderr)): |
| while True: |
| rebase_action = self._AskForData( |
| 'Cannot rebase because of unstaged changes.\n' |
| '\'git reset --hard HEAD\' ?\n' |
| 'WARNING: destroys any uncommitted work in your current branch!' |
| ' (y)es / (q)uit / (s)how : ', options) |
| if re.match(r'yes|y', rebase_action, re.I): |
| self._Scrub('HEAD', options) |
| # Should this be recursive? |
| rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path) |
| break |
| elif re.match(r'quit|q', rebase_action, re.I): |
| raise gclient_utils.Error("Please merge or rebase manually\n" |
| "cd %s && git " % self.checkout_path |
| + "%s" % ' '.join(rebase_cmd)) |
| elif re.match(r'show|s', rebase_action, re.I): |
| self.Print('%s' % e.stderr.decode('utf-8').strip()) |
| continue |
| else: |
| gclient_utils.Error("Input not recognized") |
| continue |
| elif re.search(br'^CONFLICT', e.stdout, re.M): |
| raise gclient_utils.Error("Conflict while rebasing this branch.\n" |
| "Fix the conflict and run gclient again.\n" |
| "See 'man git-rebase' for details.\n") |
| else: |
| self.Print(e.stdout.decode('utf-8').strip()) |
| self.Print('Rebase produced error output:\n%s' % |
| e.stderr.decode('utf-8').strip()) |
| raise gclient_utils.Error("Unrecognized error, please merge or rebase " |
| "manually.\ncd %s && git " % |
| self.checkout_path |
| + "%s" % ' '.join(rebase_cmd)) |
| |
| self.Print(rebase_output.strip()) |
| if not options.verbose: |
| # Make the output a little prettier. It's nice to have some |
| # whitespace between projects when syncing. |
| self.Print('') |
| |
| @staticmethod |
| def _CheckMinVersion(min_version): |
| (ok, current_version) = scm.GIT.AssertVersion(min_version) |
| if not ok: |
| raise gclient_utils.Error('git version %s < minimum required %s' % |
| (current_version, min_version)) |
| |
| def _EnsureValidHeadObjectOrCheckout(self, revision, options, url): |
| # Special case handling if all 3 conditions are met: |
| # * the mirros have recently changed, but deps destination remains same, |
| # * the git histories of mirrors are conflicting. |
| # * git cache is used |
| # This manifests itself in current checkout having invalid HEAD commit on |
| # most git operations. Since git cache is used, just deleted the .git |
| # folder, and re-create it by cloning. |
| try: |
| self._Capture(['rev-list', '-n', '1', 'HEAD']) |
| except subprocess2.CalledProcessError as e: |
| if (b'fatal: bad object HEAD' in e.stderr |
| and self.cache_dir and self.cache_dir in url): |
| self.Print(( |
| 'Likely due to DEPS change with git cache_dir, ' |
| 'the current commit points to no longer existing object.\n' |
| '%s' % e) |
| ) |
| self._DeleteOrMove(options.force) |
| self._Clone(revision, url, options) |
| else: |
| raise |
| |
| def _IsRebasing(self): |
| # Check for any of REBASE-i/REBASE-m/REBASE/AM. Unfortunately git doesn't |
| # have a plumbing command to determine whether a rebase is in progress, so |
| # for now emualate (more-or-less) git-rebase.sh / git-completion.bash |
| g = os.path.join(self.checkout_path, '.git') |
| return ( |
| os.path.isdir(os.path.join(g, "rebase-merge")) or |
| os.path.isdir(os.path.join(g, "rebase-apply"))) |
| |
| def _CheckClean(self, revision, fixup=False): |
| lockfile = os.path.join(self.checkout_path, ".git", "index.lock") |
| if os.path.exists(lockfile): |
| raise gclient_utils.Error( |
| '\n____ %s at %s\n' |
| '\tYour repo is locked, possibly due to a concurrent git process.\n' |
| '\tIf no git executable is running, then clean up %r and try again.\n' |
| % (self.relpath, revision, lockfile)) |
| |
| # Make sure the tree is clean; see git-rebase.sh for reference |
| try: |
| scm.GIT.Capture(['update-index', '--ignore-submodules', '--refresh'], |
| cwd=self.checkout_path) |
| except subprocess2.CalledProcessError: |
| raise gclient_utils.Error('\n____ %s at %s\n' |
| '\tYou have unstaged changes.\n' |
| '\tPlease commit, stash, or reset.\n' |
| % (self.relpath, revision)) |
| try: |
| scm.GIT.Capture(['diff-index', '--cached', '--name-status', '-r', |
| '--ignore-submodules', 'HEAD', '--'], |
| cwd=self.checkout_path) |
| except subprocess2.CalledProcessError: |
| raise gclient_utils.Error('\n____ %s at %s\n' |
| '\tYour index contains uncommitted changes\n' |
| '\tPlease commit, stash, or reset.\n' |
| % (self.relpath, revision)) |
| |
| def _CheckDetachedHead(self, revision, _options): |
| # HEAD is detached. Make sure it is safe to move away from (i.e., it is |
| # reference by a commit). If not, error out -- most likely a rebase is |
| # in progress, try to detect so we can give a better error. |
| try: |
| scm.GIT.Capture(['name-rev', '--no-undefined', 'HEAD'], |
| cwd=self.checkout_path) |
| except subprocess2.CalledProcessError: |
| # Commit is not contained by any rev. See if the user is rebasing: |
| if self._IsRebasing(): |
| # Punt to the user |
| raise gclient_utils.Error('\n____ %s at %s\n' |
| '\tAlready in a conflict, i.e. (no branch).\n' |
| '\tFix the conflict and run gclient again.\n' |
| '\tOr to abort run:\n\t\tgit-rebase --abort\n' |
| '\tSee man git-rebase for details.\n' |
| % (self.relpath, revision)) |
| # Let's just save off the commit so we can proceed. |
| name = ('saved-by-gclient-' + |
| self._Capture(['rev-parse', '--short', 'HEAD'])) |
| self._Capture(['branch', '-f', name]) |
| self.Print('_____ found an unreferenced commit and saved it as \'%s\'' % |
| name) |
| |
| def _GetCurrentBranch(self): |
| # Returns name of current branch or None for detached HEAD |
| branch = self._Capture(['rev-parse', '--abbrev-ref=strict', 'HEAD']) |
| if branch == 'HEAD': |
| return None |
| return branch |
| |
| def _Capture(self, args, **kwargs): |
| set_git_dir = 'cwd' not in kwargs |
| kwargs.setdefault('cwd', self.checkout_path) |
| kwargs.setdefault('stderr', subprocess2.PIPE) |
| strip = kwargs.pop('strip', True) |
| env = scm.GIT.ApplyEnvVars(kwargs) |
| # If an explicit cwd isn't set, then default to the .git/ subdir so we get |
| # stricter behavior. This can be useful in cases of slight corruption -- |
| # we don't accidentally go corrupting parent git checks too. See |
| # https://crbug.com/1000825 for an example. |
| if set_git_dir: |
| git_dir = os.path.abspath(os.path.join(self.checkout_path, '.git')) |
| # Depending on how the .gclient file was defined, self.checkout_path |
| # might be set to a unicode string, not a regular string; on Windows |
| # Python2, we can't set env vars to be unicode strings, so we |
| # forcibly cast the value to a string before setting it. |
| env.setdefault('GIT_DIR', str(git_dir)) |
| ret = subprocess2.check_output( |
| ['git'] + args, env=env, **kwargs).decode('utf-8') |
| if strip: |
| ret = ret.strip() |
| self.Print('Finished running: %s %s' % ('git', ' '.join(args))) |
| return ret |
| |
| def _Checkout(self, options, ref, force=False, quiet=None): |
| """Performs a 'git-checkout' operation. |
| |
| Args: |
| options: The configured option set |
| ref: (str) The branch/commit to checkout |
| quiet: (bool/None) Whether or not the checkout should pass '--quiet'; if |
| 'None', the behavior is inferred from 'options.verbose'. |
| Returns: (str) The output of the checkout operation |
| """ |
| if quiet is None: |
| quiet = (not options.verbose) |
| checkout_args = ['checkout'] |
| if force: |
| checkout_args.append('--force') |
| if quiet: |
| checkout_args.append('--quiet') |
| checkout_args.append(ref) |
| return self._Capture(checkout_args) |
| |
| def _Fetch(self, options, remote=None, prune=False, quiet=False, |
| refspec=None): |
| cfg = gclient_utils.DefaultIndexPackConfig(self.url) |
| # When updating, the ref is modified to be a remote ref . |
| # (e.g. refs/heads/NAME becomes refs/remotes/REMOTE/NAME). |
| # Try to reverse that mapping. |
| original_ref = scm.GIT.RemoteRefToRef(refspec, self.remote) |
| if original_ref: |
| refspec = original_ref + ':' + refspec |
| # When a mirror is configured, it only fetches |
| # refs/{heads,branch-heads,tags}/*. |
| # If asked to fetch other refs, we must fetch those directly from the |
| # repository, and not from the mirror. |
| if not original_ref.startswith( |
| ('refs/heads/', 'refs/branch-heads/', 'refs/tags/')): |
| remote, _ = gclient_utils.SplitUrlRevision(self.url) |
| fetch_cmd = cfg + [ |
| 'fetch', |
| remote or self.remote, |
| ] |
| if refspec: |
| fetch_cmd.append(refspec) |
| |
| if prune: |
| fetch_cmd.append('--prune') |
| if options.verbose: |
| fetch_cmd.append('--verbose') |
| if not hasattr(options, 'with_tags') or not options.with_tags: |
| fetch_cmd.append('--no-tags') |
| elif quiet: |
| fetch_cmd.append('--quiet') |
| self._Run(fetch_cmd, options, show_header=options.verbose, retry=True) |
| |
| def _SetFetchConfig(self, options): |
| """Adds, and optionally fetches, "branch-heads" and "tags" refspecs |
| if requested.""" |
| if options.force or options.reset: |
| try: |
| self._Run(['config', '--unset-all', 'remote.%s.fetch' % self.remote], |
| options) |
| self._Run(['config', 'remote.%s.fetch' % self.remote, |
| '+refs/heads/*:refs/remotes/%s/*' % self.remote], options) |
| except subprocess2.CalledProcessError as e: |
| # If exit code was 5, it means we attempted to unset a config that |
| # didn't exist. Ignore it. |
| if e.returncode != 5: |
| raise |
| if hasattr(options, 'with_branch_heads') and options.with_branch_heads: |
| config_cmd = ['config', 'remote.%s.fetch' % self.remote, |
| '+refs/branch-heads/*:refs/remotes/branch-heads/*', |
| '^\\+refs/branch-heads/\\*:.*$'] |
| self._Run(config_cmd, options) |
| if hasattr(options, 'with_tags') and options.with_tags: |
| config_cmd = ['config', 'remote.%s.fetch' % self.remote, |
| '+refs/tags/*:refs/tags/*', |
| '^\\+refs/tags/\\*:.*$'] |
| self._Run(config_cmd, options) |
| |
| def _AutoFetchRef(self, options, revision): |
| """Attempts to fetch |revision| if not available in local repo. |
| |
| Returns possibly updated revision.""" |
| if not scm.GIT.IsValidRevision(self.checkout_path, revision): |
| self._Fetch(options, refspec=revision) |
| revision = self._Capture(['rev-parse', 'FETCH_HEAD']) |
| return revision |
| |
| def _IsRunningUnderRosetta(self): |
| if sys.platform != 'darwin': |
| return False |
| if self._running_under_rosetta is None: |
| # If we are running under Rosetta, platform.machine() is |
| # 'x86_64'; we need to use a sysctl to see if we're being |
| # translated. |
| import ctypes |
| libSystem = ctypes.CDLL("libSystem.dylib") |
| ret = ctypes.c_int(0) |
| size = ctypes.c_size_t(4) |
| e = libSystem.sysctlbyname(ctypes.c_char_p(b'sysctl.proc_translated'), |
| ctypes.byref(ret), ctypes.byref(size), None, 0) |
| self._running_under_rosetta = e == 0 and ret.value == 1 |
| return self._running_under_rosetta |
| |
| def _Run(self, args, options, **kwargs): |
| # Disable 'unused options' warning | pylint: disable=unused-argument |
| kwargs.setdefault('cwd', self.checkout_path) |
| kwargs.setdefault('filter_fn', self.filter) |
| kwargs.setdefault('show_header', True) |
| env = scm.GIT.ApplyEnvVars(kwargs) |
| |
| cmd = ['git'] + args |
| |
| if self._IsRunningUnderRosetta(): |
| # We currently only ship an Intel Python binary in depot_tools. |
| # Intel binaries run under Rosetta on ARM Macs, and by default |
| # prefer to run their subprocesses as Intel under Rosetta too. |
| # Intel git running under Rosetta has a bug where it fails to |
| # clone src.git (rdar://7868319), so until we ship a native |
| # ARM python3 binary, explicitly use `arch` to let git run |
| # the native ARM slice instead of the Intel slice. |
| # TODO(thakis): Remove this again once we ship an arm64 python3 |
| # binary. |
| cmd = ['arch', '-arch', 'arm64e', '-arch', 'arm64'] + cmd |
| gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs) |
| |
| |
| class CipdPackage(object): |
| """A representation of a single CIPD package.""" |
| |
| def __init__(self, name, version, authority_for_subdir): |
| self._authority_for_subdir = authority_for_subdir |
| self._name = name |
| self._version = version |
| |
| @property |
| def authority_for_subdir(self): |
| """Whether this package has authority to act on behalf of its subdir. |
| |
| Some operations should only be performed once per subdirectory. A package |
| that has authority for its subdirectory is the only package that should |
| perform such operations. |
| |
| Returns: |
| bool; whether this package has subdir authority. |
| """ |
| return self._authority_for_subdir |
| |
| @property |
| def name(self): |
| return self._name |
| |
| @property |
| def version(self): |
| return self._version |
| |
| |
| class CipdRoot(object): |
| """A representation of a single CIPD root.""" |
| def __init__(self, root_dir, service_url): |
| self._all_packages = set() |
| self._mutator_lock = threading.Lock() |
| self._packages_by_subdir = collections.defaultdict(list) |
| self._root_dir = root_dir |
| self._service_url = service_url |
| |
| def add_package(self, subdir, package, version): |
| """Adds a package to this CIPD root. |
| |
| As far as clients are concerned, this grants both root and subdir authority |
| to packages arbitrarily. (The implementation grants root authority to the |
| first package added and subdir authority to the first package added for that |
| subdir, but clients should not depend on or expect that behavior.) |
| |
| Args: |
| subdir: str; relative path to where the package should be installed from |
| the cipd root directory. |
| package: str; the cipd package name. |
| version: str; the cipd package version. |
| Returns: |
| CipdPackage; the package that was created and added to this root. |
| """ |
| with self._mutator_lock: |
| cipd_package = CipdPackage( |
| package, version, |
| not self._packages_by_subdir[subdir]) |
| self._all_packages.add(cipd_package) |
| self._packages_by_subdir[subdir].append(cipd_package) |
| return cipd_package |
| |
| def packages(self, subdir): |
| """Get the list of configured packages for the given subdir.""" |
| return list(self._packages_by_subdir[subdir]) |
| |
| def clobber(self): |
| """Remove the .cipd directory. |
| |
| This is useful for forcing ensure to redownload and reinitialize all |
| packages. |
| """ |
| with self._mutator_lock: |
| cipd_cache_dir = os.path.join(self.root_dir, '.cipd') |
| try: |
| gclient_utils.rmtree(os.path.join(cipd_cache_dir)) |
| except OSError: |
| if os.path.exists(cipd_cache_dir): |
| raise |
| |
| @contextlib.contextmanager |
| def _create_ensure_file(self): |
| try: |
| contents = '$ParanoidMode CheckPresence\n\n' |
| for subdir, packages in sorted(self._packages_by_subdir.items()): |
| contents += '@Subdir %s\n' % subdir |
| for package in sorted(packages, key=lambda p: p.name): |
| contents += '%s %s\n' % (package.name, package.version) |
| contents += '\n' |
| ensure_file = None |
| with tempfile.NamedTemporaryFile( |
| suffix='.ensure', delete=False, mode='wb') as ensure_file: |
| ensure_file.write(contents.encode('utf-8', 'replace')) |
| yield ensure_file.name |
| finally: |
| if ensure_file is not None and os.path.exists(ensure_file.name): |
| os.remove(ensure_file.name) |
| |
| def ensure(self): |
| """Run `cipd ensure`.""" |
| with self._mutator_lock: |
| with self._create_ensure_file() as ensure_file: |
| cmd = [ |
| 'cipd', 'ensure', |
| '-log-level', 'error', |
| '-root', self.root_dir, |
| '-ensure-file', ensure_file, |
| ] |
| gclient_utils.CheckCallAndFilter( |
| cmd, print_stdout=True, show_header=True) |
| |
| def run(self, command): |
| if command == 'update': |
| self.ensure() |
| elif command == 'revert': |
| self.clobber() |
| self.ensure() |
| |
| def created_package(self, package): |
| """Checks whether this root created the given package. |
| |
| Args: |
| package: CipdPackage; the package to check. |
| Returns: |
| bool; whether this root created the given package. |
| """ |
| return package in self._all_packages |
| |
| @property |
| def root_dir(self): |
| return self._root_dir |
| |
| @property |
| def service_url(self): |
| return self._service_url |
| |
| |
| class CipdWrapper(SCMWrapper): |
| """Wrapper for CIPD. |
| |
| Currently only supports chrome-infra-packages.appspot.com. |
| """ |
| name = 'cipd' |
| |
| def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None, |
| out_cb=None, root=None, package=None): |
| super(CipdWrapper, self).__init__( |
| url=url, root_dir=root_dir, relpath=relpath, out_fh=out_fh, |
| out_cb=out_cb) |
| assert root.created_package(package) |
| self._package = package |
| self._root = root |
| |
| #override |
| def GetCacheMirror(self): |
| return None |
| |
| #override |
| def GetActualRemoteURL(self, options): |
| return self._root.service_url |
| |
| #override |
| def DoesRemoteURLMatch(self, options): |
| del options |
| return True |
| |
| def revert(self, options, args, file_list): |
| """Does nothing. |
| |
| CIPD packages should be reverted at the root by running |
| `CipdRoot.run('revert')`. |
| """ |
| pass |
| |
| def diff(self, options, args, file_list): |
| """CIPD has no notion of diffing.""" |
| pass |
| |
| def pack(self, options, args, file_list): |
| """CIPD has no notion of diffing.""" |
| pass |
| |
| def revinfo(self, options, args, file_list): |
| """Grab the instance ID.""" |
| try: |
| tmpdir = tempfile.mkdtemp() |
| describe_json_path = os.path.join(tmpdir, 'describe.json') |
| cmd = [ |
| 'cipd', 'describe', |
| self._package.name, |
| '-log-level', 'error', |
| '-version', self._package.version, |
| '-json-output', describe_json_path |
| ] |
| gclient_utils.CheckCallAndFilter(cmd) |
| with open(describe_json_path) as f: |
| describe_json = json.load(f) |
| return describe_json.get('result', {}).get('pin', {}).get('instance_id') |
| finally: |
| gclient_utils.rmtree(tmpdir) |
| |
| def status(self, options, args, file_list): |
| pass |
| |
| def update(self, options, args, file_list): |
| """Does nothing. |
| |
| CIPD packages should be updated at the root by running |
| `CipdRoot.run('update')`. |
| """ |
| pass |
