blob: 62d9771628eb8f39a54339ee0d326afe1e7c33c8 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This is a tool for picking patches from upstream and applying them."""
import argparse
import configparser
import functools
import mailbox
import os
import pprint
import re
import signal
import subprocess
import sys
import textwrap
import urllib.request
import xmlrpc.client
errprint = functools.partial(print, file=sys.stderr)
_PWCLIENTRC = os.path.expanduser('~/.pwclientrc')
def _git(args, stdin=None, encoding='utf-8'):
"""Calls a git subcommand.
Similar to subprocess.check_output.
stdin: a string or bytes (depending on encoding) that will be passed
to the git subcommand.
encoding: either 'utf-8' (default) or None. Override it to None if
you want both stdin and stdout to be raw bytes.
the stdout of the git subcommand, same type as stdin. The output is
also run through strip to make sure there's no extra whitespace.
subprocess.CalledProcessError: when return code is not zero.
The exception has a .returncode attribute.
['git'] + args,
def _git_returncode(*args, **kwargs):
"""Same as _git, but return returncode instead of stdout.
Similar to
Never raises subprocess.CalledProcessError.
_git(*args, **kwargs)
return 0
except subprocess.CalledProcessError as e:
return e.returncode
def _get_conflicts():
"""Report conflicting files."""
resolutions = ('DD', 'AU', 'UD', 'UA', 'DU', 'AA', 'UU')
conflicts = []
output = _git(['status', '--porcelain', '--untracked-files=no'])
for line in output.splitlines():
if not line:
resolution, name = line.split(None, 1)
if resolution in resolutions:
conflicts.append(' ' + name)
if not conflicts:
return ''
return '\nConflicts:\n%s\n' % '\n'.join(conflicts)
def _find_linux_remote():
"""Find a remote pointing to a Linux upstream repository."""
for remote in _git(['remote']).splitlines():
if _git(['remote', 'get-url', remote]) in LINUX_URLS:
return remote
except subprocess.CalledProcessError:
# Kinda weird, get-url failing on an item that git just gave us.
return None
def _pause_for_merge(conflicts):
"""Pause and go in the background till user resolves the conflicts."""
git_root = _git(['rev-parse', '--show-toplevel'])
paths = (
os.path.join(git_root, '.git', 'rebase-apply'),
os.path.join(git_root, '.git', 'CHERRY_PICK_HEAD'),
for path in paths:
if os.path.exists(path):
errprint('Found "%s".' % path)
errprint('Please resolve the conflicts and restart the '
'shell job when done. Kill this job if you '
'aborted the conflict.')
os.kill(os.getpid(), signal.SIGTSTP)
# TODO: figure out what the state is after the merging, and go based on
# that (should we abort? skip? continue?)
# Perhaps check last commit message to see if it's the one we were using.
def _get_pw_url(project):
"""Retrieve the patchwork server URL from .pwclientrc.
project: patchwork project name; if None, we retrieve the default
from pwclientrc
config = configparser.ConfigParser()[_PWCLIENTRC])
if project is None:
project = config.get('options', 'default')
except (configparser.NoSectionError, configparser.NoOptionError) as e:
errprint('Error: no default patchwork project found in %s. (%r)'
if not config.has_option(project, 'url'):
errprint("Error: patchwork URL not found for project '%s'" % project)
url = config.get(project, 'url')
# Strip trailing 'xmlrpc' and/or trailing slash.
return re.sub('/(xmlrpc/)?$', '', url)
def _wrap_commit_line(prefix, content):
line = prefix + '=' + content
indent = ' ' * (len(prefix) + 1)
return textwrap.fill(line, COMMIT_MESSAGE_WIDTH, subsequent_indent=indent)
def _pick_patchwork(url, patch_id, args):
if args['tag'] is None:
args['tag'] = 'FROMLIST: '
opener = urllib.request.urlopen('%s/patch/%d/mbox' % (url, patch_id))
if opener.getcode() != 200:
errprint('Error: could not download patch - error code %d'
% opener.getcode())
patch_contents =
if not patch_contents:
errprint('Error: No patch content found')
message_id = mailbox.Message(patch_contents)['Message-Id']
message_id = re.sub('^<|>$', '', message_id.strip())
if args['source_line'] is None:
args['source_line'] = '(am from %s/patch/%d/)' % (url, patch_id)
args['source_line'] += (
'\n(also found at' % message_id)
# Auto-snarf the Change-Id if it was encoded into the Message-Id.
mo = re.match(r'.*(I[a-f0-9]{40})@changeid$', message_id)
if mo and args['changeid'] is None:
args['changeid'] =
if args['replace']:
_git(['reset', '--hard', 'HEAD~1'])
return _git_returncode(['am', '-3'], stdin=patch_contents, encoding=None)
def _match_patchwork(match, args):
"""Match location: pw://### or pw://PROJECT/###."""
pw_project =
patch_id = int(
if args['debug']:
print('_match_patchwork: pw_project=%s, patch_id=%d' %
(pw_project, patch_id))
url = _get_pw_url(pw_project)
return _pick_patchwork(url, patch_id, args)
def _match_msgid(match, args):
"""Match location: msgid://MSGID."""
msgid =
if args['debug']:
print('_match_msgid: message_id=%s' % (msgid))
# Patchwork requires the brackets so force it
msgid = '<' + msgid + '>'
url = None
for url in PATCHWORK_URLS:
rpc = xmlrpc.client.ServerProxy(url + '/xmlrpc/')
res = rpc.patch_list({'msgid': msgid})
if res:
patch_id = res[0]['id']
errprint('Error: could not find patch based on message id')
return _pick_patchwork(url, patch_id, args)
def _match_linux(match, args):
"""Match location: linux://HASH."""
commit =
if args['debug']:
print('_match_linux: commit=%s' % commit)
# Confirm a 'linux' remote is setup.
linux_remote = _find_linux_remote()
if not linux_remote:
errprint('Error: need a valid upstream remote')
linux_master = '%s/master' % linux_remote
_git(['merge-base', '--is-ancestor', commit, linux_master])
except subprocess.CalledProcessError:
errprint('Error: Commit not in %s' % linux_master)
if args['source_line'] is None:
commit = _git(['rev-parse', commit])
args['source_line'] = ('(cherry picked from commit %s)' %
if args['tag'] is None:
args['tag'] = 'UPSTREAM: '
if args['replace']:
_git(['reset', '--hard', 'HEAD~1'])
return _git_returncode(['cherry-pick', commit])
def _match_fromgit(match, args):
"""Match location: git://remote/branch/HASH."""
remote =
branch =
commit =
if args['debug']:
print('_match_fromgit: remote=%s branch=%s commit=%s' %
(remote, branch, commit))
_git(['merge-base', '--is-ancestor', commit,
'%s/%s' % (remote, branch)])
except subprocess.CalledProcessError:
errprint('Error: Commit not in %s/%s' % (remote, branch))
url = _git(['remote', 'get-url', remote])
if args['source_line'] is None:
commit = _git(['rev-parse', commit])
args['source_line'] = (
'(cherry picked from commit %s\n %s %s)' % (commit, url, branch))
if args['tag'] is None:
args['tag'] = 'FROMGIT: '
if args['replace']:
_git(['reset', '--hard', 'HEAD~1'])
return _git_returncode(['cherry-pick', commit])
def _match_gitfetch(match, args):
"""Match location: (git|https)://repoURL#branch/HASH."""
remote =
branch =
commit =
if args['debug']:
print('_match_gitfetch: remote=%s branch=%s commit=%s' %
(remote, branch, commit))
_git(['fetch', remote, branch])
except subprocess.CalledProcessError:
errprint('Error: Branch not in %s' % remote)
url = remote
if args['source_line'] is None:
commit = _git(['rev-parse', commit])
args['source_line'] = (
'(cherry picked from commit %s\n %s %s)' % (commit, url, branch))
if args['tag'] is None:
args['tag'] = 'FROMGIT: '
if args['replace']:
_git(['reset', '--hard', 'HEAD~1'])
return _git_returncode(['cherry-pick', commit])
def main(args):
"""This is the main entrypoint for fromupstream.
args: sys.argv[1:]
An int return code.
parser = argparse.ArgumentParser()
parser.add_argument('--bug', '-b',
type=str, help='BUG= line')
parser.add_argument('--test', '-t',
type=str, help='TEST= line')
parser.add_argument('--crbug', action='append',
type=int, help='BUG=chromium: line')
parser.add_argument('--buganizer', action='append',
type=int, help='BUG=b: line')
parser.add_argument('--changeid', '-c',
help='Overrides the gerrit generated Change-Id line')
parser.add_argument('--replace', '-r',
help='Replaces the HEAD commit with this one, taking '
'its properties(BUG, TEST, Change-Id). Useful for '
'updating commits.')
dest='signoff', action='store_false')
parser.add_argument('--debug', '-d', action='store_true',
help='Prints more verbose logs.')
help='Overrides the tag from the title')
parser.add_argument('--source', '-s',
dest='source_line', type=str,
help='Overrides the source line, last line, ex: '
'(am from http://....)')
help='Patchwork ID (pw://### or pw://PROJECT/###, '
'where PROJECT is defined in ~/.pwclientrc; if no '
'PROJECT is specified, the default is retrieved from '
'~/.pwclientrc), '
'Message-ID (msgid://MSGID), '
'linux commit like linux://HASH, or '
'git reference like git://remote/branch/HASH or '
'git://repoURL#branch/HASH or '
args = vars(parser.parse_args(args))
buglist = [args['bug']] if args['bug'] else []
if args['buganizer']:
buglist += ['b:{0}'.format(x) for x in args['buganizer']]
if args['crbug']:
buglist += ['chromium:{0}'.format(x) for x in args['crbug']]
if buglist:
args['bug'] = ', '.join(buglist)
if args['replace']:
old_commit_message = _git(['show', '-s', '--format=%B', 'HEAD'])
# It is possible that multiple Change-Ids are in the commit message
# (due to cherry picking). We only want to pull out the first one.
changeid_match ='^Change-Id: (.*)$',
old_commit_message, re.MULTILINE)
if changeid_match:
args['changeid'] =
bugs = re.findall('^BUG=(.*)$', old_commit_message, re.MULTILINE)
if args['bug'] is None and bugs:
args['bug'] = '\nBUG='.join(bugs)
tests = re.findall('^TEST=(.*)$', old_commit_message, re.MULTILINE)
if args['test'] is None and tests:
args['test'] = '\nTEST='.join(tests)
# TODO: deal with multiline BUG/TEST better
if args['bug'] is None or args['test'] is None:
parser.error('BUG=/TEST= lines are required; --replace can help '
'automate, or set via --bug/--test')
if args['debug']:
re_matches = (
(re.compile(r'^pw://(([^/]+)/)?(\d+)'), _match_patchwork),
(re.compile(r'^msgid://<?([^>]*)>?'), _match_msgid),
(re.compile(r'^linux://([0-9a-f]+)'), _match_linux),
(re.compile(r'^((git|https)://.+)#(.+)/([0-9a-f]+)$'), _match_gitfetch),
for location in args['locations']:
if args['debug']:
print('location=%s' % location)
for reg, handler in re_matches:
match = reg.match(location)
if match:
ret = handler(match, args)
errprint('Don\'t know what "%s" means.' % location)
if ret != 0:
conflicts = _get_conflicts()
if args['tag'] == 'UPSTREAM: ':
args['tag'] = 'BACKPORT: '
args['tag'] = 'BACKPORT: ' + args['tag']
conflicts = ''
# extract commit message
commit_message = _git(['show', '-s', '--format=%B', 'HEAD'])
# Remove stray Change-Id, most likely from merge resolution
commit_message = re.sub(r'Change-Id:.*\n?', '', commit_message)
# Note the source location before tagging anything else
commit_message += '\n' + args['source_line']
# add automatic Change ID, BUG, and TEST (and maybe signoff too) so
# next commands know where to work on
commit_message += '\n'
commit_message += conflicts
commit_message += '\n' + 'BUG=' + args['bug']
commit_message += '\n' + _wrap_commit_line('TEST', args['test'])
if args['signoff']:
extra = ['-s']
extra = []
_git(['commit'] + extra + ['--amend', '-F', '-'], stdin=commit_message)
# re-extract commit message
commit_message = _git(['show', '-s', '--format=%B', 'HEAD'])
# If we see a "Link: " that seems to point to a Message-Id with an
# automatic Change-Id we'll snarf it out.
mo ='^Link:.*(I[a-f0-9]{40})@changeid', commit_message,
if mo and args['changeid'] is None:
args['changeid'] =
# replace changeid if needed
if args['changeid'] is not None:
commit_message = re.sub(r'(Change-Id: )(\w+)', r'\1%s' %
args['changeid'], commit_message)
args['changeid'] = None
# decorate it that it's from outside
commit_message = args['tag'] + commit_message
# commit everything
_git(['commit', '--amend', '-F', '-'], stdin=commit_message)
return 0
if __name__ == '__main__':