| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # Copyright 2020 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # pylint: disable=cros-logging-import |
| |
| """Adds a cherrypick to LLVM's PATCHES.json.""" |
| |
| from __future__ import print_function |
| |
| import argparse |
| import json |
| import logging |
| import os |
| import shlex |
| import subprocess |
| import sys |
| |
| import chroot |
| import get_llvm_hash |
| import git |
| import git_llvm_rev |
| import update_chromeos_llvm_hash |
| |
| |
| def add_cherrypick(patches_json_path: str, patches_dir: str, |
| relative_patches_dir: str, start_version: git_llvm_rev.Rev, |
| llvm_dir: str, rev: git_llvm_rev.Rev, sha: str, |
| package: str): |
| with open(patches_json_path, encoding='utf-8') as f: |
| patches_json = json.load(f) |
| |
| file_name = sha + '.patch' |
| rel_patch_path = os.path.join(relative_patches_dir, file_name) |
| |
| for p in patches_json: |
| rel_path = p['rel_patch_path'] |
| if rel_path == rel_patch_path: |
| raise ValueError('Patch at %r already exists in PATCHES.json' % rel_path) |
| if sha in rel_path: |
| logging.warning( |
| 'Similarly-named patch already exists in PATCHES.json: %r', rel_path) |
| |
| with open(os.path.join(patches_dir, file_name), 'wb') as f: |
| cmd = ['git', 'show', sha] |
| # Only apply the part of the patch that belongs to this package, expect |
| # LLVM. This is because some packages are built with LLVM ebuild on X86 but |
| # not on the other architectures. e.g. compiler-rt. Therefore always apply |
| # the entire patch to LLVM ebuild as a workaround. |
| if package != 'llvm': |
| cmd.append(package_to_project(package)) |
| subprocess.check_call(cmd, stdout=f, cwd=llvm_dir) |
| |
| commit_subject = subprocess.check_output( |
| ['git', 'log', '-n1', '--format=%s', sha], cwd=llvm_dir, encoding='utf-8') |
| |
| patches_json.append({ |
| 'comment': commit_subject.strip(), |
| 'rel_patch_path': rel_patch_path, |
| 'start_version': start_version.number, |
| 'end_version': rev.number, |
| }) |
| |
| temp_file = patches_json_path + '.tmp' |
| with open(temp_file, 'w', encoding='utf-8') as f: |
| json.dump(patches_json, f, indent=4, separators=(',', ': ')) |
| os.rename(temp_file, patches_json_path) |
| |
| |
| def parse_ebuild_for_assignment(ebuild_path: str, var_name: str) -> str: |
| # '_pre' filters the LLVM 9.0 ebuild, which we never want to target, from |
| # this list. |
| candidates = [ |
| x for x in os.listdir(ebuild_path) |
| if x.endswith('.ebuild') and '_pre' in x |
| ] |
| |
| if not candidates: |
| raise ValueError('No ebuilds found under %r' % ebuild_path) |
| |
| ebuild = os.path.join(ebuild_path, max(candidates)) |
| with open(ebuild, encoding='utf-8') as f: |
| var_name_eq = var_name + '=' |
| for orig_line in f: |
| if not orig_line.startswith(var_name_eq): |
| continue |
| |
| # We shouldn't see much variety here, so do the simplest thing possible. |
| line = orig_line[len(var_name_eq):] |
| # Remove comments |
| line = line.split('#')[0] |
| # Remove quotes |
| line = shlex.split(line) |
| if len(line) != 1: |
| raise ValueError('Expected exactly one quoted value in %r' % orig_line) |
| return line[0].strip() |
| |
| raise ValueError('No %s= line found in %r' % (var_name, ebuild)) |
| |
| |
| # Resolves a git ref (or similar) to a LLVM SHA. |
| def resolve_llvm_ref(llvm_dir: str, sha: str) -> str: |
| return subprocess.check_output( |
| ['git', 'rev-parse', sha], |
| encoding='utf-8', |
| cwd=llvm_dir, |
| ).strip() |
| |
| |
| # Get the package name of an LLVM project |
| def project_to_package(project: str) -> str: |
| if project == 'libunwind': |
| return 'llvm-libunwind' |
| return project |
| |
| |
| # Get the LLVM project name of a package |
| def package_to_project(package: str) -> str: |
| if package == 'llvm-libunwind': |
| return 'libunwind' |
| return package |
| |
| |
| # Get the LLVM projects change in the specifed sha |
| def get_package_names(sha: str, llvm_dir: str) -> list: |
| paths = subprocess.check_output( |
| ['git', 'show', '--name-only', '--format=', sha], |
| cwd=llvm_dir, |
| encoding='utf-8').splitlines() |
| # Some LLVM projects are built by LLVM ebuild on X86, so always apply the |
| # patch to LLVM ebuild |
| packages = {'llvm'} |
| # Detect if there are more packages to apply the patch to |
| for path in paths: |
| package = project_to_package(path.split('/')[0]) |
| if package in ('compiler-rt', 'libcxx', 'libcxxabi', 'llvm-libunwind'): |
| packages.add(package) |
| packages = list(sorted(packages)) |
| return packages |
| |
| |
| def main(): |
| chroot.VerifyOutsideChroot() |
| logging.basicConfig( |
| format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s', |
| level=logging.INFO, |
| ) |
| |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument( |
| '--chroot_path', |
| default=os.path.join(os.path.expanduser('~'), 'chromiumos'), |
| help='the path to the chroot (default: %(default)s)') |
| parser.add_argument( |
| '--start_sha', |
| default='llvm-next', |
| help='LLVM SHA that the patch should start applying at. You can specify ' |
| '"llvm" or "llvm-next", as well. Defaults to %(default)s.') |
| parser.add_argument( |
| '--sha', |
| required=True, |
| action='append', |
| help='The LLVM git SHA to cherry-pick.') |
| parser.add_argument( |
| '--create_cl', |
| default=False, |
| action='store_true', |
| help='Automatically create a CL if specified') |
| args = parser.parse_args() |
| |
| llvm_config = git_llvm_rev.LLVMConfig( |
| remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools()) |
| |
| llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths( |
| args.chroot_path, |
| chroot.GetChrootEbuildPaths(args.chroot_path, ['sys-devel/llvm']))[0] |
| start_sha = args.start_sha |
| if start_sha == 'llvm': |
| start_sha = parse_ebuild_for_assignment( |
| os.path.dirname(llvm_symlink), 'LLVM_HASH') |
| elif start_sha == 'llvm-next': |
| start_sha = parse_ebuild_for_assignment( |
| os.path.dirname(llvm_symlink), 'LLVM_NEXT_HASH') |
| logging.info('Base llvm hash == %s', start_sha) |
| |
| start_sha = resolve_llvm_ref(llvm_config.dir, start_sha) |
| start_rev = git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha) |
| |
| if args.create_cl: |
| branch = 'cherry-pick' |
| symlink = os.path.dirname( |
| chroot.GetChrootEbuildPaths(args.chroot_path, ['sys-devel/llvm'])[0]) |
| symlink = chroot.ConvertChrootPathsToAbsolutePaths(args.chroot_path, |
| [symlink])[0] |
| symlink_dir = os.path.dirname(symlink) |
| git.CreateBranch(symlink_dir, branch) |
| symlinks_to_uprev = [] |
| commit_messages = [ |
| 'llvm: cherry-pick CLs from upstream\n', |
| ] |
| |
| for sha in args.sha: |
| sha = resolve_llvm_ref(llvm_config.dir, sha) |
| rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) |
| # Find out the llvm projects changed in this commit |
| packages = get_package_names(sha, llvm_config.dir) |
| # Find out the ebuild symlinks of the corresponding ChromeOS packages |
| symlinks = chroot.GetChrootEbuildPaths(args.chroot_path, [ |
| 'sys-devel/llvm' if package == 'llvm' else 'sys-libs/' + package |
| for package in packages |
| ]) |
| symlinks = chroot.ConvertChrootPathsToAbsolutePaths(args.chroot_path, |
| symlinks) |
| |
| # Create a patch and add its metadata for each package |
| for package, symlink in zip(packages, symlinks): |
| symlink_dir = os.path.dirname(symlink) |
| patches_json_path = os.path.join(symlink_dir, 'files/PATCHES.json') |
| relative_patches_dir = 'cherry' if package == 'llvm' else '' |
| patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir) |
| logging.info('Cherrypicking %s (%s) into %s', rev, sha, package) |
| add_cherrypick(patches_json_path, patches_dir, relative_patches_dir, |
| start_rev, llvm_config.dir, rev, sha, package) |
| |
| if args.create_cl: |
| symlinks_to_uprev.extend(symlinks) |
| commit_messages.extend([ |
| '\n\nreviews.llvm.org/rG%s\n' % sha, |
| subprocess.check_output(['git', 'log', '-n1', '--oneline', sha], |
| cwd=llvm_config.dir, |
| encoding='utf-8') |
| ]) |
| |
| logging.info('Complete.') |
| |
| if args.create_cl: |
| symlinks_to_uprev = list(sorted(set(symlinks_to_uprev))) |
| for symlink in symlinks_to_uprev: |
| update_chromeos_llvm_hash.UprevEbuildSymlink(symlink) |
| subprocess.check_output(['git', 'add', '--all'], cwd=symlink_dir) |
| git.UploadChanges(symlink_dir, branch, commit_messages) |
| git.DeleteBranch(symlink_dir, branch) |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |