llvm_tools/update_chromeos_llvm_hash.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Updates the LLVM hash and uprevs the build of the specified packages.

For each package, a temporary repo is created and the changes are uploaded
for review.
"""

from __future__ import print_function

import argparse
import datetime
import enum
import os
import re
import subprocess

import chroot
import failure_modes
import get_llvm_hash
import git
import llvm_patch_management


# Specify which LLVM hash to update
class LLVMVariant(enum.Enum):
  """Represent the LLVM hash in an ebuild file to update."""

  current = 'LLVM_HASH'
  next = 'LLVM_NEXT_HASH'


# If set to `True`, then the contents of `stdout` after executing a command will
# be displayed to the terminal.
verbose = False


def defaultCrosRoot():
  """Get default location of chroot_path.

  The logic assumes that the cros_root is ~/chromiumos, unless llvm_tools is
  inside of a CrOS checkout, in which case that checkout should be used.

  Returns:
    The best guess location for the cros checkout.
  """
  llvm_tools_path = os.path.realpath(os.path.dirname(__file__))
  if llvm_tools_path.endswith('src/third_party/toolchain-utils/llvm_tools'):
    return os.path.join(llvm_tools_path, '../../../../')
  return '~/chromiumos'


def GetCommandLineArgs():
  """Parses the command line for the optional command line arguments.

  Returns:
    The log level to use when retrieving the LLVM hash or google3 LLVM version,
    the chroot path to use for executing chroot commands,
    a list of a package or packages to update their LLVM next hash,
    and the LLVM version to use when retrieving the LLVM hash.
  """

  # Create parser and add optional command-line arguments.
  parser = argparse.ArgumentParser(
      description="Updates the build's hash for llvm-next.")

  # Add argument for a specific chroot path.
  parser.add_argument(
      '--chroot_path',
      default=defaultCrosRoot(),
      help='the path to the chroot (default: %(default)s)')

  # Add argument for specific builds to uprev and update their llvm-next hash.
  parser.add_argument(
      '--update_packages',
      default=['sys-devel/llvm'],
      required=False,
      nargs='+',
      help='the ebuilds to update their hash for llvm-next '
      '(default: %(default)s)')

  # Add argument for whether to display command contents to `stdout`.
  parser.add_argument(
      '--verbose',
      action='store_true',
      help='display contents of a command to the terminal '
      '(default: %(default)s)')

  # Add argument for the LLVM hash to update
  parser.add_argument(
      '--is_llvm_next',
      action='store_true',
      help='which llvm hash to update. If specified, update LLVM_NEXT_HASH. '
      'Otherwise, update LLVM_HASH')

  # Add argument for the LLVM version to use.
  parser.add_argument(
      '--llvm_version',
      type=get_llvm_hash.IsSvnOption,
      required=True,
      help='which git hash to use. Either a svn revision, or one '
      'of %s' % sorted(get_llvm_hash.KNOWN_HASH_SOURCES))

  # Add argument for the mode of the patch management when handling patches.
  parser.add_argument(
      '--failure_mode',
      default=failure_modes.FailureModes.FAIL.value,
      choices=[
          failure_modes.FailureModes.FAIL.value,
          failure_modes.FailureModes.CONTINUE.value,
          failure_modes.FailureModes.DISABLE_PATCHES.value,
          failure_modes.FailureModes.REMOVE_PATCHES.value
      ],
      help='the mode of the patch manager when handling failed patches '
      '(default: %(default)s)')

  # Add argument for the patch metadata file.
  parser.add_argument(
      '--patch_metadata_file',
      default='PATCHES.json',
      help='the .json file that has all the patches and their '
      'metadata if applicable (default: PATCHES.json inside $FILESDIR)')

  # Parse the command line.
  args_output = parser.parse_args()

  # FIXME: We shouldn't be using globals here, but until we fix it, make pylint
  # stop complaining about it.
  # pylint: disable=global-statement
  global verbose

  verbose = args_output.verbose

  return args_output


def GetEbuildPathsFromSymLinkPaths(symlinks):
  """Reads the symlink(s) to get the ebuild path(s) to the package(s).

  Args:
    symlinks: A list of absolute path symlink/symlinks that point
    to the package's ebuild.

  Returns:
    A dictionary where the key is the absolute path of the symlink and the value
    is the absolute path to the ebuild that was read from the symlink.

  Raises:
    ValueError: Invalid symlink(s) were provided.
  """

  # A dictionary that holds:
  #   key: absolute symlink path
  #   value: absolute ebuild path
  resolved_paths = {}

  # Iterate through each symlink.
  #
  # For each symlink, check that it is a valid symlink,
  # and then construct the ebuild path, and
  # then add the ebuild path to the dict.
  for cur_symlink in symlinks:
    if not os.path.islink(cur_symlink):
      raise ValueError('Invalid symlink provided: %s' % cur_symlink)

    # Construct the absolute path to the ebuild.
    ebuild_path = os.path.realpath(cur_symlink)

    if cur_symlink not in resolved_paths:
      resolved_paths[cur_symlink] = ebuild_path

  return resolved_paths


def UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version):
  """Updates the LLVM hash in the ebuild.

  The build changes are staged for commit in the temporary repo.

  Args:
    ebuild_path: The absolute path to the ebuild.
    llvm_variant: Which LLVM hash to update.
    git_hash: The new git hash.
    svn_version: The SVN-style revision number of git_hash.

  Raises:
    ValueError: Invalid ebuild path provided or failed to stage the commit
    of the changes or failed to update the LLVM hash.
  """

  # Iterate through each ebuild.
  #
  # For each ebuild, read the file in
  # advance and then create a temporary file
  # that gets updated with the new LLVM hash
  # and revision number and then the ebuild file
  # gets updated to the temporary file.

  if not os.path.isfile(ebuild_path):
    raise ValueError('Invalid ebuild path provided: %s' % ebuild_path)

  temp_ebuild_file = '%s.temp' % ebuild_path

  with open(ebuild_path) as ebuild_file:
    # write updates to a temporary file in case of interrupts
    with open(temp_ebuild_file, 'w') as temp_file:
      for cur_line in ReplaceLLVMHash(ebuild_file, llvm_variant, git_hash,
                                      svn_version):
        temp_file.write(cur_line)
  os.rename(temp_ebuild_file, ebuild_path)

  # Get the path to the parent directory.
  parent_dir = os.path.dirname(ebuild_path)

  # Stage the changes.
  subprocess.check_output(['git', '-C', parent_dir, 'add', ebuild_path])


def ReplaceLLVMHash(ebuild_lines, llvm_variant, git_hash, svn_version):
  """Updates the LLVM git hash.

  Args:
    ebuild_lines: The contents of the ebuild file.
    llvm_variant: The LLVM hash to update.
    git_hash: The new git hash.
    svn_version: The SVN-style revision number of git_hash.

  Yields:
    lines of the modified ebuild file
  """
  is_updated = False
  llvm_regex = re.compile('^' + re.escape(llvm_variant.value) +
                          '=\"[a-z0-9]+\"')
  for cur_line in ebuild_lines:
    if not is_updated and llvm_regex.search(cur_line):
      # Update the git hash and revision number.
      cur_line = '%s=\"%s\" # r%d\n' % (llvm_variant.value, git_hash,
                                        svn_version)

      is_updated = True

    yield cur_line

  if not is_updated:
    raise ValueError('Failed to update %s' % llvm_variant.value)


def UprevEbuildSymlink(symlink):
  """Uprevs the symlink's revision number.

  Increases the revision number by 1 and stages the change in
  the temporary repo.

  Args:
    symlink: The absolute path of an ebuild symlink.

  Raises:
    ValueError: Failed to uprev the symlink or failed to stage the changes.
  """

  if not os.path.islink(symlink):
    raise ValueError('Invalid symlink provided: %s' % symlink)

  new_symlink, is_changed = re.subn(
      r'r([0-9]+).ebuild',
      lambda match: 'r%s.ebuild' % str(int(match.group(1)) + 1),
      symlink,
      count=1)

  if not is_changed:
    raise ValueError('Failed to uprev the symlink.')

  # rename the symlink
  subprocess.check_output(
      ['git', '-C',
       os.path.dirname(symlink), 'mv', symlink, new_symlink])


def UprevEbuildToVersion(symlink, svn_version, git_hash):
  """Uprevs the ebuild's revision number.

  Increases the revision number by 1 and stages the change in
  the temporary repo.

  Args:
    symlink: The absolute path of an ebuild symlink.
    svn_version: The SVN-style revision number of git_hash.
    git_hash: The new git hash.

  Raises:
    ValueError: Failed to uprev the ebuild or failed to stage the changes.
    AssertionError: No llvm version provided for an LLVM uprev
  """

  if not os.path.islink(symlink):
    raise ValueError('Invalid symlink provided: %s' % symlink)

  ebuild = os.path.realpath(symlink)
  llvm_major_version = get_llvm_hash.GetLLVMMajorVersion(git_hash)
  # llvm
  package = os.path.basename(os.path.dirname(symlink))
  if not package:
    raise ValueError('Tried to uprev an unknown package')
  if package == 'llvm':
    new_ebuild, is_changed = re.subn(
        r'(\d+)\.(\d+)_pre([0-9]+)_p([0-9]+)',
        '%s.\\2_pre%s_p%s' % (llvm_major_version, svn_version,
                              datetime.datetime.today().strftime('%Y%m%d')),
        ebuild,
        count=1)
  # any other package
  else:
    new_ebuild, is_changed = re.subn(
        r'(\d+)\.(\d+)_pre([0-9]+)',
        '%s.\\2_pre%s' % (llvm_major_version, svn_version), ebuild, count=1)

  if not is_changed:  # failed to increment the revision number
    raise ValueError('Failed to uprev the ebuild.')

  symlink_dir = os.path.dirname(symlink)

  # Rename the ebuild
  subprocess.check_output(['git', '-C', symlink_dir, 'mv', ebuild, new_ebuild])

  # Create a symlink of the renamed ebuild
  new_symlink = new_ebuild[:-len('.ebuild')] + '-r1.ebuild'
  subprocess.check_output(['ln', '-s', '-r', new_ebuild, new_symlink])

  if not os.path.islink(new_symlink):
    raise ValueError('Invalid symlink name: %s' % new_ebuild[:-len('.ebuild')])

  subprocess.check_output(['git', '-C', symlink_dir, 'add', new_symlink])

  # Remove the old symlink
  subprocess.check_output(['git', '-C', symlink_dir, 'rm', symlink])


def CreatePathDictionaryFromPackages(chroot_path, update_packages):
  """Creates a symlink and ebuild path pair dictionary from the packages.

  Args:
    chroot_path: The absolute path to the chroot.
    update_packages: The filtered packages to be updated.

  Returns:
    A dictionary where the key is the absolute path to the symlink
    of the package and the value is the absolute path to the ebuild of
    the package.
  """

  # Construct a list containing the chroot file paths of the package(s).
  chroot_file_paths = chroot.GetChrootEbuildPaths(chroot_path, update_packages)

  # Construct a list containing the symlink(s) of the package(s).
  symlink_file_paths = chroot.ConvertChrootPathsToAbsolutePaths(
      chroot_path, chroot_file_paths)

  # Create a dictionary where the key is the absolute path of the symlink to
  # the package and the value is the absolute path to the ebuild of the package.
  return GetEbuildPathsFromSymLinkPaths(symlink_file_paths)


def RemovePatchesFromFilesDir(patches):
  """Removes the patches from $FILESDIR of a package.

  Args:
    patches: A list of absolute pathes of patches to remove

  Raises:
    ValueError: Failed to remove a patch in $FILESDIR.
  """

  for patch in patches:
    subprocess.check_output(
        ['git', '-C', os.path.dirname(patch), 'rm', '-f', patch])


def StagePatchMetadataFileForCommit(patch_metadata_file_path):
  """Stages the updated patch metadata file for commit.

  Args:
    patch_metadata_file_path: The absolute path to the patch metadata file.

  Raises:
    ValueError: Failed to stage the patch metadata file for commit or invalid
    patch metadata file.
  """

  if not os.path.isfile(patch_metadata_file_path):
    raise ValueError('Invalid patch metadata file provided: %s' %
                     patch_metadata_file_path)

  # Cmd to stage the patch metadata file for commit.
  subprocess.check_output([
      'git', '-C',
      os.path.dirname(patch_metadata_file_path), 'add', patch_metadata_file_path
  ])


def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages):
  """Stages the patch results of the packages to the commit message.

  Args:
    package_info_dict: A dictionary where the key is the package name and the
    value is a dictionary that contains information about the patches of the
    package (key).
    commit_messages: The commit message that has the updated ebuilds and
    upreving information.

  Returns:
    commit_messages with new additions
  """

  # For each package, check if any patches for that package have
  # changed, if so, add which patches have changed to the commit
  # message.
  for package_name, patch_info_dict in package_info_dict.items():
    if (patch_info_dict['disabled_patches'] or
        patch_info_dict['removed_patches'] or
        patch_info_dict['modified_metadata']):
      cur_package_header = '\nFor the package %s:' % package_name
      commit_messages.append(cur_package_header)

    # Add to the commit message that the patch metadata file was modified.
    if patch_info_dict['modified_metadata']:
      patch_metadata_path = patch_info_dict['modified_metadata']
      commit_messages.append('The patch metadata file %s was modified' %
                             os.path.basename(patch_metadata_path))

      StagePatchMetadataFileForCommit(patch_metadata_path)

    # Add each disabled patch to the commit message.
    if patch_info_dict['disabled_patches']:
      commit_messages.append('The following patches were disabled:')

      for patch_path in patch_info_dict['disabled_patches']:
        commit_messages.append(os.path.basename(patch_path))

    # Add each removed patch to the commit message.
    if patch_info_dict['removed_patches']:
      commit_messages.append('The following patches were removed:')

      for patch_path in patch_info_dict['removed_patches']:
        commit_messages.append(os.path.basename(patch_path))

      RemovePatchesFromFilesDir(patch_info_dict['removed_patches'])

  return commit_messages


def UpdatePackages(packages, llvm_variant, git_hash, svn_version, chroot_path,
                   patch_metadata_file, mode, git_hash_source,
                   extra_commit_msg):
  """Updates an LLVM hash and uprevs the ebuild of the packages.

  A temporary repo is created for the changes. The changes are
  then uploaded for review.

  Args:
    packages: A list of all the packages that are going to be updated.
    llvm_variant: The LLVM hash to update.
    git_hash: The new git hash.
    svn_version: The SVN-style revision number of git_hash.
    chroot_path: The absolute path to the chroot.
    patch_metadata_file: The name of the .json file in '$FILESDIR/' that has
    the patches and its metadata.
    mode: The mode of the patch manager when handling an applicable patch
    that failed to apply.
      Ex. 'FailureModes.FAIL'
    git_hash_source: The source of which git hash to use based off of.
      Ex. 'google3', 'tot', or <version> such as 365123
    extra_commit_msg: extra test to append to the commit message.

  Returns:
    A nametuple that has two (key, value) pairs, where the first pair is the
    Gerrit commit URL and the second pair is the change list number.
  """

  # Determines whether to print the result of each executed command.
  llvm_patch_management.verbose = verbose

  # Construct a dictionary where the key is the absolute path of the symlink to
  # the package and the value is the absolute path to the ebuild of the package.
  paths_dict = CreatePathDictionaryFromPackages(chroot_path, packages)

  repo_path = os.path.dirname(next(iter(paths_dict.values())))

  branch = 'update-' + llvm_variant.value + '-' + git_hash

  git.CreateBranch(repo_path, branch)

  try:
    commit_message_header = 'llvm'
    if llvm_variant == LLVMVariant.next:
      commit_message_header = 'llvm-next'
    if git_hash_source in get_llvm_hash.KNOWN_HASH_SOURCES:
      commit_message_header += ('/%s: upgrade to %s (r%d)' %
                                (git_hash_source, git_hash, svn_version))
    else:
      commit_message_header += (': upgrade to %s (r%d)' %
                                (git_hash, svn_version))

    commit_messages = [
        commit_message_header + '\n',
        'The following packages have been updated:',
    ]

    # Holds the list of packages that are updating.
    packages = []

    # Iterate through the dictionary.
    #
    # For each iteration:
    # 1) Update the ebuild's LLVM hash.
    # 2) Uprev the ebuild (symlink).
    # 3) Add the modified package to the commit message.
    for symlink_path, ebuild_path in paths_dict.items():
      path_to_ebuild_dir = os.path.dirname(ebuild_path)

      UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version)

      if llvm_variant == LLVMVariant.current:
        UprevEbuildToVersion(symlink_path, svn_version, git_hash)
      else:
        UprevEbuildSymlink(symlink_path)

      cur_dir_name = os.path.basename(path_to_ebuild_dir)
      parent_dir_name = os.path.basename(os.path.dirname(path_to_ebuild_dir))

      packages.append('%s/%s' % (parent_dir_name, cur_dir_name))
      commit_messages.append('%s/%s' % (parent_dir_name, cur_dir_name))

    EnsurePackageMaskContains(chroot_path, git_hash)

    # Handle the patches for each package.
    package_info_dict = llvm_patch_management.UpdatePackagesPatchMetadataFile(
        chroot_path, svn_version, patch_metadata_file, packages, mode)

    # Update the commit message if changes were made to a package's patches.
    commit_messages = StagePackagesPatchResultsForCommit(
        package_info_dict, commit_messages)

    if extra_commit_msg:
      commit_messages.append(extra_commit_msg)

    change_list = git.UploadChanges(repo_path, branch, commit_messages)

  finally:
    git.DeleteBranch(repo_path, branch)

  return change_list


def EnsurePackageMaskContains(chroot_path, git_hash):
  """Adds the major version of llvm to package.mask if it's not already present.

  Args:
    chroot_path: The absolute path to the chroot.
    git_hash: The new git hash.

  Raises:
    FileExistsError: package.mask not found in ../../chromiumos-overlay
  """

  llvm_major_version = get_llvm_hash.GetLLVMMajorVersion(git_hash)

  overlay_dir = os.path.join(chroot_path, 'src/third_party/chromiumos-overlay')
  mask_path = os.path.join(overlay_dir,
                           'profiles/targets/chromeos/package.mask')
  with open(mask_path, 'r+') as mask_file:
    mask_contents = mask_file.read()
    expected_line = '=sys-devel/llvm-%s.0_pre*\n' % llvm_major_version
    if expected_line not in mask_contents:
      mask_file.write(expected_line)

  subprocess.check_output(['git', '-C', overlay_dir, 'add', mask_path])


def main():
  """Updates the LLVM next hash for each package.

  Raises:
    AssertionError: The script was run inside the chroot.
  """

  chroot.VerifyOutsideChroot()

  args_output = GetCommandLineArgs()

  llvm_variant = LLVMVariant.current
  if args_output.is_llvm_next:
    llvm_variant = LLVMVariant.next

  git_hash_source = args_output.llvm_version

  git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption(
      git_hash_source)

  change_list = UpdatePackages(
      args_output.update_packages,
      llvm_variant,
      git_hash,
      svn_version,
      args_output.chroot_path,
      args_output.patch_metadata_file,
      failure_modes.FailureModes(args_output.failure_mode),
      git_hash_source,
      extra_commit_msg=None)

  print('Successfully updated packages to %s (%d)' % (git_hash, svn_version))
  print('Gerrit URL: %s' % change_list.url)
  print('Change list number: %d' % change_list.cl_number)


if __name__ == '__main__':
  main()