afdo_tools/generate_afdo_from_tryjob.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Given a tryjob and perf profile, generates an AFDO profile."""

from __future__ import print_function

import argparse
import distutils.spawn
import os
import os.path
import shutil
import subprocess
import sys
import tempfile

_CREATE_LLVM_PROF = 'create_llvm_prof'
_GS_PREFIX = 'gs://'


def _fetch_gs_artifact(remote_name, local_name):
  assert remote_name.startswith(_GS_PREFIX)
  subprocess.check_call(['gsutil', 'cp', remote_name, local_name])


def _fetch_and_maybe_unpack(remote_name, local_name):
  unpackers = [
      ('.tar.bz2', ['tar', 'xaf']),
      ('.bz2', ['bunzip2']),
      ('.tar.xz', ['tar', 'xaf']),
      ('.xz', ['xz', '-d']),
  ]

  unpack_ext = None
  unpack_cmd = None
  for ext, unpack in unpackers:
    if remote_name.endswith(ext):
      unpack_ext, unpack_cmd = ext, unpack
      break

  download_to = local_name + unpack_ext if unpack_ext else local_name
  _fetch_gs_artifact(remote_name, download_to)
  if unpack_cmd is not None:
    print('Unpacking', download_to)
    subprocess.check_output(unpack_cmd + [download_to])
    assert os.path.exists(local_name)


def _generate_afdo(perf_profile_loc, tryjob_loc, output_name):
  if perf_profile_loc.startswith(_GS_PREFIX):
    local_loc = 'perf.data'
    _fetch_and_maybe_unpack(perf_profile_loc, local_loc)
    perf_profile_loc = local_loc

  chrome_in_debug_loc = 'debug/opt/google/chrome/chrome.debug'
  debug_out = 'debug.tgz'
  _fetch_gs_artifact(os.path.join(tryjob_loc, 'debug.tgz'), debug_out)

  print('Extracting chrome.debug.')
  # This has tons of artifacts, and we only want Chrome; don't waste time
  # extracting the rest in _fetch_and_maybe_unpack.
  subprocess.check_call(['tar', 'xaf', 'debug.tgz', chrome_in_debug_loc])

  # Note that the AFDO tool *requires* a binary named `chrome` to be present if
  # we're generating a profile for chrome. It's OK for this to be split debug
  # information.
  os.rename(chrome_in_debug_loc, 'chrome')

  print('Generating AFDO profile.')
  subprocess.check_call([
      _CREATE_LLVM_PROF, '--out=' + output_name, '--binary=chrome',
      '--profile=' + perf_profile_loc
  ])


def _abspath_or_gs_link(path):
  if path.startswith(_GS_PREFIX):
    return path
  return os.path.abspath(path)


def _tryjob_arg(tryjob_arg):
  # Forward gs args through
  if tryjob_arg.startswith(_GS_PREFIX):
    return tryjob_arg

  # Clicking on the 'Artifacts' link gives us a pantheon link that's basically
  # a preamble and gs path.
  pantheon = 'https://pantheon.corp.google.com/storage/browser/'
  if tryjob_arg.startswith(pantheon):
    return _GS_PREFIX + tryjob_arg[len(pantheon):]

  # Otherwise, only do things with a tryjob ID (e.g. R75-11965.0.0-b3648595)
  if not tryjob_arg.startswith('R'):
    raise ValueError('Unparseable tryjob arg; give a tryjob ID, pantheon '
                     'link, or gs:// link. Please see source for more.')

  chell_path = 'chromeos-image-archive/chell-chrome-pfq-tryjob/'
  # ...And assume it's from chell, since that's the only thing we generate
  # profiles with today.
  return _GS_PREFIX + chell_path + tryjob_arg


def main():
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument(
      '--perf_profile',
      required=True,
      help='Path to our perf profile. Accepts either a gs:// path or local '
      'filepath.')
  parser.add_argument(
      '--tryjob',
      required=True,
      type=_tryjob_arg,
      help="Path to our tryjob's artifacts. Accepts a gs:// path, pantheon "
      'link, or tryjob ID, e.g. R75-11965.0.0-b3648595. In the last case, '
      'the assumption is that you ran a chell-chrome-pfq-tryjob.')
  parser.add_argument(
      '-o',
      '--output',
      default='afdo.prof',
      help='Where to put the AFDO profile. Default is afdo.prof.')
  parser.add_argument(
      '-k',
      '--keep_artifacts_on_failure',
      action='store_true',
      help="Don't remove the tempdir on failure")
  args = parser.parse_args()

  if not distutils.spawn.find_executable(_CREATE_LLVM_PROF):
    sys.exit(_CREATE_LLVM_PROF + ' not found; are you in the chroot?')

  profile = _abspath_or_gs_link(args.perf_profile)
  afdo_output = os.path.abspath(args.output)

  initial_dir = os.getcwd()
  temp_dir = tempfile.mkdtemp(prefix='generate_afdo')
  success = True
  try:
    os.chdir(temp_dir)
    _generate_afdo(profile, args.tryjob, afdo_output)

    # The AFDO tooling is happy to generate essentially empty profiles for us.
    # Chrome's profiles are often 8+ MB; if we only see a small fraction of
    # that, something's off. 512KB was arbitrarily selected.
    if os.path.getsize(afdo_output) < 512 * 1024:
      raise ValueError('The AFDO profile is suspiciously small for Chrome. '
                       'Something might have gone wrong.')
  except:
    success = False
    raise
  finally:
    os.chdir(initial_dir)

    if success or not args.keep_artifacts_on_failure:
      shutil.rmtree(temp_dir, ignore_errors=True)
    else:
      print('Artifacts are available at', temp_dir)


if __name__ == '__main__':
  sys.exit(main())