llvm_tools/fetch_cros_sdk_rolls.py

#!/usr/bin/env python3
# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Gets info about completed chromiumos-sdk runs.

Moreover, this script exists to get versions of published sdk tarballs in
gs://chromiumos-sdk/. The hope is that it'll help answer the question "when did
the toolchain ebuild ${x} go live?"
"""

# pylint: disable=cros-logging-import

import argparse
import json
import logging
import os
import shutil
import subprocess
import sys
import tempfile
from typing import Dict, List
from pathlib import Path


def fetch_all_sdk_manifest_paths() -> List[str]:
  """Fetches all paths of SDK manifests; newer = later in the return value."""
  results = subprocess.run(
      ['gsutil', 'ls', 'gs://chromiumos-sdk/cros-sdk-20??.*.Manifest'],
      check=True,
      stdout=subprocess.PIPE,
      encoding='utf-8',
  ).stdout
  # These are named so that sorted order == newest last.
  return sorted(x.strip() for x in results.splitlines())


def fetch_manifests_into(into_dir: Path, manifests: List[str]):
  # Wrap this in a `try` block because gsutil likes to print to stdout *and*
  # stderr even on success, so we silence them & only print on failure.
  try:
    subprocess.run(
        [
            'gsutil',
            '-m',
            'cp',
            '-I',
            str(into_dir),
        ],
        check=True,
        input='\n'.join(manifests),
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        encoding='utf-8',
    )
  except subprocess.CalledProcessError as e:
    logging.exception('gsutil failed; output:\n%s', e.stdout)


def load_manifest_versions(manifest: Path) -> Dict[str, str]:
  with manifest.open(encoding='utf-8') as f:
    raw_versions = json.load(f)

  # We get a dict of list of lists of versions and some other metadata, e.g.
  # {"foo/bar": [["1.2.3", {}]]}
  # Trim out the metadata.
  return {k: v[0][0] for k, v in raw_versions['packages'].items()}


def main():
  parser = argparse.ArgumentParser(
      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument(
      '-d', '--debug', action='store_true', help='Emit debugging output')
  parser.add_argument(
      '-n',
      '--number',
      type=int,
      default=20,
      help='Number of recent manifests to fetch info about. 0 means unlimited.')
  args = parser.parse_args()

  is_debug = args.debug
  logging.basicConfig(level=logging.DEBUG if is_debug else logging.INFO)

  logging.debug('Fetching SDK manifests')
  manifest_paths = fetch_all_sdk_manifest_paths()
  logging.debug('%d SDK manifests fetched', len(manifest_paths))

  number = args.number
  if number:
    manifest_paths = manifest_paths[-number:]

  tempdir = Path(tempfile.mkdtemp(prefix='cros-sdk-rolls'))
  try:
    logging.debug('Working in tempdir %r', tempdir)
    fetch_manifests_into(tempdir, manifest_paths)

    for path in manifest_paths:
      basename = os.path.basename(path)
      versions = load_manifest_versions(tempdir.joinpath(basename))
      print(f'{basename}: {versions["sys-devel/llvm"]}')
  finally:
    if is_debug:
      logging.debug('Keeping around tempdir %r to aid debugging', tempdir)
    else:
      shutil.rmtree(tempdir)


if __name__ == '__main__':
  sys.exit(main())