llvm_tools/auto_llvm_bisection.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Performs bisection on LLVM based off a .JSON file."""

from __future__ import print_function

import enum
import json
import os
import subprocess
import sys
import time
import traceback

import chroot
from llvm_bisection import BisectionExitStatus
import llvm_bisection
import update_tryjob_status

# Used to re-try for 'llvm_bisection.py' to attempt to launch more tryjobs.
BISECTION_RETRY_TIME_SECS = 10 * 60

# Wait time to then poll each tryjob whose 'status' value is 'pending'.
POLL_RETRY_TIME_SECS = 30 * 60

# The number of attempts for 'llvm_bisection.py' to launch more tryjobs.
#
# It is reset (break out of the `for` loop/ exit the program) if successfully
# launched more tryjobs or bisection is finished (no more revisions between
# start and end of the bisection).
BISECTION_ATTEMPTS = 3

# The limit for updating all tryjobs whose 'status' is 'pending'.
#
# If the time that has passed for polling exceeds this value, then the program
# will exit with the appropriate exit code.
POLLING_LIMIT_SECS = 18 * 60 * 60


class BuilderStatus(enum.Enum):
  """Actual values given via 'cros buildresult'."""

  PASS = 'pass'
  FAIL = 'fail'
  RUNNING = 'running'


builder_status_mapping = {
    BuilderStatus.PASS.value: update_tryjob_status.TryjobStatus.GOOD.value,
    BuilderStatus.FAIL.value: update_tryjob_status.TryjobStatus.BAD.value,
    BuilderStatus.RUNNING.value: update_tryjob_status.TryjobStatus.PENDING.value
}


def GetBuildResult(chroot_path, buildbucket_id):
  """Returns the conversion of the result of 'cros buildresult'."""

  # Calls 'cros buildresult' to get the status of the tryjob.
  try:
    tryjob_json = subprocess.check_output(
        [
            'cros_sdk', '--', 'cros', 'buildresult', '--buildbucket-id',
            str(buildbucket_id), '--report', 'json'
        ],
        cwd=chroot_path,
        stderr=subprocess.STDOUT,
        encoding='UTF-8',
    )
  except subprocess.CalledProcessError as err:
    if 'No build found. Perhaps not started' not in err.output:
      raise
    return None

  tryjob_content = json.loads(tryjob_json)

  build_result = str(tryjob_content['%d' % buildbucket_id]['status'])

  # The string returned by 'cros buildresult' might not be in the mapping.
  if build_result not in builder_status_mapping:
    raise ValueError('"cros buildresult" return value is invalid: %s' %
                     build_result)

  return builder_status_mapping[build_result]


def main():
  """Bisects LLVM using the result of `cros buildresult` of each tryjob.

  Raises:
    AssertionError: The script was run inside the chroot.
  """

  chroot.VerifyOutsideChroot()

  args_output = llvm_bisection.GetCommandLineArgs()

  if os.path.isfile(args_output.last_tested):
    print('Resuming bisection for %s' % args_output.last_tested)
  else:
    print('Starting a new bisection for %s' % args_output.last_tested)

  while True:
    # Update the status of existing tryjobs
    if os.path.isfile(args_output.last_tested):
      update_start_time = time.time()
      with open(args_output.last_tested) as json_file:
        json_dict = json.load(json_file)
      while True:
        print('\nAttempting to update all tryjobs whose "status" is '
              '"pending":')
        print('-' * 40)

        completed = True
        for tryjob in json_dict['jobs']:
          if tryjob[
              'status'] == update_tryjob_status.TryjobStatus.PENDING.value:
            status = GetBuildResult(args_output.chroot_path,
                                    tryjob['buildbucket_id'])
            if status:
              tryjob['status'] = status
            else:
              completed = False

        print('-' * 40)

        # Proceed to the next step if all the existing tryjobs have completed.
        if completed:
          break

        delta_time = time.time() - update_start_time

        if delta_time > POLLING_LIMIT_SECS:
          # Something is wrong with updating the tryjobs's 'status' via
          # `cros buildresult` (e.g. network issue, etc.).
          sys.exit('Failed to update pending tryjobs.')

        print('-' * 40)
        print('Sleeping for %d minutes.' % (POLL_RETRY_TIME_SECS // 60))
        time.sleep(POLL_RETRY_TIME_SECS)

      # There should always be update from the tryjobs launched in the
      # last iteration.
      temp_filename = '%s.new' % args_output.last_tested
      with open(temp_filename, 'w') as temp_file:
        json.dump(json_dict, temp_file, indent=4, separators=(',', ': '))
      os.rename(temp_filename, args_output.last_tested)

    # Launch more tryjobs.
    for cur_try in range(1, BISECTION_ATTEMPTS + 1):
      try:
        print('\nAttempting to launch more tryjobs if possible:')
        print('-' * 40)

        bisection_ret = llvm_bisection.main(args_output)

        print('-' * 40)

        # Stop if the bisection has completed.
        if bisection_ret == BisectionExitStatus.BISECTION_COMPLETE.value:
          sys.exit(0)

        # Successfully launched more tryjobs.
        break
      except Exception:
        traceback.print_exc()

        print('-' * 40)

        # Exceeded the number of times to launch more tryjobs.
        if cur_try == BISECTION_ATTEMPTS:
          sys.exit('Unable to continue bisection.')

        num_retries_left = BISECTION_ATTEMPTS - cur_try

        print('Retries left to continue bisection %d.' % num_retries_left)

        print('Sleeping for %d minutes.' % (BISECTION_RETRY_TIME_SECS // 60))
        time.sleep(BISECTION_RETRY_TIME_SECS)


if __name__ == '__main__':
  main()