cli/cros/cros_bisect.py

# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Bisect culprit commit that causes regression."""

from __future__ import print_function

from chromite.cros_bisect import autotest_evaluator
from chromite.cros_bisect import git_bisector
from chromite.cros_bisect import simple_chrome_builder
from chromite.cli import command
from chromite.lib import commandline
from chromite.lib import cros_logging as logging
from chromite.lib import remote_access


@command.CommandDecorator('bisect')
class BisectCommand(command.CliCommand):
  """Bisect commits to find the culprit responsible for regression."""

  EPILOG = """
Bisects commits of the given repository within given range, builds Chromium for
CrOS, deploys to DUT (device under test) and run the verifier to decide if the
commit is good or bad to find the culprit commit. Right now, supported
repository is Chromium; supported verifier is Autotest.
Note that before starting "cros bisect" please install an OS image on the DUT
that is close to the failure point, in particular from the corresponding branch.
"""

  REPO = ('chromium', )
  BUILDER = {'chromium': simple_chrome_builder.SimpleChromeBuilder}
  EVALUATOR = ('autotest', )

  @classmethod
  def AddParser(cls, parser):
    super(BisectCommand, cls).AddParser(parser)
    parser.add_argument(
        '-G', '--good', metavar='good_commit', required=True,
        help='A good revision to start bisection. Should be earlier than the '
             'bad revision.')
    parser.add_argument(
        '-B', '--bad', metavar='bad_commit', default='HEAD',
        help='A bad revision to start bisection. Should be later than the good '
             'revision. Default HEAD (ToT).')
    parser.add_argument(
        '-r', '--remote', metavar='IP address / hostname', required=True,
        type=commandline.DeviceParser(commandline.DEVICE_SCHEME_SSH),
        help='The IP address/hostname of remote device which is going to test.')
    parser.add_argument(
        '-b', '--board', metavar='board_name',
        help='The board name of the ChromeOS device under test. Obtained from '
             'DUT if not assigned.')
    parser.add_argument(
        '--repo', default='chromium', choices=cls.REPO,
        help='Repository to bisect. Now it supports chormium git repository. '
             'Later it will support catapult git repository and even ChromeOS '
             'repositories.')
    parser.add_argument(
        '--evaluator', default='autotest', choices=cls.EVALUATOR,
        help='Evaluator used to determine if a commit is good or bad. Now it '
             'supports autotest. Later it will support telemetry.')
    parser.add_argument(
        '-d', '--base-dir', required=True, type='path',
        help='Base directory to store repo and results. Existing checkout can '
             'be used, see --reuse-repo.')
    parser.add_argument(
        '--chromium-dir', type='path',
        help='If specified, use it as chromium repository. Otherwise, use '
             '"[base-dir]/chromium".')
    parser.add_argument(
        '--build-dir', type='path',
        help='If specified, use it to store build results. Otherwise, use '
             '"[base-dir]/build".')
    parser.add_argument(
        '--reuse-repo', action='store_true',
        help='If set, reuse repository if it exists.')
    parser.add_argument(
        '--reuse-build', action='store_true',
        help='If set, reuse build if available.')
    parser.add_argument(
        '--reuse-eval', action='store_true',
        help='If set, reuse evaluation result if available.')
    parser.add_argument(
        '--reuse', action='store_true',
        help='If set, set reuse-repo, reuse-build, reuse-eval')
    parser.add_argument(
        '--no-archive-build', dest='archive_build', default=True,
        action='store_false',
        help='If set, do not archive the build.')
    parser.add_argument(
        '--test-name', help='Test name to run against')
    parser.add_argument(
        '--metric',
        help='Metric of test result to look at. For autotest, metric name is '
             'hierarchical, divided by "/". E.g., "avg_fps_1000_fishes/summary/'
             'value" for graphics_WebGLAquarium, or "charts/Total/Score/value" '
             'for telemetry_Benchmarks.octane autotest.')
    parser.add_argument(
        '--metric-take-average', action='store_true',
        help='If set, treat metric value as a list of numbers and calculate '
             'arithmetic average of them.')
    parser.add_argument(
        '--eval-repeat', type=int, default=3,
        help='Repeat evaluate commit for N times to calculate mean and '
             'standard deviation. Default 3.')

  def ProcessOptions(self):
    """Process self.options.

    It sets reuse_repo, reuse_build and reuse_eval if self.options.reuse is set.
    It also resolves self.options.board from DUT if it is not assigned.
    """
    if self.options.reuse:
      self.options.reuse_repo = True
      self.options.reuse_build = True
      self.options.reuse_eval = True
    if not self.options.board:
      dut = remote_access.ChromiumOSDevice(self.options.remote.hostname)
      self.options.board = dut.board
      if not self.options.board:
        raise Exception('Unable to obtain board name from DUT.')
      else:
        logging.info('Obtained board name "%s" from DUT.', self.options.board)

  def Run(self):
    self.ProcessOptions()
    # Note that for the objects consuming command line options, please evaluate
    # its SanityCheckOptions() in testAddParser in cros_bisect_unittest.
    builder = self.BUILDER[self.options.repo](self.options)
    evaluator = autotest_evaluator.AutotestEvaluator(self.options)
    bisector = git_bisector.GitBisector(self.options, builder, evaluator)
    bisector.SetUp()
    bisector.Run()