cros_bisect/evaluator.py

# -*- coding: utf-8 -*-
# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Base class of performance evaluators.

Evaluator should run specific tests on DUT to evaluate performance.
Evaluator should define methods: Evaluate() and CheckLastEvaluate().
Evaluator base class sets up data members: base_dir and report_base_dir. The
latter one is derived from base_dir, and it is created if it doesn't exist yet.
"""

from __future__ import print_function

import os

from chromite.cros_bisect import common
from chromite.lib import osutils


class Evaluator(common.OptionsChecker):
  """Base class of performance evaluator.

  If reuse_eval is set, before build/deploy, bisector can call
  CheckLastEvaluate() first to see if it has previous evaluation result to
  skip the build.
  """

  REQUIRED_ARGS = ('base_dir', 'reuse_eval')

  # If set in derived class, it means the evaluator's score's threshold
  # that tells good from bad is predefined, e.g. binary evaluator's threshold
  # is 0.5.
  THRESHOLD = None

  def __init__(self, options):
    """Constructor.

    Args:
      options: An argparse.Namespace to hold command line arguments. Should
        contain:
        * base_dir: Parent directory of repository folder
        * reuse_eval: True to reuse report if available.
    """
    super(Evaluator, self).__init__(options)
    self.base_dir = options.base_dir
    self.report_base_dir = os.path.join(self.base_dir, 'reports')
    self.reuse_eval = options.reuse_eval
    osutils.SafeMakedirs(self.report_base_dir)

  # pylint: disable=unused-argument
  def Evaluate(self, remote, build_label, repeat):
    """Evaluates performance.

    Args:
      remote: DUT to evaluate (refer lib.commandline.Device).
      build_label: Build label used for part of report filename and log message.
      repeat: run the test N times.

    Returns:
      Score object. Empty Score object if it fails to evaluate.
    """
    raise NotImplementedError()

  def CheckLastEvaluate(self, build_label, repeat=1):
    """Checks if previous evaluate report is available.

    Args:
      build_label: Build label used for part of report filename and log message.
      repeat: Run test for N times. Default 1.

    Returns:
      If reuse_eval is set and previous evaluation result for the build_label
      is found, extracts and returns score from the result. Otherwise, returns
      Score().
    """
    raise NotImplementedError()