# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Test autotest_evaluator module."""

from __future__ import print_function

import os

from chromite.cros_bisect import autotest_evaluator
from chromite.lib import commandline
from chromite.lib import cros_build_lib_unittest
from chromite.lib import cros_test_lib
from chromite.lib import osutils
from chromite.lib import partial_mock
from chromite.lib import path_util
from chromite.lib import remote_access_unittest


class RemoteShScpMock(remote_access_unittest.RemoteShMock):
  """In addition to RemoteSh, it mocks ScpToLocal."""

  ATTRS = ('RemoteSh', 'ScpToLocal')

  def ScpToLocal(self, _, remote, local):
    return self._results['ScpToLocal'].LookupResult(([remote, local], ))


class TestAutotestEvaluator(cros_test_lib.MockTempDirTestCase):
  """Tests AutotestEvaluator class."""
  BOARD = 'samus'
  TEST_NAME = 'graphics_WebGLAquarium'
  METRIC = 'avg_fps_1000_fishes/summary/value'
  REPORT_FILE = 'reports.json'
  REMOTE_REPORT_FILE = '%s/results/default/%s/results/results-chart.json' % (
      autotest_evaluator.AutotestEvaluator.AUTOTEST_BASE,
      TEST_NAME)
  DUT = commandline.DeviceParser(commandline.DEVICE_SCHEME_SSH)('192.168.1.1')
  TEST_TARGET = '%s/tests/%s/control' % (
      autotest_evaluator.AutotestEvaluator.AUTOTEST_BASE, TEST_NAME)
  AQUARIUM_REPORT_TEMPLATE = """
{"avg_fps_1000_fishes": {
   "summary": {
     "units": "fps",
     "type": "scalar",
     "value": %s,
     "improvement_direction": "up"
    }
  }
}"""
  BUILD_LABEL = 'base'

  def GetDefaultEvaluator(self):
    """Gets AutototestEvaluator instance with default options.

    Returns:
      Default AutototestEvaluator instance for test.
    """
    options = cros_test_lib.EasyAttr(
        base_dir=self.tempdir, board=self.BOARD, test_name=self.TEST_NAME,
        metric=self.METRIC, metric_take_average=False, reuse_eval=True,
        chromium_dir=None)
    return autotest_evaluator.AutotestEvaluator(options)

  def PrepareWebglAquariumReports(self, scores):
    """Prepares graphics_WebGLAquarium reports.

    It is a simplified version. What test cares is
    "avg_fps_1000_fishes/summary/value". It can produces multiple reports if
    more than one score is given.

    Args:
      scores: List of scores.

    Returns:
      A list of file names storing in report directory.
    """
    result = []
    num_reports = len(scores)
    for ith, score in enumerate(scores, start=1):
      report_file = os.path.join(
          self.tempdir, 'reports',
          'results-chart.%s.%d-%d.json' % (self.BUILD_LABEL, ith, num_reports))
      osutils.WriteFile(report_file, self.AQUARIUM_REPORT_TEMPLATE % score)
      result.append(report_file)
    return result

  def testInit(self):
    """Tests that AutotestEvaluator() works as expected."""
    base_dir = self.tempdir
    evaluator = self.GetDefaultEvaluator()
    self.assertEqual(base_dir, evaluator.base_dir)
    self.assertEqual(os.path.join(base_dir, 'reports'),
                     evaluator.report_base_dir)
    self.assertTrue(os.path.isdir(evaluator.report_base_dir))
    self.assertEqual(self.BOARD, evaluator.board)
    self.assertEqual(self.TEST_NAME, evaluator.test_name)
    self.assertEqual(self.METRIC, evaluator.metric)
    self.assertFalse(evaluator.metric_take_average)
    self.assertTrue(evaluator.reuse_eval)
    self.assertEqual(os.path.join(base_dir, 'chromium'),
                     evaluator.chromium_dir)

    # With chromium_dir specified and flip booleans.
    options = cros_test_lib.EasyAttr(
        base_dir=base_dir, board=self.BOARD, test_name=self.TEST_NAME,
        metric=self.METRIC, metric_take_average=False, reuse_eval=False,
        chromium_dir='/tmp/chromium')
    evaluator = autotest_evaluator.AutotestEvaluator(options)
    self.assertFalse(evaluator.metric_take_average)
    self.assertFalse(evaluator.reuse_eval)
    self.assertEqual('/tmp/chromium', evaluator.chromium_dir)

  def testInitMissingRequiredArgs(self):
    """Tests that AE() raises exception when required options are missing."""
    options = cros_test_lib.EasyAttr()
    with self.assertRaises(Exception) as cm:
      autotest_evaluator.AutotestEvaluator(options)
    exception_message = cm.exception.message
    self.assertTrue('Missing command line' in exception_message)
    self.assertTrue('AutotestEvaluator' in exception_message)
    for arg in autotest_evaluator.AutotestEvaluator.REQUIRED_ARGS:
      self.assertTrue(arg in exception_message)

  def testRunTestFromDut(self):
    """Tests that RunTestFromDut() invokes expected commands."""
    evaluator = self.GetDefaultEvaluator()

    rsh_mock = self.StartPatcher(RemoteShScpMock())
    rsh_mock.AddCmdResult(
        [evaluator.AUTOTEST_CLIENT, self.TEST_TARGET], returncode=0)
    rsh_mock.AddCmdResult(
        [self.REMOTE_REPORT_FILE, self.REPORT_FILE], returncode=0,
        mock_attr='ScpToLocal')

    self.assertTrue(evaluator.RunTestFromDut(self.DUT, self.REPORT_FILE))

  def testRunTestFromDutAutotestFail(self):
    """Tests RunTestFromDut() when autotest failed to run remotely."""
    evaluator = self.GetDefaultEvaluator()

    rsh_mock = self.StartPatcher(RemoteShScpMock())
    rsh_mock.AddCmdResult(
        [evaluator.AUTOTEST_CLIENT, self.TEST_TARGET], returncode=1)

    self.assertFalse(evaluator.RunTestFromDut(self.DUT, self.REPORT_FILE))

  def testRunTestFromDutScpReportFail(self):
    """Tests RunTestFromDut() when it failed to remote copy report file."""
    evaluator = self.GetDefaultEvaluator()

    rsh_mock = self.StartPatcher(RemoteShScpMock())
    rsh_mock.AddCmdResult(
        [evaluator.AUTOTEST_CLIENT, self.TEST_TARGET], returncode=0)
    rsh_mock.AddCmdResult(
        [self.REMOTE_REPORT_FILE, self.REPORT_FILE], returncode=1,
        mock_attr='ScpToLocal')

    self.assertFalse(evaluator.RunTestFromDut(self.DUT, self.REPORT_FILE))

  def testLookupReportFile(self):
    """Tests LookupReportFile().

    Tests that it invokes expected command and performs path normalization.
    """
    evaluator = self.GetDefaultEvaluator()
    command_mock = self.StartPatcher(
        cros_build_lib_unittest.RunCommandMock())
    results_base_path = os.path.join(
        self.tempdir, 'test_that_latest/results-1-telemetry_Benchmarks.octane')
    find_command_result = (
        './telemetry_Benchmarks.octane/results/results-chart.json\n')
    command_mock.AddCmdResult(
        ['find', '.', '-name', 'results-chart.json'],
        kwargs={'cwd': results_base_path, 'capture_output': True},
        output=find_command_result)
    self.PatchObject(path_util, 'FromChrootPath',
                     return_value=results_base_path)

    self.assertEqual(
        os.path.join(results_base_path,
                     'telemetry_Benchmarks.octane/results/results-chart.json'),
        evaluator.LookupReportFile())

  def testLookupReportFileMissing(self):
    """Tests LookupReportFile() when the report does not exist."""
    evaluator = self.GetDefaultEvaluator()
    command_mock = self.StartPatcher(
        cros_build_lib_unittest.RunCommandMock())
    results_base_path = os.path.join(
        self.tempdir, 'test_that_latest/results-1-telemetry_Benchmarks.octane')
    command_mock.AddCmdResult(
        ['find', '.', '-name', 'results-chart.json'],
        kwargs={'cwd': results_base_path, 'capture_output': True},
        output='')
    self.PatchObject(path_util, 'FromChrootPath',
                     return_value=results_base_path)

    self.assertIsNone(evaluator.LookupReportFile())

  def testRunTestFromHost(self):
    """Tests TestFromHost().

    Tests that it invokes expected commands and report file being copied to
    designated path.
    """
    evaluator = self.GetDefaultEvaluator()
    command_mock = self.StartPatcher(
        cros_build_lib_unittest.RunCommandMock())
    command_mock.AddCmdResult(
        ['test_that', '-b', self.BOARD, '--fast', '--args', 'local=True',
         '192.168.1.1', self.TEST_NAME], returncode=0)
    report_source_file = os.path.join(self.tempdir, 'results-chart.json')
    report_content = 'foobar'
    osutils.WriteFile(report_source_file, report_content)
    command_mock.AddCmdResult(
        ['find', '.', '-name', 'results-chart.json'],
        output=report_source_file)
    self.PatchObject(path_util, 'FromChrootPath', return_value=self.tempdir)

    # Make sure report file is copied to designated path.
    target_report_file = os.path.join(self.tempdir, 'stored-results-chart.json')
    osutils.SafeUnlink(target_report_file)
    self.assertTrue(evaluator.RunTestFromHost(self.DUT, target_report_file))
    self.assertTrue(os.path.exists(target_report_file))
    self.assertEqual(report_content, osutils.ReadFile(target_report_file))

  def testRunTestFromHostTestThatFail(self):
    """Tests TestFromHost() when autotest failed to run."""
    evaluator = self.GetDefaultEvaluator()

    command_mock = self.StartPatcher(
        cros_build_lib_unittest.RunCommandMock())
    command_mock.AddCmdResult(
        ['test_that', '-b', self.BOARD, '--fast', '--args', 'local=True',
         '192.168.1.1', self.TEST_NAME], returncode=1)

    self.assertFalse(evaluator.RunTestFromHost(self.DUT, self.REPORT_FILE))

  def testRunTestFromHostReportFileMissing(self):
    """Tests TestFromHost() when test report file does not exist."""
    evaluator = self.GetDefaultEvaluator()
    command_mock = self.StartPatcher(
        cros_build_lib_unittest.RunCommandMock())
    command_mock.AddCmdResult(
        ['test_that', '-b', self.BOARD, '--fast', '--args', 'local=True',
         '192.168.1.1', self.TEST_NAME], returncode=0)
    command_mock.AddCmdResult(
        ['find', '.', '-name', 'results-chart.json'], output='')
    self.PatchObject(path_util, 'FromChrootPath', return_value=self.tempdir)

    self.assertFalse(evaluator.RunTestFromHost(self.DUT, self.REPORT_FILE))

  def testGetAutotestMetricValue(self):
    """Tests that GetAutotestMetricValue() extracts score correctly."""
    evaluator = self.GetDefaultEvaluator()
    score = 56.73
    report_file = self.PrepareWebglAquariumReports([score])[0]
    self.assertEqual(score,
                     evaluator.GetAutotestMetricValue(report_file))

  def testGetAutotestMetricValueMetricTakeAverage(self):
    """Tests that GetAutotestMetricValue() extracts averaged scores."""
    # metric_take_average=True
    options = cros_test_lib.EasyAttr(
        base_dir=self.tempdir, board=self.BOARD, test_name=self.TEST_NAME,
        metric=self.METRIC, metric_take_average=True, reuse_eval=True,
        chromium_dir=None)
    evaluator = autotest_evaluator.AutotestEvaluator(options)

    scores = [55, 57, 58]
    # A report's value is a list of scores.
    report_file = self.PrepareWebglAquariumReports([scores])[0]
    self.assertAlmostEqual(56.66, evaluator.GetAutotestMetricValue(report_file),
                           delta=0.01)

  def testEvaluateRunTestFromDut(self):
    """Tests Evaluate() which runs test from DUT."""
    evaluator = self.GetDefaultEvaluator()

    # Mock RunTestFromDut success.
    rsh_mock = self.StartPatcher(RemoteShScpMock())
    rsh_mock.AddCmdResult(
        [evaluator.AUTOTEST_CLIENT, self.TEST_TARGET], returncode=0)

    # Prepare result for evaluate.
    score = 56.73
    report_file = self.PrepareWebglAquariumReports([score])[0]

    rsh_mock.AddCmdResult(
        [self.REMOTE_REPORT_FILE, report_file], returncode=0,
        mock_attr='ScpToLocal')

    eval_score = evaluator.Evaluate(self.DUT, self.BUILD_LABEL)
    self.assertEqual(1, len(eval_score.values))
    self.assertEqual(score, eval_score.values[0])
    self.assertEqual(score, eval_score.mean)
    self.assertEqual(0.0, eval_score.variance)
    self.assertEqual(0.0, eval_score.std)

  def testEvaluateTwiceRunTestFromDut(self):
    """Tests Evaluate() with repeat=2 which runs test from DUT."""
    evaluator = self.GetDefaultEvaluator()

    # Mock RunTestFromDut success.
    rsh_mock = self.StartPatcher(RemoteShScpMock())
    rsh_mock.AddCmdResult(
        [evaluator.AUTOTEST_CLIENT, self.TEST_TARGET], returncode=0)

    # Prepare two results for evaluate.
    scores = [56, 58]
    report_files = self.PrepareWebglAquariumReports(scores)

    for report_file in report_files:
      rsh_mock.AddCmdResult(
          [self.REMOTE_REPORT_FILE, report_file], returncode=0,
          mock_attr='ScpToLocal')

    eval_score = evaluator.Evaluate(self.DUT, self.BUILD_LABEL, repeat=2)
    self.assertEqual(2, len(eval_score.values))
    self.assertEqual(scores[0], eval_score.values[0])
    self.assertEqual(scores[1], eval_score.values[1])
    self.assertEqual(57, eval_score.mean)
    self.assertEqual(2.0, eval_score.variance)
    self.assertAlmostEqual(1.414, eval_score.std, delta=0.01)

  def testEvaluateFromHost(self):
    """Tests Evaluate() which runs test from host."""
    evaluator = self.GetDefaultEvaluator()

    # Mock RunTestFromDut fail.
    command_mock = self.StartPatcher(
        cros_build_lib_unittest.RunCommandMock())
    command_mock.AddCmdResult(
        partial_mock.InOrder([evaluator.AUTOTEST_CLIENT, self.TEST_TARGET]),
        returncode=1)

    # Mock RunTestFromHost success.
    command_mock.AddCmdResult(
        ['test_that', '-b', self.BOARD, '--fast', '--args', 'local=True',
         '192.168.1.1', self.TEST_NAME], returncode=0)

    # Mock 'find' and returns a result file for verify.
    score = 59.9
    report_file_in_chroot = os.path.join(self.tempdir, 'results-chart.json')
    osutils.WriteFile(report_file_in_chroot,
                      self.AQUARIUM_REPORT_TEMPLATE % score)
    command_mock.AddCmdResult(
        ['find', '.', '-name', 'results-chart.json'],
        output=report_file_in_chroot)
    self.PatchObject(path_util, 'FromChrootPath', return_value=self.tempdir)

    eval_score = evaluator.Evaluate(self.DUT, self.BUILD_LABEL)
    self.assertEqual(1, len(eval_score.values))
    self.assertEqual(score, eval_score.values[0])
    self.assertEqual(score, eval_score.mean)
    self.assertEqual(0.0, eval_score.variance)
    self.assertEqual(0.0, eval_score.std)

  def testCheckLastEvaluate(self):
    """Tests CheckLastEvaluate().

    Test that it extracts score from last evaluation result.
    """
    evaluator = self.GetDefaultEvaluator()

    scores = [56, 58]
    self.PrepareWebglAquariumReports(scores)

    eval_score = evaluator.CheckLastEvaluate(self.BUILD_LABEL, repeat=2)
    self.assertEqual(2, len(eval_score.values))
    self.assertEqual(scores[0], eval_score.values[0])
    self.assertEqual(scores[1], eval_score.values[1])
    self.assertEqual(57, eval_score.mean)
    self.assertEqual(2.0, eval_score.variance)
    self.assertAlmostEqual(1.414, eval_score.std, delta=0.01)

  def testCheckLastEvaluateDifferentLabel(self):
    """Tests that CheckLastEvaluate() failed to extracts score."""
    evaluator = self.GetDefaultEvaluator()

    scores = [56, 58]
    self.PrepareWebglAquariumReports(scores)

    eval_score = evaluator.CheckLastEvaluate('different_build', repeat=2)
    self.assertEqual(0, len(eval_score))

  def testCheckLastEvaluateFlagUnset(self):
    """Tests CheckLastEvaluate() when "reuse_eval" option is unset.

    Tests that it always returns empty score when "reuse_eval" option is unset.
    """
    # 'reuse_eval' set to False.
    options = cros_test_lib.EasyAttr(
        base_dir=self.tempdir, board=self.BOARD, test_name=self.TEST_NAME,
        metric=self.METRIC, metric_take_average=False, reuse_eval=False,
        chromium_dir=None)
    evaluator = autotest_evaluator.AutotestEvaluator(options)

    scores = [56, 58]
    self.PrepareWebglAquariumReports(scores)

    eval_score = evaluator.CheckLastEvaluate(self.BUILD_LABEL, repeat=2)
    self.assertEqual(0, len(eval_score))