| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| |
| # Copyright 2013 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Testing of benchmark_run.""" |
| |
| |
| import inspect |
| import unittest |
| import unittest.mock as mock |
| |
| from benchmark import Benchmark |
| import benchmark_run |
| from cros_utils import logger |
| from label import MockLabel |
| from machine_manager import MachineManager |
| from machine_manager import MockCrosMachine |
| from machine_manager import MockMachineManager |
| from results_cache import CacheConditions |
| from results_cache import MockResultsCache |
| from results_cache import Result |
| from results_cache import ResultsCache |
| from suite_runner import MockSuiteRunner |
| from suite_runner import SuiteRunner |
| |
| |
| class BenchmarkRunTest(unittest.TestCase): |
| """Unit tests for the BenchmarkRun class and all of its methods.""" |
| |
| def setUp(self): |
| self.status = [] |
| self.called_ReadCache = None |
| self.log_error = [] |
| self.log_output = [] |
| self.err_msg = None |
| self.test_benchmark = Benchmark( |
| "page_cycler.netsim.top_10", # name |
| "page_cycler.netsim.top_10", # test_name |
| "", # test_args |
| 1, # iterations |
| False, # rm_chroot_tmp |
| "", # perf_args |
| suite="telemetry_Crosperf", |
| ) # suite |
| |
| self.test_label = MockLabel( |
| "test1", |
| "build", |
| "image1", |
| "autotest_dir", |
| "debug_dir", |
| "/tmp/test_benchmark_run", |
| "x86-alex", |
| "chromeos2-row1-rack4-host9.cros", |
| image_args="", |
| cache_dir="", |
| cache_only=False, |
| log_level="average", |
| compiler="gcc", |
| crosfleet=False, |
| ) |
| |
| self.test_cache_conditions = [ |
| CacheConditions.CACHE_FILE_EXISTS, |
| CacheConditions.CHECKSUMS_MATCH, |
| ] |
| |
| self.mock_logger = logger.GetLogger(log_dir="", mock=True) |
| |
| self.mock_machine_manager = mock.Mock(spec=MachineManager) |
| |
| def testDryRun(self): |
| my_label = MockLabel( |
| "test1", |
| "build", |
| "image1", |
| "autotest_dir", |
| "debug_dir", |
| "/tmp/test_benchmark_run", |
| "x86-alex", |
| "chromeos2-row1-rack4-host9.cros", |
| image_args="", |
| cache_dir="", |
| cache_only=False, |
| log_level="average", |
| compiler="gcc", |
| crosfleet=False, |
| ) |
| |
| logging_level = "average" |
| m = MockMachineManager("/tmp/chromeos_root", 0, logging_level, "") |
| m.AddMachine("chromeos2-row1-rack4-host9.cros") |
| bench = Benchmark( |
| "page_cycler.netsim.top_10", # name |
| "page_cycler.netsim.top_10", # test_name |
| "", # test_args |
| 1, # iterations |
| False, # rm_chroot_tmp |
| "", # perf_args |
| suite="telemetry_Crosperf", |
| ) # suite |
| dut_conf = { |
| "cooldown_time": 5, |
| "cooldown_temp": 45, |
| "governor": "powersave", |
| "cpu_usage": "big_only", |
| "cpu_freq_pct": 80, |
| } |
| b = benchmark_run.MockBenchmarkRun( |
| "test run", |
| bench, |
| my_label, |
| 1, |
| [], |
| m, |
| logger.GetLogger(), |
| logging_level, |
| "", |
| dut_conf, |
| ) |
| b.cache = MockResultsCache() |
| b.suite_runner = MockSuiteRunner() |
| b.start() |
| |
| # Make sure the arguments to BenchmarkRun.__init__ have not changed |
| # since the last time this test was updated: |
| args_list = [ |
| "self", |
| "name", |
| "benchmark", |
| "label", |
| "iteration", |
| "cache_conditions", |
| "machine_manager", |
| "logger_to_use", |
| "log_level", |
| "share_cache", |
| "dut_config", |
| ] |
| arg_spec = inspect.getfullargspec(benchmark_run.BenchmarkRun.__init__) |
| self.assertEqual(len(arg_spec.args), len(args_list)) |
| self.assertEqual(arg_spec.args, args_list) |
| |
| def test_init(self): |
| # Nothing really worth testing here; just field assignments. |
| pass |
| |
| def test_read_cache(self): |
| # Nothing really worth testing here, either. |
| pass |
| |
| def test_run(self): |
| br = benchmark_run.BenchmarkRun( |
| "test_run", |
| self.test_benchmark, |
| self.test_label, |
| 1, |
| self.test_cache_conditions, |
| self.mock_machine_manager, |
| self.mock_logger, |
| "average", |
| "", |
| {}, |
| ) |
| |
| def MockLogOutput(msg, print_to_console=False): |
| """Helper function for test_run.""" |
| del print_to_console |
| self.log_output.append(msg) |
| |
| def MockLogError(msg, print_to_console=False): |
| """Helper function for test_run.""" |
| del print_to_console |
| self.log_error.append(msg) |
| |
| def MockRecordStatus(msg): |
| """Helper function for test_run.""" |
| self.status.append(msg) |
| |
| def FakeReadCache(): |
| """Helper function for test_run.""" |
| br.cache = mock.Mock(spec=ResultsCache) |
| self.called_ReadCache = True |
| return 0 |
| |
| def FakeReadCacheSucceed(): |
| """Helper function for test_run.""" |
| br.cache = mock.Mock(spec=ResultsCache) |
| br.result = mock.Mock(spec=Result) |
| br.result.out = "result.out stuff" |
| br.result.err = "result.err stuff" |
| br.result.retval = 0 |
| self.called_ReadCache = True |
| return 0 |
| |
| def FakeReadCacheException(): |
| """Helper function for test_run.""" |
| raise RuntimeError( |
| "This is an exception test; it is supposed to happen" |
| ) |
| |
| def FakeAcquireMachine(): |
| """Helper function for test_run.""" |
| mock_machine = MockCrosMachine( |
| "chromeos1-row3-rack5-host7.cros", "chromeos", "average" |
| ) |
| return mock_machine |
| |
| def FakeRunTest(_machine): |
| """Helper function for test_run.""" |
| mock_result = mock.Mock(spec=Result) |
| mock_result.retval = 0 |
| return mock_result |
| |
| def FakeRunTestFail(_machine): |
| """Helper function for test_run.""" |
| mock_result = mock.Mock(spec=Result) |
| mock_result.retval = 1 |
| return mock_result |
| |
| def ResetTestValues(): |
| """Helper function for test_run.""" |
| self.log_output = [] |
| self.log_error = [] |
| self.status = [] |
| br.result = None |
| self.called_ReadCache = False |
| |
| # Assign all the fake functions to the appropriate objects. |
| br.logger().LogOutput = MockLogOutput |
| br.logger().LogError = MockLogError |
| br.timeline.Record = MockRecordStatus |
| br.ReadCache = FakeReadCache |
| br.RunTest = FakeRunTest |
| br.AcquireMachine = FakeAcquireMachine |
| |
| # First test: No cache hit, all goes well. |
| ResetTestValues() |
| br.run() |
| self.assertTrue(self.called_ReadCache) |
| self.assertEqual( |
| self.log_output, |
| [ |
| "test_run: No cache hit.", |
| "Releasing machine: chromeos1-row3-rack5-host7.cros", |
| "Released machine: chromeos1-row3-rack5-host7.cros", |
| ], |
| ) |
| self.assertEqual(len(self.log_error), 0) |
| self.assertEqual(self.status, ["WAITING", "SUCCEEDED"]) |
| |
| # Second test: No cached result found; test run was "terminated" for some |
| # reason. |
| ResetTestValues() |
| br.terminated = True |
| br.run() |
| self.assertTrue(self.called_ReadCache) |
| self.assertEqual( |
| self.log_output, |
| [ |
| "test_run: No cache hit.", |
| "Releasing machine: chromeos1-row3-rack5-host7.cros", |
| "Released machine: chromeos1-row3-rack5-host7.cros", |
| ], |
| ) |
| self.assertEqual(len(self.log_error), 0) |
| self.assertEqual(self.status, ["WAITING"]) |
| |
| # Third test. No cached result found; RunTest failed for some reason. |
| ResetTestValues() |
| br.terminated = False |
| br.RunTest = FakeRunTestFail |
| br.run() |
| self.assertTrue(self.called_ReadCache) |
| self.assertEqual( |
| self.log_output, |
| [ |
| "test_run: No cache hit.", |
| "Releasing machine: chromeos1-row3-rack5-host7.cros", |
| "Released machine: chromeos1-row3-rack5-host7.cros", |
| ], |
| ) |
| self.assertEqual(len(self.log_error), 0) |
| self.assertEqual(self.status, ["WAITING", "FAILED"]) |
| |
| # Fourth test: ReadCache found a cached result. |
| ResetTestValues() |
| br.RunTest = FakeRunTest |
| br.ReadCache = FakeReadCacheSucceed |
| br.run() |
| self.assertTrue(self.called_ReadCache) |
| self.assertEqual( |
| self.log_output, |
| [ |
| "test_run: Cache hit.", |
| "result.out stuff", |
| "Releasing machine: chromeos1-row3-rack5-host7.cros", |
| "Released machine: chromeos1-row3-rack5-host7.cros", |
| ], |
| ) |
| self.assertEqual(self.log_error, ["result.err stuff"]) |
| self.assertEqual(self.status, ["SUCCEEDED"]) |
| |
| # Fifth test: ReadCache generates an exception; does the try/finally block |
| # work? |
| ResetTestValues() |
| br.ReadCache = FakeReadCacheException |
| br.machine = FakeAcquireMachine() |
| br.run() |
| self.assertEqual( |
| self.log_error, |
| [ |
| "Benchmark run: 'test_run' failed: This is an exception test; it is " |
| "supposed to happen" |
| ], |
| ) |
| self.assertEqual(self.status, ["FAILED"]) |
| |
| def test_terminate_pass(self): |
| br = benchmark_run.BenchmarkRun( |
| "test_run", |
| self.test_benchmark, |
| self.test_label, |
| 1, |
| self.test_cache_conditions, |
| self.mock_machine_manager, |
| self.mock_logger, |
| "average", |
| "", |
| {}, |
| ) |
| |
| def GetLastEventPassed(): |
| """Helper function for test_terminate_pass""" |
| return benchmark_run.STATUS_SUCCEEDED |
| |
| def RecordStub(status): |
| """Helper function for test_terminate_pass""" |
| self.status = status |
| |
| self.status = benchmark_run.STATUS_SUCCEEDED |
| self.assertFalse(br.terminated) |
| self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated()) |
| |
| br.timeline.GetLastEvent = GetLastEventPassed |
| br.timeline.Record = RecordStub |
| |
| br.Terminate() |
| |
| self.assertTrue(br.terminated) |
| self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated()) |
| self.assertEqual(self.status, benchmark_run.STATUS_FAILED) |
| |
| def test_terminate_fail(self): |
| br = benchmark_run.BenchmarkRun( |
| "test_run", |
| self.test_benchmark, |
| self.test_label, |
| 1, |
| self.test_cache_conditions, |
| self.mock_machine_manager, |
| self.mock_logger, |
| "average", |
| "", |
| {}, |
| ) |
| |
| def GetLastEventFailed(): |
| """Helper function for test_terminate_fail""" |
| return benchmark_run.STATUS_FAILED |
| |
| def RecordStub(status): |
| """Helper function for test_terminate_fail""" |
| self.status = status |
| |
| self.status = benchmark_run.STATUS_SUCCEEDED |
| self.assertFalse(br.terminated) |
| self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated()) |
| |
| br.timeline.GetLastEvent = GetLastEventFailed |
| br.timeline.Record = RecordStub |
| |
| br.Terminate() |
| |
| self.assertTrue(br.terminated) |
| self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated()) |
| self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED) |
| |
| def test_acquire_machine(self): |
| br = benchmark_run.BenchmarkRun( |
| "test_run", |
| self.test_benchmark, |
| self.test_label, |
| 1, |
| self.test_cache_conditions, |
| self.mock_machine_manager, |
| self.mock_logger, |
| "average", |
| "", |
| {}, |
| ) |
| |
| br.terminated = True |
| self.assertRaises(Exception, br.AcquireMachine) |
| |
| br.terminated = False |
| mock_machine = MockCrosMachine( |
| "chromeos1-row3-rack5-host7.cros", "chromeos", "average" |
| ) |
| self.mock_machine_manager.AcquireMachine.return_value = mock_machine |
| |
| machine = br.AcquireMachine() |
| self.assertEqual(machine.name, "chromeos1-row3-rack5-host7.cros") |
| |
| def test_get_extra_autotest_args(self): |
| br = benchmark_run.BenchmarkRun( |
| "test_run", |
| self.test_benchmark, |
| self.test_label, |
| 1, |
| self.test_cache_conditions, |
| self.mock_machine_manager, |
| self.mock_logger, |
| "average", |
| "", |
| {}, |
| ) |
| |
| def MockLogError(err_msg): |
| """Helper function for test_get_extra_autotest_args""" |
| self.err_msg = err_msg |
| |
| self.mock_logger.LogError = MockLogError |
| |
| result = br.GetExtraAutotestArgs() |
| self.assertEqual(result, "") |
| |
| self.test_benchmark.perf_args = "record -e cycles" |
| result = br.GetExtraAutotestArgs() |
| self.assertEqual( |
| result, |
| "--profiler=custom_perf --profiler_args='perf_options=\"record -a -e " |
| "cycles\"'", |
| ) |
| |
| self.test_benchmark.perf_args = "record -e cycles" |
| self.test_benchmark.suite = "test_that" |
| result = br.GetExtraAutotestArgs() |
| self.assertEqual(result, "") |
| self.assertEqual( |
| self.err_msg, "Non-telemetry benchmark does not support profiler." |
| ) |
| |
| self.test_benchmark.perf_args = "junk args" |
| self.test_benchmark.suite = "telemetry_Crosperf" |
| self.assertRaises(Exception, br.GetExtraAutotestArgs) |
| |
| @mock.patch.object(SuiteRunner, "Run") |
| @mock.patch.object(Result, "CreateFromRun") |
| def test_run_test(self, mock_result, mock_runner): |
| br = benchmark_run.BenchmarkRun( |
| "test_run", |
| self.test_benchmark, |
| self.test_label, |
| 1, |
| self.test_cache_conditions, |
| self.mock_machine_manager, |
| self.mock_logger, |
| "average", |
| "", |
| {}, |
| ) |
| |
| self.status = [] |
| |
| def MockRecord(status): |
| self.status.append(status) |
| |
| br.timeline.Record = MockRecord |
| mock_machine = MockCrosMachine( |
| "chromeos1-row3-rack5-host7.cros", "chromeos", "average" |
| ) |
| mock_runner.return_value = [0, "{'Score':100}", ""] |
| |
| br.RunTest(mock_machine) |
| |
| self.assertTrue(br.run_completed) |
| self.assertEqual( |
| self.status, |
| [benchmark_run.STATUS_IMAGING, benchmark_run.STATUS_RUNNING], |
| ) |
| |
| self.assertEqual(br.machine_manager.ImageMachine.call_count, 1) |
| br.machine_manager.ImageMachine.assert_called_with( |
| mock_machine, self.test_label |
| ) |
| self.assertEqual(mock_runner.call_count, 1) |
| mock_runner.assert_called_with( |
| mock_machine, br.label, br.benchmark, "", br.profiler_args |
| ) |
| |
| self.assertEqual(mock_result.call_count, 1) |
| mock_result.assert_called_with( |
| self.mock_logger, |
| "average", |
| self.test_label, |
| None, |
| "{'Score':100}", |
| "", |
| 0, |
| "page_cycler.netsim.top_10", |
| "telemetry_Crosperf", |
| "", |
| ) |
| |
| def test_set_cache_conditions(self): |
| br = benchmark_run.BenchmarkRun( |
| "test_run", |
| self.test_benchmark, |
| self.test_label, |
| 1, |
| self.test_cache_conditions, |
| self.mock_machine_manager, |
| self.mock_logger, |
| "average", |
| "", |
| {}, |
| ) |
| |
| phony_cache_conditions = [123, 456, True, False] |
| |
| self.assertEqual(br.cache_conditions, self.test_cache_conditions) |
| |
| br.SetCacheConditions(phony_cache_conditions) |
| self.assertEqual(br.cache_conditions, phony_cache_conditions) |
| |
| br.SetCacheConditions(self.test_cache_conditions) |
| self.assertEqual(br.cache_conditions, self.test_cache_conditions) |
| |
| |
| if __name__ == "__main__": |
| unittest.main() |
