| #!/usr/bin/python |
| |
| # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import StringIO |
| import socket |
| import mock |
| import unittest |
| |
| from utils.file_utils import FileUtils |
| |
| from experiment_factory import ExperimentFactory |
| from experiment_file import ExperimentFile |
| import test_flag |
| import benchmark |
| import label |
| import experiment |
| import experiment_factory |
| import machine_manager |
| import settings_factory |
| |
| EXPERIMENT_FILE_1 = """ |
| board: x86-alex |
| remote: chromeos-alex3 |
| |
| benchmark: PageCycler { |
| iterations: 3 |
| } |
| |
| image1 { |
| chromeos_image: /usr/local/google/cros_image1.bin |
| } |
| |
| image2 { |
| chromeos_image: /usr/local/google/cros_image2.bin |
| } |
| """ |
| |
| |
| class ExperimentFactoryTest(unittest.TestCase): |
| |
| def testLoadExperimentFile1(self): |
| experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) |
| experiment = ExperimentFactory().GetExperiment(experiment_file, |
| working_directory="", |
| log_dir="") |
| self.assertEqual(experiment.remote, ["chromeos-alex3"]) |
| |
| self.assertEqual(len(experiment.benchmarks), 1) |
| self.assertEqual(experiment.benchmarks[0].name, "PageCycler") |
| self.assertEqual(experiment.benchmarks[0].test_name, "PageCycler") |
| self.assertEqual(experiment.benchmarks[0].iterations, 3) |
| |
| self.assertEqual(len(experiment.labels), 2) |
| self.assertEqual(experiment.labels[0].chromeos_image, |
| "/usr/local/google/cros_image1.bin") |
| self.assertEqual(experiment.labels[0].board, |
| "x86-alex") |
| |
| |
| |
| def test_append_benchmark_set(self): |
| ef = ExperimentFactory() |
| |
| bench_list = [] |
| ef._AppendBenchmarkSet(bench_list, |
| experiment_factory.telemetry_perfv2_tests, |
| "", 1, False, "", "telemetry_Crosperf", False) |
| self.assertEqual(len(bench_list), |
| len(experiment_factory.telemetry_perfv2_tests)) |
| self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) |
| |
| bench_list = [] |
| ef._AppendBenchmarkSet(bench_list, |
| experiment_factory.telemetry_pagecycler_tests, |
| "", 1, False, "", "telemetry_Crosperf", False) |
| self.assertEqual(len(bench_list), |
| len(experiment_factory.telemetry_pagecycler_tests)) |
| self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) |
| |
| bench_list = [] |
| ef._AppendBenchmarkSet(bench_list, |
| experiment_factory.telemetry_toolchain_perf_tests, |
| "", 1, False, "", "telemetry_Crosperf", False) |
| self.assertEqual(len(bench_list), |
| len(experiment_factory.telemetry_toolchain_perf_tests)) |
| self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) |
| |
| |
| |
| @mock.patch.object(socket, 'gethostname') |
| @mock.patch.object(machine_manager.MachineManager, 'AddMachine') |
| def test_get_experiment(self, mock_machine_manager, mock_socket): |
| |
| test_flag.SetTestMode(False) |
| self.append_benchmark_call_args = [] |
| def FakeAppendBenchmarkSet(bench_list, set_list, args, iters, rm_ch, |
| perf_args, suite, show_all): |
| "Helper function for test_get_experiment" |
| arg_list = [bench_list, set_list, args, iters, rm_ch, perf_args, suite, |
| show_all] |
| self.append_benchmark_call_args.append(args_list) |
| |
| def FakeGetDefaultRemotes(board): |
| return ["fake_chromeos_machine1.cros", |
| "fake_chromeos_machine2.cros"] |
| |
| def FakeGetXbuddyPath(build, board, chroot, log_level): |
| return "fake_image_path" |
| |
| |
| ef = ExperimentFactory() |
| ef._AppendBenchmarkSet = FakeAppendBenchmarkSet |
| ef.GetDefaultRemotes = FakeGetDefaultRemotes |
| |
| label_settings = settings_factory.LabelSettings("image_label") |
| benchmark_settings = settings_factory.BenchmarkSettings("bench_test") |
| global_settings = settings_factory.GlobalSettings("test_name") |
| |
| label_settings.GetXbuddyPath = FakeGetXbuddyPath |
| |
| mock_experiment_file = ExperimentFile(StringIO.StringIO("")) |
| mock_experiment_file.all_settings = [] |
| |
| # Basic test. |
| global_settings.SetField("name","unittest_test") |
| global_settings.SetField("board", "lumpy") |
| global_settings.SetField("remote", "123.45.67.89 123.45.76.80") |
| benchmark_settings.SetField("test_name", "kraken") |
| benchmark_settings.SetField("suite", "telemetry_Crosperf") |
| benchmark_settings.SetField("iterations", 1) |
| label_settings.SetField("chromeos_image", "chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin") |
| label_settings.SetField("chrome_src", "/usr/local/google/home/chrome-top") |
| |
| |
| mock_experiment_file.global_settings = global_settings |
| mock_experiment_file.all_settings.append (label_settings) |
| mock_experiment_file.all_settings.append (benchmark_settings) |
| mock_experiment_file.all_settings.append (global_settings) |
| |
| mock_socket.return_value = "" |
| |
| # First test. General test. |
| exp = ef.GetExperiment(mock_experiment_file, "", "") |
| self.assertEqual(exp.remote, ["123.45.67.89", "123.45.76.80"]) |
| self.assertEqual(exp.cache_conditions, [0, 2, 1]) |
| self.assertEqual(exp.log_level, "average") |
| |
| self.assertEqual(len(exp.benchmarks), 1) |
| self.assertEqual(exp.benchmarks[0].name, "kraken") |
| self.assertEqual(exp.benchmarks[0].test_name, "kraken") |
| self.assertEqual(exp.benchmarks[0].iterations, 1) |
| self.assertEqual(exp.benchmarks[0].suite, "telemetry_Crosperf") |
| self.assertFalse(exp.benchmarks[0].show_all_results) |
| |
| self.assertEqual(len(exp.labels), 1) |
| self.assertEqual(exp.labels[0].chromeos_image, |
| "chromeos/src/build/images/lumpy/latest/" |
| "chromiumos_test_image.bin") |
| self.assertEqual(exp.labels[0].board, "lumpy") |
| |
| # Second test: Remotes listed in labels. |
| label_settings.SetField("remote", "chromeos1.cros chromeos2.cros") |
| exp = ef.GetExperiment(mock_experiment_file, "", "") |
| self.assertEqual(exp.remote, |
| ["chromeos1.cros", "chromeos2.cros", "123.45.67.89", |
| "123.45.76.80", ]) |
| |
| # Third test: Automatic fixing of bad logging_level param: |
| global_settings.SetField("logging_level", "really loud!") |
| exp = ef.GetExperiment(mock_experiment_file, "", "") |
| self.assertEqual(exp.log_level, "verbose") |
| |
| # Fourth test: Setting cache conditions; only 1 remote with "same_machine" |
| global_settings.SetField("rerun_if_failed", "true") |
| global_settings.SetField("rerun", "true") |
| global_settings.SetField("same_machine", "true") |
| global_settings.SetField("same_specs", "true") |
| |
| self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, "", |
| "") |
| label_settings.SetField("remote", "") |
| global_settings.SetField("remote", "123.45.67.89") |
| exp = ef.GetExperiment(mock_experiment_file, "", "") |
| self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1]) |
| |
| # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all |
| # remotes (Call GetDefaultRemotes). |
| mock_socket.return_value = "test.corp.google.com" |
| global_settings.SetField("remote", "") |
| global_settings.SetField("same_machine", "false") |
| |
| label_settings_2 = settings_factory.LabelSettings("official_image_label") |
| label_settings_2.SetField("chromeos_root", "chromeos") |
| label_settings.SetField("build", "official-dev") |
| label_settings_2.GetXbuddyPath = FakeGetXbuddyPath |
| |
| mock_experiment_file.all_settings.append (label_settings_2) |
| exp = ef.GetExperiment(mock_experiment_file, "", "") |
| self.assertEqual(len(exp.labels), 2) |
| self.assertEqual(exp.labels[1].chromeos_image, "fake_image_path") |
| self.assertEqual(exp.remote, ["fake_chromeos_machine1.cros", |
| "fake_chromeos_machine2.cros"]) |
| |
| |
| |
| def test_get_default_remotes(self): |
| board_list = ['x86-zgb', 'x86-alex', 'lumpy', 'stumpy', 'parrot', 'daisy'] |
| |
| ef = ExperimentFactory() |
| self.assertRaises(Exception, ef.GetDefaultRemotes, 'bad-board') |
| |
| # Verify that we have entries for every board, and that we get three |
| # machines back for each board. |
| for b in board_list: |
| remotes = ef.GetDefaultRemotes(b) |
| self.assertEqual(len(remotes), 3) |
| |
| if __name__ == "__main__": |
| FileUtils.Configure(True) |
| test_flag.SetTestMode(True) |
| unittest.main() |
