crosperf: move enable_aslr and turbostat option into dut_config
This patch moves enable_aslr and turbostat into dut_config dictionary,
since they are all device setup steps, and it make code cleaner and
easier to migrate to telemetry_Crosperf.
TEST=Passed unittests.
BUG=chromium:1020655
Change-Id: I29a649e76591e206efc19bb2b8a8df9aead16575
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1947903
Reviewed-by: Denis Nikitin <denik@chromium.org>
Commit-Queue: Zhizhou Yang <zhizhouy@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Auto-Submit: Zhizhou Yang <zhizhouy@google.com>
diff --git a/crosperf/benchmark.py b/crosperf/benchmark.py
index 5c11e27..0413b59 100644
--- a/crosperf/benchmark.py
+++ b/crosperf/benchmark.py
@@ -9,8 +9,8 @@
from __future__ import print_function
import math
-#FIXME(denik): Fix the import in chroot.
-#pylint: disable=import-error
+# FIXME(denik): Fix the import in chroot.
+# pylint: disable=import-error
from scipy import stats
# See crbug.com/673558 for how these are estimated.
@@ -65,12 +65,11 @@
retries=0,
run_local=False,
cwp_dso='',
- weight=0,
- turbostat=True):
+ weight=0):
self.name = name
- #For telemetry, this is the benchmark name.
+ # For telemetry, this is the benchmark name.
self.test_name = test_name
- #For telemetry, this is the data.
+ # For telemetry, this is the data.
self.test_args = test_args
self.iterations = iterations if iterations > 0 else _samples(name)
self.perf_args = perf_args
@@ -86,4 +85,3 @@
self.run_local = run_local
self.cwp_dso = cwp_dso
self.weight = weight
- self.turbostat = turbostat
diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py
index a7c3b7d..f17de1b 100644
--- a/crosperf/benchmark_run.py
+++ b/crosperf/benchmark_run.py
@@ -31,18 +31,9 @@
class BenchmarkRun(threading.Thread):
"""The benchmarkrun class."""
- def __init__(self,
- name,
- benchmark,
- label,
- iteration,
- cache_conditions,
- machine_manager,
- logger_to_use,
- log_level,
- share_cache,
- dut_config,
- enable_aslr=False):
+ def __init__(self, name, benchmark, label, iteration, cache_conditions,
+ machine_manager, logger_to_use, log_level, share_cache,
+ dut_config):
threading.Thread.__init__(self)
self.name = name
self._logger = logger_to_use
@@ -55,8 +46,7 @@
self.retval = None
self.run_completed = False
self.machine_manager = machine_manager
- self.suite_runner = SuiteRunner(
- dut_config, self._logger, self.log_level, enable_aslr=enable_aslr)
+ self.suite_runner = SuiteRunner(dut_config, self._logger, self.log_level)
self.machine = None
self.cache_conditions = cache_conditions
self.runs_complete = 0
diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py
index 4daa77b..5696c10 100755
--- a/crosperf/benchmark_run_unittest.py
+++ b/crosperf/benchmark_run_unittest.py
@@ -119,7 +119,7 @@
args_list = [
'self', 'name', 'benchmark', 'label', 'iteration', 'cache_conditions',
'machine_manager', 'logger_to_use', 'log_level', 'share_cache',
- 'dut_config', 'enable_aslr'
+ 'dut_config'
]
arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__)
self.assertEqual(len(arg_spec.args), len(args_list))
diff --git a/crosperf/benchmark_unittest.py b/crosperf/benchmark_unittest.py
index 6c0c22f..63c0a1c 100755
--- a/crosperf/benchmark_unittest.py
+++ b/crosperf/benchmark_unittest.py
@@ -60,7 +60,7 @@
args_list = [
'self', 'name', 'test_name', 'test_args', 'iterations', 'rm_chroot_tmp',
'perf_args', 'suite', 'show_all_results', 'retries', 'run_local',
- 'cwp_dso', 'weight', 'turbostat'
+ 'cwp_dso', 'weight'
]
arg_spec = inspect.getargspec(Benchmark.__init__)
self.assertEqual(len(arg_spec.args), len(args_list))
diff --git a/crosperf/experiment.py b/crosperf/experiment.py
index 1d87b6e..d6d9ee4 100644
--- a/crosperf/experiment.py
+++ b/crosperf/experiment.py
@@ -28,8 +28,8 @@
def __init__(self, name, remote, working_directory, chromeos_root,
cache_conditions, labels, benchmarks, experiment_file, email_to,
acquire_timeout, log_dir, log_level, share_cache,
- results_directory, locks_directory, cwp_dso, enable_aslr,
- ignore_min_max, skylab, dut_config):
+ results_directory, locks_directory, cwp_dso, ignore_min_max,
+ skylab, dut_config):
self.name = name
self.working_directory = working_directory
self.remote = remote
@@ -54,7 +54,6 @@
self.locked_machines = []
self.lock_mgr = None
self.cwp_dso = cwp_dso
- self.enable_aslr = enable_aslr
self.ignore_min_max = ignore_min_max
self.skylab = skylab
self.l = logger.GetLogger(log_dir)
@@ -124,11 +123,10 @@
logger_to_use = logger.Logger(self.log_dir, 'run.%s' % (full_name),
True)
benchmark_runs.append(
- benchmark_run.BenchmarkRun(benchmark_run_name, benchmark, label,
- iteration, self.cache_conditions,
- self.machine_manager, logger_to_use,
- self.log_level, self.share_cache,
- dut_config, self.enable_aslr))
+ benchmark_run.BenchmarkRun(
+ benchmark_run_name, benchmark, label, iteration,
+ self.cache_conditions, self.machine_manager, logger_to_use,
+ self.log_level, self.share_cache, dut_config))
return benchmark_runs
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index 1758f0c..7d1689c 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -70,7 +70,7 @@
'speedometer2',
'jetstream',
'loading.desktop',
- #'rendering.desktop',
+ # 'rendering.desktop',
]
crosbolt_perf_tests = [
@@ -82,7 +82,7 @@
# 'cheets_PerfBootServer',
# 'cheets_CandyCrushTest',
# 'cheets_LinpackTest',
-#]
+# ]
dso_list = [
'all',
@@ -101,14 +101,12 @@
def AppendBenchmarkSet(self, benchmarks, benchmark_list, test_args,
iterations, rm_chroot_tmp, perf_args, suite,
- show_all_results, retries, run_local, cwp_dso, weight,
- turbostat):
+ show_all_results, retries, run_local, cwp_dso, weight):
"""Add all the tests in a set to the benchmarks list."""
for test_name in benchmark_list:
- telemetry_benchmark = Benchmark(test_name, test_name, test_args,
- iterations, rm_chroot_tmp, perf_args,
- suite, show_all_results, retries,
- run_local, cwp_dso, weight, turbostat)
+ telemetry_benchmark = Benchmark(
+ test_name, test_name, test_args, iterations, rm_chroot_tmp, perf_args,
+ suite, show_all_results, retries, run_local, cwp_dso, weight)
benchmarks.append(telemetry_benchmark)
def GetExperiment(self, experiment_file, working_directory, log_dir):
@@ -166,16 +164,16 @@
cwp_dso = global_settings.GetField('cwp_dso')
if cwp_dso and not cwp_dso in dso_list:
raise RuntimeError('The DSO specified is not supported')
- enable_aslr = global_settings.GetField('enable_aslr')
ignore_min_max = global_settings.GetField('ignore_min_max')
- turbostat_opt = global_settings.GetField('turbostat')
dut_config = {
+ 'enable_aslr': global_settings.GetField('enable_aslr'),
'intel_pstate': global_settings.GetField('intel_pstate'),
'cooldown_time': global_settings.GetField('cooldown_time'),
'cooldown_temp': global_settings.GetField('cooldown_temp'),
'governor': global_settings.GetField('governor'),
'cpu_usage': global_settings.GetField('cpu_usage'),
'cpu_freq_pct': global_settings.GetField('cpu_freq_pct'),
+ 'turbostat': global_settings.GetField('turbostat'),
'top_interval': global_settings.GetField('top_interval'),
}
@@ -257,50 +255,20 @@
if suite == 'telemetry_Crosperf':
if test_name == 'all_perfv2':
- self.AppendBenchmarkSet(
- benchmarks,
- telemetry_perfv2_tests,
- test_args,
- iterations,
- rm_chroot_tmp,
- perf_args,
- suite,
- show_all_results,
- retries,
- run_local,
- cwp_dso,
- weight,
- turbostat=turbostat_opt)
+ self.AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests, test_args,
+ iterations, rm_chroot_tmp, perf_args, suite,
+ show_all_results, retries, run_local, cwp_dso,
+ weight)
elif test_name == 'all_pagecyclers':
- self.AppendBenchmarkSet(
- benchmarks,
- telemetry_pagecycler_tests,
- test_args,
- iterations,
- rm_chroot_tmp,
- perf_args,
- suite,
- show_all_results,
- retries,
- run_local,
- cwp_dso,
- weight,
- turbostat=turbostat_opt)
+ self.AppendBenchmarkSet(benchmarks, telemetry_pagecycler_tests,
+ test_args, iterations, rm_chroot_tmp,
+ perf_args, suite, show_all_results, retries,
+ run_local, cwp_dso, weight)
elif test_name == 'all_crosbolt_perf':
self.AppendBenchmarkSet(
- benchmarks,
- telemetry_crosbolt_perf_tests,
- test_args,
- iterations,
- rm_chroot_tmp,
- perf_args,
- 'telemetry_Crosperf',
- show_all_results,
- retries,
- run_local,
- cwp_dso,
- weight,
- turbostat=turbostat_opt)
+ benchmarks, telemetry_crosbolt_perf_tests, test_args, iterations,
+ rm_chroot_tmp, perf_args, 'telemetry_Crosperf', show_all_results,
+ retries, run_local, cwp_dso, weight)
self.AppendBenchmarkSet(
benchmarks,
crosbolt_perf_tests,
@@ -313,23 +281,12 @@
retries,
run_local=False,
cwp_dso=cwp_dso,
- weight=weight,
- turbostat=turbostat_opt)
+ weight=weight)
elif test_name == 'all_toolchain_perf':
- self.AppendBenchmarkSet(
- benchmarks,
- telemetry_toolchain_perf_tests,
- test_args,
- iterations,
- rm_chroot_tmp,
- perf_args,
- suite,
- show_all_results,
- retries,
- run_local,
- cwp_dso,
- weight,
- turbostat=turbostat_opt)
+ self.AppendBenchmarkSet(benchmarks, telemetry_toolchain_perf_tests,
+ test_args, iterations, rm_chroot_tmp,
+ perf_args, suite, show_all_results, retries,
+ run_local, cwp_dso, weight)
# Add non-telemetry toolchain-perf benchmarks:
benchmarks.append(
Benchmark(
@@ -344,28 +301,17 @@
retries,
run_local=False,
cwp_dso=cwp_dso,
- weight=weight,
- turbostat=turbostat_opt))
+ weight=weight))
elif test_name == 'all_toolchain_perf_old':
self.AppendBenchmarkSet(
- benchmarks,
- telemetry_toolchain_old_perf_tests,
- test_args,
- iterations,
- rm_chroot_tmp,
- perf_args,
- suite,
- show_all_results,
- retries,
- run_local,
- cwp_dso,
- weight,
- turbostat=turbostat_opt)
+ benchmarks, telemetry_toolchain_old_perf_tests, test_args,
+ iterations, rm_chroot_tmp, perf_args, suite, show_all_results,
+ retries, run_local, cwp_dso, weight)
else:
benchmark = Benchmark(benchmark_name, test_name, test_args,
iterations, rm_chroot_tmp, perf_args, suite,
show_all_results, retries, run_local, cwp_dso,
- weight, turbostat_opt)
+ weight)
benchmarks.append(benchmark)
else:
if test_name == 'all_graphics_perf':
@@ -381,8 +327,7 @@
retries,
run_local=False,
cwp_dso=cwp_dso,
- weight=weight,
- turbostat=turbostat_opt)
+ weight=weight)
else:
# Add the single benchmark.
benchmark = Benchmark(
@@ -397,8 +342,7 @@
retries,
run_local=False,
cwp_dso=cwp_dso,
- weight=weight,
- turbostat=turbostat_opt)
+ weight=weight)
benchmarks.append(benchmark)
if not benchmarks:
@@ -477,8 +421,8 @@
chromeos_root, cache_conditions, labels, benchmarks,
experiment_file.Canonicalize(), email,
acquire_timeout, log_dir, log_level, share_cache,
- results_dir, locks_dir, cwp_dso, enable_aslr,
- ignore_min_max, skylab, dut_config)
+ results_dir, locks_dir, cwp_dso, ignore_min_max,
+ skylab, dut_config)
return experiment
diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py
index a0a3d8d..69e34fc 100755
--- a/crosperf/experiment_factory_unittest.py
+++ b/crosperf/experiment_factory_unittest.py
@@ -243,7 +243,7 @@
bench_list = []
ef.AppendBenchmarkSet(bench_list, experiment_factory.telemetry_perfv2_tests,
'', 1, False, '', 'telemetry_Crosperf', False, 0,
- False, '', 0, False)
+ False, '', 0)
self.assertEqual(
len(bench_list), len(experiment_factory.telemetry_perfv2_tests))
self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark))
@@ -251,7 +251,7 @@
bench_list = []
ef.AppendBenchmarkSet(
bench_list, experiment_factory.telemetry_pagecycler_tests, '', 1, False,
- '', 'telemetry_Crosperf', False, 0, False, '', 0, False)
+ '', 'telemetry_Crosperf', False, 0, False, '', 0)
self.assertEqual(
len(bench_list), len(experiment_factory.telemetry_pagecycler_tests))
self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark))
@@ -259,7 +259,7 @@
bench_list = []
ef.AppendBenchmarkSet(
bench_list, experiment_factory.telemetry_toolchain_perf_tests, '', 1,
- False, '', 'telemetry_Crosperf', False, 0, False, '', 0, False)
+ False, '', 'telemetry_Crosperf', False, 0, False, '', 0)
self.assertEqual(
len(bench_list), len(experiment_factory.telemetry_toolchain_perf_tests))
self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark))
diff --git a/crosperf/machine_manager_unittest.py b/crosperf/machine_manager_unittest.py
index 984251c..f6a77bb 100755
--- a/crosperf/machine_manager_unittest.py
+++ b/crosperf/machine_manager_unittest.py
@@ -12,9 +12,9 @@
import os.path
import time
import hashlib
+import unittest
import mock
-import unittest
import label
import machine_manager
@@ -184,7 +184,7 @@
self.assertEqual(mock_run_cmd.call_count, 0)
self.assertEqual(mock_run_croscmd.call_count, 0)
- #Test 2: label.image_type == "trybot"
+ # Test 2: label.image_type == "trybot"
ResetValues()
LABEL_LUMPY.image_type = 'trybot'
mock_run_cmd.return_value = 0
@@ -456,7 +456,7 @@
suite='telemetry_Crosperf') # suite
test_run = MockBenchmarkRun('test run', bench, LABEL_LUMPY, 1, [], self.mm,
- mock_logger, 'verbose', '', {}, False)
+ mock_logger, 'verbose', '', {})
self.mm._machines = [
self.mock_lumpy1, self.mock_lumpy2, self.mock_lumpy3, self.mock_daisy1,
diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py
index b5649e8..418358e 100644
--- a/crosperf/suite_runner.py
+++ b/crosperf/suite_runner.py
@@ -60,8 +60,7 @@
logger_to_use=None,
log_level='verbose',
cmd_exec=None,
- cmd_term=None,
- enable_aslr=False):
+ cmd_term=None):
self.logger = logger_to_use
self.log_level = log_level
self._ce = cmd_exec or command_executer.GetCommandExecuter(
@@ -69,7 +68,6 @@
# DUT command executer.
# Will be initialized and used within Run.
self._ct = cmd_term or command_executer.CommandTerminator()
- self.enable_aslr = enable_aslr
self.dut_config = dut_config
def Run(self, cros_machine, label, benchmark, test_args, profiler_args):
@@ -134,7 +132,7 @@
with self.PauseUI(run_on_dut):
# Unless the user turns on ASLR in the flag, we first disable ASLR
# before running the benchmarks
- if not self.enable_aslr:
+ if not self.dut_config['enable_aslr']:
run_on_dut.DisableASLR()
# CPU usage setup comes first where we enable/disable cores.
@@ -370,11 +368,12 @@
args_string = "test_args='%s'" % test_args
top_interval = self.dut_config['top_interval']
+ turbostat = self.dut_config['turbostat']
cmd = ('{} {} {} --board={} --args="{} run_local={} test={} '
'turbostat={} top_interval={} {}" {} telemetry_Crosperf'.format(
TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board,
- args_string, benchmark.run_local, benchmark.test_name,
- benchmark.turbostat, top_interval, profiler_args, machine))
+ args_string, benchmark.run_local, benchmark.test_name, turbostat,
+ top_interval, profiler_args, machine))
# Use --no-ns-pid so that cros_sdk does not create a different
# process namespace and we can kill process created easily by their
diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py
index 5f377d7..c2e3322 100755
--- a/crosperf/suite_runner_unittest.py
+++ b/crosperf/suite_runner_unittest.py
@@ -132,6 +132,7 @@
self.runner.SetupDevice = mock.Mock()
DutWrapper.RunCommandOnDut = mock.Mock(return_value=FakeRunner)
+ self.runner.dut_config['enable_aslr'] = False
self.runner.dut_config['cooldown_time'] = 0
self.runner.dut_config['governor'] = 'fake_governor'
self.runner.dut_config['cpu_freq_pct'] = 65
@@ -186,6 +187,8 @@
mock_run_on_dut.RunCommandOnDut = mock.Mock(return_value=FakeRunner)
mock_run_on_dut.WaitCooldown = mock.Mock(return_value=0)
mock_run_on_dut.GetCpuOnline = mock.Mock(return_value={0: 1, 1: 1, 2: 0})
+
+ self.runner.dut_config['enable_aslr'] = False
self.runner.dut_config['cooldown_time'] = 0
self.runner.dut_config['governor'] = 'fake_governor'
self.runner.dut_config['cpu_freq_pct'] = 65
@@ -218,6 +221,7 @@
mock_run_on_dut.WaitCooldown = mock.Mock(return_value=0)
mock_run_on_dut.GetCpuOnline = mock.Mock(return_value={0: 0, 1: 1})
+ self.runner.dut_config['enable_aslr'] = False
self.runner.dut_config['cooldown_time'] = 10
self.runner.dut_config['governor'] = 'fake_governor'
self.runner.dut_config['cpu_freq_pct'] = 75
@@ -246,6 +250,7 @@
self.mock_logger)
mock_run_on_dut.SetupCpuUsage = mock.Mock(side_effect=RuntimeError())
+ self.runner.dut_config['enable_aslr'] = False
with self.assertRaises(RuntimeError):
self.runner.SetupDevice(mock_run_on_dut, cros_machine)
@@ -313,6 +318,7 @@
self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
profiler_args = ("--profiler=custom_perf --profiler_args='perf_options"
'="record -a -e cycles,instructions"\'')
+ self.runner.dut_config['turbostat'] = True
self.runner.dut_config['top_interval'] = 3
res = self.runner.Telemetry_Crosperf_Run('lumpy1.cros', self.mock_label,
self.telemetry_crosperf_bench, '',
