crosperf: Disable ASLR in crosperf by default
Disabling Address Space Layout Randomization (ASLR) can reduce
much variance when measuring performance of benchmark from run
to run. This CL will disable ASLR in crosperf by default and
add a flag '--enable_aslr' for users to enable it for other
testing purpose.
BUG=chromium:930332
TEST=Tested locally for several boards and several benchmarks
Change-Id: Icfbe477a99ef9a68b503e4979cf2ff47d07a128f
Reviewed-on: https://chromium-review.googlesource.com/1398522
Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com>
Tested-by: Tiancong Wang <tcwang@google.com>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py
index 0ab2970..4c5c75b 100644
--- a/crosperf/benchmark_run.py
+++ b/crosperf/benchmark_run.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -30,7 +31,8 @@
"""The benchmarkrun class."""
def __init__(self, name, benchmark, label, iteration, cache_conditions,
- machine_manager, logger_to_use, log_level, share_cache):
+ machine_manager, logger_to_use, log_level, share_cache,
+ enable_aslr):
threading.Thread.__init__(self)
self.name = name
self._logger = logger_to_use
@@ -43,7 +45,8 @@
self.retval = None
self.run_completed = False
self.machine_manager = machine_manager
- self.suite_runner = SuiteRunner(self._logger, self.log_level)
+ self.suite_runner = SuiteRunner(
+ self._logger, self.log_level, enable_aslr=enable_aslr)
self.machine = None
self.cache_conditions = cache_conditions
self.runs_complete = 0
@@ -167,9 +170,9 @@
machine = self.machine_manager.AcquireMachine(self.label)
if machine:
- self._logger.LogOutput('%s: Machine %s acquired at %s' %
- (self.name, machine.name,
- datetime.datetime.now()))
+ self._logger.LogOutput(
+ '%s: Machine %s acquired at %s' % (self.name, machine.name,
+ datetime.datetime.now()))
break
time.sleep(10)
return machine
diff --git a/crosperf/experiment.py b/crosperf/experiment.py
index c4ed596..d5770d4 100644
--- a/crosperf/experiment.py
+++ b/crosperf/experiment.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -27,7 +28,7 @@
def __init__(self, name, remote, working_directory, chromeos_root,
cache_conditions, labels, benchmarks, experiment_file, email_to,
acquire_timeout, log_dir, log_level, share_cache,
- results_directory, locks_directory, cwp_dso):
+ results_directory, locks_directory, cwp_dso, enable_aslr):
self.name = name
self.working_directory = working_directory
self.remote = remote
@@ -54,6 +55,7 @@
self.locks_dir = locks_directory
self.locked_machines = []
self.cwp_dso = cwp_dso
+ self.enable_aslr = enable_aslr
if not remote:
raise RuntimeError('No remote hosts specified')
@@ -128,10 +130,10 @@
logger_to_use = logger.Logger(self.log_dir, 'run.%s' % (full_name),
True)
benchmark_runs.append(
- benchmark_run.BenchmarkRun(benchmark_run_name, benchmark, label,
- iteration, self.cache_conditions,
- self.machine_manager, logger_to_use,
- self.log_level, self.share_cache))
+ benchmark_run.BenchmarkRun(
+ benchmark_run_name, benchmark, label, iteration,
+ self.cache_conditions, self.machine_manager, logger_to_use,
+ self.log_level, self.share_cache, self.enable_aslr))
return benchmark_runs
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index aa877c6..8fc4ea3 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -147,6 +147,8 @@
cwp_dso = global_settings.GetField('cwp_dso')
if cwp_dso and not cwp_dso in dso_list:
raise RuntimeError('The DSO specified is not supported')
+ enable_aslr = global_settings.GetField('enable_aslr')
+
# Default cache hit conditions. The image checksum in the cache and the
# computed checksum of the image must match. Also a cache file must exist.
cache_conditions = [
@@ -370,7 +372,7 @@
chromeos_root, cache_conditions, labels, benchmarks,
experiment_file.Canonicalize(), email,
acquire_timeout, log_dir, log_level, share_cache,
- results_dir, locks_dir, cwp_dso)
+ results_dir, locks_dir, cwp_dso, enable_aslr)
return experiment
diff --git a/crosperf/experiment_files/enable_aslr.exp b/crosperf/experiment_files/enable_aslr.exp
new file mode 100644
index 0000000..5f8f654
--- /dev/null
+++ b/crosperf/experiment_files/enable_aslr.exp
@@ -0,0 +1,37 @@
+# This example experiment file shows how to run a Telemetry test,
+# using autotest (via "suite: telemetry_Crosperf"), and also enable
+# ASLR. Note that ASLR is diabled by default
+# This turns on ASLR on the machine and runs the Telemetry's
+# "run_benchmark" for the specified test,
+#
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+name: basic_telemetry_crosperf_example
+board: <your-board-goes-here>
+
+enable_aslr: True
+
+# Note: You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+# Replace "octane" below with the name of the Telemetry benchmark you
+# want to run.
+benchmark: octane {
+ suite: telemetry_Crosperf
+ iterations: 1
+}
+
+# NOTE: You must specify at least one image; you may specify more than one.
+# Replace <path-to-your-chroot-goes-here> and <board-goes-here> below.
+vanilla_image {
+ chromeos_image:<path-to-your-chroot>/src/build/images/<board>/vanilla-image/chromiumos_test_image.bin
+}
+
+# Replace the chromeos image below with the actual path to your test image.
+test_image {
+ chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index edf269c..8bc52a4 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -302,6 +303,13 @@
'be empty.',
required=False,
default=''))
+ self.AddField(
+ BooleanField(
+ 'enable_aslr',
+ description='Enable ASLR on the machine to run the '
+ 'benchmarks. ASLR is disabled by default',
+ required=False,
+ default=False))
class SettingsFactory(object):
diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py
index b4b669a..a419194 100644
--- a/crosperf/suite_runner.py
+++ b/crosperf/suite_runner.py
@@ -1,4 +1,5 @@
-# Copyright (c) 2013~2015 The Chromium OS Authors. All rights reserved.
+# -*- coding: utf-8 -*-
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""SuiteRunner defines the interface from crosperf to test script."""
@@ -6,7 +7,6 @@
from __future__ import print_function
import os
-import time
import shlex
from cros_utils import command_executer
@@ -50,15 +50,21 @@
logger_to_use=None,
log_level='verbose',
cmd_exec=None,
- cmd_term=None):
+ cmd_term=None,
+ enable_aslr=False):
self.logger = logger_to_use
self.log_level = log_level
self._ce = cmd_exec or command_executer.GetCommandExecuter(
self.logger, log_level=self.log_level)
self._ct = cmd_term or command_executer.CommandTerminator()
+ self.enable_aslr = enable_aslr
def Run(self, machine, label, benchmark, test_args, profiler_args):
for i in range(0, benchmark.retries + 1):
+ # Unless the user turns on ASLR in the flag, we first disable ASLR
+ # before running the benchmarks
+ if not self.enable_aslr:
+ self.DisableASLR(machine, label.chromeos_root)
self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
if benchmark.suite == 'telemetry':
self.DecreaseWaitTime(machine, label.chromeos_root)
@@ -70,6 +76,7 @@
else:
ret_tup = self.Test_That_Run(machine, label, benchmark, test_args,
profiler_args)
+
if ret_tup[0] != 0:
self.logger.LogOutput('benchmark %s failed. Retries left: %s' %
(benchmark.name, benchmark.retries - i))
@@ -83,6 +90,17 @@
break
return ret_tup
+ def DisableASLR(self, machine_name, chromeos_root):
+ disable_aslr = ('set -e && '
+ 'stop ui; '
+ 'if [[ -e /proc/sys/kernel/randomize_va_space ]]; then '
+ ' echo 0 > /proc/sys/kernel/randomize_va_space; '
+ 'fi; '
+ 'start ui ')
+ self.logger.LogOutput('Disable ASLR for %s' % machine_name)
+ self._ce.CrosRunCommand(
+ disable_aslr, machine=machine_name, chromeos_root=chromeos_root)
+
def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
"""Set min and max frequencies to max static frequency."""
# pyformat: disable
@@ -112,8 +130,7 @@
#'fi ;'
#'echo $highest > scaling_max_freq; '
#'echo $highest > scaling_min_freq; '
- 'done'
- )
+ 'done')
# pyformat: enable
if self.log_level == 'average':
self.logger.LogOutput(
