[autotest] Add ablity to run script(s) before running CTS test.
Some tests require special conditions on the device before passing
such as wifi connection.
Add ability to run scripts ( that previsouly have to be installed
on the DUT ) to ensure the test conditions are correct.
Create new suite for testing this feature and partner roll out.
These changes do not affect CTS testing in the lab.
TEST=manual runs on moblab
BUG=chromium:739582
Change-Id: I688924a7a3fd7d3ba6941ffc462a2c50a5ed47c8
Reviewed-on: https://chromium-review.googlesource.com/563533
Reviewed-by: Keith Haddow <haddowk@chromium.org>
Reviewed-by: Rohit Makasana <rohitbm@chromium.org>
Tested-by: Keith Haddow <haddowk@chromium.org>
Commit-Queue: Keith Haddow <haddowk@chromium.org>
diff --git a/server/site_tests/cheets_CTS_N/cheets_CTS_N.py b/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
index bf7bd86..c0a28fb 100644
--- a/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
+++ b/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
@@ -256,6 +256,13 @@
return ((tests == passed + failed) or
(tests == passed + failed + notexecuted))
+ def _run_precondition_scripts(self, host, commands):
+ for command in commands:
+ logging.info('RUN: %s\n', command)
+ output = host.run(command, ignore_status=True)
+ logging.info('END: %s\n', output)
+
+
def run_once(self,
target_module=None,
target_plan=None,
@@ -264,6 +271,7 @@
needs_push_media=False,
max_retry=None,
cts_tradefed_args=None,
+ pre_condition_commands=[],
timeout=_CTS_TIMEOUT_SECONDS):
"""Runs the specified CTS once, but with several retries.
@@ -284,6 +292,7 @@
@param timeout: time after which tradefed can be interrupted.
@param cts_tradefed_args: a list of args to pass to tradefed.
"""
+
# On dev and beta channels timeouts are sharp, lenient on stable.
self._timeout = timeout
if self._get_release_channel == 'stable':
@@ -329,6 +338,9 @@
steps += 1
with self._login_chrome(dont_override_profile=pushed_media):
self._ready_arc()
+ self._run_precondition_scripts(
+ self._host,
+ pre_condition_commands)
# Only push media for tests that need it. b/29371037
if needs_push_media and not pushed_media:
@@ -385,6 +397,9 @@
steps += 1
with self._login_chrome(dont_override_profile=pushed_media):
self._ready_arc()
+ self._run_precondition_scripts(
+ self._host,
+ pre_condition_commands)
logging.info('Retrying failures of %s with session_id %d:',
test_name, session_id)
expected_tests = failed + notexecuted
diff --git a/server/site_tests/cheets_CTS_N/control.arm.CtsCameraTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.arm.CtsCameraTestCasesPreconditions
new file mode 100644
index 0000000..126fc2e
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.arm.CtsCameraTestCasesPreconditions
@@ -0,0 +1,33 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.arm.CtsCameraTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc, lighting'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsCameraTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using arm ABI in the ARC++ container.')
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsCameraTestCases',
+ target_module='CtsCameraTestCases',
+ bundle='arm',
+ timeout=3600)
+
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.arm.CtsMediaTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.arm.CtsMediaTestCasesPreconditions
new file mode 100644
index 0000000..9fa9c56
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.arm.CtsMediaTestCasesPreconditions
@@ -0,0 +1,33 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.arm.CtsMediaTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc, noloopback'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsMediaTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using arm ABI in the ARC++ container.')
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsMediaTestCases',
+ target_module='CtsMediaTestCases',
+ bundle='arm',
+ timeout=21600)
+
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.arm.CtsNetTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.arm.CtsNetTestCasesPreconditions
new file mode 100644
index 0000000..cb61615
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.arm.CtsNetTestCasesPreconditions
@@ -0,0 +1,43 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.arm.CtsNetTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsNetTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using arm ABI in the ARC++ container.')
+
+if not ssid:
+ ssid = 'GoogleGuest'
+
+if not wifipass:
+ wifipass = ''
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsNetTestCases',
+ target_module='CtsNetTestCases',
+ bundle='arm',
+ timeout=3600,
+ pre_condition_commands=[
+ '/usr/local/autotest/cros/scripts/wifi connect %s %s' % (ssid, wifipass),
+ '/usr/local/autotest/cros/scripts/reorder-services-moblab.sh wifi',
+ ])
+
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.arm.CtsSecurityHostTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.arm.CtsSecurityHostTestCasesPreconditions
new file mode 100644
index 0000000..4833405
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.arm.CtsSecurityHostTestCasesPreconditions
@@ -0,0 +1,36 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.arm.CtsSecurityHostTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsSecurityHostTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using arm ABI in the ARC++ container.')
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsSecurityHostTestCases',
+ target_module='CtsSecurityHostTestCases',
+ bundle='arm',
+ timeout=5400,
+ pre_condition_commands=[
+ 'echo 3 > /proc/sys/kernel/perf_event_paranoid'
+ ])
+
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.arm.CtsServicesHostTestCases b/server/site_tests/cheets_CTS_N/control.arm.CtsServicesHostTestCases
index 948b65f..a9ae937 100644
--- a/server/site_tests/cheets_CTS_N/control.arm.CtsServicesHostTestCases
+++ b/server/site_tests/cheets_CTS_N/control.arm.CtsServicesHostTestCases
@@ -22,11 +22,11 @@
'cheets_CTS_N',
host=host,
iterations=1,
- max_retry=3,
+ max_retry=10,
needs_push_media=False,
tag='CtsServicesHostTestCases',
target_module='CtsServicesHostTestCases',
bundle='arm',
timeout=3600)
-parallel_simple(run_CTS, machines)
\ No newline at end of file
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.arm.CtsUsageStatsTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.arm.CtsUsageStatsTestCasesPreconditions
new file mode 100644
index 0000000..ccaa25c
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.arm.CtsUsageStatsTestCasesPreconditions
@@ -0,0 +1,43 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.arm.CtsUsageStatsTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsUsageStatsTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using arm ABI in the ARC++ container.')
+
+if not ssid:
+ ssid = 'GoogleGuest'
+
+if not wifipass:
+ wifipass = ''
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsUsageStatsTestCases',
+ target_module='CtsUsageStatsTestCases',
+ bundle='arm',
+ timeout=3600,
+ pre_condition_commands=[
+ '/usr/local/autotest/cros/scripts/wifi connect %s %s' % (ssid, wifipass),
+ '/usr/local/autotest/cros/scripts/reorder-services-moblab.sh wifi',
+ ])
+
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.x86.CtsCameraTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.x86.CtsCameraTestCasesPreconditions
new file mode 100644
index 0000000..4791567
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.x86.CtsCameraTestCasesPreconditions
@@ -0,0 +1,33 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.x86.CtsCameraTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc, cts_abi_x86, lighting'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsCameraTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using x86 ABI in the ARC++ container.')
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsCameraTestCases',
+ target_module='CtsCameraTestCases',
+ bundle='x86',
+ timeout=3600)
+
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.x86.CtsMediaTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.x86.CtsMediaTestCasesPreconditions
new file mode 100644
index 0000000..620e90b
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.x86.CtsMediaTestCasesPreconditions
@@ -0,0 +1,33 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.x86.CtsMediaTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc, cts_abi_x86, noloopback'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsMediaTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using x86 ABI in the ARC++ container.')
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsMediaTestCases',
+ target_module='CtsMediaTestCases',
+ bundle='x86',
+ timeout=21600)
+
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.x86.CtsNetTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.x86.CtsNetTestCasesPreconditions
new file mode 100644
index 0000000..d6230f7
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.x86.CtsNetTestCasesPreconditions
@@ -0,0 +1,43 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.x86.CtsNetTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc, cts_abi_x86'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsNetTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using x86 ABI in the ARC++ container.')
+
+if not ssid:
+ ssid = 'GoogleGuest'
+
+if not wifipass:
+ wifipass = ''
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsNetTestCases',
+ target_module='CtsNetTestCases',
+ bundle='x86',
+ timeout=3600,
+ pre_condition_commands=[
+ '/usr/local/autotest/cros/scripts/wifi connect %s %s' % (ssid, wifipass),
+ '/usr/local/autotest/cros/scripts/reorder-services-moblab.sh wifi',
+ ])
+
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.x86.CtsSecurityHostTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.x86.CtsSecurityHostTestCasesPreconditions
new file mode 100644
index 0000000..c7f9853
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.x86.CtsSecurityHostTestCasesPreconditions
@@ -0,0 +1,36 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.x86.CtsSecurityHostTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc, cts_abi_x86'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsSecurityHostTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using x86 ABI in the ARC++ container.')
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsSecurityHostTestCases',
+ target_module='CtsSecurityHostTestCases',
+ bundle='x86',
+ timeout=5400,
+ pre_condition_commands=[
+ 'echo 3 > /proc/sys/kernel/perf_event_paranoid'
+ ])
+
+parallel_simple(run_CTS, machines)
diff --git a/server/site_tests/cheets_CTS_N/control.x86.CtsUsageStatsTestCasesPreconditions b/server/site_tests/cheets_CTS_N/control.x86.CtsUsageStatsTestCasesPreconditions
new file mode 100644
index 0000000..3bc25c3
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/control.x86.CtsUsageStatsTestCasesPreconditions
@@ -0,0 +1,44 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# TODO Integrate these changes back into cts_N as soon as partners have a
+# chance to configure their DUT's correctly and documentation is complete.
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_CTS_N.x86.CtsUsageStatsTestCasesPreconditions'
+ATTRIBUTES = 'suite:cts_N_Pre'
+DEPENDENCIES = 'arc, cts_abi_x86'
+JOB_RETRIES = 2
+TEST_TYPE = 'server'
+TIME = 'LENGTHY'
+
+DOC = ('Run module CtsUsageStatsTestCases of the '
+ 'Android 7.1_r6 Compatibility Test Suite (CTS), build 4009365,'
+ 'using x86 ABI in the ARC++ container.')
+
+if not ssid:
+ ssid = 'GoogleGuest'
+
+if not wifipass:
+ wifipass = ''
+
+def run_CTS(machine):
+ host = hosts.create_host(machine)
+ job.run_test(
+ 'cheets_CTS_N',
+ host=host,
+ iterations=1,
+ max_retry=3,
+ needs_push_media=False,
+ tag='CtsUsageStatsTestCases',
+ target_module='CtsUsageStatsTestCases',
+ bundle='x86',
+ timeout=3600,
+ pre_condition_commands=[
+ '/usr/local/autotest/cros/scripts/wifi connect %s %s' % (ssid, wifipass),
+ '/usr/local/autotest/cros/scripts/reorder-services-moblab.sh wifi',
+
+ ])
+
+parallel_simple(run_CTS, machines)
diff --git a/test_suites/control.cts_N_predconditions b/test_suites/control.cts_N_predconditions
new file mode 100644
index 0000000..92b9f90
--- /dev/null
+++ b/test_suites/control.cts_N_predconditions
@@ -0,0 +1,51 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = "ihf@chromium.org"
+NAME = "cts_N_Pre"
+PURPOSE = "Special suite for CTS qualification of ARC++."
+CRITERIA = "All tests with SUITE=cts_N_Pre must pass."
+
+TIME = "LONG"
+TEST_CATEGORY = "General"
+TEST_CLASS = "suite"
+TEST_TYPE = "Server"
+
+DOC = """
+ChromeOS ARC++ tests.
+
+@param build: The name of the image to test.
+ Ex: x86-mario-release/R17-1412.33.0-a1-b29
+@param board: The board to test on. Ex: x86-mario
+@param pool: The pool of machines to utilize for scheduling. If pool=None
+ board is used.
+@param check_hosts: require appropriate live hosts to exist in the lab.
+@param SKIP_IMAGE: (optional) If present and True, don't re-image devices.
+"""
+
+import common
+from autotest_lib.server.cros import provision
+from autotest_lib.server.cros.dynamic_suite import dynamic_suite
+
+
+def predicate(test):
+ if not hasattr(test, 'suite') or not hasattr(test, 'name'):
+ return False
+ if not test.suite == NAME:
+ return False
+ # Strip off the cheets_CTS_N. from the test name before comparing to args
+ if suite_args and not test.name[test.name.find('.') + 1:] in suite_args:
+ return False
+ return True
+
+args_dict['name'] = NAME
+args_dict['job'] = job
+args_dict['file_bugs'] = False
+args_dict['max_runtime_mins'] = 5040 # 5040 min = 3.5 days
+args_dict['timeout_mins'] = 5040 # Bump default 1 day to half a week as well.
+args_dict['add_experimental'] = True
+args_dict['version_prefix'] = provision.CROS_VERSION_PREFIX
+args_dict['predicate'] = predicate
+dynamic_suite.reimage_and_run(**args_dict)
+
