[autotest] add a --no-experimental flag to test_that
This CL adds a --no-experimental flag to test_that, which will cause it
to skip tests marked as experimental when scheduling a suite. A minor
change to suite.py was required in order for test_that's count of
scheduled tests to be correct when skipping experimental tests.
BUG=chromium:262135
TEST=Manually verified that experimental tests do not get scheduled when
flag is used. Unit tests pass.
Change-Id: I33247446e95915f372ced59f46b78c36fa82c1a7
Reviewed-on: https://gerrit.chromium.org/gerrit/62657
Tested-by: Aviv Keshet <akeshet@chromium.org>
Reviewed-by: Simran Basi <sbasi@chromium.org>
Commit-Queue: Aviv Keshet <akeshet@chromium.org>
diff --git a/server/cros/dynamic_suite/suite.py b/server/cros/dynamic_suite/suite.py
index 1359cf4..f2db7ee 100644
--- a/server/cros/dynamic_suite/suite.py
+++ b/server/cros/dynamic_suite/suite.py
@@ -380,22 +380,26 @@
|self._jobs|.
@param add_experimental: schedule experimental tests as well, or not.
+ @returns: The number of tests that were scheduled.
"""
logging.debug('Discovered %d stable tests.', len(self.stable_tests()))
logging.debug('Discovered %d unstable tests.',
len(self.unstable_tests()))
+ n_scheduled = 0
Status('INFO', 'Start %s' % self._tag).record_result(record)
try:
for test in self.stable_tests():
logging.debug('Scheduling %s', test.name)
self._jobs.append(self._create_job(test))
+ n_scheduled += 1
if add_experimental:
for test in self.unstable_tests():
logging.debug('Scheduling experimental %s', test.name)
test.name = constants.EXPERIMENTAL_PREFIX + test.name
self._jobs.append(self._create_job(test))
+ n_scheduled += 1
if self._results_dir:
self._remember_scheduled_job_ids()
@@ -404,6 +408,8 @@
Status('FAIL', self._tag,
'Exception while scheduling suite').record_result(record)
+ return n_scheduled
+
def should_file_bug(self, result):
"""
diff --git a/site_utils/test_that.py b/site_utils/test_that.py
index 1f46cd4..2835fad 100755
--- a/site_utils/test_that.py
+++ b/site_utils/test_that.py
@@ -44,7 +44,8 @@
def schedule_local_suite(autotest_path, suite_name, afe, build=_NO_BUILD,
- board=_NO_BOARD, results_directory=None):
+ board=_NO_BOARD, results_directory=None,
+ no_experimental=False):
"""
Schedule a suite against a mock afe object, for a local suite run.
@param autotest_path: Absolute path to autotest (in sysroot).
@@ -54,6 +55,7 @@
@param board: Board to schedule suite for.
@param results_directory: Absolute path of directory to store results in.
(results will be stored in subdirectory of this).
+ @param no_experimental: Skip experimental tests when scheduling a suite.
@returns: The number of tests scheduled.
"""
fs_getter = suite.Suite.create_fs_getter(autotest_path)
@@ -64,8 +66,9 @@
if len(my_suite.tests) == 0:
raise ValueError('Suite named %s does not exist, or contains no '
'tests.' % suite_name)
- my_suite.schedule(lambda x: None) # Schedule tests, discard record calls.
- return len(my_suite.tests)
+ # Schedule tests, discard record calls.
+ return my_suite.schedule(lambda x: None,
+ add_experimental=not no_experimental)
def schedule_local_test(autotest_path, test_name, afe, build=_NO_BUILD,
@@ -93,8 +96,8 @@
results_dir=results_directory)
if len(my_suite.tests) == 0:
raise ValueError('No tests named %s.' % test_name)
- my_suite.schedule(lambda x: None) # Schedule tests, discard record calls.
- return len(my_suite.tests)
+ # Schedule tests, discard record calls.
+ return my_suite.schedule(lambda x: None)
def run_job(job, host, sysroot_autotest_path, results_directory, fast_mode,
@@ -161,7 +164,7 @@
def perform_local_run(afe, autotest_path, tests, remote, fast_mode,
build=_NO_BUILD, board=_NO_BOARD, args=None,
- pretend=False):
+ pretend=False, no_experimental=False):
"""
@param afe: A direct_afe object used to interact with local afe database.
@param autotest_path: Absolute path of sysroot installed autotest.
@@ -175,7 +178,7 @@
and then ultimitely to test itself.
@param pretend: If True, will print out autoserv commands rather than
running them.
-
+ @param no_experimental: Skip experimental tests when scheduling a suite.
@returns: directory in which results are stored.
"""
@@ -195,7 +198,8 @@
logging.info('Scheduling suite %s...', suitename)
ntests = schedule_local_suite(autotest_path, suitename, afe,
build=build, board=board,
- results_directory=results_directory)
+ results_directory=results_directory,
+ no_experimental=no_experimental)
else:
logging.info('Scheduling test %s...', test)
ntests = schedule_local_test(autotest_path, test, afe,
@@ -279,7 +283,11 @@
help='Skip the quickmerge step and use the sysroot '
'as it currently is. May result in un-merged '
'source tree changes not being reflected in run.')
-
+ parser.add_argument('--no-experimental', action='store_true',
+ default=False, dest='no_experimental',
+ help='When scheduling a suite, skip any tests marked '
+ 'as experimental. Applies only to tests scheduled'
+ ' via suite:[SUITE].')
return parser.parse_args(argv)
@@ -390,7 +398,8 @@
res_dir= perform_local_run(afe, sysroot_autotest_path, arguments.tests,
arguments.remote, arguments.fast_mode,
args=arguments.args,
- pretend=arguments.pretend)
+ pretend=arguments.pretend,
+ no_experimental=arguments.no_experimental)
if arguments.pretend:
logging.info('Finished pretend run. Exiting.')
return 0
diff --git a/site_utils/test_that_unittest.py b/site_utils/test_that_unittest.py
index 2ba353d..2345e3c 100755
--- a/site_utils/test_that_unittest.py
+++ b/site_utils/test_that_unittest.py
@@ -152,7 +152,8 @@
self.mox.StubOutWithMock(test_that, 'schedule_local_suite')
test_that.schedule_local_suite(autotest_path, suite_name,
afe, build=build,
- board=board, results_directory=results_dir
+ board=board, results_directory=results_dir,
+ no_experimental=False
).WithSideEffects(fake_suite_callback)
self.mox.StubOutWithMock(test_that, 'run_job')
