cheets_GTS: HACK to split out unstable cases in GtsExo
A few particular unstable test cases is causing
trouble in stably running the full test module.
As a workaround until the root cause is fixed,
we split out the flaky tests to separate jobs so
that other tests can run stably.
BUG=b:178432852
TEST=cheets_GTS.8.0_r3.GtsExoPlayerTestCases.others
Change-Id: Iaf00291cc8cf8142f8b32afdd6b5ed58e3777eb1
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/2717583
Reviewed-by: Jiyoun Ha <jiyounha@chromium.org>
Commit-Queue: Kazuhiro Inaba <kinaba@chromium.org>
Tested-by: Kazuhiro Inaba <kinaba@chromium.org>
Auto-Submit: Kazuhiro Inaba <kinaba@chromium.org>
diff --git a/server/cros/tradefed/generate_controlfiles_GTS.py b/server/cros/tradefed/generate_controlfiles_GTS.py
index 8a9a7e9..d39918d 100755
--- a/server/cros/tradefed/generate_controlfiles_GTS.py
+++ b/server/cros/tradefed/generate_controlfiles_GTS.py
@@ -39,7 +39,7 @@
CONFIG['CONTROLFILE_TEST_FUNCTION_NAME'] = 'run_TS'
CONFIG['CONTROLFILE_WRITE_SIMPLE_QUAL_AND_REGRESS'] = False
CONFIG['CONTROLFILE_WRITE_CAMERA'] = False
-CONFIG['CONTROLFILE_WRITE_EXTRA'] = False
+CONFIG['CONTROLFILE_WRITE_EXTRA'] = True
CONFIG['CTS_JOB_RETRIES_IN_PUBLIC'] = 2
CONFIG['CTS_QUAL_RETRIES'] = 9
@@ -51,6 +51,7 @@
CONFIG['CTS_TIMEOUT'] = {
'GtsAssistantMicHostTestCases': 0.5,
'GtsExoPlayerTestCases': 1.5,
+ 'GtsExoPlayerTestCases.others': 1.5,
'GtsGmscoreHostTestCases': 1.0,
'GtsMediaTestCases': 4,
'GtsNetworkWatchlistTestCases': 1.0,
@@ -68,11 +69,12 @@
CONFIG['QUAL_TIMEOUT'] = 24
CONFIG['QUAL_BOOKMARKS'] = sorted([
- 'A', # A bookend to simplify partition algorithm.
- 'GtsExoPlayerTestCases',
- 'GtsMediaTestCases',
- 'GtsMediaTestCasesz', # runs the biggest module in a single job.
- 'zzzzz' # A bookend to simplify algorithm.
+ 'A', # A bookend to simplify partition algorithm.
+ 'GtsExoPlayerTestCases',
+ 'GtsExoPlayerTestCasesz', # TODO(b/178432852) runs GtsExo alone
+ 'GtsMediaTestCases',
+ 'GtsMediaTestCasesz', # runs the biggest module in a single job.
+ 'zzzzz' # A bookend to simplify algorithm.
])
CONFIG['SMOKE'] = []
@@ -138,10 +140,44 @@
# individual tests finish in the order of 10ms or less (b/118836700). Specify
# modules here to not enable the flag.
CONFIG['DISABLE_LOGCAT_ON_FAILURE'] = set([])
-CONFIG['EXTRA_MODULES'] = {}
+CONFIG['EXTRA_MODULES'] = {
+ 'GtsExoPlayerTestCases': {
+ 'SUBMODULES':
+ set([
+ 'GtsExoPlayerTestCases.cbc1',
+ 'GtsExoPlayerTestCases.cbcs',
+ 'GtsExoPlayerTestCases.cenc',
+ 'GtsExoPlayerTestCases.others',
+ ]),
+ 'SUITES': ['suite:arc-gts-qual'],
+ }
+}
CONFIG['PUBLIC_EXTRA_MODULES'] = {}
CONFIG['EXTRA_SUBMODULE_OVERRIDE'] = {}
-CONFIG['EXTRA_COMMANDLINE'] = {}
+CONFIG['EXTRA_COMMANDLINE'] = {
+ 'GtsExoPlayerTestCases.cbc1': [
+ '--include-filter',
+ 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCbc1',
+ ],
+ 'GtsExoPlayerTestCases.cbcs': [
+ '--include-filter',
+ 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCbcs',
+ ],
+ 'GtsExoPlayerTestCases.cenc': [
+ '--include-filter',
+ 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCenc',
+ ],
+ 'GtsExoPlayerTestCases.others': [
+ '--module',
+ 'GtsExoPlayerTestCases',
+ '--exclude-filter',
+ 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCbc1',
+ '--exclude-filter',
+ 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCbcs',
+ '--exclude-filter',
+ 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCenc',
+ ],
+}
CONFIG['EXTRA_ATTRIBUTES'] = {
'tradefed-run-collect-tests-only-internal': ['suite:arc-gts'],
}
diff --git a/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.cbc1 b/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.cbc1
new file mode 100644
index 0000000..e15ba63
--- /dev/null
+++ b/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.cbc1
@@ -0,0 +1,35 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file has been automatically generated. Do not edit!
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_GTS.8.0_r3.GtsExoPlayerTestCases.cbc1'
+ATTRIBUTES = 'suite:arc-gts-qual'
+DEPENDENCIES = 'arc'
+JOB_RETRIES = 1
+TEST_TYPE = 'server'
+TIME = 'MEDIUM'
+MAX_RESULT_SIZE_KB = 307200
+DOC = 'Run module GtsExoPlayerTestCases.cbc1 of the Android Google Test Suite (GTS) in the ARC++ container.'
+
+def run_TS(machine):
+ host_list = [hosts.create_host(machine)]
+ job.run_test(
+ 'cheets_GTS',
+ hosts=host_list,
+ iterations=1,
+ max_retry=9,
+ tag='8.0_r3.GtsExoPlayerTestCases.cbc1',
+ test_name='cheets_GTS.8.0_r3.GtsExoPlayerTestCases.cbc1',
+ authkey='gs://chromeos-arc-images/cts/bundle/gts-arc.json',
+ run_template=['run', 'commandAndExit', 'gts', '--include-filter', 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCbc1', '--ignore-business-logic-failure'],
+ retry_template=['run', 'commandAndExit', 'retry', '--retry', '{session_id}'],
+ target_module='GtsExoPlayerTestCases',
+ target_plan=None,
+ uri='gs://chromeos-arc-images/cts/bundle/android-gts-8-R3-P-7133222.zip',
+ use_jdk9=True,
+ timeout=86400)
+
+parallel_simple(run_TS, machines)
diff --git a/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.cbcs b/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.cbcs
new file mode 100644
index 0000000..4b6cadc
--- /dev/null
+++ b/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.cbcs
@@ -0,0 +1,35 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file has been automatically generated. Do not edit!
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_GTS.8.0_r3.GtsExoPlayerTestCases.cbcs'
+ATTRIBUTES = 'suite:arc-gts-qual'
+DEPENDENCIES = 'arc'
+JOB_RETRIES = 1
+TEST_TYPE = 'server'
+TIME = 'MEDIUM'
+MAX_RESULT_SIZE_KB = 307200
+DOC = 'Run module GtsExoPlayerTestCases.cbcs of the Android Google Test Suite (GTS) in the ARC++ container.'
+
+def run_TS(machine):
+ host_list = [hosts.create_host(machine)]
+ job.run_test(
+ 'cheets_GTS',
+ hosts=host_list,
+ iterations=1,
+ max_retry=9,
+ tag='8.0_r3.GtsExoPlayerTestCases.cbcs',
+ test_name='cheets_GTS.8.0_r3.GtsExoPlayerTestCases.cbcs',
+ authkey='gs://chromeos-arc-images/cts/bundle/gts-arc.json',
+ run_template=['run', 'commandAndExit', 'gts', '--include-filter', 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCbcs', '--ignore-business-logic-failure'],
+ retry_template=['run', 'commandAndExit', 'retry', '--retry', '{session_id}'],
+ target_module='GtsExoPlayerTestCases',
+ target_plan=None,
+ uri='gs://chromeos-arc-images/cts/bundle/android-gts-8-R3-P-7133222.zip',
+ use_jdk9=True,
+ timeout=86400)
+
+parallel_simple(run_TS, machines)
diff --git a/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.cenc b/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.cenc
new file mode 100644
index 0000000..516b363
--- /dev/null
+++ b/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.cenc
@@ -0,0 +1,35 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file has been automatically generated. Do not edit!
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_GTS.8.0_r3.GtsExoPlayerTestCases.cenc'
+ATTRIBUTES = 'suite:arc-gts-qual'
+DEPENDENCIES = 'arc'
+JOB_RETRIES = 1
+TEST_TYPE = 'server'
+TIME = 'MEDIUM'
+MAX_RESULT_SIZE_KB = 307200
+DOC = 'Run module GtsExoPlayerTestCases.cenc of the Android Google Test Suite (GTS) in the ARC++ container.'
+
+def run_TS(machine):
+ host_list = [hosts.create_host(machine)]
+ job.run_test(
+ 'cheets_GTS',
+ hosts=host_list,
+ iterations=1,
+ max_retry=9,
+ tag='8.0_r3.GtsExoPlayerTestCases.cenc',
+ test_name='cheets_GTS.8.0_r3.GtsExoPlayerTestCases.cenc',
+ authkey='gs://chromeos-arc-images/cts/bundle/gts-arc.json',
+ run_template=['run', 'commandAndExit', 'gts', '--include-filter', 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCenc', '--ignore-business-logic-failure'],
+ retry_template=['run', 'commandAndExit', 'retry', '--retry', '{session_id}'],
+ target_module='GtsExoPlayerTestCases',
+ target_plan=None,
+ uri='gs://chromeos-arc-images/cts/bundle/android-gts-8-R3-P-7133222.zip',
+ use_jdk9=True,
+ timeout=86400)
+
+parallel_simple(run_TS, machines)
diff --git a/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.others b/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.others
new file mode 100644
index 0000000..6a9f57b
--- /dev/null
+++ b/server/site_tests/cheets_GTS/control.8.0_r3.GtsExoPlayerTestCases.others
@@ -0,0 +1,35 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file has been automatically generated. Do not edit!
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_GTS.8.0_r3.GtsExoPlayerTestCases.others'
+ATTRIBUTES = 'suite:arc-gts-qual'
+DEPENDENCIES = 'arc'
+JOB_RETRIES = 1
+TEST_TYPE = 'server'
+TIME = 'MEDIUM'
+MAX_RESULT_SIZE_KB = 307200
+DOC = 'Run module GtsExoPlayerTestCases.others of the Android Google Test Suite (GTS) in the ARC++ container.'
+
+def run_TS(machine):
+ host_list = [hosts.create_host(machine)]
+ job.run_test(
+ 'cheets_GTS',
+ hosts=host_list,
+ iterations=1,
+ max_retry=9,
+ tag='8.0_r3.GtsExoPlayerTestCases.others',
+ test_name='cheets_GTS.8.0_r3.GtsExoPlayerTestCases.others',
+ authkey='gs://chromeos-arc-images/cts/bundle/gts-arc.json',
+ run_template=['run', 'commandAndExit', 'gts', '--module', 'GtsExoPlayerTestCases', '--exclude-filter', 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCbc1', '--exclude-filter', 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCbcs', '--exclude-filter', 'GtsExoPlayerTestCases com.google.android.exoplayer.gts.DashTest#testWidevineH264AdaptiveWithSeekingCenc', '--ignore-business-logic-failure'],
+ retry_template=['run', 'commandAndExit', 'retry', '--retry', '{session_id}'],
+ target_module='GtsExoPlayerTestCases',
+ target_plan=None,
+ uri='gs://chromeos-arc-images/cts/bundle/android-gts-8-R3-P-7133222.zip',
+ use_jdk9=True,
+ timeout=86400)
+
+parallel_simple(run_TS, machines)
diff --git a/server/site_tests/cheets_GTS/control.8.0_r3.all.GtsExoPlayerTestCases_-_GtsExoPlayerTestCases b/server/site_tests/cheets_GTS/control.8.0_r3.all.GtsExoPlayerTestCases_-_GtsExoPlayerTestCases
new file mode 100644
index 0000000..b66cd2b
--- /dev/null
+++ b/server/site_tests/cheets_GTS/control.8.0_r3.all.GtsExoPlayerTestCases_-_GtsExoPlayerTestCases
@@ -0,0 +1,35 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file has been automatically generated. Do not edit!
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_GTS.8.0_r3.all.GtsExoPlayerTestCases_-_GtsExoPlayerTestCases'
+ATTRIBUTES = 'suite:arc-gts-qual'
+DEPENDENCIES = 'arc'
+JOB_RETRIES = 1
+TEST_TYPE = 'server'
+TIME = 'MEDIUM'
+MAX_RESULT_SIZE_KB = 307200
+DOC = 'Run module GtsExoPlayerTestCases of the Android Google Test Suite (GTS) in the ARC++ container.'
+
+def run_TS(machine):
+ host_list = [hosts.create_host(machine)]
+ job.run_test(
+ 'cheets_GTS',
+ hosts=host_list,
+ iterations=1,
+ max_retry=9,
+ tag='8.0_r3.all.GtsExoPlayerTestCases_-_GtsExoPlayerTestCases',
+ test_name='cheets_GTS.8.0_r3.all.GtsExoPlayerTestCases_-_GtsExoPlayerTestCases',
+ authkey='gs://chromeos-arc-images/cts/bundle/gts-arc.json',
+ run_template=['run', 'commandAndExit', 'gts', '--module', 'GtsExoPlayerTestCases', '--ignore-business-logic-failure'],
+ retry_template=['run', 'commandAndExit', 'retry', '--retry', '{session_id}'],
+ target_module='all.GtsExoPlayerTestCases_-_GtsExoPlayerTestCases',
+ target_plan=None,
+ uri='gs://chromeos-arc-images/cts/bundle/android-gts-8-R3-P-7133222.zip',
+ use_jdk9=True,
+ timeout=86400)
+
+parallel_simple(run_TS, machines)
diff --git a/server/site_tests/cheets_GTS/control.8.0_r3.all.GtsFeaturesTestCases_-_GtsLocationTestCases b/server/site_tests/cheets_GTS/control.8.0_r3.all.GtsFeaturesTestCases_-_GtsLocationTestCases
new file mode 100644
index 0000000..ea927c5
--- /dev/null
+++ b/server/site_tests/cheets_GTS/control.8.0_r3.all.GtsFeaturesTestCases_-_GtsLocationTestCases
@@ -0,0 +1,36 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file has been automatically generated. Do not edit!
+
+AUTHOR = 'ARC++ Team'
+NAME = 'cheets_GTS.8.0_r3.all.GtsFeaturesTestCases_-_GtsLocationTestCases'
+ATTRIBUTES = 'suite:arc-gts-qual'
+DEPENDENCIES = 'arc'
+JOB_RETRIES = 1
+TEST_TYPE = 'server'
+TIME = 'MEDIUM'
+MAX_RESULT_SIZE_KB = 307200
+DOC = 'Run module GtsFeaturesTestCases, GtsGmscoreHostTestCases, GtsGraphicsHostTestCases, GtsHomeHostTestCases, GtsIncidentConfirmationTestCases, GtsIncidentManagerTestCases, GtsIncrementalInstallProxyHostTestCases, GtsIncrementalInstallTestCases, GtsIncrementalInstallTestCases_BackgroundProcess, GtsIncrementalInstallTriggerApp, GtsIncrementalInstallTriggerApp_BackgroundProcess, GtsInstallPackagesWhitelistDeviceTestCases, GtsInstallerV2TestCases, GtsInstallerV2TestCases_BackgroundProcess, GtsInstantAppsHostTestCases, GtsJniUncompressTestCases, GtsLargeApkHostTestCases, GtsLensTestCases, GtsLinkerConfigTestCases, GtsLinkerConfigTestCases[secondary_user], GtsLocationHostTestCases, GtsLocationTestCases of the Android Google Test Suite (GTS) in the ARC++ container.'
+
+def run_TS(machine):
+ host_list = [hosts.create_host(machine)]
+ job.run_test(
+ 'cheets_GTS',
+ hosts=host_list,
+ iterations=1,
+ max_retry=9,
+ tag='8.0_r3.all.GtsFeaturesTestCases_-_GtsLocationTestCases',
+ test_name='cheets_GTS.8.0_r3.all.GtsFeaturesTestCases_-_GtsLocationTestCases',
+ authkey='gs://chromeos-arc-images/cts/bundle/gts-arc.json',
+ run_template=['run', 'commandAndExit', 'gts', '--include-filter', 'GtsFeaturesTestCases', '--include-filter', 'GtsGmscoreHostTestCases', '--include-filter', 'GtsGraphicsHostTestCases', '--include-filter', 'GtsHomeHostTestCases', '--include-filter', 'GtsIncidentConfirmationTestCases', '--include-filter', 'GtsIncidentManagerTestCases', '--include-filter', 'GtsIncrementalInstallProxyHostTestCases', '--include-filter', 'GtsIncrementalInstallTestCases', '--include-filter', 'GtsIncrementalInstallTestCases_BackgroundProcess', '--include-filter', 'GtsIncrementalInstallTriggerApp', '--include-filter', 'GtsIncrementalInstallTriggerApp_BackgroundProcess', '--include-filter', 'GtsInstallPackagesWhitelistDeviceTestCases', '--include-filter', 'GtsInstallerV2TestCases', '--include-filter', 'GtsInstallerV2TestCases_BackgroundProcess', '--include-filter', 'GtsInstantAppsHostTestCases', '--include-filter', 'GtsJniUncompressTestCases', '--include-filter', 'GtsLargeApkHostTestCases', '--include-filter', 'GtsLensTestCases', '--include-filter', 'GtsLinkerConfigTestCases', '--include-filter', 'GtsLinkerConfigTestCases[secondary_user]', '--include-filter', 'GtsLocationHostTestCases', '--include-filter', 'GtsLocationTestCases', '--ignore-business-logic-failure'],
+ retry_template=['run', 'commandAndExit', 'retry', '--retry', '{session_id}'],
+ target_module='all.GtsFeaturesTestCases_-_GtsLocationTestCases',
+ target_plan=None,
+ uri='gs://chromeos-arc-images/cts/bundle/android-gts-8-R3-P-7133222.zip',
+ prerequisites=['bluetooth'],
+ use_jdk9=True,
+ timeout=86400)
+
+parallel_simple(run_TS, machines)
