Fixup performance suite test runs.
Add new test suite performance_optional, add less important performance tests
to that suite and schedule it for nightly run.
Add/Remove the tests from the performance suite so it matches the dashboard
exactly. Scedule these more important tests for every build, this is an
experiment to see if we have enough DUT resource available for such
frequent runs.
BUG=None
TEST=None
Change-Id: I8e9b70ee0ff58deabb5bd61bf994c0b782d9e720
Previous-Reviewed-on: https://chromium-review.googlesource.com/257010
(cherry picked from commit e83d8f1c8d4784aabb630224935c6b8b70cb9fc8)
Reviewed-on: https://chromium-review.googlesource.com/257571
Tested-by: Keith Haddow <haddowk@chromium.org>
Reviewed-by: Keith Haddow <haddowk@chromium.org>
Reviewed-by: Dan Shi <dshi@chromium.org>
Commit-Queue: Keith Haddow <haddowk@chromium.org>
diff --git a/client/site_tests/graphics_WebGLManyPlanetsDeep/control b/client/site_tests/graphics_WebGLManyPlanetsDeep/control
index 7e175cd..0edaa5d 100644
--- a/client/site_tests/graphics_WebGLManyPlanetsDeep/control
+++ b/client/site_tests/graphics_WebGLManyPlanetsDeep/control
@@ -6,7 +6,7 @@
NAME = "graphics_WebGLManyPlanetsDeep"
PURPOSE = "Execute the WebGL many planets deep test suite."
CRITERIA = "All suite tests must not crash/hang."
-SUITE = 'performance, graphics_per-day, graphics, graphics_browser'
+SUITE = 'graphics_per-day, graphics, graphics_browser'
TIME = "medium"
TEST_CATEGORY = "Performance"
TEST_CLASS = "graphics"
diff --git a/client/site_tests/platform_BootPerf/control b/client/site_tests/platform_BootPerf/control
index c035078..8f8669a 100644
--- a/client/site_tests/platform_BootPerf/control
+++ b/client/site_tests/platform_BootPerf/control
@@ -9,7 +9,7 @@
The test will fail if startup times or shut down times are exceeded. These
values are set in the suite control file.
"""
-SUITE = "performance"
+SUITE = "performance_optional"
TIME = "SHORT"
TEST_CATEGORY = "Benchmark"
TEST_CLASS = "platform"
diff --git a/client/site_tests/platform_Pkcs11LoadPerf/control b/client/site_tests/platform_Pkcs11LoadPerf/control
index 4fcdcba..3ea0a28 100644
--- a/client/site_tests/platform_Pkcs11LoadPerf/control
+++ b/client/site_tests/platform_Pkcs11LoadPerf/control
@@ -4,7 +4,6 @@
AUTHOR = "dkrahn"
NAME = "platform_Pkcs11LoadPerf"
-SUITE = "performance"
TIME = "SHORT"
TEST_CATEGORY = "Performance"
TEST_CLASS = "platform"
diff --git a/client/site_tests/video_PlaybackPerf/control.h264_all b/client/site_tests/video_PlaybackPerf/control.h264_all
index 3c3ab53..c2988fd 100644
--- a/client/site_tests/video_PlaybackPerf/control.h264_all
+++ b/client/site_tests/video_PlaybackPerf/control.h264_all
@@ -8,7 +8,7 @@
CRITERIA = """
The test outputs the cpu usage and dropped frame count of H264 video playback.
"""
-SUITE = "video"
+SUITE = "video, performance"
TIME = "MEDIUM"
TEST_CATEGORY = "Performance"
TEST_CLASS = "video"
diff --git a/client/site_tests/video_PlaybackPerf/control.vp9_all b/client/site_tests/video_PlaybackPerf/control.vp9_all
index af32d89..81b82db 100644
--- a/client/site_tests/video_PlaybackPerf/control.vp9_all
+++ b/client/site_tests/video_PlaybackPerf/control.vp9_all
@@ -8,7 +8,7 @@
CRITERIA = """
The test outputs the cpu usage and dropped frame count of VP9 video playback.
"""
-SUITE = "video"
+SUITE = "video, performance"
TIME = "MEDIUM"
TEST_CATEGORY = "Performance"
TEST_CLASS = "video"
diff --git a/client/site_tests/video_VDAPerf/control.h264 b/client/site_tests/video_VDAPerf/control.h264
index 20e749c..71d20d8 100644
--- a/client/site_tests/video_VDAPerf/control.h264
+++ b/client/site_tests/video_VDAPerf/control.h264
@@ -8,7 +8,7 @@
CRITERIA = """
The test fails if the video_decode_accelerator_unittest fails or crashes.
"""
-SUITE = "performance"
+SUITE = "performance_optional"
TIME = "SHORT"
TEST_CATEGORY = "Performance"
TEST_CLASS = "video"
diff --git a/client/site_tests/video_VDAPerf/control.vp8 b/client/site_tests/video_VDAPerf/control.vp8
index ef2c9ae..3218a33 100644
--- a/client/site_tests/video_VDAPerf/control.vp8
+++ b/client/site_tests/video_VDAPerf/control.vp8
@@ -8,7 +8,7 @@
CRITERIA = """
The test fails if the video_decode_accelerator_unittest fails or crashes.
"""
-SUITE = "performance"
+SUITE = "performance_optional"
TIME = "SHORT"
TEST_CATEGORY = "Performance"
TEST_CLASS = "video"
diff --git a/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoreattr b/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoreattr
index 894ffa0..645195d 100644
--- a/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoreattr
+++ b/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoreattr
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.dromaeo.domcoreattr'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='dromaeo.domcoreattr',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoremodify b/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoremodify
index 9b8f96f..48cae36 100644
--- a/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoremodify
+++ b/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoremodify
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.dromaeo.domcoremodify'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='dromaeo.domcoremodify',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcorequery b/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcorequery
index cfc2c1b..1823472 100644
--- a/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcorequery
+++ b/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcorequery
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.dromaeo.domcorequery'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='dromaeo.domcorequery',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoretraverse b/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoretraverse
index 447605b..a6dbfb2 100644
--- a/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoretraverse
+++ b/server/site_tests/telemetry_Benchmarks/control.dromaeo.domcoretraverse
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.dromaeo.domcoretraverse'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='dromaeo.domcoretraverse',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.media.chromeOS4kOnly.tough_video_cases b/server/site_tests/telemetry_Benchmarks/control.media.chromeOS4kOnly.tough_video_cases
index 385246d..54519b0 100644
--- a/server/site_tests/telemetry_Benchmarks/control.media.chromeOS4kOnly.tough_video_cases
+++ b/server/site_tests/telemetry_Benchmarks/control.media.chromeOS4kOnly.tough_video_cases
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.media.chromeOS4kOnly.tough_video_cases'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='media.chromeOS4kOnly.tough_video_cases',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.memory.top_7_stress b/server/site_tests/telemetry_Benchmarks/control.memory.top_7_stress
index c41a2c2..d17c02f 100644
--- a/server/site_tests/telemetry_Benchmarks/control.memory.top_7_stress
+++ b/server/site_tests/telemetry_Benchmarks/control.memory.top_7_stress
@@ -8,7 +8,6 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.memory.top_7_stress'
-SUITE = 'performance'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +28,4 @@
tag='memory.top_7_stress',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.robohornet_pro b/server/site_tests/telemetry_Benchmarks/control.robohornet_pro
index 55edd57..de146d1 100644
--- a/server/site_tests/telemetry_Benchmarks/control.robohornet_pro
+++ b/server/site_tests/telemetry_Benchmarks/control.robohornet_pro
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.robohornet_pro'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='robohornet_pro',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.session_restore.cold.typical_25 b/server/site_tests/telemetry_Benchmarks/control.session_restore.cold.typical_25
index 959b173..cedcfd6 100644
--- a/server/site_tests/telemetry_Benchmarks/control.session_restore.cold.typical_25
+++ b/server/site_tests/telemetry_Benchmarks/control.session_restore.cold.typical_25
@@ -8,7 +8,6 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.session_restore.cold.typical_25'
-SUITE = 'performance'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +28,4 @@
tag='session_restore.cold.typical_25',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_animation_cases b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_animation_cases
index a7bfc92..f4d4bdc 100644
--- a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_animation_cases
+++ b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_animation_cases
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.smoothness.tough_animation_cases'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='smoothness.tough_animation_cases',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_canvas_cases b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_canvas_cases
index b9426b0..aa94a12 100644
--- a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_canvas_cases
+++ b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_canvas_cases
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.smoothness.tough_canvas_cases'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='smoothness.tough_canvas_cases',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_filters_cases b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_filters_cases
index d30099c..f44e8a3 100644
--- a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_filters_cases
+++ b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_filters_cases
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.smoothness.tough_filters_cases'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='smoothness.tough_filters_cases',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_pinch_zoom_cases b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_pinch_zoom_cases
index 2f8a7bb..c862436 100644
--- a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_pinch_zoom_cases
+++ b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_pinch_zoom_cases
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.smoothness.tough_pinch_zoom_cases'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='smoothness.tough_pinch_zoom_cases',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_scrolling_cases b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_scrolling_cases
index 2db208a..d2650c5 100644
--- a/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_scrolling_cases
+++ b/server/site_tests/telemetry_Benchmarks/control.smoothness.tough_scrolling_cases
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.smoothness.tough_scrolling_cases'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='smoothness.tough_scrolling_cases',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.sunspider b/server/site_tests/telemetry_Benchmarks/control.sunspider
index 15bc85a..403eba5 100644
--- a/server/site_tests/telemetry_Benchmarks/control.sunspider
+++ b/server/site_tests/telemetry_Benchmarks/control.sunspider
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.sunspider'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='sunspider',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/server/site_tests/telemetry_Benchmarks/control.webrtc.webrtc_cases b/server/site_tests/telemetry_Benchmarks/control.webrtc.webrtc_cases
index cb50999..2f9883d 100644
--- a/server/site_tests/telemetry_Benchmarks/control.webrtc.webrtc_cases
+++ b/server/site_tests/telemetry_Benchmarks/control.webrtc.webrtc_cases
@@ -8,7 +8,7 @@
AUTHOR = 'sbasi, achuith, rohitbm'
NAME = 'telemetry_Benchmarks.webrtc.webrtc_cases'
-SUITE = 'performance'
+SUITE = 'performance_optional'
TIME = 'LONG'
TEST_CATEGORY = 'Benchmark'
TEST_CLASS = 'performance'
@@ -29,4 +29,4 @@
tag='webrtc.webrtc_cases',
args=utils.args_to_dict(args))
-parallel_simple(run_benchmark, machines)
\ No newline at end of file
+parallel_simple(run_benchmark, machines)
diff --git a/suite_scheduler.ini b/suite_scheduler.ini
index c10bd9b..e8b77b9 100644
--- a/suite_scheduler.ini
+++ b/suite_scheduler.ini
@@ -716,8 +716,15 @@
pool: suites
[Performance]
-run_on: nightly
+run_on: new_build
suite: performance
branch_specs: >=tot-2
pool: performance
file_bugs: True
+
+[PerformanceOptional]
+run_on: nightly
+suite: performance_optional
+branch_specs: >=tot-2
+pool: performance
+file_bugs: True
diff --git a/test_suites/control.performance_optional b/test_suites/control.performance_optional
new file mode 100644
index 0000000..db07081
--- /dev/null
+++ b/test_suites/control.performance_optional
@@ -0,0 +1,48 @@
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = "rohitbm@chromium.org, chromeos-performance@google.com"
+NAME = "performance_optional"
+PURPOSE = "Performance tests"
+CRITERIA = "All tests with SUITE=performance_optional must pass."
+
+TIME = "LONG"
+TEST_CATEGORY = "General"
+TEST_CLASS = "suite"
+TEST_TYPE = "Server"
+
+DOC = """
+ChromeOS performance tests including telemetry tests.
+
+@param build: The name of the image to test.
+ Ex: x86-mario-release/R17-1412.33.0-a1-b29
+@param board: The board to test on. Ex: x86-mario
+@param pool: The pool of machines to utilize for scheduling. If pool=None
+ board is used.
+@param check_hosts: require appropriate live hosts to exist in the lab.
+@param SKIP_IMAGE: (optional) If present and True, don't re-image devices.
+"""
+
+import common
+from autotest_lib.server.cros import provision
+from autotest_lib.server.cros.dynamic_suite import dynamic_suite
+
+
+# Values specified in this bug template will override default values when
+# filing bugs on tests that are a part of this suite. If left unspecified
+# the bug filer will fallback to its defaults.
+_BUG_TEMPLATE = {
+ 'labels': ['Type-Bug', 'Cros-Perf-Test'],
+ 'owner': 'rohitbm@chromium.org',
+ 'ccs': ['lafeenstra@chromium.org']
+}
+
+dynamic_suite.reimage_and_run(
+ build=build, board=board, name='performance_optional', job=job, pool=pool,
+ check_hosts=check_hosts, add_experimental=True, num=num,
+ file_bugs=False, priority=priority, timeout_mins=480,
+ bug_template=_BUG_TEMPLATE, devserver_url=devserver_url,
+ version_prefix=provision.CROS_VERSION_PREFIX,
+ wait_for_results=wait_for_results, job_retry=job_retry,
+ max_retries=max_retries)
