Add performance CUJ test suite for public benchmark tests.
Tauto wrappers are generated based on config JSON file.
BUG=b:182082083
TEST=test_that --board=${BOARD} ip:port suite:performance_cuj_<suite>
Change-Id: I6b7038676634590188278477e719b321577d8af5
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/3076537
Tested-by: Alex Bergman <abergman@google.com>
Reviewed-by: Derek Beckett <dbeckett@chromium.org>
Reviewed-by: Afshin Sadrieh <asadrieh@google.com>
Reviewed-by: C Shapiro <shapiroc@chromium.org>
Commit-Queue: Alex Bergman <abergman@google.com>
diff --git a/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_CRXPRT2_1 b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_CRXPRT2_1
new file mode 100644
index 0000000..fe08abf
--- /dev/null
+++ b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_CRXPRT2_1
@@ -0,0 +1,49 @@
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = 'abergman, chromeos-engprod-platform-syd'
+NAME = 'ui_CRXPRT2_1'
+ATTRIBUTES = 'suite:performance_cuj_benchmarks'
+TIME = 'long'
+TEST_CATEGORY = 'Stress'
+TEST_CLASS = 'Hardware'
+TEST_TYPE = 'Server'
+PRIORITY = 4996
+MAX_RESULT_SIZE_KB = 1024 * 1024
+JOB_RETRIES = 2
+REQUIRE_SSP = True
+DEPENDENCIES = ''
+
+DOC = '''
+Run the Tast-based MTBF performance CUJ test.
+
+Tast is an integration-testing framework analogous to the test-running portion
+of Autotest. See https://chromium.googlesource.com/chromiumos/platform/tast/ for
+more information.
+
+See http://go/tast-failures for information about investigating failures.
+'''
+
+from autotest_lib.client.common_lib import utils
+from autotest_lib.utils import labellib
+
+def report_host_info(host):
+ labels = labellib.LabelsMapping(host.host_info_store.get().labels)
+ labels['test_version'] = 1
+ labels['test'] = 'ui.CRXPRT2'
+ labels['test_iteration'] = '1'
+ utils.write_keyval(job.resultdir, labels)
+
+def run(machine):
+ host=hosts.create_host(machine)
+ report_host_info(host)
+ job.run_test('tast',
+ host=host,
+ test_exprs=['ui.CRXPRT2'],
+ ignore_test_failures=False,
+ max_run_sec=3600,
+ command_args=args,
+ vars_gs_path='config/perf_cuj/perf_cuj.config')
+
+parallel_simple(run, machines)
diff --git a/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_CRXPRT2_2 b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_CRXPRT2_2
new file mode 100644
index 0000000..7cb67a5
--- /dev/null
+++ b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_CRXPRT2_2
@@ -0,0 +1,49 @@
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = 'abergman, chromeos-engprod-platform-syd'
+NAME = 'ui_CRXPRT2_2'
+ATTRIBUTES = 'suite:performance_cuj_benchmarks'
+TIME = 'long'
+TEST_CATEGORY = 'Stress'
+TEST_CLASS = 'Hardware'
+TEST_TYPE = 'Server'
+PRIORITY = 4995
+MAX_RESULT_SIZE_KB = 1024 * 1024
+JOB_RETRIES = 2
+REQUIRE_SSP = True
+DEPENDENCIES = ''
+
+DOC = '''
+Run the Tast-based MTBF performance CUJ test.
+
+Tast is an integration-testing framework analogous to the test-running portion
+of Autotest. See https://chromium.googlesource.com/chromiumos/platform/tast/ for
+more information.
+
+See http://go/tast-failures for information about investigating failures.
+'''
+
+from autotest_lib.client.common_lib import utils
+from autotest_lib.utils import labellib
+
+def report_host_info(host):
+ labels = labellib.LabelsMapping(host.host_info_store.get().labels)
+ labels['test_version'] = 1
+ labels['test'] = 'ui.CRXPRT2'
+ labels['test_iteration'] = '2'
+ utils.write_keyval(job.resultdir, labels)
+
+def run(machine):
+ host=hosts.create_host(machine)
+ report_host_info(host)
+ job.run_test('tast',
+ host=host,
+ test_exprs=['ui.CRXPRT2'],
+ ignore_test_failures=False,
+ max_run_sec=3600,
+ command_args=args,
+ vars_gs_path='config/perf_cuj/perf_cuj.config')
+
+parallel_simple(run, machines)
diff --git a/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_GeekbenchPublicAndroidApp_1 b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_GeekbenchPublicAndroidApp_1
new file mode 100644
index 0000000..9fec298
--- /dev/null
+++ b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_GeekbenchPublicAndroidApp_1
@@ -0,0 +1,49 @@
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = 'abergman, chromeos-engprod-platform-syd'
+NAME = 'ui_GeekbenchPublicAndroidApp_1'
+ATTRIBUTES = 'suite:performance_cuj_benchmarks'
+TIME = 'long'
+TEST_CATEGORY = 'Stress'
+TEST_CLASS = 'Hardware'
+TEST_TYPE = 'Server'
+PRIORITY = 5000
+MAX_RESULT_SIZE_KB = 1024 * 1024
+JOB_RETRIES = 2
+REQUIRE_SSP = True
+DEPENDENCIES = ''
+
+DOC = '''
+Run the Tast-based MTBF performance CUJ test.
+
+Tast is an integration-testing framework analogous to the test-running portion
+of Autotest. See https://chromium.googlesource.com/chromiumos/platform/tast/ for
+more information.
+
+See http://go/tast-failures for information about investigating failures.
+'''
+
+from autotest_lib.client.common_lib import utils
+from autotest_lib.utils import labellib
+
+def report_host_info(host):
+ labels = labellib.LabelsMapping(host.host_info_store.get().labels)
+ labels['test_version'] = 1
+ labels['test'] = 'ui.GeekbenchPublicAndroidApp'
+ labels['test_iteration'] = '1'
+ utils.write_keyval(job.resultdir, labels)
+
+def run(machine):
+ host=hosts.create_host(machine)
+ report_host_info(host)
+ job.run_test('tast',
+ host=host,
+ test_exprs=['ui.GeekbenchPublicAndroidApp'],
+ ignore_test_failures=False,
+ max_run_sec=3600,
+ command_args=args,
+ vars_gs_path='config/perf_cuj/perf_cuj.config')
+
+parallel_simple(run, machines)
diff --git a/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_GeekbenchPublicAndroidApp_2 b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_GeekbenchPublicAndroidApp_2
new file mode 100644
index 0000000..ef9098e
--- /dev/null
+++ b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_GeekbenchPublicAndroidApp_2
@@ -0,0 +1,49 @@
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = 'abergman, chromeos-engprod-platform-syd'
+NAME = 'ui_GeekbenchPublicAndroidApp_2'
+ATTRIBUTES = 'suite:performance_cuj_benchmarks'
+TIME = 'long'
+TEST_CATEGORY = 'Stress'
+TEST_CLASS = 'Hardware'
+TEST_TYPE = 'Server'
+PRIORITY = 4999
+MAX_RESULT_SIZE_KB = 1024 * 1024
+JOB_RETRIES = 2
+REQUIRE_SSP = True
+DEPENDENCIES = ''
+
+DOC = '''
+Run the Tast-based MTBF performance CUJ test.
+
+Tast is an integration-testing framework analogous to the test-running portion
+of Autotest. See https://chromium.googlesource.com/chromiumos/platform/tast/ for
+more information.
+
+See http://go/tast-failures for information about investigating failures.
+'''
+
+from autotest_lib.client.common_lib import utils
+from autotest_lib.utils import labellib
+
+def report_host_info(host):
+ labels = labellib.LabelsMapping(host.host_info_store.get().labels)
+ labels['test_version'] = 1
+ labels['test'] = 'ui.GeekbenchPublicAndroidApp'
+ labels['test_iteration'] = '2'
+ utils.write_keyval(job.resultdir, labels)
+
+def run(machine):
+ host=hosts.create_host(machine)
+ report_host_info(host)
+ job.run_test('tast',
+ host=host,
+ test_exprs=['ui.GeekbenchPublicAndroidApp'],
+ ignore_test_failures=False,
+ max_run_sec=3600,
+ command_args=args,
+ vars_gs_path='config/perf_cuj/perf_cuj.config')
+
+parallel_simple(run, machines)
diff --git a/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_LMbench_1 b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_LMbench_1
new file mode 100644
index 0000000..66c0f41
--- /dev/null
+++ b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_LMbench_1
@@ -0,0 +1,49 @@
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = 'abergman, chromeos-engprod-platform-syd'
+NAME = 'ui_LMbench_1'
+ATTRIBUTES = 'suite:performance_cuj_benchmarks'
+TIME = 'long'
+TEST_CATEGORY = 'Stress'
+TEST_CLASS = 'Hardware'
+TEST_TYPE = 'Server'
+PRIORITY = 4998
+MAX_RESULT_SIZE_KB = 1024 * 1024
+JOB_RETRIES = 2
+REQUIRE_SSP = True
+DEPENDENCIES = ''
+
+DOC = '''
+Run the Tast-based MTBF performance CUJ test.
+
+Tast is an integration-testing framework analogous to the test-running portion
+of Autotest. See https://chromium.googlesource.com/chromiumos/platform/tast/ for
+more information.
+
+See http://go/tast-failures for information about investigating failures.
+'''
+
+from autotest_lib.client.common_lib import utils
+from autotest_lib.utils import labellib
+
+def report_host_info(host):
+ labels = labellib.LabelsMapping(host.host_info_store.get().labels)
+ labels['test_version'] = 1
+ labels['test'] = 'ui.LMbench'
+ labels['test_iteration'] = '1'
+ utils.write_keyval(job.resultdir, labels)
+
+def run(machine):
+ host=hosts.create_host(machine)
+ report_host_info(host)
+ job.run_test('tast',
+ host=host,
+ test_exprs=['ui.LMbench'],
+ ignore_test_failures=False,
+ max_run_sec=3600,
+ command_args=args,
+ vars_gs_path='config/perf_cuj/perf_cuj.config')
+
+parallel_simple(run, machines)
diff --git a/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_LMbench_2 b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_LMbench_2
new file mode 100644
index 0000000..ff772f8
--- /dev/null
+++ b/server/site_tests/platform_MTBF/control.performance_cuj_benchmarks_ui_LMbench_2
@@ -0,0 +1,49 @@
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = 'abergman, chromeos-engprod-platform-syd'
+NAME = 'ui_LMbench_2'
+ATTRIBUTES = 'suite:performance_cuj_benchmarks'
+TIME = 'long'
+TEST_CATEGORY = 'Stress'
+TEST_CLASS = 'Hardware'
+TEST_TYPE = 'Server'
+PRIORITY = 4997
+MAX_RESULT_SIZE_KB = 1024 * 1024
+JOB_RETRIES = 2
+REQUIRE_SSP = True
+DEPENDENCIES = ''
+
+DOC = '''
+Run the Tast-based MTBF performance CUJ test.
+
+Tast is an integration-testing framework analogous to the test-running portion
+of Autotest. See https://chromium.googlesource.com/chromiumos/platform/tast/ for
+more information.
+
+See http://go/tast-failures for information about investigating failures.
+'''
+
+from autotest_lib.client.common_lib import utils
+from autotest_lib.utils import labellib
+
+def report_host_info(host):
+ labels = labellib.LabelsMapping(host.host_info_store.get().labels)
+ labels['test_version'] = 1
+ labels['test'] = 'ui.LMbench'
+ labels['test_iteration'] = '2'
+ utils.write_keyval(job.resultdir, labels)
+
+def run(machine):
+ host=hosts.create_host(machine)
+ report_host_info(host)
+ job.run_test('tast',
+ host=host,
+ test_exprs=['ui.LMbench'],
+ ignore_test_failures=False,
+ max_run_sec=3600,
+ command_args=args,
+ vars_gs_path='config/perf_cuj/perf_cuj.config')
+
+parallel_simple(run, machines)
diff --git a/server/site_tests/platform_MTBF/spera_config.json b/server/site_tests/platform_MTBF/spera_config.json
index c1abf0f..9f64930 100644
--- a/server/site_tests/platform_MTBF/spera_config.json
+++ b/server/site_tests/platform_MTBF/spera_config.json
@@ -123,6 +123,24 @@
"test_expr": "ui.ExtendedDisplayCUJ.premium_meet_large",
"attributes": ["plus"],
"deps": ["external_display", "plus"]
+ },
+ {
+ "name": "geekbench-public-android",
+ "tool": "tast",
+ "test_expr": "ui.GeekbenchPublicAndroidApp",
+ "attributes": ["benchmark", "basic"]
+ },
+ {
+ "name": "lmbench",
+ "tool": "tast",
+ "test_expr": "ui.LMbench",
+ "attributes": ["benchmark", "basic"]
+ },
+ {
+ "name": "crxprt2",
+ "tool": "tast",
+ "test_expr": "ui.CRXPRT2",
+ "attributes": ["benchmark", "basic"]
}
],
"suites": [
@@ -190,6 +208,15 @@
{"test": "everyday-plus-ytmusic", "repeats": 10},
{"test": "extended-display-meet", "repeats": 10}
]
+ },
+ {
+ "name": "performance_cuj_benchmarks",
+ "args_file": "perf_cuj.config",
+ "tests": [
+ {"test": "geekbench-public-android", "repeats": 2},
+ {"test": "lmbench", "repeats": 2},
+ {"test": "crxprt2", "repeats": 2}
+ ]
}
]
}
diff --git a/site_utils/attribute_allowlist.txt b/site_utils/attribute_allowlist.txt
index 826c0f8..7c47de5 100644
--- a/site_utils/attribute_allowlist.txt
+++ b/site_utils/attribute_allowlist.txt
@@ -252,6 +252,7 @@
suite:partners
suite:perfalerts
suite:performance_cuj
+suite:performance_cuj_benchmarks
suite:performance_cuj_experimental
suite:performance_cuj_quick
suite:platform_internal_display
diff --git a/test_suites/control.performance_cuj_benchmarks b/test_suites/control.performance_cuj_benchmarks
new file mode 100644
index 0000000..0ece49a
--- /dev/null
+++ b/test_suites/control.performance_cuj_benchmarks
@@ -0,0 +1,33 @@
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = "abergman, chromeos-engprod-platform-syd"
+NAME = "performance_cuj_benchmarks"
+PURPOSE = "Tast wrapper suite for running public benchmarks performance CUJ Tast tests on Moblab."
+TIME = "LONG"
+TEST_CATEGORY = "General"
+TEST_CLASS = "suite"
+TEST_TYPE = "Server"
+
+DOC = """
+This suite executes a set of public benchmark performance CUJ (critical user
+journey) test cases. Supposed to be run by partners (OEMs, ODMs) using Moblab as
+a quick check that all included tests are running successfully.
+
+@param build: The name of the image to test.
+ Ex: x86-mario-release/R17-1412.33.0-a1-b29
+@param board: The board to test on. Ex: x86-mario
+@param pool: The pool of machines to utilize for scheduling. If pool=None
+ board is used.
+@param SKIP_IMAGE: (optional) If present and True, don't re-image devices.
+"""
+
+import common
+
+from autotest_lib.server.cros.dynamic_suite import dynamic_suite
+
+args_dict['name'] = NAME
+args_dict['job'] = job
+
+dynamic_suite.reimage_and_run(**args_dict)
