cbuildbot: Redirect HWTest to test_platform recipe
`skylab create-suite -bb ..` calls the test_platform recipe which in
turn will decide whether to run the tests on Skylab or Autotest.
This CL is to be ported onto all branches.
The ported version also includes the following bug fixes:
- crrev.com/c/1772693
- crrev.com/c/1773943
- crrev.com/c/1775096
BUG=chromium:992233
TEST=./run_tests cbuildbot/commands_unittest
./run_tests cbuildbot/stages/release_stages_unittest
Change-Id: I2496b66a45755d887b186226edc1c588a2e15f3d
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/1749942
Commit-Queue: Alex Zamorzaev <zamorzaev@chromium.org>
Tested-by: Alex Zamorzaev <zamorzaev@chromium.org>
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
Reviewed-by: Jason Clinton <jclinton@chromium.org>
(cherry picked from commit 5957f0c57f957df09971f38a78d5164708b68a04)
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/1772351
diff --git a/cbuildbot/builders/simple_builders.py b/cbuildbot/builders/simple_builders.py
index 98bc7ae..59d9d48 100644
--- a/cbuildbot/builders/simple_builders.py
+++ b/cbuildbot/builders/simple_builders.py
@@ -363,7 +363,7 @@
builder_run=builder_run,
suffix='[afdo_generate_min]')
for suite in builder_run.config.hw_tests:
- self._RunStage(test_stages.HWTestStage, board, suite,
+ self._RunStage(test_stages.SkylabHWTestStage, board, suite,
builder_run=builder_run)
self._RunStage(afdo_stages.AFDODataGenerateStage, board,
builder_run=builder_run)
diff --git a/cbuildbot/commands.py b/cbuildbot/commands.py
index 3871a65..a1185b2 100644
--- a/cbuildbot/commands.py
+++ b/cbuildbot/commands.py
@@ -1458,6 +1458,95 @@
return HWTestSuiteResult(to_raise, None)
+def RunSkylabHWTestPlan(test_plan=None,
+ build=None,
+ legacy_suite=None,
+ pool=None,
+ board=None,
+ model=None,
+ timeout_mins=None,
+ tags=None,
+ keyvals=None):
+ """Run a skylab test in the Autotest lab using skylab tool.
+
+ Args:
+ test_plan: A JSONpb string containing a TestPlan object.
+ build: A string full image name.
+ legacy_suite: A string suite name, if non-empty it overrides the test plan
+ on the autotest backend.
+ pool: A string pool to run the test on.
+ board: A string board to run the test on.
+ model: A string model to run the test on.
+ timeout_mins: An integer to indicate the test's timeout.
+ tags: A list of strings to tag the task in swarming.
+ keyvals: A list of strings to be passed to the test as job_keyvals.
+ """
+ if not test_plan:
+ raise ValueError('Need to specify test plan.')
+ if not build:
+ raise ValueError('Need to specify build.')
+ if not (board or model):
+ raise ValueError('Need to specify either board or model.')
+
+ args = ['-image', build]
+
+ if legacy_suite:
+ args += ['-legacy-suite', legacy_suite]
+
+ if pool:
+ args += ['-pool', pool]
+
+ if board:
+ args += ['-board', board]
+
+ if model:
+ args += ['-model', model]
+
+ if timeout_mins:
+ args += ['-timeout-mins', str(timeout_mins)]
+
+ if tags:
+ for tag in tags:
+ args += ['-tag', tag]
+
+ if keyvals is not None:
+ for k, v in keyvals.items():
+ args += ['-keyval', k+':'+v]
+
+ args += ['-service-account-json', constants.CHROMEOS_SERVICE_ACCOUNT]
+
+ args += ['-plan-file', '/dev/stdin']
+
+ skylab_path = _InstallSkylabTool()
+
+ try:
+ result = cros_build_lib.RunCommand(
+ [skylab_path, 'create-testplan'] + args,
+ redirect_stdout=True,
+ input=test_plan)
+ return HWTestSuiteResult(None, None)
+ except cros_build_lib.RunCommandError as e:
+ result = e.result
+ to_raise = failures_lib.TestFailure(
+ '** HWTest failed (code %d) **' % result.returncode)
+ return HWTestSuiteResult(to_raise, None)
+ finally:
+ # This is required to output buildbot annotations, e.g. 'STEP_LINKS'.
+ # output = json.loads(result.output)
+ output = {}
+ # The format of output is:
+ # {'task_name':'cros_test_platform',
+ # 'task_id': 'XX',
+ # 'task_url': 'YY'}
+ sys.stdout.write('%s \n' % output)
+ sys.stdout.write('######## Output for buildbot annotations ######## \n')
+ sys.stdout.write('%s \n' % str(
+ buildbot_annotations.StepLink('Test run',
+ output.get('task_url', ''))))
+ sys.stdout.write('######## END Output for buildbot annotations ######## \n')
+ sys.stdout.flush()
+
+
# pylint: disable=docstring-missing-args
def _GetRunSuiteArgs(build,
suite,
diff --git a/cbuildbot/commands_unittest.py b/cbuildbot/commands_unittest.py
index cc61ff1..fdec0f7 100644
--- a/cbuildbot/commands_unittest.py
+++ b/cbuildbot/commands_unittest.py
@@ -332,6 +332,49 @@
self.assertTrue(isinstance(error, failures_lib.TestFailure))
self.assertTrue('Suite failed' in error.message)
+ def testCreateTest(self):
+ """Test that function call args are mapped correctly to commandline args."""
+ test_plan = '{}'
+ build = 'foo-bar/R1234'
+ board = 'foo-board'
+ model = 'foo-model'
+ pool = 'foo-pool'
+ suite = 'foo-suite'
+ # An OrderedDict is used to make the keyval order on the command line
+ # deterministic for testing purposes.
+ keyvals = {'key': 'value'}
+ timeout_mins = 10
+
+ task_id = 'foo-task_id'
+
+ create_cmd = [
+ self._SKYLAB_TOOL, 'create-testplan',
+ '-image', build,
+ '-legacy-suite', suite,
+ '-pool', pool,
+ '-board', board,
+ '-model', model,
+ '-timeout-mins', str(timeout_mins),
+ '-keyval', 'key:value',
+ '-service-account-json', constants.CHROMEOS_SERVICE_ACCOUNT,
+ '-plan-file', '/dev/stdin'
+ ]
+
+ self.rc.AddCmdResult(
+ create_cmd, output=self._fakeCreateJson(task_id, 'foo://foo'))
+
+ result = commands.RunSkylabHWTestPlan(
+ test_plan=test_plan, build=build, pool=pool, board=board, model=model,
+ timeout_mins=timeout_mins, keyvals=keyvals, legacy_suite=suite)
+ self.assertTrue(isinstance(result, commands.HWTestSuiteResult))
+ self.assertEqual(result.to_raise, None)
+ self.assertEqual(result.json_dump_result, None)
+
+ self.rc.assertCommandCalled(
+ create_cmd, redirect_stdout=True, input=test_plan)
+ self.assertEqual(self.rc.call_count, 1)
+
+
class HWLabCommandsTest(cros_test_lib.RunCommandTestCase,
cros_test_lib.OutputTestCase,
diff --git a/cbuildbot/stages/test_stages.py b/cbuildbot/stages/test_stages.py
index 7ac34c7..0bfe071 100644
--- a/cbuildbot/stages/test_stages.py
+++ b/cbuildbot/stages/test_stages.py
@@ -730,18 +730,9 @@
if model.test_suites is None or suite_config.suite in model.test_suites:
stage_class = None
if suite_config.async:
- stage_class = ASyncHWTestStage
+ stage_class = ASyncSkylabHWTestStage
else:
- stage_class = HWTestStage
-
- hwtest_env = config_lib.GetHWTestEnv(builder_run.config,
- model_config=model,
- suite_config=suite_config)
- if hwtest_env == constants.ENV_SKYLAB:
- if suite_config.async:
- stage_class = ASyncSkylabHWTestStage
- else:
- stage_class = SkylabHWTestStage
+ stage_class = SkylabHWTestStage
result = stage_class(
builder_run,
diff --git a/lib/paygen/paygen_build_lib.py b/lib/paygen/paygen_build_lib.py
index 91e0cef9..8279688 100644
--- a/lib/paygen/paygen_build_lib.py
+++ b/lib/paygen/paygen_build_lib.py
@@ -21,6 +21,8 @@
import sys
import urlparse
+from chromite.api.gen.chromite.api import test_metadata_pb2
+from chromite.api.gen.test_platform import request_pb2
from chromite.cbuildbot import commands
from chromite.lib import constants
from chromite.lib import config_lib
@@ -35,6 +37,8 @@
from chromite.lib.paygen import paygen_payload_lib
from chromite.lib.paygen import utils
+from google.protobuf import json_format
+
# For crostools access.
sys.path.insert(0, constants.SOURCE_ROOT)
@@ -1148,6 +1152,7 @@
raise BoardNotConfigured(board)
+# pylint: disable=unused-argument
def ScheduleAutotestTests(suite_name, board, model, build, skip_duts_check,
debug, payload_test_configs, test_env,
job_keyvals=None):
@@ -1166,52 +1171,30 @@
value could be constants.ENV_SKYLAB, constants.ENV_AUTOTEST.
job_keyvals: A dict of job keyvals to be injected to suite control file.
"""
+ test_plan = _TestPlan(
+ payload_test_configs=payload_test_configs,
+ suite_name=suite_name,
+ build=build)
+
# Double timeout for crbug.com/930256. Will change back once paygen
# suites been migrated to skylab.
timeout_mins = 2 * config_lib.HWTestConfig.SHARED_HW_TEST_TIMEOUT / 60
- if test_env == constants.ENV_SKYLAB:
- tags = ['build:%s' % build,
- 'suite:%s' % suite_name,
- 'user:PaygenTestStage']
- for payload_test in payload_test_configs:
- test_name = test_control.get_test_name()
- # TKO parser requires that label format can be parsed by
- # site_utils.parse_job_name to get build, build_version, board and suite.
- # A parsable label format for autoupdate_EndtoEnd test should be:
- # reef-release/R74-XX.0.0/paygen_au_canary/autoupdate_E2E_***_XX.0.0
- shown_test_name = '%s_%s' % (test_name, payload_test.unique_name_suffix())
- tko_label = '%s/%s/%s' % (build, suite_name, shown_test_name)
- keyvals = ['build:%s' % build,
- 'suite:%s' % suite_name,
- 'label:%s' % tko_label]
- test_args = payload_test.get_cmdline_args()
- cmd_result = commands.RunSkylabHWTest(
- build=build,
- pool='bvt',
- test_name=test_control.get_test_name(),
- shown_test_name=shown_test_name,
- board=board,
- model=model,
- timeout_mins=timeout_mins,
- tags=tags,
- keyvals=keyvals,
- test_args=test_args)
- else:
- cmd_result = commands.RunHWTestSuite(
- board=board,
- model=model,
- build=build,
- suite=suite_name,
- file_bugs=True,
- pool='bvt',
- priority=constants.HWTEST_BUILD_PRIORITY,
- retry=True,
- wait_for_results=False,
- timeout_mins=timeout_mins,
- suite_min_duts=2,
- debug=debug,
- skip_duts_check=skip_duts_check,
- job_keyvals=job_keyvals)
+ tags = ['build:%s' % build,
+ 'suite:%s' % suite_name,
+ 'user:PaygenTestStage']
+
+ keyvals = {'build': build, 'suite': suite_name}
+
+ cmd_result = commands.RunSkylabHWTestPlan(
+ test_plan=test_plan,
+ build=build,
+ legacy_suite=suite_name,
+ pool='DUT_POOL_BVT',
+ board=board,
+ model=model,
+ timeout_mins=timeout_mins,
+ tags=tags,
+ keyvals=keyvals)
if cmd_result.to_raise:
if isinstance(cmd_result.to_raise, failures_lib.TestWarning):
@@ -1219,3 +1202,50 @@
cmd_result.to_raise)
else:
raise cmd_result.to_raise
+
+
+def _TestPlan(payload_test_configs, suite_name=None, build=None):
+ """Construct a TestPlan proto for the given payload tests.
+
+ Args:
+ payload_test_configs: A list of test_params.TestConfig objects.
+ suite_name: The name of the test suite.
+ build: A string representing the name of the archive build.
+
+ Returns:
+ A JSON-encoded string containing a TestPlan proto.
+ """
+ autotest_invocations = []
+ test_name = test_control.get_test_name()
+
+ for payload_test in payload_test_configs:
+ # TKO parser requires that label format can be parsed by
+ # site_utils.parse_job_name to get build, build_version, board and suite.
+ # A parsable label format for autoupdate_EndtoEnd test should be:
+ # reef-release/R74-XX.0.0/paygen_au_canary/autoupdate_E2E_***_XX.0.0
+ shown_test_name = '%s_%s' % (test_name, payload_test.unique_name_suffix())
+ tko_label = '%s/%s/%s' % (build, suite_name, shown_test_name)
+ test_args = payload_test.get_cmdline_args()
+
+ autotest_invocations.append(
+ request_pb2.Request.Enumeration.AutotestInvocation(
+ test=test_metadata_pb2.AutotestTest(
+ name=test_name,
+ allow_retries=True,
+ # Matching autoupdate_EndToEndTest control file.
+ max_retries=1,
+ execution_environment=(
+ test_metadata_pb2.AutotestTest.EXECUTION_ENVIRONMENT_SERVER)
+ ),
+ test_args=test_args,
+ display_name=tko_label,
+ )
+ )
+
+ test_plan = request_pb2.Request.TestPlan(
+ enumeration=request_pb2.Request.Enumeration(
+ autotest_invocations=autotest_invocations
+ )
+ )
+
+ return json_format.MessageToJson(test_plan)
diff --git a/lib/paygen/paygen_build_lib_unittest.py b/lib/paygen/paygen_build_lib_unittest.py
index 30cb77e..f340c45 100644
--- a/lib/paygen/paygen_build_lib_unittest.py
+++ b/lib/paygen/paygen_build_lib_unittest.py
@@ -11,8 +11,11 @@
import os
import mock
import tarfile
+import sys
+from chromite.cbuildbot import commands
from chromite.lib import config_lib_unittest
+from chromite.lib import constants
from chromite.lib import cros_test_lib
from chromite.lib import gs
from chromite.lib import parallel
@@ -22,6 +25,11 @@
from chromite.lib.paygen import paygen_build_lib
from chromite.lib.paygen import paygen_payload_lib
+AUTOTEST_DIR = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party',
+ 'autotest', 'files')
+sys.path.insert(0, AUTOTEST_DIR)
+# pylint: disable=import-error,wrong-import-position
+from site_utils.autoupdate.lib import test_params
# We access a lot of protected members during testing.
# pylint: disable=protected-access
@@ -1238,3 +1246,89 @@
REQUIRE_SSP = False
DOC ="""))
+
+
+class HWTest(cros_test_lib.MockTestCase):
+ """Test HW test invocation."""
+
+ def testScheduleAutotestTests(self):
+ run_test_mock = self.PatchObject(
+ commands,
+ 'RunSkylabHWTestPlan',
+ return_value=commands.HWTestSuiteResult(None, None))
+
+ paygen_build_lib.ScheduleAutotestTests(
+ suite_name='dummy-suite',
+ board='dummy-board',
+ model='dummy-model',
+ build='dummy-build',
+ skip_duts_check=None, # ignored
+ debug=None, # ignored
+ payload_test_configs=[],
+ test_env=None, # ignored
+ job_keyvals=None) #ignored
+
+ run_test_mock.assert_called_once_with(
+ test_plan=paygen_build_lib._TestPlan(
+ payload_test_configs=[],
+ suite_name='dummy-suite',
+ build='dummy-build'),
+ build='dummy-build',
+ pool='DUT_POOL_BVT',
+ legacy_suite='dummy-suite',
+ board='dummy-board',
+ model='dummy-model',
+ timeout_mins=2*3*60,
+ tags=['build:dummy-build',
+ 'suite:dummy-suite',
+ 'user:PaygenTestStage'],
+ keyvals={'build': 'dummy-build',
+ 'suite': 'dummy-suite'})
+
+ def testTestPlan(self):
+ payload_test_configs = [
+ test_params.TestConfig(
+ board='dummy-board',
+ name='dummy-test',
+ is_delta_update=True,
+ source_release='source-release',
+ target_release='target-release',
+ source_payload_uri='source-uri',
+ target_payload_uri='target-uri',
+ suite_name='dummy-suite')]
+
+ test_plan_string = paygen_build_lib._TestPlan(
+ payload_test_configs=payload_test_configs,
+ suite_name='dummy-suite',
+ build='dummy-build')
+ test_plan_dict = json.loads(test_plan_string)
+ test_args_string = test_plan_dict[
+ 'enumeration']['autotestInvocations'][0].pop('testArgs')
+ # There's no guarantee on the order.
+ test_args_set = set(test_args_string.split(' '))
+
+ expected_test_plan = {
+ 'enumeration': {
+ 'autotestInvocations': [{
+ 'test': {
+ 'name': 'autoupdate_EndToEndTest',
+ 'allowRetries': True,
+ 'maxRetries': 1,
+ 'executionEnvironment': 'EXECUTION_ENVIRONMENT_SERVER'
+ },
+ 'displayName': 'dummy-build/dummy-suite/' +
+ 'autoupdate_EndToEndTest_' +
+ 'dummy-test_delta_source-release'
+ }]
+ }
+ }
+ expected_test_args = set(['name=dummy-test',
+ 'update_type=delta',
+ 'source_release=source-release',
+ 'target_release=target-release',
+ 'target_payload_uri=target-uri',
+ 'SUITE=dummy-suite',
+ 'source_payload_uri=source-uri'])
+
+ self.assertDictEqual(test_plan_dict, expected_test_plan)
+ self.assertSetEqual(test_args_set, expected_test_args)
