test_stages: use skylab create-suite/wait-task for hwtests
Cherrypicked to address crbug.com/973654
Use `skylab create-suite` and `skylab wait-task` to trigger skylab
hwtests and collect their results.
Do not land this until
https://chromium-review.googlesource.com/c/infra/infra/+/1593690 is
pushed to prod.
BUG=chromium:957805
TEST=tryjob at https://cros-goldeneye.corp.google.com/chromeos/healthmonitoring/buildDetails?buildbucketId=8914633363238609216
shows both passing and failing suites are handled correctly
Change-Id: I1328b09d717af558379483944f9cb2fe9c3522a0
Reviewed-on: https://chromium-review.googlesource.com/1589146
Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com>
Tested-by: Aviv Keshet <akeshet@chromium.org>
Legacy-Commit-Queue: Commit Bot <commit-bot@chromium.org>
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
(cherry picked from commit ae4c830cc69e9774361a94bb2b72780132f97ee3)
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/1658751
Reviewed-by: Bernie Thompson <bhthompson@chromium.org>
Commit-Queue: Aviv Keshet <akeshet@chromium.org>
diff --git a/cbuildbot/commands.py b/cbuildbot/commands.py
index f7bf404..aed14f8 100644
--- a/cbuildbot/commands.py
+++ b/cbuildbot/commands.py
@@ -878,6 +878,7 @@
return os.path.basename(target)
+# TODO(akeshet): Deprecate this undocumented type.
HWTestSuiteResult = collections.namedtuple('HWTestSuiteResult',
['to_raise', 'json_dump_result'])
@@ -1066,74 +1067,90 @@
return HWTestSuiteResult(to_raise, json_dump_result)
-# pylint: disable=docstring-missing-args
-def _GetRunSkylabSuiteArgs(
- build, suite, board,
- model=None,
- pool=None,
- priority=None,
- timeout_mins=None,
- retry=None,
- max_retries=None,
- suite_args=None,
- job_keyvals=None,
- quota_account=None):
- """Get a list of args for run_suite.
-
- TODO (xixuan): Add other features as required, e.g.
- subsystems
- test_args
- skip_duts_check
- suite_min_duts
- minimum_duts
+def _GetSkylabWaitTaskArgs(task_id, timeout_mins=None):
+ """Get arguments for the `skylab wait-task` command.
Args:
- See args of |RunHWTestSuite|.
+ timeout_mins: maximum number of minutes to wait for task completion.
+ task_id: id of the task to wait for.
Returns:
- A list of args for SKYLAB_RUN_SUITE_PATH.
+ List of args for `skylab wait-task`, not including the subcommand itself.
"""
- # HACK(pwang): Delete this once better solution is out.
+ args = ['-service-account-json', constants.CHROMEOS_SERVICE_ACCOUNT]
+ if timeout_mins is not None:
+ args += ['-timeout-mins', str(timeout_mins)]
+ args += [task_id]
+ return args
+
+
+def _GetSkylabCreateSuiteArgs(
+ build, suite, board, pool,
+ model=None,
+ priority=None,
+ timeout_mins=None,
+ max_retries=None,
+ job_keyvals=None,
+ quota_account=None):
+ """Get arguments for the `skylab create-suite` command.
+
+ Command will run in -json mode.
+
+ Args:
+ build: string image name to use, e.g. elm-paladin/R99-12345
+ suite: suite name to run
+ board: board name to run suite for
+ pool: pool to run the suite in
+ model: (optional) model name to run suite for
+ priority: (optional) integer priority for the suite. Higher number is a
+ lower priority
+ timeout_mins: (optional) suite timeout
+ max_retries: (optional) max retries allowed across all child tasks
+ job_keyvals: (optional) dictionary of {'key': 'value'} keyvals to be
+ injected into all children of suite.
+ quota_account: (optional) quotascheduler account to use for child tasks
+
+ Returns:
+ A list of args for the `skylab create-suite` subcommand (not including)
+ the subcommand itself.
+ """
+ # TODO(crbug.com/958037): Figure out a way to remove this board-replacement
+ # hack.
board = board.replace('-arcnext', '')
board = board.replace('-arcvm', '')
board = board.replace('-kernelnext', '')
- args = ['--build', build, '--board', board]
+
+ args = ['-image', build, '-board', board]
if model:
- args += ['--model', model]
+ args += ['-model', model]
- args += ['--suite_name', suite]
-
- # Add optional arguments to command, if present.
if pool is not None:
- args += ['--pool', pool]
+ args += ['-pool', pool]
if priority is not None:
- args += ['--priority', str(priority)]
+ args += ['-priority', str(priority)]
if timeout_mins is not None:
- args += ['--timeout_mins', str(timeout_mins)]
-
- if retry is not None:
- args += ['--test_retry']
+ args += ['-timeout-mins', str(timeout_mins)]
if max_retries is not None:
- args += ['--max_retries', str(max_retries)]
-
- if suite_args is not None:
- args += ['--suite_args', repr(suite_args)]
+ args += ['-max-retries', str(max_retries)]
if job_keyvals is not None:
- args += ['--job_keyvals', repr(job_keyvals)]
+ for key, value in job_keyvals.items():
+ args += ['-keyval', '%s:%s' % (key, value)]
if quota_account is not None:
- args += ['--quota_account', quota_account]
+ args += ['-qs-account', quota_account]
- # Use fallback request for every skylab suite.
- args += ['--use_fallback']
+ args += ['-json']
+
+ args += ['-service-account-json', constants.CHROMEOS_SERVICE_ACCOUNT]
+
+ args += [suite]
return args
-# pylint: enable=docstring-missing-args
def _remove_seeded_steps(output):
@@ -1179,64 +1196,6 @@
return return_output
-def _SkylabHWTestCreate(cmd, **kwargs):
- """Create HWTest with Skylab.
-
- Args:
- cmd: A list of args for proxied run_suite_skylab command.
- kwargs: args to be passed to RunSwarmingCommand.
-
- Returns:
- A string swarming task id, which is regarded as the suite_id.
- """
- create_cmd = cmd + ['--create_and_return']
- logging.info('Suite creation command: \n%s',
- cros_build_lib.CmdToStr(create_cmd))
- result = swarming_lib.RunSwarmingCommandWithRetries(
- max_retry=_MAX_HWTEST_START_CMD_RETRY,
- is_skylab=True,
- error_check=swarming_lib.SwarmingRetriableErrorCheck,
- cmd=create_cmd, capture_output=True,
- combine_stdout_stderr=True, **kwargs)
- # If the command succeeds, result.task_summary_json will have the right
- # content. So result.GetValue is able to return valid values from its
- # json summary.
- for output in result.GetValue('outputs', []):
- sys.stdout.write(_remove_seeded_steps(output))
- sys.stdout.flush()
- return result.GetValue('run_id')
-
-
-def _SkylabHWTestWait(cmd, suite_id, **kwargs):
- """Wait for Skylab HWTest to finish.
-
- Args:
- cmd: Proxied run_suite command.
- suite_id: The string suite_id to wait for.
- kwargs: args to be passed to RunSwarmingCommand.
- """
- wait_cmd = list(cmd) + ['--suite_id', str(suite_id)]
- logging.info('RunSkylabHWTestSuite will wait for suite %s: %s',
- suite_id, cros_build_lib.CmdToStr(wait_cmd))
- try:
- result = swarming_lib.RunSwarmingCommandWithRetries(
- max_retry=_MAX_HWTEST_START_CMD_RETRY,
- is_skylab=True,
- error_check=swarming_lib.SwarmingRetriableErrorCheck,
- cmd=wait_cmd, capture_output=True, combine_stdout_stderr=True,
- **kwargs)
- except cros_build_lib.RunCommandError as e:
- result = e.result
- raise
- finally:
- # This is required to output buildbot annotations, e.g. 'STEP_LINKS'.
- sys.stdout.write('######## Output for buildbot annotations ######## \n')
- for output in result.GetValue('outputs', []):
- sys.stdout.write(_remove_seeded_steps(output))
- sys.stdout.write('######## END Output for buildbot annotations ######## \n')
- sys.stdout.flush()
-
-
def _InstallSkylabTool():
"""Install skylab tool.
@@ -1361,18 +1320,22 @@
sys.stdout.write('######## END Output for buildbot annotations ######## \n')
sys.stdout.flush()
+
# pylint: disable=docstring-missing-args
@failures_lib.SetFailureType(failures_lib.SuiteTimedOut,
timeout_util.TimeoutError)
def RunSkylabHWTestSuite(
build, suite, board,
model=None,
+ # TODO(akeshet): Make this required argument a positional arg.
pool=None,
- wait_for_results=None,
+ wait_for_results=False,
priority=None,
timeout_mins=None,
+ # TODO(akeshet): Delete this ignored argument.
retry=None,
max_retries=None,
+ # TODO(akeshet): Delete this ignored argument.
suite_args=None,
job_keyvals=None,
quota_account=None):
@@ -1384,36 +1347,76 @@
Returns:
See returns of RunHWTestSuite.
"""
- priority_str = priority
+ if suite_args:
+ logging.warning('Skylab suites do not support suite_args, although they '
+ 'were specified: %s', suite_args)
+
+ if retry:
+ logging.warning('Skylab suites do not support retry arg, although it '
+ 'was specified: %s', retry)
+
+ if not pool:
+ raise ValueError('|pool| argument is required in Skylab, but was not '
+ 'supplied.')
+
if priority:
priority = constants.SKYLAB_HWTEST_PRIORITIES_MAP[str(priority)]
- cmd = [SKYLAB_RUN_SUITE_PATH]
- cmd += _GetRunSkylabSuiteArgs(
- build, suite, board,
+ skylab_tool = _InstallSkylabTool()
+
+ cmd = [skylab_tool, 'create-suite']
+
+ cmd += _GetSkylabCreateSuiteArgs(
+ build, suite, board, pool,
model=model,
- pool=pool,
priority=priority,
timeout_mins=timeout_mins,
- retry=retry,
max_retries=max_retries,
- suite_args=suite_args,
job_keyvals=job_keyvals,
quota_account=quota_account)
- swarming_args = _CreateSwarmingArgs(build, suite, board, priority_str,
- timeout_mins, run_skylab=True)
- try:
- suite_id = _SkylabHWTestCreate(cmd, **swarming_args)
- if wait_for_results:
- _SkylabHWTestWait(cmd, suite_id, **swarming_args)
- return HWTestSuiteResult(None, None)
+ try:
+ output = cros_build_lib.RunCommand(cmd, redirect_stdout=True)
+ report = json.loads(output.output)
+ task_id = report['task_id']
+ task_url = report['task_url']
+
+ logging.info('Launched suite task %s', task_url)
+ logging.PrintBuildbotLink('Suite task: %s' % suite, task_url)
+ if not wait_for_results:
+ return HWTestSuiteResult(None, None)
+
+ wait_cmd = [skylab_tool, 'wait-task'] + _GetSkylabWaitTaskArgs(
+ task_id, timeout_mins=timeout_mins)
+ output = cros_build_lib.RunCommand(wait_cmd, redirect_stdout=True)
+ try:
+ report = json.loads(output.output)
+ except:
+ logging.error('Error when json parsing:\n%s', output.output)
+ raise
+
+ # TODO(crbug.com/958142): Once the schema change in this bug is landed,
+ # remove this if block.
+ if task_id in report:
+ report = report[task_id]
+ logging.info(
+ 'Suite ended in state %s (failure=%s) with output:\n%s',
+ report['task-result']['state'],
+ report['task-result']['failure'],
+ report['stdout'],
+ )
+
+ error = None
+ if report['task-result']['failure']:
+ error = failures_lib.TestFailure('Suite failed.')
+
+ return HWTestSuiteResult(error, None)
+
except cros_build_lib.RunCommandError as e:
result = e.result
to_raise = failures_lib.TestFailure(
'** HWTest failed (code %d) **' % result.returncode)
return HWTestSuiteResult(to_raise, None)
-# pylint: enable=docstring-missing-args
# pylint: disable=docstring-missing-args
diff --git a/cbuildbot/commands_unittest.py b/cbuildbot/commands_unittest.py
index 2d117ca..319410e 100644
--- a/cbuildbot/commands_unittest.py
+++ b/cbuildbot/commands_unittest.py
@@ -8,6 +8,7 @@
from __future__ import print_function
import base64
+import collections
import datetime as dt
import json
import hashlib
@@ -175,196 +176,160 @@
'chromiumos_preflight'])
-# pylint: disable=protected-access
-class SkylabHWLabCommandsTest(cros_test_lib.RunCommandTestCase,
- cros_test_lib.OutputTestCase,
- cros_test_lib.MockTempDirTestCase):
- """Test commands related to HWLab tests with Skylab via swarming proxy."""
- SWARMING_TIMEOUT_DEFAULT = str(
- commands._DEFAULT_HWTEST_TIMEOUT_MINS * 60 +
- commands._SWARMING_ADDITIONAL_TIMEOUT)
- SWARMING_EXPIRATION = str(commands._SWARMING_EXPIRATION)
+class SkylabHWLabCommandsTest(cros_test_lib.RunCommandTestCase):
+ """Test commands that launch Skylab suites."""
- WAIT_OUTPUT = '''
-WAIT OUTPUT: Finished.
-'''
-
- CREATE_OUTPUT = '''
-FAKE OUTPUT. Will be filled in later.
-'''
+ _SKYLAB_TOOL = '/opt/skylab'
def setUp(self):
- self._build = 'test-build'
- self._board = 'test-board'
- self._suite = 'test-suite'
- self.temp_json_path = os.path.join(self.tempdir, 'temp_summary.json')
+ self.PatchObject(commands, '_InstallSkylabTool',
+ return_value=self._SKYLAB_TOOL)
- self.create_cmd = None
- self.wait_cmd = None
+ @staticmethod
+ def _fakeWaitJson(state, failure, stdout=''):
+ """Return a fake json serialized suite report, in wait-task format."""
+ report = {
+ 'task-result': {
+ 'state': state,
+ 'failure': failure,
+ },
+ 'stdout': stdout,
+ }
+ return json.dumps(report)
- def SetCmdResults(self, swarming_cli_cmd='run',
- create_return_code=0,
- wait_return_code=0,
- args=(),
- swarming_timeout_secs=SWARMING_TIMEOUT_DEFAULT,
- swarming_io_timeout_secs=SWARMING_TIMEOUT_DEFAULT,
- swarming_hard_timeout_secs=SWARMING_TIMEOUT_DEFAULT,
- swarming_expiration_secs=SWARMING_EXPIRATION):
- """Set the expected results from the specified commands.
+ @staticmethod
+ def _fakeCreateJson(task_id, task_url):
+ """Return a fake json serialized suite report, in create-suite format."""
+ report = {
+ 'task_id': task_id,
+ 'task_url': task_url,
+ }
+ return json.dumps(report)
- Args:
- swarming_cli_cmd: The swarming client command to kick off.
- create_return_code: Return code from create command.
- wait_return_code: Return code from wait command.
- args: Additional args to pass to create and wait commands.
- swarming_timeout_secs: swarming client timeout.
- swarming_io_timeout_secs: swarming client io timeout.
- swarming_hard_timeout_secs: swarming client hard timeout.
- swarming_expiration_secs: swarming task expiration.
- """
- # Pull out the test priority for the swarming tag.
- priority = None
- priority_flag = '--priority'
- if priority_flag in args:
- priority = args[args.index(priority_flag) + 1]
+ def testCreateSuiteNoPoolRaisesError(self):
+ """Attempting to create a suite without specifying a pool should raise."""
+ build = 'foo-bar/R1234'
+ suite = 'foo-suite'
+ board = 'foo-board'
+ with self.assertRaises(ValueError):
+ commands.RunSkylabHWTestSuite(build, suite, board)
- base_cmd = [swarming_lib._SWARMING_PROXY_CLIENT, swarming_cli_cmd,
- '--swarming', topology.topology.get(
- topology.CHROME_SWARMING_PROXY_HOST_KEY)]
- if swarming_cli_cmd == 'run':
- base_cmd += ['--task-summary-json', self.temp_json_path,
- '--print-status-updates',
- '--timeout', swarming_timeout_secs]
- elif swarming_cli_cmd == 'trigger':
- base_cmd += ['--dump-json', self.temp_json_path]
+ def testCreateSuite(self):
+ """Test that function call args are mapped correctly to commandline args."""
+ build = 'foo-bar/R1234'
+ suite = 'foo-suite'
+ board = 'foo-board'
+ pool = 'foo-pool'
+ # An OrderedDict is used to make the keyval order on the command line
+ # deterministic for testing purposes.
+ keyvals = collections.OrderedDict([('k1', 'v1'), ('k2', 'v2')])
+ priority = constants.HWTEST_DEFAULT_PRIORITY
+ quota_account = 'foo-account'
+ max_retries = 10
+ timeout_mins = 10
- base_cmd += ['--raw-cmd',
- '--task-name', 'test-build-test-suite',
- '--dimension', 'os', 'Ubuntu-14.04',
- '--dimension', 'pool', commands.SKYLAB_SUITE_BOT_POOL,
- '--io-timeout', swarming_io_timeout_secs,
- '--hard-timeout', swarming_hard_timeout_secs,
- '--expiration', swarming_expiration_secs,
- '--tags=skylab:run_suite',
- '--tags=priority:%s' % priority,
- '--tags=build:test-build',
- '--tags=task_name:test-build-test-suite',
- '--tags=luci_project:chromeos',
- '--tags=suite:test-suite',
- '--tags=board:test-board',
- '--auth-service-account-json',
- constants.CHROMEOS_SERVICE_ACCOUNT,
- '--', commands.SKYLAB_RUN_SUITE_PATH,
- '--build', self._build, '--board', self._board,
- '--suite_name', self._suite,
- '--use_fallback']
- args = list(args)
- base_cmd = base_cmd + args
+ task_id = 'foo-task_id'
- self.create_cmd = base_cmd + ['--create_and_return',
- '--passed_mins', '0']
- create_results = iter([
- self.rc.CmdResult(returncode=create_return_code,
- output=self.CREATE_OUTPUT,
- error=''),
- ])
+ create_cmd = [
+ self._SKYLAB_TOOL, 'create-suite',
+ '-image', build,
+ '-board', board,
+ '-pool', pool,
+ '-priority', '140',
+ '-timeout-mins', str(timeout_mins),
+ '-max-retries', str(max_retries),
+ '-keyval', 'k1:v1',
+ '-keyval', 'k2:v2',
+ '-qs-account', quota_account,
+ '-json',
+ '-service-account-json', constants.CHROMEOS_SERVICE_ACCOUNT,
+ suite
+ ]
+
self.rc.AddCmdResult(
- self.create_cmd,
- side_effect=lambda *args, **kwargs: create_results.next(),
- )
+ create_cmd, output=self._fakeCreateJson(task_id, 'foo://foo'))
- self.wait_cmd = base_cmd + ['--suite_id', 'fake_parent_id',
- '--passed_mins', '0']
- wait_results = iter([
- self.rc.CmdResult(returncode=wait_return_code,
- output=self.WAIT_OUTPUT,
- error=''),
- ])
+ result = commands.RunSkylabHWTestSuite(
+ build, suite, board, pool=pool, job_keyvals=keyvals, priority=priority,
+ quota_account=quota_account, max_retries=max_retries,
+ timeout_mins=timeout_mins)
+
+ self.assertTrue(isinstance(result, commands.HWTestSuiteResult))
+ self.assertEqual(result.to_raise, None)
+ self.assertEqual(result.json_dump_result, None)
+
+ self.rc.assertCommandCalled(create_cmd, redirect_stdout=True)
+ self.assertEqual(self.rc.call_count, 1)
+
+ def testCreateSuiteAndWait(self):
+ """Test that suite can be created and then waited for."""
+ build = 'foo-bar/R1234'
+ suite = 'foo-suite'
+ board = 'foo-board'
+ pool = 'foo-pool'
+
+ task_id = 'foo-task_id'
+
+ create_cmd = [
+ self._SKYLAB_TOOL, 'create-suite',
+ '-image', build,
+ '-board', board,
+ '-pool', pool,
+ '-json',
+ '-service-account-json', constants.CHROMEOS_SERVICE_ACCOUNT,
+ suite,
+ ]
+ wait_cmd = [
+ self._SKYLAB_TOOL, 'wait-task',
+ '-service-account-json', constants.CHROMEOS_SERVICE_ACCOUNT,
+ task_id
+ ]
self.rc.AddCmdResult(
- self.wait_cmd,
- side_effect=lambda *args, **kwargs: wait_results.next(),
- )
+ create_cmd, output=self._fakeCreateJson(task_id, 'foo://foo'))
+ self.rc.AddCmdResult(
+ wait_cmd, output=self._fakeWaitJson('COMPLETED', False))
- def PatchJson(self, task_outputs):
- """Mock out the code that loads from json.
+ result = commands.RunSkylabHWTestSuite(
+ build, suite, board, pool=pool, wait_for_results=True)
+ self.assertEqual(result.to_raise, None)
+ self.assertEqual(result.json_dump_result, None)
- Args:
- task_outputs: See explaination in PatchJson of HWLabCommandsTest.
- """
- orig_func = commands._CreateSwarmingArgs
+ self.rc.assertCommandCalled(create_cmd, redirect_stdout=True)
+ self.rc.assertCommandCalled(wait_cmd, redirect_stdout=True)
+ self.assertEqual(self.rc.call_count, 2)
- def replacement(*args, **kargs):
- swarming_args = orig_func(*args, **kargs)
- swarming_args['temp_json_path'] = self.temp_json_path
- return swarming_args
+ def testSuiteFailure(self):
+ """Test that nonzero return code is reported as suite failure."""
+ build = 'foo-bar/R1234'
+ suite = 'foo-suite'
+ board = 'foo-board'
+ pool = 'foo-pool'
- self.PatchObject(commands, '_CreateSwarmingArgs', side_effect=replacement)
+ task_id = 'foo-task_id'
- if task_outputs:
- return_values = []
- for s in task_outputs:
- j = {'shards':[{'name': 'fake_name', 'bot_id': 'chromeos-server990',
- 'created_ts': '2015-06-12 12:00:00',
- 'internal_failure': s[1],
- 'state': s[2],
- 'outputs': [s[0]],
- 'run_id': s[3]}]}
- return_values.append(j)
- return_values_iter = iter(return_values)
- self.PatchObject(swarming_lib.SwarmingCommandResult, 'LoadJsonSummary',
- side_effect=lambda json_file: return_values_iter.next())
- else:
- self.PatchObject(swarming_lib.SwarmingCommandResult, 'LoadJsonSummary',
- return_value=None)
+ create_cmd = [
+ self._SKYLAB_TOOL, 'create-suite',
+ '-image', build,
+ '-board', board,
+ '-pool', pool,
+ '-json',
+ '-service-account-json', constants.CHROMEOS_SERVICE_ACCOUNT,
+ suite,
+ ]
+ wait_cmd = [
+ self._SKYLAB_TOOL, 'wait-task',
+ '-service-account-json', constants.CHROMEOS_SERVICE_ACCOUNT,
+ task_id,
+ ]
+ self.rc.AddCmdResult(
+ create_cmd, output=self._fakeCreateJson(task_id, 'foo://foo'))
+ self.rc.AddCmdResult(wait_cmd, output=self._fakeWaitJson('COMPLETED', True))
- def RunSkylabHWTestSuite(self, *args, **kwargs):
- """Run the hardware test suite, printing logs to stdout."""
- with cros_test_lib.LoggingCapturer() as logs:
- try:
- cmd_result = commands.RunSkylabHWTestSuite(self._build, self._suite,
- self._board, *args, **kwargs)
- return cmd_result
- finally:
- print(logs.messages)
-
- def testRemoveSeededSteps(self):
- output = ('2018-xx-xx info | kicked off a test\n'
- '@@@SEED_STEP Scheduled Tests@@@\n'
- '@@@STEP_CURSOR Scheduled Tests@@@\n'
- '@@@STEP_STARTED@@@\n'
- '@@@STEP_LINK@[Test-logs]: test 1'
- '@https://chrome-swarming.appspot.com/user/task/123@@@\n'
- '@@@STEP_CLOSED@@@\n'
- '@@@SEED_STEP Scheduled Tests 2@@@\n'
- '@@@STEP_CURSOR Scheduled Tests 2@@@\n'
- '@@@STEP_STARTED@@@\n'
- '@@@STEP_LINK@[Test-logs]: test 2'
- '@https://chrome-swarming.appspot.com/user/task/456@@@\n'
- '@@@STEP_CLOSED@@@\n'
- '@@@STEP_LINK@[Test-logs]: test 3'
- '@https://chrome-swarming.appspot.com/user/task/789@@@\n')
- expected_output = (
- '2018-xx-xx info | kicked off a test\n'
- '@@@STEP_LINK@[Test-logs]: test 3'
- '@https://chrome-swarming.appspot.com/user/task/789@@@\n')
- self.assertEqual(commands._remove_seeded_steps(output), expected_output)
-
- def testRunSkylabHWTestSuiteWithWait(self):
- """Test RunSkylabHWTestSuite with waiting for results."""
- self.SetCmdResults(swarming_cli_cmd='run')
- # When run without optional arguments, wait and dump_json cmd will not run.
- self.PatchJson([(self.CREATE_OUTPUT, False, None, 'fake_parent_id'),
- (self.WAIT_OUTPUT, False, None, 'fake_wait_id')])
-
- with self.OutputCapturer() as output:
- cmd_result = self.RunSkylabHWTestSuite(wait_for_results=True)
- self.assertEqual(cmd_result, (None, None))
- self.assertCommandCalled(self.create_cmd, capture_output=True,
- combine_stdout_stderr=True, env=mock.ANY)
- self.assertCommandCalled(self.wait_cmd, capture_output=True,
- combine_stdout_stderr=True, env=mock.ANY)
- self.assertIn(self.CREATE_OUTPUT, '\n'.join(output.GetStdoutLines()))
- self.assertIn(self.WAIT_OUTPUT, '\n'.join(output.GetStdoutLines()))
+ result = commands.RunSkylabHWTestSuite(
+ build, suite, board, pool=pool, wait_for_results=True)
+ error = result.to_raise
+ self.assertTrue(isinstance(error, failures_lib.TestFailure))
+ self.assertTrue('Suite failed' in error.message)
class HWLabCommandsTest(cros_test_lib.RunCommandTestCase,
