Revert "Add support for handling device/arch/abi specific waivers."
This reverts commit 3aa24409ea679e9968a2cc2efd6efe210997db0c.
BUG=chromium:796743
Change-Id: I7b68db4cc3f4ffcafbc0ce69d682ec480576217c
Reviewed-on: https://chromium-review.googlesource.com/838409
Reviewed-by: Ilja H. Friedel <ihf@chromium.org>
Tested-by: Ilja H. Friedel <ihf@chromium.org>
(cherry picked from commit cfcd30157a1779ed31621295f253e7f69b17a87f)
Reviewed-on: https://chromium-review.googlesource.com/838519
Reviewed-by: Josafat Garcia <josafat@chromium.org>
Commit-Queue: Josafat Garcia <josafat@chromium.org>
Tested-by: Josafat Garcia <josafat@chromium.org>
diff --git a/server/cros/cts_expected_failure_parser.py b/server/cros/cts_expected_failure_parser.py
deleted file mode 100644
index 37baff1..0000000
--- a/server/cros/cts_expected_failure_parser.py
+++ /dev/null
@@ -1,59 +0,0 @@
-# Copyright 2017 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import logging
-import yaml
-
-class ParseKnownCTSFailures(object):
- """A class to parse known failures in CTS test."""
-
- def __init__(self, failure_files):
- self.waivers_yaml = self._load_failures(failure_files)
-
- def _validate_waiver_config(self, arch, board, config):
- """Validate if the test environment matches the test config.
-
- @param arch: DUT's arch type.
- @param board: DUT's board name.
- @param config: config for an expected failing test.
- @return True if test arch or board is part of the config, else False.
- """
- dut_config = ['all', arch, board]
- return len(set(dut_config).intersection(config)) > 0
-
- def _load_failures(self, failure_files):
- """Load failures from files.
-
- @param failure_files: files with failure configs.
- @return a dictionary of failures config in yaml format.
- """
- waivers_yaml = {}
- for failure_file in failure_files:
- try:
- logging.info('Loading expected failure file: %s.', failure_file)
- with open(failure_file) as wf:
- waivers_yaml.update(yaml.load(wf.read()))
- except IOError as e:
- logging.error('Error loading %s (%s).',
- failure_file,
- e.strerror)
- continue
- logging.info('Finished loading expected failure file: %s',
- failure_file)
- return waivers_yaml
-
- def find_waivers(self, arch, board):
- """Finds waivers for the test board.
-
- @param arch: DUT's arch type.
- @param board: DUT's board name.
- @return a set of waivers/no-test-modules applied to the test board.
- """
- applied_waiver_list = set()
- for test, config in self.waivers_yaml.iteritems():
- if self._validate_waiver_config(arch, board, config):
- applied_waiver_list.add(test)
- logging.info('Excluding tests/packages from rerun: %s.',
- applied_waiver_list)
- return applied_waiver_list
diff --git a/server/cros/tradefed_test.py b/server/cros/tradefed_test.py
index 8e05e2b..094bc44 100644
--- a/server/cros/tradefed_test.py
+++ b/server/cros/tradefed_test.py
@@ -41,7 +41,6 @@
from autotest_lib.server import autotest
from autotest_lib.server import test
from autotest_lib.server import utils
-from autotest_lib.server.cros import cts_expected_failure_parser
# TODO(ihf): If akeshet doesn't fix crbug.com/691046 delete metrics again.
try:
@@ -52,7 +51,6 @@
# TODO(ihf): Find a home for all these paths. This is getting out of hand.
_SDK_TOOLS_DIR = 'gs://chromeos-arc-images/builds/git_nyc-mr1-arc-linux-static_sdk_tools/3544738'
_SDK_TOOLS_FILES = ['aapt']
-
# To stabilize adb behavior, we use dynamically linked adb.
_ADB_DIR = 'gs://chromeos-arc-images/builds/git_nyc-mr1-arc-linux-cheets_arm-user/3544738'
_ADB_FILES = ['adb']
@@ -472,7 +470,6 @@
uri=None,
host=None,
max_retry=None,
- retry_manual_tests=False,
warn_on_test_retry=True):
"""Sets up the tools and binary bundles for the test."""
logging.info('Hostname: %s', host.hostname)
@@ -521,12 +518,9 @@
uri or self._get_default_bundle_url(bundle))
self._repository = os.path.join(bundle_install_path,
self._get_tradefed_base_dir())
-
- # Load expected test failures to exclude them from re-runs.
+ # Load waivers and manual tests so TF doesn't re-run them.
self._waivers = self._get_expected_failures('expectations')
- if not retry_manual_tests:
- self._waivers.update(self._get_expected_failures('manual_tests'))
-
+ self._manual_tests = self._get_expected_failures('manual_tests')
# Load modules with no tests.
self._notest_modules = self._get_expected_failures('notest_modules')
@@ -1060,7 +1054,7 @@
tradefed. It is up to the caller to handle inconsistencies.
@param result: The result object from utils.run.
- @param waivers: a set[] of tests which are permitted to fail.
+ @param waivers: a set() of tests which are permitted to fail.
"""
return parse_tradefed_result(result.stdout, waivers)
@@ -1107,36 +1101,34 @@
shutil.copytree(
os.path.join(repository_logs, datetime), destination_logs_datetime)
- def _get_expected_failures(self, *directories):
- """Return a list of expected failures or no test module.
+ def _get_expected_failures(self, directory):
+ """Return a list of expected failures.
- @param directories: A list of directories with expected no tests
- or failures files.
- @return: A list of expected failures or no test modules for the current
- testing device.
+ @return: a list of expected failures.
"""
- # Load waivers and manual tests so TF doesn't re-run them.
- expected_fail_files = []
- test_board = self._get_board_name(self._host)
- test_arch = self._get_board_arch(self._host)
- for directory in directories:
- expected_fail_dir = os.path.join(self.bindir, directory)
- if os.path.exists(expected_fail_dir):
- expected_fail_files += glob.glob(expected_fail_dir + '/*.yaml')
-
- waivers = cts_expected_failure_parser.ParseKnownCTSFailures(
- expected_fail_files)
- return waivers.find_waivers(test_board, test_arch)
+ logging.info('Loading expected failures from %s.', directory)
+ expected_fail_dir = os.path.join(self.bindir, directory)
+ expected_fail_files = glob.glob(expected_fail_dir + '/*.' + self._abi)
+ expected_failures = set()
+ for expected_fail_file in expected_fail_files:
+ try:
+ file_path = os.path.join(expected_fail_dir, expected_fail_file)
+ with open(file_path) as f:
+ lines = set(f.read().splitlines())
+ logging.info('Loaded %d expected failures from %s',
+ len(lines), expected_fail_file)
+ expected_failures |= lines
+ except IOError as e:
+ logging.error('Error loading %s (%s).', file_path, e.strerror)
+ logging.info('Finished loading expected failures: %s',
+ expected_failures)
+ return expected_failures
def _get_release_channel(self, host):
"""Returns the DUT channel of the image ('dev', 'beta', 'stable')."""
channel = host.get_channel()
return channel or 'dev'
- def _get_board_arch(self, host):
- """ Return target DUT arch name."""
- return 'arm' if host.get_cpu_arch() == 'arm' else 'x86'
-
def _get_board_name(self, host):
"""Return target DUT board name."""
return host.get_board().split(':')[1]
diff --git a/server/site_tests/cheets_CTS_N/cheets_CTS_N.py b/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
index 064ede3..dc1ad26 100644
--- a/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
+++ b/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
@@ -151,7 +151,8 @@
# newbie and novato are x86 VMs without binary translation. Skip the ARM
# tests.
no_ARM_ABI_test_boards = ('newbie', 'novato', 'novato-arc64')
- if self._get_board_name(self._host) in no_ARM_ABI_test_boards:
+ board = self._host.get_board().split(':')[1] # Remove 'board:' prefix.
+ if board in no_ARM_ABI_test_boards:
if self._abi == 'arm':
return True
return False
@@ -200,6 +201,7 @@
target_class=None,
target_method=None,
needs_push_media=False,
+ retry_manual_tests=False,
cts_tradefed_args=None,
precondition_commands=[],
login_precondition_commands=[],
@@ -219,6 +221,9 @@
@param target_class: the name of the class to be tested.
@param target_method: the name of the method to be tested.
@param needs_push_media: need to push test media streams.
+ @param retry_manual_tests: a flag to track whether manual tests
+ need to be retried or not. Autotest lab skips manual tests,
+ while moblab runs them.
@param timeout: time after which tradefed can be interrupted.
@param precondition_commands: a list of scripts to be run on the
dut before the test is run, the scripts must already be installed.
@@ -234,6 +239,11 @@
# Retries depend on channel.
self._timeoutfactor = None
+ # TODO(kinaba): Move this logic to tradefed_test so that cheets_GTS can
+ # deal with manual tests.
+ if not retry_manual_tests:
+ self._waivers.update(self._manual_tests)
+
test_command, test_name = self.generate_test_command(target_module,
target_plan,
target_class,
@@ -244,3 +254,4 @@
target_plan, needs_push_media, _CTS_URI,
login_precondition_commands,
precondition_commands)
+
diff --git a/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.arm b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.arm
new file mode 100644
index 0000000..3f47752
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.arm
@@ -0,0 +1,6 @@
+CtsAlarmClockTestCases
+CtsCallLogTestCases
+CtsCarTestCases
+CtsSampleDeviceTestCases
+CtsSampleHostTestCases
+CtsSystemUiTestCases
diff --git a/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.x86 b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.x86
new file mode 100644
index 0000000..3f47752
--- /dev/null
+++ b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.x86
@@ -0,0 +1,6 @@
+CtsAlarmClockTestCases
+CtsCallLogTestCases
+CtsCarTestCases
+CtsSampleDeviceTestCases
+CtsSampleHostTestCases
+CtsSystemUiTestCases
diff --git a/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.yaml b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.yaml
deleted file mode 100644
index 82c4026..0000000
--- a/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-CtsAlarmClockTestCases: [all]
-CtsCallLogTestCases: [all]
-CtsCarTestCases: [all]
-CtsSampleDeviceTestCases: [all]
-CtsSampleHostTestCases: [all]
-CtsSystemUiTestCases: [all]
diff --git a/server/site_tests/cheets_GTS/notest_modules/notest_modules.arm b/server/site_tests/cheets_GTS/notest_modules/notest_modules.arm
new file mode 100644
index 0000000..192f25e
--- /dev/null
+++ b/server/site_tests/cheets_GTS/notest_modules/notest_modules.arm
@@ -0,0 +1,5 @@
+GtsAccountsHostTestCases
+GtsDexModuleRegistrationTestCases
+GtsSampleDeviceTestCases
+GtsSampleDynamicConfigTestCases
+GtsSampleHostTestCases
diff --git a/server/site_tests/cheets_GTS/notest_modules/notest_modules.x86 b/server/site_tests/cheets_GTS/notest_modules/notest_modules.x86
new file mode 100644
index 0000000..192f25e
--- /dev/null
+++ b/server/site_tests/cheets_GTS/notest_modules/notest_modules.x86
@@ -0,0 +1,5 @@
+GtsAccountsHostTestCases
+GtsDexModuleRegistrationTestCases
+GtsSampleDeviceTestCases
+GtsSampleDynamicConfigTestCases
+GtsSampleHostTestCases
diff --git a/server/site_tests/cheets_GTS/notest_modules/notest_modules.yaml b/server/site_tests/cheets_GTS/notest_modules/notest_modules.yaml
deleted file mode 100644
index e6ccc50..0000000
--- a/server/site_tests/cheets_GTS/notest_modules/notest_modules.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-GtsAccountsHostTestCases: [all]
-GtsDexModuleRegistrationTestCases: [all]
-GtsSampleDeviceTestCases: [all]
-GtsSampleDynamicConfigTestCases: [all]
-GtsSampleHostTestCases: [all]
