Revert "Add support for handling device/arch/abi specific waivers." This reverts commit 3aa24409ea679e9968a2cc2efd6efe210997db0c. BUG=chromium:796743 Change-Id: I7b68db4cc3f4ffcafbc0ce69d682ec480576217c Reviewed-on: https://chromium-review.googlesource.com/838409 Reviewed-by: Ilja H. Friedel <ihf@chromium.org> Tested-by: Ilja H. Friedel <ihf@chromium.org> (cherry picked from commit cfcd30157a1779ed31621295f253e7f69b17a87f) Reviewed-on: https://chromium-review.googlesource.com/838519 Reviewed-by: Josafat Garcia <josafat@chromium.org> Commit-Queue: Josafat Garcia <josafat@chromium.org> Tested-by: Josafat Garcia <josafat@chromium.org>
diff --git a/server/cros/cts_expected_failure_parser.py b/server/cros/cts_expected_failure_parser.py deleted file mode 100644 index 37baff1..0000000 --- a/server/cros/cts_expected_failure_parser.py +++ /dev/null
@@ -1,59 +0,0 @@ -# Copyright 2017 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import logging -import yaml - -class ParseKnownCTSFailures(object): - """A class to parse known failures in CTS test.""" - - def __init__(self, failure_files): - self.waivers_yaml = self._load_failures(failure_files) - - def _validate_waiver_config(self, arch, board, config): - """Validate if the test environment matches the test config. - - @param arch: DUT's arch type. - @param board: DUT's board name. - @param config: config for an expected failing test. - @return True if test arch or board is part of the config, else False. - """ - dut_config = ['all', arch, board] - return len(set(dut_config).intersection(config)) > 0 - - def _load_failures(self, failure_files): - """Load failures from files. - - @param failure_files: files with failure configs. - @return a dictionary of failures config in yaml format. - """ - waivers_yaml = {} - for failure_file in failure_files: - try: - logging.info('Loading expected failure file: %s.', failure_file) - with open(failure_file) as wf: - waivers_yaml.update(yaml.load(wf.read())) - except IOError as e: - logging.error('Error loading %s (%s).', - failure_file, - e.strerror) - continue - logging.info('Finished loading expected failure file: %s', - failure_file) - return waivers_yaml - - def find_waivers(self, arch, board): - """Finds waivers for the test board. - - @param arch: DUT's arch type. - @param board: DUT's board name. - @return a set of waivers/no-test-modules applied to the test board. - """ - applied_waiver_list = set() - for test, config in self.waivers_yaml.iteritems(): - if self._validate_waiver_config(arch, board, config): - applied_waiver_list.add(test) - logging.info('Excluding tests/packages from rerun: %s.', - applied_waiver_list) - return applied_waiver_list
diff --git a/server/cros/tradefed_test.py b/server/cros/tradefed_test.py index 8e05e2b..094bc44 100644 --- a/server/cros/tradefed_test.py +++ b/server/cros/tradefed_test.py
@@ -41,7 +41,6 @@ from autotest_lib.server import autotest from autotest_lib.server import test from autotest_lib.server import utils -from autotest_lib.server.cros import cts_expected_failure_parser # TODO(ihf): If akeshet doesn't fix crbug.com/691046 delete metrics again. try: @@ -52,7 +51,6 @@ # TODO(ihf): Find a home for all these paths. This is getting out of hand. _SDK_TOOLS_DIR = 'gs://chromeos-arc-images/builds/git_nyc-mr1-arc-linux-static_sdk_tools/3544738' _SDK_TOOLS_FILES = ['aapt'] - # To stabilize adb behavior, we use dynamically linked adb. _ADB_DIR = 'gs://chromeos-arc-images/builds/git_nyc-mr1-arc-linux-cheets_arm-user/3544738' _ADB_FILES = ['adb'] @@ -472,7 +470,6 @@ uri=None, host=None, max_retry=None, - retry_manual_tests=False, warn_on_test_retry=True): """Sets up the tools and binary bundles for the test.""" logging.info('Hostname: %s', host.hostname) @@ -521,12 +518,9 @@ uri or self._get_default_bundle_url(bundle)) self._repository = os.path.join(bundle_install_path, self._get_tradefed_base_dir()) - - # Load expected test failures to exclude them from re-runs. + # Load waivers and manual tests so TF doesn't re-run them. self._waivers = self._get_expected_failures('expectations') - if not retry_manual_tests: - self._waivers.update(self._get_expected_failures('manual_tests')) - + self._manual_tests = self._get_expected_failures('manual_tests') # Load modules with no tests. self._notest_modules = self._get_expected_failures('notest_modules') @@ -1060,7 +1054,7 @@ tradefed. It is up to the caller to handle inconsistencies. @param result: The result object from utils.run. - @param waivers: a set[] of tests which are permitted to fail. + @param waivers: a set() of tests which are permitted to fail. """ return parse_tradefed_result(result.stdout, waivers) @@ -1107,36 +1101,34 @@ shutil.copytree( os.path.join(repository_logs, datetime), destination_logs_datetime) - def _get_expected_failures(self, *directories): - """Return a list of expected failures or no test module. + def _get_expected_failures(self, directory): + """Return a list of expected failures. - @param directories: A list of directories with expected no tests - or failures files. - @return: A list of expected failures or no test modules for the current - testing device. + @return: a list of expected failures. """ - # Load waivers and manual tests so TF doesn't re-run them. - expected_fail_files = [] - test_board = self._get_board_name(self._host) - test_arch = self._get_board_arch(self._host) - for directory in directories: - expected_fail_dir = os.path.join(self.bindir, directory) - if os.path.exists(expected_fail_dir): - expected_fail_files += glob.glob(expected_fail_dir + '/*.yaml') - - waivers = cts_expected_failure_parser.ParseKnownCTSFailures( - expected_fail_files) - return waivers.find_waivers(test_board, test_arch) + logging.info('Loading expected failures from %s.', directory) + expected_fail_dir = os.path.join(self.bindir, directory) + expected_fail_files = glob.glob(expected_fail_dir + '/*.' + self._abi) + expected_failures = set() + for expected_fail_file in expected_fail_files: + try: + file_path = os.path.join(expected_fail_dir, expected_fail_file) + with open(file_path) as f: + lines = set(f.read().splitlines()) + logging.info('Loaded %d expected failures from %s', + len(lines), expected_fail_file) + expected_failures |= lines + except IOError as e: + logging.error('Error loading %s (%s).', file_path, e.strerror) + logging.info('Finished loading expected failures: %s', + expected_failures) + return expected_failures def _get_release_channel(self, host): """Returns the DUT channel of the image ('dev', 'beta', 'stable').""" channel = host.get_channel() return channel or 'dev' - def _get_board_arch(self, host): - """ Return target DUT arch name.""" - return 'arm' if host.get_cpu_arch() == 'arm' else 'x86' - def _get_board_name(self, host): """Return target DUT board name.""" return host.get_board().split(':')[1]
diff --git a/server/site_tests/cheets_CTS_N/cheets_CTS_N.py b/server/site_tests/cheets_CTS_N/cheets_CTS_N.py index 064ede3..dc1ad26 100644 --- a/server/site_tests/cheets_CTS_N/cheets_CTS_N.py +++ b/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
@@ -151,7 +151,8 @@ # newbie and novato are x86 VMs without binary translation. Skip the ARM # tests. no_ARM_ABI_test_boards = ('newbie', 'novato', 'novato-arc64') - if self._get_board_name(self._host) in no_ARM_ABI_test_boards: + board = self._host.get_board().split(':')[1] # Remove 'board:' prefix. + if board in no_ARM_ABI_test_boards: if self._abi == 'arm': return True return False @@ -200,6 +201,7 @@ target_class=None, target_method=None, needs_push_media=False, + retry_manual_tests=False, cts_tradefed_args=None, precondition_commands=[], login_precondition_commands=[], @@ -219,6 +221,9 @@ @param target_class: the name of the class to be tested. @param target_method: the name of the method to be tested. @param needs_push_media: need to push test media streams. + @param retry_manual_tests: a flag to track whether manual tests + need to be retried or not. Autotest lab skips manual tests, + while moblab runs them. @param timeout: time after which tradefed can be interrupted. @param precondition_commands: a list of scripts to be run on the dut before the test is run, the scripts must already be installed. @@ -234,6 +239,11 @@ # Retries depend on channel. self._timeoutfactor = None + # TODO(kinaba): Move this logic to tradefed_test so that cheets_GTS can + # deal with manual tests. + if not retry_manual_tests: + self._waivers.update(self._manual_tests) + test_command, test_name = self.generate_test_command(target_module, target_plan, target_class, @@ -244,3 +254,4 @@ target_plan, needs_push_media, _CTS_URI, login_precondition_commands, precondition_commands) +
diff --git a/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.arm b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.arm new file mode 100644 index 0000000..3f47752 --- /dev/null +++ b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.arm
@@ -0,0 +1,6 @@ +CtsAlarmClockTestCases +CtsCallLogTestCases +CtsCarTestCases +CtsSampleDeviceTestCases +CtsSampleHostTestCases +CtsSystemUiTestCases
diff --git a/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.x86 b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.x86 new file mode 100644 index 0000000..3f47752 --- /dev/null +++ b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.x86
@@ -0,0 +1,6 @@ +CtsAlarmClockTestCases +CtsCallLogTestCases +CtsCarTestCases +CtsSampleDeviceTestCases +CtsSampleHostTestCases +CtsSystemUiTestCases
diff --git a/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.yaml b/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.yaml deleted file mode 100644 index 82c4026..0000000 --- a/server/site_tests/cheets_CTS_N/notest_modules/notest_packages.yaml +++ /dev/null
@@ -1,6 +0,0 @@ -CtsAlarmClockTestCases: [all] -CtsCallLogTestCases: [all] -CtsCarTestCases: [all] -CtsSampleDeviceTestCases: [all] -CtsSampleHostTestCases: [all] -CtsSystemUiTestCases: [all]
diff --git a/server/site_tests/cheets_GTS/notest_modules/notest_modules.arm b/server/site_tests/cheets_GTS/notest_modules/notest_modules.arm new file mode 100644 index 0000000..192f25e --- /dev/null +++ b/server/site_tests/cheets_GTS/notest_modules/notest_modules.arm
@@ -0,0 +1,5 @@ +GtsAccountsHostTestCases +GtsDexModuleRegistrationTestCases +GtsSampleDeviceTestCases +GtsSampleDynamicConfigTestCases +GtsSampleHostTestCases
diff --git a/server/site_tests/cheets_GTS/notest_modules/notest_modules.x86 b/server/site_tests/cheets_GTS/notest_modules/notest_modules.x86 new file mode 100644 index 0000000..192f25e --- /dev/null +++ b/server/site_tests/cheets_GTS/notest_modules/notest_modules.x86
@@ -0,0 +1,5 @@ +GtsAccountsHostTestCases +GtsDexModuleRegistrationTestCases +GtsSampleDeviceTestCases +GtsSampleDynamicConfigTestCases +GtsSampleHostTestCases
diff --git a/server/site_tests/cheets_GTS/notest_modules/notest_modules.yaml b/server/site_tests/cheets_GTS/notest_modules/notest_modules.yaml deleted file mode 100644 index e6ccc50..0000000 --- a/server/site_tests/cheets_GTS/notest_modules/notest_modules.yaml +++ /dev/null
@@ -1,5 +0,0 @@ -GtsAccountsHostTestCases: [all] -GtsDexModuleRegistrationTestCases: [all] -GtsSampleDeviceTestCases: [all] -GtsSampleDynamicConfigTestCases: [all] -GtsSampleHostTestCases: [all]