| # Copyright 2016 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # repohooks/pre-upload.py currently does not run pylint. But for developers who |
| # want to check their code manually we disable several harmless pylint warnings |
| # which just distract from more serious remaining issues. |
| # |
| # The instance variable _android_cts is not defined in __init__(). |
| # pylint: disable=attribute-defined-outside-init |
| # |
| # Many short variable names don't follow the naming convention. |
| # pylint: disable=invalid-name |
| |
| import logging |
| import os |
| |
| from autotest_lib.server import utils |
| from autotest_lib.server.cros import tradefed_test |
| |
| # Maximum default time allowed for each individual CTS module. |
| _CTS_TIMEOUT_SECONDS = 3600 |
| |
| # Public download locations for android cts bundles. |
| _DL_CTS = 'https://dl.google.com/dl/android/cts/' |
| _CTS_URI = { |
| 'arm': _DL_CTS + 'android-cts-7.1_r11-linux_x86-arm.zip', |
| 'x86': _DL_CTS + 'android-cts-7.1_r11-linux_x86-x86.zip', |
| 'media': _DL_CTS + 'android-cts-media-1.4.zip', |
| } |
| |
| |
| class cheets_CTS_N(tradefed_test.TradefedTest): |
| """Sets up tradefed to run CTS tests.""" |
| version = 1 |
| |
| # TODO(bmgordon): Remove kahlee once the bulk of failing tests are fixed. |
| _BOARD_RETRY = {'betty': 0, 'kahlee': 0} |
| _CHANNEL_RETRY = {'dev': 5, 'beta': 5, 'stable': 5} |
| |
| def _get_default_bundle_url(self, bundle): |
| return _CTS_URI[bundle] |
| |
| def _get_tradefed_base_dir(self): |
| return 'android-cts' |
| |
| def _tradefed_run_command(self, |
| module=None, |
| plan=None, |
| session_id=None, |
| test_class=None, |
| test_method=None): |
| """Builds the CTS tradefed 'run' command line. |
| |
| There are five different usages: |
| |
| 1. Test a module: assign the module name via |module|. |
| 2. Test a plan: assign the plan name via |plan|. |
| 3. Continue a session: assign the session ID via |session_id|. |
| 4. Run all test cases of a class: assign the class name via |
| |test_class|. |
| 5. Run a specific test case: assign the class name and method name in |
| |test_class| and |test_method|. |
| |
| @param module: the name of test module to be run. |
| @param plan: name of the plan to be run. |
| @param session_id: tradefed session id to continue. |
| @param test_class: the name of the class of which all test cases will |
| be run. |
| @param test_name: the name of the method of |test_class| to be run. |
| Must be used with |test_class|. |
| @return: list of command tokens for the 'run' command. |
| """ |
| if module is not None: |
| # Run a particular module (used to be called package in M). |
| cmd = ['run', 'commandAndExit', 'cts', '--module', module] |
| if test_class is not None: |
| if test_method is not None: |
| cmd += ['-t', test_class + '#' + test_method] |
| else: |
| cmd += ['-t', test_class] |
| elif plan is not None and session_id is not None: |
| # In 7.1 R2 we can only retry session_id with the original plan. |
| cmd = ['run', 'commandAndExit', 'cts', '--plan', plan, |
| '--retry', '%d' % session_id] |
| elif plan is not None: |
| # TODO(ihf): This needs testing to support media team. |
| cmd = ['run', 'commandAndExit', 'cts', '--plan', plan] |
| else: |
| logging.warning('Running all tests. This can take several days.') |
| cmd = ['run', 'commandAndExit', 'cts'] |
| # We handle media download ourselves in the lab, as lazy as possible. |
| cmd.append('--precondition-arg') |
| cmd.append('skip-media-download') |
| # If we are running outside of the lab we can collect more data. |
| if not utils.is_in_container(): |
| logging.info('Running outside of lab, adding extra debug options.') |
| cmd.append('--log-level-display=DEBUG') |
| cmd.append('--screenshot-on-failure') |
| # TODO(ihf): Add log collection once b/28333587 fixed. |
| #cmd.append('--collect-deqp-logs') |
| # TODO(ihf): Add tradefed_test.adb_keepalive() and remove |
| # --disable-reboot. This might be more efficient. |
| # At early stage, cts-tradefed tries to reboot the device by |
| # "adb reboot" command. In a real Android device case, when the |
| # rebooting is completed, adb connection is re-established |
| # automatically, and cts-tradefed expects that behavior. |
| # However, in ARC, it doesn't work, so the whole test process |
| # is just stuck. Here, disable the feature. |
| cmd.append('--disable-reboot') |
| # Create a logcat file for each individual failure. |
| cmd.append('--logcat-on-failure') |
| return cmd |
| |
| def _get_timeout_factor(self): |
| """Returns the factor to be multiplied to the timeout parameter. |
| The factor is determined by the number of ABIs to run.""" |
| if self._timeoutfactor is None: |
| abilist = self._run('adb', args=('shell', 'getprop', |
| 'ro.product.cpu.abilist')).stdout.split(',') |
| prefix = {'x86': 'x86', 'arm': 'armeabi-'}.get(self._abi) |
| self._timeoutfactor = (1 if prefix is None else |
| sum(1 for abi in abilist if abi.startswith(prefix))) |
| return self._timeoutfactor |
| |
| def _run_tradefed(self, commands): |
| """Kick off CTS. |
| |
| @param commands: the command(s) to pass to CTS. |
| @param datetime_id: For 'continue' datetime of previous run is known. |
| @return: The result object from utils.run. |
| """ |
| cts_tradefed = os.path.join(self._repository, 'tools', 'cts-tradefed') |
| for command in commands: |
| logging.info('RUN: ./cts-tradefed %s', ' '.join(command)) |
| output = self._run( |
| cts_tradefed, |
| args=tuple(command), |
| timeout=self._timeout * self._get_timeout_factor(), |
| verbose=True, |
| ignore_status=False, |
| # Make sure to tee tradefed stdout/stderr to autotest logs |
| # continuously during the test run. |
| stdout_tee=utils.TEE_TO_LOGS, |
| stderr_tee=utils.TEE_TO_LOGS) |
| logging.info('END: ./cts-tradefed %s\n', ' '.join(command)) |
| return output |
| |
| def _should_skip_test(self): |
| """Some tests are expected to fail and are skipped.""" |
| # newbie and novato are x86 VMs without binary translation. Skip the ARM |
| # tests. |
| no_ARM_ABI_test_boards = ('newbie', 'novato', 'novato-arc64') |
| board = self._host.get_board().split(':')[1] # Remove 'board:' prefix. |
| if board in no_ARM_ABI_test_boards: |
| if self._abi == 'arm': |
| return True |
| return False |
| |
| def generate_test_command(self, target_module, target_plan, target_class, |
| target_method, cts_tradefed_args, session_id=0): |
| """Generates the CTS command and name to use based on test arguments. |
| |
| @param target_module: the name of test module to run. |
| @param target_plan: the name of the test plan to run. |
| @param target_class: the name of the class to be tested. |
| @param target_method: the name of the method to be tested. |
| @param cts_tradefed_args: a list of args to pass to tradefed. |
| @param session_id: tradefed session_id. |
| """ |
| if target_module is not None: |
| if target_class is not None: |
| test_name = 'testcase.%s' % target_class |
| if target_method is not None: |
| test_name += '.' + target_method |
| test_command = self._tradefed_run_command( |
| module=target_module, |
| test_class=target_class, |
| test_method=target_method, |
| session_id=session_id) |
| else: |
| test_name = 'module.%s' % target_module |
| test_command = self._tradefed_run_command( |
| module=target_module, session_id=session_id) |
| elif target_plan is not None: |
| test_name = 'plan.%s' % target_plan |
| test_command = self._tradefed_run_command( |
| plan=target_plan, session_id=session_id) |
| elif cts_tradefed_args is not None: |
| test_name = 'run tradefed %s' % ' '.join(cts_tradefed_args) |
| test_command = cts_tradefed_args |
| else: |
| test_command = self._tradefed_run_command() |
| test_name = 'all_CTS' |
| |
| return test_command, test_name |
| |
| def run_once(self, |
| target_module=None, |
| target_plan=None, |
| target_class=None, |
| target_method=None, |
| needs_push_media=False, |
| retry_manual_tests=False, |
| cts_tradefed_args=None, |
| precondition_commands=[], |
| login_precondition_commands=[], |
| timeout=_CTS_TIMEOUT_SECONDS): |
| """Runs the specified CTS once, but with several retries. |
| |
| There are four usages: |
| 1. Test the whole module named |target_module|. |
| 2. Test with a plan named |target_plan|. |
| 3. Run all the test cases of class named |target_class|. |
| 4. Run a specific test method named |target_method| of class |
| |target_class|. |
| 5. Run an arbitrary tradefed command. |
| |
| @param target_module: the name of test module to run. |
| @param target_plan: the name of the test plan to run. |
| @param target_class: the name of the class to be tested. |
| @param target_method: the name of the method to be tested. |
| @param needs_push_media: need to push test media streams. |
| @param retry_manual_tests: a flag to track whether manual tests |
| need to be retried or not. Autotest lab skips manual tests, |
| while moblab runs them. |
| @param timeout: time after which tradefed can be interrupted. |
| @param precondition_commands: a list of scripts to be run on the |
| dut before the test is run, the scripts must already be installed. |
| @param login_precondition_commands: a list of scripts to be run on the |
| dut before the log-in for the test is performed. |
| @param cts_tradefed_args: a list of args to pass to tradefed. |
| """ |
| |
| # On dev and beta channels timeouts are sharp, lenient on stable. |
| self._timeout = timeout |
| if self._get_release_channel == 'stable': |
| self._timeout += 3600 |
| # Retries depend on channel. |
| self._timeoutfactor = None |
| |
| # TODO(kinaba): Move this logic to tradefed_test so that cheets_GTS can |
| # deal with manual tests. |
| if not retry_manual_tests: |
| self._waivers.update(self._manual_tests) |
| |
| test_command, test_name = self.generate_test_command(target_module, |
| target_plan, |
| target_class, |
| target_method, |
| cts_tradefed_args) |
| |
| self._run_tradefed_with_retries(target_module, test_command, test_name, |
| target_plan, needs_push_media, _CTS_URI, |
| login_precondition_commands, |
| precondition_commands) |
| |