| #!/usr/bin/python2 |
| # |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| |
| """Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" |
| |
| from __future__ import absolute_import |
| from __future__ import division |
| from __future__ import print_function |
| |
| import collections |
| from collections import OrderedDict |
| import mock |
| import mox |
| import os |
| import six |
| from six.moves import range |
| from six.moves import zip |
| import shutil |
| import tempfile |
| import unittest |
| |
| import common |
| |
| from autotest_lib.client.common_lib import base_job |
| from autotest_lib.client.common_lib import control_data |
| from autotest_lib.client.common_lib import error |
| from autotest_lib.client.common_lib import priorities |
| from autotest_lib.client.common_lib import utils |
| from autotest_lib.client.common_lib.cros import dev_server |
| from autotest_lib.server import frontend |
| from autotest_lib.server.cros import provision |
| from autotest_lib.server.cros.dynamic_suite import control_file_getter |
| from autotest_lib.server.cros.dynamic_suite import constants |
| from autotest_lib.server.cros.dynamic_suite import job_status |
| from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase |
| from autotest_lib.server.cros.dynamic_suite import suite_common |
| from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains |
| from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData |
| from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob |
| from autotest_lib.server.cros.dynamic_suite.fakes import FakeMultiprocessingPool |
| from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler |
| from autotest_lib.server.cros.dynamic_suite.suite import Suite |
| |
| |
| class SuiteTest(mox.MoxTestBase): |
| """Unit tests for dynamic_suite Suite class. |
| |
| @var _BUILDS: fake build |
| @var _TAG: fake suite tag |
| """ |
| |
| _BOARD = 'board:board' |
| _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', |
| provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} |
| _TAG = 'au' |
| _ATTR = {'attr:attr'} |
| _DEVSERVER_HOST = 'http://dontcare:8080' |
| _FAKE_JOB_ID = 10 |
| |
| |
| def setUp(self): |
| """Setup.""" |
| super(SuiteTest, self).setUp() |
| self.maxDiff = None |
| self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH |
| suite_common.ENABLE_CONTROLS_IN_BATCH = False |
| self.afe = self.mox.CreateMock(frontend.AFE) |
| self.tko = self.mox.CreateMock(frontend.TKO) |
| |
| self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) |
| |
| self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) |
| self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) |
| |
| self.files = OrderedDict( |
| [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', |
| 'FAST', job_retries=None)), |
| ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', |
| 'SHORT', dependencies=['feta'])), |
| ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', |
| 'MEDIUM')), |
| ('four', FakeControlData('other', self._ATTR, 'data_four', |
| 'LONG', dependencies=['arugula'])), |
| ('five', FakeControlData(self._TAG, {'other'}, 'data_five', |
| 'LONG', dependencies=['arugula', |
| 'caligula'])), |
| ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', |
| 'LENGTHY')), |
| ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', |
| 'FAST', job_retries=1))]) |
| |
| self.files_to_filter = { |
| 'with/deps/...': FakeControlData(self._TAG, self._ATTR, |
| 'gets filtered'), |
| 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, |
| 'gets filtered')} |
| |
| |
| def tearDown(self): |
| """Teardown.""" |
| suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch |
| super(SuiteTest, self).tearDown() |
| shutil.rmtree(self.tmpdir, ignore_errors=True) |
| |
| |
| def expect_control_file_parsing(self, suite_name=_TAG): |
| """Expect an attempt to parse the 'control files' in |self.files|. |
| |
| @param suite_name: The suite name to parse control files for. |
| """ |
| all_files = list(self.files.keys()) + list(self.files_to_filter.keys()) |
| self._set_control_file_parsing_expectations(False, all_files, |
| self.files, suite_name) |
| |
| |
| def _set_control_file_parsing_expectations(self, already_stubbed, |
| file_list, files_to_parse, |
| suite_name): |
| """Expect an attempt to parse the 'control files' in |files|. |
| |
| @param already_stubbed: parse_control_string already stubbed out. |
| @param file_list: the files the dev server returns |
| @param files_to_parse: the {'name': FakeControlData} dict of files we |
| expect to get parsed. |
| """ |
| if not already_stubbed: |
| self.mox.StubOutWithMock(control_data, 'parse_control_string') |
| |
| self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') |
| suite_common.multiprocessing.Pool( |
| processes=suite_common.get_process_limit()).AndReturn( |
| FakeMultiprocessingPool()) |
| |
| self.getter.get_control_file_list( |
| suite_name=suite_name).AndReturn(file_list) |
| for file, data in six.iteritems(files_to_parse): |
| self.getter.get_control_file_contents( |
| file).InAnyOrder().AndReturn(data.string) |
| control_data.parse_control_string( |
| data.string, |
| raise_warnings=True, |
| path=file).InAnyOrder().AndReturn(data) |
| |
| |
| def expect_control_file_parsing_in_batch(self, suite_name=_TAG): |
| """Expect an attempt to parse the contents of all control files in |
| |self.files| and |self.files_to_filter|, form them to a dict. |
| |
| @param suite_name: The suite name to parse control files for. |
| """ |
| self.getter = self.mox.CreateMock(control_file_getter.DevServerGetter) |
| self.mox.StubOutWithMock(control_data, 'parse_control_string') |
| |
| self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') |
| suite_common.multiprocessing.Pool( |
| processes=suite_common.get_process_limit()).AndReturn( |
| FakeMultiprocessingPool()) |
| |
| suite_info = {} |
| for k, v in six.iteritems(self.files): |
| suite_info[k] = v.string |
| control_data.parse_control_string( |
| v.string, |
| raise_warnings=True, |
| path=k).InAnyOrder().AndReturn(v) |
| for k, v in six.iteritems(self.files_to_filter): |
| suite_info[k] = v.string |
| self.getter._dev_server = self._DEVSERVER_HOST |
| self.getter.get_suite_info( |
| suite_name=suite_name).AndReturn(suite_info) |
| |
| |
| def testFindAllTestInBatch(self): |
| """Test switch on enable_getting_controls_in_batch for function |
| find_all_test.""" |
| self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH |
| self.expect_control_file_parsing_in_batch() |
| suite_common.ENABLE_CONTROLS_IN_BATCH = True |
| |
| self.mox.ReplayAll() |
| |
| predicate = lambda d: d.suite == self._TAG |
| tests = SuiteBase.find_and_parse_tests(self.getter, |
| predicate, |
| self._TAG) |
| self.assertEquals(len(tests), 6) |
| self.assertTrue(self.files['one'] in tests) |
| self.assertTrue(self.files['two'] in tests) |
| self.assertTrue(self.files['three'] in tests) |
| self.assertTrue(self.files['five'] in tests) |
| self.assertTrue(self.files['six'] in tests) |
| self.assertTrue(self.files['seven'] in tests) |
| suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch |
| |
| |
| def testFindAndParseStableTests(self): |
| """Should find only tests that match a predicate.""" |
| self.expect_control_file_parsing() |
| self.mox.ReplayAll() |
| |
| predicate = lambda d: d.text == self.files['two'].string |
| tests = SuiteBase.find_and_parse_tests(self.getter, |
| predicate, |
| self._TAG) |
| self.assertEquals(len(tests), 1) |
| self.assertEquals(tests[0], self.files['two']) |
| |
| |
| def testFindSuiteSyntaxErrors(self): |
| """Check all control files for syntax errors. |
| |
| This test actually parses all control files in the autotest directory |
| for syntax errors, by using the un-forgiving parser and pretending to |
| look for all control files with the suite attribute. |
| """ |
| autodir = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), '..', '..', '..')) |
| fs_getter = SuiteBase.create_fs_getter(autodir) |
| predicate = lambda t: hasattr(t, 'suite') |
| SuiteBase.find_and_parse_tests(fs_getter, predicate, |
| forgiving_parser=False) |
| |
| |
| def testFindAndParseTestsSuite(self): |
| """Should find all tests that match a predicate.""" |
| self.expect_control_file_parsing() |
| self.mox.ReplayAll() |
| |
| predicate = lambda d: d.suite == self._TAG |
| tests = SuiteBase.find_and_parse_tests(self.getter, |
| predicate, |
| self._TAG) |
| self.assertEquals(len(tests), 6) |
| self.assertTrue(self.files['one'] in tests) |
| self.assertTrue(self.files['two'] in tests) |
| self.assertTrue(self.files['three'] in tests) |
| self.assertTrue(self.files['five'] in tests) |
| self.assertTrue(self.files['six'] in tests) |
| self.assertTrue(self.files['seven'] in tests) |
| |
| |
| def testFindAndParseTestsAttr(self): |
| """Should find all tests that match a predicate.""" |
| self.expect_control_file_parsing() |
| self.mox.ReplayAll() |
| |
| predicate = SuiteBase.matches_attribute_expression_predicate('attr:attr') |
| tests = SuiteBase.find_and_parse_tests(self.getter, |
| predicate, |
| self._TAG) |
| self.assertEquals(len(tests), 6) |
| self.assertTrue(self.files['one'] in tests) |
| self.assertTrue(self.files['two'] in tests) |
| self.assertTrue(self.files['three'] in tests) |
| self.assertTrue(self.files['four'] in tests) |
| self.assertTrue(self.files['six'] in tests) |
| self.assertTrue(self.files['seven'] in tests) |
| |
| |
| def testAdHocSuiteCreation(self): |
| """Should be able to schedule an ad-hoc suite by specifying |
| a single test name.""" |
| self.expect_control_file_parsing(suite_name='ad_hoc_suite') |
| self.mox.ReplayAll() |
| predicate = SuiteBase.test_name_equals_predicate('name-data_five') |
| suite = Suite.create_from_predicates([predicate], self._BUILDS, |
| self._BOARD, devserver=None, |
| cf_getter=self.getter, |
| afe=self.afe, tko=self.tko) |
| |
| self.assertFalse(self.files['one'] in suite.tests) |
| self.assertFalse(self.files['two'] in suite.tests) |
| self.assertFalse(self.files['four'] in suite.tests) |
| self.assertTrue(self.files['five'] in suite.tests) |
| |
| |
| def mock_control_file_parsing(self): |
| """Fake out find_and_parse_tests(), returning content from |self.files|. |
| """ |
| for test in self.files.values(): |
| test.text = test.string # mimic parsing. |
| self.mox.StubOutWithMock(SuiteBase, 'find_and_parse_tests') |
| SuiteBase.find_and_parse_tests( |
| mox.IgnoreArg(), |
| mox.IgnoreArg(), |
| mox.IgnoreArg(), |
| forgiving_parser=True, |
| run_prod_code=False, |
| test_args=None).AndReturn(list(self.files.values())) |
| |
| |
| def expect_job_scheduling(self, recorder, |
| tests_to_skip=[], ignore_deps=False, |
| raises=False, suite_deps=[], suite=None, |
| extra_keyvals={}): |
| """Expect jobs to be scheduled for 'tests' in |self.files|. |
| |
| @param recorder: object with a record_entry to be used to record test |
| results. |
| @param tests_to_skip: [list, of, test, names] that we expect to skip. |
| @param ignore_deps: If true, ignore tests' dependencies. |
| @param raises: If True, expect exceptions. |
| @param suite_deps: If True, add suite level dependencies. |
| @param extra_keyvals: Extra keyvals set to tests. |
| """ |
| record_job_id = suite and suite._results_dir |
| if record_job_id: |
| self.mox.StubOutWithMock(suite, '_remember_job_keyval') |
| recorder.record_entry( |
| StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), |
| log_in_subdir=False) |
| tests = list(self.files.values()) |
| n = 1 |
| for test in tests: |
| if test.name in tests_to_skip: |
| continue |
| dependencies = [] |
| if not ignore_deps: |
| dependencies.extend(test.dependencies) |
| if suite_deps: |
| dependencies.extend(suite_deps) |
| dependencies.append(self._BOARD) |
| build = self._BUILDS[provision.CROS_VERSION_PREFIX] |
| keyvals = { |
| 'build': build, |
| 'suite': self._TAG, |
| 'builds': SuiteTest._BUILDS, |
| 'experimental':test.experimental, |
| } |
| keyvals.update(extra_keyvals) |
| job_mock = self.afe.create_job( |
| control_file=test.text, |
| name=mox.And(mox.StrContains(build), |
| mox.StrContains(test.name)), |
| control_type=mox.IgnoreArg(), |
| meta_hosts=[self._BOARD], |
| dependencies=dependencies, |
| keyvals=keyvals, |
| max_runtime_mins=24*60, |
| timeout_mins=1440, |
| parent_job_id=None, |
| reboot_before=mox.IgnoreArg(), |
| run_reset=mox.IgnoreArg(), |
| priority=priorities.Priority.DEFAULT, |
| synch_count=test.sync_count, |
| require_ssp=test.require_ssp |
| ) |
| if raises: |
| job_mock.AndRaise(error.NoEligibleHostException()) |
| recorder.record_entry( |
| StatusContains.CreateFromStrings('START', test.name), |
| log_in_subdir=False) |
| recorder.record_entry( |
| StatusContains.CreateFromStrings('TEST_NA', test.name), |
| log_in_subdir=False) |
| recorder.record_entry( |
| StatusContains.CreateFromStrings('END', test.name), |
| log_in_subdir=False) |
| else: |
| fake_job = FakeJob(id=n) |
| job_mock.AndReturn(fake_job) |
| if record_job_id: |
| suite._remember_job_keyval(fake_job) |
| n += 1 |
| |
| |
| def testScheduleTestsAndRecord(self): |
| """Should schedule stable and experimental tests with the AFE.""" |
| name_list = ['name-data_one', 'name-data_two', 'name-data_three', |
| 'name-data_four', 'name-data_five', 'name-data_six', |
| 'name-data_seven'] |
| keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, |
| constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} |
| |
| self.mock_control_file_parsing() |
| self.mox.ReplayAll() |
| suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, |
| self.devserver, |
| afe=self.afe, tko=self.tko, |
| results_dir=self.tmpdir) |
| self.mox.ResetAll() |
| recorder = self.mox.CreateMock(base_job.base_job) |
| self.expect_job_scheduling(recorder, suite=suite) |
| |
| self.mox.StubOutWithMock(utils, 'write_keyval') |
| utils.write_keyval(self.tmpdir, keyval_dict) |
| self.mox.ReplayAll() |
| suite.schedule(recorder.record_entry) |
| for job in suite._jobs: |
| self.assertTrue(hasattr(job, 'test_name')) |
| |
| |
| def testScheduleTests(self): |
| """Should schedule tests with the AFE.""" |
| name_list = ['name-data_one', 'name-data_two', 'name-data_three', |
| 'name-data_four', 'name-data_five', 'name-data_six', |
| 'name-data_seven'] |
| keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), |
| constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} |
| |
| self.mock_control_file_parsing() |
| recorder = self.mox.CreateMock(base_job.base_job) |
| self.expect_job_scheduling(recorder) |
| self.mox.StubOutWithMock(utils, 'write_keyval') |
| utils.write_keyval(None, keyval_dict) |
| |
| self.mox.ReplayAll() |
| suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, |
| self.devserver, |
| afe=self.afe, tko=self.tko) |
| suite.schedule(recorder.record_entry) |
| |
| |
| def testScheduleTestsIgnoreDeps(self): |
| """Test scheduling tests ignoring deps.""" |
| name_list = ['name-data_one', 'name-data_two', 'name-data_three', |
| 'name-data_four', 'name-data_five', 'name-data_six', |
| 'name-data_seven'] |
| keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), |
| constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} |
| |
| self.mock_control_file_parsing() |
| recorder = self.mox.CreateMock(base_job.base_job) |
| self.expect_job_scheduling(recorder, ignore_deps=True) |
| self.mox.StubOutWithMock(utils, 'write_keyval') |
| utils.write_keyval(None, keyval_dict) |
| |
| self.mox.ReplayAll() |
| suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, |
| self.devserver, |
| afe=self.afe, tko=self.tko, |
| ignore_deps=True) |
| suite.schedule(recorder.record_entry) |
| |
| |
| def testScheduleUnrunnableTestsTESTNA(self): |
| """Tests which fail to schedule should be TEST_NA.""" |
| # Since all tests will be fail to schedule, the num of scheduled tests |
| # will be zero. |
| name_list = [] |
| keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, |
| constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} |
| |
| self.mock_control_file_parsing() |
| recorder = self.mox.CreateMock(base_job.base_job) |
| self.expect_job_scheduling(recorder, raises=True) |
| self.mox.StubOutWithMock(utils, 'write_keyval') |
| utils.write_keyval(None, keyval_dict) |
| self.mox.ReplayAll() |
| suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, |
| self.devserver, |
| afe=self.afe, tko=self.tko) |
| suite.schedule(recorder.record_entry) |
| |
| |
| def testRetryMapAfterScheduling(self): |
| """Test job-test and test-job mapping are correctly updated.""" |
| name_list = ['name-data_one', 'name-data_two', 'name-data_three', |
| 'name-data_four', 'name-data_five', 'name-data_six', |
| 'name-data_seven'] |
| keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, |
| constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} |
| |
| self.mock_control_file_parsing() |
| recorder = self.mox.CreateMock(base_job.base_job) |
| self.expect_job_scheduling(recorder) |
| self.mox.StubOutWithMock(utils, 'write_keyval') |
| utils.write_keyval(None, keyval_dict) |
| |
| all_files = list(self.files.items()) |
| # Sort tests in self.files so that they are in the same |
| # order as they are scheduled. |
| expected_retry_map = {} |
| for n in range(len(all_files)): |
| test = all_files[n][1] |
| job_id = n + 1 |
| job_retries = 1 if test.job_retries is None else test.job_retries |
| if job_retries > 0: |
| expected_retry_map[job_id] = { |
| 'state': RetryHandler.States.NOT_ATTEMPTED, |
| 'retry_max': job_retries} |
| |
| self.mox.ReplayAll() |
| suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, |
| self.devserver, |
| afe=self.afe, tko=self.tko, |
| job_retry=True) |
| suite.schedule(recorder.record_entry) |
| |
| self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) |
| |
| |
| def testSuiteMaxRetries(self): |
| """Test suite max retries.""" |
| name_list = ['name-data_one', 'name-data_two', 'name-data_three', |
| 'name-data_four', 'name-data_five', |
| 'name-data_six', 'name-data_seven'] |
| keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, |
| constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} |
| |
| self.mock_control_file_parsing() |
| recorder = self.mox.CreateMock(base_job.base_job) |
| self.expect_job_scheduling(recorder) |
| self.mox.StubOutWithMock(utils, 'write_keyval') |
| utils.write_keyval(None, keyval_dict) |
| self.mox.ReplayAll() |
| suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, |
| self.devserver, |
| afe=self.afe, tko=self.tko, |
| job_retry=True, max_retries=1) |
| suite.schedule(recorder.record_entry) |
| self.assertEqual(suite._retry_handler._max_retries, 1) |
| # Find the job_id of the test that allows retry |
| job_id = next(six.iterkeys(suite._retry_handler._retry_map)) |
| suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) |
| self.assertEqual(suite._retry_handler._max_retries, 0) |
| |
| |
| def testSuiteDependencies(self): |
| """Should add suite dependencies to tests scheduled.""" |
| name_list = ['name-data_one', 'name-data_two', 'name-data_three', |
| 'name-data_four', 'name-data_five', 'name-data_six', |
| 'name-data_seven'] |
| keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), |
| constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} |
| |
| self.mock_control_file_parsing() |
| recorder = self.mox.CreateMock(base_job.base_job) |
| self.expect_job_scheduling(recorder, suite_deps=['extra']) |
| self.mox.StubOutWithMock(utils, 'write_keyval') |
| utils.write_keyval(None, keyval_dict) |
| |
| self.mox.ReplayAll() |
| suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, |
| self.devserver, extra_deps=['extra'], |
| afe=self.afe, tko=self.tko) |
| suite.schedule(recorder.record_entry) |
| |
| |
| def testInheritedKeyvals(self): |
| """Tests should inherit some allowlisted job keyvals.""" |
| # Only keyvals in constants.INHERITED_KEYVALS are inherited to tests. |
| job_keyvals = { |
| constants.KEYVAL_CIDB_BUILD_ID: '111', |
| constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', |
| constants.KEYVAL_BRANCH: 'dummy_branch', |
| constants.KEYVAL_BUILDER_NAME: 'model-dummy', |
| constants.KEYVAL_MASTER_BUILDER_NAME: 'master-dummy', |
| 'your': 'name', |
| } |
| test_keyvals = { |
| constants.KEYVAL_CIDB_BUILD_ID: '111', |
| constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', |
| constants.KEYVAL_BRANCH: 'dummy_branch', |
| constants.KEYVAL_BUILDER_NAME: 'model-dummy', |
| constants.KEYVAL_MASTER_BUILDER_NAME: 'master-dummy', |
| } |
| |
| self.mock_control_file_parsing() |
| recorder = self.mox.CreateMock(base_job.base_job) |
| self.expect_job_scheduling( |
| recorder, |
| extra_keyvals=test_keyvals) |
| self.mox.StubOutWithMock(utils, 'write_keyval') |
| utils.write_keyval(None, job_keyvals) |
| utils.write_keyval(None, mox.IgnoreArg()) |
| |
| self.mox.ReplayAll() |
| suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, |
| self.devserver, |
| afe=self.afe, tko=self.tko, |
| job_keyvals=job_keyvals) |
| suite.schedule(recorder.record_entry) |
| |
| |
| def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): |
| """Create a Suite, using mocked tests and control file contents. |
| |
| @return Suite object, after mocking out behavior needed to create it. |
| """ |
| self.result_reporter = _MemoryResultReporter() |
| self.expect_control_file_parsing() |
| self.mox.ReplayAll() |
| suite = Suite.create_from_name( |
| self._TAG, |
| self._BUILDS, |
| self._BOARD, |
| self.devserver, |
| self.getter, |
| afe=self.afe, |
| tko=self.tko, |
| file_bugs=file_bugs, |
| job_retry=True, |
| result_reporter=self.result_reporter, |
| ) |
| self.mox.ResetAll() |
| return suite |
| |
| |
| def _createSuiteMockResults(self, results_dir=None, result_status='FAIL'): |
| """Create a suite, returned a set of mocked results to expect. |
| |
| @param results_dir: A mock results directory. |
| @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. |
| |
| @return List of mocked results to wait on. |
| """ |
| self.suite = self._createSuiteWithMockedTestsAndControlFiles( |
| file_bugs=True) |
| self.suite._results_dir = results_dir |
| test_report = self._get_bad_test_report(result_status) |
| test_predicates = test_report.predicates |
| test_fallout = test_report.fallout |
| |
| self.recorder = self.mox.CreateMock(base_job.base_job) |
| self.recorder.record_entry = self.mox.CreateMock( |
| base_job.base_job.record_entry) |
| self._mock_recorder_with_results([test_predicates], self.recorder) |
| return [test_predicates, test_fallout] |
| |
| |
| def _mock_recorder_with_results(self, results, recorder): |
| """ |
| Checks that results are recoded in order, eg: |
| START, (status, name, reason) END |
| |
| @param results: list of results |
| @param recorder: status recorder |
| """ |
| for result in results: |
| status = result[0] |
| test_name = result[1] |
| recorder.record_entry( |
| StatusContains.CreateFromStrings('START', test_name), |
| log_in_subdir=False) |
| recorder.record_entry( |
| StatusContains.CreateFromStrings(*result), |
| log_in_subdir=False).InAnyOrder('results') |
| recorder.record_entry( |
| StatusContains.CreateFromStrings('END %s' % status, test_name), |
| log_in_subdir=False) |
| |
| |
| def schedule_and_expect_these_results(self, suite, results, recorder): |
| """Create mox stubs for call to suite.schedule and |
| job_status.wait_for_results |
| |
| @param suite: suite object for which to stub out schedule(...) |
| @param results: results object to be returned from |
| job_stats_wait_for_results(...) |
| @param recorder: mocked recorder object to replay status messages |
| """ |
| def result_generator(results): |
| """A simple generator which generates results as Status objects. |
| |
| This generator handles 'send' by simply ignoring it. |
| |
| @param results: results object to be returned from |
| job_stats_wait_for_results(...) |
| @yield: job_status.Status objects. |
| """ |
| results = [job_status.Status(*r) for r in results] |
| for r in results: |
| new_input = (yield r) |
| if new_input: |
| yield None |
| |
| self.mox.StubOutWithMock(suite, 'schedule') |
| suite.schedule(recorder.record_entry) |
| suite._retry_handler = RetryHandler({}) |
| |
| waiter_patch = mock.patch.object( |
| job_status.JobResultWaiter, 'wait_for_results', autospec=True) |
| waiter_mock = waiter_patch.start() |
| waiter_mock.return_value = result_generator(results) |
| self.addCleanup(waiter_patch.stop) |
| |
| |
| def testRunAndWaitSuccess(self): |
| """Should record successful results.""" |
| suite = self._createSuiteWithMockedTestsAndControlFiles() |
| |
| recorder = self.mox.CreateMock(base_job.base_job) |
| |
| results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] |
| self._mock_recorder_with_results(results, recorder) |
| self.schedule_and_expect_these_results(suite, results, recorder) |
| self.mox.ReplayAll() |
| |
| suite.schedule(recorder.record_entry) |
| suite.wait(recorder.record_entry) |
| |
| |
| def testRunAndWaitFailure(self): |
| """Should record failure to gather results.""" |
| suite = self._createSuiteWithMockedTestsAndControlFiles() |
| |
| recorder = self.mox.CreateMock(base_job.base_job) |
| recorder.record_entry( |
| StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), |
| log_in_subdir=False) |
| |
| self.mox.StubOutWithMock(suite, 'schedule') |
| suite.schedule(recorder.record_entry) |
| self.mox.ReplayAll() |
| |
| with mock.patch.object( |
| job_status.JobResultWaiter, 'wait_for_results', |
| autospec=True) as wait_mock: |
| wait_mock.side_effect = Exception |
| suite.schedule(recorder.record_entry) |
| suite.wait(recorder.record_entry) |
| |
| |
| def testRunAndWaitScheduleFailure(self): |
| """Should record failure to schedule jobs.""" |
| suite = self._createSuiteWithMockedTestsAndControlFiles() |
| |
| recorder = self.mox.CreateMock(base_job.base_job) |
| recorder.record_entry( |
| StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), |
| log_in_subdir=False) |
| |
| recorder.record_entry( |
| StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), |
| log_in_subdir=False) |
| |
| self.mox.StubOutWithMock(suite._job_creator, 'create_job') |
| suite._job_creator.create_job( |
| mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( |
| Exception('Expected during test.')) |
| self.mox.ReplayAll() |
| |
| suite.schedule(recorder.record_entry) |
| suite.wait(recorder.record_entry) |
| |
| |
| def testGetTestsSortedByTime(self): |
| """Should find all tests and sorted by TIME setting.""" |
| self.expect_control_file_parsing() |
| self.mox.ReplayAll() |
| # Get all tests. |
| tests = SuiteBase.find_and_parse_tests(self.getter, |
| lambda d: True, |
| self._TAG) |
| self.assertEquals(len(tests), 7) |
| times = [control_data.ControlData.get_test_time_index(test.time) |
| for test in tests] |
| self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), |
| 'Tests are not ordered correctly.') |
| |
| |
| def _get_bad_test_report(self, result_status='FAIL'): |
| """ |
| Fetch the predicates of a failing test, and the parameters |
| that are a fallout of this test failing. |
| """ |
| predicates = collections.namedtuple('predicates', |
| 'status, testname, reason') |
| fallout = collections.namedtuple('fallout', |
| ('time_start, time_end, job_id,' |
| 'username, hostname')) |
| test_report = collections.namedtuple('test_report', |
| 'predicates, fallout') |
| return test_report(predicates(result_status, 'bad_test', |
| 'dreadful_reason'), |
| fallout('2014-01-01 01:01:01', 'None', |
| self._FAKE_JOB_ID, 'user', 'myhost')) |
| |
| |
| def testJobRetryTestFail(self): |
| """Test retry works.""" |
| test_to_retry = self.files['seven'] |
| fake_new_job_id = self._FAKE_JOB_ID + 1 |
| fake_job = FakeJob(id=self._FAKE_JOB_ID) |
| fake_new_job = FakeJob(id=fake_new_job_id) |
| |
| test_results = self._createSuiteMockResults() |
| self.schedule_and_expect_these_results( |
| self.suite, |
| [test_results[0] + test_results[1]], |
| self.recorder) |
| self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') |
| self.suite._job_creator.create_job( |
| test_to_retry, |
| retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) |
| self.mox.ReplayAll() |
| self.suite.schedule(self.recorder.record_entry) |
| self.suite._retry_handler._retry_map = { |
| self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, |
| 'retry_max': 1} |
| } |
| self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry |
| self.suite.wait(self.recorder.record_entry) |
| expected_retry_map = { |
| self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, |
| 'retry_max': 1}, |
| fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, |
| 'retry_max': 0} |
| } |
| # Check retry map is correctly updated |
| self.assertEquals(self.suite._retry_handler._retry_map, |
| expected_retry_map) |
| # Check _jobs_to_tests is correctly updated |
| self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], |
| test_to_retry) |
| |
| |
| def testJobRetryTestWarn(self): |
| """Test that no retry is scheduled if test warns.""" |
| test_to_retry = self.files['seven'] |
| fake_job = FakeJob(id=self._FAKE_JOB_ID) |
| test_results = self._createSuiteMockResults(result_status='WARN') |
| self.schedule_and_expect_these_results( |
| self.suite, |
| [test_results[0] + test_results[1]], |
| self.recorder) |
| self.mox.ReplayAll() |
| self.suite.schedule(self.recorder.record_entry) |
| self.suite._retry_handler._retry_map = { |
| self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, |
| 'retry_max': 1} |
| } |
| self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry |
| expected_jobs_to_tests = self.suite._jobs_to_tests.copy() |
| expected_retry_map = self.suite._retry_handler._retry_map.copy() |
| self.suite.wait(self.recorder.record_entry) |
| self.assertTrue(self.result_reporter.results) |
| # Check retry map and _jobs_to_tests, ensure no retry was scheduled. |
| self.assertEquals(self.suite._retry_handler._retry_map, |
| expected_retry_map) |
| self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) |
| |
| |
| def testFailedJobRetry(self): |
| """Make sure the suite survives even if the retry failed.""" |
| test_to_retry = self.files['seven'] |
| fake_job = FakeJob(id=self._FAKE_JOB_ID) |
| |
| test_results = self._createSuiteMockResults() |
| self.schedule_and_expect_these_results( |
| self.suite, |
| [test_results[0] + test_results[1]], |
| self.recorder) |
| self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') |
| self.suite._job_creator.create_job( |
| test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( |
| error.RPCException('Expected during test')) |
| # Do not file a bug. |
| self.mox.StubOutWithMock(self.suite, '_should_report') |
| self.suite._should_report(mox.IgnoreArg()).AndReturn(False) |
| |
| self.mox.ReplayAll() |
| |
| self.suite.schedule(self.recorder.record_entry) |
| self.suite._retry_handler._retry_map = { |
| self._FAKE_JOB_ID: { |
| 'state': RetryHandler.States.NOT_ATTEMPTED, |
| 'retry_max': 1}} |
| self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry |
| self.suite.wait(self.recorder.record_entry) |
| expected_retry_map = { |
| self._FAKE_JOB_ID: { |
| 'state': RetryHandler.States.ATTEMPTED, |
| 'retry_max': 1}} |
| expected_jobs_to_tests = self.suite._jobs_to_tests.copy() |
| self.assertEquals(self.suite._retry_handler._retry_map, |
| expected_retry_map) |
| self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) |
| |
| |
| class _MemoryResultReporter(SuiteBase._ResultReporter): |
| """Reporter that stores results internally for testing.""" |
| def __init__(self): |
| self.results = [] |
| |
| def report(self, result): |
| """Reports the result by storing it internally.""" |
| self.results.append(result) |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |
