| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # Copyright 2019 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Tests for LLVM bisection.""" |
| |
| from __future__ import print_function |
| |
| import json |
| import unittest |
| import unittest.mock as mock |
| |
| from get_llvm_hash import LLVMHash |
| from test_helpers import ArgsOutputTest |
| from test_helpers import CallCountsToMockFunctions |
| from test_helpers import CreateTemporaryJsonFile |
| from test_helpers import WritePrettyJsonFile |
| import llvm_bisection |
| |
| |
| class LLVMBisectionTest(unittest.TestCase): |
| """Unittests for LLVM bisection.""" |
| |
| def testStartAndEndDoNotMatchJsonStartAndEnd(self): |
| start = 100 |
| end = 150 |
| |
| json_start = 110 |
| json_end = 150 |
| |
| # Verify the exception is raised when the start and end revision for LLVM |
| # bisection do not match the .JSON's 'start' and 'end' values. |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection._ValidateStartAndEndAgainstJSONStartAndEnd( |
| start, end, json_start, json_end) |
| |
| expected_error_message = ('The start %d or the end %d version provided is ' |
| 'different than "start" %d or "end" %d in the ' |
| '.JSON file' % (start, end, json_start, json_end)) |
| |
| self.assertEqual(str(err.exception), expected_error_message) |
| |
| def testStartAndEndMatchJsonStartAndEnd(self): |
| start = 100 |
| end = 150 |
| |
| json_start = 100 |
| json_end = 150 |
| |
| llvm_bisection._ValidateStartAndEndAgainstJSONStartAndEnd( |
| start, end, json_start, json_end) |
| |
| def testTryjobStatusIsMissing(self): |
| start = 100 |
| end = 150 |
| |
| test_tryjobs = [{ |
| 'rev': 105, |
| 'status': 'good', |
| 'link': 'https://some_tryjob_1_url.com' |
| }, { |
| 'rev': 120, |
| 'status': None, |
| 'link': 'https://some_tryjob_2_url.com' |
| }, { |
| 'rev': 140, |
| 'status': 'bad', |
| 'link': 'https://some_tryjob_3_url.com' |
| }] |
| |
| # Verify the exception is raised when a tryjob does not have a value for |
| # the 'status' key or the 'status' key is missing. |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs) |
| |
| expected_error_message = ( |
| '"status" is missing or has no value, please ' |
| 'go to %s and update it' % test_tryjobs[1]['link']) |
| |
| self.assertEqual(str(err.exception), expected_error_message) |
| |
| def testGoodRevisionGreaterThanBadRevision(self): |
| start = 100 |
| end = 150 |
| |
| test_tryjobs = [{ |
| 'rev': 110, |
| 'status': 'bad', |
| 'link': 'https://some_tryjob_1_url.com' |
| }, { |
| 'rev': 125, |
| 'status': 'skip', |
| 'link': 'https://some_tryjob_2_url.com' |
| }, { |
| 'rev': 140, |
| 'status': 'good', |
| 'link': 'https://some_tryjob_3_url.com' |
| }] |
| |
| # Verify the exception is raised when the new 'start' revision is greater |
| # than the new 'bad' revision for bisection (i.e. bisection is broken). |
| with self.assertRaises(AssertionError) as err: |
| llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs) |
| |
| expected_error_message = ( |
| 'Bisection is broken because %d (good) is >= ' |
| '%d (bad)' % (test_tryjobs[2]['rev'], test_tryjobs[0]['rev'])) |
| |
| self.assertEqual(str(err.exception), expected_error_message) |
| |
| def testSuccessfullyGetNewStartAndNewEndRevision(self): |
| start = 100 |
| end = 150 |
| |
| test_tryjobs = [{ |
| 'rev': 110, |
| 'status': 'good', |
| 'link': 'https://some_tryjob_1_url.com' |
| }, { |
| 'rev': 120, |
| 'status': 'good', |
| 'link': 'https://some_tryjob_2_url.com' |
| }, { |
| 'rev': 130, |
| 'status': 'pending', |
| 'link': 'https://some_tryjob_3_url.com' |
| }, { |
| 'rev': 135, |
| 'status': 'skip', |
| 'link': 'https://some_tryjob_4_url.com' |
| }, { |
| 'rev': 140, |
| 'status': 'bad', |
| 'link': 'https://some_tryjob_5_url.com' |
| }] |
| |
| # Tuple consists of the new good revision, the new bad revision, a set of |
| # 'pending' revisions, and a set of 'skip' revisions. |
| expected_revisions_tuple = 120, 140, {130}, {135} |
| |
| self.assertTupleEqual( |
| llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs), |
| expected_revisions_tuple) |
| |
| @mock.patch.object(LLVMHash, 'GetGitHashForVersion') |
| def testNoRevisionsBetweenStartAndEnd(self, mock_get_git_hash): |
| start = 100 |
| end = 110 |
| |
| test_pending_revisions = {107} |
| test_skip_revisions = {101, 102, 103, 104, 108, 109} |
| |
| # Simulate behavior of `GetGitHashForVersion()` when the revision does not |
| # exist in the LLVM source tree. |
| def MockGetGitHashForRevisionRaiseException(src_path, revision): |
| raise ValueError('Revision does not exist') |
| |
| mock_get_git_hash.side_effect = MockGetGitHashForRevisionRaiseException |
| |
| parallel = 3 |
| |
| abs_path_to_src = '/abs/path/to/src' |
| |
| self.assertListEqual( |
| llvm_bisection.GetRevisionsBetweenBisection( |
| start, end, parallel, abs_path_to_src, test_pending_revisions, |
| test_skip_revisions), []) |
| |
| @mock.patch.object(LLVMHash, 'GetGitHashForVersion') |
| def testSuccessfullyRetrievedRevisionsBetweenStartAndEnd( |
| self, mock_get_git_hash): |
| |
| start = 100 |
| end = 110 |
| |
| test_pending_revisions = set() |
| test_skip_revisions = {101, 102, 103, 104, 106, 108, 109} |
| |
| parallel = 3 |
| |
| abs_path_to_src = '/abs/path/to/src' |
| |
| # Valid revision that exist in the LLVM source tree between 'start' and |
| # 'end' and were not in the 'pending' set or 'skip' set. |
| expected_revisions_between_start_and_end = [105, 107] |
| |
| self.assertListEqual( |
| llvm_bisection.GetRevisionsBetweenBisection( |
| start, end, parallel, abs_path_to_src, test_pending_revisions, |
| test_skip_revisions), expected_revisions_between_start_and_end) |
| |
| self.assertEqual(mock_get_git_hash.call_count, 2) |
| |
| # Simulate behavior of `GetGitHashForVersion()` when successfully retrieved |
| # a list git hashes for each revision in the revisions list. |
| @mock.patch.object(LLVMHash, 'GetGitHashForVersion') |
| # Simulate behavior of `GetRevisionsBetweenBisection()` when successfully |
| # retrieved a list of valid revisions between 'start' and 'end'. |
| @mock.patch.object(llvm_bisection, 'GetRevisionsBetweenBisection') |
| # Simulate behavior of `CreatTempLLVMRepo()` when successfully created a |
| # worktree when a source path was not provided. |
| @mock.patch.object(llvm_bisection, 'CreateTempLLVMRepo') |
| def testSuccessfullyGetRevisionsListAndHashList( |
| self, mock_create_temp_llvm_repo, mock_get_revisions_between_bisection, |
| mock_get_git_hash): |
| |
| expected_revisions_and_hash_tuple = ([102, 105, 108], [ |
| 'a123testhash1', 'a123testhash2', 'a123testhash3' |
| ]) |
| |
| @CallCountsToMockFunctions |
| def MockGetGitHashForRevision(call_count, src_path, rev): |
| # Simulate retrieving the git hash for the revision. |
| if call_count < 3: |
| return expected_revisions_and_hash_tuple[1][call_count] |
| |
| assert False, 'Called `GetGitHashForVersion()` more than expected.' |
| |
| temp_worktree = '/abs/path/to/tmpDir' |
| |
| mock_create_temp_llvm_repo.return_value.__enter__.return_value.name = \ |
| temp_worktree |
| |
| # Simulate the valid revisions list. |
| mock_get_revisions_between_bisection.return_value = \ |
| expected_revisions_and_hash_tuple[0] |
| |
| # Simulate behavior of `GetGitHashForVersion()` by using the testing |
| # function. |
| mock_get_git_hash.side_effect = MockGetGitHashForRevision |
| |
| start = 100 |
| end = 110 |
| parallel = 3 |
| src_path = None |
| pending_revisions = {103, 104} |
| skip_revisions = {101, 106, 107, 109} |
| |
| self.assertTupleEqual( |
| llvm_bisection.GetRevisionsListAndHashList( |
| start, end, parallel, src_path, pending_revisions, skip_revisions), |
| expected_revisions_and_hash_tuple) |
| |
| mock_get_revisions_between_bisection.assert_called_once() |
| |
| self.assertEqual(mock_get_git_hash.call_count, 3) |
| |
| def testSuccessfullyDieWithNoRevisionsError(self): |
| start = 100 |
| end = 110 |
| |
| pending_revisions = {105, 108} |
| skip_revisions = {101, 102, 103, 104, 106, 107, 109} |
| |
| expected_no_revisions_message = ('No revisions between start %d and end ' |
| '%d to create tryjobs' % (start, end)) |
| |
| expected_no_revisions_message += '\nThe following tryjobs are pending:\n' \ |
| + '\n'.join(str(rev) for rev in pending_revisions) |
| |
| expected_no_revisions_message += '\nThe following tryjobs were skipped:\n' \ |
| + '\n'.join(str(rev) for rev in skip_revisions) |
| |
| # Verify that an exception is raised when there are no revisions to launch |
| # tryjobs for between 'start' and 'end' and some tryjobs are 'pending'. |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.DieWithNoRevisionsError(start, end, skip_revisions, |
| pending_revisions) |
| |
| self.assertEqual(str(err.exception), expected_no_revisions_message) |
| |
| # Simulate behavior of `FindTryjobIndex()` when the index of the tryjob was |
| # found. |
| @mock.patch.object(llvm_bisection, 'FindTryjobIndex', return_value=0) |
| def testTryjobExistsInRevisionsToLaunch(self, mock_find_tryjob_index): |
| test_existing_jobs = [{'rev': 102, 'status': 'good'}] |
| |
| revision_to_launch = [102] |
| |
| expected_revision_that_exists = 102 |
| |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.CheckForExistingTryjobsInRevisionsToLaunch( |
| revision_to_launch, test_existing_jobs) |
| |
| expected_found_tryjob_index_error_message = ( |
| 'Revision %d exists already ' |
| 'in "jobs"' % expected_revision_that_exists) |
| |
| self.assertEqual( |
| str(err.exception), expected_found_tryjob_index_error_message) |
| |
| mock_find_tryjob_index.assert_called_once() |
| |
| @mock.patch.object(llvm_bisection, 'AddTryjob') |
| def testSuccessfullyUpdatedStatusFileWhenExceptionIsRaised( |
| self, mock_add_tryjob): |
| |
| git_hash_list = ['a123testhash1', 'a123testhash2', 'a123testhash3'] |
| revisions_list = [102, 104, 106] |
| |
| # Simulate behavior of `AddTryjob()` when successfully launched a tryjob for |
| # the updated packages. |
| @CallCountsToMockFunctions |
| def MockAddTryjob(call_count, packages, git_hash, revision, chroot_path, |
| patch_file, extra_cls, options, builder, verbose, |
| svn_revision): |
| |
| if call_count < 2: |
| return {'rev': revisions_list[call_count], 'status': 'pending'} |
| |
| # Simulate an exception happened along the way when updating the |
| # packages' `LLVM_NEXT_HASH`. |
| if call_count == 2: |
| raise ValueError('Unable to launch tryjob') |
| |
| assert False, 'Called `AddTryjob()` more than expected.' |
| |
| # Use the test function to simulate `AddTryjob()`. |
| mock_add_tryjob.side_effect = MockAddTryjob |
| |
| start = 100 |
| end = 110 |
| |
| bisection_contents = {'start': start, 'end': end, 'jobs': []} |
| |
| args_output = ArgsOutputTest() |
| |
| packages = ['sys-devel/llvm'] |
| patch_file = '/abs/path/to/PATCHES.json' |
| |
| # Create a temporary .JSON file to simulate a status file for bisection. |
| with CreateTemporaryJsonFile() as temp_json_file: |
| with open(temp_json_file, 'w') as f: |
| WritePrettyJsonFile(bisection_contents, f) |
| |
| # Verify that the status file is updated when an exception happened when |
| # attempting to launch a revision (i.e. progress is not lost). |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.UpdateBisection( |
| revisions_list, git_hash_list, bisection_contents, temp_json_file, |
| packages, args_output.chroot_path, patch_file, |
| args_output.extra_change_lists, args_output.options, |
| args_output.builders, args_output.verbose) |
| |
| expected_bisection_contents = { |
| 'start': |
| start, |
| 'end': |
| end, |
| 'jobs': [{ |
| 'rev': revisions_list[0], |
| 'status': 'pending' |
| }, { |
| 'rev': revisions_list[1], |
| 'status': 'pending' |
| }] |
| } |
| |
| # Verify that the launched tryjobs were added to the status file when |
| # an exception happened. |
| with open(temp_json_file) as f: |
| json_contents = json.load(f) |
| |
| self.assertDictEqual(json_contents, expected_bisection_contents) |
| |
| self.assertEqual(str(err.exception), 'Unable to launch tryjob') |
| |
| self.assertEqual(mock_add_tryjob.call_count, 3) |
| |
| # Simulate behavior of `GetGitHashForVersion()` when successfully retrieved |
| # the git hash of the bad revision. |
| @mock.patch.object( |
| LLVMHash, 'GetGitHashForVersion', return_value='a123testhash4') |
| def testCompletedBisectionWhenProvidedSrcPath(self, mock_get_git_hash): |
| last_tested = '/some/last/tested_file.json' |
| |
| src_path = '/abs/path/to/src/path' |
| |
| # The bad revision. |
| end = 150 |
| |
| llvm_bisection._NoteCompletedBisection(last_tested, src_path, end) |
| |
| mock_get_git_hash.assert_called_once() |
| |
| # Simulate behavior of `GetLLVMHash()` when successfully retrieved |
| # the git hash of the bad revision. |
| @mock.patch.object(LLVMHash, 'GetLLVMHash', return_value='a123testhash5') |
| def testCompletedBisectionWhenNotProvidedSrcPath(self, mock_get_git_hash): |
| last_tested = '/some/last/tested_file.json' |
| |
| src_path = None |
| |
| # The bad revision. |
| end = 200 |
| |
| llvm_bisection._NoteCompletedBisection(last_tested, src_path, end) |
| |
| mock_get_git_hash.assert_called_once() |
| |
| def testSuccessfullyLoadedStatusFile(self): |
| start = 100 |
| end = 150 |
| |
| test_bisect_contents = {'start': start, 'end': end, 'jobs': []} |
| |
| # Simulate that the status file exists. |
| with CreateTemporaryJsonFile() as temp_json_file: |
| with open(temp_json_file, 'w') as f: |
| WritePrettyJsonFile(test_bisect_contents, f) |
| |
| self.assertDictEqual( |
| llvm_bisection.LoadStatusFile(temp_json_file, start, end), |
| test_bisect_contents) |
| |
| def testLoadedStatusFileThatDoesNotExist(self): |
| start = 200 |
| end = 250 |
| |
| expected_bisect_contents = {'start': start, 'end': end, 'jobs': []} |
| |
| last_tested = '/abs/path/to/file_that_does_not_exist.json' |
| |
| self.assertDictEqual( |
| llvm_bisection.LoadStatusFile(last_tested, start, end), |
| expected_bisect_contents) |
| |
| # Simulate behavior of `_NoteCompletedBisection()` when there are no more |
| # tryjobs to launch between start and end, so bisection is complete. |
| @mock.patch.object(llvm_bisection, '_NoteCompletedBisection') |
| @mock.patch.object(llvm_bisection, 'GetRevisionsListAndHashList') |
| @mock.patch.object(llvm_bisection, 'GetStartAndEndRevision') |
| # Simulate behavior of `_ValidateStartAndEndAgainstJSONStartAndEnd()` when |
| # both start and end revisions match. |
| @mock.patch.object(llvm_bisection, |
| '_ValidateStartAndEndAgainstJSONStartAndEnd') |
| @mock.patch.object(llvm_bisection, 'LoadStatusFile') |
| # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the |
| # script outside of the chroot. |
| @mock.patch.object(llvm_bisection, 'VerifyOutsideChroot', return_value=True) |
| def testSuccessfullyBisectedLLVM( |
| self, mock_outside_chroot, mock_load_status_file, |
| mock_validate_start_and_end, mock_get_start_and_end_revision, |
| mock_get_revision_and_hash_list, mock_note_completed_bisection): |
| |
| start = 500 |
| end = 502 |
| |
| bisect_contents = { |
| 'start': start, |
| 'end': end, |
| 'jobs': [{ |
| 'rev': 501, |
| 'status': 'skip' |
| }] |
| } |
| |
| skip_revisions = {501} |
| pending_revisions = {} |
| |
| # Simulate behavior of `LoadStatusFile()` when successfully loaded the |
| # status file. |
| mock_load_status_file.return_value = bisect_contents |
| |
| # Simulate behavior of `GetStartAndEndRevision()` when successfully found |
| # the new start and end revision of the bisection. |
| # |
| # Returns new start revision, new end revision, a set of pending revisions, |
| # and a set of skip revisions. |
| mock_get_start_and_end_revision.return_value = (start, end, |
| pending_revisions, |
| skip_revisions) |
| |
| # Simulate behavior of `GetRevisionsListAndHashList()` when successfully |
| # retrieved valid revisions (along with their git hashes) between start and |
| # end (in this case, none). |
| mock_get_revision_and_hash_list.return_value = [], [] |
| |
| args_output = ArgsOutputTest() |
| args_output.start_rev = start |
| args_output.end_rev = end |
| args_output.parallel = 3 |
| args_output.src_path = None |
| |
| self.assertEqual( |
| llvm_bisection.main(args_output), |
| llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value) |
| |
| mock_outside_chroot.assert_called_once() |
| |
| mock_load_status_file.assert_called_once() |
| |
| mock_validate_start_and_end.assert_called_once() |
| |
| mock_get_start_and_end_revision.assert_called_once() |
| |
| mock_get_revision_and_hash_list.assert_called_once() |
| |
| mock_note_completed_bisection.assert_called_once() |
| |
| @mock.patch.object(llvm_bisection, 'DieWithNoRevisionsError') |
| # Simulate behavior of `_NoteCompletedBisection()` when there are no more |
| # tryjobs to launch between start and end, so bisection is complete. |
| @mock.patch.object(llvm_bisection, 'GetRevisionsListAndHashList') |
| @mock.patch.object(llvm_bisection, 'GetStartAndEndRevision') |
| # Simulate behavior of `_ValidateStartAndEndAgainstJSONStartAndEnd()` when |
| # both start and end revisions match. |
| @mock.patch.object(llvm_bisection, |
| '_ValidateStartAndEndAgainstJSONStartAndEnd') |
| @mock.patch.object(llvm_bisection, 'LoadStatusFile') |
| # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the |
| # script outside of the chroot. |
| @mock.patch.object(llvm_bisection, 'VerifyOutsideChroot', return_value=True) |
| def testNoMoreTryjobsToLaunch( |
| self, mock_outside_chroot, mock_load_status_file, |
| mock_validate_start_and_end, mock_get_start_and_end_revision, |
| mock_get_revision_and_hash_list, mock_die_with_no_revisions_error): |
| |
| start = 500 |
| end = 502 |
| |
| bisect_contents = { |
| 'start': start, |
| 'end': end, |
| 'jobs': [{ |
| 'rev': 501, |
| 'status': 'pending' |
| }] |
| } |
| |
| skip_revisions = {} |
| pending_revisions = {501} |
| |
| no_revisions_error_message = ('No more tryjobs to launch between %d and ' |
| '%d' % (start, end)) |
| |
| def MockNoRevisionsErrorException(start, end, skip, pending): |
| raise ValueError(no_revisions_error_message) |
| |
| # Simulate behavior of `LoadStatusFile()` when successfully loaded the |
| # status file. |
| mock_load_status_file.return_value = bisect_contents |
| |
| # Simulate behavior of `GetStartAndEndRevision()` when successfully found |
| # the new start and end revision of the bisection. |
| # |
| # Returns new start revision, new end revision, a set of pending revisions, |
| # and a set of skip revisions. |
| mock_get_start_and_end_revision.return_value = (start, end, |
| pending_revisions, |
| skip_revisions) |
| |
| # Simulate behavior of `GetRevisionsListAndHashList()` when successfully |
| # retrieved valid revisions (along with their git hashes) between start and |
| # end (in this case, none). |
| mock_get_revision_and_hash_list.return_value = [], [] |
| |
| # Use the test function to simulate `DieWithNoRevisionsWithError()` |
| # behavior. |
| mock_die_with_no_revisions_error.side_effect = MockNoRevisionsErrorException |
| |
| # Simulate behavior of arguments passed into the command line. |
| args_output = ArgsOutputTest() |
| args_output.start_rev = start |
| args_output.end_rev = end |
| args_output.parallel = 3 |
| args_output.src_path = None |
| |
| # Verify the exception is raised when there are no more tryjobs to launch |
| # between start and end when there are tryjobs that are 'pending', so |
| # the actual bad revision can change when those tryjobs's 'status' are |
| # updated. |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.main(args_output) |
| |
| self.assertEqual(str(err.exception), no_revisions_error_message) |
| |
| mock_outside_chroot.assert_called_once() |
| |
| mock_load_status_file.assert_called_once() |
| |
| mock_validate_start_and_end.assert_called_once() |
| |
| mock_get_start_and_end_revision.assert_called_once() |
| |
| mock_get_revision_and_hash_list.assert_called_once() |
| |
| mock_die_with_no_revisions_error.assert_called_once() |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |