| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # Copyright 2019 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # pylint: disable=protected-access |
| |
| """Tests for LLVM bisection.""" |
| |
| from __future__ import print_function |
| |
| import json |
| import os |
| import subprocess |
| import unittest |
| import unittest.mock as mock |
| |
| import chroot |
| import get_llvm_hash |
| import git_llvm_rev |
| import llvm_bisection |
| import modify_a_tryjob |
| import test_helpers |
| |
| |
| class LLVMBisectionTest(unittest.TestCase): |
| """Unittests for LLVM bisection.""" |
| |
| def testGetRemainingRangePassed(self): |
| start = 100 |
| end = 150 |
| |
| test_tryjobs = [{ |
| 'rev': 110, |
| 'status': 'good', |
| 'link': 'https://some_tryjob_1_url.com' |
| }, { |
| 'rev': 120, |
| 'status': 'good', |
| 'link': 'https://some_tryjob_2_url.com' |
| }, { |
| 'rev': 130, |
| 'status': 'pending', |
| 'link': 'https://some_tryjob_3_url.com' |
| }, { |
| 'rev': 135, |
| 'status': 'skip', |
| 'link': 'https://some_tryjob_4_url.com' |
| }, { |
| 'rev': 140, |
| 'status': 'bad', |
| 'link': 'https://some_tryjob_5_url.com' |
| }] |
| |
| # Tuple consists of the new good revision, the new bad revision, a set of |
| # 'pending' revisions, and a set of 'skip' revisions. |
| expected_revisions_tuple = 120, 140, {130}, {135} |
| |
| self.assertEqual( |
| llvm_bisection.GetRemainingRange(start, end, test_tryjobs), |
| expected_revisions_tuple) |
| |
| def testGetRemainingRangeFailedWithMissingStatus(self): |
| start = 100 |
| end = 150 |
| |
| test_tryjobs = [{ |
| 'rev': 105, |
| 'status': 'good', |
| 'link': 'https://some_tryjob_1_url.com' |
| }, { |
| 'rev': 120, |
| 'status': None, |
| 'link': 'https://some_tryjob_2_url.com' |
| }, { |
| 'rev': 140, |
| 'status': 'bad', |
| 'link': 'https://some_tryjob_3_url.com' |
| }] |
| |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.GetRemainingRange(start, end, test_tryjobs) |
| |
| error_message = ('"status" is missing or has no value, please ' |
| 'go to %s and update it' % test_tryjobs[1]['link']) |
| self.assertEqual(str(err.exception), error_message) |
| |
| def testGetRemainingRangeFailedWithInvalidRange(self): |
| start = 100 |
| end = 150 |
| |
| test_tryjobs = [{ |
| 'rev': 110, |
| 'status': 'bad', |
| 'link': 'https://some_tryjob_1_url.com' |
| }, { |
| 'rev': 125, |
| 'status': 'skip', |
| 'link': 'https://some_tryjob_2_url.com' |
| }, { |
| 'rev': 140, |
| 'status': 'good', |
| 'link': 'https://some_tryjob_3_url.com' |
| }] |
| |
| with self.assertRaises(AssertionError) as err: |
| llvm_bisection.GetRemainingRange(start, end, test_tryjobs) |
| |
| expected_error_message = ('Bisection is broken because %d (good) is >= ' |
| '%d (bad)' % |
| (test_tryjobs[2]['rev'], test_tryjobs[0]['rev'])) |
| |
| self.assertEqual(str(err.exception), expected_error_message) |
| |
| @mock.patch.object(get_llvm_hash, 'GetGitHashFrom') |
| def testGetCommitsBetweenPassed(self, mock_get_git_hash): |
| start = git_llvm_rev.base_llvm_revision |
| end = start + 10 |
| test_pending_revisions = {start + 7} |
| test_skip_revisions = { |
| start + 1, start + 2, start + 4, start + 8, start + 9 |
| } |
| parallel = 3 |
| abs_path_to_src = '/abs/path/to/src' |
| |
| revs = ['a123testhash3', 'a123testhash5'] |
| mock_get_git_hash.side_effect = revs |
| |
| git_hashes = [ |
| git_llvm_rev.base_llvm_revision + 3, git_llvm_rev.base_llvm_revision + 5 |
| ] |
| |
| self.assertEqual( |
| llvm_bisection.GetCommitsBetween(start, end, parallel, abs_path_to_src, |
| test_pending_revisions, |
| test_skip_revisions), |
| (git_hashes, revs)) |
| |
| def testLoadStatusFilePassedWithExistingFile(self): |
| start = 100 |
| end = 150 |
| |
| test_bisect_state = {'start': start, 'end': end, 'jobs': []} |
| |
| # Simulate that the status file exists. |
| with test_helpers.CreateTemporaryJsonFile() as temp_json_file: |
| with open(temp_json_file, 'w') as f: |
| test_helpers.WritePrettyJsonFile(test_bisect_state, f) |
| |
| self.assertEqual( |
| llvm_bisection.LoadStatusFile(temp_json_file, start, end), |
| test_bisect_state) |
| |
| def testLoadStatusFilePassedWithoutExistingFile(self): |
| start = 200 |
| end = 250 |
| |
| expected_bisect_state = {'start': start, 'end': end, 'jobs': []} |
| |
| last_tested = '/abs/path/to/file_that_does_not_exist.json' |
| |
| self.assertEqual( |
| llvm_bisection.LoadStatusFile(last_tested, start, end), |
| expected_bisect_state) |
| |
| @mock.patch.object(modify_a_tryjob, 'AddTryjob') |
| def testBisectPassed(self, mock_add_tryjob): |
| |
| git_hash_list = ['a123testhash1', 'a123testhash2', 'a123testhash3'] |
| revisions_list = [102, 104, 106] |
| |
| # Simulate behavior of `AddTryjob()` when successfully launched a tryjob for |
| # the updated packages. |
| @test_helpers.CallCountsToMockFunctions |
| def MockAddTryjob(call_count, _packages, _git_hash, _revision, _chroot_path, |
| _patch_file, _extra_cls, _options, _builder, _verbose, |
| _svn_revision): |
| |
| if call_count < 2: |
| return {'rev': revisions_list[call_count], 'status': 'pending'} |
| |
| # Simulate an exception happened along the way when updating the |
| # packages' `LLVM_NEXT_HASH`. |
| if call_count == 2: |
| raise ValueError('Unable to launch tryjob') |
| |
| assert False, 'Called `AddTryjob()` more than expected.' |
| |
| # Use the test function to simulate `AddTryjob()`. |
| mock_add_tryjob.side_effect = MockAddTryjob |
| |
| start = 100 |
| end = 110 |
| |
| bisection_contents = {'start': start, 'end': end, 'jobs': []} |
| |
| args_output = test_helpers.ArgsOutputTest() |
| |
| packages = ['sys-devel/llvm'] |
| patch_file = '/abs/path/to/PATCHES.json' |
| |
| # Create a temporary .JSON file to simulate a status file for bisection. |
| with test_helpers.CreateTemporaryJsonFile() as temp_json_file: |
| with open(temp_json_file, 'w') as f: |
| test_helpers.WritePrettyJsonFile(bisection_contents, f) |
| |
| # Verify that the status file is updated when an exception happened when |
| # attempting to launch a revision (i.e. progress is not lost). |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.Bisect(revisions_list, git_hash_list, bisection_contents, |
| temp_json_file, packages, args_output.chroot_path, |
| patch_file, args_output.extra_change_lists, |
| args_output.options, args_output.builders, |
| args_output.verbose) |
| |
| expected_bisection_contents = { |
| 'start': |
| start, |
| 'end': |
| end, |
| 'jobs': [{ |
| 'rev': revisions_list[0], |
| 'status': 'pending' |
| }, { |
| 'rev': revisions_list[1], |
| 'status': 'pending' |
| }] |
| } |
| |
| # Verify that the launched tryjobs were added to the status file when |
| # an exception happened. |
| with open(temp_json_file) as f: |
| json_contents = json.load(f) |
| |
| self.assertEqual(json_contents, expected_bisection_contents) |
| |
| self.assertEqual(str(err.exception), 'Unable to launch tryjob') |
| |
| self.assertEqual(mock_add_tryjob.call_count, 3) |
| |
| @mock.patch.object(subprocess, 'check_output', return_value=None) |
| @mock.patch.object( |
| get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4') |
| @mock.patch.object(llvm_bisection, 'GetCommitsBetween') |
| @mock.patch.object(llvm_bisection, 'GetRemainingRange') |
| @mock.patch.object(llvm_bisection, 'LoadStatusFile') |
| @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) |
| def testMainPassed(self, mock_outside_chroot, mock_load_status_file, |
| mock_get_range, mock_get_revision_and_hash_list, |
| _mock_get_bad_llvm_hash, mock_abandon_cl): |
| |
| start = 500 |
| end = 502 |
| cl = 1 |
| |
| bisect_state = { |
| 'start': start, |
| 'end': end, |
| 'jobs': [{ |
| 'rev': 501, |
| 'status': 'bad', |
| 'cl': cl |
| }] |
| } |
| |
| skip_revisions = {501} |
| pending_revisions = {} |
| |
| mock_load_status_file.return_value = bisect_state |
| |
| mock_get_range.return_value = (start, end, pending_revisions, |
| skip_revisions) |
| |
| mock_get_revision_and_hash_list.return_value = [], [] |
| |
| args_output = test_helpers.ArgsOutputTest() |
| args_output.start_rev = start |
| args_output.end_rev = end |
| args_output.parallel = 3 |
| args_output.src_path = None |
| args_output.chroot_path = 'somepath' |
| args_output.cleanup = True |
| |
| self.assertEqual( |
| llvm_bisection.main(args_output), |
| llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value) |
| |
| mock_outside_chroot.assert_called_once() |
| |
| mock_load_status_file.assert_called_once() |
| |
| mock_get_range.assert_called_once() |
| |
| mock_get_revision_and_hash_list.assert_called_once() |
| |
| mock_abandon_cl.assert_called_once() |
| self.assertEqual( |
| mock_abandon_cl.call_args, |
| mock.call( |
| [ |
| os.path.join(args_output.chroot_path, 'chromite/bin/gerrit'), |
| 'abandon', |
| str(cl), |
| ], |
| stderr=subprocess.STDOUT, |
| encoding='utf-8', |
| )) |
| |
| @mock.patch.object(llvm_bisection, 'LoadStatusFile') |
| @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) |
| def testMainFailedWithInvalidRange(self, mock_outside_chroot, |
| mock_load_status_file): |
| |
| start = 500 |
| end = 502 |
| |
| bisect_state = { |
| 'start': start - 1, |
| 'end': end, |
| } |
| |
| mock_load_status_file.return_value = bisect_state |
| |
| args_output = test_helpers.ArgsOutputTest() |
| args_output.start_rev = start |
| args_output.end_rev = end |
| args_output.parallel = 3 |
| args_output.src_path = None |
| |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.main(args_output) |
| |
| error_message = (f'The start {start} or the end {end} version provided is ' |
| f'different than "start" {bisect_state["start"]} or "end" ' |
| f'{bisect_state["end"]} in the .JSON file') |
| |
| self.assertEqual(str(err.exception), error_message) |
| |
| mock_outside_chroot.assert_called_once() |
| |
| mock_load_status_file.assert_called_once() |
| |
| @mock.patch.object(llvm_bisection, 'GetCommitsBetween') |
| @mock.patch.object(llvm_bisection, 'GetRemainingRange') |
| @mock.patch.object(llvm_bisection, 'LoadStatusFile') |
| @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) |
| def testMainFailedWithPendingBuilds(self, mock_outside_chroot, |
| mock_load_status_file, mock_get_range, |
| mock_get_revision_and_hash_list): |
| |
| start = 500 |
| end = 502 |
| rev = 501 |
| |
| bisect_state = { |
| 'start': start, |
| 'end': end, |
| 'jobs': [{ |
| 'rev': rev, |
| 'status': 'pending' |
| }] |
| } |
| |
| skip_revisions = {} |
| pending_revisions = {rev} |
| |
| mock_load_status_file.return_value = bisect_state |
| |
| mock_get_range.return_value = (start, end, pending_revisions, |
| skip_revisions) |
| |
| mock_get_revision_and_hash_list.return_value = [], [] |
| |
| args_output = test_helpers.ArgsOutputTest() |
| args_output.start_rev = start |
| args_output.end_rev = end |
| args_output.parallel = 3 |
| args_output.src_path = None |
| |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.main(args_output) |
| |
| error_message = (f'No revisions between start {start} and end {end} to ' |
| 'create tryjobs\nThe following tryjobs are pending:\n' |
| f'{rev}\n') |
| |
| self.assertEqual(str(err.exception), error_message) |
| |
| mock_outside_chroot.assert_called_once() |
| |
| mock_load_status_file.assert_called_once() |
| |
| mock_get_range.assert_called_once() |
| |
| mock_get_revision_and_hash_list.assert_called_once() |
| |
| @mock.patch.object(llvm_bisection, 'GetCommitsBetween') |
| @mock.patch.object(llvm_bisection, 'GetRemainingRange') |
| @mock.patch.object(llvm_bisection, 'LoadStatusFile') |
| @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) |
| def testMainFailedWithDuplicateBuilds(self, mock_outside_chroot, |
| mock_load_status_file, mock_get_range, |
| mock_get_revision_and_hash_list): |
| |
| start = 500 |
| end = 502 |
| rev = 501 |
| git_hash = 'a123testhash1' |
| |
| bisect_state = { |
| 'start': start, |
| 'end': end, |
| 'jobs': [{ |
| 'rev': rev, |
| 'status': 'pending' |
| }] |
| } |
| |
| skip_revisions = {} |
| pending_revisions = {rev} |
| |
| mock_load_status_file.return_value = bisect_state |
| |
| mock_get_range.return_value = (start, end, pending_revisions, |
| skip_revisions) |
| |
| mock_get_revision_and_hash_list.return_value = [rev], [git_hash] |
| |
| args_output = test_helpers.ArgsOutputTest() |
| args_output.start_rev = start |
| args_output.end_rev = end |
| args_output.parallel = 3 |
| args_output.src_path = None |
| |
| with self.assertRaises(ValueError) as err: |
| llvm_bisection.main(args_output) |
| |
| error_message = ('Revision %d exists already in "jobs"' % rev) |
| self.assertEqual(str(err.exception), error_message) |
| |
| mock_outside_chroot.assert_called_once() |
| |
| mock_load_status_file.assert_called_once() |
| |
| mock_get_range.assert_called_once() |
| |
| mock_get_revision_and_hash_list.assert_called_once() |
| |
| @mock.patch.object(subprocess, 'check_output', return_value=None) |
| @mock.patch.object( |
| get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4') |
| @mock.patch.object(llvm_bisection, 'GetCommitsBetween') |
| @mock.patch.object(llvm_bisection, 'GetRemainingRange') |
| @mock.patch.object(llvm_bisection, 'LoadStatusFile') |
| @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) |
| def testMainFailedToAbandonCL(self, mock_outside_chroot, |
| mock_load_status_file, mock_get_range, |
| mock_get_revision_and_hash_list, |
| _mock_get_bad_llvm_hash, mock_abandon_cl): |
| |
| start = 500 |
| end = 502 |
| |
| bisect_state = { |
| 'start': start, |
| 'end': end, |
| 'jobs': [{ |
| 'rev': 501, |
| 'status': 'bad', |
| 'cl': 0 |
| }] |
| } |
| |
| skip_revisions = {501} |
| pending_revisions = {} |
| |
| mock_load_status_file.return_value = bisect_state |
| |
| mock_get_range.return_value = (start, end, pending_revisions, |
| skip_revisions) |
| |
| mock_get_revision_and_hash_list.return_value = ([], []) |
| |
| error_message = 'Error message.' |
| mock_abandon_cl.side_effect = subprocess.CalledProcessError( |
| returncode=1, cmd=[], output=error_message) |
| |
| args_output = test_helpers.ArgsOutputTest() |
| args_output.start_rev = start |
| args_output.end_rev = end |
| args_output.parallel = 3 |
| args_output.src_path = None |
| args_output.cleanup = True |
| |
| with self.assertRaises(subprocess.CalledProcessError) as err: |
| llvm_bisection.main(args_output) |
| |
| self.assertEqual(err.exception.output, error_message) |
| |
| mock_outside_chroot.assert_called_once() |
| |
| mock_load_status_file.assert_called_once() |
| |
| mock_get_range.assert_called_once() |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |
