| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # Copyright 2020 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Tests for bisecting tool.""" |
| |
| |
| __author__ = "shenhan@google.com (Han Shen)" |
| |
| import os |
| import random |
| import sys |
| import unittest |
| |
| from cros_utils import command_executer |
| from binary_search_tool import binary_search_state |
| from binary_search_tool import run_bisect |
| |
| from binary_search_tool.test import common |
| from binary_search_tool.test import gen_obj |
| |
| |
| def GenObj(): |
| obj_num = random.randint(100, 1000) |
| bad_obj_num = random.randint(obj_num // 100, obj_num // 20) |
| if bad_obj_num == 0: |
| bad_obj_num = 1 |
| gen_obj.Main(["--obj_num", str(obj_num), "--bad_obj_num", str(bad_obj_num)]) |
| |
| |
| def CleanObj(): |
| os.remove(common.OBJECTS_FILE) |
| os.remove(common.WORKING_SET_FILE) |
| print( |
| 'Deleted "{0}" and "{1}"'.format( |
| common.OBJECTS_FILE, common.WORKING_SET_FILE |
| ) |
| ) |
| |
| |
| class BisectTest(unittest.TestCase): |
| """Tests for run_bisect.py""" |
| |
| def setUp(self): |
| with open("./is_setup", "w", encoding="utf-8"): |
| pass |
| |
| try: |
| os.remove(binary_search_state.STATE_FILE) |
| except OSError: |
| pass |
| |
| def tearDown(self): |
| try: |
| os.remove("./is_setup") |
| os.remove(os.readlink(binary_search_state.STATE_FILE)) |
| os.remove(binary_search_state.STATE_FILE) |
| except OSError: |
| pass |
| |
| class FullBisector(run_bisect.Bisector): |
| """Test bisector to test run_bisect.py with""" |
| |
| def __init__(self, options, overrides): |
| super(BisectTest.FullBisector, self).__init__(options, overrides) |
| |
| def PreRun(self): |
| GenObj() |
| return 0 |
| |
| def Run(self): |
| return binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| prune=True, |
| file_args=True, |
| ) |
| |
| def PostRun(self): |
| CleanObj() |
| return 0 |
| |
| def test_full_bisector(self): |
| ret = run_bisect.Run(self.FullBisector({}, {})) |
| self.assertEqual(ret, 0) |
| self.assertFalse(os.path.exists(common.OBJECTS_FILE)) |
| self.assertFalse(os.path.exists(common.WORKING_SET_FILE)) |
| |
| def check_output(self): |
| _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( |
| ( |
| 'grep "Bad items are: " logs/binary_search_tool_test.py.out | ' |
| "tail -n1" |
| ) |
| ) |
| ls = out.splitlines() |
| self.assertEqual(len(ls), 1) |
| line = ls[0] |
| |
| _, _, bad_ones = line.partition("Bad items are: ") |
| bad_ones = bad_ones.split() |
| expected_result = common.ReadObjectsFile() |
| |
| # Reconstruct objects file from bad_ones and compare |
| actual_result = [0] * len(expected_result) |
| for bad_obj in bad_ones: |
| actual_result[int(bad_obj)] = 1 |
| |
| self.assertEqual(actual_result, expected_result) |
| |
| |
| class BisectingUtilsTest(unittest.TestCase): |
| """Tests for bisecting tool.""" |
| |
| def setUp(self): |
| """Generate [100-1000] object files, and 1-5% of which are bad ones.""" |
| GenObj() |
| |
| with open("./is_setup", "w", encoding="utf-8"): |
| pass |
| |
| try: |
| os.remove(binary_search_state.STATE_FILE) |
| except OSError: |
| pass |
| |
| def tearDown(self): |
| """Cleanup temp files.""" |
| CleanObj() |
| |
| try: |
| os.remove(os.readlink(binary_search_state.STATE_FILE)) |
| except OSError: |
| pass |
| |
| cleanup_list = [ |
| "./is_setup", |
| binary_search_state.STATE_FILE, |
| "noinc_prune_bad", |
| "noinc_prune_good", |
| "./cmd_script.sh", |
| ] |
| for f in cleanup_list: |
| if os.path.exists(f): |
| os.remove(f) |
| |
| def runTest(self): |
| ret = binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| prune=True, |
| file_args=True, |
| ) |
| self.assertEqual(ret, 0) |
| self.check_output() |
| |
| def test_arg_parse(self): |
| args = [ |
| "--get_initial_items", |
| "./gen_init_list.py", |
| "--switch_to_good", |
| "./switch_to_good.py", |
| "--switch_to_bad", |
| "./switch_to_bad.py", |
| "--test_script", |
| "./is_good.py", |
| "--prune", |
| "--file_args", |
| ] |
| ret = binary_search_state.Main(args) |
| self.assertEqual(ret, 0) |
| self.check_output() |
| |
| def test_test_setup_script(self): |
| os.remove("./is_setup") |
| with self.assertRaises(AssertionError): |
| ret = binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| prune=True, |
| file_args=True, |
| ) |
| |
| ret = binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| test_setup_script="./test_setup.py", |
| prune=True, |
| file_args=True, |
| ) |
| self.assertEqual(ret, 0) |
| self.check_output() |
| |
| def test_bad_test_setup_script(self): |
| with self.assertRaises(AssertionError): |
| binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| test_setup_script="./test_setup_bad.py", |
| prune=True, |
| file_args=True, |
| ) |
| |
| def test_bad_save_state(self): |
| state_file = binary_search_state.STATE_FILE |
| hidden_state_file = os.path.basename( |
| binary_search_state.HIDDEN_STATE_FILE |
| ) |
| |
| with open(state_file, "w", encoding="utf-8") as f: |
| f.write("test123") |
| |
| bss = binary_search_state.MockBinarySearchState() |
| with self.assertRaises(OSError): |
| bss.SaveState() |
| |
| with open(state_file, "r", encoding="utf-8") as f: |
| self.assertEqual(f.read(), "test123") |
| |
| os.remove(state_file) |
| |
| # Cleanup generated save state that has no symlink |
| files = os.listdir(os.getcwd()) |
| save_states = [x for x in files if x.startswith(hidden_state_file)] |
| _ = [os.remove(x) for x in save_states] |
| |
| def test_save_state(self): |
| state_file = binary_search_state.STATE_FILE |
| |
| bss = binary_search_state.MockBinarySearchState() |
| bss.SaveState() |
| self.assertTrue(os.path.exists(state_file)) |
| first_state = os.readlink(state_file) |
| |
| bss.SaveState() |
| second_state = os.readlink(state_file) |
| self.assertTrue(os.path.exists(state_file)) |
| self.assertTrue(second_state != first_state) |
| self.assertFalse(os.path.exists(first_state)) |
| |
| bss.RemoveState() |
| self.assertFalse(os.path.islink(state_file)) |
| self.assertFalse(os.path.exists(second_state)) |
| |
| def test_load_state(self): |
| test_items = [1, 2, 3, 4, 5] |
| |
| bss = binary_search_state.MockBinarySearchState() |
| bss.all_items = test_items |
| bss.currently_good_items = set([1, 2, 3]) |
| bss.currently_bad_items = set([4, 5]) |
| bss.SaveState() |
| |
| bss = None |
| |
| bss2 = binary_search_state.MockBinarySearchState.LoadState() |
| self.assertEqual(bss2.all_items, test_items) |
| self.assertEqual(bss2.currently_good_items, set([])) |
| self.assertEqual(bss2.currently_bad_items, set([])) |
| |
| def test_tmp_cleanup(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items='echo "0\n1\n2\n3"', |
| switch_to_good="./switch_tmp.py", |
| file_args=True, |
| ) |
| bss.SwitchToGood(["0", "1", "2", "3"]) |
| |
| tmp_file = None |
| with open("tmp_file", "r", encoding="utf-8") as f: |
| tmp_file = f.read() |
| os.remove("tmp_file") |
| |
| self.assertFalse(os.path.exists(tmp_file)) |
| ws = common.ReadWorkingSet() |
| for i in range(3): |
| self.assertEqual(ws[i], 42) |
| |
| def test_verify_fail(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_bad.py", |
| switch_to_bad="./switch_to_good.py", |
| test_script="./is_good.py", |
| prune=True, |
| file_args=True, |
| verify=True, |
| ) |
| with self.assertRaises(AssertionError): |
| bss.DoVerify() |
| |
| def test_early_terminate(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| prune=True, |
| file_args=True, |
| iterations=1, |
| ) |
| bss.DoSearchBadItems() |
| self.assertFalse(bss.found_items) |
| |
| def test_no_prune(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| test_setup_script="./test_setup.py", |
| prune=False, |
| file_args=True, |
| ) |
| bss.DoSearchBadItems() |
| self.assertEqual(len(bss.found_items), 1) |
| |
| bad_objs = common.ReadObjectsFile() |
| found_obj = int(bss.found_items.pop()) |
| self.assertEqual(bad_objs[found_obj], 1) |
| |
| def test_set_file(self): |
| binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good_set_file.py", |
| switch_to_bad="./switch_to_bad_set_file.py", |
| test_script="./is_good.py", |
| prune=True, |
| file_args=True, |
| verify=True, |
| ) |
| self.check_output() |
| |
| def test_noincremental_prune(self): |
| ret = binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good_noinc_prune.py", |
| switch_to_bad="./switch_to_bad_noinc_prune.py", |
| test_script="./is_good_noinc_prune.py", |
| test_setup_script="./test_setup.py", |
| prune=True, |
| noincremental=True, |
| file_args=True, |
| verify=False, |
| ) |
| self.assertEqual(ret, 0) |
| self.check_output() |
| |
| def check_output(self): |
| _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( |
| ( |
| 'grep "Bad items are: " logs/binary_search_tool_test.py.out | ' |
| "tail -n1" |
| ) |
| ) |
| ls = out.splitlines() |
| self.assertEqual(len(ls), 1) |
| line = ls[0] |
| |
| _, _, bad_ones = line.partition("Bad items are: ") |
| bad_ones = bad_ones.split() |
| expected_result = common.ReadObjectsFile() |
| |
| # Reconstruct objects file from bad_ones and compare |
| actual_result = [0] * len(expected_result) |
| for bad_obj in bad_ones: |
| actual_result[int(bad_obj)] = 1 |
| |
| self.assertEqual(actual_result, expected_result) |
| |
| |
| class BisectingUtilsPassTest(BisectingUtilsTest): |
| """Tests for bisecting tool at pass/transformation level.""" |
| |
| def check_pass_output(self, pass_name, pass_num, trans_num): |
| _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( |
| ( |
| 'grep "Bad pass: " logs/binary_search_tool_test.py.out | ' |
| "tail -n1" |
| ) |
| ) |
| ls = out.splitlines() |
| self.assertEqual(len(ls), 1) |
| line = ls[0] |
| _, _, bad_info = line.partition("Bad pass: ") |
| actual_info = pass_name + " at number " + str(pass_num) |
| self.assertEqual(actual_info, bad_info) |
| |
| _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( |
| ( |
| 'grep "Bad transformation number: ' |
| '" logs/binary_search_tool_test.py.out | ' |
| "tail -n1" |
| ) |
| ) |
| ls = out.splitlines() |
| self.assertEqual(len(ls), 1) |
| line = ls[0] |
| _, _, bad_info = line.partition("Bad transformation number: ") |
| actual_info = str(trans_num) |
| self.assertEqual(actual_info, bad_info) |
| |
| def test_with_prune(self): |
| ret = binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| pass_bisect="./generate_cmd.py", |
| prune=True, |
| file_args=True, |
| ) |
| self.assertEqual(ret, 1) |
| |
| def test_gen_cmd_script(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| pass_bisect="./generate_cmd.py", |
| prune=False, |
| file_args=True, |
| ) |
| bss.DoSearchBadItems() |
| cmd_script_path = bss.cmd_script |
| self.assertTrue(os.path.exists(cmd_script_path)) |
| |
| def test_no_pass_support(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| pass_bisect="./generate_cmd.py", |
| prune=False, |
| file_args=True, |
| ) |
| bss.cmd_script = "./cmd_script_no_support.py" |
| # No support for -opt-bisect-limit |
| with self.assertRaises(RuntimeError): |
| bss.BuildWithPassLimit(-1) |
| |
| def test_no_transform_support(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| pass_bisect="./generate_cmd.py", |
| prune=False, |
| file_args=True, |
| ) |
| bss.cmd_script = "./cmd_script_no_support.py" |
| # No support for -print-debug-counter |
| with self.assertRaises(RuntimeError): |
| bss.BuildWithTransformLimit(-1, "counter_name") |
| |
| def test_pass_transform_bisect(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| pass_bisect="./generate_cmd.py", |
| prune=False, |
| file_args=True, |
| ) |
| pass_num = 4 |
| trans_num = 19 |
| bss.cmd_script = "./cmd_script.py %d %d" % (pass_num, trans_num) |
| bss.DoSearchBadPass() |
| self.check_pass_output("instcombine-visit", pass_num, trans_num) |
| |
| def test_result_not_reproduced_pass(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| pass_bisect="./generate_cmd.py", |
| prune=False, |
| file_args=True, |
| ) |
| # Fails reproducing at pass level. |
| pass_num = 0 |
| trans_num = 19 |
| bss.cmd_script = "./cmd_script.py %d %d" % (pass_num, trans_num) |
| with self.assertRaises(ValueError): |
| bss.DoSearchBadPass() |
| |
| def test_result_not_reproduced_transform(self): |
| bss = binary_search_state.MockBinarySearchState( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| pass_bisect="./generate_cmd.py", |
| prune=False, |
| file_args=True, |
| ) |
| # Fails reproducing at transformation level. |
| pass_num = 4 |
| trans_num = 0 |
| bss.cmd_script = "./cmd_script.py %d %d" % (pass_num, trans_num) |
| with self.assertRaises(ValueError): |
| bss.DoSearchBadPass() |
| |
| |
| class BisectStressTest(unittest.TestCase): |
| """Stress tests for bisecting tool.""" |
| |
| def test_every_obj_bad(self): |
| amt = 25 |
| gen_obj.Main(["--obj_num", str(amt), "--bad_obj_num", str(amt)]) |
| ret = binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_script="./is_good.py", |
| prune=True, |
| file_args=True, |
| verify=False, |
| ) |
| self.assertEqual(ret, 0) |
| self.check_output() |
| |
| def test_every_index_is_bad(self): |
| amt = 25 |
| for i in range(amt): |
| obj_list = ["0"] * amt |
| obj_list[i] = "1" |
| obj_list = ",".join(obj_list) |
| gen_obj.Main(["--obj_list", obj_list]) |
| ret = binary_search_state.Run( |
| get_initial_items="./gen_init_list.py", |
| switch_to_good="./switch_to_good.py", |
| switch_to_bad="./switch_to_bad.py", |
| test_setup_script="./test_setup.py", |
| test_script="./is_good.py", |
| prune=True, |
| file_args=True, |
| ) |
| self.assertEqual(ret, 0) |
| self.check_output() |
| |
| def check_output(self): |
| _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( |
| ( |
| 'grep "Bad items are: " logs/binary_search_tool_test.py.out | ' |
| "tail -n1" |
| ) |
| ) |
| ls = out.splitlines() |
| self.assertEqual(len(ls), 1) |
| line = ls[0] |
| |
| _, _, bad_ones = line.partition("Bad items are: ") |
| bad_ones = bad_ones.split() |
| expected_result = common.ReadObjectsFile() |
| |
| # Reconstruct objects file from bad_ones and compare |
| actual_result = [0] * len(expected_result) |
| for bad_obj in bad_ones: |
| actual_result[int(bad_obj)] = 1 |
| |
| self.assertEqual(actual_result, expected_result) |
| |
| |
| def Main(argv): |
| num_tests = 2 |
| if len(argv) > 1: |
| num_tests = int(argv[1]) |
| |
| suite = unittest.TestSuite() |
| for _ in range(0, num_tests): |
| suite.addTest(BisectingUtilsTest()) |
| suite.addTest(BisectingUtilsTest("test_arg_parse")) |
| suite.addTest(BisectingUtilsTest("test_test_setup_script")) |
| suite.addTest(BisectingUtilsTest("test_bad_test_setup_script")) |
| suite.addTest(BisectingUtilsTest("test_bad_save_state")) |
| suite.addTest(BisectingUtilsTest("test_save_state")) |
| suite.addTest(BisectingUtilsTest("test_load_state")) |
| suite.addTest(BisectingUtilsTest("test_tmp_cleanup")) |
| suite.addTest(BisectingUtilsTest("test_verify_fail")) |
| suite.addTest(BisectingUtilsTest("test_early_terminate")) |
| suite.addTest(BisectingUtilsTest("test_no_prune")) |
| suite.addTest(BisectingUtilsTest("test_set_file")) |
| suite.addTest(BisectingUtilsTest("test_noincremental_prune")) |
| suite.addTest(BisectingUtilsPassTest("test_with_prune")) |
| suite.addTest(BisectingUtilsPassTest("test_gen_cmd_script")) |
| suite.addTest(BisectingUtilsPassTest("test_no_pass_support")) |
| suite.addTest(BisectingUtilsPassTest("test_no_transform_support")) |
| suite.addTest(BisectingUtilsPassTest("test_pass_transform_bisect")) |
| suite.addTest(BisectingUtilsPassTest("test_result_not_reproduced_pass")) |
| suite.addTest( |
| BisectingUtilsPassTest("test_result_not_reproduced_transform") |
| ) |
| suite.addTest(BisectTest("test_full_bisector")) |
| suite.addTest(BisectStressTest("test_every_obj_bad")) |
| suite.addTest(BisectStressTest("test_every_index_is_bad")) |
| runner = unittest.TextTestRunner() |
| runner.run(suite) |
| |
| |
| if __name__ == "__main__": |
| Main(sys.argv) |
