bisect tool: Adding unit tests for pass/transform level bisect
This patch added unit tests for pass and transformation level bisect to
the bisecting tool. Since no compiler is involved in unit test, it used
cmd_script.py to simulate the output of compiler (to stderr).
BUG=chromium:878954
TEST=Passed all unit tests.
Change-Id: I599c079497333ec24f08f37c3315a16f28c0f887
Reviewed-on: https://chromium-review.googlesource.com/1298384
Commit-Ready: Zhizhou Yang <zhizhouy@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Caroline Tice <cmtice@chromium.org>
diff --git a/binary_search_tool/test/binary_search_tool_tester.py b/binary_search_tool/test/binary_search_tool_tester.py
index 923ea11..aff45a8 100755
--- a/binary_search_tool/test/binary_search_tool_tester.py
+++ b/binary_search_tool/test/binary_search_tool_tester.py
@@ -132,7 +132,7 @@
cleanup_list = [
'./is_setup', binary_search_state.STATE_FILE, 'noinc_prune_bad',
- 'noinc_prune_good'
+ 'noinc_prune_good', './cmd_script.sh'
]
for f in cleanup_list:
if os.path.exists(f):
@@ -304,24 +304,6 @@
found_obj = int(bss.found_items.pop())
self.assertEquals(bad_objs[found_obj], 1)
- def test_pass_bisect(self):
- bss = binary_search_state.MockBinarySearchState(
- get_initial_items='./gen_init_list.py',
- switch_to_good='./switch_to_good.py',
- switch_to_bad='./switch_to_bad.py',
- pass_bisect='./generate_cmd.py',
- test_script='./is_good.py',
- test_setup_script='./test_setup.py',
- prune=False,
- file_args=True)
- # TODO: Need to design unit tests for pass level bisection
- bss.DoSearchBadItems()
- self.assertEquals(len(bss.found_items), 1)
-
- bad_objs = common.ReadObjectsFile()
- found_obj = int(bss.found_items.pop())
- self.assertEquals(bad_objs[found_obj], 1)
-
def test_set_file(self):
binary_search_state.Run(
get_initial_items='./gen_init_list.py',
@@ -367,6 +349,131 @@
self.assertEqual(actual_result, expected_result)
+class BisectingUtilsPassTest(BisectingUtilsTest):
+ """Tests for bisecting tool at pass/transformation level."""
+
+ def check_pass_output(self, pass_name, pass_num, trans_num):
+ _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
+ ('grep "Bad pass: " logs/binary_search_tool_tester.py.out | '
+ 'tail -n1'))
+ ls = out.splitlines()
+ self.assertEqual(len(ls), 1)
+ line = ls[0]
+ _, _, bad_info = line.partition('Bad pass: ')
+ actual_info = pass_name + ' at number ' + str(pass_num)
+ self.assertEqual(actual_info, bad_info)
+
+ _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
+ ('grep "Bad transformation number: '
+ '" logs/binary_search_tool_tester.py.out | '
+ 'tail -n1'))
+ ls = out.splitlines()
+ self.assertEqual(len(ls), 1)
+ line = ls[0]
+ _, _, bad_info = line.partition('Bad transformation number: ')
+ actual_info = str(trans_num)
+ self.assertEqual(actual_info, bad_info)
+
+ def test_with_prune(self):
+ ret = binary_search_state.Run(
+ get_initial_items='./gen_init_list.py',
+ switch_to_good='./switch_to_good.py',
+ switch_to_bad='./switch_to_bad.py',
+ test_script='./is_good.py',
+ pass_bisect='./generate_cmd.py',
+ prune=True,
+ file_args=True)
+ self.assertEquals(ret, 1)
+
+ def test_gen_cmd_script(self):
+ bss = binary_search_state.MockBinarySearchState(
+ get_initial_items='./gen_init_list.py',
+ switch_to_good='./switch_to_good.py',
+ switch_to_bad='./switch_to_bad.py',
+ test_script='./is_good.py',
+ pass_bisect='./generate_cmd.py',
+ prune=False,
+ file_args=True)
+ bss.DoSearchBadItems()
+ cmd_script_path = bss.cmd_script
+ self.assertTrue(os.path.exists(cmd_script_path))
+
+ def test_no_pass_support(self):
+ bss = binary_search_state.MockBinarySearchState(
+ get_initial_items='./gen_init_list.py',
+ switch_to_good='./switch_to_good.py',
+ switch_to_bad='./switch_to_bad.py',
+ test_script='./is_good.py',
+ pass_bisect='./generate_cmd.py',
+ prune=False,
+ file_args=True)
+ bss.cmd_script = './cmd_script_no_support.py'
+ # No support for -opt-bisect-limit
+ with self.assertRaises(RuntimeError):
+ bss.BuildWithPassLimit(-1)
+
+ def test_no_transform_support(self):
+ bss = binary_search_state.MockBinarySearchState(
+ get_initial_items='./gen_init_list.py',
+ switch_to_good='./switch_to_good.py',
+ switch_to_bad='./switch_to_bad.py',
+ test_script='./is_good.py',
+ pass_bisect='./generate_cmd.py',
+ prune=False,
+ file_args=True)
+ bss.cmd_script = './cmd_script_no_support.py'
+ # No support for -print-debug-counter
+ with self.assertRaises(RuntimeError):
+ bss.BuildWithTransformLimit(-1, 'counter_name')
+
+ def test_pass_transform_bisect(self):
+ bss = binary_search_state.MockBinarySearchState(
+ get_initial_items='./gen_init_list.py',
+ switch_to_good='./switch_to_good.py',
+ switch_to_bad='./switch_to_bad.py',
+ test_script='./is_good.py',
+ pass_bisect='./generate_cmd.py',
+ prune=False,
+ file_args=True)
+ pass_num = 4
+ trans_num = 19
+ bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
+ bss.DoSearchBadPass()
+ self.check_pass_output('instcombine-visit', pass_num, trans_num)
+
+ def test_result_not_reproduced_pass(self):
+ bss = binary_search_state.MockBinarySearchState(
+ get_initial_items='./gen_init_list.py',
+ switch_to_good='./switch_to_good.py',
+ switch_to_bad='./switch_to_bad.py',
+ test_script='./is_good.py',
+ pass_bisect='./generate_cmd.py',
+ prune=False,
+ file_args=True)
+ # Fails reproducing at pass level.
+ pass_num = 0
+ trans_num = 19
+ bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
+ with self.assertRaises(ValueError):
+ bss.DoSearchBadPass()
+
+ def test_result_not_reproduced_transform(self):
+ bss = binary_search_state.MockBinarySearchState(
+ get_initial_items='./gen_init_list.py',
+ switch_to_good='./switch_to_good.py',
+ switch_to_bad='./switch_to_bad.py',
+ test_script='./is_good.py',
+ pass_bisect='./generate_cmd.py',
+ prune=False,
+ file_args=True)
+ # Fails reproducing at transformation level.
+ pass_num = 4
+ trans_num = 0
+ bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
+ with self.assertRaises(ValueError):
+ bss.DoSearchBadPass()
+
+
class BisectStressTest(unittest.TestCase):
"""Stress tests for bisecting tool."""
@@ -442,6 +549,13 @@
suite.addTest(BisectingUtilsTest('test_no_prune'))
suite.addTest(BisectingUtilsTest('test_set_file'))
suite.addTest(BisectingUtilsTest('test_noincremental_prune'))
+ suite.addTest(BisectingUtilsPassTest('test_with_prune'))
+ suite.addTest(BisectingUtilsPassTest('test_gen_cmd_script'))
+ suite.addTest(BisectingUtilsPassTest('test_no_pass_support'))
+ suite.addTest(BisectingUtilsPassTest('test_no_transform_support'))
+ suite.addTest(BisectingUtilsPassTest('test_pass_transform_bisect'))
+ suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_pass'))
+ suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_transform'))
suite.addTest(BisectTest('test_full_bisector'))
suite.addTest(BisectStressTest('test_every_obj_bad'))
suite.addTest(BisectStressTest('test_every_index_is_bad'))
diff --git a/binary_search_tool/test/cmd_script.py b/binary_search_tool/test/cmd_script.py
new file mode 100755
index 0000000..6940eaa
--- /dev/null
+++ b/binary_search_tool/test/cmd_script.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python2
+"""Command script without compiler support for pass level bisection.
+
+This script generates a pseudo log which a workable compiler should print out.
+It assumes that -opt-bisect-limit and -print-debug-counter are supported by the
+compiler.
+"""
+
+from __future__ import print_function
+
+import os
+import sys
+
+import common
+
+
+def Main(argv):
+ if not os.path.exists('./is_setup'):
+ return 1
+
+ if len(argv) != 3:
+ return 1
+
+ limit_flags = os.environ['LIMIT_FLAGS']
+ opt_bisect_exist = False
+ debug_counter_exist = False
+
+ for option in limit_flags.split():
+ if '-opt-bisect-limit' in option:
+ opt_bisect_limit = int(option.split('=')[-1])
+ opt_bisect_exist = True
+ if '-debug-counter=' in option:
+ debug_counter = int(option.split('=')[-1])
+ debug_counter_exist = True
+
+ if not opt_bisect_exist:
+ return 1
+
+ # Manually set total number and bad number
+ total_pass = 10
+ total_transform = 20
+ bad_pass = int(argv[1])
+ bad_transform = int(argv[2])
+
+ if opt_bisect_limit == -1:
+ opt_bisect_limit = total_pass
+
+ for i in xrange(1, total_pass + 1):
+ bisect_str = 'BISECT: %srunning pass (%d) Combine redundant ' \
+ 'instructions on function (f1)' \
+ % ('NOT ' if i > opt_bisect_limit else '', i)
+ print(bisect_str, file=sys.stderr)
+
+ if debug_counter_exist:
+ print('Counters and values:', file=sys.stderr)
+ print(
+ 'instcombine-visit : {%d, 0, %d}' % (total_transform, debug_counter),
+ file=sys.stderr)
+
+ if opt_bisect_limit > bad_pass or \
+ (debug_counter_exist and debug_counter > bad_transform):
+ common.WriteWorkingSet([1])
+ else:
+ common.WriteWorkingSet([0])
+
+ return 0
+
+
+if __name__ == '__main__':
+ retval = Main(sys.argv)
+ sys.exit(retval)
diff --git a/binary_search_tool/test/cmd_script_no_support.py b/binary_search_tool/test/cmd_script_no_support.py
new file mode 100644
index 0000000..a817f30
--- /dev/null
+++ b/binary_search_tool/test/cmd_script_no_support.py
@@ -0,0 +1,23 @@
+"""Command script without compiler support for pass level bisection.
+
+This script generates a pseudo log when certain bisecting flags are not
+supported by compiler.
+"""
+
+from __future__ import print_function
+
+import os
+import sys
+
+
+def Main():
+ if not os.path.exists('./is_setup'):
+ return 1
+ print('No support for -opt-bisect-limit or -print-debug-counter.',
+ file=sys.stderr)
+ return 0
+
+
+if __name__ == '__main__':
+ retval = Main()
+ sys.exit(retval)
diff --git a/binary_search_tool/test/gen_obj.py b/binary_search_tool/test/gen_obj.py
index d17e93f..a2bc7d9 100755
--- a/binary_search_tool/test/gen_obj.py
+++ b/binary_search_tool/test/gen_obj.py
@@ -85,7 +85,7 @@
f.close()
obj_num = len(obj_list)
- bad_obj_num = obj_list.count('1')
+ bad_obj_num = obj_list.count(1)
print('Generated {0} object files, with {1} bad ones.'.format(
obj_num, bad_obj_num))
diff --git a/binary_search_tool/test/generate_cmd.py b/binary_search_tool/test/generate_cmd.py
new file mode 100755
index 0000000..f6876ed
--- /dev/null
+++ b/binary_search_tool/test/generate_cmd.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python2
+"""Generate a virtual cmd script for pass level bisection.
+
+This is a required argument for pass level bisecting. For unit test, we use
+this script to verify if cmd_script.sh is generated correctly.
+"""
+
+from __future__ import print_function
+
+import os
+import sys
+
+
+def Main():
+ if not os.path.exists('./is_setup'):
+ return 1
+ file_name = 'cmd_script.sh'
+ with open(file_name, 'w') as f:
+ f.write('Generated by generate_cmd.py')
+ return 0
+
+
+if __name__ == '__main__':
+ retval = Main()
+ sys.exit(retval)
