#!/usr/bin/env python2

# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Tests for bisecting tool."""

from __future__ import print_function

__author__ = 'shenhan@google.com (Han Shen)'

import os
import random
import sys
import unittest

from cros_utils import command_executer
from binary_search_tool import binary_search_state
from binary_search_tool import bisect

import common
import gen_obj


def GenObj():
  obj_num = random.randint(100, 1000)
  bad_obj_num = random.randint(obj_num / 100, obj_num / 20)
  if bad_obj_num == 0:
    bad_obj_num = 1
  gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)])


def CleanObj():
  os.remove(common.OBJECTS_FILE)
  os.remove(common.WORKING_SET_FILE)
  print('Deleted "{0}" and "{1}"'.format(common.OBJECTS_FILE,
                                         common.WORKING_SET_FILE))


class BisectTest(unittest.TestCase):
  """Tests for bisect.py"""

  def setUp(self):
    with open('./is_setup', 'w'):
      pass

    try:
      os.remove(binary_search_state.STATE_FILE)
    except OSError:
      pass

  def tearDown(self):
    try:
      os.remove('./is_setup')
      os.remove(os.readlink(binary_search_state.STATE_FILE))
      os.remove(binary_search_state.STATE_FILE)
    except OSError:
      pass

  class FullBisector(bisect.Bisector):
    """Test bisector to test bisect.py with"""

    def __init__(self, options, overrides):
      super(BisectTest.FullBisector, self).__init__(options, overrides)

    def PreRun(self):
      GenObj()
      return 0

    def Run(self):
      return binary_search_state.Run(
          get_initial_items='./gen_init_list.py',
          switch_to_good='./switch_to_good.py',
          switch_to_bad='./switch_to_bad.py',
          test_script='./is_good.py',
          prune=True,
          file_args=True)

    def PostRun(self):
      CleanObj()
      return 0

  def test_full_bisector(self):
    ret = bisect.Run(self.FullBisector({}, {}))
    self.assertEquals(ret, 0)
    self.assertFalse(os.path.exists(common.OBJECTS_FILE))
    self.assertFalse(os.path.exists(common.WORKING_SET_FILE))

  def check_output(self):
    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
        ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
         'tail -n1'))
    ls = out.splitlines()
    self.assertEqual(len(ls), 1)
    line = ls[0]

    _, _, bad_ones = line.partition('Bad items are: ')
    bad_ones = bad_ones.split()
    expected_result = common.ReadObjectsFile()

    # Reconstruct objects file from bad_ones and compare
    actual_result = [0] * len(expected_result)
    for bad_obj in bad_ones:
      actual_result[int(bad_obj)] = 1

    self.assertEqual(actual_result, expected_result)


class BisectingUtilsTest(unittest.TestCase):
  """Tests for bisecting tool."""

  def setUp(self):
    """Generate [100-1000] object files, and 1-5% of which are bad ones."""
    GenObj()

    with open('./is_setup', 'w'):
      pass

    try:
      os.remove(binary_search_state.STATE_FILE)
    except OSError:
      pass

  def tearDown(self):
    """Cleanup temp files."""
    CleanObj()

    try:
      os.remove(os.readlink(binary_search_state.STATE_FILE))
    except OSError:
      pass

    cleanup_list = [
        './is_setup', binary_search_state.STATE_FILE, 'noinc_prune_bad',
        'noinc_prune_good', './cmd_script.sh'
    ]
    for f in cleanup_list:
      if os.path.exists(f):
        os.remove(f)

  def runTest(self):
    ret = binary_search_state.Run(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        prune=True,
        file_args=True)
    self.assertEquals(ret, 0)
    self.check_output()

  def test_arg_parse(self):
    args = [
        '--get_initial_items', './gen_init_list.py', '--switch_to_good',
        './switch_to_good.py', '--switch_to_bad', './switch_to_bad.py',
        '--test_script', './is_good.py', '--prune', '--file_args'
    ]
    ret = binary_search_state.Main(args)
    self.assertEquals(ret, 0)
    self.check_output()

  def test_test_setup_script(self):
    os.remove('./is_setup')
    with self.assertRaises(AssertionError):
      ret = binary_search_state.Run(
          get_initial_items='./gen_init_list.py',
          switch_to_good='./switch_to_good.py',
          switch_to_bad='./switch_to_bad.py',
          test_script='./is_good.py',
          prune=True,
          file_args=True)

    ret = binary_search_state.Run(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        test_setup_script='./test_setup.py',
        prune=True,
        file_args=True)
    self.assertEquals(ret, 0)
    self.check_output()

  def test_bad_test_setup_script(self):
    with self.assertRaises(AssertionError):
      binary_search_state.Run(
          get_initial_items='./gen_init_list.py',
          switch_to_good='./switch_to_good.py',
          switch_to_bad='./switch_to_bad.py',
          test_script='./is_good.py',
          test_setup_script='./test_setup_bad.py',
          prune=True,
          file_args=True)

  def test_bad_save_state(self):
    state_file = binary_search_state.STATE_FILE
    hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE)

    with open(state_file, 'w') as f:
      f.write('test123')

    bss = binary_search_state.MockBinarySearchState()
    with self.assertRaises(binary_search_state.Error):
      bss.SaveState()

    with open(state_file, 'r') as f:
      self.assertEquals(f.read(), 'test123')

    os.remove(state_file)

    # Cleanup generated save state that has no symlink
    files = os.listdir(os.getcwd())
    save_states = [x for x in files if x.startswith(hidden_state_file)]
    _ = [os.remove(x) for x in save_states]

  def test_save_state(self):
    state_file = binary_search_state.STATE_FILE

    bss = binary_search_state.MockBinarySearchState()
    bss.SaveState()
    self.assertTrue(os.path.exists(state_file))
    first_state = os.readlink(state_file)

    bss.SaveState()
    second_state = os.readlink(state_file)
    self.assertTrue(os.path.exists(state_file))
    self.assertTrue(second_state != first_state)
    self.assertFalse(os.path.exists(first_state))

    bss.RemoveState()
    self.assertFalse(os.path.islink(state_file))
    self.assertFalse(os.path.exists(second_state))

  def test_load_state(self):
    test_items = [1, 2, 3, 4, 5]

    bss = binary_search_state.MockBinarySearchState()
    bss.all_items = test_items
    bss.currently_good_items = set([1, 2, 3])
    bss.currently_bad_items = set([4, 5])
    bss.SaveState()

    bss = None

    bss2 = binary_search_state.MockBinarySearchState.LoadState()
    self.assertEquals(bss2.all_items, test_items)
    self.assertEquals(bss2.currently_good_items, set([]))
    self.assertEquals(bss2.currently_bad_items, set([]))

  def test_tmp_cleanup(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='echo "0\n1\n2\n3"',
        switch_to_good='./switch_tmp.py',
        file_args=True)
    bss.SwitchToGood(['0', '1', '2', '3'])

    tmp_file = None
    with open('tmp_file', 'r') as f:
      tmp_file = f.read()
    os.remove('tmp_file')

    self.assertFalse(os.path.exists(tmp_file))
    ws = common.ReadWorkingSet()
    for i in range(3):
      self.assertEquals(ws[i], 42)

  def test_verify_fail(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_bad.py',
        switch_to_bad='./switch_to_good.py',
        test_script='./is_good.py',
        prune=True,
        file_args=True,
        verify=True)
    with self.assertRaises(AssertionError):
      bss.DoVerify()

  def test_early_terminate(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        prune=True,
        file_args=True,
        iterations=1)
    bss.DoSearchBadItems()
    self.assertFalse(bss.found_items)

  def test_no_prune(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        test_setup_script='./test_setup.py',
        prune=False,
        file_args=True)
    bss.DoSearchBadItems()
    self.assertEquals(len(bss.found_items), 1)

    bad_objs = common.ReadObjectsFile()
    found_obj = int(bss.found_items.pop())
    self.assertEquals(bad_objs[found_obj], 1)

  def test_set_file(self):
    binary_search_state.Run(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good_set_file.py',
        switch_to_bad='./switch_to_bad_set_file.py',
        test_script='./is_good.py',
        prune=True,
        file_args=True,
        verify=True)
    self.check_output()

  def test_noincremental_prune(self):
    ret = binary_search_state.Run(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good_noinc_prune.py',
        switch_to_bad='./switch_to_bad_noinc_prune.py',
        test_script='./is_good_noinc_prune.py',
        test_setup_script='./test_setup.py',
        prune=True,
        noincremental=True,
        file_args=True,
        verify=False)
    self.assertEquals(ret, 0)
    self.check_output()

  def check_output(self):
    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
        ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
         'tail -n1'))
    ls = out.splitlines()
    self.assertEqual(len(ls), 1)
    line = ls[0]

    _, _, bad_ones = line.partition('Bad items are: ')
    bad_ones = bad_ones.split()
    expected_result = common.ReadObjectsFile()

    # Reconstruct objects file from bad_ones and compare
    actual_result = [0] * len(expected_result)
    for bad_obj in bad_ones:
      actual_result[int(bad_obj)] = 1

    self.assertEqual(actual_result, expected_result)


class BisectingUtilsPassTest(BisectingUtilsTest):
  """Tests for bisecting tool at pass/transformation level."""

  def check_pass_output(self, pass_name, pass_num, trans_num):
    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
        ('grep "Bad pass: " logs/binary_search_tool_tester.py.out | '
         'tail -n1'))
    ls = out.splitlines()
    self.assertEqual(len(ls), 1)
    line = ls[0]
    _, _, bad_info = line.partition('Bad pass: ')
    actual_info = pass_name + ' at number ' + str(pass_num)
    self.assertEqual(actual_info, bad_info)

    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
        ('grep "Bad transformation number: '
         '" logs/binary_search_tool_tester.py.out | '
         'tail -n1'))
    ls = out.splitlines()
    self.assertEqual(len(ls), 1)
    line = ls[0]
    _, _, bad_info = line.partition('Bad transformation number: ')
    actual_info = str(trans_num)
    self.assertEqual(actual_info, bad_info)

  def test_with_prune(self):
    ret = binary_search_state.Run(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        pass_bisect='./generate_cmd.py',
        prune=True,
        file_args=True)
    self.assertEquals(ret, 1)

  def test_gen_cmd_script(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        pass_bisect='./generate_cmd.py',
        prune=False,
        file_args=True)
    bss.DoSearchBadItems()
    cmd_script_path = bss.cmd_script
    self.assertTrue(os.path.exists(cmd_script_path))

  def test_no_pass_support(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        pass_bisect='./generate_cmd.py',
        prune=False,
        file_args=True)
    bss.cmd_script = './cmd_script_no_support.py'
    # No support for -opt-bisect-limit
    with self.assertRaises(RuntimeError):
      bss.BuildWithPassLimit(-1)

  def test_no_transform_support(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        pass_bisect='./generate_cmd.py',
        prune=False,
        file_args=True)
    bss.cmd_script = './cmd_script_no_support.py'
    # No support for -print-debug-counter
    with self.assertRaises(RuntimeError):
      bss.BuildWithTransformLimit(-1, 'counter_name')

  def test_pass_transform_bisect(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        pass_bisect='./generate_cmd.py',
        prune=False,
        file_args=True)
    pass_num = 4
    trans_num = 19
    bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
    bss.DoSearchBadPass()
    self.check_pass_output('instcombine-visit', pass_num, trans_num)

  def test_result_not_reproduced_pass(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        pass_bisect='./generate_cmd.py',
        prune=False,
        file_args=True)
    # Fails reproducing at pass level.
    pass_num = 0
    trans_num = 19
    bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
    with self.assertRaises(ValueError):
      bss.DoSearchBadPass()

  def test_result_not_reproduced_transform(self):
    bss = binary_search_state.MockBinarySearchState(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        pass_bisect='./generate_cmd.py',
        prune=False,
        file_args=True)
    # Fails reproducing at transformation level.
    pass_num = 4
    trans_num = 0
    bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
    with self.assertRaises(ValueError):
      bss.DoSearchBadPass()


class BisectStressTest(unittest.TestCase):
  """Stress tests for bisecting tool."""

  def test_every_obj_bad(self):
    amt = 25
    gen_obj.Main(['--obj_num', str(amt), '--bad_obj_num', str(amt)])
    ret = binary_search_state.Run(
        get_initial_items='./gen_init_list.py',
        switch_to_good='./switch_to_good.py',
        switch_to_bad='./switch_to_bad.py',
        test_script='./is_good.py',
        prune=True,
        file_args=True,
        verify=False)
    self.assertEquals(ret, 0)
    self.check_output()

  def test_every_index_is_bad(self):
    amt = 25
    for i in range(amt):
      obj_list = ['0'] * amt
      obj_list[i] = '1'
      obj_list = ','.join(obj_list)
      gen_obj.Main(['--obj_list', obj_list])
      ret = binary_search_state.Run(
          get_initial_items='./gen_init_list.py',
          switch_to_good='./switch_to_good.py',
          switch_to_bad='./switch_to_bad.py',
          test_setup_script='./test_setup.py',
          test_script='./is_good.py',
          prune=True,
          file_args=True)
      self.assertEquals(ret, 0)
      self.check_output()

  def check_output(self):
    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
        ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
         'tail -n1'))
    ls = out.splitlines()
    self.assertEqual(len(ls), 1)
    line = ls[0]

    _, _, bad_ones = line.partition('Bad items are: ')
    bad_ones = bad_ones.split()
    expected_result = common.ReadObjectsFile()

    # Reconstruct objects file from bad_ones and compare
    actual_result = [0] * len(expected_result)
    for bad_obj in bad_ones:
      actual_result[int(bad_obj)] = 1

    self.assertEqual(actual_result, expected_result)


def Main(argv):
  num_tests = 2
  if len(argv) > 1:
    num_tests = int(argv[1])

  suite = unittest.TestSuite()
  for _ in range(0, num_tests):
    suite.addTest(BisectingUtilsTest())
  suite.addTest(BisectingUtilsTest('test_arg_parse'))
  suite.addTest(BisectingUtilsTest('test_test_setup_script'))
  suite.addTest(BisectingUtilsTest('test_bad_test_setup_script'))
  suite.addTest(BisectingUtilsTest('test_bad_save_state'))
  suite.addTest(BisectingUtilsTest('test_save_state'))
  suite.addTest(BisectingUtilsTest('test_load_state'))
  suite.addTest(BisectingUtilsTest('test_tmp_cleanup'))
  suite.addTest(BisectingUtilsTest('test_verify_fail'))
  suite.addTest(BisectingUtilsTest('test_early_terminate'))
  suite.addTest(BisectingUtilsTest('test_no_prune'))
  suite.addTest(BisectingUtilsTest('test_set_file'))
  suite.addTest(BisectingUtilsTest('test_noincremental_prune'))
  suite.addTest(BisectingUtilsPassTest('test_with_prune'))
  suite.addTest(BisectingUtilsPassTest('test_gen_cmd_script'))
  suite.addTest(BisectingUtilsPassTest('test_no_pass_support'))
  suite.addTest(BisectingUtilsPassTest('test_no_transform_support'))
  suite.addTest(BisectingUtilsPassTest('test_pass_transform_bisect'))
  suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_pass'))
  suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_transform'))
  suite.addTest(BisectTest('test_full_bisector'))
  suite.addTest(BisectStressTest('test_every_obj_bad'))
  suite.addTest(BisectStressTest('test_every_index_is_bad'))
  runner = unittest.TextTestRunner()
  runner.run(suite)


if __name__ == '__main__':
  Main(sys.argv)