llvm_tools/auto_llvm_bisection_unittest.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Tests for auto bisection of LLVM."""

from __future__ import print_function

import os
import subprocess
import time
import traceback
import unittest
import unittest.mock as mock

import auto_llvm_bisection
import chroot
import llvm_bisection
import test_helpers


class AutoLLVMBisectionTest(unittest.TestCase):
  """Unittests for auto bisection of LLVM."""

  # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking
  # the script outside of the chroot.
  @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
  # Simulate behavior of `time.sleep()` when waiting for errors to settle caused
  # by `llvm_bisection.main()` (e.g. network issue, etc.).
  @mock.patch.object(time, 'sleep')
  # Simulate behavior of `traceback.print_exc()` when an exception happened in
  # `llvm_bisection.main()`.
  @mock.patch.object(traceback, 'print_exc')
  # Simulate behavior of `llvm_bisection.main()` when failed to launch tryjobs
  # (exception happened along the way, etc.).
  @mock.patch.object(llvm_bisection, 'main')
  # Simulate behavior of `os.path.isfile()` when starting a new bisection.
  @mock.patch.object(os.path, 'isfile', return_value=False)
  # Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when
  # returning the absolute path to that script that updates all 'pending'
  # tryjobs to the result of `cros buildresult`.
  @mock.patch.object(
      auto_llvm_bisection,
      'GetPathToUpdateAllTryjobsWithAutoScript',
      return_value='/abs/path/to/update_tryjob.py')
  # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line
  # arguments required by the bisection script.
  @mock.patch.object(
      llvm_bisection,
      'GetCommandLineArgs',
      return_value=test_helpers.ArgsOutputTest())
  def testFailedToStartBisection(
      self, mock_get_args, mock_get_auto_script, mock_is_file,
      mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot):

    def MockLLVMBisectionRaisesException(_args_output):
      raise ValueError('Failed to launch more tryjobs.')

    # Use the test function to simulate the behavior of an exception happening
    # when launching more tryjobs.
    mock_llvm_bisection.side_effect = MockLLVMBisectionRaisesException

    # Verify the exception is raised when the number of attempts to launched
    # more tryjobs is exceeded, so unable to continue
    # bisection.
    with self.assertRaises(SystemExit) as err:
      auto_llvm_bisection.main()

    self.assertEqual(err.exception.code, 1)

    mock_outside_chroot.assert_called_once()
    mock_get_args.assert_called_once()
    mock_get_auto_script.assert_called_once()
    self.assertEqual(mock_is_file.call_count, 2)
    self.assertEqual(mock_llvm_bisection.call_count, 3)
    self.assertEqual(mock_traceback.call_count, 3)
    self.assertEqual(mock_sleep.call_count, 2)

  # Simulate the behavior of `subprocess.call()` when successfully updated all
  # tryjobs whose 'status' value is 'pending'.
  @mock.patch.object(subprocess, 'call', return_value=0)
  # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking
  # the script outside of the chroot.
  @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
  # Simulate behavior of `time.sleep()` when waiting for errors to settle caused
  # by `llvm_bisection.main()` (e.g. network issue, etc.).
  @mock.patch.object(time, 'sleep')
  # Simulate behavior of `traceback.print_exc()` when an exception happened in
  # `llvm_bisection.main()`.
  @mock.patch.object(traceback, 'print_exc')
  # Simulate behavior of `llvm_bisection.main()` when failed to launch tryjobs
  # (exception happened along the way, etc.).
  @mock.patch.object(llvm_bisection, 'main')
  # Simulate behavior of `os.path.isfile()` when starting a new bisection.
  @mock.patch.object(os.path, 'isfile')
  # Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when
  # returning the absolute path to that script that updates all 'pending'
  # tryjobs to the result of `cros buildresult`.
  @mock.patch.object(
      auto_llvm_bisection,
      'GetPathToUpdateAllTryjobsWithAutoScript',
      return_value='/abs/path/to/update_tryjob.py')
  # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line
  # arguments required by the bisection script.
  @mock.patch.object(
      llvm_bisection,
      'GetCommandLineArgs',
      return_value=test_helpers.ArgsOutputTest())
  def testSuccessfullyBisectedLLVMRevision(
      self, mock_get_args, mock_get_auto_script, mock_is_file,
      mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot,
      mock_update_tryjobs):

    # Simulate the behavior of `os.path.isfile()` when checking whether the
    # status file provided exists.
    @test_helpers.CallCountsToMockFunctions
    def MockStatusFileCheck(call_count, _last_tested):
      # Simulate that the status file does not exist, so the LLVM bisection
      # script would create the status file and launch tryjobs.
      if call_count < 2:
        return False

      # Simulate when the status file exists and `subprocess.call()` executes
      # the script that updates all the 'pending' tryjobs to the result of `cros
      # buildresult`.
      if call_count == 2:
        return True

      assert False, 'os.path.isfile() called more times than expected.'

    # Simulate behavior of `llvm_bisection.main()` when successfully bisected
    # between the good and bad LLVM revision.
    @test_helpers.CallCountsToMockFunctions
    def MockLLVMBisectionReturnValue(call_count, _args_output):
      # Simulate that successfully launched more tryjobs.
      if call_count == 0:
        return 0

      # Simulate that failed to launch more tryjobs.
      if call_count == 1:
        raise ValueError('Failed to launch more tryjobs.')

      # Simulate that the bad revision has been found.
      if call_count == 2:
        return llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value

      assert False, 'Called `llvm_bisection.main()` more than expected.'

    # Use the test function to simulate the behavior of `llvm_bisection.main()`.
    mock_llvm_bisection.side_effect = MockLLVMBisectionReturnValue

    # Use the test function to simulate the behavior of `os.path.isfile()`.
    mock_is_file.side_effect = MockStatusFileCheck

    # Verify the excpetion is raised when successfully found the bad revision.
    # Uses `sys.exit(0)` to indicate success.
    with self.assertRaises(SystemExit) as err:
      auto_llvm_bisection.main()

    self.assertEqual(err.exception.code, 0)

    mock_outside_chroot.assert_called_once()
    mock_get_args.assert_called_once()
    mock_get_auto_script.assert_called_once()
    self.assertEqual(mock_is_file.call_count, 3)
    self.assertEqual(mock_llvm_bisection.call_count, 3)
    mock_traceback.assert_called_once()
    mock_sleep.assert_called_once()
    mock_update_tryjobs.assert_called_once()

  # Simulate behavior of `subprocess.call()` when failed to update tryjobs to
  # `cros buildresult` (script failed).
  @mock.patch.object(subprocess, 'call', return_value=1)
  # Simulate behavior of `time.time()` when determining the time passed when
  # updating tryjobs whose 'status' is 'pending'.
  @mock.patch.object(time, 'time')
  # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking
  # the script outside of the chroot.
  @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
  # Simulate behavior of `time.sleep()` when waiting for errors to settle caused
  # by `llvm_bisection.main()` (e.g. network issue, etc.).
  @mock.patch.object(time, 'sleep')
  # Simulate behavior of `traceback.print_exc()` when resuming bisection.
  @mock.patch.object(os.path, 'isfile', return_value=True)
  # Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when
  # returning the absolute path to that script that updates all 'pending'
  # tryjobs to the result of `cros buildresult`.
  @mock.patch.object(
      auto_llvm_bisection,
      'GetPathToUpdateAllTryjobsWithAutoScript',
      return_value='/abs/path/to/update_tryjob.py')
  # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line
  # arguments required by the bisection script.
  @mock.patch.object(
      llvm_bisection,
      'GetCommandLineArgs',
      return_value=test_helpers.ArgsOutputTest())
  def testFailedToUpdatePendingTryJobs(
      self, mock_get_args, mock_get_auto_script, mock_is_file, mock_sleep,
      mock_outside_chroot, mock_time, mock_update_tryjobs):

    # Simulate behavior of `time.time()` for time passed.
    @test_helpers.CallCountsToMockFunctions
    def MockTimePassed(call_count):
      if call_count < 3:
        return call_count

      assert False, 'Called `time.time()` more than expected.'

    # Use the test function to simulate the behavior of `time.time()`.
    mock_time.side_effect = MockTimePassed

    # Reduce the polling limit for the test case to terminate faster.
    auto_llvm_bisection.POLLING_LIMIT_SECS = 1

    # Verify the exception is raised when unable to update tryjobs whose
    # 'status' value is 'pending'.
    with self.assertRaises(SystemExit) as err:
      auto_llvm_bisection.main()

    self.assertEqual(err.exception.code, 1)

    mock_outside_chroot.assert_called_once()
    mock_get_args.assert_called_once()
    mock_get_auto_script.assert_called_once()
    self.assertEqual(mock_is_file.call_count, 2)
    mock_sleep.assert_called_once()
    self.assertEqual(mock_time.call_count, 3)
    self.assertEqual(mock_update_tryjobs.call_count, 2)


if __name__ == '__main__':
  unittest.main()