cbuildbot/builders/simple_builders_unittest.py

# Copyright 2015 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Unittests for simpler builders."""

import copy
import os

from chromite.cbuildbot import cbuildbot_run
from chromite.cbuildbot.builders import generic_builders
from chromite.cbuildbot.builders import simple_builders
from chromite.cbuildbot.stages import generic_stages
from chromite.cbuildbot.stages import tast_test_stages
from chromite.cbuildbot.stages import test_stages
from chromite.cbuildbot.stages import vm_test_stages
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import cros_test_lib
from chromite.lib import failures_lib
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib.buildstore import FakeBuildStore
from chromite.scripts import cbuildbot


# pylint: disable=protected-access


class SimpleBuilderTest(cros_test_lib.MockTempDirTestCase):
    """Tests for the main code paths in simple_builders.SimpleBuilder"""

    def setUp(self):
        # List of all stages that would have been called as part of this run.
        self.called_stages = []

        # Map from stage class to exception to be raised when stage is run.
        self.stage_exceptions = {}

        # VM test stages that are run by SimpleBuilder._RunVMTests.
        self.all_vm_test_stages = [
            vm_test_stages.VMTestStage,
            tast_test_stages.TastVMTestStage,
        ]

        self.buildstore = FakeBuildStore()
        # Simple new function that redirects RunStage to record all stages to be
        # run rather than mock them completely. These can be used in a test to
        # assert something has been called.
        def run_stage(_class_instance, stage_name, *args, **_kwargs):
            # It's more useful to record the actual stage that's wrapped within
            # RepeatStage or RetryStage.
            if stage_name in [
                generic_stages.RepeatStage,
                generic_stages.RetryStage,
            ]:
                stage_name = args[1]

            self.called_stages.append(stage_name)
            if stage_name in self.stage_exceptions:
                raise self.stage_exceptions[stage_name]

        # Parallel version.
        def run_parallel_stages(_class_instance, *_args):
            # Since parallel stages are forked processes, we can't actually
            # update anything here unless we want to do interprocesses comms.
            pass

        self.buildroot = os.path.join(self.tempdir, "buildroot")
        chroot_path = os.path.join(self.buildroot, constants.DEFAULT_CHROOT_DIR)
        osutils.SafeMakedirs(os.path.join(chroot_path, "tmp"))

        self.PatchObject(generic_builders.Builder, "_RunStage", new=run_stage)
        self.PatchObject(
            simple_builders.SimpleBuilder,
            "_RunParallelStages",
            new=run_parallel_stages,
        )
        self.PatchObject(
            cbuildbot_run._BuilderRunBase,
            "GetVersion",
            return_value="R32-1234.0.0",
        )

        self._manager = parallel.Manager()
        # Pylint-1.9 has a false positive on this for some reason.
        self._manager.__enter__()  # pylint: disable=no-value-for-parameter

    def tearDown(self):
        # Mimic exiting a 'with' statement.
        self._manager.__exit__(None, None, None)

    def _initConfig(
        self,
        bot_id,
        master=False,
        extra_argv=None,
        override_hw_test_config=None,
        models=None,
    ):
        """Return normal options/build_config for |bot_id|"""
        site_config = config_lib.GetConfig()
        build_config = copy.deepcopy(site_config[bot_id])
        build_config["master"] = master
        build_config["important"] = False
        if models:
            build_config["models"] = models

        # Use the cbuildbot parser to create properties and populate default values.
        parser = cbuildbot._CreateParser()
        argv = (
            ["-r", self.buildroot, "--buildbot", "--debug", "--nochromesdk"]
            + (extra_argv if extra_argv else [])
            + [bot_id]
        )
        options = cbuildbot.ParseCommandLine(parser, argv)

        # Yikes.
        options.managed_chrome = build_config["sync_chrome"]

        # Iterate through override and update HWTestConfig attributes.
        if override_hw_test_config:
            for key, val in override_hw_test_config.items():
                for hw_test in build_config.hw_tests:
                    setattr(hw_test, key, val)

        return cbuildbot_run.BuilderRun(
            options, site_config, build_config, self._manager
        )

    def _RunVMTests(self):
        """Helper method that runs VM tests and returns exceptions.

        Returns:
          List of exception classes in CompoundFailure.
        """
        board = "betty-release"
        builder_run = self._initConfig(board)
        exception_types = []

        try:
            simple_builders.SimpleBuilder(
                builder_run, self.buildstore
            )._RunVMTests(builder_run, board)
        except failures_lib.CompoundFailure as f:
            exception_types = [e.type for e in f.exc_infos]
        return exception_types

    def testRunStagesChrootBuilder(self):
        """Verify RunStages for CHROOT_BUILDER_TYPE builders"""
        builder_run = self._initConfig("chromiumos-sdk")
        simple_builders.SimpleBuilder(builder_run, self.buildstore).RunStages()

    def testRunStagesDefaultBuild(self):
        """Verify RunStages for standard board builders"""
        builder_run = self._initConfig("amd64-generic-full")
        builder_run.attrs.chrome_version = "TheChromeVersion"
        simple_builders.SimpleBuilder(builder_run, self.buildstore).RunStages()

    def testRunStagesDefaultBuildCompileCheck(self):
        """Verify RunStages for standard board builders (compile only)"""
        extra_argv = ["--compilecheck"]
        builder_run = self._initConfig(
            "amd64-generic-full", extra_argv=extra_argv
        )
        builder_run.attrs.chrome_version = "TheChromeVersion"
        simple_builders.SimpleBuilder(builder_run, self.buildstore).RunStages()

    def testRunStagesDefaultBuildHwTests(self):
        """Verify RunStages for boards w/hwtests"""
        extra_argv = ["--hwtest"]
        builder_run = self._initConfig("eve-release", extra_argv=extra_argv)
        builder_run.attrs.chrome_version = "TheChromeVersion"
        simple_builders.SimpleBuilder(builder_run, self.buildstore).RunStages()

    def testThatWeScheduleHWTestsRegardlessOfBlocking(self):
        """Verify RunStages for boards w/hwtests (blocking).

        Make sure the same stages get scheduled regardless of whether their hwtest
        suites are marked blocking or not.
        """
        extra_argv = ["--hwtest"]
        builder_run_without_blocking = self._initConfig(
            "eve-release",
            extra_argv=extra_argv,
            override_hw_test_config=dict(blocking=False),
        )
        builder_run_with_blocking = self._initConfig(
            "eve-release",
            extra_argv=extra_argv,
            override_hw_test_config=dict(blocking=True),
        )
        builder_run_without_blocking.attrs.chrome_version = "TheChromeVersion"
        builder_run_with_blocking.attrs.chrome_version = "TheChromeVersion"

        simple_builders.SimpleBuilder(
            builder_run_without_blocking, self.buildstore
        ).RunStages()
        without_blocking_stages = list(self.called_stages)

        self.called_stages = []
        simple_builders.SimpleBuilder(
            builder_run_with_blocking, self.buildstore
        ).RunStages()
        self.assertEqual(without_blocking_stages, self.called_stages)

    def testUnifiedBuildsRunHwTestsForAllModels(self):
        """Verify hwtests run for model fanout with unified builds"""
        extra_argv = ["--hwtest"]
        unified_build = self._initConfig(
            "eve-release",
            extra_argv=extra_argv,
            models=[
                config_lib.ModelTestConfig("model1", "model1"),
                config_lib.ModelTestConfig(
                    "model2", "model2", ["sanity", "bvt-inline"]
                ),
            ],
        )
        unified_build.attrs.chrome_version = "TheChromeVersion"
        simple_builders.SimpleBuilder(
            unified_build, self.buildstore
        ).RunStages()

    def testAllVMTestStagesSucceed(self):
        """Verify all VM test stages are run."""
        self.assertEqual([], self._RunVMTests())
        self.assertEqual(self.all_vm_test_stages, self.called_stages)

    def testAllVMTestStagesFail(self):
        """Verify failures are reported when all VM test stages fail."""
        self.stage_exceptions = {
            vm_test_stages.VMTestStage: failures_lib.InfrastructureFailure(),
            tast_test_stages.TastVMTestStage: failures_lib.TestFailure(),
        }
        self.assertEqual(
            [failures_lib.InfrastructureFailure, failures_lib.TestFailure],
            self._RunVMTests(),
        )
        self.assertEqual(self.all_vm_test_stages, self.called_stages)

    def testVMTestStageFails(self):
        """Verify TastVMTestStage is still run when VMTestStage fails."""
        self.stage_exceptions = {
            vm_test_stages.VMTestStage: failures_lib.TestFailure(),
        }
        self.assertEqual([failures_lib.TestFailure], self._RunVMTests())
        self.assertEqual(self.all_vm_test_stages, self.called_stages)

    def testTastVMTestStageFails(self):
        """Verify VMTestStage is still run when TastVMTestStage fails."""
        self.stage_exceptions = {
            tast_test_stages.TastVMTestStage: failures_lib.TestFailure(),
        }
        self.assertEqual([failures_lib.TestFailure], self._RunVMTests())
        self.assertEqual(self.all_vm_test_stages, self.called_stages)

    def testBoardsForSimpleBuilderWithDUTOverride(self):
        """Test the BoardsForSimpleBuilder function with a DUT board override."""
        builder_run = self._initConfig("amd64-generic-full")
        builder_run.options.hwtest_dut_override = test_stages.HWTestDUTOverride(
            "bar-board", "bar-model", "bar-pool"
        )
        simple_builder = simple_builders.SimpleBuilder(
            builder_run, self.buildstore
        )
        self.assertEqual(
            simple_builder.BoardsForSimpleBuilder(builder_run), ["bar-board"]
        )

    def testBoardsForSimpleBuilderWithoutDUTOverride(self):
        """Test the BoardsForSimpleBuilder function withhut a DUT board override."""
        builder_run = self._initConfig("amd64-generic-full")
        simple_builder = simple_builders.SimpleBuilder(
            builder_run, self.buildstore
        )
        self.assertEqual(
            simple_builder.BoardsForSimpleBuilder(builder_run),
            ["amd64-generic"],
        )