blob: 0a722f3ddb2a049f222f66c263dfce6c1a7f8e65 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Unittests for report stages."""
import datetime as dt
import json
from unittest import mock
import pytest # pylint: disable=import-error
from chromite.cbuildbot import cbuildbot_alerts
from chromite.cbuildbot import cbuildbot_run
from chromite.cbuildbot import cbuildbot_unittest
from chromite.cbuildbot import commands
from chromite.cbuildbot import manifest_version
from chromite.cbuildbot import topology
from chromite.cbuildbot import topology_unittest
from chromite.cbuildbot.stages import generic_stages_unittest
from chromite.cbuildbot.stages import report_stages
from chromite.lib import alerts
from chromite.lib import cidb
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import failure_message_lib_unittest
from chromite.lib import fake_cidb
from chromite.lib import gs_unittest
from chromite.lib import metrics
from chromite.lib import osutils
from chromite.lib import results_lib
from chromite.lib import retry_stats
from chromite.lib import toolchain
from chromite.lib.buildstore import FakeBuildStore
# pylint: disable=protected-access
# pylint: disable=too-many-ancestors
class BuildReexecutionStageTest(generic_stages_unittest.AbstractStageTestCase):
"""Tests that BuildReexecutionFinishedStage behaves as expected."""
def setUp(self):
self.fake_db = fake_cidb.FakeCIDBConnection()
self.buildstore = FakeBuildStore(self.fake_db)
cidb.CIDBConnectionFactory.SetupMockCidb(self.fake_db)
build_id = self.fake_db.InsertBuild(
'builder name', 1, 'build config', 'bot hostname')
self._Prepare(build_id=build_id)
release_tag = '4815.0.0-rc1'
self._run.attrs.release_tag = '4815.0.0-rc1'
fake_versioninfo = manifest_version.VersionInfo(release_tag, '39')
self.gs_mock = self.StartPatcher(gs_unittest.GSContextMock())
self.gs_mock.SetDefaultCmdResult()
self.PatchObject(cbuildbot_run._BuilderRunBase, 'GetVersionInfo',
return_value=fake_versioninfo)
self.PatchObject(toolchain, 'GetToolchainsForBoard')
self.PatchObject(toolchain, 'GetToolchainTupleForBoard',
return_value=['i686-pc-linux-gnu', 'arm-none-eabi'])
def tearDown(self):
cidb.CIDBConnectionFactory.SetupMockCidb()
def testPerformStage(self):
"""Test that a normal runs completes without error."""
self.RunStage()
tags = self._run.attrs.metadata.GetValue(constants.METADATA_TAGS)
self.assertEqual(tags['version_full'], 'R39-4815.0.0-rc1')
def ConstructStage(self):
return report_stages.BuildReexecutionFinishedStage(self._run,
self.buildstore)
class ConfigDumpStageTest(generic_stages_unittest.AbstractStageTestCase):
"""Tests that ConfigDumpStage runs without syntax error."""
def setUp(self):
self.buildstore = FakeBuildStore()
def ConstructStage(self):
return report_stages.ConfigDumpStage(self._run, self.buildstore)
def testPerformStage(self):
self._Prepare()
self.RunStage()
class SlaveFailureSummaryStageTest(
generic_stages_unittest.AbstractStageTestCase):
"""Tests that SlaveFailureSummaryStage behaves as expected."""
def setUp(self):
self.db = mock.MagicMock()
self.buildstore = FakeBuildStore(self.db)
cidb.CIDBConnectionFactory.SetupMockCidb(self.db)
self._Prepare(build_id=1)
# Our API here is not great when it comes to kwargs passing.
def _Prepare(self, **kwargs): # pylint: disable=arguments-differ
"""Prepare stage with config['master']=True."""
super()._Prepare(**kwargs)
self._run.config['master'] = True
def ConstructStage(self):
return report_stages.SlaveFailureSummaryStage(self._run, self.buildstore)
def testPerformStage(self):
"""Tests that stage runs without syntax errors."""
fake_failure = (
failure_message_lib_unittest.StageFailureHelper.CreateStageFailure(
build_id=10,
build_stage_id=11,
builder_name='builder_name',
build_number=12,
build_config='build-config',
stage_name='FailingStage',
stage_status=constants.BUILDER_STATUS_FAILED,
build_status=constants.BUILDER_STATUS_FAILED))
self.PatchObject(self.buildstore, 'GetBuildsFailures',
return_value=[fake_failure])
self.PatchObject(cbuildbot_alerts, 'PrintBuildbotLink')
self.RunStage()
self.assertEqual(cbuildbot_alerts.PrintBuildbotLink.call_count, 1)
@pytest.mark.usefixtures('singleton_manager')
class BuildStartStageTest(generic_stages_unittest.AbstractStageTestCase):
"""Tests that BuildStartStage behaves as expected."""
def setUp(self):
self.db = fake_cidb.FakeCIDBConnection()
self.buildstore = FakeBuildStore(self.db)
cidb.CIDBConnectionFactory.SetupMockCidb(self.db)
retry_stats.SetupStats()
master_build_id = self.db.InsertBuild(
'master_build', 1,
'master_build_config', 'bot_hostname')
self.PatchObject(toolchain, 'GetToolchainsForBoard')
self.PatchObject(toolchain, 'GetArchForTarget', return_value='x86')
self._Prepare(build_id=None, master_build_id=master_build_id)
def testPerformStage(self):
"""Test that a normal run of the stage does a database insert."""
self.RunStage()
build_id = self._run.attrs.metadata.GetValue('build_id')
self.assertGreater(build_id, 0)
self.assertEqual(self._run.attrs.metadata.GetValue('db_type'),
cidb.CONNECTION_TYPE_MOCK)
def testSuiteSchedulingEqualsFalse(self):
"""Test that a run of the stage makes suite_scheduling False."""
# Test suite_scheduling for **-paladin
self._Prepare(bot_id='amd64-generic-full')
self.RunStage()
self.assertFalse(self._run.attrs.metadata.GetValue('suite_scheduling'))
def testSuiteSchedulingEqualsTrue(self):
"""Test that a run of the stage makes suite_scheduling True."""
# Test suite_scheduling for **-release
self._Prepare(bot_id='eve-release')
self.RunStage()
self.assertTrue(self._run.attrs.metadata.GetValue('suite_scheduling'))
def testHandleSkipWithInstanceChange(self):
"""Test that HandleSkip disables cidb and dies when necessary."""
# This test verifies that switching to a 'mock' database type once
# metadata already has an id in 'previous_db_type' will fail.
self._run.attrs.metadata.UpdateWithDict({'build_id': 31337,
'db_type': 'previous_db_type'})
stage = self.ConstructStage()
self.assertRaises(AssertionError, stage.HandleSkip)
self.assertEqual(cidb.CIDBConnectionFactory.GetCIDBConnectionType(),
cidb.CONNECTION_TYPE_INV)
# The above test has the side effect of invalidating CIDBConnectionFactory.
# Undo that side effect so other unit tests can run.
cidb.CIDBConnectionFactory.SetupMockCidb()
def testHandleSkipWithNoDbType(self):
"""Test that HandleSkip passes when db_type is missing."""
self._run.attrs.metadata.UpdateWithDict({'build_id': 31337})
stage = self.ConstructStage()
stage.HandleSkip()
def testHandleSkipWithDbType(self):
"""Test that HandleSkip passes when db_type is specified."""
self._run.attrs.metadata.UpdateWithDict(
{'build_id': 31337,
'db_type': cidb.CONNECTION_TYPE_MOCK})
stage = self.ConstructStage()
stage.HandleSkip()
def ConstructStage(self):
return report_stages.BuildStartStage(self._run, self.buildstore)
class AbstractReportStageTestCase(
generic_stages_unittest.AbstractStageTestCase,
cbuildbot_unittest.SimpleBuilderTestCase):
"""Base class for testing the Report stage."""
def setUp(self):
for cmd in ((osutils, 'WriteFile'),
(commands, 'UploadArchivedFile'),
(alerts, 'SendEmail')):
self.StartPatcher(mock.patch.object(*cmd, autospec=True))
retry_stats.SetupStats()
self.PatchObject(report_stages.ReportStage, '_GetBuildDuration',
return_value=1000)
self.PatchObject(toolchain, 'GetToolchainsForBoard')
self.PatchObject(toolchain, 'GetArchForTarget', return_value='x86')
# Set up a general purpose cidb mock. Tests with more specific
# mock requirements can replace this with a separate call to
# SetupMockCidb
self.mock_cidb = mock.MagicMock()
self.buildstore = FakeBuildStore(self.mock_cidb)
cidb.CIDBConnectionFactory.SetupMockCidb(self.mock_cidb)
# Setup topology for unittests
keyvals = {topology.DATASTORE_WRITER_CREDS_KEY: './foo/bar.cert'}
topology_unittest.FakeFetchTopology(keyvals=keyvals)
self._Prepare()
def _SetupUpdateStreakCounter(self, counter_value=-1):
self.PatchObject(report_stages.ReportStage, '_UpdateStreakCounter',
autospec=True, return_value=counter_value)
def ConstructStage(self):
return report_stages.ReportStage(self._run, self.buildstore, None)
@pytest.mark.usefixtures('singleton_manager')
class ReportStageTest(AbstractReportStageTestCase):
"""Test the Report stage."""
RELEASE_TAG = ''
def setUp(self):
self.mock_cidb.GetSlaveStatuses = mock.Mock(return_value=None)
def testCheckResults(self):
"""Basic sanity check for results stage functionality"""
self.CreateMockOverlay('amd64-generic')
stages = [
{
'build_config': 'build1',
'name': 'stage1',
'start_time': dt.datetime.now() - dt.timedelta(0, 500),
'finish_time': dt.datetime.now() - dt.timedelta(0, 300),
'status': constants.BUILDER_STATUS_PASSED,
},
{
'build_config': 'build1',
'name': 'stage2',
'start_time': dt.datetime.now() - dt.timedelta(0, 500),
'finish_time': dt.datetime.now() - dt.timedelta(0, 200),
'status': constants.BUILDER_STATUS_PASSED,
},
{
'build_config': 'build1',
'name': 'stage3',
'start_time': dt.datetime.now() - dt.timedelta(0, 200),
'finish_time': dt.datetime.now() - dt.timedelta(0, 100),
'status': constants.BUILDER_STATUS_PASSED,
},
]
statuses = [
{
'build_config': 'build1',
'build_number': '64',
'start_time': dt.datetime.now() - dt.timedelta(0, 600),
'finish_time': dt.datetime.now() - dt.timedelta(0, 330),
'status': constants.BUILDER_STATUS_PASSED,
},
{
'build_config': 'build2',
'build_number': '27',
'start_time': dt.datetime.now() - dt.timedelta(0, 300),
'finish_time': dt.datetime.now() - dt.timedelta(0, 100),
'status': constants.BUILDER_STATUS_PASSED,
},
{
'build_config': 'build3',
'build_number': '288282',
'start_time': dt.datetime.now() - dt.timedelta(0, 400),
'finish_time': dt.datetime.now() - dt.timedelta(0, 200),
'status': constants.BUILDER_STATUS_PASSED,
},
]
self.buildstore.GetBuildsStages = mock.Mock(return_value=stages)
self.mock_cidb.GetSlaveStatuses = mock.Mock(return_value=statuses)
self._SetupUpdateStreakCounter()
self.PatchObject(report_stages.ReportStage, '_LinkArtifacts')
self.RunStage()
filenames = (
'LATEST-%s' % self.TARGET_MANIFEST_BRANCH,
'LATEST-%s' % self.VERSION,
)
calls = [mock.call(mock.ANY, mock.ANY, 'metadata.json', False,
update_list=True, acl=mock.ANY)]
calls += [mock.call(mock.ANY, mock.ANY, 'timeline-stages.html',
debug=False, update_list=True, acl=mock.ANY)]
calls += [mock.call(mock.ANY, mock.ANY, 'timeline-slaves.html',
debug=False, update_list=True, acl=mock.ANY)]
calls += [mock.call(mock.ANY, mock.ANY, filename, False,
acl=mock.ANY) for filename in filenames]
# Verify build stages timeline contains the stages that were mocked.
self.assertEqual(calls, commands.UploadArchivedFile.call_args_list)
timeline_content = osutils.WriteFile.call_args_list[2][0][1]
for s in stages:
self.assertIn('["%s", new Date' % s['name'], timeline_content)
# Verify slaves timeline contains the slaves that were mocked.
self.assertEqual(calls, commands.UploadArchivedFile.call_args_list)
timeline_content = osutils.WriteFile.call_args_list[3][0][1]
for s in statuses:
self.assertIn('["%s - %s", new Date' %
(s['build_config'], s['build_number']), timeline_content)
def testDoNotUpdateLATESTMarkersWhenBuildFailed(self):
"""Check that we do not update the latest markers on failed build."""
self._SetupUpdateStreakCounter()
self.PatchObject(report_stages.ReportStage, '_LinkArtifacts')
self.PatchObject(results_lib.Results, 'BuildSucceededSoFar',
return_value=False)
stage = self.ConstructStage()
self.PatchObject(stage, 'GetBuildFailureMessage')
stage.Run()
calls = [mock.call(mock.ANY, mock.ANY, 'metadata.json', False,
update_list=True, acl=mock.ANY)]
calls += [mock.call(mock.ANY, mock.ANY, 'timeline-stages.html',
debug=False, update_list=True, acl=mock.ANY)]
self.assertEqual(calls, commands.UploadArchivedFile.call_args_list)
def testEmailNotifyMasterBranch(self):
"""Send out alerts when streak counter reaches the threshold."""
self.PatchObject(cbuildbot_run._BuilderRunBase,
'InEmailReportingEnvironment', return_value=True)
self.PatchObject(cros_build_lib, 'HostIsCIBuilder', return_value=True)
self.buildstore.UpdateLuciNotifyProperties = mock.Mock()
notification_config_1 = config_lib.NotificationConfig(
'test1@chromium.org', threshold=1)
notification_config_2 = config_lib.NotificationConfig(
'test2@chromium.org', threshold=2)
notification_config_3 = config_lib.NotificationConfig(
'test3@chromium.org', threshold=3, template='explicit_template')
self._Prepare(
extra_config={
'notification_configs': [
notification_config_1,
notification_config_2,
notification_config_3,
]
},
cmd_args=['-r', self.build_root, '--branch', 'master', self.BOT_ID])
self._SetupUpdateStreakCounter(counter_value=-2)
self.RunStage()
self.buildstore.UpdateLuciNotifyProperties.assert_called_once_with(
email_notify=[
notification_config_1.email_notify,
notification_config_2.email_notify,
])
def testEmailNotifyNonMasterBranch(self):
"""Do not send out email alerts for non-master branches."""
self.PatchObject(
cbuildbot_run._BuilderRunBase,
'InEmailReportingEnvironment',
return_value=True)
self.PatchObject(cros_build_lib, 'HostIsCIBuilder', return_value=True)
self.buildstore.UpdateLuciNotifyProperties = mock.Mock()
self._Prepare(
extra_config={
'notification_configs': [
config_lib.NotificationConfig(
'test1@chromium.org', threshold=1)
]
})
self._SetupUpdateStreakCounter(counter_value=-1)
self.RunStage()
self.buildstore.UpdateLuciNotifyProperties.assert_not_called()
def testWriteBasicMetadata(self):
"""Test that WriteBasicMetadata writes expected keys correctly."""
report_stages.WriteBasicMetadata(self._run)
metadata_dict = self._run.attrs.metadata.GetDict()
self.assertEqual(metadata_dict['build-number'],
generic_stages_unittest.DEFAULT_BUILD_NUMBER)
self.assertIn('builder-name', metadata_dict)
self.assertIn('bot-hostname', metadata_dict)
def testWriteTagMetadata(self):
"""Test that WriteTagMetadata writes expected keys correctly."""
self.PatchObject(cros_build_lib, 'GetHostName', return_value='cros-wimpy2')
self._SetupUpdateStreakCounter()
report_stages.WriteTagMetadata(self._run)
tags_dict = self._run.attrs.metadata.GetValue(constants.METADATA_TAGS)
self.assertEqual(tags_dict['build_number'],
generic_stages_unittest.DEFAULT_BUILD_NUMBER)
self.assertIn('builder_name', tags_dict)
self.assertIn('bot_hostname', tags_dict)
self.RunStage()
tags_content = osutils.WriteFile.call_args_list[1][0][1]
tags_content_dict = json.loads(tags_content)
self.assertEqual(tags_content_dict['build_number'],
generic_stages_unittest.DEFAULT_BUILD_NUMBER)
def testGetChildConfigsMetadataList(self):
"""Test that GetChildConfigListMetadata generates child config metadata."""
child_configs = [{'name': 'config1', 'boards': ['board1']},
{'name': 'config2', 'boards': ['board2']}]
config_status_map = {'config1': True,
'config2': False}
expected = [{'name': 'config1', 'boards': ['board1'],
'status': constants.BUILDER_STATUS_PASSED},
{'name': 'config2', 'boards': ['board2'],
'status': constants.BUILDER_STATUS_FAILED}]
child_config_list = report_stages.GetChildConfigListMetadata(
child_configs, config_status_map)
self.assertEqual(expected, child_config_list)
def testPerformStage(self):
"""Test PerformStage."""
mock_sd = self.PatchObject(metrics, 'CumulativeSecondsDistribution')
self.PatchObject(report_stages.ReportStage, 'ArchiveResults')
stage = self.ConstructStage()
stage.PerformStage()
self.assertEqual(mock_sd.call_count, 1)
@pytest.mark.usefixtures('singleton_manager')
class ReportStageNoSyncTest(AbstractReportStageTestCase):
"""Test the Report stage if SyncStage didn't complete.
If SyncStage doesn't complete, we don't know the release tag, and can't
archive results.
"""
RELEASE_TAG = None
def testCommitQueueResults(self):
"""Check that we can run with a RELEASE_TAG of None."""
self._SetupUpdateStreakCounter()
self.RunStage()