blob: 31cef9f3a4c01eea72ce651818bad55be9cd1287 [file] [log] [blame]
# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Integration tests for module.
Running these tests requires and assumes:
1) You are running from a machine with whitelisted access to the CIDB
database test instance.
2) You have a checkout of the crostools repo, which provides credentials
to the above test instance.
# pylint: disable= W0212
# pylint: disable=bad-continuation
from __future__ import print_function
import datetime
import glob
import logging
import os
import sys
import time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
from chromite.cbuildbot import constants
from chromite.cbuildbot import metadata_lib
from chromite.lib import cidb
from chromite.lib import clactions
from chromite.lib import cros_build_lib
from chromite.lib import cros_test_lib
from chromite.lib import osutils
from chromite.lib import parallel
SERIES_0_TEST_DATA_PATH = os.path.join(
constants.CHROMITE_DIR, 'cidb', 'test_data', 'series_0')
SERIES_1_TEST_DATA_PATH = os.path.join(
constants.CHROMITE_DIR, 'cidb', 'test_data', 'series_1')
TEST_DB_CRED_ROOT = os.path.join(constants.SOURCE_ROOT,
'crostools', 'cidb',
TEST_DB_CRED_READONLY = os.path.join(constants.SOURCE_ROOT,
'crostools', 'cidb',
TEST_DB_CRED_BOT = os.path.join(constants.SOURCE_ROOT,
'crostools', 'cidb',
class CIDBIntegrationTest(cros_test_lib.TestCase):
"""Base class for cidb tests that connect to a test MySQL instance."""
def _PrepareFreshDatabase(self, max_schema_version=None):
"""Create an empty database with migrations applied.
max_schema_version: The highest schema version migration to apply,
defaults to None in which case all migrations will be applied.
A CIDBConnection instance, connected to a an empty database as the
root user.
# Note: We do not use the cidb.CIDBConnectionFactory
# in this module. That factory method is used only to construct
# connections as the bot user, which is how the builders will always
# connect to the database. In this module, however, we need to test
# database connections as other mysql users.
# Connect to database and drop its contents.
db = cidb.CIDBConnection(TEST_DB_CRED_ROOT)
# Connect to now fresh database and apply migrations.
db = cidb.CIDBConnection(TEST_DB_CRED_ROOT)
return db
class CIDBMigrationsTest(CIDBIntegrationTest):
"""Test that all migrations apply correctly."""
def testMigrations(self):
"""Test that all migrations apply in bulk correctly."""
def testIncrementalMigrations(self):
"""Test that all migrations apply incrementally correctly."""
db = self._PrepareFreshDatabase(0)
migrations = db._GetMigrationScripts()
max_version = migrations[-1][0]
for i in range(1, max_version+1):
def testActions(self):
"""Test that InsertCLActions accepts 0-, 1-, and multi-item lists."""
db = self._PrepareFreshDatabase()
build_id = db.InsertBuild('my builder', 'chromiumos', 12, 'my config',
'my bot hostname')
a1 = clactions.CLAction.FromGerritPatchAndAction(
metadata_lib.GerritPatchTuple(1, 1, True),
a2 = clactions.CLAction.FromGerritPatchAndAction(
metadata_lib.GerritPatchTuple(1, 1, True),
a3 = clactions.CLAction.FromGerritPatchAndAction(
metadata_lib.GerritPatchTuple(1, 1, True),
db.InsertCLActions(build_id, [])
db.InsertCLActions(build_id, [a1])
db.InsertCLActions(build_id, [a2, a3])
action_count = db._GetEngine().execute('select count(*) from clActionTable'
self.assertEqual(action_count, 3)
# Test that all known CL action types can be inserted
fakepatch = metadata_lib.GerritPatchTuple(1, 1, True)
all_actions_list = [
clactions.CLAction.FromGerritPatchAndAction(fakepatch, action)
for action in constants.CL_ACTIONS]
db.InsertCLActions(build_id, all_actions_list)
class CIDBAPITest(CIDBIntegrationTest):
"""Tests of the CIDB API."""
def testSchemaVersionTooLow(self):
"""Tests that the minimum_schema decorator works as expected."""
db = self._PrepareFreshDatabase(2)
with self.assertRaises(cidb.UnsupportedMethodException):
db.InsertCLActions(0, [])
def testSchemaVersionOK(self):
"""Tests that the minimum_schema decorator works as expected."""
db = self._PrepareFreshDatabase(4)
db.InsertCLActions(0, [])
def testGetTime(self):
db = self._PrepareFreshDatabase(1)
current_db_time = db.GetTime()
self.assertEqual(type(current_db_time), datetime.datetime)
def GetTestDataSeries(test_data_path):
"""Get metadata from json files at |test_data_path|.
A list of CBuildbotMetadata objects, sorted by their start time.
filenames = glob.glob(os.path.join(test_data_path, '*.json'))
metadatas = []
for fname in filenames:
# Convert start time values, which are stored in RFC 2822 string format,
# to seconds since epoch.
timestamp_from_dict = lambda x: cros_build_lib.ParseUserDateTimeFormat(
return metadatas
class DataSeries0Test(CIDBIntegrationTest):
"""Simulate a set of 630 master/slave CQ builds."""
def testCQWithSchema32(self):
"""Run the CQ test with schema version 32."""
def _runCQTest(self):
"""Simulate a set of 630 master/slave CQ builds.
Note: This test takes about 2.5 minutes to populate its 630 builds
and their corresponding cl actions into the test database.
metadatas = GetTestDataSeries(SERIES_0_TEST_DATA_PATH)
self.assertEqual(len(metadatas), 630, 'Did not load expected amount of '
'test data')
bot_db = cidb.CIDBConnection(TEST_DB_CRED_BOT)
# Simulate the test builds, using a database connection as the
# bot user.
self.simulate_builds(bot_db, metadatas)
# Perform some sanity check queries against the database, connected
# as the readonly user.
readonly_db = cidb.CIDBConnection(TEST_DB_CRED_READONLY)
build_types = readonly_db._GetEngine().execute(
'select build_type from buildTable').fetchall()
self.assertTrue(all(x == ('paladin',) for x in build_types))
build_config_count = readonly_db._GetEngine().execute(
'select COUNT(distinct build_config) from buildTable').fetchall()[0][0]
self.assertEqual(build_config_count, 30)
# Test the _Select method, and verify that the first inserted
# build is a master-paladin build.
first_row = readonly_db._Select('buildTable', 1, ['id', 'build_config'])
self.assertEqual(first_row['build_config'], 'master-paladin')
# First master build has 29 slaves. Build with id 2 is a slave
# build with no slaves of its own.
self.assertEqual(len(readonly_db.GetSlaveStatuses(1)), 29)
self.assertEqual(len(readonly_db.GetSlaveStatuses(2)), 0)
# Make sure we can get build status by build id.
self.assertEqual(readonly_db.GetBuildStatus(2).get('id'), 2)
# Make sure we can get build statuses by build ids.
build_dicts = readonly_db.GetBuildStatuses([1, 2])
self.assertEqual([x.get('id') for x in build_dicts], [1, 2])
#| Test get build_status from -- here's the relevant data from
# master-paladin
#| id | status |
#| 601 | pass |
#| 571 | pass |
#| 541 | fail |
#| 511 | pass |
#| 481 | pass |
# From 1929 because we always go back one build first.
last_status = readonly_db.GetLastBuildStatuses('master-paladin', 1)
self.assertEqual(len(last_status), 1)
last_status = readonly_db.GetLastBuildStatuses('master-paladin', 5)
self.assertEqual(len(last_status), 5)
# Make sure keys are sorted correctly.
build_ids = []
for index, status in enumerate(last_status):
# Add these to list to confirm they are sorted afterwards correctly.
# Should be descending.
if index == 2:
self.assertEqual(status['status'], 'fail')
self.assertEqual(status['status'], 'pass')
# Check the sort order.
self.assertEqual(sorted(build_ids, reverse=True), build_ids)
def _last_updated_time_checks(self, db):
"""Sanity checks on the last_updated column."""
# We should have a diversity of last_updated times. Since the timestamp
# resolution is only 1 second, and we have lots of parallelism in the test,
# we won't have a distinct last_updated time per row.
# As the test db gets beefier, we're more likely to get collisions. So we
# check for a small number of distinct timestamps.
distinct_last_updated = db._GetEngine().execute(
'select count(distinct last_updated) from buildTable').fetchall()[0][0]
self.assertTrue(distinct_last_updated > 20)
ids_by_last_updated = db._GetEngine().execute(
'select id from buildTable order by last_updated').fetchall()
ids_by_last_updated = [id_tuple[0] for id_tuple in ids_by_last_updated]
# Build #1 should have been last updated before build # 200.
# However, build #1 (which was a master build) should have been last updated
# AFTER build #2 which was its slave.
def _cl_action_checks(self, db):
"""Sanity checks that correct cl actions were recorded."""
submitted_cl_count = db._GetEngine().execute(
'select count(*) from clActionTable where action="submitted"'
rejected_cl_count = db._GetEngine().execute(
'select count(*) from clActionTable where action="kicked_out"'
total_actions = db._GetEngine().execute(
'select count(*) from clActionTable').fetchall()[0][0]
self.assertEqual(submitted_cl_count, 56)
self.assertEqual(rejected_cl_count, 8)
self.assertEqual(total_actions, 1877)
actions_for_change = db.GetActionsForChanges(
[metadata_lib.GerritChangeTuple(205535, False)])
self.assertEqual(len(actions_for_change), 60)
last_action_dict = dict(actions_for_change[-1]._asdict())
self.assertEqual(last_action_dict, {'action': 'submitted',
'build_config': 'master-paladin',
'build_id': 511L,
'change_number': 205535L,
'change_source': 'external',
'patch_number': 1L,
'reason': ''})
def _start_and_finish_time_checks(self, db):
"""Sanity checks that correct data was recorded, and can be retrieved."""
max_start_time = db._GetEngine().execute(
'select max(start_time) from buildTable').fetchall()[0][0]
min_start_time = db._GetEngine().execute(
'select min(start_time) from buildTable').fetchall()[0][0]
max_fin_time = db._GetEngine().execute(
'select max(finish_time) from buildTable').fetchall()[0][0]
min_fin_time = db._GetEngine().execute(
'select min(finish_time) from buildTable').fetchall()[0][0]
self.assertGreater(max_start_time, min_start_time)
self.assertGreater(max_fin_time, min_fin_time)
# For all builds, finish_time should equal last_updated.
mismatching_times = db._GetEngine().execute(
'select count(*) from buildTable where finish_time != last_updated'
self.assertEqual(mismatching_times, 0)
def simulate_builds(self, db, metadatas):
"""Simulate a series of Commit Queue master and slave builds.
This method use the metadata objects in |metadatas| to simulate those
builds insertions and updates to the cidb. All metadatas encountered
after a particular master build will be assumed to be slaves of that build,
until a new master build is encountered. Slave builds for a particular
master will be simulated in parallel.
The first element in |metadatas| must be a CQ master build.
db: A CIDBConnection instance.
metadatas: A list of CBuildbotMetadata instances, sorted by start time.
m_iter = iter(metadatas)
def is_master(m):
return m.GetDict()['bot-config'] == 'master-paladin'
next_master =
while next_master:
master = next_master
next_master = None
assert is_master(master)
master_build_id = _SimulateBuildStart(db, master)
def simulate_slave(slave_metadata):
build_id = _SimulateBuildStart(db, slave_metadata,
_SimulateCQBuildFinish(db, slave_metadata, build_id)
logging.debug('Simulated slave build %s on pid %s', build_id,
return build_id
slave_metadatas = []
for slave in m_iter:
if is_master(slave):
next_master = slave
with parallel.BackgroundTaskRunner(simulate_slave, processes=15) as queue:
for slave in slave_metadatas:
_SimulateCQBuildFinish(db, master, master_build_id)
logging.debug('Simulated master build %s', master_build_id)
class BuildStagesAndFailureTest(CIDBIntegrationTest):
"""Test buildStageTable functionality."""
def runTest(self):
"""Test basic buildStageTable and failureTable functionality."""
bot_db = cidb.CIDBConnection(TEST_DB_CRED_BOT)
build_id = bot_db.InsertBuild('builder name',
build_stage_id = bot_db.InsertBuildStage(build_id,
'My Stage',
values = bot_db._Select('buildStageTable', build_stage_id, ['start_time'])
self.assertEqual(None, values['start_time'])
values = bot_db._Select('buildStageTable', build_stage_id,
['start_time', 'status'])
self.assertNotEqual(None, values['start_time'])
self.assertEqual(constants.BUILDER_STATUS_INFLIGHT, values['status'])
bot_db.FinishBuildStage(build_stage_id, constants.BUILDER_STATUS_PASSED)
values = bot_db._Select('buildStageTable', build_stage_id,
['finish_time', 'status', 'final'])
self.assertNotEqual(None, values['finish_time'])
self.assertEqual(True, values['final'])
self.assertEqual(constants.BUILDER_STATUS_PASSED, values['status'])
for category in constants.EXCEPTION_CATEGORY_ALL_CATEGORIES:
e = ValueError('The value was erroneous.')
bot_db.InsertFailure(build_stage_id, e, category)
class BuildTableTest(CIDBIntegrationTest):
"""Test buildTable functionality not tested by the DataSeries tests."""
def testDeadlineAPI(self):
"""Test deadline setting/querying API."""
bot_db = cidb.CIDBConnection(TEST_DB_CRED_BOT)
build_id = bot_db.InsertBuild('build_name',
timeout_seconds=30 * 60)
time_remaining = bot_db.GetTimeToDeadline(build_id)
# This will flake if the few cidb calls above take hours. Unlikely.
self.assertGreater(time_remaining, 10)
build_id = bot_db.InsertBuild('build_name',
# Sleep till the deadline expires.
time_remaining = bot_db.GetTimeToDeadline(build_id)
self.assertEqual(0, time_remaining)
build_id = bot_db.InsertBuild('build_name',
time_remaining = bot_db.GetTimeToDeadline(build_id)
self.assertEqual(None, time_remaining)
time_remaining = bot_db.GetTimeToDeadline(-1)
self.assertEqual(None, time_remaining)
class DataSeries1Test(CIDBIntegrationTest):
"""Simulate a single set of canary builds."""
def runTest(self):
"""Simulate a single set of canary builds with database schema v28."""
metadatas = GetTestDataSeries(SERIES_1_TEST_DATA_PATH)
self.assertEqual(len(metadatas), 18, 'Did not load expected amount of '
'test data')
# Migrate db to specified version. As new schema versions are added,
# migrations to later version can be applied after the test builds are
# simulated, to test that db contents are correctly migrated.
bot_db = cidb.CIDBConnection(TEST_DB_CRED_BOT)
def is_master(m):
return m.GetValue('bot-config') == 'master-release'
master_index = metadatas.index(next(m for m in metadatas if is_master(m)))
master_metadata = metadatas.pop(master_index)
self.assertEqual(master_metadata.GetValue('bot-config'), 'master-release')
master_id = self._simulate_canary(bot_db, master_metadata)
for m in metadatas:
self._simulate_canary(bot_db, m, master_id)
# Verify that expected data was inserted
num_boards = bot_db._GetEngine().execute(
'select count(*) from boardPerBuildTable'
self.assertEqual(num_boards, 40)
main_firmware_versions = bot_db._GetEngine().execute(
'select count(distinct main_firmware_version) from boardPerBuildTable'
self.assertEqual(main_firmware_versions, 29)
# For all builds, finish_time should equal last_updated.
mismatching_times = bot_db._GetEngine().execute(
'select count(*) from buildTable where finish_time != last_updated'
self.assertEqual(mismatching_times, 0)
def _simulate_canary(self, db, metadata, master_build_id=None):
"""Helper method to simulate an individual canary build.
db: cidb instance to use for simulation
metadata: CBuildbotMetadata instance of build to simulate.
master_build_id: Optional id of master build.
build_id of build that was simulated.
build_id = _SimulateBuildStart(db, metadata, master_build_id)
metadata_dict = metadata.GetDict()
# Insert child configs and boards
for child_config_dict in metadata_dict['child-configs']:
db.InsertChildConfigPerBuild(build_id, child_config_dict['name'])
for board in metadata_dict['board-metadata'].keys():
db.InsertBoardPerBuild(build_id, board)
for board, bm in metadata_dict['board-metadata'].items():
db.UpdateBoardPerBuildMetadata(build_id, board, bm)
db.UpdateMetadata(build_id, metadata)
status = metadata_dict['status']['status']
status = _TranslateStatus(status)
for child_config_dict in metadata_dict['child-configs']:
# Note, we are not using test data here, because the test data
# we have predates the existence of child-config status being
# stored in metadata.json. Instead, we just pretend all child
# configs had the same status as the main config.
db.FinishChildConfig(build_id, child_config_dict['name'],
db.FinishBuild(build_id, status)
return build_id
def _TranslateStatus(status):
# TODO(akeshet): The status strings used in BuildStatus are not the same as
# those recorded in CBuildbotMetadata. Use a general purpose adapter.
if status == 'passed':
return 'pass'
if status == 'failed':
return 'fail'
return status
def _SimulateBuildStart(db, metadata, master_build_id=None):
"""Returns build_id for the inserted buildTable entry."""
metadata_dict = metadata.GetDict()
# TODO(akeshet): We are pretending that all these builds were on the internal
# waterfall at the moment, for testing purposes. This is because we don't
# actually save in the metadata.json any way to know which waterfall the
# build was on.
waterfall = 'chromeos'
build_id = db.InsertBuild(metadata_dict['builder-name'],
return build_id
def _SimulateCQBuildFinish(db, metadata, build_id):
metadata_dict = metadata.GetDict()
for e in metadata_dict['cl_actions']])
db.UpdateMetadata(build_id, metadata)
status = metadata_dict['status']['status']
status = _TranslateStatus(status)
# The build summary reported by a real CQ run is more complicated -- it is
# computed from slave summaries by a master. For sanity checking, we just
# insert the current builer's summary.
summary = metadata_dict['status'].get('reason', None)
db.FinishBuild(build_id, status, summary)
# TODO(akeshet): Allow command line args to specify alternate CIDB instance
# for testing.
if __name__ == '__main__':