blob: 7e3c7a11785098d89ba5a771225f5a98773cd1b2 [file] [log] [blame]
# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Integration tests for module.
Running these tests requires and assumes:
1) You are running from a machine with whitelisted access to the CIDB
database test instance.
2) You have a checkout of the crostools repo, which provides credentials
to the above test instance.
# pylint: disable-msg= W0212
from __future__ import print_function
import glob
import logging
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
from chromite.cbuildbot import constants
from chromite.cbuildbot import metadata_lib
from chromite.lib import cidb
from chromite.lib import cros_build_lib
from chromite.lib import cros_test_lib
from chromite.lib import osutils
from chromite.lib import parallel
SERIES_0_TEST_DATA_PATH = os.path.join(
constants.CHROMITE_DIR, 'cidb', 'test_data', 'series_0')
SERIES_1_TEST_DATA_PATH = os.path.join(
constants.CHROMITE_DIR, 'cidb', 'test_data', 'series_1')
TEST_DB_CRED_ROOT = os.path.join(constants.SOURCE_ROOT,
'crostools', 'cidb',
TEST_DB_CRED_READONLY = os.path.join(constants.SOURCE_ROOT,
'crostools', 'cidb',
TEST_DB_CRED_BOT = os.path.join(constants.SOURCE_ROOT,
'crostools', 'cidb',
class CIDBIntegrationTest(cros_test_lib.TestCase):
"""Base class for cidb tests that connect to a test MySQL instance."""
def _PrepareFreshDatabase(self, max_schema_version=None):
"""Create an empty database with migrations applied.
max_schema_version: The highest schema version migration to apply,
defaults to None in which case all migrations will be applied.
A CIDBConnection instance, connected to a an empty database as the
root user.
# Note: We do not use the cidb.CIDBConnectionFactory
# in this module. That factory method is used only to construct
# connections as the bot user, which is how the builders will always
# connect to the database. In this module, however, we need to test
# database connections as other mysql users.
# Connect to database and drop its contents.
db = cidb.CIDBConnection(TEST_DB_CRED_ROOT)
# Connect to now fresh database and apply migrations.
db = cidb.CIDBConnection(TEST_DB_CRED_ROOT)
return db
class CIDBMigrationsTest(CIDBIntegrationTest):
"""Test that all migrations apply correctly."""
def testMigrations(self):
"""Test that all migrations apply in bulk correctly."""
def testIncrementalMigrations(self):
"""Test that all migrations apply incrementally correctly."""
db = self._PrepareFreshDatabase(0)
migrations = db._GetMigrationScripts()
max_version = migrations[-1][0]
for i in range(1, max_version+1):
def testActions(self):
"""Test that InsertCLActions accepts 0-, 1-, and multi-item lists."""
db = self._PrepareFreshDatabase()
build_id = db.InsertBuild('my builder', 'chromiumos', 12, 'my config',
'my bot hostname')
a1 = metadata_lib.GetCLActionTuple(
metadata_lib.GerritPatchTuple(1, 1, True),
a2 = metadata_lib.GetCLActionTuple(
metadata_lib.GerritPatchTuple(1, 1, True),
a3 = metadata_lib.GetCLActionTuple(
metadata_lib.GerritPatchTuple(1, 1, True),
db.InsertCLActions(build_id, [])
db.InsertCLActions(build_id, [a1])
db.InsertCLActions(build_id, [a2, a3])
action_count = db._GetEngine().execute('select count(*) from clActionTable'
self.assertEqual(action_count, 3)
# Test that all known CL action types can be inserted
fakepatch = metadata_lib.GerritPatchTuple(1, 1, True)
all_actions_list = [metadata_lib.GetCLActionTuple(fakepatch, action)
for action in constants.CL_ACTIONS]
db.InsertCLActions(build_id, all_actions_list)
class CIDBAPITest(CIDBIntegrationTest):
"""Tests of the CIDB API."""
def testSchemaVersionTooLow(self):
"""Tests that the minimum_schema decorator works as expected."""
db = self._PrepareFreshDatabase(3)
db.InsertBuildStages, [])
def testSchemaVersionOK(self):
"""Tests that the minimum_schema decorator works as expected."""
db = self._PrepareFreshDatabase(4)
def GetTestDataSeries(test_data_path):
"""Get metadata from json files at |test_data_path|.
A list of CBuildbotMetadata objects, sorted by their start time.
filenames = glob.glob(os.path.join(test_data_path, '*.json'))
metadatas = []
for fname in filenames:
# Convert start time values, which are stored in RFC 2822 string format,
# to seconds since epoch.
timestamp_from_dict = lambda x: cros_build_lib.ParseUserDateTimeFormat(
return metadatas
class DataSeries0Test(CIDBIntegrationTest):
"""Simulate a set of 630 master/slave CQ builds."""
def testCQWithSchema14(self):
"""Run the CQ test with schema version 14."""
# Run the CQ test at schema version 14
def _runCQTest(self):
"""Simulate a set of 630 master/slave CQ builds.
Note: This test takes about 2.5 minutes to populate its 630 builds
and their corresponding cl actions into the test database.
metadatas = GetTestDataSeries(SERIES_0_TEST_DATA_PATH)
self.assertEqual(len(metadatas), 630, 'Did not load expected amount of '
'test data')
bot_db = cidb.CIDBConnection(TEST_DB_CRED_BOT)
# Simulate the test builds, using a database connection as the
# bot user.
self.simulate_builds(bot_db, metadatas)
# Perform some sanity check queries against the database, connected
# as the readonly user.
readonly_db = cidb.CIDBConnection(TEST_DB_CRED_READONLY)
build_types = readonly_db._GetEngine().execute(
'select build_type from buildTable').fetchall()
self.assertTrue(all(x == ('paladin',) for x in build_types))
build_config_count = readonly_db._GetEngine().execute(
'select COUNT(distinct build_config) from buildTable').fetchall()[0][0]
self.assertEqual(build_config_count, 30)
# Test the _Select method, and verify that the first inserted
# build is a master-paladin build.
first_row = readonly_db._Select('buildTable', 1, ['id', 'build_config'])
self.assertEqual(first_row['build_config'], 'master-paladin')
# First master build has 29 slaves. Build with id 2 is a slave
# build with no slaves of its own.
self.assertEqual(len(readonly_db.GetSlaveStatuses(1)), 29)
self.assertEqual(len(readonly_db.GetSlaveStatuses(2)), 0)
# Make sure we can get build status by build id.
self.assertEqual(readonly_db.GetBuildStatus(2).get('id'), 2)
def _last_updated_time_checks(self, db):
"""Sanity checks on the last_updated column."""
# We should have a diversity of last_updated times. Since the timestamp
# resolution is only 1 second, and we have lots of parallelism in the test,
# we won't have a distring last_updated time per row. But we will have at
# least 100 distinct last_updated times.
distinct_last_updated = db._GetEngine().execute(
'select count(distinct last_updated) from buildTable').fetchall()[0][0]
self.assertTrue(distinct_last_updated > 80)
ids_by_last_updated = db._GetEngine().execute(
'select id from buildTable order by last_updated').fetchall()
ids_by_last_updated = [id_tuple[0] for id_tuple in ids_by_last_updated]
# Build #1 should have been last updated before build # 200.
# However, build #1 (which was a master build) should have been last updated
# AFTER build #2 which was its slave.
def _cl_action_checks(self, db):
"""Sanity checks that correct cl actions were recorded."""
submitted_cl_count = db._GetEngine().execute(
'select count(*) from clActionTable where action="submitted"'
rejected_cl_count = db._GetEngine().execute(
'select count(*) from clActionTable where action="kicked_out"'
total_actions = db._GetEngine().execute(
'select count(*) from clActionTable').fetchall()[0][0]
self.assertEqual(submitted_cl_count, 56)
self.assertEqual(rejected_cl_count, 8)
self.assertEqual(total_actions, 1877)
actions_for_change = db.GetActionsForChange(
metadata_lib.GerritChangeTuple(205535, False))
self.assertEqual(len(actions_for_change), 60)
last_action = actions_for_change[-1]
self.assertEqual(last_action, {'action': 'submitted',
'build_config': 'master-paladin',
'build_id': 511L,
'change_number': 205535L,
'change_source': 'external',
'patch_number': 1L})
def _start_and_finish_time_checks(self, db):
"""Sanity checks that correct data was recorded, and can be retrieved."""
max_start_time = db._GetEngine().execute(
'select max(start_time) from buildTable').fetchall()[0][0]
min_start_time = db._GetEngine().execute(
'select min(start_time) from buildTable').fetchall()[0][0]
max_fin_time = db._GetEngine().execute(
'select max(finish_time) from buildTable').fetchall()[0][0]
min_fin_time = db._GetEngine().execute(
'select min(finish_time) from buildTable').fetchall()[0][0]
self.assertGreater(max_start_time, min_start_time)
self.assertGreater(max_fin_time, min_fin_time)
# For all builds, finish_time should equal last_updated.
mismatching_times = db._GetEngine().execute(
'select count(*) from buildTable where finish_time != last_updated'
self.assertEqual(mismatching_times, 0)
def simulate_builds(self, db, metadatas):
"""Simulate a serires of Commit Queue master and slave builds.
This method use the metadata objects in |metadatas| to simulate those
builds insertions and updates to the cidb. All metadatas encountered
after a particular master build will be assumed to be slaves of that build,
until a new master build is encountered. Slave builds for a particular
master will be simulated in parallel.
The first element in |metadatas| must be a CQ master build.
db: A CIDBConnection instance.
metadatas: A list of CBuildbotMetadata instances, sorted by start time.
m_iter = iter(metadatas)
def is_master(m):
return m.GetDict()['bot-config'] == 'master-paladin'
next_master =
while next_master:
master = next_master
next_master = None
assert is_master(master)
master_build_id = _SimulateBuildStart(db, master)
def simulate_slave(slave_metadata):
build_id = _SimulateBuildStart(db, slave_metadata,
_SimulateCQBuildFinish(db, slave_metadata, build_id)
logging.debug('Simulated slave build %s on pid %s', build_id,
return build_id
slave_metadatas = []
for slave in m_iter:
if is_master(slave):
next_master = slave
with parallel.BackgroundTaskRunner(simulate_slave, processes=15) as queue:
for slave in slave_metadatas:
_SimulateCQBuildFinish(db, master, master_build_id)
logging.debug('Simulated master build %s', master_build_id)
class DataSeries1Test(CIDBIntegrationTest):
"""Simulate a single set of canary builds."""
def runTest(self):
"""Simulate a single set of canary builds with database schema v7."""
metadatas = GetTestDataSeries(SERIES_1_TEST_DATA_PATH)
self.assertEqual(len(metadatas), 18, 'Did not load expected amount of '
'test data')
# Migrate db to specified version. As new schema versions are added,
# migrations to later version can be applied after the test builds are
# simulated, to test that db contents are correctly migrated.
bot_db = cidb.CIDBConnection(TEST_DB_CRED_BOT)
def is_master(m):
return m.GetValue('bot-config') == 'master-release'
master_index = metadatas.index(next(m for m in metadatas if is_master(m)))
master_metadata = metadatas.pop(master_index)
self.assertEqual(master_metadata.GetValue('bot-config'), 'master-release')
master_id = self._simulate_canary(bot_db, master_metadata)
for m in metadatas:
self._simulate_canary(bot_db, m, master_id)
# Verify that expected data was inserted
num_boards = bot_db._GetEngine().execute(
'select count(*) from boardPerBuildTable'
self.assertEqual(num_boards, 40)
main_firmware_versions = bot_db._GetEngine().execute(
'select count(distinct main_firmware_version) from boardPerBuildTable'
self.assertEqual(main_firmware_versions, 29)
# For all builds, finish_time should equal last_updated.
mismatching_times = bot_db._GetEngine().execute(
'select count(*) from buildTable where finish_time != last_updated'
self.assertEqual(mismatching_times, 0)
def _simulate_canary(self, db, metadata, master_build_id=None):
"""Helper method to simulate an individual canary build.
db: cidb instance to use for simulation
metadata: CBuildbotMetadata instance of build to simulate.
master_build_id: Optional id of master build.
build_id of build that was simulated.
build_id = _SimulateBuildStart(db, metadata, master_build_id)
metadata_dict = metadata.GetDict()
# Insert child configs and boards
for child_config_dict in metadata_dict['child-configs']:
db.InsertChildConfigPerBuild(build_id, child_config_dict['name'])
for board in metadata_dict['board-metadata'].keys():
db.InsertBoardPerBuild(build_id, board)
for board, bm in metadata_dict['board-metadata'].items():
db.UpdateBoardPerBuildMetadata(build_id, board, bm)
db.UpdateMetadata(build_id, metadata)
status = metadata_dict['status']['status']
status = _TranslateStatus(status)
db.FinishBuild(build_id, status)
return build_id
def _TranslateStatus(status):
# TODO(akeshet): The status strings used in BuildStatus are not the same as
# those recorded in CBuildbotMetadata. Use a general purpose adapter.
if status == 'passed':
return 'pass'
if status == 'failed':
return 'fail'
return status
def _SimulateBuildStart(db, metadata, master_build_id=None):
"""Returns (build_id, metadata_id) tuple."""
metadata_dict = metadata.GetDict()
# TODO(akeshet): We are pretending that all these builds were on the internal
# waterfall at the moment, for testing purposes. This is because we don't
# actually save in the metadata.json any way to know which waterfall the
# build was on.
waterfall = 'chromeos'
build_id = db.InsertBuild(metadata_dict['builder-name'],
return build_id
def _SimulateCQBuildFinish(db, metadata, build_id):
metadata_dict = metadata.GetDict()
# Insert the first build stage using InsertBuildStage, then batch-insert
# the rest with InsertBuildStages. This allows us to test InsertBuildStage
# without taking too much performance loss in the test.
stage_results = metadata_dict['results']
if len(stage_results) > 0:
r = stage_results[0]
db.InsertBuildStage(build_id, r['name'], r['board'],
_TranslateStatus(r['status']), r['log'],
if len(stage_results) > 1:
stages = [{'build_id': build_id,
'name': r['name'],
'board': r['board'],
'status': _TranslateStatus(r['status']),
'log_url': r['log'],
'summary': r['summary']}
for r in stage_results[1:]]
db.InsertCLActions(build_id, metadata_dict['cl_actions'])
db.UpdateMetadata(build_id, metadata)
status = metadata_dict['status']['status']
status = _TranslateStatus(status)
db.FinishBuild(build_id, status)
# TODO(akeshet): Allow command line args to specify alternate CIDB instance
# for testing.
if __name__ == '__main__':