| # Copyright 2015 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Script to summarize stats for different builds in prod.""" |
| |
| from __future__ import print_function |
| |
| import datetime |
| import itertools |
| import numpy |
| import re |
| import sys |
| |
| from chromite.lib import constants |
| from chromite.lib import cidb |
| from chromite.lib import commandline |
| from chromite.lib import cros_build_lib |
| from chromite.lib import cros_logging as logging |
| |
| |
| # These are the preferred base URLs we use to canonicalize bugs/CLs. |
| BUGANIZER_BASE_URL = 'b/' |
| GUTS_BASE_URL = 't/' |
| CROS_BUG_BASE_URL = 'crbug.com/' |
| INTERNAL_CL_BASE_URL = 'crosreview.com/i/' |
| EXTERNAL_CL_BASE_URL = 'crosreview.com/' |
| CHROMIUM_CL_BASE_URL = 'codereview.chromium.org/' |
| |
| class CLStatsEngine(object): |
| """Engine to generate stats about CL actions taken by the Commit Queue.""" |
| |
| def __init__(self, db): |
| self.db = db |
| self.builds = [] |
| self.claction_history = None |
| self.reasons = {} |
| self.blames = {} |
| self.summary = {} |
| self.builds_by_build_id = {} |
| self.slave_builds_by_master_id = {} |
| self.slave_builds_by_config = {} |
| |
| def GatherBuildAnnotations(self): |
| """Gather the failure annotations for builds from cidb.""" |
| annotations_by_builds = self.db.GetAnnotationsForBuilds( |
| [b['id'] for b in self.builds]) |
| for b in self.builds: |
| build_id = b['id'] |
| build_number = b['build_number'] |
| annotations = annotations_by_builds.get(build_id, []) |
| if not annotations: |
| self.reasons[build_number] = ['None'] |
| self.blames[build_number] = [] |
| else: |
| # TODO(pprabhu) crbug.com/458275 |
| # We currently squash together multiple annotations into one to ease |
| # co-existence with the spreadsheet based logic. Once we've moved off of |
| # using the spreadsheet, we should update all uses of the annotations to |
| # expect one or more annotations. |
| self.reasons[build_number] = [ |
| a['failure_category'] for a in annotations] |
| self.blames[build_number] = [] |
| for annotation in annotations: |
| self.blames[build_number] += self.ProcessBlameString( |
| annotation['blame_url']) |
| |
| @staticmethod |
| def ProcessBlameString(blame_string): |
| """Parse a human-created |blame_string| from the spreadsheet. |
| |
| Returns: |
| A list of canonicalized URLs for bugs or CLs that appear in the blame |
| string. Canonicalized form will be 'crbug.com/1234', |
| 'crosreview.com/1234', 'b/1234', 't/1234', or 'crosreview.com/i/1234' as |
| applicable. |
| """ |
| urls = [] |
| tokens = blame_string.split() |
| |
| # Format to generate the regex patterns. Matches one of provided domain |
| # names, followed by lazy wildcard, followed by greedy digit wildcard, |
| # followed by optional slash and optional comma and optional (# + |
| # alphanum wildcard). |
| general_regex = r'^.*(%s).*?([0-9]+)/?,?(#\S*)?$' |
| |
| crbug = general_regex % r'crbug.com|bugs.chromium.org' |
| internal_review = general_regex % ( |
| r'chrome-internal-review.googlesource.com|crosreview.com/i') |
| external_review = general_regex % ( |
| r'crosreview.com|chromium-review.googlesource.com') |
| guts = general_regex % r't/|gutsv\d.corp.google.com/#ticket/' |
| chromium_review = general_regex % r'codereview.chromium.org' |
| |
| # Buganizer regex is different, as buganizer urls do not end with the bug |
| # number. |
| buganizer = r'^.*(b/|b.corp.google.com/issue\?id=)([0-9]+).*$' |
| |
| # Patterns need to be tried in a specific order -- internal review needs |
| # to be tried before external review, otherwise urls like crosreview.com/i |
| # will be incorrectly parsed as external. |
| patterns = [crbug, |
| internal_review, |
| external_review, |
| buganizer, |
| guts, |
| chromium_review] |
| url_patterns = [CROS_BUG_BASE_URL, |
| INTERNAL_CL_BASE_URL, |
| EXTERNAL_CL_BASE_URL, |
| BUGANIZER_BASE_URL, |
| GUTS_BASE_URL, |
| CHROMIUM_CL_BASE_URL] |
| |
| for t in tokens: |
| for p, u in zip(patterns, url_patterns): |
| m = re.match(p, t) |
| if m: |
| urls.append(u + m.group(2)) |
| break |
| |
| return urls |
| |
| def Gather(self, start_date, end_date, |
| master_config=constants.CQ_MASTER, |
| sort_by_build_number=True, |
| starting_build_number=None, |
| ending_build_number=None): |
| """Fetches build data and failure reasons. |
| |
| Args: |
| start_date: A datetime.date instance for the earliest build to |
| examine. |
| end_date: A datetime.date instance for the latest build to |
| examine. |
| master_config: Config name of master to gather data for. |
| Default to CQ_MASTER. |
| sort_by_build_number: Optional boolean. If True, builds will be |
| sorted by build number. |
| starting_build_number: (optional) The lowest build number to |
| include in the results. |
| ending_build_number: (optional) The highest build number to include |
| in the results. |
| """ |
| logging.info('Gathering data for %s from %s until %s', master_config, |
| start_date, end_date) |
| self.builds = self.db.GetBuildHistory( |
| master_config, |
| start_date=start_date, |
| end_date=end_date, |
| starting_build_number=starting_build_number, |
| num_results=self.db.NUM_RESULTS_NO_LIMIT) |
| if ending_build_number is not None: |
| self.builds = [x for x in self.builds |
| if x['build_number'] <= ending_build_number] |
| if self.builds: |
| logging.info('Fetched %d builds (build_id: %d to %d)', len(self.builds), |
| self.builds[0]['id'], self.builds[-1]['id']) |
| else: |
| logging.info('Fetched no builds.') |
| if sort_by_build_number: |
| logging.info('Sorting by build number.') |
| self.builds.sort(key=lambda x: x['build_number']) |
| |
| self.claction_history = self.db.GetActionHistory(start_date, end_date) |
| self.GatherBuildAnnotations() |
| |
| self.builds_by_build_id.update( |
| {b['id'] : b for b in self.builds}) |
| |
| # Gather slave statuses for each of the master builds. For now this is a |
| # separate query per CQ run, but this could be consolidated to a single |
| # query if necessary (requires adding a cidb.py API method). |
| for bid in self.builds_by_build_id: |
| self.slave_builds_by_master_id[bid] = self.db.GetSlaveStatuses(bid) |
| |
| self.slave_builds_by_config = cros_build_lib.GroupByKey( |
| itertools.chain(*self.slave_builds_by_master_id.values()), |
| 'build_config') |
| |
| def _PrintCounts(self, reasons, fmt): |
| """Print a sorted list of reasons in descending order of frequency. |
| |
| Args: |
| reasons: A key/value mapping mapping the reason to the count. |
| fmt: A format string for our log message, containing %(cnt)d |
| and %(reason)s. |
| """ |
| d = reasons |
| for cnt, reason in sorted(((v, k) for (k, v) in d.items()), reverse=True): |
| logging.info(fmt, dict(cnt=cnt, reason=reason)) |
| if not d: |
| logging.info(' None') |
| |
| def FalseRejectionRate(self, good_patch_count, false_rejection_count): |
| """Calculate the false rejection ratio. |
| |
| This is the chance that a good patch will be rejected by the Pre-CQ or CQ |
| in a given run. |
| |
| Args: |
| good_patch_count: The number of good patches in the run. |
| false_rejection_count: A dict containing the number of false rejections |
| for the CQ and PRE_CQ. |
| |
| Returns: |
| A dict containing the false rejection ratios for CQ, PRE_CQ, and combined. |
| """ |
| false_rejection_rate = dict() |
| for bot, rejection_count in false_rejection_count.iteritems(): |
| false_rejection_rate[bot] = ( |
| rejection_count * 100. / (rejection_count + good_patch_count) |
| ) |
| false_rejection_rate['combined'] = 0 |
| if good_patch_count: |
| rejection_count = sum(false_rejection_count.values()) |
| false_rejection_rate['combined'] = ( |
| rejection_count * 100. / (good_patch_count + rejection_count) |
| ) |
| return false_rejection_rate |
| |
| def GetBuildRunTimes(self, builds): |
| """Gets the elapsed run times of the completed builds within |builds|. |
| |
| Args: |
| builds: Iterable of build statuses as returned by cidb. |
| |
| Returns: |
| A list of the elapsed times (in seconds) of the builds that completed. |
| """ |
| times = [] |
| for b in builds: |
| if b['finish_time']: |
| td = (b['finish_time'] - b['start_time']).total_seconds() |
| times.append(td) |
| return times |
| |
| def Summarize(self, build_type, bad_patch_candidates=False): |
| """Process, print, and return a summary of statistics. |
| |
| As a side effect, save summary to self.summary. |
| |
| Returns: |
| A dictionary summarizing the statistics. |
| """ |
| if build_type == 'cq': |
| return self.SummarizeCQ(bad_patch_candidates=bad_patch_candidates) |
| else: |
| return self.SummarizePFQ() |
| |
| def SummarizeCQ(self, bad_patch_candidates=False): |
| """Process, print, and return a summary of cl action statistics. |
| |
| As a side effect, save summary to self.summary. |
| |
| Returns: |
| A dictionary summarizing the statistics. |
| """ |
| if self.builds: |
| logging.info('%d total runs included, from build %d to %d.', |
| len(self.builds), self.builds[-1]['build_number'], |
| self.builds[0]['build_number']) |
| total_passed = len([b for b in self.builds |
| if b['status'] == constants.BUILDER_STATUS_PASSED]) |
| logging.info('%d of %d runs passed.', total_passed, len(self.builds)) |
| else: |
| logging.info('No runs included.') |
| |
| build_times_sec = sorted(self.GetBuildRunTimes(self.builds)) |
| |
| build_reason_counts = {} |
| for reasons in self.reasons.values(): |
| for reason in reasons: |
| if reason != 'None': |
| build_reason_counts[reason] = build_reason_counts.get(reason, 0) + 1 |
| |
| unique_blames = set() |
| build_blame_counts = {} |
| for blames in self.blames.itervalues(): |
| unique_blames.update(blames) |
| for blame in blames: |
| build_blame_counts[blame] = build_blame_counts.get(blame, 0) + 1 |
| unique_cl_blames = {blame for blame in unique_blames if |
| EXTERNAL_CL_BASE_URL in blame} |
| |
| # Shortcuts to some time aggregates about action history. |
| patch_handle_times = self.claction_history.GetPatchHandlingTimes().values() |
| pre_cq_handle_times = self.claction_history.GetPreCQHandlingTimes().values() |
| cq_wait_times = self.claction_history.GetCQWaitingTimes().values() |
| cq_handle_times = self.claction_history.GetCQHandlingTimes().values() |
| |
| # Calculate how many good patches were falsely rejected and why. |
| good_patch_rejections = self.claction_history.GetFalseRejections() |
| patch_reason_counts = {} |
| patch_blame_counts = {} |
| for k, v in good_patch_rejections.iteritems(): |
| for a in v: |
| build = self.builds_by_build_id.get(a.build_id) |
| if a.bot_type == constants.CQ and build is not None: |
| build_number = build['build_number'] |
| reasons = self.reasons.get(build_number, ['None']) |
| blames = self.blames.get(build_number, ['None']) |
| for x in reasons: |
| patch_reason_counts[x] = patch_reason_counts.get(x, 0) + 1 |
| for x in blames: |
| patch_blame_counts[x] = patch_blame_counts.get(x, 0) + 1 |
| |
| good_patch_count = len(self.claction_history.GetSubmittedPatches(False)) |
| false_rejection_count = {} |
| bad_cl_candidates = {} |
| for bot_type in [constants.CQ, constants.PRE_CQ]: |
| rejections = self.claction_history.GetFalseRejections(bot_type) |
| false_rejection_count[bot_type] = sum(map(len, |
| rejections.values())) |
| |
| rejections = self.claction_history.GetTrueRejections(bot_type) |
| rejected_cls = set([x.GetChangeTuple() for x in rejections.keys()]) |
| bad_cl_candidates[bot_type] = self.claction_history.SortBySubmitTimes( |
| rejected_cls) |
| |
| false_rejection_rate = self.FalseRejectionRate(good_patch_count, |
| false_rejection_count) |
| |
| # This list counts how many times each good patch was rejected. |
| rejection_counts = [0] * (good_patch_count - len(good_patch_rejections)) |
| rejection_counts += [len(x) for x in good_patch_rejections.values()] |
| |
| # Break down the frequency of how many times each patch is rejected. |
| good_patch_rejection_breakdown = [] |
| if rejection_counts: |
| for x in range(max(rejection_counts) + 1): |
| good_patch_rejection_breakdown.append((x, rejection_counts.count(x))) |
| |
| # For CQ runs that passed, track which slave was the long pole, i.e. the |
| # last to finish. |
| long_pole_slave_counts = {} |
| for bid, master_build in self.builds_by_build_id.items(): |
| if master_build['status'] == constants.BUILDER_STATUS_PASSED: |
| if not self.slave_builds_by_master_id[bid]: |
| continue |
| _, long_config = max((slave['finish_time'], slave['build_config']) |
| for slave in self.slave_builds_by_master_id[bid] |
| if slave['finish_time'] and slave['important']) |
| long_pole_slave_counts[long_config] = ( |
| long_pole_slave_counts.get(long_config, 0) + 1) |
| |
| # Calc list of slowest slaves and their corresponding build stats |
| total_counts = sum(long_pole_slave_counts.values()) |
| slowest_cq_slaves = [] |
| for (count, config) in sorted( |
| {v: k for k, v in long_pole_slave_counts.items()}.items(), |
| reverse=True): |
| if count < (total_counts / 20.0): |
| continue |
| build_times = self.GetBuildRunTimes(self.slave_builds_by_config[config]) |
| slave = str(config) |
| percentile_50 = numpy.percentile(build_times, 50) / 3600.0 |
| percentile_90 = numpy.percentile(build_times, 90) / 3600.0 |
| slowest_cq_slaves.append((slave, count, percentile_50, percentile_90)) |
| |
| summary = { |
| 'total_cl_actions': len(self.claction_history), |
| 'total_builds': len(self.builds), |
| 'first_build_num': self.builds[0]['build_number'], |
| 'last_build_num': self.builds[-1]['build_number'], |
| 'unique_cls': len(self.claction_history.affected_cls), |
| 'unique_patches': len(self.claction_history.affected_patches), |
| 'submitted_patches': len(self.claction_history.submit_actions), |
| 'rejections': len(self.claction_history.reject_actions), |
| 'submit_fails': len(self.claction_history.submit_fail_actions), |
| 'good_patch_rejections': sum(rejection_counts), |
| 'mean_good_patch_rejections': numpy.mean(rejection_counts), |
| 'good_patch_rejection_breakdown': good_patch_rejection_breakdown, |
| 'good_patch_rejection_count': false_rejection_count, |
| 'false_rejection_rate': false_rejection_rate, |
| 'median_handling_time': numpy.median(patch_handle_times), |
| 'patch_handling_time': patch_handle_times, |
| 'cl_handling_time_50': numpy.percentile(patch_handle_times, 50)/3600.0, |
| 'cl_handling_time_90': numpy.percentile(patch_handle_times, 90)/3600.0, |
| 'cq_time_50': numpy.percentile(cq_handle_times, 50) / 3600.0, |
| 'cq_time_90': numpy.percentile(cq_handle_times, 90) / 3600.0, |
| 'wait_time_50': numpy.percentile(cq_wait_times, 50) / 3600.0, |
| 'wait_time_90': numpy.percentile(cq_wait_times, 90) / 3600.0, |
| 'cq_run_time_50': numpy.percentile(build_times_sec, 50) / 3600.0, |
| 'cq_run_time_90': numpy.percentile(build_times_sec, 90) / 3600.0, |
| 'bad_cl_candidates': bad_cl_candidates, |
| 'unique_blames_change_count': len(unique_cl_blames), |
| 'long_pole_slave_counts': long_pole_slave_counts, |
| 'slowest_cq_slaves': slowest_cq_slaves, |
| 'patch_flake_rejections': len(good_patch_rejections), |
| 'bad_cl_precq_rejected': len(bad_cl_candidates['pre-cq']), |
| 'false_rejection_total': sum(false_rejection_count.values()), |
| 'false_rejection_pre_cq': false_rejection_count[constants.PRE_CQ], |
| 'false_rejection_cq': false_rejection_count[constants.CQ], |
| 'build_blame_counts': build_blame_counts, |
| 'patch_blame_counts': patch_blame_counts, |
| } |
| |
| logging.info('CQ committed %s changes', summary['submitted_patches']) |
| logging.info('CQ correctly rejected %s unique changes', |
| summary['unique_blames_change_count']) |
| logging.info('pre-CQ and CQ incorrectly rejected %s changes a total of ' |
| '%s times (pre-CQ: %s; CQ: %s)', |
| summary['patch_flake_rejections'], |
| summary['false_rejection_total'], |
| summary['false_rejection_pre_cq'], |
| summary['false_rejection_cq']) |
| |
| logging.info(' Total CL actions: %d.', summary['total_cl_actions']) |
| logging.info(' Unique CLs touched: %d.', summary['unique_cls']) |
| logging.info('Unique patches touched: %d.', summary['unique_patches']) |
| logging.info(' Total CLs submitted: %d.', summary['submitted_patches']) |
| logging.info(' Total rejections: %d.', summary['rejections']) |
| logging.info(' Total submit failures: %d.', summary['submit_fails']) |
| logging.info(' Good patches rejected: %d.', |
| summary['patch_flake_rejections']) |
| logging.info(' Mean rejections per') |
| logging.info(' good patch: %.2f', |
| summary['mean_good_patch_rejections']) |
| logging.info(' False rejection rate for CQ: %.1f%%', |
| summary['false_rejection_rate'].get(constants.CQ, 0)) |
| logging.info(' False rejection rate for Pre-CQ: %.1f%%', |
| summary['false_rejection_rate'].get(constants.PRE_CQ, 0)) |
| logging.info(' Combined false rejection rate: %.1f%%', |
| summary['false_rejection_rate']['combined']) |
| |
| for x, p in summary['good_patch_rejection_breakdown']: |
| logging.info('%d good patches were rejected %d times.', p, x) |
| logging.info('') |
| logging.info('Good patch handling time:') |
| logging.info(' 10th percentile: %.2f hours', |
| numpy.percentile(patch_handle_times, 10) / 3600.0) |
| logging.info(' 25th percentile: %.2f hours', |
| numpy.percentile(patch_handle_times, 25) / 3600.0) |
| logging.info(' 50th percentile: %.2f hours', |
| summary['median_handling_time'] / 3600.0) |
| logging.info(' 75th percentile: %.2f hours', |
| numpy.percentile(patch_handle_times, 75) / 3600.0) |
| logging.info(' 90th percentile: %.2f hours', |
| numpy.percentile(patch_handle_times, 90) / 3600.0) |
| logging.info('') |
| logging.info('Time spent in Pre-CQ:') |
| logging.info(' 10th percentile: %.2f hours', |
| numpy.percentile(pre_cq_handle_times, 10) / 3600.0) |
| logging.info(' 25th percentile: %.2f hours', |
| numpy.percentile(pre_cq_handle_times, 25) / 3600.0) |
| logging.info(' 50th percentile: %.2f hours', |
| numpy.percentile(pre_cq_handle_times, 50) / 3600.0) |
| logging.info(' 75th percentile: %.2f hours', |
| numpy.percentile(pre_cq_handle_times, 75) / 3600.0) |
| logging.info(' 90th percentile: %.2f hours', |
| numpy.percentile(pre_cq_handle_times, 90) / 3600.0) |
| logging.info('') |
| logging.info('Time spent waiting for CQ:') |
| logging.info(' 10th percentile: %.2f hours', |
| numpy.percentile(cq_wait_times, 10) / 3600.0) |
| logging.info(' 25th percentile: %.2f hours', |
| numpy.percentile(cq_wait_times, 25) / 3600.0) |
| logging.info(' 50th percentile: %.2f hours', |
| numpy.percentile(cq_wait_times, 50) / 3600.0) |
| logging.info(' 75th percentile: %.2f hours', |
| numpy.percentile(cq_wait_times, 75) / 3600.0) |
| logging.info(' 90th percentile: %.2f hours', |
| numpy.percentile(cq_wait_times, 90) / 3600.0) |
| logging.info('') |
| logging.info('Time spent in CQ:') |
| logging.info(' 10th percentile: %.2f hours', |
| numpy.percentile(cq_handle_times, 10) / 3600.0) |
| logging.info(' 25th percentile: %.2f hours', |
| numpy.percentile(cq_handle_times, 25) / 3600.0) |
| logging.info(' 50th percentile: %.2f hours', |
| numpy.percentile(cq_handle_times, 50) / 3600.0) |
| logging.info(' 75th percentile: %.2f hours', |
| numpy.percentile(cq_handle_times, 75) / 3600.0) |
| logging.info(' 90th percentile: %.2f hours', |
| numpy.percentile(cq_handle_times, 90) / 3600.0) |
| logging.info('') |
| |
| # Log some statistics about cq-master run-time. |
| logging.info('CQ-master run time:') |
| logging.info(' 50th percentile: %.2f hours', |
| numpy.percentile(build_times_sec, 50) / 3600.0) |
| logging.info(' 90th percenfile: %.2f hours', |
| numpy.percentile(build_times_sec, 90) / 3600.0) |
| |
| for bot_type, patches in summary['bad_cl_candidates'].items(): |
| logging.info('%d bad patch candidates were rejected by the %s', |
| len(patches), bot_type) |
| if bad_patch_candidates: |
| for k in patches: |
| logging.info('Bad patch candidate in: %s', k) |
| |
| fmt_fai = ' %(cnt)d failures in %(reason)s' |
| fmt_rej = ' %(cnt)d rejections due to %(reason)s' |
| |
| logging.info('Reasons why good patches were rejected:') |
| self._PrintCounts(patch_reason_counts, fmt_rej) |
| |
| logging.info('Bugs or CLs responsible for good patches rejections:') |
| self._PrintCounts(patch_blame_counts, fmt_rej) |
| |
| logging.info('Reasons why builds failed:') |
| self._PrintCounts(build_reason_counts, fmt_fai) |
| |
| logging.info('Bugs or CLs responsible for build failures:') |
| self._PrintCounts(build_blame_counts, fmt_fai) |
| |
| logging.info('Slowest CQ slaves out of %s passing runs:', total_counts) |
| |
| for slave, count, per_50, per_90 in summary['slowest_cq_slaves']: |
| logging.info('%s times the slowest slave was %s', count, slave) |
| logging.info(' 50th percentile: %.2f hours, 90th percentile: %.2f hours', |
| per_50, |
| per_90) |
| return summary |
| |
| # TODO(akeshet): some of this logic is copied directly from SummarizeCQ. |
| # Refactor to reuse that code instead. |
| def SummarizePFQ(self): |
| """Process, print, and return a summary of pfq bug and failure statistics. |
| |
| As a side effect, save summary to self.summary. |
| |
| Returns: |
| A dictionary summarizing the statistics. |
| """ |
| if self.builds: |
| logging.info('%d total runs included, from build %d to %d.', |
| len(self.builds), self.builds[-1]['build_number'], |
| self.builds[0]['build_number']) |
| total_passed = len([b for b in self.builds |
| if b['status'] == constants.BUILDER_STATUS_PASSED]) |
| logging.info('%d of %d runs passed.', total_passed, len(self.builds)) |
| else: |
| logging.info('No runs included.') |
| |
| # TODO(akeshet): This is the end of the verbatim copied code. |
| |
| # Count the number of times each particular (canonicalized) blame url was |
| # given. |
| unique_blame_counts = {} |
| for blames in self.blames.itervalues(): |
| for b in blames: |
| unique_blame_counts[b] = unique_blame_counts.get(b, 0) + 1 |
| |
| top_blames = sorted([(count, blame) for |
| blame, count in unique_blame_counts.iteritems()], |
| reverse=True) |
| logging.info('Top blamed issues:') |
| if top_blames: |
| for tb in top_blames: |
| logging.info(' %s x %s', tb[0], tb[1]) |
| else: |
| logging.info('None!') |
| |
| return {} |
| |
| ReportHTMLTemplate = """ |
| <head> |
| <style> |
| table {{border-collapse: collapse; width: 50%}} |
| table, td, th {{border: 1px solid black}} |
| th, td {{padding: 5px}} |
| td {{text-align: left}} |
| replace {{background-color: red; font-style: bold}} |
| #note {{background-color: green; font-style: italic}} |
| </style> |
| <title>Summary of CQ Performance for the past week</title> |
| </head> |
| |
| <body> |
| <p id="note"> |
| Instruction text in italic green. Places to replace are in <replace>bold red</replace>.<br> |
| |
| <ol id="note"> |
| <li>Copy-paste this page into an email</li> |
| <li>Set subject line to "Summary of CQ performance for the past week"</li> |
| <li>Follow instructions in italic green</li> |
| <li>Delete any green text</li> |
| <li>Insure that all <replace>REPLACE</replace> text is replaced</li> |
| <li>Email to <a href="mailto:chromeos-infra-discuss@google.com">chromeos-infra-discuss@google.com</a> |
| </ol> |
| </p> |
| |
| <p> |
| <b>{total_builds}</b> CQ runs included, from build <b>{first_build_num}</b> to <b>{last_build_num}</b>. |
| </p> |
| |
| <p> |
| The CQ <b>committed {submitted_patches} changes</b> this week.<br> |
| The CQ <b>correctly rejected {unique_blames_change_count} unique changes</b> this week, which would otherwise have broken the tree and required investigation and revert.<br> |
| </p> |
| |
| <p> |
| The pre-CQ <b>rejected {bad_cl_precq_rejected} changes this week</b>, which would otherwise have broken a CQ run and affected other developers.<br> |
| </p> |
| |
| <p> |
| The CL handling time was <b>{cl_handling_time_50:.2f} hours</b> 50%ile <b>{cl_handling_time_90:.2f} hours</b> 90%ile.<br> |
| Time spent in the CQ was <b>{cq_time_50:.2f} hours</b> 50%ile <b>{cq_time_90:.2f} hours</b> 90%ile.<br> |
| Time spent waiting was <b>{wait_time_50:.2f} hours</b> 50%ile <b>{wait_time_90:.2f} hours</b> 90%ile.<br> |
| CQ run time was <b>{cq_run_time_50:.2f} hours</b> 50%ile <b>{cq_run_time_90:.2f} hours</b> 90%ile.<br> |
| </p> |
| |
| <h2>Slowest Passing Slaves</h2> |
| <table> |
| <tr> |
| <th>Slave</th> |
| <th>Times Slowest Slave</th> |
| <th>Median Time (hours)</th> |
| <th>90th Percentile (hours)</th> |
| </tr> |
| {slow_slaves_html} |
| </table> |
| </p> |
| |
| The tree <b>was:</b><br> |
| <p id="note"> |
| Get this value from <a href="https://chromiumos-status.appspot.com/status_viewer?curView=stats&startTime=TODAY&numDays=7">here</a> (subtract 100% - %open - %throttled) |
| </p> |
| <b> |
| <table> |
| <tr> |
| <td>Open</td> |
| <td>_<replace>REPLACE</replace>_</td> |
| </tr> |
| <tr> |
| <td>Throttle</td> |
| <td>_<replace>REPLACE</replace>_</td> |
| </tr> |
| <tr> |
| <td>Close</td> |
| <td>_<replace>REPLACE</replace>_</td> |
| </tr> |
| </table> |
| </b> |
| |
| <p> |
| The pre-CQ + CQ <b>incorrectly rejected [{patch_flake_rejections}] unique changes a total of [{false_rejection_total}] times</b> this week. (Pre-CQ:[{false_rejection_pre_cq}]; CQ: [{false_rejection_cq}])<br> |
| The probability of a good patch being incorrectly rejected by the CQ or Pre-CQ is [{false_rejection_rate[combined]:.2f}]%. (Pre-CQ:[{false_rejection_rate[pre-cq]:.2f}]%; CQ: [{false_rejection_rate[cq]:.2f}])<br> |
| </p> |
| |
| <h2>Top reasons that good changes were rejected</h2> |
| <table> |
| <tr> |
| <th><b>Number of rejections</b></th> |
| <th><b>Explanation</b></th> |
| </tr> |
| <tr> |
| <td>[_<replace>X</replace>_]</td> |
| <td>_<replace>REPLACE</replace>_</td> |
| </tr> |
| <tr> |
| <td>[_<replace>X</replace>_]</td> |
| <td>_<replace>REPLACE</replace>_</td> |
| </tr> |
| <tr> |
| <td>[_<replace>X</replace>_]</td> |
| <td><replace>REPLACE</replace></td> |
| </tr> |
| <tr> |
| <td>[_<replace>X</replace>_]</td> |
| <td>_<replace>REPLACE</replace>_</td> |
| </tr> |
| </table> |
| |
| <h2>What issues caused the most CQ flakiness this week?</h2> |
| <i>Note: test flake may be caused by flake in the test itself, or flake in the product.</i><br> |
| <p id="note">At your discretion, pick the top few which caused the most failures</p> |
| <ul> |
| {cq_flakes_html} |
| </ul> |
| |
| <h2>What CLs caused issues for other developers?</h2> |
| <p id="note">At your discretion, pull the top blamed CLs from the summarize_build_stats output. When appropriate, summarize what action we plan to take to block these changes in the pre-CQ.</p> |
| <ul> |
| {cl_flakes_html} |
| </ul> |
| |
| <h2>What was the patch turnaround time?</h2> |
| <p id="note"> Copy/Paste in the histogram from to top of <a href="http://go/chromiumos-build-annotator">go/chromiumos-build-annotator</a>. Update your visible list to match the build ID's listed at the top of your email first, or the values will be randomized. </p> |
| _<replace>IMAGE_PLACEHOLDER</replace>_<br> |
| <br> |
| |
| <i>Generated on {datetime}</i> |
| </body> |
| """ |
| |
| def GenerateReport(file_out, summary): |
| """Write formated summary to file_out""" |
| |
| report = summary.copy() |
| report['datetime'] = str(datetime.datetime.now()) |
| |
| def gen_html_row(l): |
| s = "" |
| for r in l: |
| s += ' <tr>\n' |
| for i in r: |
| s += " <td>{}</td>".format(i) |
| s += ' </tr>\n' |
| return s |
| |
| report['slow_slaves_html'] = gen_html_row(summary['slowest_cq_slaves']) |
| |
| def gen_html_list(fmt, items): |
| s = "" |
| for i in items: |
| s += " <li>{}</li>\n".format(fmt.format(**i)) |
| return s |
| |
| cq_flakes = [] |
| for rejections, b_id in sorted(((v, k) for (k, v) \ |
| in summary['patch_blame_counts'].items()), reverse=True): |
| cq_flakes.append({'id': b_id, 'rejections': rejections}) |
| |
| flake_fmt = '<a href="http://{id}">{id}</a> (<b>[{rejections}] false ' + \ |
| 'rejections</b>): _<replace>Brief explanation of bug. If ' + \ |
| 'fixed, or describe workarounds</replace>_' |
| |
| report['cq_flakes_html'] = gen_html_list(flake_fmt, cq_flakes) |
| |
| cl_fails = [] |
| for rejections, b_id in \ |
| sorted(((v, k) for (k, v) in summary['build_blame_counts'].items()), |
| reverse=True): |
| cl_fails.append({'id': b_id, 'rejections': rejections}) |
| |
| cl_flake_fmt = '<a href="http://{id}">{id}</a> (<b>[{rejections}] false ' + \ |
| 'rejection of build</b>): _<replace>explanation</replace>_' |
| report['cl_flakes_html'] = gen_html_list(cl_flake_fmt, cl_fails) |
| |
| file_out.write(ReportHTMLTemplate.format(**report)) |
| |
| |
| def _CheckOptions(options): |
| # Ensure that specified start date is in the past. |
| now = datetime.datetime.now() |
| if options.start_date and now.date() < options.start_date: |
| logging.error('Specified start date is in the future: %s', |
| options.start_date) |
| return False |
| |
| return True |
| |
| |
| def GetParser(): |
| """Creates the argparse parser.""" |
| parser = commandline.ArgumentParser(description=__doc__) |
| |
| ex_group = parser.add_mutually_exclusive_group(required=True) |
| ex_group.add_argument('--start-date', action='store', type='date', |
| default=None, |
| help='Limit scope to a start date in the past.') |
| ex_group.add_argument('--past-month', action='store_true', default=False, |
| help='Limit scope to the past 30 days up to now.') |
| ex_group.add_argument('--past-week', action='store_true', default=False, |
| help='Limit scope to the past week up to now.') |
| ex_group.add_argument('--past-day', action='store_true', default=False, |
| help='Limit scope to the past day up to now.') |
| |
| parser.add_argument('--cred-dir', action='store', required=True, |
| metavar='CIDB_CREDENTIALS_DIR', |
| help='Database credentials directory with certificates ' |
| 'and other connection information. Obtain your ' |
| 'credentials at go/cros-cidb-admin .') |
| parser.add_argument('--starting-build', action='store', type=int, |
| default=None, help='Filter to builds after given number' |
| '(inclusive).') |
| parser.add_argument('--ending-build', action='store', type=int, |
| default=None, help='Builder to builds before a given ' |
| 'number (inclusive).') |
| parser.add_argument('--end-date', action='store', type='date', default=None, |
| help='Limit scope to an end date in the past.') |
| |
| parser.add_argument('--build-type', choices=['cq', 'chrome-pfq'], |
| default='cq', |
| help='Build type to summarize. Default: cq.') |
| parser.add_argument('--bad-patch-candidates', action='store_true', |
| default=False, |
| help='In CQ mode, whether to print bad patch ' |
| 'candidates.') |
| parser.add_argument('--report-file', action='store', |
| help='Write HTML formatted report to given file') |
| return parser |
| |
| |
| def main(argv): |
| parser = GetParser() |
| options = parser.parse_args(argv) |
| |
| if not _CheckOptions(options): |
| sys.exit(1) |
| |
| db = cidb.CIDBConnection(options.cred_dir) |
| |
| if options.end_date: |
| end_date = options.end_date |
| else: |
| end_date = datetime.datetime.now().date() |
| |
| # Determine the start date to use, which is required. |
| if options.start_date: |
| start_date = options.start_date |
| else: |
| assert options.past_month or options.past_week or options.past_day |
| if options.past_month: |
| start_date = end_date - datetime.timedelta(days=30) |
| elif options.past_week: |
| start_date = end_date - datetime.timedelta(days=7) |
| else: |
| start_date = end_date - datetime.timedelta(days=1) |
| |
| if options.build_type == 'cq': |
| master_config = constants.CQ_MASTER |
| else: |
| master_config = constants.PFQ_MASTER |
| |
| cl_stats_engine = CLStatsEngine(db) |
| cl_stats_engine.Gather(start_date, end_date, master_config, |
| starting_build_number=options.starting_build, |
| ending_build_number=options.ending_build) |
| summary = cl_stats_engine.Summarize(options.build_type, |
| options.bad_patch_candidates) |
| |
| if options.report_file: |
| with open(options.report_file, "w") as f: |
| logging.info("Writing report to %s", options.report_file) |
| GenerateReport(f, summary) |
