| #!/usr/bin/env python |
| |
| # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # Script to check the history of stage calls made to devserver. |
| # Following are some sample use cases: |
| # |
| # 1. Find all stage request for autotest and image nyan_big-release/R38-6055.0.0 |
| # in the last 10 days across all devservers. |
| # ./devserver_history.py --image_filters nyan_big 38 6055.0.0 -l 240 \ |
| # --artifact_filters autotest -v |
| # output: |
| # ============================================================================== |
| # 170.21.64.22 |
| # ============================================================================== |
| # Number of calls: 1 |
| # Number of unique images: 1 |
| # 2014-08-23 12:45:00: nyan_big-release/R38-6055.0.0 autotest |
| # ============================================================================== |
| # 170.21.64.23 |
| # ============================================================================== |
| # Number of calls: 2 |
| # Number of unique images: 1 |
| # 2014-08-23 12:45:00: nyan_big-release/R38-6055.0.0 autotest, test_suites |
| # 2014-08-23 12:55:00: nyan_big-release/R38-6055.0.0 autotest, test_suites |
| # |
| # 2. Find all duplicated stage request for the last 10 days. |
| # ./devserver_history.py -d -l 240 |
| # output: |
| # Detecting artifacts staged in multiple devservers. |
| # ============================================================================== |
| # nyan_big-release/R38-6055.0.0 |
| # ============================================================================== |
| # 170.21.64.22: 23 requests 2014-09-04 22:44:28 -- 2014-09-05 00:03:23 |
| # 170.21.64.23: 6 requests 2014-09-04 22:48:58 -- 2014-09-04 22:49:42 |
| # |
| # Count of images with duplicated stages on each devserver: |
| # 170.21.64.22 : 22 |
| # 170.21.64.23 : 11 |
| |
| |
| import argparse |
| import datetime |
| import logging |
| import operator |
| import re |
| import time |
| from itertools import groupby |
| |
| import common |
| from autotest_lib.client.common_lib import global_config |
| from autotest_lib.client.common_lib import time_utils |
| from autotest_lib.client.common_lib.cros.graphite import es_utils |
| |
| |
| class devserver_call(object): |
| """A container to store the information of devserver stage call. |
| """ |
| |
| def __init__(self, hit): |
| """Retrieve information from a ES query hit. |
| """ |
| self.devserver = hit['_source']['devserver'] |
| self.subname = hit['_source']['subname'] |
| self.artifacts = hit['_source']['artifacts'].split(' ') |
| self.image = hit['_source']['image'] |
| self.value = hit['_source']['value'] |
| self.time_recorded = time_utils.epoch_time_to_date_string( |
| hit['_source']['time_recorded']) |
| |
| |
| def __str__(self): |
| pairs = ['%-20s: %s' % (attr, getattr(self, attr)) for attr in dir(self) |
| if not attr.startswith('__') and |
| not callable(getattr(self, attr))] |
| return '\n'.join(pairs) |
| |
| |
| def get_calls(time_start, time_end, artifact_filters=None, |
| regex_constraints=None, devserver=None, size=1e7): |
| """Gets all devserver calls from es db with the given constraints. |
| |
| @param time_start: Earliest time entry was recorded. |
| @param time_end: Latest time entry was recorded. |
| @param artifact_filters: A list of names to match artifacts. |
| @param regex_constraints: A list of regex constraints for ES query. |
| @param devserver: name of devserver to query for. If it's set to None, |
| return calls for all devservers. Default is set to None. |
| @param size: Max number of entries to return, default to 1 million. |
| |
| @returns: Entries from esdb. |
| """ |
| eqs = [('_type', 'devserver')] |
| if devserver: |
| eqs.append(('devserver', devserver)) |
| if artifact_filters: |
| for artifact in artifact_filters: |
| eqs.append(('artifacts', artifact)) |
| query = es_utils.create_range_eq_query_multiple( |
| fields_returned=None, |
| equality_constraints=eqs, |
| range_constraints=[('time_recorded', time_start, time_end)], |
| size=size, |
| sort_specs=[{'time_recorded': 'desc'}], |
| regex_constraints=regex_constraints) |
| results = es_utils.execute_query(query, timeout=30) |
| devserver_calls = [] |
| for hit in results['hits']['hits']: |
| devserver_calls.append(devserver_call(hit)) |
| logging.info('Found %d calls.', len(devserver_calls)) |
| return devserver_calls |
| |
| |
| def print_call_details(calls, verbose): |
| """Print details of each call to devserver to stage artifacts. |
| |
| @param calls: A list of devserver stage requests. |
| @param verbose: Set to True to print out all devserver calls. |
| """ |
| calls = sorted(calls, key=lambda c: c.devserver) |
| for devserver,calls_for_devserver in groupby(calls, lambda c: c.devserver): |
| calls_for_devserver = list(calls_for_devserver) |
| print '='*80 |
| print devserver |
| print '='*80 |
| print 'Number of calls: %d' % len(calls_for_devserver) |
| print ('Number of unique images: %d' % |
| len(set([call.image for call in calls_for_devserver]))) |
| if verbose: |
| for call in sorted(calls_for_devserver, |
| key=lambda c: c.time_recorded): |
| print ('%s %s %s' % (call.time_recorded, call.image, |
| ', '.join(call.artifacts))) |
| |
| |
| def detect_duplicated_stage(calls): |
| """Detect any artifact for same build was staged in multiple devservers. |
| |
| @param calls: A list of devserver stage requests. |
| """ |
| print '\nDetecting artifacts staged in multiple devservers.' |
| calls = sorted(calls, key=lambda c: c.image) |
| # Count how many times a devserver staged duplicated artifacts. A number |
| # significantly larger then others can indicate that the devserver failed |
| # check_health too often and needs to be removed from production. |
| duplicated_stage_count = {} |
| for image,calls_for_image in groupby(calls, lambda c: c.image): |
| calls_for_image = list(calls_for_image) |
| devservers = set([call.devserver for call in calls_for_image]) |
| if len(devservers) > 1: |
| print '='*80 |
| print image |
| print '='*80 |
| calls_for_image = sorted(calls_for_image, key=lambda c: c.devserver) |
| for devserver,calls_for_devserver in groupby(calls_for_image, |
| lambda c: c.devserver): |
| timestamps = [c.time_recorded for c in calls_for_devserver] |
| print ('%s: %-3d requests %s -- %s' % |
| (devserver, len(timestamps), min(timestamps), |
| max(timestamps))) |
| duplicated_stage_count[devserver] = ( |
| duplicated_stage_count.get(devserver, 0) + 1) |
| print '\nCount of images with duplicated stages on each devserver:' |
| counts = sorted(duplicated_stage_count.iteritems(), |
| key=operator.itemgetter(1), reverse=True) |
| for k,v in counts: |
| print '%-15s: %d' % (k, v) |
| |
| |
| def main(): |
| """main script. """ |
| t_now = time.time() |
| t_now_minus_one_day = t_now - 3600 * 24 |
| parser = argparse.ArgumentParser() |
| parser.add_argument('-l', type=float, dest='last', |
| help='last hours to search results across', |
| default=None) |
| parser.add_argument('--start', type=str, dest='start', |
| help=('Enter start time as: yyyy-mm-dd hh-mm-ss,' |
| 'defualts to 24h ago. This option is ignored when' |
| ' -l is used.'), |
| default=time_utils.epoch_time_to_date_string( |
| t_now_minus_one_day)) |
| parser.add_argument('--end', type=str, dest='end', |
| help=('Enter end time in as: yyyy-mm-dd hh-mm-ss,' |
| 'defualts to current time. This option is ignored' |
| ' when -l is used.'), |
| default=time_utils.epoch_time_to_date_string(t_now)) |
| parser.add_argument('--devservers', nargs='+', dest='devservers', |
| help=('Enter space deliminated devservers. Default are' |
| ' all devservers specified in global config.'), |
| default=[]) |
| parser.add_argument('--artifact_filters', nargs='+', |
| dest='artifact_filters', |
| help=('Enter space deliminated filters on artifact ' |
| 'name. For example "autotest test_suites". The ' |
| 'filter does not support regex.'), |
| default=[]) |
| parser.add_argument('--image_filters', nargs='+', dest='image_filters', |
| help=('Enter space deliminated filters on image name. ' |
| 'For example "nyan 38 6566", search will use ' |
| 'regex to match each filter. Do not use filters ' |
| 'with mixed letter and number, e.g., R38.'), |
| default=[]) |
| parser.add_argument('-d', '--detect_duplicated_stage', action='store_true', |
| dest='detect_duplicated_stage', |
| help=('Set to True to detect if an artifacts for a same' |
| ' build was staged in multiple devservers. ' |
| 'Default is True.'), |
| default=False) |
| parser.add_argument('-v', action='store_true', dest='verbose', |
| default=False, |
| help='-v to print out ALL entries.') |
| options = parser.parse_args() |
| if options.verbose: |
| logging.getLogger().setLevel(logging.INFO) |
| |
| if options.last: |
| end_time = datetime.datetime.now() |
| start_time = end_time - datetime.timedelta(seconds=3600 * options.last) |
| else: |
| start_time = datetime.datetime.strptime(options.start, |
| time_utils.TIME_FMT) |
| end_time = datetime.datetime.strptime(options.end, time_utils.TIME_FMT) |
| logging.info('Searching devserver calls from %s to %s', start_time, |
| end_time) |
| |
| devservers = options.devservers |
| if not devservers: |
| devserver_urls = global_config.global_config.get_config_value( |
| 'CROS', 'dev_server', type=list, default=[]) |
| devservers = [] |
| for url in devserver_urls: |
| match = re.match('http://([^:]*):*\d*', url) |
| devservers.append(match.groups(0)[0] if match else url) |
| logging.info('Found devservers: %s', devservers) |
| |
| regex_constraints = [] |
| for filter in options.image_filters: |
| regex_constraints.append(('image', '.*%s.*' % filter)) |
| calls = [] |
| for devserver in devservers: |
| calls.extend(get_calls(start_time, end_time, options.artifact_filters, |
| regex_constraints, devserver=devserver)) |
| |
| print_call_details(calls, options.verbose) |
| |
| if options.detect_duplicated_stage: |
| detect_duplicated_stage(calls) |
| |
| |
| if __name__ == '__main__': |
| main() |
