| # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # This file contains utility functions for host_history. |
| |
| import collections |
| import copy |
| |
| import common |
| from autotest_lib.client.common_lib import time_utils |
| from autotest_lib.client.common_lib.cros.graphite import es_utils |
| from autotest_lib.frontend import setup_django_environment |
| from autotest_lib.frontend.afe import models |
| from autotest_lib.site_utils import job_history |
| |
| def prepopulate_dict(keys, value, extras=None): |
| """Creates a dictionary with val=value for each key. |
| |
| @param keys: list of keys |
| @param value: the value of each entry in the dict. |
| @param extras: list of additional keys |
| @returns: dictionary |
| """ |
| result = collections.OrderedDict() |
| extra_keys = tuple(extras if extras else []) |
| for key in keys + extra_keys: |
| result[key] = value |
| return result |
| |
| |
| def lock_history_to_intervals(initial_lock_val, t_start, t_end, lock_history): |
| """Converts lock history into a list of intervals of locked times. |
| |
| @param initial_lock_val: Initial value of the lock (False or True) |
| @param t_start: beginning of the time period we are interested in. |
| @param t_end: end of the time period we are interested in. |
| @param lock_history: Result of querying es for locks (dict) |
| This dictionary should contain keys 'locked' and 'time_recorded' |
| @returns: Returns a list of tuples where the elements of each tuples |
| represent beginning and end of intervals of locked, respectively. |
| """ |
| locked_intervals = [] |
| t_prev = t_start |
| state_prev = initial_lock_val |
| for entry in lock_history['hits']['hits']: |
| t_curr = entry['fields']['time_recorded'][0] |
| |
| #If it is locked, then we put into locked_intervals |
| if state_prev: |
| locked_intervals.append((t_prev, t_curr)) |
| |
| # update vars |
| t_prev = t_curr |
| state_prev = entry['fields']['locked'][0] |
| if state_prev: |
| locked_intervals.append((t_prev, t_end)) |
| return locked_intervals |
| |
| |
| def find_most_recent_entry_before(t, type_str, hostname, fields, index): |
| """Returns the fields of the most recent entry before t. |
| |
| @param t: time we are interested in. |
| @param type_str: _type in esdb, such as 'host_history' (string) |
| @param hostname: hostname of DUT (string) |
| @param fields: list of fields we are interested in |
| @param index: index in elasticsearch to query data for. |
| @returns: time, field_value of the latest entry. |
| """ |
| query = es_utils.create_range_eq_query_multiple( |
| fields_returned=fields, |
| equality_constraints=[('_type', type_str), |
| ('hostname', hostname)], |
| range_constraints=[('time_recorded', None, t)], |
| size=1, |
| sort_specs=[{'time_recorded': 'desc'}]) |
| result = es_utils.execute_query( |
| query, index, |
| es_utils.METADATA_ES_SERVER, es_utils.ES_PORT) |
| if result['hits']['total'] > 0: |
| # If fields are not specified, the query returns all data for the |
| # record under key "_source" |
| key = 'fields' if fields else '_source' |
| return es_utils.convert_hit(result['hits']['hits'][0][key]) |
| return {} |
| |
| |
| def host_history_intervals(t_start, t_end, hostname, size, index): |
| """Gets stats for a host. |
| |
| @param t_start: beginning of time period we are interested in. |
| @param t_end: end of time period we are interested in. |
| @param hostname: hostname for the host we are interested in (string) |
| @param size: maximum number of entries returned per query |
| @param index: index in elasticsearch to query data for. |
| @returns: dictionary, num_entries_found |
| dictionary of status: time spent in that status |
| num_entries_found: number of host history entries |
| found in [t_start, t_end] |
| |
| """ |
| lock_history_recent = find_most_recent_entry_before( |
| t=t_start, type_str='lock_history', hostname=hostname, |
| fields=['time_recorded', 'locked'], index=index) |
| # I use [0] and [None] because lock_history_recent's type is list. |
| t_lock = lock_history_recent.get('time_recorded', None) |
| t_lock_val = lock_history_recent.get('locked', None) |
| host_history_recent = find_most_recent_entry_before( |
| t=t_start, type_str='host_history', hostname=hostname, |
| fields=None, index=index) |
| t_host = host_history_recent.get('time_recorded', None) |
| t_host_stat = host_history_recent.get('status', None) |
| t_metadata = es_utils.get_metadata(host_history_recent, |
| ['time_recorded', 'status']) |
| |
| status_first = t_host_stat if t_host else 'Ready' |
| t = min([t for t in [t_lock, t_host, t_start] if t]) |
| |
| query_lock_history = es_utils.create_range_eq_query_multiple( |
| fields_returned=['locked', 'time_recorded'], |
| equality_constraints=[('_type', 'lock_history'), |
| ('hostname', hostname)], |
| range_constraints=[('time_recorded', t, t_end)], |
| size=size, |
| sort_specs=[{'time_recorded': 'asc'}]) |
| |
| lock_history_entries = es_utils.execute_query( |
| query_lock_history, index, |
| es_utils.METADATA_ES_SERVER, es_utils.ES_PORT) |
| |
| locked_intervals = lock_history_to_intervals(t_lock_val, t, t_end, |
| lock_history_entries) |
| query_host_history = es_utils.create_range_eq_query_multiple( |
| fields_returned=None, |
| equality_constraints=[("_type", "host_history"), |
| ("hostname", hostname)], |
| range_constraints=[("time_recorded", t_start, t_end)], |
| size=size, |
| sort_specs=[{"time_recorded": "asc"}]) |
| host_history_entries = es_utils.execute_query( |
| query_host_history, index, |
| es_utils.METADATA_ES_SERVER, es_utils.ES_PORT) |
| num_entries_found = host_history_entries['hits']['total'] |
| t_prev = t_start |
| status_prev = status_first |
| metadata_prev = t_metadata |
| intervals_of_statuses = collections.OrderedDict() |
| |
| for entry in host_history_entries['hits']['hits']: |
| t_curr = entry['_source']['time_recorded'] |
| status_curr = entry['_source']['status'] |
| metadata = es_utils.get_metadata(entry['_source'], |
| ['time_recorded', 'status']) |
| intervals_of_statuses.update(calculate_status_times( |
| t_prev, t_curr, status_prev, metadata_prev, locked_intervals)) |
| # Update vars |
| t_prev = t_curr |
| status_prev = status_curr |
| metadata_prev = metadata |
| |
| # Do final as well. |
| intervals_of_statuses.update(calculate_status_times( |
| t_prev, t_end, status_prev, metadata_prev, locked_intervals)) |
| return intervals_of_statuses, num_entries_found |
| |
| |
| def calculate_total_times(intervals_of_statuses): |
| """Calculates total times in each status. |
| |
| @param intervals_of_statuses: ordereddict where key=(ti, tf) and val=status |
| @returns: dictionary where key=status value=time spent in that status |
| """ |
| total_times = prepopulate_dict(models.Host.Status.names, 0.0, |
| extras=['Locked']) |
| for key, status_info in intervals_of_statuses.iteritems(): |
| ti, tf = key |
| total_times[status_info['status']] += tf - ti |
| return total_times |
| |
| |
| def aggregate_hosts(intervals_of_statuses_list): |
| """Aggregates history of multiple hosts |
| |
| @param intervals_of_statuses_list: A list of dictionaries where keys |
| are tuple (ti, tf), and value is the status along with other metadata. |
| @returns: A dictionary where keys are strings, e.g. 'status' and |
| value is total time spent in that status among all hosts. |
| """ |
| stats_all = prepopulate_dict(models.Host.Status.names, 0.0, |
| extras=['Locked']) |
| num_hosts = len(intervals_of_statuses_list) |
| for intervals_of_statuses in intervals_of_statuses_list: |
| total_times = calculate_total_times(intervals_of_statuses) |
| for status, delta in total_times.iteritems(): |
| stats_all[status] += delta |
| return stats_all, num_hosts |
| |
| |
| def get_stats_string_aggregate(labels, t_start, t_end, aggregated_stats, |
| num_hosts): |
| """Returns string reporting overall host history for a group of hosts. |
| |
| @param labels: A list of labels useful for describing the group |
| of hosts these overall stats represent. |
| @param t_start: beginning of time period we are interested in. |
| @param t_end: end of time period we are interested in. |
| @param aggregated_stats: A dictionary where keys are string, e.g. 'status' |
| value is total time spent in that status among all hosts. |
| @returns: string representing the aggregate stats report. |
| """ |
| result = 'Overall stats for hosts: %s \n' % (', '.join(labels)) |
| result += ' %s - %s \n' % (time_utils.epoch_time_to_date_string(t_start), |
| time_utils.epoch_time_to_date_string(t_end)) |
| result += ' Number of total hosts: %s \n' % (num_hosts) |
| # This is multiplied by time_spent to get percentage_spent |
| multiplication_factor = 100.0 / ((t_end - t_start) * num_hosts) |
| for status, time_spent in aggregated_stats.iteritems(): |
| # Normalize by the total time we are interested in among ALL hosts. |
| spaces = ' ' * (15 - len(status)) |
| percent_spent = multiplication_factor * time_spent |
| result += ' %s: %s %.2f %%\n' % (status, spaces, percent_spent) |
| result += '- -- --- ---- ----- ---- --- -- -\n' |
| return result |
| |
| |
| def get_overall_report(label, t_start, t_end, intervals_of_statuses_list): |
| """Returns string reporting overall host history for a group of hosts. |
| |
| @param label: A string that can be useful for showing what type group |
| of hosts these overall stats represent. |
| @param t_start: beginning of time period we are interested in. |
| @param t_end: end of time period we are interested in. |
| @param intervals_of_statuses_list: A list of dictionaries where keys |
| are tuple (ti, tf), and value is the status along with other metadata, |
| e.g., task_id, task_name, job_id etc. |
| """ |
| stats_all, num_hosts = aggregate_hosts( |
| intervals_of_statuses_list) |
| return get_stats_string_aggregate( |
| label, t_start, t_end, stats_all, num_hosts) |
| |
| |
| def get_report_for_host(t_start, t_end, hostname, size, |
| print_each_interval, index): |
| """Gets stats report for a host |
| |
| @param t_start: beginning of time period we are interested in. |
| @param t_end: end of time period we are interested in. |
| @param hostname: hostname for the host we are interested in (string) |
| @param print_each_interval: True or False, whether we want to |
| display all intervals |
| @param index: index in elasticsearch to query data for. |
| @returns: stats report for this particular host (string) |
| """ |
| intervals_of_statuses, num_entries_found = host_history_intervals( |
| t_start, t_end, hostname, size, index) |
| total_times = calculate_total_times(intervals_of_statuses) |
| return (get_stats_string( |
| t_start, t_end, total_times, intervals_of_statuses, |
| hostname, num_entries_found, print_each_interval), |
| intervals_of_statuses) |
| |
| |
| def get_stats_string(t_start, t_end, total_times, intervals_of_statuses, |
| hostname, num_entries_found, print_each_interval): |
| """Returns string reporting host_history for this host. |
| @param t_start: beginning of time period we are interested in. |
| @param t_end: end of time period we are interested in. |
| @param total_times: dictionary where key=status, |
| value=(time spent in that status) |
| @param intervals_of_statuses: dictionary where keys is tuple (ti, tf), |
| and value is the status along with other metadata. |
| @param hostname: hostname for the host we are interested in (string) |
| @param num_entries_found: Number of entries found for the host in es |
| @param print_each_interval: boolean, whether to print each interval |
| """ |
| delta = t_end - t_start |
| result = 'usage stats for host: %s \n' % (hostname) |
| result += ' %s - %s \n' % (time_utils.epoch_time_to_date_string(t_start), |
| time_utils.epoch_time_to_date_string(t_end)) |
| result += ' Num entries found in this interval: %s\n' % (num_entries_found) |
| for status, value in total_times.iteritems(): |
| spaces = (15 - len(status)) * ' ' |
| result += ' %s: %s %.2f %%\n' % (status, spaces, 100*value/delta) |
| result += '- -- --- ---- ----- ---- --- -- -\n' |
| if print_each_interval: |
| for interval, status_info in intervals_of_statuses.iteritems(): |
| t0, t1 = interval |
| t0_string = time_utils.epoch_time_to_date_string(t0) |
| t1_string = time_utils.epoch_time_to_date_string(t1) |
| status = status_info['status'] |
| spaces = (15 - len(status)) * ' ' |
| delta = int(t1-t0) |
| result += ' %s : %s %s %s %ss\n' % (t0_string, t1_string, |
| status_info['status'], |
| spaces, |
| delta, |
| ) |
| return result |
| |
| |
| def calculate_status_times(t_start, t_end, int_status, metadata, |
| locked_intervals): |
| """Returns a list of intervals along w/ statuses associated with them. |
| |
| @param t_start: start time |
| @param t_end: end time |
| @param int_status: status of [t_start, t_end] if not locked |
| @param metadata: metadata of the status change, e.g., task_id, task_name. |
| @param locked_intervals: list of tuples denoting intervals of locked states |
| @returns: dictionary where key = (t_interval_start, t_interval_end), |
| val = (status, metadata) |
| t_interval_start: beginning of interval for that status |
| t_interval_end: end of the interval for that status |
| status: string such as 'Repair Failed', 'Locked', etc. |
| metadata: A dictionary of metadata, e.g., |
| {'task_id':123, 'task_name':'Reset'} |
| """ |
| statuses = collections.OrderedDict() |
| |
| prev_interval_end = t_start |
| |
| # TODO: Put allow more information here in info/locked status |
| status_info = {'status': int_status, |
| 'metadata': metadata} |
| locked_info = {'status': 'Locked', |
| 'metadata': {}} |
| if not locked_intervals: |
| statuses[(t_start, t_end)] = status_info |
| return statuses |
| for lock_start, lock_end in locked_intervals: |
| if lock_start > t_end: |
| # optimization to break early |
| # case 0 |
| # Timeline of status change: t_start t_end |
| # Timeline of lock action: lock_start lock_end |
| break |
| elif lock_end < t_start: |
| # case 1 |
| # t_start t_end |
| # lock_start lock_end |
| continue |
| elif lock_end < t_end and lock_start > t_start: |
| # case 2 |
| # t_start t_end |
| # lock_start lock_end |
| statuses[(prev_interval_end, lock_start)] = status_info |
| statuses[(lock_start, lock_end)] = locked_info |
| elif lock_end > t_start and lock_start < t_start: |
| # case 3 |
| # t_start t_end |
| # lock_start lock_end |
| statuses[(t_start, lock_end)] = locked_info |
| elif lock_start < t_end and lock_end > t_end: |
| # case 4 |
| # t_start t_end |
| # lock_start lock_end |
| statuses[(prev_interval_end, lock_start)] = status_info |
| statuses[(lock_start, t_end)] = locked_info |
| prev_interval_end = lock_end |
| # Otherwise we are in the case where lock_end < t_start OR |
| # lock_start > t_end, which means the lock doesn't apply. |
| if t_end > prev_interval_end: |
| # This is to avoid logging the same time |
| statuses[(prev_interval_end, t_end)] = status_info |
| return statuses |
| |
| |
| def get_log_url(hostname, metadata): |
| """Compile a url to job's debug log from debug string. |
| |
| @param hostname: Hostname of the dut. |
| @param metadata: A dictionary of other metadata, e.g., |
| {'task_id':123, 'task_name':'Reset'} |
| @return: Url of the debug log for special task or job url for test job. |
| """ |
| log_url = None |
| if 'task_id' in metadata and 'task_name' in metadata: |
| log_url = job_history.TASK_URL % {'hostname': hostname, |
| 'task_id': metadata['task_id'], |
| 'task_name': metadata['task_name']} |
| elif 'job_id' in metadata and 'owner' in metadata: |
| log_url = job_history.JOB_URL % {'hostname': hostname, |
| 'job_id': metadata['job_id'], |
| 'owner': metadata['owner']} |
| |
| return log_url |
| |
| |
| def build_history(hostname, status_intervals): |
| """Get host history information from given state intervals. |
| |
| @param hostname: Hostname of the dut. |
| @param status_intervals: A ordered dictionary with |
| key as (t_start, t_end) and value as (status, metadata) |
| status = status of the host. e.g. 'Repair Failed' |
| t_start is the beginning of the interval where the DUT's has |
| that status |
| t_end is the end of the interval where the DUT has that |
| status |
| metadata: A dictionary of other metadata, e.g., |
| {'task_id':123, 'task_name':'Reset'} |
| @return: A list of host history, e.g., |
| [{'status': 'Resetting' |
| 'start_time': '2014-08-07 10:02:16', |
| 'end_time': '2014-08-07 10:03:16', |
| 'log_url': 'http://autotest/reset-546546/debug', |
| 'task_id': 546546}, |
| {'status': 'Running' |
| 'start_time': '2014-08-07 10:03:18', |
| 'end_time': '2014-08-07 10:13:00', |
| 'log_url': 'http://autotest/afe/#tab_id=view_job&object_id=1683', |
| 'job_id': 1683} |
| ] |
| """ |
| history = [] |
| for time_interval, status_info in status_intervals.items(): |
| start_time = time_utils.epoch_time_to_date_string(time_interval[0]) |
| end_time = time_utils.epoch_time_to_date_string(time_interval[1]) |
| interval = {'status': status_info['status'], |
| 'start_time': start_time, |
| 'end_time': end_time} |
| interval['log_url'] = get_log_url(hostname, status_info['metadata']) |
| interval.update(status_info['metadata']) |
| history.append(interval) |
| return history |
| |
| |
| def get_status_intervals(history_details): |
| """Get a list of status interval from history details. |
| |
| This is a reverse method of above build_history. Caller gets the history |
| details from RPC get_host_history, and use this method to get the list of |
| status interval, which can be used to calculate stats from |
| host_history_utils.aggregate_hosts. |
| |
| @param history_details: A dictionary of host history for each host, e.g., |
| {'172.22.33.51': [{'status': 'Resetting' |
| 'start_time': '2014-08-07 10:02:16', |
| 'end_time': '2014-08-07 10:03:16', |
| 'log_url': 'http://autotest/reset-546546/debug', |
| 'task_id': 546546},] |
| } |
| @return: A list of dictionaries where keys are tuple (start_time, end_time), |
| and value is a dictionary containing at least key 'status'. |
| """ |
| status_intervals = [] |
| for host,history in history_details.iteritems(): |
| intervals = collections.OrderedDict() |
| for interval in history: |
| start_time = time_utils.to_epoch_time(interval['start_time']) |
| end_time = time_utils.to_epoch_time(interval['end_time']) |
| metadata = copy.deepcopy(interval) |
| metadata['hostname'] = host |
| intervals[(start_time, end_time)] = {'status': interval['status'], |
| 'metadata': metadata} |
| status_intervals.append(intervals) |
| return status_intervals |
| |
| |
| def get_machine_utilization_rate(stats): |
| """Get machine utilization rate from given stats. |
| |
| @param stats: A dictionary with a status as key and value is the total |
| number of seconds spent on the status. |
| @return: The percentage of time when dut is running test jobs. |
| """ |
| not_utilized_status = ['Repairing', 'Repair Failed', 'Ready', 'Verifying'] |
| excluded_status = ['Locked'] |
| total_time = 0 |
| total_time_not_utilized = 0.0 |
| for status, interval in stats.iteritems(): |
| if status in excluded_status: |
| continue |
| total_time += interval |
| if status in not_utilized_status: |
| total_time_not_utilized += interval |
| if total_time == 0: |
| # All duts are locked, assume MUR is 0% |
| return 0 |
| else: |
| return 1 - total_time_not_utilized/total_time |
| |
| |
| def get_machine_availability_rate(stats): |
| """Get machine availability rate from given stats. |
| |
| @param stats: A dictionary with a status as key and value is the total |
| number of seconds spent on the status. |
| @return: The percentage of time when dut is available to run jobs. |
| """ |
| not_available_status = ['Repairing', 'Repair Failed', 'Verifying'] |
| excluded_status = ['Locked'] |
| total_time = 0 |
| total_time_not_available = 0.0 |
| for status, interval in stats.iteritems(): |
| if status in excluded_status: |
| continue |
| total_time += interval |
| if status in not_available_status: |
| total_time_not_available += interval |
| if total_time == 0: |
| # All duts are locked, assume MAR is 0% |
| return 0 |
| else: |
| return 1 - total_time_not_available/total_time |
