| #!/usr/bin/env python2 |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Script to compare the performance of two different chromeOS builds. |
| |
| This script is meant to be used when the performance impact of a change in |
| chromeOS needs to be analyzed. It requires that you have already created two |
| chromeOS test images (one with the change, and one without), and that you have |
| at least one device available on which to run performance tests. |
| |
| This script is actually a light-weight wrapper around crosperf, a tool for |
| automatically imaging one or more chromeOS devices with particular builds, |
| running a set of tests on those builds, and then notifying the user of test |
| results (along with some statistical analysis of perf keyvals). This wrapper |
| script performs the following tasks: |
| |
| 1) Creates a crosperf "experiment" file to be consumed by crosperf. |
| 2) Invokes crosperf using the created experiment file. Crosperf produces 2 |
| outputs: an e-mail that is sent to the user who invoked it; and an output |
| folder that is named based on the given --experiment-name, which is created in |
| the directory in which this script was run. |
| 3) Parses the results of crosperf and outputs a summary of relevant data. This |
| script produces output in a CSV file, as well as in stdout. |
| |
| Before running this script for the first time, you should set up your system to |
| run sudo without prompting for a password (otherwise, crosperf prompts for a |
| sudo password). You should only have to do that once per host machine. |
| |
| Once you're set up with passwordless sudo, you can run the script (preferably |
| from an empty directory, since several output files are produced): |
| |
| > python perf_compare.py --crosperf=CROSPERF_EXE --image-1=IMAGE_1 \ |
| --image-2=IMAGE_2 --board-1=BOARD_1 --board-2=BOARD_2 --remote-1=REMOTE_1 \ |
| --remote-2=REMOTE_2 |
| |
| You'll need to specify the following inputs: the full path to the crosperf |
| executable; the absolute paths to 2 locally-built chromeOS images (which must |
| reside in the "typical location" relative to the chroot, as required by |
| crosperf); the name of the boards associated with the 2 images (if both images |
| have the same board, you can specify that single board with --board=BOARD); and |
| the IP addresses of the 2 remote devices on which to run crosperf (if you have |
| only a single device available, specify it with --remote=REMOTE). Run with -h to |
| see the full set of accepted command-line arguments. |
| |
| Notes: |
| |
| 1) When you run this script, it will delete any previously-created crosperf |
| output directories and created CSV files based on the specified |
| --experiment-name. If you don't want to lose any old crosperf/CSV data, either |
| move it to another location, or run this script with a different |
| --experiment-name. |
| 2) This script will only run the benchmarks and process the perf keys specified |
| in the file "perf_benchmarks.json". Some benchmarks output more perf keys than |
| what are specified in perf_benchmarks.json, and these will appear in the |
| crosperf outputs, but not in the outputs produced specifically by this script. |
| """ |
| |
| |
| import json |
| import logging |
| import math |
| import optparse |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| |
| |
| _ITERATIONS = 5 |
| _IMAGE_1_NAME = 'Image1' |
| _IMAGE_2_NAME = 'Image2' |
| _DEFAULT_EXPERIMENT_NAME = 'perf_comparison' |
| _ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| _BENCHMARK_INFO_FILE_NAME = os.path.join(_ROOT_DIR, 'perf_benchmarks.json') |
| _CROSPERF_REPORT_LINE_DELIMITER = '\t' |
| _EXPERIMENT_FILE_NAME = 'experiment.txt' |
| |
| _BENCHMARK_INFO_TEMPLATE = """ |
| benchmark: {benchmark} {{ |
| autotest_name: {autotest_name} |
| autotest_args: --use_emerged {autotest_args} |
| iterations: {iterations} |
| }} |
| """ |
| |
| _IMAGE_INFO_TEMPLATE = """ |
| label: {label} {{ |
| chromeos_image: {image} |
| board: {board} |
| remote: {remote} |
| }} |
| """ |
| |
| |
| def prompt_for_input(prompt_message): |
| """Prompts for user input and returns the inputted text as a string.""" |
| return raw_input('%s:> ' % prompt_message) |
| |
| |
| def identify_benchmarks_to_run(benchmark_info, iteration_nums, perf_keys): |
| """Identifies which benchmarks to run, and for how many iterations. |
| |
| @param benchmark_info: A list of dictionaries containing information about |
| the complete set of default perf benchmarks to run. |
| @param iteration_nums: See output_benchmarks_info(). |
| @param perf_keys: See output_benchmarks_info(). |
| |
| @return A tuple (X, Y), where X is a list of dictionaries containing |
| information about the set of benchmarks to run, and Y is the set of |
| perf keys requested to be run. |
| """ |
| perf_keys_requested = set() |
| benchmarks_to_run = [] |
| if not perf_keys: |
| # Run every benchmark for the specified number of iterations. |
| benchmarks_to_run = benchmark_info |
| for benchmark in benchmarks_to_run: |
| benchmark['iterations'] = iteration_nums[0] |
| for perf_key in benchmark['perf_keys']: |
| perf_keys_requested.add(perf_key) |
| else: |
| # Identify which benchmarks to run, and for how many iterations. |
| identified_benchmarks = {} |
| for i, perf_key in enumerate(perf_keys): |
| perf_keys_requested.add(perf_key) |
| benchmarks = [benchmark for benchmark in benchmark_info |
| if perf_key in benchmark['perf_keys']] |
| if not benchmarks: |
| logging.error('Perf key "%s" isn\'t associated with a known ' |
| 'benchmark.', perf_key) |
| sys.exit(1) |
| elif len(benchmarks) > 1: |
| logging.error('Perf key "%s" is associated with more than one ' |
| 'benchmark, but should be unique.', perf_key) |
| sys.exit(1) |
| benchmark_to_add = benchmarks[0] |
| benchmark_to_add = identified_benchmarks.setdefault( |
| benchmark_to_add['benchmark'], benchmark_to_add) |
| if len(iteration_nums) == 1: |
| # If only a single iteration number is specified, we assume |
| # that applies to every benchmark. |
| benchmark_to_add['iterations'] = iteration_nums[0] |
| else: |
| # The user must have specified a separate iteration number for |
| # each perf key. If the benchmark associated with the current |
| # perf key already has an interation number associated with it, |
| # choose the maximum of the two. |
| iter_num = iteration_nums[i] |
| if 'iterations' in benchmark_to_add: |
| benchmark_to_add['iterations'] = ( |
| iter_num if iter_num > benchmark_to_add['iterations'] |
| else benchmark_to_add['iterations']) |
| else: |
| benchmark_to_add['iterations'] = iter_num |
| benchmarks_to_run = identified_benchmarks.values() |
| |
| return benchmarks_to_run, perf_keys_requested |
| |
| |
| def output_benchmarks_info(f, iteration_nums, perf_keys): |
| """Identifies details of benchmarks to run, and writes that info to a file. |
| |
| @param f: A file object that is writeable. |
| @param iteration_nums: A list of one or more integers representing the |
| number of iterations to run for one or more benchmarks. |
| @param perf_keys: A list of one or more string perf keys we need to |
| run, or None if we should use the complete set of default perf keys. |
| |
| @return Set of perf keys actually requested to be run in the output file. |
| """ |
| benchmark_info = [] |
| with open(_BENCHMARK_INFO_FILE_NAME, 'r') as f_bench: |
| benchmark_info = json.load(f_bench) |
| |
| benchmarks_to_run, perf_keys_requested = identify_benchmarks_to_run( |
| benchmark_info, iteration_nums, perf_keys) |
| |
| for benchmark in benchmarks_to_run: |
| f.write(_BENCHMARK_INFO_TEMPLATE.format( |
| benchmark=benchmark['benchmark'], |
| autotest_name=benchmark['autotest_name'], |
| autotest_args=benchmark.get('autotest_args', ''), |
| iterations=benchmark['iterations'])) |
| |
| return perf_keys_requested |
| |
| |
| def output_image_info(f, label, image, board, remote): |
| """Writes information about a given image to an output file. |
| |
| @param f: A file object that is writeable. |
| @param label: A string label for the given image. |
| @param image: The string path to the image on disk. |
| @param board: The string board associated with the image. |
| @param remote: The string IP address on which to install the image. |
| """ |
| f.write(_IMAGE_INFO_TEMPLATE.format( |
| label=label, image=image, board=board, remote=remote)) |
| |
| |
| def invoke_crosperf(crosperf_exe, result_dir, experiment_name, board_1, board_2, |
| remote_1, remote_2, iteration_nums, perf_keys, image_1, |
| image_2, image_1_name, image_2_name): |
| """Invokes crosperf with a set of benchmarks and waits for it to complete. |
| |
| @param crosperf_exe: The string path to a crosperf executable. |
| @param result_dir: The string name of the directory in which crosperf is |
| expected to write its output. |
| @param experiment_name: A string name to give the crosperf invocation. |
| @param board_1: The string board associated with the first image. |
| @param board_2: The string board associated with the second image. |
| @param remote_1: The string IP address/name of the first remote device. |
| @param remote_2: The string IP address/name of the second remote device. |
| @param iteration_nums: A list of integers representing the number of |
| iterations to run for the different benchmarks. |
| @param perf_keys: A list of perf keys to run, or None to run the full set |
| of default perf benchmarks. |
| @param image_1: The string path to the first image. |
| @param image_2: The string path to the second image. |
| @param image_1_name: A string label to give the first image. |
| @param image_2_name: A string label to give the second image. |
| |
| @return A tuple (X, Y), where X is the path to the created crosperf report |
| file, and Y is the set of perf keys actually requested to be run. |
| """ |
| # Create experiment file for crosperf. |
| with open(_EXPERIMENT_FILE_NAME, 'w') as f: |
| f.write('name: {name}\n'.format(name=experiment_name)) |
| perf_keys_requested = output_benchmarks_info( |
| f, iteration_nums, perf_keys) |
| output_image_info(f, image_1_name, image_1, board_1, remote_1) |
| output_image_info(f, image_2_name, image_2, board_2, remote_2) |
| |
| # Invoke crosperf with the experiment file. |
| logging.info('Invoking crosperf with created experiment file...') |
| p = subprocess.Popen([crosperf_exe, _EXPERIMENT_FILE_NAME], |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| |
| # Pass through crosperf output as debug messages until crosperf run is |
| # complete. |
| while True: |
| next_line = p.stdout.readline().strip() |
| if not next_line and p.poll() != None: |
| break |
| logging.debug(next_line) |
| sys.stdout.flush() |
| p.communicate() |
| exit_code = p.returncode |
| |
| if exit_code: |
| logging.error('Crosperf returned exit code %s', exit_code) |
| sys.exit(1) |
| |
| report_file = os.path.join(result_dir, 'results.html') |
| if not os.path.exists(report_file): |
| logging.error('Crosperf report file missing, cannot proceed.') |
| sys.exit(1) |
| |
| logging.info('Crosperf run complete.') |
| logging.info('Crosperf results available in "%s"', result_dir) |
| return report_file, perf_keys_requested |
| |
| |
| def parse_crosperf_report_file(report_file, perf_keys_requested): |
| """Reads in and parses a crosperf report file for relevant perf data. |
| |
| @param report_file: See generate_results(). |
| @param perf_keys_requested: See generate_results(). |
| |
| @return A dictionary containing perf information extracted from the crosperf |
| report file. |
| """ |
| results = {} |
| with open(report_file, 'r') as f: |
| contents = f.read() |
| |
| match = re.search(r'summary-tsv.+?/pre', contents, flags=re.DOTALL) |
| contents = match.group(0) |
| |
| curr_benchmark = None |
| for line in contents.splitlines(): |
| delimiter = r'\s+?' |
| match = re.search( |
| r'Benchmark:%s(?P<benchmark>\w+?);%sIterations:%s' |
| '(?P<iterations>\w+?)\s' % (delimiter, delimiter, delimiter), |
| line) |
| if match: |
| curr_benchmark = match.group('benchmark') |
| iterations = match.group('iterations') |
| results[curr_benchmark] = {'iterations': iterations, |
| 'p_values': []} |
| continue |
| split = line.strip().split(_CROSPERF_REPORT_LINE_DELIMITER) |
| if (len(split) == 12 and split[-2] == '--' and |
| split[0] not in ['retval', 'iterations'] and |
| split[0] in perf_keys_requested): |
| results[curr_benchmark]['p_values'].append( |
| (split[0], split[-1])) |
| |
| return results |
| |
| |
| def generate_results(report_file, result_file, perf_keys_requested): |
| """Output relevant crosperf results to a CSV file, and to stdout. |
| |
| This code parses the "results.html" output file of crosperf. It then creates |
| a CSV file that has the following format per line: |
| |
| benchmark_name,num_iterations,perf_key,p_value[,perf_key,p_value] |
| |
| @param report_file: The string name of the report file created by crosperf. |
| @param result_file: A string name for the CSV file to output. |
| @param perf_keys_requested: The set of perf keys originally requested to be |
| run. |
| """ |
| results = parse_crosperf_report_file(report_file, perf_keys_requested) |
| |
| # Output p-value data to a CSV file. |
| with open(result_file, 'w') as f: |
| for bench in results: |
| perf_key_substring = ','.join( |
| ['%s,%s' % (x[0], x[1]) for x in results[bench]['p_values']]) |
| f.write('%s,%s,%s\n' % ( |
| bench, results[bench]['iterations'], perf_key_substring)) |
| |
| logging.info('P-value results available in "%s"', result_file) |
| |
| # Collect and output some additional summary results to stdout. |
| small_p_value = [] |
| nan_p_value = [] |
| perf_keys_obtained = set() |
| for benchmark in results: |
| p_values = results[benchmark]['p_values'] |
| for key, p_val in p_values: |
| perf_keys_obtained.add(key) |
| if float(p_val) <= 0.05: |
| small_p_value.append((benchmark, key, p_val)) |
| elif math.isnan(float(p_val)): |
| nan_p_value.append((benchmark, key, p_val)) |
| |
| if small_p_value: |
| logging.info('The following perf keys showed statistically significant ' |
| 'result differences (p-value <= 0.05):') |
| for item in small_p_value: |
| logging.info('* [%s] %s (p-value %s)', item[0], item[1], item[2]) |
| else: |
| logging.info('No perf keys showed statistically significant result ' |
| 'differences (p-value <= 0.05)') |
| |
| if nan_p_value: |
| logging.info('The following perf keys had "NaN" p-values:') |
| for item in nan_p_value: |
| logging.info('* [%s] %s (p-value %s)', item[0], item[1], item[2]) |
| |
| # Check if any perf keys are missing from what was requested, and notify |
| # the user if so. |
| for key_requested in perf_keys_requested: |
| if key_requested not in perf_keys_obtained: |
| logging.warning('Could not find results for requested perf key ' |
| '"%s".', key_requested) |
| |
| |
| def parse_options(): |
| """Parses command-line arguments.""" |
| parser = optparse.OptionParser() |
| |
| parser.add_option('--crosperf', metavar='PATH', type='string', default=None, |
| help='Absolute path to the crosperf executable ' |
| '(required).') |
| parser.add_option('--image-1', metavar='PATH', type='string', default=None, |
| help='Absolute path to the first image .bin file ' |
| '(required).') |
| parser.add_option('--image-2', metavar='PATH', type='string', default=None, |
| help='Absolute path to the second image .bin file ' |
| '(required).') |
| |
| board_group = optparse.OptionGroup( |
| parser, 'Specifying the boards (required)') |
| board_group.add_option('--board', metavar='BOARD', type='string', |
| default=None, |
| help='Name of the board associated with the images, ' |
| 'if both images have the same board. If each ' |
| 'image has a different board, use ' |
| 'options --board-1 and --board-2 instead.') |
| board_group.add_option('--board-1', metavar='BOARD', type='string', |
| default=None, |
| help='Board associated with the first image.') |
| board_group.add_option('--board-2', metavar='BOARD', type='string', |
| default=None, |
| help='Board associated with the second image.') |
| parser.add_option_group(board_group) |
| |
| remote_group = optparse.OptionGroup( |
| parser, 'Specifying the remote devices (required)') |
| remote_group.add_option('--remote', metavar='IP', type='string', |
| default=None, |
| help='IP address/name of remote device to use, if ' |
| 'only one physical device is to be used. If ' |
| 'using two devices, use options --remote-1 ' |
| 'and --remote-2 instead.') |
| remote_group.add_option('--remote-1', metavar='IP', type='string', |
| default=None, |
| help='IP address/name of first device to use.') |
| remote_group.add_option('--remote-2', metavar='IP', type='string', |
| default=None, |
| help='IP address/name of second device to use.') |
| parser.add_option_group(remote_group) |
| |
| optional_group = optparse.OptionGroup(parser, 'Optional settings') |
| optional_group.add_option('--image-1-name', metavar='NAME', type='string', |
| default=_IMAGE_1_NAME, |
| help='Descriptive name for the first image. ' |
| 'Defaults to "%default".') |
| optional_group.add_option('--image-2-name', metavar='NAME', type='string', |
| default=_IMAGE_2_NAME, |
| help='Descriptive name for the second image. ' |
| 'Defaults to "%default".') |
| optional_group.add_option('--experiment-name', metavar='NAME', |
| type='string', default=_DEFAULT_EXPERIMENT_NAME, |
| help='A descriptive name for the performance ' |
| 'comparison experiment to run. Defaults to ' |
| '"%default".') |
| optional_group.add_option('--perf-keys', metavar='KEY1[,KEY2...]', |
| type='string', default=None, |
| help='Comma-separated list of perf keys to ' |
| 'evaluate, if you do not want to run the ' |
| 'complete set. By default, will evaluate ' |
| 'with the complete set of perf keys.') |
| optional_group.add_option('--iterations', metavar='N1[,N2...]', |
| type='string', default=str(_ITERATIONS), |
| help='Number of iterations to use to evaluate ' |
| 'each perf key (defaults to %default). If ' |
| 'specifying a custom list of perf keys ' |
| '(with --perf-keys) and you want to have a ' |
| 'different number of iterations for each ' |
| 'perf key, specify a comma-separated list ' |
| 'of iteration numbers where N1 corresponds ' |
| 'to KEY1, N2 corresponds to KEY2, etc.') |
| optional_group.add_option('-v', '--verbose', action='store_true', |
| default=False, help='Use verbose logging.') |
| parser.add_option_group(optional_group) |
| |
| options, _ = parser.parse_args() |
| return options |
| |
| |
| def verify_command_line_options(options, iteration_nums, perf_keys): |
| """Verifies there are no errors in the specified command-line options. |
| |
| @param options: An optparse.Options object. |
| @param iteration_nums: An array of numbers representing the number of |
| iterations to perform to evaluate each perf key. |
| @param perf_keys: A list of strings representing perf keys to evaluate, or |
| None if no particular perf keys are specified. |
| |
| @return True, if there were no errors in the command-line options, or |
| False if any error was detected. |
| """ |
| success = True |
| if not options.crosperf: |
| logging.error('You must specify the path to a crosperf executable.') |
| success = False |
| if options.crosperf and not os.path.isfile(options.crosperf): |
| logging.error('Could not locate crosperf executable "%s".', |
| options.crosperf) |
| if options.crosperf.startswith('/google'): |
| logging.error('Did you remember to run prodaccess?') |
| success = False |
| if not options.image_1 or not options.image_2: |
| logging.error('You must specify the paths for 2 image .bin files.') |
| success = False |
| if not options.board and (not options.board_1 or not options.board_2): |
| logging.error('You must specify the board name(s): either a single ' |
| 'board with --board, or else two board names with ' |
| '--board-1 and --board-2.') |
| success = False |
| if options.board and options.board_1 and options.board_2: |
| logging.error('Specify either one board with --board, or two boards ' |
| 'with --board-1 and --board-2, but not both.') |
| success = False |
| if not options.remote and (not options.remote_1 or not options.remote_2): |
| logging.error('You must specify the remote device(s) to use: either a ' |
| 'single device with --remote, or else two devices with ' |
| '--remote-1 and --remote-2.') |
| success = False |
| if options.remote and options.remote_1 and options.remote_2: |
| logging.error('Specify either one remote device with --remote, or two ' |
| 'devices with --remote-1 and --remote-2, but not both.') |
| success = False |
| if len(iteration_nums) > 1 and not perf_keys: |
| logging.error('You should only specify multiple iteration numbers ' |
| 'if you\'re specifying a custom list of perf keys to ' |
| 'evaluate.') |
| success = False |
| if (options.perf_keys and len(iteration_nums) > 1 and |
| len(options.perf_keys.split(',')) > len(iteration_nums)): |
| logging.error('You specified %d custom perf keys, but only %d ' |
| 'iteration numbers.', len(options.perf_keys.split(',')), |
| len(iteration_nums)) |
| success = False |
| return success |
| |
| |
| def main(): |
| options = parse_options() |
| |
| log_level = logging.DEBUG if options.verbose else logging.INFO |
| logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', |
| level=log_level) |
| |
| iteration_nums = [int(i) for i in options.iterations.split(',')] |
| perf_keys = options.perf_keys.split(',') if options.perf_keys else None |
| |
| # Verify there are no errors in the specified command-line options. |
| if not verify_command_line_options(options, iteration_nums, perf_keys): |
| return 1 |
| |
| # Clean up any old results that will be overwritten. |
| result_dir = options.experiment_name + '_results' |
| if os.path.isdir(result_dir): |
| shutil.rmtree(result_dir) |
| result_file = options.experiment_name + '_results.csv' |
| if os.path.isfile(result_file): |
| os.remove(result_file) |
| |
| if options.remote: |
| remote_1, remote_2 = options.remote, options.remote |
| else: |
| remote_1, remote_2 = options.remote_1, options.remote_2 |
| |
| if options.board: |
| board_1, board_2 = options.board, options.board |
| else: |
| board_1, board_2 = options.board_1, options.board_2 |
| |
| report_file, perf_keys_requested = invoke_crosperf( |
| options.crosperf, result_dir, options.experiment_name, board_1, board_2, |
| remote_1, remote_2, iteration_nums, perf_keys, options.image_1, |
| options.image_2, options.image_1_name, options.image_2_name) |
| generate_results(report_file, result_file, perf_keys_requested) |
| |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
