| # -*- coding: utf-8 -*- |
| # Copyright 2013 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """A module to generate experiments.""" |
| |
| |
| import os |
| import re |
| import socket |
| import sys |
| |
| from benchmark import Benchmark |
| from cros_utils import command_executer |
| from cros_utils import logger |
| from experiment import Experiment |
| import file_lock_machine |
| from label import Label |
| from label import MockLabel |
| from results_cache import CacheConditions |
| import test_flag |
| |
| import config |
| |
| |
| # Users may want to run Telemetry tests either individually, or in |
| # specified sets. Here we define sets of tests that users may want |
| # to run together. |
| |
| telemetry_perfv2_tests = [ |
| "kraken", |
| "octane", |
| ] |
| |
| telemetry_pagecycler_tests = [ |
| "page_cycler_v2.intl_ar_fa_he", |
| "page_cycler_v2.intl_es_fr_pt-BR", |
| "page_cycler_v2.intl_hi_ru", |
| "page_cycler_v2.intl_ja_zh", |
| "page_cycler_v2.intl_ko_th_vi", |
| "page_cycler_v2.typical_25", |
| ] |
| |
| telemetry_toolchain_old_perf_tests = [ |
| "page_cycler_v2.intl_es_fr_pt-BR", |
| "page_cycler_v2.intl_hi_ru", |
| "page_cycler_v2.intl_ja_zh", |
| "page_cycler_v2.intl_ko_th_vi", |
| "page_cycler_v2.netsim.top_10", |
| "page_cycler_v2.typical_25", |
| "spaceport", |
| "tab_switching.top_10", |
| ] |
| telemetry_toolchain_perf_tests = [ |
| "octane", |
| "kraken", |
| "speedometer", |
| "speedometer2", |
| "jetstream2", |
| ] |
| graphics_perf_tests = [ |
| "graphics_GLBench", |
| "graphics_GLMark2", |
| "graphics_SanAngeles", |
| "graphics_WebGLAquarium", |
| "graphics_WebGLPerformance", |
| ] |
| # TODO: disable rendering.desktop by default as the benchmark is |
| # currently in a bad state |
| # page_cycler_v2.typical_25 is deprecated and the recommend replacement is |
| # loading.desktop@@typical (crbug.com/916340) |
| telemetry_crosbolt_perf_tests = [ |
| "octane", |
| "kraken", |
| "speedometer2", |
| "jetstream", |
| "loading.desktop", |
| # 'rendering.desktop', |
| ] |
| |
| crosbolt_perf_tests = [ |
| "graphics_WebGLAquarium", |
| "tast.video.PlaybackPerfVP91080P30FPS", |
| ] |
| |
| # 'cheets_AntutuTest', |
| # 'cheets_PerfBootServer', |
| # 'cheets_CandyCrushTest', |
| # 'cheets_LinpackTest', |
| # ] |
| |
| dso_list = [ |
| "all", |
| "chrome", |
| "kallsyms", |
| ] |
| |
| |
| class ExperimentFactory(object): |
| """Factory class for building an Experiment, given an ExperimentFile as input. |
| |
| This factory is currently hardcoded to produce an experiment for running |
| ChromeOS benchmarks, but the idea is that in the future, other types |
| of experiments could be produced. |
| """ |
| |
| def AppendBenchmarkSet( |
| self, |
| benchmarks, |
| benchmark_list, |
| test_args, |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| suite, |
| show_all_results, |
| retries, |
| run_local, |
| cwp_dso, |
| weight, |
| ): |
| """Add all the tests in a set to the benchmarks list.""" |
| for test_name in benchmark_list: |
| telemetry_benchmark = Benchmark( |
| test_name, |
| test_name, |
| test_args, |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| suite, |
| show_all_results, |
| retries, |
| run_local, |
| cwp_dso, |
| weight, |
| ) |
| benchmarks.append(telemetry_benchmark) |
| |
| def GetExperiment(self, experiment_file, working_directory, log_dir): |
| """Construct an experiment from an experiment file.""" |
| global_settings = experiment_file.GetGlobalSettings() |
| experiment_name = global_settings.GetField("name") |
| board = global_settings.GetField("board") |
| chromeos_root = global_settings.GetField("chromeos_root") |
| log_level = global_settings.GetField("logging_level") |
| if log_level not in ("quiet", "average", "verbose"): |
| log_level = "verbose" |
| |
| crosfleet = global_settings.GetField("crosfleet") |
| no_lock = bool(global_settings.GetField("no_lock")) |
| # Check whether crosfleet tool is installed correctly for crosfleet mode. |
| if crosfleet and not self.CheckCrosfleetTool(chromeos_root, log_level): |
| sys.exit(0) |
| |
| remote = global_settings.GetField("remote") |
| # This is used to remove the ",' from the remote if user |
| # add them to the remote string. |
| new_remote = [] |
| if remote: |
| for i in remote: |
| c = re.sub("[\"']", "", i) |
| new_remote.append(c) |
| remote = new_remote |
| rm_chroot_tmp = global_settings.GetField("rm_chroot_tmp") |
| perf_args = global_settings.GetField("perf_args") |
| download_debug = global_settings.GetField("download_debug") |
| # Do not download debug symbols when perf_args is not specified. |
| if not perf_args and download_debug: |
| download_debug = False |
| acquire_timeout = global_settings.GetField("acquire_timeout") |
| cache_dir = global_settings.GetField("cache_dir") |
| cache_only = global_settings.GetField("cache_only") |
| config.AddConfig("no_email", global_settings.GetField("no_email")) |
| share_cache = global_settings.GetField("share_cache") |
| results_dir = global_settings.GetField("results_dir") |
| compress_results = global_settings.GetField("compress_results") |
| # Warn user that option use_file_locks is deprecated. |
| use_file_locks = global_settings.GetField("use_file_locks") |
| if use_file_locks: |
| l = logger.GetLogger() |
| l.LogWarning( |
| "Option use_file_locks is deprecated, please remove it " |
| "from your experiment settings." |
| ) |
| locks_dir = global_settings.GetField("locks_dir") |
| # If not specified, set the locks dir to the default locks dir in |
| # file_lock_machine. |
| if not locks_dir: |
| locks_dir = file_lock_machine.Machine.LOCKS_DIR |
| if not os.path.exists(locks_dir): |
| raise RuntimeError( |
| "Cannot access default lock directory. " |
| "Please run prodaccess or specify a local directory" |
| ) |
| chrome_src = global_settings.GetField("chrome_src") |
| show_all_results = global_settings.GetField("show_all_results") |
| cwp_dso = global_settings.GetField("cwp_dso") |
| if cwp_dso and not cwp_dso in dso_list: |
| raise RuntimeError("The DSO specified is not supported") |
| ignore_min_max = global_settings.GetField("ignore_min_max") |
| dut_config = { |
| "enable_aslr": global_settings.GetField("enable_aslr"), |
| "intel_pstate": global_settings.GetField("intel_pstate"), |
| "cooldown_time": global_settings.GetField("cooldown_time"), |
| "cooldown_temp": global_settings.GetField("cooldown_temp"), |
| "governor": global_settings.GetField("governor"), |
| "cpu_usage": global_settings.GetField("cpu_usage"), |
| "cpu_freq_pct": global_settings.GetField("cpu_freq_pct"), |
| "turbostat": global_settings.GetField("turbostat"), |
| "top_interval": global_settings.GetField("top_interval"), |
| } |
| |
| # Default cache hit conditions. The image checksum in the cache and the |
| # computed checksum of the image must match. Also a cache file must exist. |
| cache_conditions = [ |
| CacheConditions.CACHE_FILE_EXISTS, |
| CacheConditions.CHECKSUMS_MATCH, |
| ] |
| if global_settings.GetField("rerun_if_failed"): |
| cache_conditions.append(CacheConditions.RUN_SUCCEEDED) |
| if global_settings.GetField("rerun"): |
| cache_conditions.append(CacheConditions.FALSE) |
| if global_settings.GetField("same_machine"): |
| cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH) |
| if global_settings.GetField("same_specs"): |
| cache_conditions.append(CacheConditions.MACHINES_MATCH) |
| |
| # Construct benchmarks. |
| # Some fields are common with global settings. The values are |
| # inherited and/or merged with the global settings values. |
| benchmarks = [] |
| all_benchmark_settings = experiment_file.GetSettings("benchmark") |
| |
| # Check if there is duplicated benchmark name |
| benchmark_names = {} |
| # Check if in cwp_dso mode, all benchmarks should have same iterations |
| cwp_dso_iterations = 0 |
| |
| for benchmark_settings in all_benchmark_settings: |
| benchmark_name = benchmark_settings.name |
| test_name = benchmark_settings.GetField("test_name") |
| if not test_name: |
| test_name = benchmark_name |
| test_args = benchmark_settings.GetField("test_args") |
| |
| # Rename benchmark name if 'story-filter' or 'story-tag-filter' specified |
| # in test_args. Make sure these two tags only appear once. |
| story_count = 0 |
| for arg in test_args.split(): |
| if "--story-filter=" in arg or "--story-tag-filter=" in arg: |
| story_count += 1 |
| if story_count > 1: |
| raise RuntimeError( |
| "Only one story or story-tag filter allowed in " |
| "a single benchmark run" |
| ) |
| # Rename benchmark name with an extension of 'story'-option |
| benchmark_name = "%s@@%s" % ( |
| benchmark_name, |
| arg.split("=")[-1], |
| ) |
| |
| # Check for duplicated benchmark name after renaming |
| if not benchmark_name in benchmark_names: |
| benchmark_names[benchmark_name] = True |
| else: |
| raise SyntaxError( |
| "Duplicate benchmark name: '%s'." % benchmark_name |
| ) |
| |
| iterations = benchmark_settings.GetField("iterations") |
| if cwp_dso: |
| if cwp_dso_iterations not in (0, iterations): |
| raise RuntimeError( |
| "Iterations of each benchmark run are not the " "same" |
| ) |
| cwp_dso_iterations = iterations |
| |
| suite = benchmark_settings.GetField("suite") |
| retries = benchmark_settings.GetField("retries") |
| run_local = benchmark_settings.GetField("run_local") |
| weight = benchmark_settings.GetField("weight") |
| if weight: |
| if not cwp_dso: |
| raise RuntimeError( |
| "Weight can only be set when DSO specified" |
| ) |
| if suite != "telemetry_Crosperf": |
| raise RuntimeError( |
| "CWP approximation weight only works with " |
| "telemetry_Crosperf suite" |
| ) |
| if run_local: |
| raise RuntimeError( |
| "run_local must be set to False to use CWP " |
| "approximation" |
| ) |
| if weight < 0: |
| raise RuntimeError("Weight should be a float >=0") |
| elif cwp_dso: |
| raise RuntimeError( |
| "With DSO specified, each benchmark should have a " "weight" |
| ) |
| |
| if suite == "telemetry_Crosperf": |
| if test_name == "all_perfv2": |
| self.AppendBenchmarkSet( |
| benchmarks, |
| telemetry_perfv2_tests, |
| test_args, |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| suite, |
| show_all_results, |
| retries, |
| run_local, |
| cwp_dso, |
| weight, |
| ) |
| elif test_name == "all_pagecyclers": |
| self.AppendBenchmarkSet( |
| benchmarks, |
| telemetry_pagecycler_tests, |
| test_args, |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| suite, |
| show_all_results, |
| retries, |
| run_local, |
| cwp_dso, |
| weight, |
| ) |
| elif test_name == "all_crosbolt_perf": |
| self.AppendBenchmarkSet( |
| benchmarks, |
| telemetry_crosbolt_perf_tests, |
| test_args, |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| "telemetry_Crosperf", |
| show_all_results, |
| retries, |
| run_local, |
| cwp_dso, |
| weight, |
| ) |
| self.AppendBenchmarkSet( |
| benchmarks, |
| crosbolt_perf_tests, |
| "", |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| "", |
| show_all_results, |
| retries, |
| run_local=False, |
| cwp_dso=cwp_dso, |
| weight=weight, |
| ) |
| elif test_name == "all_toolchain_perf": |
| self.AppendBenchmarkSet( |
| benchmarks, |
| telemetry_toolchain_perf_tests, |
| test_args, |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| suite, |
| show_all_results, |
| retries, |
| run_local, |
| cwp_dso, |
| weight, |
| ) |
| # Add non-telemetry toolchain-perf benchmarks: |
| |
| # Tast test platform.ReportDiskUsage for image size. |
| benchmarks.append( |
| Benchmark( |
| "platform.ReportDiskUsage", |
| "platform.ReportDiskUsage", |
| "", |
| 1, # This is not a performance benchmark, only run once. |
| rm_chroot_tmp, |
| "", |
| "tast", # Specify the suite to be 'tast' |
| show_all_results, |
| retries, |
| ) |
| ) |
| |
| # TODO: crbug.com/1057755 Do not enable graphics_WebGLAquarium until |
| # it gets fixed. |
| # |
| # benchmarks.append( |
| # Benchmark( |
| # 'graphics_WebGLAquarium', |
| # 'graphics_WebGLAquarium', |
| # '', |
| # iterations, |
| # rm_chroot_tmp, |
| # perf_args, |
| # 'crosperf_Wrapper', # Use client wrapper in Autotest |
| # show_all_results, |
| # retries, |
| # run_local=False, |
| # cwp_dso=cwp_dso, |
| # weight=weight)) |
| elif test_name == "all_toolchain_perf_old": |
| self.AppendBenchmarkSet( |
| benchmarks, |
| telemetry_toolchain_old_perf_tests, |
| test_args, |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| suite, |
| show_all_results, |
| retries, |
| run_local, |
| cwp_dso, |
| weight, |
| ) |
| else: |
| benchmark = Benchmark( |
| benchmark_name, |
| test_name, |
| test_args, |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| suite, |
| show_all_results, |
| retries, |
| run_local, |
| cwp_dso, |
| weight, |
| ) |
| benchmarks.append(benchmark) |
| else: |
| if test_name == "all_graphics_perf": |
| self.AppendBenchmarkSet( |
| benchmarks, |
| graphics_perf_tests, |
| "", |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| "", |
| show_all_results, |
| retries, |
| run_local=False, |
| cwp_dso=cwp_dso, |
| weight=weight, |
| ) |
| else: |
| # Add the single benchmark. |
| benchmark = Benchmark( |
| benchmark_name, |
| test_name, |
| test_args, |
| iterations, |
| rm_chroot_tmp, |
| perf_args, |
| suite, |
| show_all_results, |
| retries, |
| run_local=False, |
| cwp_dso=cwp_dso, |
| weight=weight, |
| ) |
| benchmarks.append(benchmark) |
| |
| if not benchmarks: |
| raise RuntimeError("No benchmarks specified") |
| |
| # Construct labels. |
| # Some fields are common with global settings. The values are |
| # inherited and/or merged with the global settings values. |
| labels = [] |
| all_label_settings = experiment_file.GetSettings("label") |
| all_remote = list(remote) |
| for label_settings in all_label_settings: |
| label_name = label_settings.name |
| image = label_settings.GetField("chromeos_image") |
| build = label_settings.GetField("build") |
| autotest_path = label_settings.GetField("autotest_path") |
| debug_path = label_settings.GetField("debug_path") |
| chromeos_root = label_settings.GetField("chromeos_root") |
| my_remote = label_settings.GetField("remote") |
| compiler = label_settings.GetField("compiler") |
| new_remote = [] |
| if my_remote: |
| for i in my_remote: |
| c = re.sub("[\"']", "", i) |
| new_remote.append(c) |
| my_remote = new_remote |
| |
| if image: |
| if crosfleet: |
| raise RuntimeError( |
| "In crosfleet mode, local image should not be used." |
| ) |
| if build: |
| raise RuntimeError( |
| "Image path and build are provided at the same " |
| "time, please use only one of them." |
| ) |
| else: |
| if not build: |
| raise RuntimeError("Can not have empty 'build' field!") |
| image, autotest_path, debug_path = label_settings.GetXbuddyPath( |
| build, |
| autotest_path, |
| debug_path, |
| board, |
| chromeos_root, |
| log_level, |
| download_debug, |
| ) |
| |
| cache_dir = label_settings.GetField("cache_dir") |
| chrome_src = label_settings.GetField("chrome_src") |
| |
| # TODO(yunlian): We should consolidate code in machine_manager.py |
| # to derermine whether we are running from within google or not |
| if ( |
| "corp.google.com" in socket.gethostname() |
| and not my_remote |
| and not crosfleet |
| ): |
| my_remote = self.GetDefaultRemotes(board) |
| if global_settings.GetField("same_machine") and len(my_remote) > 1: |
| raise RuntimeError( |
| "Only one remote is allowed when same_machine " |
| "is turned on" |
| ) |
| all_remote += my_remote |
| image_args = label_settings.GetField("image_args") |
| if test_flag.GetTestMode(): |
| # pylint: disable=too-many-function-args |
| label = MockLabel( |
| label_name, |
| build, |
| image, |
| autotest_path, |
| debug_path, |
| chromeos_root, |
| board, |
| my_remote, |
| image_args, |
| cache_dir, |
| cache_only, |
| log_level, |
| compiler, |
| crosfleet, |
| chrome_src, |
| ) |
| else: |
| label = Label( |
| label_name, |
| build, |
| image, |
| autotest_path, |
| debug_path, |
| chromeos_root, |
| board, |
| my_remote, |
| image_args, |
| cache_dir, |
| cache_only, |
| log_level, |
| compiler, |
| crosfleet, |
| chrome_src, |
| ) |
| labels.append(label) |
| |
| if not labels: |
| raise RuntimeError("No labels specified") |
| |
| email = global_settings.GetField("email") |
| all_remote += list(set(my_remote)) |
| all_remote = list(set(all_remote)) |
| if crosfleet: |
| for remote in all_remote: |
| self.CheckRemotesInCrosfleet(remote) |
| experiment = Experiment( |
| experiment_name, |
| all_remote, |
| working_directory, |
| chromeos_root, |
| cache_conditions, |
| labels, |
| benchmarks, |
| experiment_file.Canonicalize(), |
| email, |
| acquire_timeout, |
| log_dir, |
| log_level, |
| share_cache, |
| results_dir, |
| compress_results, |
| locks_dir, |
| cwp_dso, |
| ignore_min_max, |
| crosfleet, |
| dut_config, |
| no_lock=no_lock, |
| ) |
| |
| return experiment |
| |
| def GetDefaultRemotes(self, board): |
| default_remotes_file = os.path.join( |
| os.path.dirname(__file__), "default_remotes" |
| ) |
| try: |
| with open(default_remotes_file) as f: |
| for line in f: |
| key, v = line.split(":") |
| if key.strip() == board: |
| remotes = v.strip().split() |
| if remotes: |
| return remotes |
| else: |
| raise RuntimeError( |
| "There is no remote for {0}".format(board) |
| ) |
| except IOError: |
| # TODO: rethrow instead of throwing different exception. |
| raise RuntimeError( |
| "IOError while reading file {0}".format(default_remotes_file) |
| ) |
| else: |
| raise RuntimeError("There is no remote for {0}".format(board)) |
| |
| def CheckRemotesInCrosfleet(self, remote): |
| # TODO: (AI:zhizhouy) need to check whether a remote is a local or lab |
| # machine. If not lab machine, raise an error. |
| pass |
| |
| def CheckCrosfleetTool(self, chromeos_root, log_level): |
| CROSFLEET_PATH = "crosfleet" |
| if os.path.exists(CROSFLEET_PATH): |
| return True |
| l = logger.GetLogger() |
| l.LogOutput("Crosfleet tool not installed, trying to install it.") |
| ce = command_executer.GetCommandExecuter(l, log_level=log_level) |
| setup_lab_tools = os.path.join( |
| chromeos_root, "chromeos-admin", "lab-tools", "setup_lab_tools" |
| ) |
| cmd = "%s" % setup_lab_tools |
| status = ce.RunCommand(cmd) |
| if status != 0: |
| raise RuntimeError( |
| "Crosfleet tool not installed correctly, please try to " |
| "manually install it from %s" % setup_lab_tools |
| ) |
| l.LogOutput( |
| "Crosfleet is installed at %s, please login before first use. " |
| 'Login by running "crosfleet login" and follow instructions.' |
| % CROSFLEET_PATH |
| ) |
| return False |