New script for using buildbots for nightly tests.
This script could eventually replace the current test_toolchains.py
script. Instead of downloading and building ChromeOS locally, it
launches a buildbot to build ChromeOS with the latest mobile
toolchain compiler. It then downloads the trybot image (and the
corresponding vanilla image), generates the crosperf experiment file,
and launches crosperf.
I have been running this as a second cron job on chrotomation2,
running daisy tests, and it has been working just fine.
BUG=None
TEST=Ran in cron jobs on chrotomation2.
Change-Id: I3b17d0373e2fae359fc8f4e0188dcec2943c7b71
Reviewed-on: https://chrome-internal-review.googlesource.com/187735
Reviewed-by: Caroline Tice <cmtice@google.com>
Trybot-Ready: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>
Commit-Queue: Caroline Tice <cmtice@google.com>
diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py
new file mode 100644
index 0000000..bc2c056
--- /dev/null
+++ b/buildbot_test_toolchains.py
@@ -0,0 +1,247 @@
+#!/usr/bin/python
+"""
+Script for running nightly compiler tests on ChromeOS.
+
+This script launches a buildbot to build ChromeOS with the latest compiler on
+a particular board; then it finds and downloads the trybot image and the
+corresponding official image, and runs crosperf performance tests comparing
+the two. It then generates a report, emails it to the c-compiler-chrome, as
+well as copying the images into the seven-day reports directory.
+"""
+
+# Script to test different toolchains against ChromeOS benchmarks.
+import optparse
+import os
+import sys
+import time
+import urllib2
+
+from utils import command_executer
+from utils import logger
+
+from utils import buildbot_utils
+
+# CL that updated GCC ebuilds to use 'next_gcc'.
+USE_NEXT_GCC_PATCH ="230260"
+
+WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data"
+ROLE_ACCOUNT = "mobiletc-prebuild"
+TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class ToolchainComparator():
+ """
+ Class for doing the nightly tests work.
+ """
+
+ def __init__(self, board, remotes, chromeos_root, weekday):
+ self._board = board
+ self._remotes = remotes
+ self._chromeos_root = chromeos_root
+ self._base_dir = os.getcwd()
+ self._ce = command_executer.GetCommandExecuter()
+ self._l = logger.GetLogger()
+ self._build = "%s-release" % board
+ if not weekday:
+ self._weekday = time.strftime("%a")
+ else:
+ self._weekday = weekday
+
+ def _ParseVanillaImage(self, trybot_image):
+ """
+ Parse a trybot artifact name to get corresponding vanilla image.
+
+ This function takes an artifact name, such as
+ 'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
+ corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
+ """
+ start_pos = trybot_image.find(self._build)
+ end_pos = trybot_image.rfind("-b")
+ vanilla_image = trybot_image[start_pos:end_pos]
+ return vanilla_image
+
+
+ def _FinishSetup(self):
+ """
+ Make sure testing_rsa file is properly set up.
+ """
+ # Fix protections on ssh key
+ command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
+ "/chrome-src-internal/src/third_party/chromite/ssh_keys"
+ "/testing_rsa")
+ ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
+ if ret_val:
+ raise Exception("chmod for testing_rsa failed")
+
+ def _TestImages(self, trybot_image, vanilla_image):
+ """
+ Create crosperf experiment file.
+
+ Given the names of the trybot and vanilla images, create the
+ appropriate crosperf experiment file and launch crosperf on it.
+ """
+ experiment_file_dir = os.path.join (self._chromeos_root, "..",
+ self._weekday)
+ experiment_file_name = "%s_toolchain_experiment.txt" % self._board
+ experiment_file = os.path.join (experiment_file_dir,
+ experiment_file_name)
+ experiment_header = """
+ board: %s
+ remote: %s
+ """ % (self._board, self._remotes)
+ experiment_tests = """
+ benchmark: all_perfv2 {
+ suite: telemetry_Crosperf
+ iterations: 3
+ }
+ """
+ with open(experiment_file, "w") as f:
+ print >> f, experiment_header
+ print >> f, experiment_tests
+
+ # Now add vanilla to test file.
+ official_image = """
+ vanilla_image {
+ chromeos_root: %s
+ build: %s
+ }
+ """ % (self._chromeos_root, vanilla_image)
+ print >> f, official_image
+
+ experiment_image = """
+ test_image {
+ chromeos_root: %s
+ build: %s
+ }
+ """ % (self._chromeos_root, trybot_image)
+ print >> f, experiment_image
+
+ crosperf = os.path.join(TOOLCHAIN_DIR,
+ "crosperf",
+ "crosperf")
+ command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
+ ret = self._ce.RunCommand(command)
+ if ret:
+ raise Exception("Couldn't run crosperf!")
+ return ret
+
+
+ def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
+ """
+ Put files in place for running seven-day reports.
+
+ Create tar files of the custom and official images and copy them
+ to the weekly reports directory, so they exist when the weekly report
+ gets generated. IMPORTANT NOTE: This function must run *after*
+ crosperf has been run; otherwise the vanilla images will not be there.
+ """
+
+ dry_run = False
+ if (os.getlogin() != ROLE_ACCOUNT):
+ self._l.LogOutput("Running this from non-role account; not copying "
+ "tar files for weekly reports.")
+ dry_run = True
+
+ images_path = os.path.join(os.path.realpath(self._chromeos_root),
+ "chroot/tmp")
+
+ data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
+ dest_dir = os.path.join (data_dir, self._weekday)
+ if not os.path.exists(dest_dir):
+ os.makedirs(dest_dir)
+
+ # Make sure dest_dir is empty (clean out last week's data).
+ cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
+ if dry_run:
+ print "CMD: %s" % cmd
+ else:
+ self._ce.RunCommand(cmd)
+
+ # Now create new tar files and copy them over.
+ labels = [ "test", "vanilla" ]
+ for label_name in labels:
+ if label_name == "test":
+ test_path = trybot_image
+ else:
+ test_path = vanilla_image
+ tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
+ cmd = "cd %s; tar -cvf %s %s/*; cp %s %s/." % (images_path,
+ tar_file_name,
+ test_path,
+ tar_file_name,
+ dest_dir)
+ if dry_run:
+ print "CMD: %s" % cmd
+ tar_ret = 0
+ else:
+ tar_ret = self._ce.RunCommand(cmd)
+ if tar_ret:
+ self._l.LogOutput("Error while creating/copying test tar file(%s)."
+ % tar_file_name)
+
+
+ def DoAll(self):
+ """
+ Main function inside ToolchainComparator class.
+
+ Launch trybot, get image names, create crosperf experiment file, run
+ crosperf, and copy images into seven-day report directories.
+ """
+
+ description = "master_%s_%s" % (USE_NEXT_GCC_PATCH, self._build)
+ trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
+ self._build,
+ [ USE_NEXT_GCC_PATCH ],
+ description)
+
+ vanilla_image = self._ParseVanillaImage(trybot_image)
+
+ print ("trybot_image: %s" % trybot_image)
+ print ("vanilla_image: %s" % vanilla_image)
+ if os.getlogin() == ROLE_ACCOUNT:
+ self._FinishSetup()
+
+ if not self._TestImages(trybot_image, vanilla_image):
+ # Only try to copy the image files if the test runs ran successfully.
+ self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
+ return 0
+
+
+def Main(argv):
+ """The main function."""
+
+ # Common initializations
+ command_executer.InitCommandExecuter()
+ parser = optparse.OptionParser()
+ parser.add_option("--remote",
+ dest="remote",
+ help="Remote machines to run tests on.")
+ parser.add_option("--board",
+ dest="board",
+ default="x86-zgb",
+ help="The target board.")
+ parser.add_option("--chromeos_root",
+ dest="chromeos_root",
+ help="The chromeos root from which to run tests.")
+ parser.add_option("--weekday", default="",
+ dest="weekday",
+ help="The day of the week for which to run tests.")
+ options, _ = parser.parse_args(argv)
+ if not options.board:
+ print "Please give a board."
+ return 1
+ if not options.remote:
+ print "Please give at least one remote machine."
+ return 1
+ if not options.chromeos_root:
+ print "Please specify the ChromeOS root directory."
+ return 1
+
+ fc = ToolchainComparator(options.board, options.remote,
+ options.chromeos_root, options.weekday)
+ return fc.DoAll()
+
+
+if __name__ == "__main__":
+ retval = Main(sys.argv)
+ sys.exit(retval)
diff --git a/utils/buildbot_utils.py b/utils/buildbot_utils.py
new file mode 100644
index 0000000..e41490f
--- /dev/null
+++ b/utils/buildbot_utils.py
@@ -0,0 +1,199 @@
+#!/usr/bin/python
+
+# Copyright 2014 Google Inc. All Rights Reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import time
+import urllib2
+
+from utils import command_executer
+from utils import logger
+from utils import buildbot_json
+
+SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot.
+TIME_OUT = 18000 # Decide the build is dead or will never finish
+ # after this time (5 hours).
+
+"""Utilities for launching and accessing ChromeOS buildbots."""
+
+def ParseReportLog (url, build):
+ """
+ Scrape the trybot image name off the Reports log page.
+
+ This takes the URL for a trybot Reports Stage web page,
+ and a trybot build type, such as 'daisy-release'. It
+ opens the web page and parses it looking for the trybot
+ artifact name (e.g. something like
+ 'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the
+ artifact name, if found.
+ """
+ trybot_image = ""
+ url += "/text"
+ newurl = url.replace ("uberchromegw", "chromegw")
+ webpage = urllib2.urlopen(newurl)
+ data = webpage.read()
+ lines = data.split('\n')
+ for l in lines:
+ if l.find("Artifacts") and l.find("trybot"):
+ trybot_name = "trybot-%s" % build
+ start_pos = l.find(trybot_name)
+ end_pos = l.find("@https://storage")
+ trybot_image = l[start_pos:end_pos]
+
+ return trybot_image
+
+
+def GetBuildData (buildbot_queue, build_id):
+ """
+ Find the Reports stage web page for a trybot build.
+
+ This takes the name of a buildbot_queue, such as 'daisy-release'
+ and a build id (the build number), and uses the json buildbot api to
+ find the Reports stage web page for that build, if it exists.
+ """
+ builder = buildbot_json.Buildbot(
+ "http://chromegw/p/tryserver.chromiumos/").builders[buildbot_queue]
+ build_data = builder.builds[build_id].data
+ logs = build_data["logs"]
+ for l in logs:
+ fname = l[1]
+ if "steps/Report/" in fname:
+ return fname
+
+ return ""
+
+
+def FindBuildRecordFromLog(description, log_info):
+ """
+ Find the right build record in the build logs.
+
+ Get the first build record from build log with a reason field
+ that matches 'description'. ('description' is a special tag we
+ created when we launched the buildbot, so we could find it at this
+ point.)
+ """
+
+ current_line = 1
+ while current_line < len(log_info):
+ my_dict = {}
+ # Read all the lines from one "Build" to the next into my_dict
+ while True:
+ key = log_info[current_line].split(":")[0].strip()
+ value = log_info[current_line].split(":", 1)[1].strip()
+ my_dict[key] = value
+ current_line += 1
+ if "Build" in key or current_line == len(log_info):
+ break
+ try:
+ # Check to see of the build record is the right one.
+ if str(description) in my_dict["reason"]:
+ # We found a match; we're done.
+ return my_dict
+ except:
+ print "reason is not in dictionary: '%s'" % repr(my_dict)
+ else:
+ # Keep going.
+ continue
+
+ # We hit the bottom of the log without a match.
+ return {}
+
+
+def GetBuildInfo(file_dir):
+ """
+ Get all the build records for the trybot builds.
+
+ file_dir is the toolchain_utils directory.
+ """
+ ce = command_executer.GetCommandExecuter()
+ commands = ("{0}/utils/buildbot_json.py builds "
+ "http://chromegw/p/tryserver.chromiumos/"
+ .format(file_dir))
+
+ _, buildinfo, _ = ce.RunCommand(commands, return_output=True,
+ print_to_console=False)
+ build_log = buildinfo.splitlines()
+ return build_log
+
+
+def GetTrybotImage(chromeos_root, buildbot_name, patch_list, build_tag):
+ """
+ Launch buildbot and get resulting trybot artifact name.
+
+ This function launches a buildbot with the appropriate flags to
+ build the test ChromeOS image, with the current ToT mobile compiler. It
+ checks every 10 minutes to see if the trybot has finished. When the trybot
+ has finished, it parses the resulting report logs to find the trybot
+ artifact (if one was created), and returns that artifact name.
+
+ chromeos_root is the path to the ChromeOS root, needed for finding chromite
+ and launching the buildbot.
+
+ buildbot_name is the name of the buildbot queue, such as lumpy-release or
+ daisy-paladin.
+
+ patch_list a python list of the patches, if any, for the buildbot to use.
+
+ build_tag is a (unique) string to be used to look up the buildbot results
+ from among all the build records.
+ """
+ ce = command_executer.GetCommandExecuter()
+ cbuildbot_path = os.path.join(chromeos_root, "chromite/cbuildbot")
+ base_dir = os.getcwd()
+ patch_arg = ""
+ if patch_list:
+ patch_arg = "-g "
+ for p in patch_list:
+ patch_arg = patch_arg + " " + repr(p)
+ branch = "master"
+ os.chdir(cbuildbot_path)
+
+ # Launch buildbot with appropriate flags.
+ build = buildbot_name
+ description = build_tag
+ command = ("./cbuildbot --remote --nochromesdk --notests %s %s"
+ " --remote-description=%s"
+ " --chrome_rev=tot" % (patch_arg, build, description))
+ ce.RunCommand(command)
+ os.chdir(base_dir)
+
+ build_id = 0
+ # Wait for buildbot to finish running (check every 10 minutes)
+ done = False
+ running_time = 0
+ while not done:
+ done = True
+ build_info = GetBuildInfo(base_dir)
+ if not build_info:
+ logger.GetLogger().LogFatal("Unable to get build logs for target %s"
+ % build)
+
+ data_dict = FindBuildRecordFromLog(description, build_info)
+ if not data_dict:
+ logger.GetLogger().LogFatal("Unable to find build record for trybot %s"
+ % description)
+
+ if "True" in data_dict["completed"]:
+ build_id = data_dict["number"]
+ else:
+ done = False
+
+ if not done:
+ logger.GetLogger().LogOutput("{0} minutes passed.".format(
+ running_time / 60))
+ logger.GetLogger().LogOutput("Sleeping {0} seconds.".format(SLEEP_TIME))
+ time.sleep(SLEEP_TIME)
+ running_time += SLEEP_TIME
+ if running_time > TIME_OUT:
+ done = True
+
+ trybot_image = ""
+ # Buildbot has finished. Look for the log and the trybot image.
+ if build_id:
+ log_name = GetBuildData(build, build_id)
+ if log_name:
+ trybot_image = ParseReportLog(log_name, build)
+
+ return trybot_image
