Add 'build' settings option and xbuddy image format.
This patch updates Crosperf to add the 'build' tag in the
experiment file, and to allow that field to contain xbuddy
syntax for using trybot and/or official builds in the test runs.
It also adds a bit more checking to make sure we have everything
necessary for running 'cros flash' before attempting to use it.
BUG=None
TEST=I have run this using an experiment file that compares a local
image, a trybot image and an official image against each other. It
all worked.
Change-Id: Ia896799061508fb5718a3201b1599d8622de0b3f
Reviewed-on: https://chrome-internal-review.googlesource.com/154097
Reviewed-by: Yunlian Jiang <yunlian@google.com>
Commit-Queue: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index f6c8205..86118a3 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -108,6 +108,7 @@
share_users = global_settings.GetField("share_users")
results_dir = global_settings.GetField("results_dir")
chrome_src = global_settings.GetField("chrome_src")
+ build = global_settings.GetField("build")
use_test_that = global_settings.GetField("use_test_that")
show_all_results = global_settings.GetField("show_all_results")
# Default cache hit conditions. The image checksum in the cache and the
@@ -182,9 +183,12 @@
all_remote = list(remote)
for label_settings in all_label_settings:
label_name = label_settings.name
- image = label_settings.GetField("chromeos_image")
- chromeos_root = label_settings.GetField("chromeos_root")
board = label_settings.GetField("board")
+ image = label_settings.GetField("chromeos_image")
+ if image == "":
+ build = label_settings.GetField("build")
+ image = label_settings.GetXbuddyPath (build, board)
+ chromeos_root = label_settings.GetField("chromeos_root")
my_remote = label_settings.GetField("remote")
new_remote = []
for i in my_remote:
diff --git a/crosperf/experiment_file.py b/crosperf/experiment_file.py
index 293e822..0a3c3b9 100644
--- a/crosperf/experiment_file.py
+++ b/crosperf/experiment_file.py
@@ -131,10 +131,13 @@
def Canonicalize(self):
"""Convert parsed experiment file back into an experiment file."""
res = ""
+ board = ""
for field_name in self.global_settings.fields:
field = self.global_settings.fields[field_name]
if field.assigned:
res += "%s: %s\n" % (field.name, field.GetString())
+ if field.name == "board":
+ board = field.GetString()
res += "\n"
for settings in self.all_settings:
@@ -149,6 +152,10 @@
(os.path.expanduser(field.GetString())))
if real_file != field.GetString():
res += "\t#actual_image: %s\n" % real_file
+ if field.name == "build":
+ value = field.GetString()
+ xbuddy_path = settings.GetXbuddyPath (value, board)
+ res += "\t#actual_image: %s\n" % xbuddy_path
res += "}\n\n"
return res
diff --git a/crosperf/experiment_files/official-image.exp b/crosperf/experiment_files/official-image.exp
new file mode 100644
index 0000000..72be02b
--- /dev/null
+++ b/crosperf/experiment_files/official-image.exp
@@ -0,0 +1,31 @@
+# This is an example experiment file for Crosperf, showing how to run
+# a basic test, using a (previously made) trybot image.
+
+name: trybot_example
+# Replace board and remote values below appropriately. e.g. "lumpy" and
+# "123.45.678.901" or "my-machine.blah.com".
+board: <your-board-goes-here>
+remote: <your-remote-ip-address-here>
+
+# You can replace 'canvasmark' below with the name of the Telemetry
+# benchmakr you want to run.
+benchmark: canvasmark {
+ suite:telemetry_Crosperf
+ iterations: 1
+}
+
+
+# Replace <path-to-your-chroot-goes-here> with the actual directory path
+# to the top of your ChromimumOS chroot.
+trybot_image {
+ chromeos_root:<path-to-your-chroot-goes-here>
+ # Replace <xbuddy-official-image-designation> with the xbuddy syntax
+ # for the official image you want to use (see
+ # http://www.chromium.org/chromium-os/how-tos-and-troubleshooting/using-the-dev-server/xbuddy-for-devserver#TOC-XBuddy-Paths
+ # for xbuddy syntax). Omit the "http://xbuddy/remote/<board>/" prefix.
+ # For example, if you want to use the "latest-dev" official image,
+ # your build field would look like:
+ # build:latest-dev
+ build:<xbuddy-official-image-designation>
+}
+
diff --git a/crosperf/experiment_files/trybot-image.exp b/crosperf/experiment_files/trybot-image.exp
new file mode 100644
index 0000000..d80f418
--- /dev/null
+++ b/crosperf/experiment_files/trybot-image.exp
@@ -0,0 +1,32 @@
+# This is an example experiment file for Crosperf, showing how to run
+# a basic test, using a (previously made) trybot image.
+
+name: trybot_example
+# Replace board and remote values below appropriately. e.g. "lumpy" and
+# "123.45.678.901" or "my-machine.blah.com".
+board: <your-board-goes-here>
+remote: <your-remote-ip-address-here>
+
+# You can replace 'canvasmark' below with the name of the Telemetry
+# benchmakr you want to run.
+benchmark: canvasmark {
+ suite:telemetry_Crosperf
+ iterations: 1
+}
+
+
+# Replace <path-to-your-chroot-goes-here> with the actual directory path
+# to the top of your ChromimumOS chroot.
+trybot_image {
+ chromeos_root:<path-to-your-chroot-goes-here>
+ # Replace <trybot-image-name> with the actual name of the trybot image
+ # that you wish to use. You can find this by going to the trybot build
+ # log, going # to the 'Report' stage, and looking for "Build Artifacts'
+ # at the bottom. You will see something like:
+ # 'lumpy: https://storage.cloud.google.com/chromeos-image-archive/trybot-lumpy-paladin/R34-5393.0.0-b1504/index.html'
+ # From that you can extract the trybot image name and put it in the build
+ # field:
+ # build:trybot-lumpy-paladin/R34-5417.0.0-b1506
+ build:<trybot-image-name>
+}
+
diff --git a/crosperf/image_checksummer.py b/crosperf/image_checksummer.py
index dcc1cb0..eeb4466 100644
--- a/crosperf/image_checksummer.py
+++ b/crosperf/image_checksummer.py
@@ -22,6 +22,8 @@
logger.GetLogger().LogOutput("Acquiring checksum for '%s'." %
self.label.name)
self._checksum = None
+ if self.label.image_type != "local":
+ raise Exception("Called Checksum on non-local image!")
if self.label.chromeos_image:
if os.path.exists(self.label.chromeos_image):
self._checksum = FileUtils().Md5File(self.label.chromeos_image)
@@ -49,6 +51,8 @@
return cls._instance
def Checksum(self, label):
+ if label.image_type != "local":
+ raise Exception("Attempt to call Checksum on non-local image.")
with self._lock:
if label.name not in self._per_image_checksummers:
self._per_image_checksummers[label.name] = (ImageChecksummer.
diff --git a/crosperf/label.py b/crosperf/label.py
index 6169865..c212125 100644
--- a/crosperf/label.py
+++ b/crosperf/label.py
@@ -14,9 +14,13 @@
class Label(object):
def __init__(self, name, chromeos_image, chromeos_root, board, remote,
image_args, image_md5sum, cache_dir, chrome_src=None):
+
+ self.image_type = self._GetImageType(chromeos_image)
+
# Expand ~
chromeos_root = os.path.expanduser(chromeos_root)
- chromeos_image = os.path.expanduser(chromeos_image)
+ if self.image_type == "local":
+ chromeos_image = os.path.expanduser(chromeos_image)
self.name = name
self.chromeos_image = chromeos_image
@@ -27,7 +31,8 @@
self.cache_dir = cache_dir
if not chromeos_root:
- chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image)
+ if self.image_type == "local":
+ chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image)
if not chromeos_root:
raise Exception("No ChromeOS root given for label '%s' and could not "
"determine one from image path: '%s'." %
@@ -52,6 +57,15 @@
% (name, chrome_src))
self.chrome_src = chromeos_src
+ def _GetImageType(self, chromeos_image):
+ image_type = None
+ if chromeos_image.find("xbuddy://") < 0:
+ image_type = "local"
+ elif chromeos_image.find("trybot") >= 0:
+ image_type = "trybot"
+ else:
+ image_type = "official"
+ return image_type
class MockLabel(object):
def __init__(self, name, chromeos_image, chromeos_root, board, remote,
@@ -67,3 +81,4 @@
self.chromeos_root = chromeos_root
self.image_args = image_args
self.image_md5sum = image_md5sum
+ self.chrome_src = chrome_src
diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py
index 282bbc5..51e1055 100644
--- a/crosperf/machine_manager.py
+++ b/crosperf/machine_manager.py
@@ -180,8 +180,14 @@
self.chromeos_root = chromeos_root
def ImageMachine(self, machine, label):
- checksum = ImageChecksummer().Checksum(label)
- if machine.checksum == checksum:
+ if label.image_type == "local":
+ checksum = ImageChecksummer().Checksum(label)
+ elif label.image_type == "trybot":
+ checksum = machine._GetMD5Checksum(label.chromeos_image)
+ else:
+ checksum = None
+
+ if checksum and (machine.checksum == checksum):
return
chromeos_root = label.chromeos_root
if not chromeos_root:
@@ -271,7 +277,12 @@
% m.name)
def AcquireMachine(self, chromeos_image, label):
- image_checksum = ImageChecksummer().Checksum(label)
+ if label.image_type == "local":
+ image_checksum = ImageChecksummer().Checksum(label)
+ elif label.image_type == "trybot":
+ image_checksum = hashlib.md5(chromeos_image).hexdigest()
+ else:
+ image_checksum = None
machines = self.GetMachines(label)
check_interval_time = 120
with self._lock:
@@ -305,7 +316,7 @@
### return None
for m in [machine for machine in self.GetAvailableMachines(label)
if not machine.locked]:
- if m.checksum == image_checksum:
+ if image_checksum and (m.checksum == image_checksum):
m.locked = True
m.test_run = threading.current_thread()
return m
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index 4c1cf4c..fb66e5c 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -539,6 +539,10 @@
machine_checksum = self.machine_manager.machine_checksum[self.label.name]
if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
checksum = "*"
+ elif self.label.image_type == "trybot":
+ checksum = hashlib.md5(self.label.chromeos_image).hexdigest()
+ elif self.label.image_type == "official":
+ checksum = "*"
else:
checksum = ImageChecksummer().Checksum(self.label)
diff --git a/crosperf/settings.py b/crosperf/settings.py
index e407a14..2ed18c3 100644
--- a/crosperf/settings.py
+++ b/crosperf/settings.py
@@ -60,3 +60,11 @@
for name in self.fields:
if not self.fields[name].assigned and self.fields[name].required:
raise Exception("Field %s is invalid." % name)
+
+ def GetXbuddyPath(self, path_str, board):
+ prefix = "xbuddy://remote"
+ if path_str.find("trybot") < 0:
+ xbuddy_path = "%s/%s/%s" % (prefix, board, path_str)
+ else:
+ xbuddy_path = "%s/%s" % (prefix, path_str)
+ return xbuddy_path
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index ba3fcee..6be3481 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -54,9 +54,10 @@
class LabelSettings(Settings):
def __init__(self, name):
super(LabelSettings, self).__init__(name, "label")
- self.AddField(TextField("chromeos_image", required=True,
+ self.AddField(TextField("chromeos_image", required=False,
description="The path to the image to run tests "
- "on."))
+ "on, for local/custom-built images. See 'build' "
+ "option for official or trybot images."))
self.AddField(TextField("chromeos_root",
description="The path to a chromeos checkout which "
"contains a src/scripts directory. Defaults to "
@@ -80,6 +81,13 @@
"This is used to run telemetry benchmarks. "
"The default one is the src inside chroot.",
required=False, default=""))
+ self.AddField(TextField("build",
+ description="The xbuddy specification for an "
+ "official or trybot image to use for tests. "
+ "'/remote' is assumed, and the board is given "
+ "elsewhere, so omit the '/remote/<board>/' xbuddy"
+ "prefix.",
+ required=False, default=""))
class GlobalSettings(Settings):
@@ -153,7 +161,13 @@
description="The path to the source of chrome. "
"This is used to run telemetry benchmarks. "
"The default one is the src inside chroot.",
-
+ required=False, default=""))
+ self.AddField(TextField("build",
+ description="The xbuddy specification for an "
+ "official or trybot image to use for tests. "
+ "'/remote' is assumed, and the board is given "
+ "elsewhere, so omit the '/remote/<board>/' xbuddy"
+ "prefix.",
required=False, default=""))
diff --git a/image_chromeos.py b/image_chromeos.py
index 17588d9..001f4a6 100755
--- a/image_chromeos.py
+++ b/image_chromeos.py
@@ -33,6 +33,32 @@
sys.exit(0)
+def CheckForCrosFlash (chromeos_root, remote):
+ cmd_executer = command_executer.GetCommandExecuter()
+
+ chroot_has_cros_flash = False
+ remote_has_cherrypy = False
+
+ # Check to see if chroot contains cros flash.
+ cros_flash_path = os.path.join(os.path.realpath(chromeos_root),
+ "chromite/cros/commands/cros_flash.py")
+
+ if os.path.exists(cros_flash_path):
+ chroot_has_cros_flash = True
+
+ # Check to see if remote machine has cherrypy.
+ keypath = os.path.join (os.path.realpath(chromeos_root),
+ "src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa")
+
+ command = ("ssh -i %s -o StrictHostKeyChecking=no -o CheckHostIP=no "
+ "-o BatchMode=yes root@%s \"python -c 'import cherrypy'\" " %
+ (keypath,remote) )
+ retval = cmd_executer.RunCommand (command)
+ if retval == 0:
+ remote_has_cherrypy = True
+
+ return (chroot_has_cros_flash and remote_has_cherrypy)
+
def DoImage(argv):
"""Build ChromeOS."""
@@ -84,41 +110,54 @@
"chromiumos_image.bin")
else:
image = options.image
- image = os.path.expanduser(image)
+ if image.find("xbuddy://") < 0:
+ image = os.path.expanduser(image)
- image = os.path.realpath(image)
+ if image.find("xbuddy://") < 0:
+ image = os.path.realpath(image)
- if not os.path.exists(image):
+ if not os.path.exists(image) and image.find("xbuddy://") < 0:
Usage(parser, "Image file: " + image + " does not exist!")
- image_checksum = FileUtils().Md5File(image)
+ reimage = False
+ local_image = False
+ if image.find("xbuddy://") < 0:
+ local_image = True
+ image_checksum = FileUtils().Md5File(image)
- command = "cat " + checksum_file
- retval, device_checksum, err = cmd_executer.CrosRunCommand(command,
- return_output=True,
- chromeos_root=options.chromeos_root,
- machine=options.remote)
+ command = "cat " + checksum_file
+ retval, device_checksum, err = cmd_executer.CrosRunCommand(command,
+ return_output=True,
+ chromeos_root=options.chromeos_root,
+ machine=options.remote)
- device_checksum = device_checksum.strip()
- image_checksum = str(image_checksum)
+ device_checksum = device_checksum.strip()
+ image_checksum = str(image_checksum)
- l.LogOutput("Image checksum: " + image_checksum)
- l.LogOutput("Device checksum: " + device_checksum)
+ l.LogOutput("Image checksum: " + image_checksum)
+ l.LogOutput("Device checksum: " + device_checksum)
- if image_checksum != device_checksum:
- [found, located_image] = LocateOrCopyImage(options.chromeos_root,
- image,
- board=board)
+ if image_checksum != device_checksum:
+ [found, located_image] = LocateOrCopyImage(options.chromeos_root,
+ image,
+ board=board)
- l.LogOutput("Checksums do not match. Re-imaging...")
+ reimage = True
+ l.LogOutput("Checksums do not match. Re-imaging...")
- is_test_image = IsImageModdedForTest(options.chromeos_root,
- located_image)
+ is_test_image = IsImageModdedForTest(options.chromeos_root,
+ located_image)
- if not is_test_image and not options.force:
- logger.GetLogger().LogFatal("Have to pass --force to image a non-test "
- "image!")
+ if not is_test_image and not options.force:
+ logger.GetLogger().LogFatal("Have to pass --force to image a non-test "
+ "image!")
+ else:
+ reimage = True
+ found = True
+ l.LogOutput("Using non-local image; Re-imaging...")
+
+ if reimage:
# If the device has /tmp mounted as noexec, image_to_live.sh can fail.
command = "mount -o remount,rw,exec /tmp"
cmd_executer.CrosRunCommand(command,
@@ -127,27 +166,35 @@
real_src_dir = os.path.join(os.path.realpath(options.chromeos_root),
"src")
- if located_image.find(real_src_dir) != 0:
- raise Exception("Located image: %s not in chromeos_root: %s" %
- (located_image, options.chromeos_root))
- chroot_image = os.path.join(
- "..",
- located_image[len(real_src_dir):].lstrip("/"))
+ if local_image:
+ if located_image.find(real_src_dir) != 0:
+ raise Exception("Located image: %s not in chromeos_root: %s" %
+ (located_image, options.chromeos_root))
+ chroot_image = os.path.join(
+ "..",
+ located_image[len(real_src_dir):].lstrip("/"))
# Check to see if cros flash is in the chroot or not.
- cros_flash_path = os.path.join(options.chromeos_root,
- "chromite/cros/commands/cros_flash.py")
- if os.path.exists(cros_flash_path):
+ use_cros_flash = CheckForCrosFlash (options.chromeos_root,
+ options.remote)
+
+ if use_cros_flash:
# Use 'cros flash'
- cros_flash_args = ["--board=%s" % board,
- "--clobber-stateful",
- options.remote,
- chroot_image]
+ if local_image:
+ cros_flash_args = ["--board=%s" % board,
+ "--clobber-stateful",
+ options.remote,
+ chroot_image]
+ else:
+
+ cros_flash_args = ["--board=%s" % board,
+ "--clobber-stateful",
+ options.remote,
+ image]
command = ("cros flash %s" % " ".join(cros_flash_args))
- else:
+ elif local_image:
# Use 'cros_image_to_target.py'
-
cros_image_to_target_args = ["--remote=%s" % options.remote,
"--board=%s" % board,
"--from=%s" % os.path.dirname(chroot_image),
@@ -158,6 +205,10 @@
" ".join(cros_image_to_target_args))
if options.image_args:
command += " %s" % options.image_args
+ else:
+ raise Exception("Unable to find 'cros flash' in chroot; cannot use "
+ "non-local image (%s) with cros_image_to_target.py" %
+ image)
# Workaround for crosbug.com/35684.
os.chmod(misc.GetChromeOSKeyFile(options.chromeos_root), 0600)
@@ -181,19 +232,29 @@
# machine isn't fully up yet.
retval = EnsureMachineUp(options.chromeos_root, options.remote)
- command = "echo %s > %s && chmod -w %s" % (image_checksum, checksum_file,
- checksum_file)
- retval = cmd_executer.CrosRunCommand(command,
- chromeos_root=options.chromeos_root,
- machine=options.remote)
- logger.GetLogger().LogFatalIf(retval, "Writing checksum failed.")
+ # If this is a non-local image, then the retval returned from
+ # EnsureMachineUp is the one that will be returned by this function;
+ # in that case, make sure the value in 'retval' is appropriate.
+ if not local_image and retval == True:
+ retval = 0
+ else:
+ retval = 1
- successfully_imaged = VerifyChromeChecksum(options.chromeos_root,
- image,
- options.remote)
- logger.GetLogger().LogFatalIf(not successfully_imaged,
- "Image verification failed!")
- TryRemountPartitionAsRW(options.chromeos_root, options.remote)
+ if local_image:
+ command = "echo %s > %s && chmod -w %s" % (image_checksum,
+ checksum_file,
+ checksum_file)
+ retval = cmd_executer.CrosRunCommand(command,
+ chromeos_root=options.chromeos_root,
+ machine=options.remote)
+ logger.GetLogger().LogFatalIf(retval, "Writing checksum failed.")
+
+ successfully_imaged = VerifyChromeChecksum(options.chromeos_root,
+ image,
+ options.remote)
+ logger.GetLogger().LogFatalIf(not successfully_imaged,
+ "Image verification failed!")
+ TryRemountPartitionAsRW(options.chromeos_root, options.remote)
else:
l.LogOutput("Checksums match. Skipping reimage")
return retval
