server/cros/telemetry_runner.py - mirrors/cros/chromiumos/third_party/autotest - Git at Google

 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 import logging
 import os
 import re
 import StringIO

 import common
 from autotest_lib.client.common_lib import error, utils
 from autotest_lib.client.common_lib.cros import dev_server


 TELEMETRY_RUN_BENCHMARKS_SCRIPT = 'tools/perf/run_multipage_benchmarks'
 TELEMETRY_RUN_TESTS_SCRIPT = 'tools/telemetry/run_tests'
 TELEMETRY_TIMEOUT_MINS = 15

 # Result Statuses
 SUCCESS_STATUS = 'SUCCESS'
 WARNING_STATUS = 'WARNING'
 FAILED_STATUS = 'FAILED'


 class TelemetryResult(object):
     """Class to represent the results of a telemetry run.

     This class represents the results of a telemetry run, whether it ran
     successful, failed or had warnings.
     """


     def __init__(self, exit_code=0, stdout='', stderr=''):
         """Initializes this TelemetryResultObject instance.

         @param status: Status of the telemtry run.
         @param stdout: Stdout of the telemetry run.
         @param stderr: Stderr of the telemetry run.
         """
         if exit_code == 0:
             self.status = SUCCESS_STATUS
         else:
             self.status = FAILED_STATUS

         self.perf_keyvals = {}
         self._stdout = stdout
         self._stderr = stderr
         self.output = '\n'.join([stdout, stderr])


     def _cleanup_value(self, value):
         """Cleanup a value string.

         Given a string representing a value clean it up by removing the space
         and parenthesis around the units, and either append the units or get
         rid of them.

         Examples:
             loadtime (ms) -> loadtime_ms
             image_count () -> image_count
             image_count (count) -> image_count
             CodeLoad (score (bigger is better)) -> CodeLoad_score
             load_percent (%) -> load_percent
             score (runs/s) -> score_runs_per_s

         @param value: Value we are cleaning up.

         @result a String representing the cleaned up value.
         """
         value_sections = value.split(' (')
         value_name = value_sections[0]
         # There can be sub-parens in the units -> if so remove them.
         units = value_sections[1].split('(')[0]
         units = units.split(')')[0]
         if units is '%':
             units = 'percent'
         if '/' in units:
             units = units.replace('/','_per_')
         if not units:
             return value_name
         if value_name.endswith(units):
             return value_name
         return '_'.join([value_name, units])


     def parse_benchmark_results(self):
         """Parse the results of a telemetry benchmark run.

         Stdout has the format of CSV at the top and then the output repeated
         in RESULT block format below.

         We will parse the CSV part to get the perf key-value pairs we are
         interested in.

         Example stdout:
         url,average_commit_time (ms),average_image_gathering_time (ms)
         file:///tough_scrolling_cases/cust_scrollbar.html,1.3644,0
         RESULT average_commit_time: <URL>= <SCORE> score
         RESULT average_image_gathering_time: <URL>= <SCORE> score

         We want to generate perf keys in the format of value-url i.e.:
         average_commit_time-http____www.google.com
         Where we also removed non non-alphanumeric characters except '.', '_',
         and '-'.

         Stderr has the format of Warnings/Tracebacks. There is always a default
         warning of the display enviornment setting. Followed by warnings of
         page timeouts or a traceback.

         If there are any other warnings we flag the test as warning. If there
         is a traceback we consider this test a failure.

         @param exit_code: Exit code of the the telemetry run. 0 == SUCCESS,
                           otherwise it is a warning or failure.
         @param stdout: Stdout of the telemetry run.
         @param stderr: Stderr of the telemetry run.

         @returns A TelemetryResult instance with the results of the telemetry
                  run.
         """
         # The output will be in CSV format.
         if not self._stdout:
             # Nothing in stdout implies a test failure.
             logging.error('No stdout, test failed.')
             self.status = FAILED_STATUS
             return

         stdout_lines = self._stdout.splitlines()
         value_names = None
         for line in stdout_lines:
             if not line:
                 continue
             if not value_names and line.startswith('url,'):
                 # This line lists out all the values we care about and we drop
                 # the first one as it is the url name.
                 value_names = line.split(',')[1:]
                 # Clean up each value name.
                 value_names = [self._cleanup_value(v) for v in value_names]
                 logging.debug('Value_names: %s', value_names)
             if not value_names:
                 continue
             if ' ' in line:
                 # We are in a non-CSV part of the output, ignore this line.
                 continue
             # We are now a CSV line we care about, parse it accordingly.
             line_values = line.split(',')
             # Grab the URL
             url = line_values[0]
             # We want the perf keys to be format value|url. Example:
             # load_time-http___www.google.com
             # Andd replace all non-alphanumeric characters except
             # '-', '.' and '_' with '_'
             url_values_names = [re.sub(r'[^\w.-]', '_', '-'.join([v, url]))
                     for v in value_names]
             self.perf_keyvals.update(dict(zip(url_values_names,
                                               line_values[1:])))
         logging.debug('Perf Keyvals: %s', self.perf_keyvals)

         if self.status is SUCCESS_STATUS:
             return

         # Otherwise check if simply a Warning occurred or a Failure,
         # i.e. a Traceback is listed.
         self.status = WARNING_STATUS
         for line in self._stderr.splitlines():
             if line.startswith('Traceback'):
                 self.status = FAILED_STATUS


 class TelemetryRunner(object):
     """Class responsible for telemetry for a given build.

     This class will extract and install telemetry on the devserver and is
     responsible for executing the telemetry benchmarks and returning their
     output to the caller.
     """

     def __init__(self, host):
         """Initializes this telemetry runner instance.

         If telemetry is not installed for this build, it will be.
         """
         self._host = host
         logging.debug('Grabbing build from AFE.')

         build = host.get_build()
         if not build:
             logging.error('Unable to locate build label for host: %s.',
                           self._host.hostname)
             raise error.AutotestError('Failed to grab build for host %s.' %
                                       self._host.hostname)

         logging.debug('Setting up telemetry for build: %s', build)

         self._devserver = dev_server.ImageServer.resolve(build)
         self._telemetry_path = self._devserver.setup_telemetry(build=build)
         logging.debug('Telemetry Path: %s',self._telemetry_path)


     def _run_telemetry(self, script, test_or_benchmark):
         """Runs telemetry on a dut.

         @param script: Telemetry script we want to run. For example:
                        [path_to_telemetry_src]/src/tools/telemetry/run_tests
         @param test_or_benchmark: Name of the test or benchmark we want to run,
                                  with the page_set (if required) as part of the
                                  string.

         @returns A TelemetryResult Instance with the results of this telemetry
                  execution.
         """
         devserver_hostname = self._devserver.url().split(
                 'http://')[1].split(':')[0]
         telemetry_args = ['ssh',
                           devserver_hostname,
                           'python',
                           script,
                           '--browser=cros-chrome',
                           '--remote=%s' % self._host.hostname,
                           test_or_benchmark]

         logging.debug('Running Telemetry: %s', ' '.join(telemetry_args))
         output = StringIO.StringIO()
         error_output = StringIO.StringIO()
         exit_code = 0
         try:
             result = utils.run(' '.join(telemetry_args), stdout_tee=output,
                                stderr_tee=error_output,
                                timeout=TELEMETRY_TIMEOUT_MINS*60)
             exit_code = result.exit_status
         except error.CmdError as e:
             # Telemetry returned a return code of not 0; for benchmarks this
             # can be due to a timeout on one of the pages of the page set and
             # we may still have data on the rest. For a test however this
             # indicates failure.
             logging.debug('Error occurred executing telemetry.')
             exit_code = e.result_obj.exit_status

         stdout = output.getvalue()
         stderr = error_output.getvalue()
         logging.debug('Telemetry completed with exit code: %d.\nstdout:%s\n'
                       'stderr:%s', exit_code, stdout, stderr)

         return TelemetryResult(exit_code=exit_code, stdout=stdout,
                                stderr=stderr)


     def run_telemetry_test(self, test):
         """Runs a telemetry test on a dut.

         @param test: Telemetry test we want to run.

         @returns A TelemetryResult Instance with the results of this telemetry
                  execution.
         """
         logging.debug('Running telemetry test: %s', test)
         telemetry_script = os.path.join(self._telemetry_path,
                                         TELEMETRY_RUN_TESTS_SCRIPT)
         result = self._run_telemetry(telemetry_script, test)
         if result.status is FAILED_STATUS:
             raise error.TestFail('Telemetry test: %s failed.',
                                  test)
         return result


     def run_telemetry_benchmark(self, benchmark, page_set, keyval_writer=None):
         """Runs a telemetry benchmark on a dut.

         @param benchmark: Benchmark we want to run.
         @param page_set: Page set we want to use.
         @param keyval_writer: Should be a instance with the function
                               write_perf_keyval(), if None, no keyvals will be
                               written. Typically this will be the job object
                               from a autotest test.

         @returns A TelemetryResult Instance with the results of this telemetry
                  execution.
         """
         logging.debug('Running telemetry benchmark: %s with page set: %s.',
                       benchmark, page_set)
         telemetry_script = os.path.join(self._telemetry_path,
                                         TELEMETRY_RUN_BENCHMARKS_SCRIPT)
         page_set_path = os.path.join(self._telemetry_path,
                                      'tools/perf/page_sets/%s' % page_set)
         benchmark_with_pageset = ' '.join([benchmark, page_set_path])
         result = self._run_telemetry(telemetry_script, benchmark_with_pageset)
         result.parse_benchmark_results()

         if keyval_writer:
             keyval_writer.write_perf_keyval(result.perf_keyvals)

         if result.status is WARNING_STATUS:
             raise error.TestWarn('Telemetry Benchmark: %s with page set: %s'
                                  ' exited with Warnings.' % (benchmark,
                                                              page_set))
         if result.status is FAILED_STATUS:
             raise error.TestFail('Telemetry Benchmark: %s with page set: %s'
                                  ' failed to run.' % (benchmark,
                                                       page_set))

         return result
	# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	import logging
	import os
	import re
	import StringIO

	import common
	from autotest_lib.client.common_lib import error, utils
	from autotest_lib.client.common_lib.cros import dev_server


	TELEMETRY_RUN_BENCHMARKS_SCRIPT = 'tools/perf/run_multipage_benchmarks'
	TELEMETRY_RUN_TESTS_SCRIPT = 'tools/telemetry/run_tests'
	TELEMETRY_TIMEOUT_MINS = 15

	# Result Statuses
	SUCCESS_STATUS = 'SUCCESS'
	WARNING_STATUS = 'WARNING'
	FAILED_STATUS = 'FAILED'


	class TelemetryResult(object):
	"""Class to represent the results of a telemetry run.

	This class represents the results of a telemetry run, whether it ran
	successful, failed or had warnings.
	"""


	def __init__(self, exit_code=0, stdout='', stderr=''):
	"""Initializes this TelemetryResultObject instance.

	@param status: Status of the telemtry run.
	@param stdout: Stdout of the telemetry run.
	@param stderr: Stderr of the telemetry run.
	"""
	if exit_code == 0:
	self.status = SUCCESS_STATUS
	else:
	self.status = FAILED_STATUS

	self.perf_keyvals = {}
	self._stdout = stdout
	self._stderr = stderr
	self.output = '\n'.join([stdout, stderr])


	def _cleanup_value(self, value):
	"""Cleanup a value string.

	Given a string representing a value clean it up by removing the space
	and parenthesis around the units, and either append the units or get
	rid of them.

	Examples:
	loadtime (ms) -> loadtime_ms
	image_count () -> image_count
	image_count (count) -> image_count
	CodeLoad (score (bigger is better)) -> CodeLoad_score
	load_percent (%) -> load_percent
	score (runs/s) -> score_runs_per_s

	@param value: Value we are cleaning up.

	@result a String representing the cleaned up value.
	"""
	value_sections = value.split(' (')
	value_name = value_sections[0]
	# There can be sub-parens in the units -> if so remove them.
	units = value_sections[1].split('(')[0]
	units = units.split(')')[0]
	if units is '%':
	units = 'percent'
	if '/' in units:
	units = units.replace('/','_per_')
	if not units:
	return value_name
	if value_name.endswith(units):
	return value_name
	return '_'.join([value_name, units])


	def parse_benchmark_results(self):
	"""Parse the results of a telemetry benchmark run.

	Stdout has the format of CSV at the top and then the output repeated
	in RESULT block format below.

	We will parse the CSV part to get the perf key-value pairs we are
	interested in.

	Example stdout:
	url,average_commit_time (ms),average_image_gathering_time (ms)
	file:///tough_scrolling_cases/cust_scrollbar.html,1.3644,0
	RESULT average_commit_time: <URL>= <SCORE> score
	RESULT average_image_gathering_time: <URL>= <SCORE> score

	We want to generate perf keys in the format of value-url i.e.:
	average_commit_time-http____www.google.com
	Where we also removed non non-alphanumeric characters except '.', '_',
	and '-'.

	Stderr has the format of Warnings/Tracebacks. There is always a default
	warning of the display enviornment setting. Followed by warnings of
	page timeouts or a traceback.

	If there are any other warnings we flag the test as warning. If there
	is a traceback we consider this test a failure.

	@param exit_code: Exit code of the the telemetry run. 0 == SUCCESS,
	otherwise it is a warning or failure.
	@param stdout: Stdout of the telemetry run.
	@param stderr: Stderr of the telemetry run.

	@returns A TelemetryResult instance with the results of the telemetry
	run.
	"""
	# The output will be in CSV format.
	if not self._stdout:
	# Nothing in stdout implies a test failure.
	logging.error('No stdout, test failed.')
	self.status = FAILED_STATUS
	return

	stdout_lines = self._stdout.splitlines()
	value_names = None
	for line in stdout_lines:
	if not line:
	continue
	if not value_names and line.startswith('url,'):
	# This line lists out all the values we care about and we drop
	# the first one as it is the url name.
	value_names = line.split(',')[1:]
	# Clean up each value name.
	value_names = [self._cleanup_value(v) for v in value_names]
	logging.debug('Value_names: %s', value_names)
	if not value_names:
	continue
	if ' ' in line:
	# We are in a non-CSV part of the output, ignore this line.
	continue
	# We are now a CSV line we care about, parse it accordingly.
	line_values = line.split(',')
	# Grab the URL
	url = line_values[0]
	# We want the perf keys to be format value\|url. Example:
	# load_time-http___www.google.com
	# Andd replace all non-alphanumeric characters except
	# '-', '.' and '_' with '_'
	url_values_names = [re.sub(r'[^\w.-]', '_', '-'.join([v, url]))
	for v in value_names]
	self.perf_keyvals.update(dict(zip(url_values_names,
	line_values[1:])))
	logging.debug('Perf Keyvals: %s', self.perf_keyvals)

	if self.status is SUCCESS_STATUS:
	return

	# Otherwise check if simply a Warning occurred or a Failure,
	# i.e. a Traceback is listed.
	self.status = WARNING_STATUS
	for line in self._stderr.splitlines():
	if line.startswith('Traceback'):
	self.status = FAILED_STATUS


	class TelemetryRunner(object):
	"""Class responsible for telemetry for a given build.

	This class will extract and install telemetry on the devserver and is
	responsible for executing the telemetry benchmarks and returning their
	output to the caller.
	"""

	def __init__(self, host):
	"""Initializes this telemetry runner instance.

	If telemetry is not installed for this build, it will be.
	"""
	self._host = host
	logging.debug('Grabbing build from AFE.')

	build = host.get_build()
	if not build:
	logging.error('Unable to locate build label for host: %s.',
	self._host.hostname)
	raise error.AutotestError('Failed to grab build for host %s.' %
	self._host.hostname)

	logging.debug('Setting up telemetry for build: %s', build)

	self._devserver = dev_server.ImageServer.resolve(build)
	self._telemetry_path = self._devserver.setup_telemetry(build=build)
	logging.debug('Telemetry Path: %s',self._telemetry_path)


	def _run_telemetry(self, script, test_or_benchmark):
	"""Runs telemetry on a dut.

	@param script: Telemetry script we want to run. For example:
	[path_to_telemetry_src]/src/tools/telemetry/run_tests
	@param test_or_benchmark: Name of the test or benchmark we want to run,
	with the page_set (if required) as part of the
	string.

	@returns A TelemetryResult Instance with the results of this telemetry
	execution.
	"""
	devserver_hostname = self._devserver.url().split(
	'http://')[1].split(':')[0]
	telemetry_args = ['ssh',
	devserver_hostname,
	'python',
	script,
	'--browser=cros-chrome',
	'--remote=%s' % self._host.hostname,
	test_or_benchmark]

	logging.debug('Running Telemetry: %s', ' '.join(telemetry_args))
	output = StringIO.StringIO()
	error_output = StringIO.StringIO()
	exit_code = 0
	try:
	result = utils.run(' '.join(telemetry_args), stdout_tee=output,
	stderr_tee=error_output,
	timeout=TELEMETRY_TIMEOUT_MINS*60)
	exit_code = result.exit_status
	except error.CmdError as e:
	# Telemetry returned a return code of not 0; for benchmarks this
	# can be due to a timeout on one of the pages of the page set and
	# we may still have data on the rest. For a test however this
	# indicates failure.
	logging.debug('Error occurred executing telemetry.')
	exit_code = e.result_obj.exit_status

	stdout = output.getvalue()
	stderr = error_output.getvalue()
	logging.debug('Telemetry completed with exit code: %d.\nstdout:%s\n'
	'stderr:%s', exit_code, stdout, stderr)

	return TelemetryResult(exit_code=exit_code, stdout=stdout,
	stderr=stderr)


	def run_telemetry_test(self, test):
	"""Runs a telemetry test on a dut.

	@param test: Telemetry test we want to run.

	@returns A TelemetryResult Instance with the results of this telemetry
	execution.
	"""
	logging.debug('Running telemetry test: %s', test)
	telemetry_script = os.path.join(self._telemetry_path,
	TELEMETRY_RUN_TESTS_SCRIPT)
	result = self._run_telemetry(telemetry_script, test)
	if result.status is FAILED_STATUS:
	raise error.TestFail('Telemetry test: %s failed.',
	test)
	return result


	def run_telemetry_benchmark(self, benchmark, page_set, keyval_writer=None):
	"""Runs a telemetry benchmark on a dut.

	@param benchmark: Benchmark we want to run.
	@param page_set: Page set we want to use.
	@param keyval_writer: Should be a instance with the function
	write_perf_keyval(), if None, no keyvals will be
	written. Typically this will be the job object
	from a autotest test.

	@returns A TelemetryResult Instance with the results of this telemetry
	execution.
	"""
	logging.debug('Running telemetry benchmark: %s with page set: %s.',
	benchmark, page_set)
	telemetry_script = os.path.join(self._telemetry_path,
	TELEMETRY_RUN_BENCHMARKS_SCRIPT)
	page_set_path = os.path.join(self._telemetry_path,
	'tools/perf/page_sets/%s' % page_set)
	benchmark_with_pageset = ' '.join([benchmark, page_set_path])
	result = self._run_telemetry(telemetry_script, benchmark_with_pageset)
	result.parse_benchmark_results()

	if keyval_writer:
	keyval_writer.write_perf_keyval(result.perf_keyvals)

	if result.status is WARNING_STATUS:
	raise error.TestWarn('Telemetry Benchmark: %s with page set: %s'
	' exited with Warnings.' % (benchmark,
	page_set))
	if result.status is FAILED_STATUS:
	raise error.TestFail('Telemetry Benchmark: %s with page set: %s'
	' failed to run.' % (benchmark,
	page_set))

	return result