crosperf: migration to python 3
This patch migrates crosperf and its utils to python 3.
TEST=Passed presubmit check; tested with simple experiment locally.
BUG=chromium:1011676
Change-Id: Ib2a9f9c7cf6a1bb1d0b42a1dd3d9e3cbb4d70a36
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2003796
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Caroline Tice <cmtice@chromium.org>
Commit-Queue: Zhizhou Yang <zhizhouy@google.com>
Auto-Submit: Zhizhou Yang <zhizhouy@google.com>
diff --git a/cros_utils/command_executer.py b/cros_utils/command_executer.py
old mode 100644
new mode 100755
index c4d3fde..002aa75
--- a/cros_utils/command_executer.py
+++ b/cros_utils/command_executer.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -17,11 +18,11 @@
import tempfile
import time
-import logger
-import misc
+from cros_utils import logger
mock_default = False
+CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts'
LOG_LEVEL = ('none', 'quiet', 'average', 'verbose')
@@ -101,6 +102,7 @@
# In this way the child cannot mess the parent's terminal.
p = None
try:
+ # pylint: disable=bad-option-value, subprocess-popen-preexec-fn
p = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
@@ -238,11 +240,13 @@
return command
def WriteToTempShFile(self, contents):
- handle, command_file = tempfile.mkstemp(prefix=os.uname()[1], suffix='.sh')
- os.write(handle, '#!/bin/bash\n')
- os.write(handle, contents)
- os.close(handle)
- return command_file
+ with tempfile.NamedTemporaryFile(
+ 'w', encoding='utf-8', delete=False, prefix=os.uname()[1],
+ suffix='.sh') as f:
+ f.write('#!/bin/bash\n')
+ f.write(contents)
+ f.flush()
+ return f.name
def CrosLearnBoard(self, chromeos_root, machine):
command = self.RemoteAccessInitCommand(chromeos_root, machine)
@@ -362,15 +366,19 @@
if self.logger:
self.logger.LogCmd(command, print_to_console=print_to_console)
- handle, command_file = tempfile.mkstemp(
+ with tempfile.NamedTemporaryFile(
+ 'w',
+ encoding='utf-8',
+ delete=False,
dir=os.path.join(chromeos_root, 'src/scripts'),
suffix='.sh',
- prefix='in_chroot_cmd')
- os.write(handle, '#!/bin/bash\n')
- os.write(handle, command)
- os.write(handle, '\n')
- os.close(handle)
+ prefix='in_chroot_cmd') as f:
+ f.write('#!/bin/bash\n')
+ f.write(command)
+ f.write('\n')
+ f.flush()
+ command_file = f.name
os.chmod(command_file, 0o777)
# if return_output is set, run a dummy command first to make sure that
@@ -386,7 +394,7 @@
# Run command_file inside the chroot, making sure that any "~" is expanded
# by the shell inside the chroot, not outside.
command = ("cd %s; cros_sdk %s -- bash -c '%s/%s'" %
- (chromeos_root, cros_sdk_options, misc.CHROMEOS_SCRIPTS_DIR,
+ (chromeos_root, cros_sdk_options, CHROMEOS_SCRIPTS_DIR,
os.path.basename(command_file)))
ret = self.RunCommandGeneric(
command,
@@ -601,6 +609,7 @@
# In this way the child cannot mess the parent's terminal.
pobject = None
try:
+ # pylint: disable=bad-option-value, subprocess-popen-preexec-fn
pobject = subprocess.Popen(
cmd,
cwd=cwd,
diff --git a/cros_utils/email_sender.py b/cros_utils/email_sender.py
index 3f724f6..0019982 100755
--- a/cros_utils/email_sender.py
+++ b/cros_utils/email_sender.py
@@ -97,12 +97,14 @@
if not text_to_send:
text_to_send = 'Empty message body.'
- body_fd, body_filename = tempfile.mkstemp()
- to_be_deleted = [body_filename]
+ to_be_deleted = []
try:
- os.write(body_fd, text_to_send)
- os.close(body_fd)
+ with tempfile.NamedTemporaryFile(
+ 'w', encoding='utf-8', delete=False) as f:
+ f.write(text_to_send)
+ f.flush()
+ to_be_deleted.append(f.name)
# Fix single-quotes inside the subject. In bash, to escape a single quote
# (e.g 'don't') you need to replace it with '\'' (e.g. 'don'\''t'). To
@@ -113,11 +115,10 @@
if msg_type == 'html':
command = ("sendgmr --to='%s' --from='%s' --subject='%s' "
"--html_file='%s' --body_file=/dev/null" %
- (to_list, email_from, subject, body_filename))
+ (to_list, email_from, subject, f.name))
else:
- command = (
- "sendgmr --to='%s' --from='%s' --subject='%s' "
- "--body_file='%s'" % (to_list, email_from, subject, body_filename))
+ command = ("sendgmr --to='%s' --from='%s' --subject='%s' "
+ "--body_file='%s'" % (to_list, email_from, subject, f.name))
if email_cc:
cc_list = ','.join(email_cc)
@@ -133,10 +134,11 @@
report_suffix = '_report.html'
else:
report_suffix = '_report.txt'
- fd, fname = tempfile.mkstemp(suffix=report_suffix)
- os.write(fd, attachment.content)
- os.close(fd)
- attachment_files.append(fname)
+ with tempfile.NamedTemporaryFile(
+ 'w', encoding='utf-8', delete=False, suffix=report_suffix) as f:
+ f.write(attachment.content)
+ f.flush()
+ attachment_files.append(f.name)
files = ','.join(attachment_files)
command += " --attachment_files='%s'" % files
to_be_deleted += attachment_files
diff --git a/cros_utils/file_utils.py b/cros_utils/file_utils.py
index fe6fc16..f0e4064 100644
--- a/cros_utils/file_utils.py
+++ b/cros_utils/file_utils.py
@@ -10,7 +10,8 @@
import errno
import os
import shutil
-import command_executer
+
+from cros_utils import command_executer
class FileUtils(object):
@@ -65,7 +66,7 @@
shutil.rmtree(path, ignore_errors=True)
def WriteFile(self, path, contents):
- with open(path, 'wb') as f:
+ with open(path, 'w', encoding='utf-8') as f:
f.write(contents)
diff --git a/cros_utils/locks.py b/cros_utils/locks.py
index 4ecbe0a..848e23f 100644
--- a/cros_utils/locks.py
+++ b/cros_utils/locks.py
@@ -12,7 +12,7 @@
import lock_machine
-import logger
+from cros_utils import logger
def AcquireLock(machines, chromeos_root, timeout=1200):
diff --git a/cros_utils/misc.py b/cros_utils/misc.py
index 48e3c72..246767f 100644
--- a/cros_utils/misc.py
+++ b/cros_utils/misc.py
@@ -17,10 +17,10 @@
import sys
import traceback
-import command_executer
-import logger
+from cros_utils import command_executer
+from cros_utils import logger
-CHROMEOS_SCRIPTS_DIR = '~/trunk/src/scripts'
+CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts'
TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/'
'cros_utils/toolchain_utils.sh')
diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py
index 9527fde..300c2d7 100644
--- a/cros_utils/tabulator.py
+++ b/cros_utils/tabulator.py
@@ -72,8 +72,8 @@
import numpy
import scipy
-from email_sender import EmailSender
-import misc
+from cros_utils.email_sender import EmailSender
+from cros_utils import misc
def _AllFloat(values):
diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py
index f17de1b..74d461a 100644
--- a/crosperf/benchmark_run.py
+++ b/crosperf/benchmark_run.py
@@ -131,7 +131,7 @@
self.failure_reason = 'Return value of test suite was non-zero.'
self.timeline.Record(STATUS_FAILED)
- except Exception, e:
+ except Exception as e:
self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e))
traceback.print_exc()
if self.timeline.GetLastEvent() != STATUS_FAILED:
diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py
index 5696c10..4cadc35 100755
--- a/crosperf/benchmark_run_unittest.py
+++ b/crosperf/benchmark_run_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
@@ -9,10 +9,9 @@
from __future__ import print_function
-import unittest
import inspect
-
-import mock
+import unittest
+import unittest.mock as mock
import benchmark_run
@@ -121,7 +120,7 @@
'machine_manager', 'logger_to_use', 'log_level', 'share_cache',
'dut_config'
]
- arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__)
+ arg_spec = inspect.getfullargspec(benchmark_run.BenchmarkRun.__init__)
self.assertEqual(len(arg_spec.args), len(args_list))
self.assertEqual(arg_spec.args, args_list)
diff --git a/crosperf/benchmark_unittest.py b/crosperf/benchmark_unittest.py
index 63c0a1c..70508b1 100755
--- a/crosperf/benchmark_unittest.py
+++ b/crosperf/benchmark_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2014 The Chromium OS Authors. All rights reserved.
@@ -62,7 +62,7 @@
'perf_args', 'suite', 'show_all_results', 'retries', 'run_local',
'cwp_dso', 'weight'
]
- arg_spec = inspect.getargspec(Benchmark.__init__)
+ arg_spec = inspect.getfullargspec(Benchmark.__init__)
self.assertEqual(len(arg_spec.args), len(args_list))
for arg in args_list:
self.assertIn(arg, arg_spec.args)
diff --git a/crosperf/config_unittest.py b/crosperf/config_unittest.py
index 9e71539..208f44d 100755
--- a/crosperf/config_unittest.py
+++ b/crosperf/config_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
diff --git a/crosperf/crosperf b/crosperf/crosperf
index a29dcbf..c98f2dd 100755
--- a/crosperf/crosperf
+++ b/crosperf/crosperf
@@ -1,2 +1,7 @@
#!/bin/bash
-PYTHONPATH=$(dirname $0)/..:$PYTHONPATH exec python $(dirname $0)/crosperf.py "$@"
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+PYTHONPATH=$(dirname "$0")/..:${PYTHONPATH} exec \
+python3 "$(dirname "$0")/crosperf.py" "$@"
diff --git a/crosperf/crosperf.py b/crosperf/crosperf.py
index 7549437..ec07e7c 100755
--- a/crosperf/crosperf.py
+++ b/crosperf/crosperf.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -110,7 +110,7 @@
test_flag.SetTestMode(True)
experiment_file = ExperimentFile(
- open(experiment_filename, 'rb'), option_settings)
+ open(experiment_filename, encoding='utf-8'), option_settings)
if not experiment_file.GetGlobalSettings().GetField('name'):
experiment_name = os.path.basename(experiment_filename)
experiment_file.GetGlobalSettings().SetField('name', experiment_name)
diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py
index f7ffa1b..ffd964a 100755
--- a/crosperf/crosperf_unittest.py
+++ b/crosperf/crosperf_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
@@ -12,11 +12,9 @@
import argparse
import io
-import os
import tempfile
import unittest
-
-import mock
+import unittest.mock as mock
import crosperf
import settings_factory
@@ -46,14 +44,14 @@
"""Crosperf test class."""
def setUp(self):
- input_file = io.BytesIO(EXPERIMENT_FILE_1)
+ input_file = io.StringIO(EXPERIMENT_FILE_1)
self.exp_file = experiment_file.ExperimentFile(input_file)
def testDryRun(self):
- filehandle, filename = tempfile.mkstemp()
- os.write(filehandle, EXPERIMENT_FILE_1)
- crosperf.Main(['', filename, '--dry_run'])
- os.remove(filename)
+ with tempfile.NamedTemporaryFile('w', encoding='utf-8') as f:
+ f.write(EXPERIMENT_FILE_1)
+ f.flush()
+ crosperf.Main(['', f.name, '--dry_run'])
def testConvertOptionsToSettings(self):
parser = argparse.ArgumentParser()
diff --git a/crosperf/download_images.py b/crosperf/download_images.py
index f855cb6..9bd4a8b 100644
--- a/crosperf/download_images.py
+++ b/crosperf/download_images.py
@@ -56,8 +56,8 @@
def GetBuildID(self, chromeos_root, xbuddy_label):
# Get the translation of the xbuddy_label into the real Google Storage
# image name.
- command = ('cd ~/trunk/src/third_party/toolchain-utils/crosperf; '
- "python translate_xbuddy.py '%s'" % xbuddy_label)
+ command = ('cd /mnt/host/source/src/third_party/toolchain-utils/crosperf; '
+ "./translate_xbuddy.py '%s'" % xbuddy_label)
_, build_id_tuple_str, _ = self._ce.ChrootRunCommandWOutput(
chromeos_root, command)
if not build_id_tuple_str:
@@ -226,7 +226,7 @@
status = self.VerifyFileExists(chromeos_root, build_id,
autotest_packages_name)
if status != 0:
- default_autotest_dir = '~/trunk/src/third_party/autotest/files'
+ default_autotest_dir = '/mnt/host/source/src/third_party/autotest/files'
print(
'(Warning: Could not find autotest packages .)\n'
'(Warning: Defaulting autotest path to %s .' % default_autotest_dir)
diff --git a/crosperf/download_images_buildid_test.py b/crosperf/download_images_buildid_test.py
index a376691..fc37f2c 100755
--- a/crosperf/download_images_buildid_test.py
+++ b/crosperf/download_images_buildid_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
diff --git a/crosperf/download_images_unittest.py b/crosperf/download_images_unittest.py
index 94504d3..62b8d89 100755
--- a/crosperf/download_images_unittest.py
+++ b/crosperf/download_images_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -10,8 +10,7 @@
import os
import unittest
-
-import mock
+import unittest.mock as mock
import download_images
from cros_utils import command_executer
diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py
index 69e34fc..f5b17ee 100755
--- a/crosperf/experiment_factory_unittest.py
+++ b/crosperf/experiment_factory_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
@@ -13,8 +13,7 @@
import os
import socket
import unittest
-
-import mock
+import unittest.mock as mock
from cros_utils import command_executer
from cros_utils.file_utils import FileUtils
@@ -85,7 +84,7 @@
self.append_benchmark_call_args = []
def testLoadExperimentFile1(self):
- experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_1))
+ experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1))
exp = ExperimentFactory().GetExperiment(
experiment_file, working_directory='', log_dir='')
self.assertEqual(exp.remote, ['chromeos-alex3'])
@@ -104,7 +103,7 @@
self.assertEqual(exp.labels[0].board, 'x86-alex')
def testLoadExperimentFile2CWP(self):
- experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_2))
+ experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_2))
exp = ExperimentFactory().GetExperiment(
experiment_file, working_directory='', log_dir='')
self.assertEqual(exp.cwp_dso, 'kallsyms')
@@ -113,7 +112,7 @@
self.assertEqual(exp.benchmarks[1].weight, 0.2)
def testDuplecateBenchmark(self):
- mock_experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_1))
+ mock_experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1))
mock_experiment_file.all_settings = []
benchmark_settings1 = settings_factory.BenchmarkSettings('name')
mock_experiment_file.all_settings.append(benchmark_settings1)
@@ -125,7 +124,7 @@
ef.GetExperiment(mock_experiment_file, '', '')
def testCWPExceptions(self):
- mock_experiment_file = ExperimentFile(io.BytesIO(''))
+ mock_experiment_file = ExperimentFile(io.StringIO(''))
mock_experiment_file.all_settings = []
global_settings = settings_factory.GlobalSettings('test_name')
global_settings.SetField('locks_dir', '/tmp')
@@ -305,7 +304,7 @@
label_settings.GetXbuddyPath = FakeGetXbuddyPath
- mock_experiment_file = ExperimentFile(io.BytesIO(''))
+ mock_experiment_file = ExperimentFile(io.StringIO(''))
mock_experiment_file.all_settings = []
test_flag.SetTestMode(True)
@@ -332,7 +331,7 @@
# First test. General test.
exp = ef.GetExperiment(mock_experiment_file, '', '')
- self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80'])
+ self.assertCountEqual(exp.remote, ['123.45.67.89', '123.45.76.80'])
self.assertEqual(exp.cache_conditions, [0, 2, 1])
self.assertEqual(exp.log_level, 'average')
@@ -354,9 +353,9 @@
test_flag.SetTestMode(True)
label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros')
exp = ef.GetExperiment(mock_experiment_file, '', '')
- self.assertEqual(
+ self.assertCountEqual(
exp.remote,
- ['chromeos1.cros', 'chromeos2.cros', '123.45.67.89', '123.45.76.80'])
+ ['123.45.67.89', '123.45.76.80', 'chromeos1.cros', 'chromeos2.cros'])
# Third test: Automatic fixing of bad logging_level param:
global_settings.SetField('logging_level', 'really loud!')
@@ -392,7 +391,7 @@
self.assertEqual(len(exp.labels), 2)
self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path')
self.assertEqual(exp.labels[1].autotest_path, 'fake_autotest_path')
- self.assertEqual(
+ self.assertCountEqual(
exp.remote,
['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'])
@@ -431,7 +430,7 @@
ef.CheckSkylabTool(chromeos_root, log_level)
self.assertEqual(mock_runcmd.call_count, 1)
self.assertEqual(
- err.exception.message, 'Skylab tool not installed '
+ str(err.exception), 'Skylab tool not installed '
'correctly, please try to manually install it from '
'/tmp/chromeos/chromeos-admin/lab-tools/setup_lab_tools')
diff --git a/crosperf/experiment_file.py b/crosperf/experiment_file.py
index 33459d1..1d89eda 100644
--- a/crosperf/experiment_file.py
+++ b/crosperf/experiment_file.py
@@ -129,7 +129,7 @@
self.global_settings.SetField(field[0], field[1], field[2])
else:
raise IOError('Unexpected line.')
- except Exception, err:
+ except Exception as err:
raise RuntimeError('Line %d: %s\n==> %s' % (reader.LineNo(), str(err),
reader.CurrentLine(False)))
diff --git a/crosperf/experiment_file_unittest.py b/crosperf/experiment_file_unittest.py
index 12b6822..0d4e1e6 100755
--- a/crosperf/experiment_file_unittest.py
+++ b/crosperf/experiment_file_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
@@ -147,13 +147,13 @@
}
label: image1 {
-\tremote: chromeos-alex3
\tchromeos_image: /usr/local/google/cros_image1.bin
+\tremote: chromeos-alex3
}
label: image2 {
-\tremote: chromeos-lumpy1
\tchromeos_image: /usr/local/google/cros_image2.bin
+\tremote: chromeos-lumpy1
}\n\n"""
@@ -161,7 +161,7 @@
"""The main class for Experiment File test."""
def testLoadExperimentFile1(self):
- input_file = io.BytesIO(EXPERIMENT_FILE_1)
+ input_file = io.StringIO(EXPERIMENT_FILE_1)
experiment_file = ExperimentFile(input_file)
global_settings = experiment_file.GetGlobalSettings()
self.assertEqual(global_settings.GetField('remote'), ['chromeos-alex3'])
@@ -181,7 +181,7 @@
self.assertEqual(label_settings[0].GetField('remote'), ['chromeos-alex3'])
def testOverrideSetting(self):
- input_file = io.BytesIO(EXPERIMENT_FILE_2)
+ input_file = io.StringIO(EXPERIMENT_FILE_2)
experiment_file = ExperimentFile(input_file)
global_settings = experiment_file.GetGlobalSettings()
self.assertEqual(global_settings.GetField('remote'), ['chromeos-alex3'])
@@ -194,11 +194,11 @@
self.assertEqual(benchmark_settings[1].GetField('iterations'), 2)
def testDuplicateLabel(self):
- input_file = io.BytesIO(EXPERIMENT_FILE_3)
+ input_file = io.StringIO(EXPERIMENT_FILE_3)
self.assertRaises(Exception, ExperimentFile, input_file)
def testDuplicateBenchmark(self):
- input_file = io.BytesIO(EXPERIMENT_FILE_4)
+ input_file = io.StringIO(EXPERIMENT_FILE_4)
experiment_file = ExperimentFile(input_file)
benchmark_settings = experiment_file.GetSettings('benchmark')
self.assertEqual(benchmark_settings[0].name, 'webrtc')
@@ -209,13 +209,13 @@
'--story-tag-filter=smoothness')
def testCanonicalize(self):
- input_file = io.BytesIO(EXPERIMENT_FILE_1)
+ input_file = io.StringIO(EXPERIMENT_FILE_1)
experiment_file = ExperimentFile(input_file)
res = experiment_file.Canonicalize()
self.assertEqual(res, OUTPUT_FILE)
def testLoadDutConfigExperimentFile_Good(self):
- input_file = io.BytesIO(DUT_CONFIG_EXPERIMENT_FILE_GOOD)
+ input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_GOOD)
experiment_file = ExperimentFile(input_file)
global_settings = experiment_file.GetGlobalSettings()
self.assertEqual(global_settings.GetField('turbostat'), False)
@@ -228,22 +228,21 @@
self.assertEqual(global_settings.GetField('top_interval'), 5)
def testLoadDutConfigExperimentFile_WrongGovernor(self):
- input_file = io.BytesIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV)
+ input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV)
with self.assertRaises(RuntimeError) as msg:
ExperimentFile(input_file)
- self.assertRegexpMatches(
- str(msg.exception), 'governor: misspelled_governor')
- self.assertRegexpMatches(
+ self.assertRegex(str(msg.exception), 'governor: misspelled_governor')
+ self.assertRegex(
str(msg.exception), "Invalid enum value for field 'governor'."
r' Must be one of \(performance, powersave, userspace, ondemand,'
r' conservative, schedutils, sched, interactive\)')
def testLoadDutConfigExperimentFile_WrongCpuUsage(self):
- input_file = io.BytesIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE)
+ input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE)
with self.assertRaises(RuntimeError) as msg:
ExperimentFile(input_file)
- self.assertRegexpMatches(str(msg.exception), 'cpu_usage: unknown')
- self.assertRegexpMatches(
+ self.assertRegex(str(msg.exception), 'cpu_usage: unknown')
+ self.assertRegex(
str(msg.exception), "Invalid enum value for field 'cpu_usage'."
r' Must be one of \(all, big_only, little_only, exclusive_cores\)')
diff --git a/crosperf/experiment_runner_unittest.py b/crosperf/experiment_runner_unittest.py
index 8747de4..4905975 100755
--- a/crosperf/experiment_runner_unittest.py
+++ b/crosperf/experiment_runner_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
@@ -15,7 +15,7 @@
import time
import unittest
-import mock
+import unittest.mock as mock
import experiment_runner
import experiment_status
@@ -109,7 +109,7 @@
def make_fake_experiment(self):
test_flag.SetTestMode(True)
- experiment_file = ExperimentFile(io.BytesIO(EXPERIMENT_FILE_1))
+ experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1))
experiment = ExperimentFactory().GetExperiment(
experiment_file, working_directory='', log_dir='')
return experiment
@@ -409,7 +409,7 @@
@mock.patch.object(Result, 'CopyResultsTo')
@mock.patch.object(Result, 'CleanUp')
@mock.patch.object(Result, 'FormatStringTop5')
- @mock.patch('__builtin__.open', new_callable=mock.mock_open)
+ @mock.patch('builtins.open', new_callable=mock.mock_open)
def test_store_results(self, mock_open, mock_top5, mock_cleanup, mock_copy,
_mock_text_report, mock_report, mock_writefile,
mock_mkdir, mock_rmdir):
diff --git a/crosperf/flag_test_unittest.py b/crosperf/flag_test_unittest.py
index cb0e59e..1e77c8a 100755
--- a/crosperf/flag_test_unittest.py
+++ b/crosperf/flag_test_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
diff --git a/crosperf/generate_report.py b/crosperf/generate_report.py
index cea95a2..bae365d 100755
--- a/crosperf/generate_report.py
+++ b/crosperf/generate_report.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -104,7 +104,7 @@
}
"""
actually_updated = False
- for bench_results in results.itervalues():
+ for bench_results in results.values():
for platform_results in bench_results:
for i, result in enumerate(platform_results):
# Keep the keys that come earliest when sorted alphabetically.
@@ -127,26 +127,6 @@
return results
-def _ConvertToASCII(obj):
- """Convert an object loaded from JSON to ASCII; JSON gives us unicode."""
-
- # In python 3, we can directly return the unicode loaded from json.
- if sys.version_info.major == 3:
- return obj
- # TODO(zhizhouy): Drop the following code once migrated to python 3.
- # Using something like `object_hook` is insufficient, since it only fires on
- # actual JSON objects. `encoding` fails, too, since the default decoder always
- # uses unicode() to decode strings.
- # pylint: disable=unicode-builtin
- if isinstance(obj, unicode):
- return str(obj)
- if isinstance(obj, dict):
- return {_ConvertToASCII(k): _ConvertToASCII(v) for k, v in obj.iteritems()}
- if isinstance(obj, list):
- return [_ConvertToASCII(v) for v in obj]
- return obj
-
-
def _PositiveInt(s):
i = int(s)
if i < 0:
@@ -279,10 +259,8 @@
def Main(argv):
args = _ParseArgs(argv)
- # JSON likes to load UTF-8; our results reporter *really* doesn't like
- # UTF-8.
with PickInputFile(args.input) as in_file:
- raw_results = _ConvertToASCII(json.load(in_file))
+ raw_results = json.load(in_file)
platform_names = raw_results['platforms']
results = raw_results['data']
diff --git a/crosperf/generate_report_unittest.py b/crosperf/generate_report_unittest.py
index 465db29..e19d469 100755
--- a/crosperf/generate_report_unittest.py
+++ b/crosperf/generate_report_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -12,7 +12,7 @@
import copy
import json
import unittest
-import mock
+import unittest.mock as mock
import generate_report
import results_report
@@ -49,7 +49,7 @@
}
results = generate_report.CountBenchmarks(runs)
expected_results = [('foo', 4), ('bar', 0), ('baz', 3)]
- self.assertEqual(sorted(expected_results), sorted(results))
+ self.assertCountEqual(expected_results, results)
def testCutResultsInPlace(self):
bench_data = {
@@ -82,8 +82,8 @@
bench_data, max_keys=max_keys, complain_on_update=False)
# Cuts should be in-place.
self.assertIs(results, bench_data)
- self.assertEqual(
- sorted(original_bench_data.keys()), sorted(bench_data.keys()))
+ self.assertCountEqual(
+ list(original_bench_data.keys()), list(bench_data.keys()))
for bench_name, original_runs in original_bench_data.items():
bench_runs = bench_data[bench_name]
self.assertEqual(len(original_runs), len(bench_runs))
@@ -135,13 +135,11 @@
self.assertEqual(0, return_code)
self.assertEqual(mock_run_actions.call_count, 1)
ctors = [ctor for ctor, _ in mock_run_actions.call_args[0][0]]
- self.assertEqual(
- sorted(ctors),
- sorted([
- results_report.JSONResultsReport,
- results_report.TextResultsReport,
- results_report.HTMLResultsReport,
- ]))
+ self.assertEqual(ctors, [
+ results_report.JSONResultsReport,
+ results_report.TextResultsReport,
+ results_report.HTMLResultsReport,
+ ])
@mock.patch('generate_report.RunActions')
def testMainSelectsHTMLIfNoReportsGiven(self, mock_run_actions):
diff --git a/crosperf/label.py b/crosperf/label.py
index bebc270..b812261 100644
--- a/crosperf/label.py
+++ b/crosperf/label.py
@@ -93,7 +93,8 @@
if self.image_type == 'local':
self.checksum = ImageChecksummer().Checksum(self, self.log_level)
elif self.image_type == 'trybot':
- self.checksum = hashlib.md5(self.chromeos_image).hexdigest()
+ self.checksum = hashlib.md5(
+ self.chromeos_image.encode('utf-8')).hexdigest()
def _GetImageType(self, chromeos_image):
image_type = None
diff --git a/crosperf/machine_image_manager_unittest.py b/crosperf/machine_image_manager_unittest.py
index 062c185..fbbca7b 100755
--- a/crosperf/machine_image_manager_unittest.py
+++ b/crosperf/machine_image_manager_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -53,13 +53,6 @@
duts.append(MockDut(n))
return duts
- def print_matrix(self, matrix):
- # pylint: disable=expression-not-assigned
- for r in matrix:
- for v in r:
- print('{} '.format('.' if v == ' ' else v)),
- print('')
-
def create_labels_and_duts_from_pattern(self, pattern):
labels = []
duts = []
diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py
index 6ed7e51..7211662 100644
--- a/crosperf/machine_manager.py
+++ b/crosperf/machine_manager.py
@@ -151,9 +151,8 @@
def _GetMD5Checksum(self, ss):
if ss:
- return hashlib.md5(ss).hexdigest()
- else:
- return ''
+ return hashlib.md5(ss.encode('utf-8')).hexdigest()
+ return ''
def _GetMachineID(self):
command = 'dump_vpd_log --full --stdout'
diff --git a/crosperf/machine_manager_unittest.py b/crosperf/machine_manager_unittest.py
index f6a77bb..26eacbd 100755
--- a/crosperf/machine_manager_unittest.py
+++ b/crosperf/machine_manager_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
@@ -13,8 +13,7 @@
import time
import hashlib
import unittest
-
-import mock
+import unittest.mock as mock
import label
import machine_manager
@@ -778,7 +777,7 @@
self.assertEqual(cm.phys_kbytes, 4194304)
mock_run_cmd.return_value = [1, MEMINFO_STRING, '']
- self.assertRaises(cm._GetMemoryInfo)
+ self.assertRaises(Exception, cm._GetMemoryInfo)
@mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommandWOutput')
@mock.patch.object(machine_manager.CrosMachine, 'SetUpChecksumInfo')
@@ -838,7 +837,7 @@
'44:6d:57:20:4a:c5 txqueuelen 1000 (Ethernet)')
mock_run_cmd.return_value = [0, 'invalid hardware config', '']
- self.assertRaises(cm._GetMachineID)
+ self.assertRaises(Exception, cm._GetMachineID)
def test_add_cooldown_waittime(self):
cm = machine_manager.CrosMachine('1.2.3.4.cros', '/usr/local/chromeos',
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index bd80e8a..9806219 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -198,8 +198,8 @@
command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir)
self.ce.RunCommand(command, print_to_console=False)
- command = ('python generate_test_report --no-color --csv %s' %
- (os.path.join('/tmp', os.path.basename(self.temp_dir))))
+ command = ('./generate_test_report --no-color --csv %s' % (os.path.join(
+ '/tmp', os.path.basename(self.temp_dir))))
_, out, _ = self.ce.ChrootRunCommandWOutput(
self.chromeos_root, command, print_to_console=False)
keyvals_dict = {}
@@ -869,7 +869,7 @@
self.suite = suite
self.cwp_dso = cwp_dso
# Read in everything from the cache directory.
- with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f:
+ with open(os.path.join(cache_dir, RESULTS_FILE), 'rb') as f:
self.out = pickle.load(f)
self.err = pickle.load(f)
self.retval = pickle.load(f)
@@ -907,7 +907,7 @@
temp_dir = tempfile.mkdtemp()
# Store to the temp directory.
- with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f:
+ with open(os.path.join(temp_dir, RESULTS_FILE), 'wb') as f:
pickle.dump(self.out, f)
pickle.dump(self.err, f)
pickle.dump(self.retval, f)
@@ -1040,7 +1040,7 @@
self.test_name = test
self.suite = suite
self.cwp_dso = cwp_dso
- with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f:
+ with open(os.path.join(cache_dir, RESULTS_FILE), 'rb') as f:
self.out = pickle.load(f)
self.err = pickle.load(f)
self.retval = pickle.load(f)
@@ -1183,7 +1183,8 @@
if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
checksum = '*'
elif self.label.image_type == 'trybot':
- checksum = hashlib.md5(self.label.chromeos_image).hexdigest()
+ checksum = hashlib.md5(
+ self.label.chromeos_image.encode('utf-8')).hexdigest()
elif self.label.image_type == 'official':
checksum = '*'
else:
@@ -1192,7 +1193,8 @@
if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
image_path_checksum = '*'
else:
- image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
+ image_path_checksum = hashlib.md5(
+ self.chromeos_image.encode('utf-8')).hexdigest()
machine_id_checksum = ''
if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
@@ -1208,7 +1210,7 @@
temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args,
self.run_local)
- test_args_checksum = hashlib.md5(temp_test_args).hexdigest()
+ test_args_checksum = hashlib.md5(temp_test_args.encode('utf-8')).hexdigest()
return (image_path_checksum, self.test_name, str(self.iteration),
test_args_checksum, checksum, machine_checksum, machine_id_checksum,
str(self.CACHE_VERSION))
diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py
index c1c2726..1e7f04a 100755
--- a/crosperf/results_cache_unittest.py
+++ b/crosperf/results_cache_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
@@ -13,8 +13,7 @@
import shutil
import tempfile
import unittest
-
-import mock
+import unittest.mock as mock
import image_checksummer
import machine_manager
@@ -236,13 +235,13 @@
'top5': [2.0],
},
{
- 'cmd': 'sshd',
+ 'cmd': 'spi5',
'cpu_avg': 0.5,
'count': 1,
'top5': [1.0],
},
{
- 'cmd': 'spi5',
+ 'cmd': 'sshd',
'cpu_avg': 0.5,
'count': 1,
'top5': [1.0],
@@ -689,8 +688,7 @@
self.assertEqual(mock_chrootruncmd.call_count, 1)
self.assertEqual(
mock_chrootruncmd.call_args_list[0][0],
- ('/tmp',
- ('python generate_test_report --no-color --csv %s') % TMP_DIR1))
+ ('/tmp', ('./generate_test_report --no-color --csv %s') % TMP_DIR1))
self.assertEqual(mock_getpath.call_count, 1)
self.assertEqual(mock_mkdtemp.call_count, 1)
self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']})
@@ -893,7 +891,7 @@
def test_process_turbostat_results_with_valid_data(self):
"""Normal case when log exists and contains valid data."""
self.result.turbostat_log_file = '/tmp/somelogfile.log'
- with mock.patch('__builtin__.open',
+ with mock.patch('builtins.open',
mock.mock_open(read_data=TURBOSTAT_LOG_OUTPUT)) as mo:
cpustats = self.result.ProcessTurbostatResults()
# Check that the log got opened and data were read/parsed.
@@ -904,7 +902,7 @@
def test_process_turbostat_results_from_empty_file(self):
"""Error case when log exists but file is empty."""
self.result.turbostat_log_file = '/tmp/emptylogfile.log'
- with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo:
+ with mock.patch('builtins.open', mock.mock_open(read_data='')) as mo:
cpustats = self.result.ProcessTurbostatResults()
# Check that the log got opened and parsed successfully and empty data
# returned.
@@ -932,7 +930,7 @@
returned cpustats.
"""
self.result.cpustats_log_file = '/tmp/somelogfile.log'
- with mock.patch('__builtin__.open',
+ with mock.patch('builtins.open',
mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT)) as mo:
cpustats = self.result.ProcessCpustatsResults()
# Check that the log got opened and data were read/parsed.
@@ -949,7 +947,7 @@
returned cpustats.
"""
self.result.cpustats_log_file = '/tmp/somelogfile.log'
- with mock.patch('__builtin__.open',
+ with mock.patch('builtins.open',
mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT)) as mo:
cpustats = self.result.ProcessCpustatsResults()
# Check that the log got opened and data were read/parsed.
@@ -960,7 +958,7 @@
def test_process_cpustats_results_from_empty_file(self):
"""Error case when log exists but file is empty."""
self.result.cpustats_log_file = '/tmp/emptylogfile.log'
- with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo:
+ with mock.patch('builtins.open', mock.mock_open(read_data='')) as mo:
cpustats = self.result.ProcessCpustatsResults()
# Check that the log got opened and parsed successfully and empty data
# returned.
@@ -972,8 +970,7 @@
"""Process top log with valid data."""
self.result.top_log_file = '/tmp/fakelogfile.log'
- with mock.patch('__builtin__.open',
- mock.mock_open(read_data=TOP_LOG)) as mo:
+ with mock.patch('builtins.open', mock.mock_open(read_data=TOP_LOG)) as mo:
topproc = self.result.ProcessTopResults()
# Check that the log got opened and data were read/parsed.
calls = [mock.call('/tmp/fakelogfile.log')]
@@ -983,7 +980,7 @@
def test_process_top_results_from_empty_file(self):
"""Error case when log exists but file is empty."""
self.result.top_log_file = '/tmp/emptylogfile.log'
- with mock.patch('__builtin__.open', mock.mock_open(read_data='')) as mo:
+ with mock.patch('builtins.open', mock.mock_open(read_data='')) as mo:
topcalls = self.result.ProcessTopResults()
# Check that the log got opened and parsed successfully and empty data
# returned.
diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py
index 4879cae..674745f 100644
--- a/crosperf/results_organizer.py
+++ b/crosperf/results_organizer.py
@@ -80,7 +80,7 @@
# pylint: disable=cell-var-from-loop
added_runs = _Repeat(
lambda: _DictWithReturnValues(run_retval, run_pass_fail), max_dup)
- for key, value in run.iteritems():
+ for key, value in run.items():
match = _DUP_KEY_REGEX.match(key)
if not match:
new_run[key] = value
@@ -94,7 +94,7 @@
def _DuplicatePass(result, benchmarks):
"""Properly expands keys like `foo{1}` in `result`."""
- for bench, data in result.iteritems():
+ for bench, data in result.items():
max_dup = _GetMaxDup(data)
# If there's nothing to expand, there's nothing to do.
if not max_dup:
diff --git a/crosperf/results_organizer_unittest.py b/crosperf/results_organizer_unittest.py
index 39a8cce..f259879 100755
--- a/crosperf/results_organizer_unittest.py
+++ b/crosperf/results_organizer_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
diff --git a/crosperf/results_report.py b/crosperf/results_report.py
index 81591db..ff6c4f9 100644
--- a/crosperf/results_report.py
+++ b/crosperf/results_report.py
@@ -100,10 +100,10 @@
def filter_dict(m):
return {
- fn_name: pct for fn_name, pct in m.iteritems() if pct >= event_threshold
+ fn_name: pct for fn_name, pct in m.items() if pct >= event_threshold
}
- return {event: filter_dict(m) for event, m in report.iteritems()}
+ return {event: filter_dict(m) for event, m in report.items()}
class _PerfTable(object):
@@ -186,7 +186,7 @@
iter_counts = benchmark_results.iter_counts
result = benchmark_results.run_keyvals
tables = []
- for bench_name, runs in result.iteritems():
+ for bench_name, runs in result.items():
iterations = iter_counts[bench_name]
ben_table = _GetResultsTableHeader(bench_name, iterations)
@@ -438,7 +438,7 @@
def _GetHTMLCharts(label_names, test_results):
charts = []
- for item, runs in test_results.iteritems():
+ for item, runs in test_results.items():
# Fun fact: label_names is actually *entirely* useless as a param, since we
# never add headers. We still need to pass it anyway.
table = TableGenerator(runs, label_names).GetTable()
@@ -734,7 +734,7 @@
label_names = benchmark_results.label_names
summary_field_defaults = self.summary_field_defaults
final_results = []
- for test, test_results in benchmark_results.run_keyvals.iteritems():
+ for test, test_results in benchmark_results.run_keyvals.items():
for label_name, label_results in zip(label_names, test_results):
for iter_results in label_results:
passed = iter_results.get('retval') == 0
@@ -767,7 +767,7 @@
# Get detailed results.
detail_results = {}
json_results['detailed_results'] = detail_results
- for k, v in iter_results.iteritems():
+ for k, v in iter_results.items():
if k == 'retval' or k == 'PASS' or k == ['PASS'] or v == 'PASS':
continue
diff --git a/crosperf/results_report_unittest.py b/crosperf/results_report_unittest.py
index dfcce72..e03ea43 100755
--- a/crosperf/results_report_unittest.py
+++ b/crosperf/results_report_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2016 The Chromium OS Authors. All rights reserved.
@@ -14,8 +14,8 @@
import io
import os
import unittest
+import unittest.mock as mock
-import mock
import test_flag
from benchmark_run import MockBenchmarkRun
@@ -83,7 +83,7 @@
def MakeMockExperiment(compiler='gcc'):
"""Mocks an experiment using the given compiler."""
- mock_experiment_file = io.BytesIO("""
+ mock_experiment_file = io.StringIO("""
board: x86-alex
remote: 127.0.0.1
locks_dir: /tmp
@@ -312,7 +312,7 @@
# Nothing succeeded; we don't send anything more than what's required.
required_keys = self._GetRequiredKeys(is_experiment=True)
for result in results:
- self.assertItemsEqual(result.iterkeys(), required_keys)
+ self.assertCountEqual(result.keys(), required_keys)
def testJSONReportOutputWithSuccesses(self):
success_keyvals = {
@@ -429,7 +429,7 @@
def testParserParsesRealWorldPerfReport(self):
report = ParseStandardPerfReport(self._ReadRealPerfReport())
- self.assertItemsEqual(['cycles', 'instructions'], sorted(report.keys()))
+ self.assertCountEqual(['cycles', 'instructions'], list(report.keys()))
# Arbitrarily selected known percentages from the perf report.
known_cycles_percentages = {
diff --git a/crosperf/schedv2_unittest.py b/crosperf/schedv2_unittest.py
index 3ccf6fa..7b56d72 100755
--- a/crosperf/schedv2_unittest.py
+++ b/crosperf/schedv2_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2015 The Chromium OS Authors. All rights reserved.
@@ -12,8 +12,7 @@
import functools
import io
import unittest
-
-import mock
+import unittest.mock as mock
import benchmark_run
import test_flag
@@ -81,7 +80,7 @@
Note - we mock out BenchmarkRun in this step.
"""
- experiment_file = ExperimentFile(io.BytesIO(expstr))
+ experiment_file = ExperimentFile(io.StringIO(expstr))
experiment = ExperimentFactory().GetExperiment(
experiment_file, working_directory='', log_dir='')
return experiment
@@ -200,7 +199,7 @@
# The non-cache-hit brs are put into Schedv2._label_brl_map.
self.assertEqual(
functools.reduce(lambda a, x: a + len(x[1]),
- my_schedv2.get_label_map().iteritems(), 0), 30)
+ my_schedv2.get_label_map().items(), 0), 30)
def test_nocachehit(self):
"""Test no cache-hit."""
@@ -218,7 +217,7 @@
# The non-cache-hit brs are put into Schedv2._label_brl_map.
self.assertEqual(
functools.reduce(lambda a, x: a + len(x[1]),
- my_schedv2.get_label_map().iteritems(), 0), 60)
+ my_schedv2.get_label_map().items(), 0), 60)
if __name__ == '__main__':
diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py
index a303a06..12a87ae 100755
--- a/crosperf/settings_factory_unittest.py
+++ b/crosperf/settings_factory_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2017 The Chromium OS Authors. All rights reserved.
diff --git a/crosperf/settings_unittest.py b/crosperf/settings_unittest.py
index e612692..e127552 100755
--- a/crosperf/settings_unittest.py
+++ b/crosperf/settings_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -9,7 +9,7 @@
from __future__ import print_function
import unittest
-import mock
+import unittest.mock as mock
import settings
import settings_factory
diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py
index 579e30b..71ca7e7 100644
--- a/crosperf/suite_runner.py
+++ b/crosperf/suite_runner.py
@@ -21,7 +21,7 @@
# TODO: Need to check whether Skylab is installed and set up correctly.
SKYLAB_PATH = '/usr/local/bin/skylab'
GS_UTIL = 'src/chromium/depot_tools/gsutil.py'
-AUTOTEST_DIR = '~/trunk/src/third_party/autotest/files'
+AUTOTEST_DIR = '/mnt/host/source/src/third_party/autotest/files'
CHROME_MOUNT_DIR = '/tmp/chrome_root'
diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py
index 413822a..7f3a7bc 100755
--- a/crosperf/suite_runner_unittest.py
+++ b/crosperf/suite_runner_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
@@ -14,7 +14,7 @@
import time
import unittest
-import mock
+import unittest.mock as mock
import suite_runner
import label
@@ -227,7 +227,7 @@
self.assertEqual(
args_list[1],
('/usr/bin/test_that --autotest_dir '
- '~/trunk/src/third_party/autotest/files --fast '
+ '/mnt/host/source/src/third_party/autotest/files --fast '
"--board=lumpy --args=' run_local=False test=octane "
'dut_config=\'"\'"\'{"turbostat": true, "top_interval": 3}\'"\'"\' '
'profiler=custom_perf profiler_args=\'"\'"\'record -a -e '
diff --git a/crosperf/test_cache/compare_output/results.txt b/crosperf/test_cache/compare_output/results.txt
index db6803c..592e716 100644
--- a/crosperf/test_cache/compare_output/results.txt
+++ b/crosperf/test_cache/compare_output/results.txt
Binary files differ
diff --git a/crosperf/translate_xbuddy.py b/crosperf/translate_xbuddy.py
old mode 100644
new mode 100755
index ca7058e..80187f9
--- a/crosperf/translate_xbuddy.py
+++ b/crosperf/translate_xbuddy.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
diff --git a/file_lock_machine.py b/file_lock_machine.py
index 8493b08..5bba443 100755
--- a/file_lock_machine.py
+++ b/file_lock_machine.py
@@ -1,8 +1,12 @@
-#!/usr/bin/env python2
-#
-# Copyright 2010 Google Inc. All Rights Reserved.
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Script to lock/unlock machines."""
+from __future__ import division
from __future__ import print_function
__author__ = 'asharif@google.com (Ahmad Sharif)'
@@ -24,8 +28,8 @@
# The locks file directory REQUIRES that 'group' only has read/write
# privileges and 'world' has no privileges. So the mask must be
-# '0027': 0777 - 0027 = 0750.
-LOCK_MASK = 0027
+# '0o27': 0o777 - 0o27 = 0o750.
+LOCK_MASK = 0o27
def FileCheckName(name):
@@ -37,7 +41,8 @@
fd = open(file_name, 'a')
try:
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- except IOError:
+ except IOError as e:
+ logger.GetLogger().LogError(e)
raise
return fd
@@ -80,9 +85,12 @@
def __str__(self):
return ' '.join([
- 'Owner: %s' % self.owner, 'Exclusive: %s' % self.exclusive,
- 'Counter: %s' % self.counter, 'Time: %s' % self.time,
- 'Reason: %s' % self.reason, 'Auto: %s' % self.auto
+ 'Owner: %s' % self.owner,
+ 'Exclusive: %s' % self.exclusive,
+ 'Counter: %s' % self.counter,
+ 'Time: %s' % self.time,
+ 'Reason: %s' % self.reason,
+ 'Auto: %s' % self.auto,
])
@@ -97,6 +105,12 @@
self._file = None
self._description = None
+ self.exclusive = None
+ self.auto = None
+ self.reason = None
+ self.time = None
+ self.owner = None
+
def getDescription(self):
return self._description
@@ -118,12 +132,12 @@
seconds=int(time.time() - file_lock.getDescription().time))
elapsed_time = '%s ago' % elapsed_time
lock_strings.append(
- stringify_fmt %
- (os.path.basename(file_lock.getFilePath),
- file_lock.getDescription().owner,
- file_lock.getDescription().exclusive,
- file_lock.getDescription().counter, elapsed_time,
- file_lock.getDescription().reason, file_lock.getDescription().auto))
+ stringify_fmt % (os.path.basename(file_lock.getFilePath),
+ file_lock.getDescription().owner,
+ file_lock.getDescription().exclusive,
+ file_lock.getDescription().counter, elapsed_time,
+ file_lock.getDescription().reason,
+ file_lock.getDescription().auto))
table = '\n'.join(lock_strings)
return '\n'.join([header, table])
@@ -286,7 +300,7 @@
sleep = timeout / 10
while True:
locked = self.Lock(exclusive, reason)
- if locked or not timeout >= 0:
+ if locked or timeout < 0:
break
print('Lock not acquired for {0}, wait {1} seconds ...'.format(
self._name, sleep))
diff --git a/file_lock_machine_test.py b/file_lock_machine_test.py
index 340f914..bc20a88 100755
--- a/file_lock_machine_test.py
+++ b/file_lock_machine_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
diff --git a/image_chromeos.py b/image_chromeos.py
index 726b565..d5c404d 100755
--- a/image_chromeos.py
+++ b/image_chromeos.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 The Chromium OS Authors. All rights reserved.
@@ -260,7 +260,7 @@
command = ' '.join(cros_flash_args)
# Workaround for crosbug.com/35684.
- os.chmod(misc.GetChromeOSKeyFile(options.chromeos_root), 0600)
+ os.chmod(misc.GetChromeOSKeyFile(options.chromeos_root), 0o600)
if log_level == 'average':
cmd_executer.SetLogLevel('verbose')
@@ -369,7 +369,7 @@
def GetImageMountCommand(image, rootfs_mp, stateful_mp):
image_dir = os.path.dirname(image)
image_file = os.path.basename(image)
- mount_command = ('cd ~/trunk/src/scripts &&'
+ mount_command = ('cd /mnt/host/source/src/scripts &&'
'./mount_gpt_image.sh --from=%s --image=%s'
' --safe --read_only'
' --rootfs_mountpt=%s'
@@ -462,7 +462,7 @@
## Safely ignore.
l.LogWarning('Failed to remount partition as rw, '
'probably the image was not built with '
- "\"--noenable_rootfs_verification\", "
+ '"--noenable_rootfs_verification", '
'you can safely ignore this.')
else:
l.LogOutput('Re-mounted partition as writable.')
diff --git a/lock_machine.py b/lock_machine.py
index 87230b7..244edfb 100755
--- a/lock_machine.py
+++ b/lock_machine.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 The Chromium OS Authors. All rights reserved.
@@ -298,7 +298,7 @@
cros_machine = cros_machine + '.cros'
self.machines = [
- m for m in self.machines if m != cros_machine and m != machine
+ m for m in self.machines if m not in (cros_machine, machine)
]
def CheckMachineLocks(self, machine_states, cmd):
@@ -316,7 +316,7 @@
Raises:
DontOwnLock: The lock on a requested machine is owned by someone else.
"""
- for k, state in machine_states.iteritems():
+ for k, state in machine_states.items():
if cmd == 'unlock':
if not state['locked']:
self.logger.LogWarning('Attempt to unlock already unlocked machine '
