Reland "[PY3][Autotest] First go at setting test_that flag for py3"
This is a reland of 39fc2292aaeeafff91add2465bf5f6edc7299f38
Original change's description:
> [PY3][Autotest] First go at setting test_that flag for py3
>
> NOTE: its gaurded with a force back to py2 for now. But this is roughly
> what I think I am going for. There is a bit of debate I am having still
> about turning on the "no restart" flag going forward. IMO the launching
> script, (autoserv, test that, or other bins) should be able to control
> the version they want to run in going forward. 1 pager about this likely
> coming
>
> BUG=chromium:990593
> TEST=test_that with the py_version flag in both 2/3 and verfied tests
> are in py2, as well as without the flag
>
> Change-Id: If34d6866061f93e72728a3e49c02e2faf73ae628
> Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/2638720
> Reviewed-by: Greg Edelston <gredelston@google.com>
> Commit-Queue: Derek Beckett <dbeckett@chromium.org>
> Tested-by: Derek Beckett <dbeckett@chromium.org>
Bug: chromium:990593
Change-Id: I498b734e00bb57bc6ec0e38ff686583544444b61
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/2690764
Reviewed-by: Greg Edelston <gredelston@google.com>
Commit-Queue: Derek Beckett <dbeckett@chromium.org>
Tested-by: Derek Beckett <dbeckett@chromium.org>
(cherry picked from commit 80b7f446e66e74a2dbba01ef53b6c074bac0a1ca)
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/2715647
Reviewed-by: C Shapiro <shapiroc@chromium.org>
Auto-Submit: Derek Beckett <dbeckett@chromium.org>
diff --git a/client/common_lib/check_version.py b/client/common_lib/check_version.py
index 68ee16c..7600f66 100644
--- a/client/common_lib/check_version.py
+++ b/client/common_lib/check_version.py
@@ -1,19 +1,28 @@
# This file must use Python 1.5 syntax.
import glob
+import logging
import os
import sys
+PY_GLOBS = {
+ 3: ['/usr/bin/python3*', '/usr/local/bin/python3*'],
+ 2: ['/usr/bin/python2*', '/usr/local/bin/python2*']
+}
+
class check_python_version:
- def __init__(self):
+ def __init__(self, desired_version=2):
# In order to ease the migration to Python3, disable the restart logic
# when AUTOTEST_NO_RESTART is set. This makes it possible to run
# autotest locally as Python3 before any other environment is switched
# to Python3.
if os.getenv("AUTOTEST_NO_RESTART"):
return
-
+ self.desired_version = desired_version
+ if self.desired_version == 3:
+ logging.warning("Python3 not not ready yet. Swapping to Python 2.")
+ self.desired_version = 2
# The change to prefer 2.4 really messes up any systems which have both
# the new and old version of Python, but where the newer is default.
# This is because packages, libraries, etc are all installed into the
@@ -23,26 +32,22 @@
# runs) 'import common' it restarts my shell. Overall, the change was
# fairly annoying for me (and I can't get around having 2.4 and 2.5
# installed with 2.5 being default).
- if sys.version_info.major >= 3:
+ if sys.version_info.major != self.desired_version:
try:
# We can't restart when running under mod_python.
from mod_python import apache
except ImportError:
self.restart()
-
- PYTHON_BIN_GLOB_STRINGS = ['/usr/bin/python2*', '/usr/local/bin/python2*']
-
-
def find_desired_python(self):
"""Returns the path of the desired python interpreter."""
# CrOS only ever has Python 2.7 available, so pick whatever matches.
+ pyv_strings = PY_GLOBS[self.desired_version]
pythons = []
- for glob_str in self.PYTHON_BIN_GLOB_STRINGS:
+ for glob_str in pyv_strings:
pythons.extend(glob.glob(glob_str))
return pythons[0]
-
def restart(self):
python = self.find_desired_python()
sys.stderr.write('NOTE: %s switching to %s\n' %
diff --git a/client/setup_modules.py b/client/setup_modules.py
index 9cb59eb..82e0924 100644
--- a/client/setup_modules.py
+++ b/client/setup_modules.py
@@ -1,14 +1,71 @@
-__author__ = "jadmanski@google.com (John Admanski)"
-
-import os, sys
+import os
+import re
+import sys
# This must run on Python versions less than 2.4.
dirname = os.path.dirname(sys.modules[__name__].__file__)
-common_dir = os.path.abspath(os.path.join(dirname, "common_lib"))
+common_dir = os.path.abspath(os.path.join(dirname, 'common_lib'))
sys.path.insert(0, common_dir)
import check_version
sys.path.pop(0)
-check_version.check_python_version()
+
+
+def _get_pyversion_from_args():
+ """Extract, format, & pop the current py_version from args, if provided."""
+ py_version = 2
+ py_version_re = re.compile(r'--py_version=(\w+)\b')
+
+ version_found = False
+ for i, arg in enumerate(sys.argv):
+ if not arg.startswith('--py_version'):
+ continue
+ result = py_version_re.search(arg)
+ if result:
+ if version_found:
+ raise ValueError('--py_version may only be specified once.')
+ py_version = result.group(1)
+ version_found = True
+ if py_version not in ('2', '3'):
+ raise ValueError('Python version must be "2" or "3".')
+
+ # Remove the arg so other argparsers don't get grumpy.
+ sys.argv.pop(i)
+
+ return py_version
+
+
+def _desired_version():
+ """
+ Returns desired python version.
+
+ If the PY_VERSION env var is set, just return that. This is the case
+ when autoserv kicks of autotest on the server side via a job.run(), or
+ a process created a subprocess.
+
+ Otherwise, parse & pop the sys.argv for the '--py_version' flag. If no
+ flag is set, default to python 2 (for now).
+
+ """
+ # Even if the arg is in the env vars, we will attempt to get it from the
+ # args, so that it can be popped prior to other argparsers hitting.
+ py_version = _get_pyversion_from_args()
+
+ if os.getenv('PY_VERSION'):
+ return int(os.getenv('PY_VERSION'))
+
+ os.environ['PY_VERSION'] = str(py_version)
+ return int(py_version)
+
+
+desired_version = _desired_version()
+if desired_version == sys.version_info.major:
+ os.environ['AUTOTEST_NO_RESTART'] = 'True'
+else:
+ # There are cases were this can be set (ie by test_that), but a subprocess
+ # is launched in the incorrect version.
+ if os.getenv('AUTOTEST_NO_RESTART'):
+ del os.environ['AUTOTEST_NO_RESTART']
+ check_version.check_python_version(desired_version)
import glob, traceback, types
@@ -22,7 +79,7 @@
def _create_module_and_parents(name):
"""Create a module, and all the necessary parents"""
- parts = name.split(".")
+ parts = name.split('.')
# first create the top-level module
parent = _create_module(parts[0])
created_parts = [parts[0]]
@@ -33,7 +90,7 @@
module = types.ModuleType(child_name)
setattr(parent, child_name, module)
created_parts.append(child_name)
- sys.modules[".".join(created_parts)] = module
+ sys.modules['.'.join(created_parts)] = module
parent = module
@@ -45,11 +102,11 @@
full_name = os.path.join(path, filename)
if not os.path.isdir(full_name):
continue # skip files
- if "." in filename:
- continue # if "." is in the name it's not a valid package name
+ if '.' in filename:
+ continue # if '.' is in the name it's not a valid package name
if not os.access(full_name, os.R_OK | os.X_OK):
continue # need read + exec access to make a dir importable
- if "__init__.py" in os.listdir(full_name):
+ if '__init__.py' in os.listdir(full_name):
names.append(filename)
# import all the packages and insert them into 'parent_module'
sys.path.insert(0, path)
@@ -58,7 +115,7 @@
# add the package to the parent
parent_module = sys.modules[parent_module_name]
setattr(parent_module, name, module)
- full_name = parent_module_name + "." + name
+ full_name = parent_module_name + '.' + name
sys.modules[full_name] = module
# restore the system path
sys.path.pop(0)
@@ -91,10 +148,10 @@
def _monkeypatch_logging_handle_error():
# Hack out logging.py*
- logging_py = os.path.join(os.path.dirname(__file__), "common_lib",
- "logging.py*")
+ logging_py = os.path.join(os.path.dirname(__file__), 'common_lib',
+ 'logging.py*')
if glob.glob(logging_py):
- os.system("rm -f %s" % logging_py)
+ os.system('rm -f %s' % logging_py)
# Monkey patch our own handleError into the logging module's StreamHandler.
# A nicer way of doing this -might- be to have our own logging module define
@@ -134,6 +191,6 @@
# running as a client.
# This is primarily for the benefit of frontend and tko so that they
# may use libraries other than those available as system packages.
- sys.path.insert(0, os.path.join(base_path, "site-packages"))
+ sys.path.insert(0, os.path.join(base_path, 'site-packages'))
_monkeypatch_logging_handle_error()
diff --git a/server/autoserv_parser.py b/server/autoserv_parser.py
index 6733d70..0216100 100644
--- a/server/autoserv_parser.py
+++ b/server/autoserv_parser.py
@@ -238,6 +238,12 @@
' enabled. The default value is provided via the global'
' config setting for AUTOSERV/container_base_name.'
)
+ self.parser.add_argument('--py_version',
+ action='store',
+ dest='py_version',
+ default='2',
+ type=str,
+ choices=['2', '3'])
#
# Warning! Please read before adding any new arguments!
diff --git a/server/autoserv_utils.py b/server/autoserv_utils.py
index 76e8601..f609fdc 100644
--- a/server/autoserv_utils.py
+++ b/server/autoserv_utils.py
@@ -71,7 +71,16 @@
"""
script_name = 'virtualenv_autoserv' if use_virtualenv else 'autoserv'
- command = [os.path.join(autoserv_directory, script_name)]
+
+ full_script_path = os.path.join(autoserv_directory, script_name)
+
+ # virtualenv_autoserv is a `POSIX shell script, ASCII text executable`.
+ # Calling with `sys.executable` would fail because python doesn't
+ # interpret shebangs itself.
+ if use_virtualenv:
+ command = [full_script_path]
+ else:
+ command = [sys.executable, full_script_path]
if write_pidfile:
command.append('-p')
@@ -131,4 +140,8 @@
if in_lab:
command.extend(['--lab', 'True'])
+ py_version = os.getenv('PY_VERSION')
+ if py_version:
+ command.extend(['--py_version', py_version])
+
return command + extra_args
diff --git a/server/autotest.py b/server/autotest.py
index d195bd0..7a35fba 100644
--- a/server/autotest.py
+++ b/server/autotest.py
@@ -55,6 +55,11 @@
LOG_BUFFER_SIZE_BYTES = 64
+def _set_py_version():
+ """Return the py_version flag obtained from the set environmental var."""
+ return '--py_version=%s' % int(os.getenv('PY_VERSION'))
+
+
class AutodirNotFoundError(Exception):
"""No Autotest installation could be found."""
@@ -735,23 +740,36 @@
def get_background_cmd(self, section):
- cmd = ['nohup', os.path.join(self.autodir, 'bin/autotest_client')]
+ cmd = [
+ 'nohup',
+ os.path.join(self.autodir, 'bin/autotest_client'),
+ _set_py_version()
+ ]
cmd += self.get_base_cmd_args(section)
cmd += ['>/dev/null', '2>/dev/null', '&']
return ' '.join(cmd)
def get_daemon_cmd(self, section, monitor_dir):
- cmd = ['nohup', os.path.join(self.autodir, 'bin/autotestd'),
- monitor_dir, '-H autoserv']
+ cmd = [
+ 'nohup',
+ os.path.join(self.autodir, 'bin/autotestd'), monitor_dir,
+ '-H autoserv',
+ _set_py_version()
+ ]
cmd += self.get_base_cmd_args(section)
cmd += ['>/dev/null', '2>/dev/null', '&']
return ' '.join(cmd)
def get_monitor_cmd(self, monitor_dir, stdout_read, stderr_read):
- cmd = [os.path.join(self.autodir, 'bin', 'autotestd_monitor'),
- monitor_dir, str(stdout_read), str(stderr_read)]
+ cmd = [
+ os.path.join(self.autodir, 'bin', 'autotestd_monitor'),
+ monitor_dir,
+ str(stdout_read),
+ str(stderr_read),
+ _set_py_version()
+ ]
return ' '.join(cmd)
@@ -1419,8 +1437,8 @@
server_package = os.path.join(self.job.pkgmgr.pkgmgr_dir,
'packages', pkg_name)
if os.path.exists(server_package):
- self.host.send_file(server_package, remote_dest)
- return
+ self.host.send_file(server_package, remote_dest)
+ return
except error.AutoservRunError:
msg = ("Package %s could not be sent from the package cache." %
