[Autotest][PY3] Migrating/Refactoring control_data.py for py2/3
compiler isn't present on python3, but AST is. Reworked control_data.py
to use this instead. Tested using unittest, dummy_Pass, and others. Ran
it both in python2 and 3 with success.
BUG=chromium:990593
TEST=dummy_Pass(in python2 & 3), control_data_unittest
Change-Id: Ifae1833af0c617b634279f77bcde83febfb22b75
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/2508464
Reviewed-by: Seewai Fu <seewaifu@google.com>
Reviewed-by: Greg Edelston <gredelston@google.com>
Commit-Queue: Derek Beckett <dbeckett@chromium.org>
Tested-by: Derek Beckett <dbeckett@chromium.org>
diff --git a/client/common_lib/control_data.py b/client/common_lib/control_data.py
index de8ea83..54d4f4f 100644
--- a/client/common_lib/control_data.py
+++ b/client/common_lib/control_data.py
@@ -6,12 +6,7 @@
from __future__ import division
from __future__ import print_function
-import warnings
-with warnings.catch_warnings():
- # The 'compiler' module is gone in Python 3.0. Let's not say
- # so in every log file.
- warnings.simplefilter("ignore", DeprecationWarning)
- import compiler
+import ast
import logging
import textwrap
import re
@@ -21,6 +16,7 @@
from autotest_lib.client.common_lib import global_config
from autotest_lib.client.common_lib import priorities
+
REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type'])
OBSOLETE_VARS = set(['experimental'])
@@ -323,16 +319,19 @@
def _extract_const(expr):
- assert(expr.__class__ == compiler.ast.Const)
- assert(expr.value.__class__ in (str, int, float, six.text_type))
- return str(expr.value).strip()
+ assert (expr.__class__ == ast.Str)
+ if six.PY2:
+ assert (expr.s.__class__ in (str, int, float, unicode))
+ else:
+ assert (expr.s.__class__ in (str, int, float))
+ return str(expr.s).strip()
def _extract_dict(expr):
- assert(expr.__class__ == compiler.ast.Dict)
- assert(expr.items.__class__ == list)
+ assert (expr.__class__ == ast.Dict)
+ assert (expr.keys.__class__ == list)
cf_dict = {}
- for key, value in expr.items:
+ for key, value in zip(expr.keys, expr.values):
try:
key = _extract_const(key)
val = _extract_expression(value)
@@ -344,9 +343,9 @@
def _extract_list(expr):
- assert(expr.__class__ == compiler.ast.List)
+ assert (expr.__class__ == ast.List)
list_values = []
- for value in expr.nodes:
+ for value in expr.elts:
try:
list_values.append(_extract_expression(value))
except (AssertionError, ValueError):
@@ -355,34 +354,38 @@
def _extract_name(expr):
- assert(expr.__class__ == compiler.ast.Name)
- assert(expr.name in ('False', 'True', 'None'))
- return str(expr.name)
+ assert (expr.__class__ == ast.Name)
+ assert (expr.id in ('False', 'True', 'None'))
+ return str(expr.id)
def _extract_expression(expr):
- if expr.__class__ == compiler.ast.Const:
+ if expr.__class__ == ast.Str:
return _extract_const(expr)
- if expr.__class__ == compiler.ast.Name:
+ if expr.__class__ == ast.Name:
return _extract_name(expr)
- if expr.__class__ == compiler.ast.Dict:
+ if expr.__class__ == ast.Dict:
return _extract_dict(expr)
- if expr.__class__ == compiler.ast.List:
+ if expr.__class__ == ast.List:
return _extract_list(expr)
+ if expr.__class__ == ast.Num:
+ return expr.n
+ if six.PY3 and expr.__class__ == ast.NameConstant:
+ return expr.value
+ if six.PY3 and expr.__class__ == ast.Constant:
+ try:
+ return expr.value.strip()
+ except Exception:
+ return expr.value
raise ValueError('Unknown rval %s' % expr)
def _extract_assignment(n):
- assert(n.__class__ == compiler.ast.Assign)
- assert(n.nodes.__class__ == list)
- assert(len(n.nodes) == 1)
- assert(n.nodes[0].__class__ == compiler.ast.AssName)
- assert(n.nodes[0].flags.__class__ == str)
- assert(n.nodes[0].name.__class__ == str)
-
- val = _extract_expression(n.expr)
- key = n.nodes[0].name.lower()
-
+ assert (n.__class__ == ast.Assign)
+ assert (len(n.targets) == 1)
+ assert (n.targets[0].__class__ == ast.Name)
+ val = _extract_expression(n.value)
+ key = n.targets[0].id.lower()
return (key, val)
@@ -396,7 +399,7 @@
"""
try:
- mod = compiler.parse(control)
+ mod = ast.parse(control)
except SyntaxError as e:
logging.error('Syntax error (%s) while parsing control string:', e)
lines = control.split('\n')
@@ -408,7 +411,8 @@
def parse_control(path, raise_warnings=False):
try:
- mod = compiler.parseFile(path)
+ with open(path, 'r') as r:
+ mod = ast.parse(r.read())
except SyntaxError as e:
raise ControlVariableException("Error parsing %s because %s" %
(path, e))
@@ -424,22 +428,20 @@
try:
key, val = _extract_assignment(node)
variables[key] = val
- except (AssertionError, ValueError):
+ except (AssertionError, ValueError) as e:
pass
def finish_parse(mod, path, raise_warnings):
- assert(mod.__class__ == compiler.ast.Module)
- assert(mod.node.__class__ == compiler.ast.Stmt)
- assert(mod.node.nodes.__class__ == list)
+ assert (mod.__class__ == ast.Module)
+ assert (mod.body.__class__ == list)
variables = {}
injection_variables = {}
- for n in mod.node.nodes:
- if (n.__class__ == compiler.ast.Function and
- re.match('step\d+', n.name)):
+ for n in mod.body:
+ if (n.__class__ == ast.FunctionDef and re.match('step\d+', n.name)):
vars_in_step = {}
- for sub_node in n.code.nodes:
+ for sub_node in n.body:
_try_extract_assignment(sub_node, vars_in_step)
if vars_in_step:
# Empty the vars collection so assignments from multiple steps
diff --git a/client/common_lib/control_data_unittest.py b/client/common_lib/control_data_unittest.py
index 2089e61..a96d1ad 100755
--- a/client/common_lib/control_data_unittest.py
+++ b/client/common_lib/control_data_unittest.py
@@ -6,14 +6,16 @@
from __future__ import print_function
import json
-import os, unittest
+import os
import six
from six.moves import range
+import unittest
import common
from autotest_lib.client.common_lib import control_data, autotemp
+
ControlData = control_data.ControlData
CONTROL = """
@@ -86,7 +88,7 @@
def setUp(self):
self.control_tmp = autotemp.tempfile(unique_id='control_unit',
text=True)
- os.write(self.control_tmp.fd, CONTROL)
+ os.write(self.control_tmp.fd, str.encode(CONTROL))
def tearDown(self):
@@ -95,21 +97,20 @@
def test_parse_control(self):
cd = control_data.parse_control(self.control_tmp.name, True)
- self.assertEquals(cd.author, "Author")
- self.assertEquals(cd.dependencies, set(['console', 'power']))
- self.assertEquals(cd.doc, "doc stuff")
- self.assertEquals(cd.experimental, False)
- self.assertEquals(cd.name, "nAmE")
- self.assertEquals(cd.run_verify, False)
- self.assertEquals(cd.sync_count, 2)
- self.assertEquals(cd.time, "short")
- self.assertEquals(cd.test_class, "kernel")
- self.assertEquals(cd.test_category, "stress")
- self.assertEquals(cd.test_type, "client")
- self.assertEquals(cd.require_ssp, False)
- self.assertEquals(cd.attributes, set(["suite:smoke","suite:bvt"]))
- self.assertEquals(cd.suite,
- "bvt,smoke,suite-listed-only-in-suite-line")
+ self.assertEqual(cd.author, "Author")
+ self.assertEqual(cd.dependencies, set(['console', 'power']))
+ self.assertEqual(cd.doc, "doc stuff")
+ self.assertEqual(cd.experimental, False)
+ self.assertEqual(cd.name, "nAmE")
+ self.assertEqual(cd.run_verify, False)
+ self.assertEqual(cd.sync_count, 2)
+ self.assertEqual(cd.time, "short")
+ self.assertEqual(cd.test_class, "kernel")
+ self.assertEqual(cd.test_category, "stress")
+ self.assertEqual(cd.test_type, "client")
+ self.assertEqual(cd.require_ssp, False)
+ self.assertEqual(cd.attributes, set(["suite:smoke", "suite:bvt"]))
+ self.assertEqual(cd.suite, "bvt,smoke,suite-listed-only-in-suite-line")
class ParseWrappedControlTest(unittest.TestCase):
@@ -117,7 +118,7 @@
def setUp(self):
self.control_tmp = autotemp.tempfile(unique_id='wrapped_control_unit',
text=True)
- os.write(self.control_tmp.fd, WRAPPED_CONTROL)
+ os.write(self.control_tmp.fd, str.encode(WRAPPED_CONTROL))
def tearDown(self):
@@ -126,22 +127,21 @@
def test_parse_control(self):
cd = control_data.parse_control(self.control_tmp.name, True)
- self.assertEquals(cd.author, "Author")
- self.assertEquals(cd.dependencies, set(['console', 'power']))
- self.assertEquals(cd.doc, "doc stuff")
- self.assertEquals(cd.experimental, False)
- self.assertEquals(cd.name, "nAmE")
- self.assertEquals(cd.run_verify, False)
- self.assertEquals(cd.sync_count, 2)
- self.assertEquals(cd.time, "short")
- self.assertEquals(cd.test_class, "kernel")
- self.assertEquals(cd.test_category, "stress")
- self.assertEquals(cd.test_type, "client")
- self.assertEquals(cd.require_ssp, False)
- self.assertEquals(cd.attributes, set(["suite:smoke","suite:bvt"]))
- self.assertEquals(cd.suite,
- "bvt,smoke,suite-listed-only-in-suite-line")
- self.assertEquals(cd.max_result_size_KB, 20000)
+ self.assertEqual(cd.author, "Author")
+ self.assertEqual(cd.dependencies, set(['console', 'power']))
+ self.assertEqual(cd.doc, "doc stuff")
+ self.assertEqual(cd.experimental, False)
+ self.assertEqual(cd.name, "nAmE")
+ self.assertEqual(cd.run_verify, False)
+ self.assertEqual(cd.sync_count, 2)
+ self.assertEqual(cd.time, "short")
+ self.assertEqual(cd.test_class, "kernel")
+ self.assertEqual(cd.test_category, "stress")
+ self.assertEqual(cd.test_type, "client")
+ self.assertEqual(cd.require_ssp, False)
+ self.assertEqual(cd.attributes, set(["suite:smoke", "suite:bvt"]))
+ self.assertEqual(cd.suite, "bvt,smoke,suite-listed-only-in-suite-line")
+ self.assertEqual(cd.max_result_size_KB, 20000)
class ParseControlFileBugTemplate(unittest.TestCase):
@@ -190,7 +190,8 @@
def test_bug_template_parsing(self):
"""Basic parsing test for a bug templates in a test control file."""
- os.write(self.control_tmp.fd, self.insert_bug_template(CONTROL))
+ os.write(self.control_tmp.fd,
+ str.encode(self.insert_bug_template(CONTROL)))
cd = control_data.parse_control(self.control_tmp.name, True)
self.verify_bug_template(cd.bug_template)
@@ -198,7 +199,8 @@
def test_bug_template_list(self):
"""Test that lists in the bug template can handle other datatypes."""
self.bug_template['labels'].append({'foo': 'bar'})
- os.write(self.control_tmp.fd, self.insert_bug_template(CONTROL))
+ os.write(self.control_tmp.fd,
+ str.encode(self.insert_bug_template(CONTROL)))
cd = control_data.parse_control(self.control_tmp.name, True)
self.verify_bug_template(cd.bug_template)
@@ -206,7 +208,8 @@
def test_bad_template(self):
"""Test that a bad bug template doesn't result in a bad control data."""
self.bug_template = 'foobarbug_template'
- os.write(self.control_tmp.fd, self.insert_bug_template(CONTROL))
+ os.write(self.control_tmp.fd,
+ str.encode(self.insert_bug_template(CONTROL)))
cd = control_data.parse_control(self.control_tmp.name, True)
self.assertFalse(hasattr(cd, 'bug_template'))
@@ -224,13 +227,13 @@
def test_bool(self):
cd = ControlData({}, 'filename')
cd._set_bool('foo', 'False')
- self.assertEquals(cd.foo, False)
+ self.assertEqual(cd.foo, False)
cd._set_bool('foo', True)
- self.assertEquals(cd.foo, True)
+ self.assertEqual(cd.foo, True)
cd._set_bool('foo', 'FALSE')
- self.assertEquals(cd.foo, False)
+ self.assertEqual(cd.foo, False)
cd._set_bool('foo', 'true')
- self.assertEquals(cd.foo, True)
+ self.assertEqual(cd.foo, True)
self.assertRaises(ValueError, cd._set_bool, 'foo', '')
self.assertRaises(ValueError, cd._set_bool, 'foo', 1)
self.assertRaises(ValueError, cd._set_bool, 'foo', [])
@@ -240,11 +243,11 @@
def test_int(self):
cd = ControlData({}, 'filename')
cd._set_int('foo', 0)
- self.assertEquals(cd.foo, 0)
+ self.assertEqual(cd.foo, 0)
cd._set_int('foo', '0')
- self.assertEquals(cd.foo, 0)
+ self.assertEqual(cd.foo, 0)
cd._set_int('foo', '-1', min=-2, max=10)
- self.assertEquals(cd.foo, -1)
+ self.assertEqual(cd.foo, -1)
self.assertRaises(ValueError, cd._set_int, 'foo', 0, min=1)
self.assertRaises(ValueError, cd._set_int, 'foo', 1, max=0)
self.assertRaises(ValueError, cd._set_int, 'foo', 'x')
@@ -255,40 +258,40 @@
def test_set(self):
cd = ControlData({}, 'filename')
cd._set_set('foo', 'a')
- self.assertEquals(cd.foo, set(['a']))
+ self.assertEqual(cd.foo, set(['a']))
cd._set_set('foo', 'a,b,c')
- self.assertEquals(cd.foo, set(['a', 'b', 'c']))
+ self.assertEqual(cd.foo, set(['a', 'b', 'c']))
cd._set_set('foo', ' a , b , c ')
- self.assertEquals(cd.foo, set(['a', 'b', 'c']))
+ self.assertEqual(cd.foo, set(['a', 'b', 'c']))
cd._set_set('foo', None)
- self.assertEquals(cd.foo, set(['None']))
+ self.assertEqual(cd.foo, set(['None']))
def test_string(self):
cd = ControlData({}, 'filename')
cd._set_string('foo', 'a')
- self.assertEquals(cd.foo, 'a')
+ self.assertEqual(cd.foo, 'a')
cd._set_string('foo', 'b')
- self.assertEquals(cd.foo, 'b')
+ self.assertEqual(cd.foo, 'b')
cd._set_string('foo', 'B')
- self.assertEquals(cd.foo, 'B')
+ self.assertEqual(cd.foo, 'B')
cd._set_string('foo', 1)
- self.assertEquals(cd.foo, '1')
+ self.assertEqual(cd.foo, '1')
cd._set_string('foo', None)
- self.assertEquals(cd.foo, 'None')
+ self.assertEqual(cd.foo, 'None')
cd._set_string('foo', [])
- self.assertEquals(cd.foo, '[]')
+ self.assertEqual(cd.foo, '[]')
def test_option(self):
options = ['a', 'b']
cd = ControlData({}, 'filename')
cd._set_option('foo', 'a', options)
- self.assertEquals(cd.foo, 'a')
+ self.assertEqual(cd.foo, 'a')
cd._set_option('foo', 'b', options)
- self.assertEquals(cd.foo, 'b')
+ self.assertEqual(cd.foo, 'b')
cd._set_option('foo', 'B', options)
- self.assertEquals(cd.foo, 'B')
+ self.assertEqual(cd.foo, 'B')
self.assertRaises(ValueError, cd._set_option,
'foo', 'x', options)
self.assertRaises(ValueError, cd._set_option,
@@ -302,7 +305,7 @@
def test_set_attributes(self):
cd = ControlData({}, 'filename')
cd.set_attributes('suite:bvt')
- self.assertEquals(cd.attributes, set(['suite:bvt']))
+ self.assertEqual(cd.attributes, set(['suite:bvt']))
def test_get_test_time_index(self):
