blob: cfcac42f0ba8ff084fd336087ec59d6554b6c61d [file] [log] [blame]
# pylint: disable=missing-docstring
import os
import re
import common
from autotest_lib.tko import models
from autotest_lib.tko import status_lib
from autotest_lib.tko import utils as tko_utils
from autotest_lib.tko.parsers import base
class NoHostnameError(Exception):
class BoardLabelError(Exception):
class job(models.job):
def __init__(self, dir):
job_dict = job.load_from_dir(dir)
super(job, self).__init__(dir, **job_dict)
def load_from_dir(cls, dir):
keyval = cls.read_keyval(dir)
user = keyval.get("user", None)
label = keyval.get("label", None)
queued_time = tko_utils.get_timestamp(keyval, "job_queued")
started_time = tko_utils.get_timestamp(keyval, "job_started")
finished_time = tko_utils.get_timestamp(keyval, "job_finished")
machine = cls.determine_hostname(keyval, dir)
machine_group = cls.determine_machine_group(machine, dir)
machine_owner = keyval.get("owner", None)
aborted_by = keyval.get("aborted_by", None)
aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
return {"user": user, "label": label, "machine": machine,
"queued_time": queued_time, "started_time": started_time,
"finished_time": finished_time, "machine_owner": machine_owner,
"machine_group": machine_group, "aborted_by": aborted_by,
"aborted_on": aborted_at, "keyval_dict": keyval}
def determine_hostname(cls, keyval, job_dir):
host_group_name = keyval.get("host_group_name", None)
machine = keyval.get("hostname", "")
is_multimachine = "," in machine
# determine what hostname to use
if host_group_name:
if is_multimachine or not machine:
tko_utils.dprint("Using host_group_name %r instead of "
"machine name." % host_group_name)
machine = host_group_name
elif is_multimachine:
machine = job.find_hostname(job_dir) # find a unique hostname
except NoHostnameError:
pass # just use the comma-separated name
tko_utils.dprint("MACHINE NAME: %s" % machine)
return machine
def determine_machine_group(cls, hostname, job_dir):
machine_groups = set()
for individual_hostname in hostname.split(","):
host_keyval = models.test.parse_host_keyval(job_dir,
if not host_keyval:
tko_utils.dprint('Unable to parse host keyval for %s'
% individual_hostname)
elif 'labels' in host_keyval:
# Use `model` label as machine group. This is to avoid the
# confusion of multiple boards mapping to the same platform in
# wmatrix. With this change, wmatrix will group tests with the
# same model, rather than the same platform.
labels = host_keyval['labels'].split(',')
board_labels = [l[8:] for l in labels
if l.startswith('model%3A')]
# If the host doesn't have `model:` label, fall back to `board:`
# label.
if not board_labels:
board_labels = [l[8:] for l in labels
if l.startswith('board%3A')]
if board_labels:
# Multiple board/model labels aren't supposed to
# happen, but let's report something sane rather
# than just failing.
error = ('Failed to retrieve board label from host labels: '
'%s' % host_keyval['labels'])
raise BoardLabelError(error)
elif "platform" in host_keyval:
machine_group = ",".join(sorted(machine_groups))
tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
return machine_group
def find_hostname(path):
hostname = os.path.join(path, "sysinfo", "hostname")
machine = open(hostname).readline().rstrip()
return machine
except Exception:
tko_utils.dprint("Could not read a hostname from "
uname = os.path.join(path, "sysinfo", "uname_-a")
machine = open(uname).readline().split()[1]
return machine
except Exception:
tko_utils.dprint("Could not read a hostname from "
raise NoHostnameError("Unable to find a machine name")
class kernel(models.kernel):
def __init__(self, job, verify_ident=None):
kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
super(kernel, self).__init__(**kernel_dict)
def load_from_dir(dir, verify_ident=None):
# try and load the booted kernel version
attributes = False
i = 1
build_dir = os.path.join(dir, "build")
while True:
if not os.path.exists(build_dir):
build_log = os.path.join(build_dir, "debug", "build_log")
attributes = kernel.load_from_build_log(build_log)
if attributes:
i += 1
build_dir = os.path.join(dir, "build.%d" % (i))
if not attributes:
if verify_ident:
base = verify_ident
base = kernel.load_from_sysinfo(dir)
patches = []
hashes = []
base, patches, hashes = attributes
tko_utils.dprint("kernel.__init__() found kernel version %s"
% base)
# compute the kernel hash
if base == "UNKNOWN":
kernel_hash = "UNKNOWN"
kernel_hash = kernel.compute_hash(base, hashes)
return {"base": base, "patches": patches,
"kernel_hash": kernel_hash}
def load_from_sysinfo(path):
for subdir in ("reboot1", ""):
uname_path = os.path.join(path, "sysinfo", subdir,
if not os.path.exists(uname_path):
uname = open(uname_path).readline().split()
return re.sub("-autotest$", "", uname[2])
return "UNKNOWN"
def load_from_build_log(path):
if not os.path.exists(path):
return None
base, patches, hashes = "UNKNOWN", [], []
for line in file(path):
head, rest = line.split(": ", 1)
rest = rest.split()
if head == "BASE":
base = rest[0]
elif head == "PATCH":
return base, patches, hashes
class test(models.test):
def __init__(self, subdir, testname, status, reason, test_kernel,
machine, started_time, finished_time, iterations,
attributes, labels):
# for backwards compatibility with the original parser
# implementation, if there is no test version we need a NULL
# value to be used; also, if there is a version it should
# be terminated by a newline
if "version" in attributes:
attributes["version"] = str(attributes["version"])
attributes["version"] = None
super(test, self).__init__(subdir, testname, status, reason,
test_kernel, machine, started_time,
finished_time, iterations,
attributes, labels)
def load_iterations(keyval_path):
return iteration.load_from_keyval(keyval_path)
class patch(models.patch):
def __init__(self, spec, reference, hash):
tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
super(patch, self).__init__(spec, reference, hash)
self.spec = spec
self.reference = reference
self.hash = hash
class iteration(models.iteration):
def parse_line_into_dicts(line, attr_dict, perf_dict):
key, value = line.split("=", 1)
perf_dict[key] = value
class status_line(object):
def __init__(self, indent, status, subdir, testname, reason,
# pull out the type & status of the line
if status == "START":
self.type = "START"
self.status = None
elif status.startswith("END "):
self.type = "END"
self.status = status[4:]
self.type = "STATUS"
self.status = status
assert (self.status is None or
self.status in status_lib.statuses)
# save all the other parameters
self.indent = indent
self.subdir = self.parse_name(subdir)
self.testname = self.parse_name(testname)
self.reason = reason
self.optional_fields = optional_fields
def parse_name(name):
if name == "----":
return None
return name
def is_status_line(line):
return"^\t*(\S[^\t]*\t){3}", line) is not None
def parse_line(cls, line):
if not status_line.is_status_line(line):
return None
match ="^(\t*)(.*)$", line, flags=re.DOTALL)
if not match:
# A more useful error message than:
# AttributeError: 'NoneType' object has no attribute 'groups'
# to help us debug WTF happens on occasion here.
raise RuntimeError("line %r could not be parsed." % line)
indent, line = match.groups()
indent = len(indent)
# split the line into the fixed and optional fields
parts = line.rstrip("\n").split("\t")
part_index = 3
status, subdir, testname = parts[0:part_index]
# all optional parts should be of the form "key=value". once we've found
# a non-matching part, treat it and the rest of the parts as the reason.
optional_fields = {}
while part_index < len(parts):
kv ="^(\w+)=(.+)", parts[part_index])
if not kv:
optional_fields[] =
part_index += 1
reason = "\t".join(parts[part_index:])
# build up a new status_line and return it
return cls(indent, status, subdir, testname, reason,
class parser(base.parser):
def make_job(dir):
return job(dir)
def state_iterator(self, buffer):
new_tests = []
boot_count = 0
group_subdir = None
sought_level = 0
stack = status_lib.status_stack()
current_kernel = kernel(self.job)
boot_in_progress = False
alert_pending = None
started_time = None
while not self.finished or buffer.size():
# stop processing once the buffer is empty
if buffer.size() == 0:
yield new_tests
new_tests = []
# parse the next line
line = buffer.get()
tko_utils.dprint('\nSTATUS: ' + line.strip())
line = status_line.parse_line(line)
if line is None:
tko_utils.dprint('non-status line, ignoring')
continue # ignore non-status lines
# have we hit the job start line?
if (line.type == "START" and not line.subdir and
not line.testname):
sought_level = 1
tko_utils.dprint("found job level start "
"marker, looking for level "
"1 groups now")
# have we hit the job end line?
if (line.type == "END" and not line.subdir and
not line.testname):
tko_utils.dprint("found job level end "
"marker, looking for level "
"0 lines now")
sought_level = 0
# START line, just push another layer on to the stack
# and grab the start time if this is at the job level
# we're currently seeking
if line.type == "START":
group_subdir = None
if line.indent == sought_level:
started_time = \
line.optional_fields, "timestamp")
tko_utils.dprint("start line, ignoring")
# otherwise, update the status on the stack
tko_utils.dprint("GROPE_STATUS: %s" %
line.status, line.subdir,
line.testname, line.reason])
if line.status == "ALERT":
tko_utils.dprint("job level alert, recording")
alert_pending = line.reason
# ignore Autotest.install => GOOD lines
if (line.testname == "Autotest.install" and
line.status == "GOOD"):
tko_utils.dprint("Successful Autotest "
"install, ignoring")
# ignore END lines for a reboot group
if (line.testname == "reboot" and line.type == "END"):
tko_utils.dprint("reboot group, ignoring")
# convert job-level ABORTs into a 'CLIENT_JOB' test, and
# ignore other job-level events
if line.testname is None:
if (line.status == "ABORT" and
line.type != "END"):
line.testname = "CLIENT_JOB"
tko_utils.dprint("job level event, "
# use the group subdir for END lines
if line.type == "END":
line.subdir = group_subdir
# are we inside a block group?
if (line.indent != sought_level and
line.status != "ABORT" and
not line.testname.startswith('reboot.')):
if line.subdir:
tko_utils.dprint("set group_subdir: "
+ line.subdir)
group_subdir = line.subdir
tko_utils.dprint("ignoring incorrect indent "
"level %d != %d," %
(line.indent, sought_level))
# use the subdir as the testname, except for
# boot.* and kernel.* tests
if (line.testname is None or
if line.subdir and '.' in line.subdir:
line.testname = line.subdir
# has a reboot started?
if line.testname == "reboot.start":
started_time = tko_utils.get_timestamp(
line.optional_fields, "timestamp")
tko_utils.dprint("reboot start event, "
boot_in_progress = True
# has a reboot finished?
if line.testname == "reboot.verify":
line.testname = "boot.%d" % boot_count
tko_utils.dprint("reboot verified")
boot_in_progress = False
verify_ident = line.reason.strip()
current_kernel = kernel(self.job, verify_ident)
boot_count += 1
if alert_pending:
line.status = "ALERT"
line.reason = alert_pending
alert_pending = None
# create the actual test object
finished_time = tko_utils.get_timestamp(
line.optional_fields, "timestamp")
final_status = stack.end()
tko_utils.dprint("Adding: "
"%s\nSubdir:%s\nTestname:%s\n%s" %
(final_status, line.subdir,
line.testname, line.reason))
new_test = test.parse_test(self.job, line.subdir,
final_status, line.reason,
started_time = None
# the job is finished, but we never came back from reboot
if boot_in_progress:
testname = "boot.%d" % boot_count
reason = "machine did not return from reboot"
tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
% (testname, reason))
new_test = test.parse_test(self.job, None, testname,
"ABORT", reason,
current_kernel, None, None)
yield new_tests