blob: b2e0178a5b12afb1f7c2040289e3be83e8ca838b [file] [log] [blame] [edit]
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import logging
import os
from autotest_lib.client.common_lib import utils as client_utils
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib.cros import dev_server
from autotest_lib.client.cros import constants
from autotest_lib.server.cros.dynamic_suite.constants import JOB_BUILD_KEY
from autotest_lib.server import utils
def generate_minidump_stacktrace(minidump_path):
"""
Generates a stacktrace for the specified minidump.
This function expects the debug symbols to reside under:
/build/<board>/usr/lib/debug
@param minidump_path: absolute path to minidump to by symbolicated.
@raise client_utils.error.CmdError if minidump_stackwalk return code != 0.
"""
symbol_dir = '%s/../../../lib/debug' % utils.get_server_dir()
logging.info('symbol_dir: %s', symbol_dir)
client_utils.run('minidump_stackwalk "%s" "%s" > "%s.txt"' %
(minidump_path, symbol_dir, minidump_path))
def symbolicate_minidump_with_devserver(minidump_path, resultdir):
"""
Generates a stack trace for the specified minidump by consulting devserver.
This function assumes the debug symbols have been staged on the devserver.
@param minidump_path: absolute path to minidump to by symbolicated.
@param resultdir: server job's result directory.
@raise DevServerException upon failure, HTTP or otherwise.
"""
# First, look up what build we tested. If we can't find this, we can't
# get the right debug symbols, so we might as well give up right now.
keyvals = client_utils.read_keyval(resultdir)
if JOB_BUILD_KEY not in keyvals:
raise dev_server.DevServerException(
'Cannot determine build being tested.')
devserver = dev_server.CrashServer.resolve(keyvals[JOB_BUILD_KEY])
trace_text = devserver.symbolicate_dump(
minidump_path, keyvals[JOB_BUILD_KEY])
if not trace_text:
raise dev_server.DevServerException('Unknown error!!')
with open(minidump_path + '.txt', 'w') as trace_file:
trace_file.write(trace_text)
def find_and_generate_minidump_stacktraces(host_resultdir):
"""
Finds all minidump files and generates a stack trace for each.
Enumerates all files under the test results directory (recursively)
and generates a stack trace file for the minidumps. Minidump files are
identified as files with .dmp extension. The stack trace filename is
composed by appending the .txt extension to the minidump filename.
@param host_resultdir: Directory to walk looking for dmp files.
@returns The list of generated minidumps.
"""
minidumps = []
for dir, subdirs, files in os.walk(host_resultdir):
for file in files:
if not file.endswith('.dmp'):
continue
minidump = os.path.join(dir, file)
# First, try to symbolicate locally.
try:
generate_minidump_stacktrace(minidump)
logging.info('Generated stack trace for dump %s', minidump)
minidumps.append(minidump)
continue
except client_utils.error.CmdError as err:
logging.warning('Failed to generate stack trace locally for '
'dump %s (rc=%d):\n%r',
minidump, err.result_obj.exit_status, err)
# If that did not succeed, try to symbolicate using the dev server.
try:
minidumps.append(minidump)
symbolicate_minidump_with_devserver(minidump, host_resultdir)
logging.info('Generated stack trace for dump %s', minidump)
continue
except dev_server.DevServerException as e:
logging.warning('Failed to generate stack trace on devserver for '
'dump %s:\n%r', minidump, e)
return minidumps
def fetch_orphaned_crashdumps(host, host_resultdir):
"""
Copy all of the crashes in the crash directory over to the results folder.
@param host A host object of the device we're to pull crashes from.
@param host_resultdir The result directory for this host for this test run.
@return The list of minidumps that we pulled back from the host.
"""
minidumps = []
for file in host.list_files_glob(os.path.join(constants.CRASH_DIR, '*')):
logging.info('Collecting %s...', file)
host.get_file(file, host_resultdir, preserve_perm=False)
minidumps.append(file)
return minidumps
def get_site_crashdumps(host, test_start_time):
"""
Copy all of the crashdumps from a host to the results directory.
@param host The host object from which to pull crashes
@param test_start_time When the test we just ran started.
@return A list of all the minidumps
"""
host_resultdir = getattr(getattr(host, 'job', None), 'resultdir', None)
infodir = os.path.join(host_resultdir, 'crashinfo.%s' % host.hostname)
if not os.path.exists(infodir):
os.mkdir(infodir)
# TODO(milleral): handle orphans differently. crosbug.com/38202
try:
orphans = fetch_orphaned_crashdumps(host, infodir)
except Exception as e:
orphans = []
logging.warning('Collection of orphaned crash dumps failed %s', e)
minidumps = find_and_generate_minidump_stacktraces(host_resultdir)
# Record all crashdumps in status.log of the job:
# - If one server job runs several client jobs we will only record
# crashdumps in the status.log of the high level server job.
# - We will record these crashdumps whether or not we successfully
# symbolicate them.
if host.job and minidumps or orphans:
host.job.record('INFO', None, None, 'Start crashcollection record')
for minidump in minidumps:
host.job.record('INFO', None, 'New Crash Dump', minidump)
for orphan in orphans:
host.job.record('INFO', None, 'Orphaned Crash Dump', orphan)
host.job.record('INFO', None, None, 'End crashcollection record')
orphans.extend(minidumps)
for minidump in orphans:
report_bug_from_crash(host, minidump)
return orphans
def find_packages_of(host, exec_name):
"""
Find the package that an executable came from.
@param host A host object that has the executable.
@param exec_name The name of the executable.
@return The name of the package that installed the executable.
"""
packages = []
# TODO(milleral): It would be significantly faster to iterate through
# $PATH and run this than to point it at all of /
find = host.run('find / -executable -type f -name %s' % exec_name)
for full_path in find.stdout.splitlines():
# TODO(milleral): This currently shows scary looking error messages
# in the debug logs via stderr. We only look at stdout, so those
# get filtered, but it would be good to silence them.
portageq = host.run('portageq owners / %s' % full_path)
if portageq.stdout:
packages.append(portageq.stdout.splitlines()[0].strip())
# TODO(milleral): This chunk of code is here to verify that mapping
# executable name to package gives you one and only one package.
# It is highly questionable as to if this should be left in the
# production version of this code or not.
if len(packages) == 0:
raise error.NoUniquePackageFound('no package for %s' % exec_name)
if len(packages) > 1:
# Running through all of /usr/bin in the chroot showed this should
# never happen, but still maybe possible?
raise error.NoUniquePackageFound('Crash detection found more than one'
'package for %s: %s' % exec_name, packages)
# |len(packages) == 1| at this point, as it should be anyway
return packages[0]
def report_bug_from_crash(host, minidump_path):
"""
Given a host to query and a minidump, file a bug about the crash.
@param host A host object that is where the dump came from
@param minidump_path The path to the dump file that should be reported.
"""
# TODO(milleral): Once this has actually been tested, remove the
# try/except. In the meantime, let's make sure nothing dies because of
# the fact that this code isn't very heavily tested.
try:
meta_path = os.path.splitext(minidump_path)[0] + '.meta'
with open(meta_path, 'r') as f:
for line in f.readlines():
parts = line.split('=')
if parts[0] == 'exec_name':
packages = find_packages_of(host, parts[1].strip())
logging.info('Would report crash on %s.', packages)
break
except Exception as e:
logging.warning('Crash detection failed with: %s', e)