| # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import logging |
| import os |
| from autotest_lib.client.common_lib import utils as client_utils |
| from autotest_lib.client.common_lib import error |
| from autotest_lib.client.common_lib.cros import dev_server |
| from autotest_lib.client.cros import constants |
| from autotest_lib.server.cros.dynamic_suite.constants import JOB_BUILD_KEY |
| from autotest_lib.server import utils |
| |
| |
| def generate_minidump_stacktrace(minidump_path): |
| """ |
| Generates a stacktrace for the specified minidump. |
| |
| This function expects the debug symbols to reside under: |
| /build/<board>/usr/lib/debug |
| |
| @param minidump_path: absolute path to minidump to by symbolicated. |
| @raise client_utils.error.CmdError if minidump_stackwalk return code != 0. |
| """ |
| symbol_dir = '%s/../../../lib/debug' % utils.get_server_dir() |
| logging.info('symbol_dir: %s', symbol_dir) |
| client_utils.run('minidump_stackwalk "%s" "%s" > "%s.txt"' % |
| (minidump_path, symbol_dir, minidump_path)) |
| |
| |
| def symbolicate_minidump_with_devserver(minidump_path, resultdir): |
| """ |
| Generates a stack trace for the specified minidump by consulting devserver. |
| |
| This function assumes the debug symbols have been staged on the devserver. |
| |
| @param minidump_path: absolute path to minidump to by symbolicated. |
| @param resultdir: server job's result directory. |
| @raise DevServerException upon failure, HTTP or otherwise. |
| """ |
| # First, look up what build we tested. If we can't find this, we can't |
| # get the right debug symbols, so we might as well give up right now. |
| keyvals = client_utils.read_keyval(resultdir) |
| if JOB_BUILD_KEY not in keyvals: |
| raise dev_server.DevServerException( |
| 'Cannot determine build being tested.') |
| |
| devserver = dev_server.CrashServer.resolve(keyvals[JOB_BUILD_KEY]) |
| trace_text = devserver.symbolicate_dump( |
| minidump_path, keyvals[JOB_BUILD_KEY]) |
| if not trace_text: |
| raise dev_server.DevServerException('Unknown error!!') |
| with open(minidump_path + '.txt', 'w') as trace_file: |
| trace_file.write(trace_text) |
| |
| |
| def find_and_generate_minidump_stacktraces(host_resultdir): |
| """ |
| Finds all minidump files and generates a stack trace for each. |
| |
| Enumerates all files under the test results directory (recursively) |
| and generates a stack trace file for the minidumps. Minidump files are |
| identified as files with .dmp extension. The stack trace filename is |
| composed by appending the .txt extension to the minidump filename. |
| |
| @param host_resultdir: Directory to walk looking for dmp files. |
| |
| @returns The list of generated minidumps. |
| """ |
| minidumps = [] |
| for dir, subdirs, files in os.walk(host_resultdir): |
| for file in files: |
| if not file.endswith('.dmp'): |
| continue |
| minidump = os.path.join(dir, file) |
| |
| # First, try to symbolicate locally. |
| try: |
| generate_minidump_stacktrace(minidump) |
| logging.info('Generated stack trace for dump %s', minidump) |
| minidumps.append(minidump) |
| continue |
| except client_utils.error.CmdError as err: |
| logging.warning('Failed to generate stack trace locally for ' |
| 'dump %s (rc=%d):\n%r', |
| minidump, err.result_obj.exit_status, err) |
| |
| # If that did not succeed, try to symbolicate using the dev server. |
| try: |
| minidumps.append(minidump) |
| symbolicate_minidump_with_devserver(minidump, host_resultdir) |
| logging.info('Generated stack trace for dump %s', minidump) |
| continue |
| except dev_server.DevServerException as e: |
| logging.warning('Failed to generate stack trace on devserver for ' |
| 'dump %s:\n%r', minidump, e) |
| return minidumps |
| |
| |
| def fetch_orphaned_crashdumps(host, host_resultdir): |
| """ |
| Copy all of the crashes in the crash directory over to the results folder. |
| |
| @param host A host object of the device we're to pull crashes from. |
| @param host_resultdir The result directory for this host for this test run. |
| @return The list of minidumps that we pulled back from the host. |
| """ |
| minidumps = [] |
| for file in host.list_files_glob(os.path.join(constants.CRASH_DIR, '*')): |
| logging.info('Collecting %s...', file) |
| host.get_file(file, host_resultdir, preserve_perm=False) |
| minidumps.append(file) |
| return minidumps |
| |
| |
| def get_site_crashdumps(host, test_start_time): |
| """ |
| Copy all of the crashdumps from a host to the results directory. |
| |
| @param host The host object from which to pull crashes |
| @param test_start_time When the test we just ran started. |
| @return A list of all the minidumps |
| """ |
| host_resultdir = getattr(getattr(host, 'job', None), 'resultdir', None) |
| infodir = os.path.join(host_resultdir, 'crashinfo.%s' % host.hostname) |
| if not os.path.exists(infodir): |
| os.mkdir(infodir) |
| |
| # TODO(milleral): handle orphans differently. crosbug.com/38202 |
| try: |
| orphans = fetch_orphaned_crashdumps(host, infodir) |
| except Exception as e: |
| orphans = [] |
| logging.warning('Collection of orphaned crash dumps failed %s', e) |
| |
| minidumps = find_and_generate_minidump_stacktraces(host_resultdir) |
| |
| # Record all crashdumps in status.log of the job: |
| # - If one server job runs several client jobs we will only record |
| # crashdumps in the status.log of the high level server job. |
| # - We will record these crashdumps whether or not we successfully |
| # symbolicate them. |
| if host.job and minidumps or orphans: |
| host.job.record('INFO', None, None, 'Start crashcollection record') |
| for minidump in minidumps: |
| host.job.record('INFO', None, 'New Crash Dump', minidump) |
| for orphan in orphans: |
| host.job.record('INFO', None, 'Orphaned Crash Dump', orphan) |
| host.job.record('INFO', None, None, 'End crashcollection record') |
| |
| orphans.extend(minidumps) |
| |
| for minidump in orphans: |
| report_bug_from_crash(host, minidump) |
| |
| return orphans |
| |
| |
| def find_packages_of(host, exec_name): |
| """ |
| Find the package that an executable came from. |
| |
| @param host A host object that has the executable. |
| @param exec_name The name of the executable. |
| @return The name of the package that installed the executable. |
| """ |
| packages = [] |
| |
| # TODO(milleral): It would be significantly faster to iterate through |
| # $PATH and run this than to point it at all of / |
| find = host.run('find / -executable -type f -name %s' % exec_name) |
| for full_path in find.stdout.splitlines(): |
| # TODO(milleral): This currently shows scary looking error messages |
| # in the debug logs via stderr. We only look at stdout, so those |
| # get filtered, but it would be good to silence them. |
| portageq = host.run('portageq owners / %s' % full_path) |
| if portageq.stdout: |
| packages.append(portageq.stdout.splitlines()[0].strip()) |
| |
| # TODO(milleral): This chunk of code is here to verify that mapping |
| # executable name to package gives you one and only one package. |
| # It is highly questionable as to if this should be left in the |
| # production version of this code or not. |
| if len(packages) == 0: |
| raise error.NoUniquePackageFound('no package for %s' % exec_name) |
| if len(packages) > 1: |
| # Running through all of /usr/bin in the chroot showed this should |
| # never happen, but still maybe possible? |
| raise error.NoUniquePackageFound('Crash detection found more than one' |
| 'package for %s: %s' % exec_name, packages) |
| |
| # |len(packages) == 1| at this point, as it should be anyway |
| return packages[0] |
| |
| |
| def report_bug_from_crash(host, minidump_path): |
| """ |
| Given a host to query and a minidump, file a bug about the crash. |
| |
| @param host A host object that is where the dump came from |
| @param minidump_path The path to the dump file that should be reported. |
| """ |
| # TODO(milleral): Once this has actually been tested, remove the |
| # try/except. In the meantime, let's make sure nothing dies because of |
| # the fact that this code isn't very heavily tested. |
| try: |
| meta_path = os.path.splitext(minidump_path)[0] + '.meta' |
| with open(meta_path, 'r') as f: |
| for line in f.readlines(): |
| parts = line.split('=') |
| if parts[0] == 'exec_name': |
| packages = find_packages_of(host, parts[1].strip()) |
| logging.info('Would report crash on %s.', packages) |
| break |
| except Exception as e: |
| logging.warning('Crash detection failed with: %s', e) |