blob: 3c9035239cb2b508146d0d4128e44af56de06fb0 [file] [log] [blame]
#!/bin/bash
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Script to upload all debug symbols required for crash reporting
# purposes. This script need only be used to upload release builds
# symbols or to debug crashes on non-release builds (in which case try
# to only upload the symbols for those executables involved).
SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
. "${SCRIPT_ROOT}/common.sh" || exit 1
# Script must be run inside the chroot if not in "testing" mode.
if [[ "$1" != "--testing" ]]; then
restart_in_chroot_if_needed "$@"
fi
# Flags
DEFINE_string board "$DEFAULT_BOARD" "The board to build packages for."
DEFINE_string breakpad_root "" "Root directory for breakpad symbols."
DEFINE_boolean official_build ${FLAGS_FALSE} "Point to official symbol server."
DEFINE_boolean regenerate ${FLAGS_FALSE} "Regenerate all symbols."
# Default of 290M is relative to current 300M limit the Crash server enforces.
DEFINE_integer strip_cfi 290000000 "Strip CFI data for files above this size."
DEFINE_boolean testing ${FLAGS_FALSE} \
"Run in testing mode (should be first argument)."
DEFINE_boolean verbose ${FLAGS_FALSE} "Be verbose."
DEFINE_boolean yes ${FLAGS_FALSE} "Answer yes to all prompts."
# Number of seconds to wait before retrying an upload. The delay will double
# for each subsequent retry of the same symbol file.
INITIAL_RETRY_DELAY=1
# Allow up to 7 attempts to upload a symbol file (total delay may be
# 1+2+4+8+16+32=63 seconds).
MAX_RETRIES=6
# Number of total errors, ${TOTAL_ERROR_COUNT}, before retries are no longer
# attempted. This is used to avoid lots of errors causing unreasonable delays.
MAX_TOTAL_ERRORS_FOR_RETRY=3 # don't bother retrying after 3 errors
# Testing parameters. These are only relevant if the "--testing" command-line
# option is passed.
# Specifies how many attempts should pretend to give an error before
# succeeding. NOTE: If this number is greater than ${TEST_MAX_RETRIES}, then
# it will never succeed.
TEST_ERRORS_FOR_THIS_MANY_ATTEMPTS=3
# Overrides ${MAX_RETRIES} in "testing" mode.
TEST_MAX_RETRIES=2
# Overrides ${MAX_TOTAL_ERRORS_FOR_RETRY} in "testing" mode.
TEST_MAX_TOTAL_ERRORS_FOR_RETRY=2
SYM_UPLOAD="sym_upload"
TOTAL_ERROR_COUNT=0
OUT_DIR=$(mktemp -d "/tmp/err.XXXX")
cleanup() {
rm -rf "${OUT_DIR}"
}
really_upload() {
if [ ${FLAGS_yes} -eq ${FLAGS_TRUE} ]; then
return 0
fi
echo "Uploading symbols for an entire Chromium OS build is really only "
echo "necessary for release builds and in a few cases for developers "
echo "to debug problems. It will take considerable time to run. For "
echo "developer debugging purposes, consider instead passing specific files "
echo "to upload."
read -p "Are you sure you want to upload all build symbols (y/N)? " SURE
SURE="${SURE:0:1}" # Get just the first character
if [ "${SURE}" != "y" ]; then
echo "Ok, better safe than sorry."
return 1
fi
return 0
}
# Upload the given symbol file to given URL.
upload_file() {
local symbol_file="$1"
local upload_url="$2"
local upload_file="${symbol_file}"
# If the symbols size is too big, strip out the call frame info. The CFI is
# unnecessary for 32b x86 targets where the frame pointer is used (as all of
# ours have) and it accounts for over half the size of the symbols uploaded.
if [ ${FLAGS_strip_cfi} -ne 0 ]; then
local symbol_size="$(stat -c%s ${symbol_file})"
if [ ${symbol_size} -gt ${FLAGS_strip_cfi} ]; then
if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then
warn "Stripping CFI for ${symbol_file} due to size ${symbol_size} > \
${FLAGS_strip_cfi}."
fi
upload_file="${OUT_DIR}/stripped.sym"
sed '/^STACK CFI/d' < "${symbol_file}" > "${upload_file}"
fi
fi
if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then
info "Uploading ${symbol_file}"
fi
local upload_size="$(stat -c%s ${upload_file})"
if [ ${upload_size} -gt ${FLAGS_strip_cfi} ]; then
# Emit an annotation in order to flag the current step in buildbots.
# NOTE: Must be on a line by itself.
echo "@@@STEP_WARNINGS@@@" >&2
error "Upload file ${upload_file} is awfully large, risking rejection by \
symbol server (${upload_size} > ${FLAGS_strip_cfi})"
let ++TOTAL_ERROR_COUNT
fi
# Upload the symbol file, allowing for ${MAX_RETRIES} number of retries
# before giving an error. However, don't retry if the total errors have
# reached ${MAX_TOTAL_ERRORS_FOR_RETRY}.
local success=0
local attempts=0
local retry_delay=${INITIAL_RETRY_DELAY}
while [ ${attempts} -le ${MAX_RETRIES} ]; do
if [ ${attempts} -gt 0 ]; then
if [ ${TOTAL_ERROR_COUNT} -ge ${MAX_TOTAL_ERRORS_FOR_RETRY} ]; then
warn "Not retrying to upload symbols in ${symbol_file} \
due to too many total errors"
break
fi
warn "Retry #${attempts} to upload symbols in ${symbol_file} \
(sleeping ${retry_delay} seconds)"
sleep "${retry_delay}"
let retry_delay=retry_delay*2
fi
# In testing mode show command that would be run.
if [ ${FLAGS_testing} -eq ${FLAGS_TRUE} ]; then
echo "TEST: ${SYM_UPLOAD}" "${upload_file}" "${upload_url}"
fi
# Run the sym_upload command, redirecting its output to files so we can
# check them.
{
if [ ${FLAGS_testing} -eq ${FLAGS_FALSE} ]; then
"${SYM_UPLOAD}" "${upload_file}" "${upload_url}"
elif [ ${attempts} -lt ${TEST_ERRORS_FOR_THIS_MANY_ATTEMPTS} ]; then
# Pretend to fail with an error message.
(echo "INFO: Testing info message";
echo "ERROR: Testing error message" >&2;
exit 1)
else
# Pretend to succeed.
(echo "Successfully sent the symbol file.")
fi
} > "${OUT_DIR}/stdout" 2> "${OUT_DIR}/stderr"
# Check if sym_upload command succeeded.
if grep -q "Successfully sent the symbol file." "${OUT_DIR}/stdout"; then
success=1
break
fi
let ++attempts
done
if [ ${success} -ne 1 ]; then
error "Unable to upload symbols in ${symbol_file}:"
cat "${OUT_DIR}/stderr" >&2
let ++TOTAL_ERROR_COUNT
return 0
fi
if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then
size=$(wc -c "${upload_file}" | cut -d' ' -f1)
info "Successfully uploaded ${size}B."
fi
return 0
}
main() {
trap cleanup EXIT
# Parse command line
FLAGS_HELP="usage: $0 [flags] [<files...>]"
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"
if [ ${FLAGS_testing} -eq ${FLAGS_TRUE} ]; then
info "Running in testing mode:"
info " MAX_RETRIES=${TEST_ERRORS_FOR_THIS_MANY_ATTEMPTS}"
MAX_RETRIES=${TEST_MAX_RETRIES}
info " MAX_TOTAL_ERRORS_FOR_RETRY=${TEST_MAX_TOTAL_ERRORS_FOR_RETRY}"
MAX_TOTAL_ERRORS_FOR_RETRY=${TEST_MAX_TOTAL_ERRORS_FOR_RETRY}
fi
[ -n "$FLAGS_board" ] || die_notrace "--board is required."
SYSROOT="/build/${FLAGS_board}"
local upload_url=""
if [ $FLAGS_official_build -eq $FLAGS_TRUE ]; then
upload_url="http://clients2.google.com/cr/symbol"
else
upload_url="http://clients2.google.com/cr/staging_symbol"
warn "This is an unofficial build, uploading to staging server."
fi
info "Uploading symbols to ${upload_url} from ${SYSROOT}."
DEFAULT_BREAKPAD_ROOT="${SYSROOT}/usr/lib/debug/breakpad"
if [ -z "${FLAGS_breakpad_root}" ]; then
FLAGS_breakpad_root="${DEFAULT_BREAKPAD_ROOT}"
else
if [ ${FLAGS_regenerate} -eq ${FLAGS_TRUE} ]; then
warn "Assuming --noregenerate when --breakpad_root is specified"
FLAGS_regenerate=${FLAGS_FALSE}
fi
fi
if [ -z "${FLAGS_ARGV}" ]; then
if [ ${FLAGS_regenerate} -eq ${FLAGS_TRUE} ]; then
really_upload || exit 1
info "Clearing ${DEFAULT_BREAKPAD_ROOT}"
sudo rm -rf "${DEFAULT_BREAKPAD_ROOT}"
info "Generating all breakpad symbol files."
local verbosity=""
local generate_script="${SCRIPTS_DIR}/cros_generate_breakpad_symbols"
[ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ] && verbosity="--verbose"
if ! "${generate_script}" --board=${FLAGS_board} ${verbosity}; then
error "Some errors while generating symbols; uploading anyway"
let ++TOTAL_ERROR_COUNT
fi
fi
info "Uploading all breakpad symbol files."
for sym_file in $(find "${FLAGS_breakpad_root}" -name \*.sym); do
# sleep for 200ms to avoid DoS'ing symbol server (crosbug.com/26596)
sleep .2
upload_file "${sym_file}" "${upload_url}"
done
else
error "Unexpected args ${FLAGS_ARGV}"
fi
if [ ${TOTAL_ERROR_COUNT} -ne 0 ]; then
die "Encountered ${TOTAL_ERROR_COUNT} problem(s)"
fi
return 0
}
main "$@"