| #!/bin/bash |
| # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # Script to upload all debug symbols required for crash reporting |
| # purposes. This script need only be used to upload release builds |
| # symbols or to debug crashes on non-release builds (in which case try |
| # to only upload the symbols for those executables involved). |
| |
| SCRIPT_ROOT=$(dirname $(readlink -f "$0")) |
| . "${SCRIPT_ROOT}/common.sh" || exit 1 |
| |
| # Script must be run inside the chroot if not in "testing" mode. |
| if [[ "$1" != "--testing" ]]; then |
| restart_in_chroot_if_needed "$@" |
| fi |
| |
| # Flags |
| DEFINE_string board "$DEFAULT_BOARD" "The board to build packages for." |
| DEFINE_string breakpad_root "" "Root directory for breakpad symbols." |
| DEFINE_boolean official_build ${FLAGS_FALSE} "Point to official symbol server." |
| DEFINE_boolean regenerate ${FLAGS_FALSE} "Regenerate all symbols." |
| # Default of 290M is relative to current 300M limit the Crash server enforces. |
| DEFINE_integer strip_cfi 290000000 "Strip CFI data for files above this size." |
| DEFINE_boolean testing ${FLAGS_FALSE} \ |
| "Run in testing mode (should be first argument)." |
| DEFINE_boolean verbose ${FLAGS_FALSE} "Be verbose." |
| DEFINE_boolean yes ${FLAGS_FALSE} "Answer yes to all prompts." |
| |
| # Number of seconds to wait before retrying an upload. The delay will double |
| # for each subsequent retry of the same symbol file. |
| INITIAL_RETRY_DELAY=1 |
| # Allow up to 7 attempts to upload a symbol file (total delay may be |
| # 1+2+4+8+16+32=63 seconds). |
| MAX_RETRIES=6 |
| # Number of total errors, ${TOTAL_ERROR_COUNT}, before retries are no longer |
| # attempted. This is used to avoid lots of errors causing unreasonable delays. |
| MAX_TOTAL_ERRORS_FOR_RETRY=3 # don't bother retrying after 3 errors |
| |
| # Testing parameters. These are only relevant if the "--testing" command-line |
| # option is passed. |
| # Specifies how many attempts should pretend to give an error before |
| # succeeding. NOTE: If this number is greater than ${TEST_MAX_RETRIES}, then |
| # it will never succeed. |
| TEST_ERRORS_FOR_THIS_MANY_ATTEMPTS=3 |
| # Overrides ${MAX_RETRIES} in "testing" mode. |
| TEST_MAX_RETRIES=2 |
| # Overrides ${MAX_TOTAL_ERRORS_FOR_RETRY} in "testing" mode. |
| TEST_MAX_TOTAL_ERRORS_FOR_RETRY=2 |
| |
| SYM_UPLOAD="sym_upload" |
| |
| TOTAL_ERROR_COUNT=0 |
| |
| OUT_DIR=$(mktemp -d "/tmp/err.XXXX") |
| |
| cleanup() { |
| rm -rf "${OUT_DIR}" |
| } |
| |
| really_upload() { |
| if [ ${FLAGS_yes} -eq ${FLAGS_TRUE} ]; then |
| return 0 |
| fi |
| echo "Uploading symbols for an entire Chromium OS build is really only " |
| echo "necessary for release builds and in a few cases for developers " |
| echo "to debug problems. It will take considerable time to run. For " |
| echo "developer debugging purposes, consider instead passing specific files " |
| echo "to upload." |
| read -p "Are you sure you want to upload all build symbols (y/N)? " SURE |
| SURE="${SURE:0:1}" # Get just the first character |
| if [ "${SURE}" != "y" ]; then |
| echo "Ok, better safe than sorry." |
| return 1 |
| fi |
| return 0 |
| } |
| |
| # Upload the given symbol file to given URL. |
| upload_file() { |
| local symbol_file="$1" |
| local upload_url="$2" |
| local upload_file="${symbol_file}" |
| # If the symbols size is too big, strip out the call frame info. The CFI is |
| # unnecessary for 32b x86 targets where the frame pointer is used (as all of |
| # ours have) and it accounts for over half the size of the symbols uploaded. |
| if [ ${FLAGS_strip_cfi} -ne 0 ]; then |
| local symbol_size="$(stat -c%s ${symbol_file})" |
| if [ ${symbol_size} -gt ${FLAGS_strip_cfi} ]; then |
| if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then |
| warn "Stripping CFI for ${symbol_file} due to size ${symbol_size} > \ |
| ${FLAGS_strip_cfi}." |
| fi |
| upload_file="${OUT_DIR}/stripped.sym" |
| sed '/^STACK CFI/d' < "${symbol_file}" > "${upload_file}" |
| fi |
| fi |
| if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then |
| info "Uploading ${symbol_file}" |
| fi |
| local upload_size="$(stat -c%s ${upload_file})" |
| if [ ${upload_size} -gt ${FLAGS_strip_cfi} ]; then |
| # Emit an annotation in order to flag the current step in buildbots. |
| # NOTE: Must be on a line by itself. |
| echo "@@@STEP_WARNINGS@@@" >&2 |
| error "Upload file ${upload_file} is awfully large, risking rejection by \ |
| symbol server (${upload_size} > ${FLAGS_strip_cfi})" |
| let ++TOTAL_ERROR_COUNT |
| fi |
| |
| # Upload the symbol file, allowing for ${MAX_RETRIES} number of retries |
| # before giving an error. However, don't retry if the total errors have |
| # reached ${MAX_TOTAL_ERRORS_FOR_RETRY}. |
| local success=0 |
| local attempts=0 |
| local retry_delay=${INITIAL_RETRY_DELAY} |
| while [ ${attempts} -le ${MAX_RETRIES} ]; do |
| if [ ${attempts} -gt 0 ]; then |
| if [ ${TOTAL_ERROR_COUNT} -ge ${MAX_TOTAL_ERRORS_FOR_RETRY} ]; then |
| warn "Not retrying to upload symbols in ${symbol_file} \ |
| due to too many total errors" |
| break |
| fi |
| warn "Retry #${attempts} to upload symbols in ${symbol_file} \ |
| (sleeping ${retry_delay} seconds)" |
| sleep "${retry_delay}" |
| let retry_delay=retry_delay*2 |
| fi |
| # In testing mode show command that would be run. |
| if [ ${FLAGS_testing} -eq ${FLAGS_TRUE} ]; then |
| echo "TEST: ${SYM_UPLOAD}" "${upload_file}" "${upload_url}" |
| fi |
| # Run the sym_upload command, redirecting its output to files so we can |
| # check them. |
| { |
| if [ ${FLAGS_testing} -eq ${FLAGS_FALSE} ]; then |
| "${SYM_UPLOAD}" "${upload_file}" "${upload_url}" |
| elif [ ${attempts} -lt ${TEST_ERRORS_FOR_THIS_MANY_ATTEMPTS} ]; then |
| # Pretend to fail with an error message. |
| (echo "INFO: Testing info message"; |
| echo "ERROR: Testing error message" >&2; |
| exit 1) |
| else |
| # Pretend to succeed. |
| (echo "Successfully sent the symbol file.") |
| fi |
| } > "${OUT_DIR}/stdout" 2> "${OUT_DIR}/stderr" |
| # Check if sym_upload command succeeded. |
| if grep -q "Successfully sent the symbol file." "${OUT_DIR}/stdout"; then |
| success=1 |
| break |
| fi |
| let ++attempts |
| done |
| if [ ${success} -ne 1 ]; then |
| error "Unable to upload symbols in ${symbol_file}:" |
| cat "${OUT_DIR}/stderr" >&2 |
| let ++TOTAL_ERROR_COUNT |
| return 0 |
| fi |
| |
| if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then |
| size=$(wc -c "${upload_file}" | cut -d' ' -f1) |
| info "Successfully uploaded ${size}B." |
| fi |
| return 0 |
| } |
| |
| main() { |
| trap cleanup EXIT |
| |
| # Parse command line |
| FLAGS_HELP="usage: $0 [flags] [<files...>]" |
| FLAGS "$@" || exit 1 |
| eval set -- "${FLAGS_ARGV}" |
| |
| if [ ${FLAGS_testing} -eq ${FLAGS_TRUE} ]; then |
| info "Running in testing mode:" |
| info " MAX_RETRIES=${TEST_ERRORS_FOR_THIS_MANY_ATTEMPTS}" |
| MAX_RETRIES=${TEST_MAX_RETRIES} |
| info " MAX_TOTAL_ERRORS_FOR_RETRY=${TEST_MAX_TOTAL_ERRORS_FOR_RETRY}" |
| MAX_TOTAL_ERRORS_FOR_RETRY=${TEST_MAX_TOTAL_ERRORS_FOR_RETRY} |
| fi |
| |
| [ -n "$FLAGS_board" ] || die_notrace "--board is required." |
| |
| SYSROOT="/build/${FLAGS_board}" |
| |
| local upload_url="" |
| if [ $FLAGS_official_build -eq $FLAGS_TRUE ]; then |
| upload_url="http://clients2.google.com/cr/symbol" |
| else |
| upload_url="http://clients2.google.com/cr/staging_symbol" |
| warn "This is an unofficial build, uploading to staging server." |
| fi |
| info "Uploading symbols to ${upload_url} from ${SYSROOT}." |
| |
| DEFAULT_BREAKPAD_ROOT="${SYSROOT}/usr/lib/debug/breakpad" |
| if [ -z "${FLAGS_breakpad_root}" ]; then |
| FLAGS_breakpad_root="${DEFAULT_BREAKPAD_ROOT}" |
| else |
| if [ ${FLAGS_regenerate} -eq ${FLAGS_TRUE} ]; then |
| warn "Assuming --noregenerate when --breakpad_root is specified" |
| FLAGS_regenerate=${FLAGS_FALSE} |
| fi |
| fi |
| |
| if [ -z "${FLAGS_ARGV}" ]; then |
| if [ ${FLAGS_regenerate} -eq ${FLAGS_TRUE} ]; then |
| really_upload || exit 1 |
| info "Clearing ${DEFAULT_BREAKPAD_ROOT}" |
| sudo rm -rf "${DEFAULT_BREAKPAD_ROOT}" |
| info "Generating all breakpad symbol files." |
| local verbosity="" |
| local generate_script="${SCRIPTS_DIR}/cros_generate_breakpad_symbols" |
| [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ] && verbosity="--verbose" |
| if ! "${generate_script}" --board=${FLAGS_board} ${verbosity}; then |
| error "Some errors while generating symbols; uploading anyway" |
| let ++TOTAL_ERROR_COUNT |
| fi |
| fi |
| |
| info "Uploading all breakpad symbol files." |
| for sym_file in $(find "${FLAGS_breakpad_root}" -name \*.sym); do |
| # sleep for 200ms to avoid DoS'ing symbol server (crosbug.com/26596) |
| sleep .2 |
| upload_file "${sym_file}" "${upload_url}" |
| done |
| else |
| error "Unexpected args ${FLAGS_ARGV}" |
| fi |
| |
| if [ ${TOTAL_ERROR_COUNT} -ne 0 ]; then |
| die "Encountered ${TOTAL_ERROR_COUNT} problem(s)" |
| fi |
| return 0 |
| } |
| |
| main "$@" |