| #!/bin/sh |
| # shellcheck disable=SC2039 |
| # |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| set -e |
| |
| WIFI_CRED=/usr/local/etc/wifi_cred |
| |
| # See do_suspend() in src/third_party/autotest/files/client/cros/power/sys_power.py |
| PAUSE_FILE=/run/autotest_pause_ethernet_hook |
| paused_time=0 |
| |
| |
| # Critical messages should be sent to /var/log/messages. Other messages should |
| # be sent via echo to be harvested by recover_duts.py. |
| # |
| # TODO(tbroch) Relocate this to common hook library if/when there's more than |
| # one hook. |
| critical_msg() { |
| echo "$(date --rfc-3339=seconds) $*" >&2 |
| logger -t "$(basename "$0")" -- "$*" |
| } |
| |
| info_msg() { |
| echo "$(date --rfc-3339=seconds) $*" >&2 |
| } |
| |
| # Returns the default gateway. |
| get_default_gateway() { |
| local ip_route |
| ip_route="$(ip route get 1.0.0.0)" |
| echo "${ip_route}" | head -n 1 | cut -f 3 -d ' ' |
| } |
| |
| # Shows the list of Ethernet interfaces found on the system. |
| # Optional arg: when non-empty, only check for operational links. |
| search_devices() { |
| local device_path |
| local device |
| local operational="$1" |
| |
| for device_path in /sys/class/net/*; do |
| device="$(basename "${device_path}")" |
| |
| # Skip non-operational links. |
| if [ -n "${operational}" ] && \ |
| [ "$(cat "${device_path}/operstate")" != "up" ]; then |
| continue |
| fi |
| |
| # ignore "wwan*" devices: they are 3G/4G/LTE modems. |
| case "${device}" in |
| wwan*|lo) continue;; |
| esac |
| |
| # ignore virtual devices (e.g. lxcbr0 or arcbr0) |
| if [ "$(cat "${device_path}/addr_assign_type" 2>&1)" != "0" ] ; then |
| continue |
| fi |
| |
| # ignore "wlan" devices. |
| if iw "${device}" info > /dev/null 2>&1 ; then |
| continue |
| fi |
| |
| echo "${device}" |
| done |
| } |
| |
| # Pings the given ipaddress through all operational ethernet devices |
| # $1 - IP address to ping. |
| do_ping() { |
| local ip_addr=$1 |
| local eth |
| |
| for eth in $(search_devices 1); do |
| ping -q -I "${eth}" -c 9 "${ip_addr}" && return 0 |
| done |
| |
| return 1 |
| } |
| |
| # Return the remote IP address of an established SSH connection. |
| find_ssh_client() { |
| netstat -ntW | awk '$1 ~ /^tcp[6]?$/ && $4 ~ /:22$/ && $6 == "ESTABLISHED" { |
| sub(/:[^:]*$/, "", $5); |
| if ($5 != "127.0.0.1" && $5 != "::1") { |
| print $5 |
| exit |
| } |
| }' |
| } |
| |
| # Return the IP address of a neighbor reachable via ethernet |
| find_ethernet_neighbor() { |
| local eth |
| local neighbor_ip |
| |
| for eth in $(search_devices 1); do |
| neighbor_ip=$(ip -4 neigh show dev "${eth}" | |
| awk '/REACHABLE|DELAY|STALE/ {print $1; exit}') |
| [ -n "${neighbor_ip}" ] && echo "${neighbor_ip}" && return 0 |
| done |
| return 1 |
| } |
| |
| # Try to find a connected SSH client (our autotest server) and ping it |
| ping_controlling_server() { |
| local default_gateway |
| default_gateway="$(get_default_gateway)" || default_gateway= |
| if [ -n "${default_gateway}" ]; then |
| do_ping ${default_gateway} && return 0 |
| fi |
| |
| local ssh_client |
| ssh_client="$(find_ssh_client)" || ssh_client= |
| if [ -n "${ssh_client}" ]; then |
| do_ping ${ssh_client} && return 0 |
| fi |
| |
| # Last attempt: any recently seen neighbor |
| local neighbor |
| neighbor="$(find_ethernet_neighbor)" || neighbor= |
| if [ -n "${neighbor}" ]; then |
| do_ping ${neighbor} && return 0 |
| fi |
| return 1 |
| } |
| |
| reload_ethernet_drivers() { |
| local eth |
| local ret=1 |
| for eth in $(search_devices); do |
| info_msg "Reload driver for interface ${eth}" |
| reload_network_device "${eth}" |
| ret=0 |
| done |
| return ${ret} |
| } |
| |
| toggle_usb_ports() { |
| local device_path |
| local ret=1 |
| for device_path in /sys/class/net/*; do |
| local usbpath |
| usbpath=$(readlink -f "${device_path}") |
| # Example of what usbpath is expect to look like: |
| # usbpath=/sys/devices/pci0000:00/0000:00:14.0/usb2/2-2/2-2:1.0/net/eth1 |
| # |
| # But we want the port for that device: |
| # echo ${usbpath%/*/net/*} |
| # /sys/devices/pci0000:00/0000:00:14.0/usb2/2-2 |
| usbpath=${usbpath%/*/net/*} |
| |
| # Only USB devices have "authorized" field in /sys |
| if [ -w "${usbpath}/authorized" ]; then |
| ret=0 |
| # disable port: sort of like unplugging/plugging the dongle |
| echo 0 > "${usbpath}/authorized" |
| sleep 2 |
| echo 1 > "${usbpath}/authorized" |
| sleep 1 |
| fi |
| done |
| return ${ret} |
| } |
| |
| # If there are no devices available, rescan all hubs. |
| rescan_usb_hubs() { |
| local port |
| local portnum |
| local usbhub=/sys/bus/usb/drivers/hub |
| |
| # If there's an operational Ethernet interface, no need to reset any hubs. |
| if [ -n "$(search_devices 1)" ]; then |
| return 1 |
| fi |
| |
| # Didn't find any eth devices. |
| # Some possible causes are: |
| # crbug.com/452686 RT8153 (USB3-GigE) dongle coming up as USB-storage |
| # (Fixed: "mist" will reset back to ethernet device) |
| # crbug.com/733425 USB enumeration fails with |
| # "device not accepting address 2, error -71" |
| info_msg "No ethernet found: Rescanning USB hubs" |
| |
| for port in "${usbhub}"/[0-9]*-0:1.0; do |
| # Doesn't exist. Glob didn't match anything? |
| [ -e "${port}" ] || continue |
| |
| critical_msg "Rescanning ${port}" |
| portnum="$(basename "${port}")" |
| echo "${portnum}" > "${usbhub}"/unbind |
| sleep 1 |
| echo "${portnum}" > "${usbhub}"/bind |
| done |
| |
| # Return status: *now* do we have any devices? |
| for _ in $(seq 1 5); do |
| # USB needs a bit of time to scan the "hub". |
| sleep 3 |
| [ -n "$(search_devices 1)" ] && return 0 |
| done |
| return 1 |
| } |
| |
| restart_connection_manager() { |
| initctl stop shill || info_msg "Shill was not running." |
| initctl start shill |
| } |
| |
| try_pause_lock() { |
| # Append, to avoid changing the mtime of an existing lock if we don't acquire |
| # it. |
| if ! exec 9>>"${PAUSE_FILE}"; then |
| critical_msg "Failed to open ${PAUSE_FILE}" |
| return 1 |
| fi |
| if ! flock -xn 9; then |
| critical_msg "Failed to acquire ${PAUSE_FILE}" |
| return 1 |
| fi |
| } |
| |
| # Clear old locks and try to grab the lock. Does not block, and aborts if we |
| # fail. |
| force_pause_lock() { |
| critical_msg "Clobbering lock (${PAUSE_FILE})" |
| rm -f "${PAUSE_FILE}" |
| try_pause_lock || exit 1 |
| } |
| |
| # Return 0 if we need to pause (abort). Return non-zero and grab the "pause" |
| # lock if we can continue. |
| pause_check_ethernet() { |
| # power_SuspendStress tests requires many minutes of network timeout |
| # tolerance since the SSH connection to the autotest server will be |
| # disrupted. *** See http://crbug.com/334951 *** |
| # |
| # "Pause" the ethernet check for up to 30 minutes at the |
| # request of any test that creates and flocks PAUSE_FILE. |
| |
| # Acquire the lock and hold it until exit, if possible. |
| if try_pause_lock; then |
| # File wasn't locked - no need to pause. |
| return 1 |
| fi |
| |
| local now |
| local start_time |
| |
| now="$(date +%s)" |
| start_time=$(stat -c%Z "${PAUSE_FILE}") || true |
| |
| if [ -z "${start_time}" ]; then |
| # Couldn't figure out lock time - just clobber it. |
| force_pause_lock |
| return 1 |
| fi |
| |
| local paused_time |
| paused_time=$((now - start_time)) |
| if [ ${paused_time} -gt $((30*60)) ] ; then |
| critical_msg "Pause request exceeded 30 minutes. Checking lab network link." |
| force_pause_lock |
| return 1 |
| fi |
| |
| info_msg "Ethernet Check Pause started ${paused_time} seconds ago." |
| return 0 |
| } |
| |
| main() { |
| # Special check for devices running power autotests that connect to moblab via |
| # WiFi. This check saves devices from rebooting even though they don't have |
| # wired ethernet connection. |
| if [ -f "${WIFI_CRED}" ]; then |
| info_msg "${WIFI_CRED} found. No need to check ethernet." |
| return 0 |
| fi |
| |
| if pause_check_ethernet; then |
| return 0 |
| fi |
| |
| # Attempt to ping our controlling autotest server over ethernet. |
| if ping_controlling_server; then |
| return 0 |
| fi |
| |
| local recovery_method |
| for recovery_method in rescan_usb_hubs \ |
| toggle_usb_ports \ |
| reload_ethernet_drivers; do |
| critical_msg "Attempting recovery method \"${recovery_method}\"" |
| |
| # A success return from the recovery method implies that it successfully |
| # performed some action that makes it worth re-checking to see whether |
| # our connectivity was remediated. Otherwise, we move on to the next |
| # recovery method without delay. |
| "${recovery_method}" || continue |
| |
| local now |
| local start_time |
| local method_timeout |
| |
| now="$(date +%s)" |
| start_time="${now}" |
| method_timeout=$((now+30)) |
| |
| if ! initctl status shill | grep -q running ; then |
| restart_connection_manager |
| fi |
| |
| # poll "controlling_server" until timeout |
| # NB: Our Lab DHCP servers must respond in < 30 seconds. |
| while [ "${now}" -lt "${method_timeout}" ]; do |
| if ping_controlling_server; then |
| critical_msg "${recovery_method} successful after $((now-start_time)) seconds." |
| return 0 |
| fi |
| sleep 1 |
| now="$(date +%s)" |
| done |
| done |
| |
| critical_msg "All ethernet recovery methods have failed. Rebooting." |
| sync |
| |
| # Give powerd a chance to reboot via the standard path (and log messages that |
| # are helpful for debugging) before calling 'reboot' directly. |
| dbus-send --system --type=method_call --dest=org.chromium.PowerManager \ |
| /org/chromium/PowerManager org.chromium.PowerManager.RequestRestart \ |
| int32:2 string:'recover_duts check_ethernet hook failed' & |
| sleep 30 |
| reboot |
| |
| return 1 |
| } |
| |
| main |