commit 45f277e1b0ef33d59bc959c64f2f3726234c1d39
author Todd Broch <tbroch@chromium.org> Wed Nov 20 12:49:56 2013 -0800
committer chrome-internal-fetch <chrome-internal-fetch@google.com> Mon Nov 25 23:10:23 2013 +0000
tree a5d42cdda7b0f9963c46db8f2532d8f6524b0315
parent 5004c1b88e46760e7898cb8c407af8da0cfaef72

Better logging for ethernet related petting.

recover_duts ethernet hook periodically pets the connection of lab machine to
guarantee connectivity or force a reboot.  This change adds logging to
/var/log/messages in the event this petting fails.

Change also quiets the ping output to only print the summary info.

BUG=chromium:321666
TEST=manual
1. stop recover_duts
2. touch /mnt/stateful_partition/.labmachine
3. start recover_duts
4. <remove ethernet & wait for reboot>
5. <attach ethernet>
6. rm /mnt/stateful_partition/.labmachine
7. grep recover_duts /var/log/messages

   2013-11-20T12:18:50.434889-08:00 localhost recover_duts:check_ethernet.hook:
     All efforts to recover ethernet have been exhausted. Rebooting.

Change-Id: Ic12ebccd89132b7186b3051a35e5b5dfc7e1851b
Reviewed-on: https://chromium-review.googlesource.com/177425
Reviewed-by: Richard Barnette <jrbarnette@chromium.org>
Commit-Queue: Todd Broch <tbroch@chromium.org>
Tested-by: Todd Broch <tbroch@chromium.org>

recover_duts/hooks/check_ethernet.hook

diff --git a/recover_duts/hooks/check_ethernet.hook b/recover_duts/hooks/check_ethernet.hook
index b6abcb5..ff676cb 100755
--- a/recover_duts/hooks/check_ethernet.hook
+++ b/recover_duts/hooks/check_ethernet.hook
@@ -14,6 +14,15 @@
 
 NON_ETHERNET_DRIVERS="cdc_ether"
 
+# Critical messages should be sent to /var/log/messages.  Other messages should
+# be sent via echo to be harvested by recover_duts.py.
+#
+# TODO(tbroch) Relocate this to common hook library if/when there's more than
+# one hook.
+critical_msg() {
+  logger -t "recover_duts:$(basename $0)" -- "$@"
+}
+
 # Returns the default gateway.
 get_default_gateway() {
   local ip_route="$(ip route get 1.0.0.0)"
@@ -55,7 +64,7 @@
 do_ping() {
   local ip_addr=$1
   for eth in $(find_ethernet_interfaces); do
-    ping -I ${eth} -c 9 ${ip_addr} && return 0
+    ping -q -I ${eth} -c 9 ${ip_addr} && return 0
   done
   return 1
 }
@@ -70,6 +79,7 @@
   initctl stop shill || echo "Shill was not running."
   initctl start shill
   sleep 30
+  ifconfig -a
 }
 
 # Return the remote IP address of the first established SSH connection
@@ -111,7 +121,8 @@
   fi
 
   # Last chance - reboot if we can't get any connectivity.
-  echo "All efforts to recover ethernet have been exhausted. Rebooting."
+  critical_msg "All efforts to recover ethernet have been exhausted. Rebooting."
+  sync
   (sleep 5 && reboot) &
   return 1
 }