commit 47804922679c997804283b6dcdb751e087f3eee0
author Don Garrett <dgarrett@google.com> Mon Mar 24 20:11:41 2014 -0700
committer Don Garrett <dgarrett@chromium.org> Tue Mar 25 07:07:23 2014 +0000
tree e34c836372dc8bdde196f3b1706f383ed5ef4f70
parent ba2f3a96df4106b09cb9e4466d14870094fbe055

cros_generate_update_payload: Add retries to umount operations.

Add retries around umount operations that have been failing
under heavy load. Add additional logging of system stats
before each retry.

BUG=chromium:353558
TEST=cros flash to local dev box.

Change-Id: Id23cd5cc50ff0d216122e42e0356f1d1b61e1c8b
Reviewed-on: https://chromium-review.googlesource.com/191365
Reviewed-by: David James <davidjames@chromium.org>
Commit-Queue: Don Garrett <dgarrett@chromium.org>
Tested-by: Don Garrett <dgarrett@chromium.org>

host/cros_generate_update_payload

diff --git a/host/cros_generate_update_payload b/host/cros_generate_update_payload
index 06d022c..df04253 100755
--- a/host/cros_generate_update_payload
+++ b/host/cros_generate_update_payload
@@ -46,27 +46,52 @@
 DST_ROOT=""
 STATE_MNT=""
 
+umount_with_retry() {
+  local mnt_point="$1"
+  local ret=0
+
+  if [ ! -d "$mnt_point" ]; then
+    return
+  fi
+
+  for x in {1..10}; do
+    sudo umount -v "$mnt_point"
+    echo "umount ${mnt_point} exited with: $?"
+    rmdir "$mnt_point"
+    ret=$?
+    if [ ${ret} -eq 0 ]; then
+      break
+    fi
+
+    echo "fndmnt:"
+    sudo findmnt
+    echo
+
+    echo "lsof +D:"
+    sudo lsof +D "$mnt_point"
+    echo
+
+    echo "fuser -vm:"
+    sudo fuser -vm "$mnt_point"
+    echo
+
+    sleep 60
+    echo "Retrying umount ${mnt_point}"
+  done
+
+  echo "sudo umount -v ${mnt_point}: ${result}"
+
+}
+
 # Pass an arg to not exit 1 at the end
 cleanup() {
   set +e
-  if [ -n "$SRC_MNT" ]; then
-    sudo umount -v "$SRC_MNT"
-    echo "sudo umount -v ${SRC_MNT}: $?"
-    [ -d "$SRC_MNT" ] && rmdir "$SRC_MNT"
-    SRC_MNT=""
-  fi
-  if [ -n "$DST_MNT" ]; then
-    sudo umount -v "$DST_MNT"
-    echo "sudo umount -v ${DST_MNT}: $?"
-    [ -d "$DST_MNT" ] && rmdir "$DST_MNT"
-    DST_MNT=""
-  fi
-  if [ -n "$STATE_MNT" ]; then
-    sudo umount -v "$STATE_MNT"
-    echo "sudo umount -v ${STATE_MNT}: $?"
-    [ -d "$STATE_MNT" ] && rmdir "$STATE_MNT"
-    STATE_MNT=""
-  fi
+  umount_with_retry "$SRC_MNT"
+  SRC_MNT=""
+  umount_with_retry "$DST_MNT"
+  DST_MNT=""
+  umount_with_retry "$STATE_MNT"
+  STATE_MNT=""
   if [ -z "$FLAGS_src_kern_path" ]; then
     rm -f "$SRC_KERNEL"
   fi