commit 66e51e90bb700d7bfdd5632cb04b7f8f8ef0cd4f
author Mike Frysinger <vapier@chromium.org> Sat May 03 16:52:00 2014 -0400
committer Mike Frysinger <vapier@chromium.org> Mon May 05 19:32:16 2014 +0000
tree abef47285e9697207ac0e24f9be1ad75a96fb5df
parent c976305af43798e366eee932772d0c403f95dd21

upload_symbols: add more robust killing

Looks like even though we tried to terminate the background process, it
didn't respect that.  So use a series of signals to get it back to a state
we can work with.

BUG=chromium:355843
TEST=`./buildbot/run_tests` passes
TEST=`cbuildbot daisy-release` passes

Change-Id: Ie06b438031ae5e72ab3b3de2ccaa67dbdded3763
Reviewed-on: https://chromium-review.googlesource.com/198121
Reviewed-by: Yu-Ju Hong <yjhong@chromium.org>
Commit-Queue: Mike Frysinger <vapier@chromium.org>
Tested-by: Mike Frysinger <vapier@chromium.org>

scripts/upload_symbols.py

diff --git a/scripts/upload_symbols.py b/scripts/upload_symbols.py
index e4f4871..5bcb928 100644
--- a/scripts/upload_symbols.py
+++ b/scripts/upload_symbols.py
@@ -19,7 +19,15 @@
 import multiprocessing
 import os
 import poster
+try:
+  import Queue
+except ImportError:
+  # Python-3 renamed to "queue".  We still use Queue to avoid collisions
+  # with naming variables as "queue".  Maybe we'll transition at some point.
+  # pylint: disable=F0401
+  import queue as Queue
 import random
+import signal
 import socket
 import textwrap
 import tempfile
@@ -35,6 +43,7 @@
 from chromite.lib import osutils
 from chromite.lib import parallel
 from chromite.lib import retry_util
+from chromite.lib import signals
 from chromite.lib import timeout_util
 from chromite.scripts import cros_generate_breakpad_symbols
 
@@ -379,6 +388,7 @@
       with timeout_util.Timeout(DEDUPE_TIMEOUT):
         cros_build_lib.Debug('sending %s to dedupe server', item.sym_file)
         storage.push(item, item.content(0))
+        cros_build_lib.Debug('sent %s', item.sym_file)
     cros_build_lib.Info('dedupe notification finished; exiting')
   except Exception:
     sym_file = item.sym_file if (item and item.sym_file) else ''
@@ -700,10 +710,25 @@
 
     # The process is taking too long, so kill it and complain.
     if storage_notify_proc.is_alive():
-      storage_notify_proc.terminate()
       cros_build_lib.Warning('notification process took too long')
       cros_build_lib.PrintBuildbotStepWarnings()
 
+      # Kill it gracefully first (traceback) before tacking it down harder.
+      pid = storage_notify_proc.pid
+      for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGKILL):
+        cros_build_lib.Warning('sending %s to %i', signals.StrSignal(sig), pid)
+        os.kill(pid, sig)
+        time.sleep(5)
+        if storage_notify_proc.is_alive():
+          break
+
+      # Drain the queue so we don't hang when we finish.
+      try:
+        while dedupe_queue.get_nowait():
+          pass
+      except Queue.Empty:
+        pass
+
   cros_build_lib.Info('uploaded %i symbols (%i were deduped) which took: %s',
                       counters.uploaded_count, counters.deduped_count,
                       datetime.datetime.now() - start_time)