cros_build_lib: _Popen: workaround Python 3.4.1+ subprocess locking bug

Python 3.4.1 changed behavior where Popen APIs cannot be used from a
signal handler when the Popen object was in use when the signal was
delivered due to holding a threading lock.  Add some internal helpers
to workaround it.

See the upstream bug report for more details:
https://bugs.python.org/issue25960

BUG=chromium:1022187
TEST=CQ passes

Change-Id: I76d71351cde8061bd6b50ec4512209e09bbf543d
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/2068782
Reviewed-by: Chris McDonald <cjmcdonald@chromium.org>
Commit-Queue: Mike Frysinger <vapier@chromium.org>
Tested-by: Mike Frysinger <vapier@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/2341320
Reviewed-by: Mike Frysinger <vapier@chromium.org>
diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py
index 8b0d4c5..584c4df 100644
--- a/lib/cros_build_lib.py
+++ b/lib/cros_build_lib.py
@@ -340,16 +340,16 @@
   # the Popen instance was created, but no process was generated.
   if proc.returncode is None and proc.pid is not None:
     try:
-      while proc.poll() is None and int_timeout >= 0:
+      while proc.poll_lock_breaker() is None and int_timeout >= 0:
         time.sleep(0.1)
         int_timeout -= 0.1
 
       proc.terminate()
-      while proc.poll() is None and kill_timeout >= 0:
+      while proc.poll_lock_breaker() is None and kill_timeout >= 0:
         time.sleep(0.1)
         kill_timeout -= 0.1
 
-      if proc.poll() is None:
+      if proc.poll_lock_breaker() is None:
         # Still doesn't want to die.  Too bad, so sad, time to die.
         proc.kill()
     except EnvironmentError as e:
@@ -357,7 +357,11 @@
                       e)
 
     # Ensure our child process has been reaped.
-    proc.wait()
+    kwargs = {}
+    if sys.version_info.major >= 3:
+      # ... but don't wait forever.
+      kwargs['timeout'] = 60
+    proc.wait_lock_breaker(**kwargs)
 
   if not signals.RelaySignal(original_handler, signum, frame):
     # Mock up our own, matching exit code for signaling.
@@ -411,6 +415,31 @@
       else:
         raise
 
+  def _lock_breaker(self, func, *args, **kwargs):
+    """Helper to manage the waitpid lock.
+
+    Workaround https://bugs.python.org/issue25960.
+    """
+    # If the lock doesn't exist, or is not locked, call the func directly.
+    lock = getattr(self, '_waitpid_lock', None)
+    if lock is not None and lock.locked():
+      try:
+        lock.release()
+        return func(*args, **kwargs)
+      finally:
+        if not lock.locked():
+          lock.acquire()
+    else:
+      return func(*args, **kwargs)
+
+  def poll_lock_breaker(self, *args, **kwargs):
+    """Wrapper around poll() to break locks if needed."""
+    return self._lock_breaker(self.poll, *args, **kwargs)
+
+  def wait_lock_breaker(self, *args, **kwargs):
+    """Wrapper around wait() to break locks if needed."""
+    return self._lock_breaker(self.wait, *args, **kwargs)
+
 
 # pylint: disable=redefined-builtin
 def RunCommand(cmd, print_cmd=True, error_message=None, redirect_stdout=False,