paygen: Use multiprocessing semaphore, not threading semaphore!
We were using multithreading semaphores which where not really working
in a multi process environemnt, duh!!!
Also allow again more than two parallel paygen processes.
BUG=chromium:955145
TEST=payloads tryjob and this time worked.
Change-Id: Id5bf572edcc96971bf0e0f6763a6c549c05ad38d
Reviewed-on: https://chromium-review.googlesource.com/1578579
Commit-Ready: Amin Hassani <ahassani@chromium.org>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Nicolas Norvez <norvez@chromium.org>
(cherry picked from commit f5612c3e9e8bc28a6aa00dc5fac723ecd4d37417)
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/1585601
Reviewed-by: Amin Hassani <ahassani@chromium.org>
Commit-Queue: Amin Hassani <ahassani@chromium.org>
diff --git a/lib/paygen/paygen_build_lib.py b/lib/paygen/paygen_build_lib.py
index b2b537d..e89e33e 100644
--- a/lib/paygen/paygen_build_lib.py
+++ b/lib/paygen/paygen_build_lib.py
@@ -729,12 +729,16 @@
True)
for payload in payloads]
- # delta_generator can eat gigs of RAM so we need to constrain the total
- # number of such processes running at the same time. Also note that some
- # other instances of this could be running at the same time so this
- # number could have an additional multiplier applied to it.
+ # Most of the operations in paygen for one single payload is single threaded
+ # and mostly IO bound (downloading images, extracting partitions, waiting
+ # for signers, signing payload, etc). The only part that requires special
+ # attention is generating an unsigned payload which internally has a
+ # massively parallel implementation. So, here we allow multiple processes to
+ # run simultenously and we restrict the number of processes that do the
+ # unsigned payload generation to only two (look at _semaphore in
+ # paygen_payload_lib.py).
parallel.RunTasksInProcessPool(paygen_payload_lib.CreateAndUploadPayload,
- payloads_args, processes=2)
+ payloads_args, processes=8)
def _FindFullTestPayloads(self, channel, version):
"""Returns a list of full test payloads for a given version.
diff --git a/lib/paygen/paygen_build_lib_unittest.py b/lib/paygen/paygen_build_lib_unittest.py
index bdf8ea0..d694d8e 100644
--- a/lib/paygen/paygen_build_lib_unittest.py
+++ b/lib/paygen/paygen_build_lib_unittest.py
@@ -826,7 +826,7 @@
[(self.mp_full_payload, True, True),
(self.mp_delta_payload, True, True),
(self.test_delta_payload, False, True)],
- processes=2)])
+ processes=8)])
def testCleanupBuild(self):
"""Test PaygenBuild._CleanupBuild."""
diff --git a/lib/paygen/paygen_payload_lib.py b/lib/paygen/paygen_payload_lib.py
index 1ec3af2..7b07786 100644
--- a/lib/paygen/paygen_payload_lib.py
+++ b/lib/paygen/paygen_payload_lib.py
@@ -10,10 +10,10 @@
import base64
import datetime
import json
+import multiprocessing
import os
import shutil
import tempfile
-import threading
from chromite.lib import chroot_util
from chromite.lib import constants
@@ -32,7 +32,8 @@
DESCRIPTION_FILE_VERSION = 2
# Only two parallel paygen for now.
-_semaphore = threading.Semaphore(2)
+_semaphore = multiprocessing.Semaphore(2)
+
class Error(Exception):
"""Base class for payload generation errors."""
@@ -370,9 +371,8 @@
# Do not run the delta_generator in parallel. It already has full
# parallelism inside.
- _semaphore.acquire()
- self._RunGeneratorCmd(cmd)
- _semaphore.release()
+ with _semaphore:
+ self._RunGeneratorCmd(cmd)
def _GenerateHashes(self):
"""Generate a payload hash and a metadata hash.
