autotest: Prune empty per-request directories
BUG=chromium:955175
TEST=unittests
Change-Id: Ie799f678321b0cd3c1d9beda8c15005b2702906d
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/1808720
Legacy-Commit-Queue: Commit Bot <commit-bot@chromium.org>
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
Reviewed-by: Xinan Lin <linxinan@chromium.org>
Tested-by: Xinan Lin <linxinan@chromium.org>
diff --git a/site_utils/gs_offloader.py b/site_utils/gs_offloader.py
index fbd4979..ec83a50 100755
--- a/site_utils/gs_offloader.py
+++ b/site_utils/gs_offloader.py
@@ -473,6 +473,23 @@
"""
return TEST_LIST_COLLECTOR in package
+
+def _get_swarming_req_dir(path):
+ """
+ Returns the parent directory of |path|, if |path| is a swarming task result.
+
+ @param path: Full path to the result of a task.
+ e.g. /tmp/results/swarming-44466815c4bc951/1
+
+ @return string of the parent dir or None if not a swarming task.
+ """
+ m_parent = re.match(
+ '(?P<parent_dir>.*/swarming-[0-9a-fA-F]*0)/[1-9a-fA-F]$', path)
+ if m_parent:
+ return m_parent.group('parent_dir')
+ return None
+
+
def _parse_cts_job_results_file_path(path):
"""Parse CTS file paths an extract required information from them."""
@@ -735,6 +752,10 @@
correct_results_folder_permission(dir_entry)
else:
self._prune(dir_entry, job_complete_time)
+ swarming_req_dir = _get_swarming_req_dir(dir_entry)
+ if swarming_req_dir:
+ self._prune_swarming_req_dir(swarming_req_dir)
+
def _try_offload(self, dir_entry, dest_path,
stdout_file, stderr_file):
@@ -825,6 +846,20 @@
# Try again after the permission issue is fixed.
shutil.rmtree(dir_entry)
+ def _prune_swarming_req_dir(self, swarming_req_dir):
+ """Prune swarming request directory, if it is empty.
+
+ @param swarming_req_dir: Directory entry of a swarming request.
+ """
+ try:
+ logging.debug('Pruning swarming request directory %s',
+ swarming_req_dir)
+ os.rmdir(swarming_req_dir)
+ except OSError as e:
+ # Do nothing and leave this directory to next attempt to remove.
+ logging.debug('Failed to prune swarming request directory %s',
+ swarming_req_dir)
+
class _OffloadError(Exception):
"""Google Storage offload failed."""
diff --git a/site_utils/gs_offloader_unittest.py b/site_utils/gs_offloader_unittest.py
index 3ca815e..598fb80 100755
--- a/site_utils/gs_offloader_unittest.py
+++ b/site_utils/gs_offloader_unittest.py
@@ -441,7 +441,7 @@
`self._resultsroot`.
"""
- os.mkdir(jobdir)
+ os.makedirs(jobdir)
return _MockJobDirectory(jobdir)
@@ -554,6 +554,7 @@
alarm.start()
self.addCleanup(alarm.stop)
self.mox.StubOutWithMock(models.test, 'parse_job_keyval')
+ self.should_remove_sarming_req_dir = False
def tearDown(self):
@@ -609,6 +610,11 @@
self.mox.VerifyAll()
self.assertEqual(not should_succeed,
os.path.isdir(self._job.queue_args[0]))
+ swarming_req_dir = gs_offloader._get_swarming_req_dir(
+ self._job.queue_args[0])
+ if swarming_req_dir:
+ self.assertEqual(not self.should_remove_sarming_req_dir,
+ os.path.exists(swarming_req_dir))
def test_offload_success(self):
@@ -627,6 +633,33 @@
self._run_offload_dir(False, 0)
+ def test_offload_swarming_req_dir_remove(self):
+ """Test that `offload_dir()` can prune the empty swarming task dir."""
+ should_remove = os.path.join('results', 'swarming-123abc0')
+ self._job = self.make_job(os.path.join(should_remove, '1'))
+ self._mock_offload_dir_calls(['test', '-d'],
+ self._job.queue_args)
+
+ os.path.isfile(mox.IgnoreArg()).AndReturn(True)
+ self.should_remove_sarming_req_dir = True
+ self._mock_create_marker_file()
+ self._run_offload_dir(True, 0)
+
+
+ def test_offload_swarming_req_dir_exist(self):
+ """Test that `offload_dir()` keeps the non-empty swarming task dir."""
+ should_not_remove = os.path.join('results', 'swarming-456edf0')
+ self._job = self.make_job(os.path.join(should_not_remove, '1'))
+ self.make_job(os.path.join(should_not_remove, '2'))
+ self._mock_offload_dir_calls(['test', '-d'],
+ self._job.queue_args)
+
+ os.path.isfile(mox.IgnoreArg()).AndReturn(True)
+ self.should_remove_sarming_req_dir = False
+ self._mock_create_marker_file()
+ self._run_offload_dir(True, 0)
+
+
def test_sanitize_dir(self):
"""Test that folder/file name with invalid character can be corrected.
"""
