Add a new file listing all uploaded files in a Google Storage bucket
This change records all uploaded files to a Google Storage (GS) bucket
in a file "UPLOADED", which resides in that bucket, too.
Currently, the devserver relies on "ls" to get a list of payloads to
download from a GS bucket. This occasionally causes a race condition
where "ls" does not list all uploaded files. The root cause of this is
that GS guarantees only eventual consistency for file metadata, i.e., we
do not know when it will give us the full list of files through "ls".
To avoid constantly polling GS until the list shows up, this change adds
and updates a file called "UPLOADED" in the bucket so that devserver can
retreive it for the list of uploaded files.
BUG=chromium-os:32361
TEST=unittest + trybot
Change-Id: Id328a92b6337c7cc7afa21676d8ff50c82861f37
Reviewed-on: https://gerrit.chromium.org/gerrit/30124
Commit-Ready: Yu-Ju Hong <yjhong@chromium.org>
Reviewed-by: Yu-Ju Hong <yjhong@chromium.org>
Tested-by: Yu-Ju Hong <yjhong@chromium.org>
Reviewed-on: https://gerrit.chromium.org/gerrit/30745
diff --git a/buildbot/cbuildbot_commands.py b/buildbot/cbuildbot_commands.py
index 84d3a9e..4958bc8 100644
--- a/buildbot/cbuildbot_commands.py
+++ b/buildbot/cbuildbot_commands.py
@@ -17,6 +17,7 @@
from chromite.buildbot import cbuildbot_config
from chromite.lib import cros_build_lib as cros_lib
from chromite.lib import locking
+from chromite.lib import osutils
_DEFAULT_RETRIES = 3
_PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
@@ -36,6 +37,7 @@
_AUTOTEST_RPC_CLIENT = ('/b/build_internal/scripts/slave-internal/autotest_rpc/'
'autotest_rpc_client.py')
_LOCAL_BUILD_FLAGS = ['--nousepkg', '--reuse_pkgs_from_local_boards']
+_UPLOADED_LIST_FILENAME = 'UPLOADED'
class TestException(Exception):
pass
@@ -780,13 +782,49 @@
return os.path.basename(debug_tgz)
-def UploadArchivedFile(archive_path, upload_url, filename, debug):
+def AppendToFile(file_path, string):
+ """Append the string to the given file.
+
+ This method provides atomic appends if the string is smaller than
+ PIPE_BUF (> 512 bytes). It does not guarantee atomicity once the
+ string is greater than that.
+
+ Args:
+ file_path: File to be appended to.
+ string: String to append to the file.
+ """
+ osutils.WriteFile(file_path, string, mode='a')
+
+
+def UpdateUploadedList(last_uploaded, archive_path, upload_url, debug):
+ """Updates the list of files uploaded to Google Storage.
+
+ Args:
+ last_uploaded: Filename of the last uploaded file.
+ archive_path: Path to archive_dir.
+ upload_url: Location where tarball should be uploaded.
+ debug: Whether we are in debug mode.
+ """
+
+ # Append to the uploaded list.
+ filename = _UPLOADED_LIST_FILENAME
+ AppendToFile(os.path.join(archive_path, filename), last_uploaded+'\n')
+
+ # Upload the updated list to Google Storage.
+ UploadArchivedFile(archive_path, upload_url, filename, debug,
+ update_list=False)
+
+
+def UploadArchivedFile(archive_path, upload_url, filename, debug,
+ update_list=False):
"""Upload the specified tarball from the archive dir to Google Storage.
Args:
archive_path: Path to archive dir.
upload_url: Location where tarball should be uploaded.
debug: Whether we are in debug mode.
+ filename: Filename of the tarball to upload.
+ update_list: Flag to update the list of uploaded files.
"""
if upload_url and not debug:
@@ -797,6 +835,10 @@
cros_lib.RunCommandCaptureOutput([_GSUTIL_PATH, 'setacl', _GS_ACL,
full_url])
+ # Update the list of uploaded files.
+ if update_list:
+ UpdateUploadedList(filename, archive_path, upload_url, debug)
+
def UploadSymbols(buildroot, board, official):
"""Upload debug symbols for this build."""
@@ -1072,9 +1114,7 @@
set_version: Version of output directory.
"""
latest_path = os.path.join(bot_archive_root, 'LATEST')
- latest_file = open(latest_path, 'w')
- print >> latest_file, set_version
- latest_file.close()
+ osutils.WriteFile(latest_path, set_version, mode='w')
def RemoveOldArchives(bot_archive_root, keep_max):
diff --git a/buildbot/cbuildbot_stages.py b/buildbot/cbuildbot_stages.py
index 06de066..a0d7417 100644
--- a/buildbot/cbuildbot_stages.py
+++ b/buildbot/cbuildbot_stages.py
@@ -1158,12 +1158,17 @@
def _SetupArchivePath(self):
"""Create a fresh directory for archiving a build."""
archive_path = self._GetArchivePath()
+
if not archive_path:
return None
if self._options.buildbot:
# Buildbot: Clear out any leftover build artifacts, if present.
shutil.rmtree(archive_path, ignore_errors=True)
+ else:
+ # Clear the list of uploaded file if it exists
+ osutils.SafeUnlink(os.path.join(archive_path,
+ commands._UPLOADED_LIST_FILENAME))
os.makedirs(archive_path)
@@ -1335,7 +1340,8 @@
def UploadArtifact(filename):
"""Upload generated artifact to Google Storage."""
- commands.UploadArchivedFile(archive_path, upload_url, filename, debug)
+ commands.UploadArchivedFile(archive_path, upload_url, filename, debug,
+ update_list=True)
def ArchiveArtifactsForHWTesting(num_upload_processes=6):
"""Archives artifacts required for HWTest stage."""
