Reland "Reland "Add optional enforce_fetch kwarg to fetch and refresh git cache""
This is a reland of 3774c589cb395e9949f5e7d92f837e4a424617d8
Added missing options.enforce_fetch arg
Confirmed via a fake test roll with this CL's patch commit that the
bot_update step passes with this change:
https://chromium-review.googlesource.com/c/chromium/tools/build/+/2443649
Original change's description:
> Reland "Add optional enforce_fetch kwarg to fetch and refresh git cache"
>
> This is a reland of f38f54f2f1a3121643d86abab9e6ebe597238d9b
>
> Original change's description:
> > Add optional enforce_fetch kwarg to fetch and refresh git cache
> >
> > Allows builders to do a force refresh of the git cache to ensure a new
> > fetch happens. android-binary-size needs this to get an accurate diff
> > of the passed in src revision and the applied patch:
> > https://bugs.chromium.org/p/chromium/issues/detail?id=1129235#c19
> >
> > Bug: 1129235
> > Change-Id: I993521cdde2a491fc9d42d75ff28dedcd2cea64e
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2438802
> > Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> > Reviewed-by: Josip Sokcevic <sokcevic@google.com>
>
> Bug: 1129235
> Change-Id: I034a91a8b57821d08b10042f7fc94d32350540ba
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2443235
> Reviewed-by: Edward Lesmes <ehmaldonado@chromium.org>
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Bug: 1129235
Change-Id: Ibbd4535f8529243d2dc1d4d32881d8a61efe374a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2443574
Reviewed-by: Josip Sokcevic <sokcevic@google.com>
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md
index 0e4b50d..5692738 100644
--- a/recipes/README.recipes.md
+++ b/recipes/README.recipes.md
@@ -58,12 +58,12 @@
Wrapper for easy calling of bot_update.
-— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#519)(self, bot_update_step):**
+— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#522)(self, bot_update_step):**
Deapplies a patch, taking care of DEPS and solution revisions properly.
-— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#76)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, \*\*kwargs):**
+— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#76)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, enforce_fetch=False, \*\*kwargs):**
Args:
gclient_config: The gclient configuration to use when running bot_update.
@@ -88,8 +88,10 @@
Requires falsy ignore_input_commit.
step_test_data: a null function that returns test bot_update.py output.
Use test_api.output_json to generate test data.
+ enforce_fetch: Enforce a new fetch to refresh the git cache, even if the
+ solution revision passed in already exists in the current git cache.
-— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#496)(self, project_name, gclient_config=None):**
+— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#499)(self, project_name, gclient_config=None):**
Returns all property names used for storing the checked-out revision of
a given project.
@@ -107,7 +109,7 @@
  **@property**<br>— **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#44)(self):**
-— **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#446)(self, bot_update_json, name):**
+— **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#449)(self, bot_update_json, name):**
Set a fixed revision for a single dependency using project revision
properties.
diff --git a/recipes/recipe_modules/bot_update/api.py b/recipes/recipe_modules/bot_update/api.py
index 03af60d..ffa426b 100644
--- a/recipes/recipe_modules/bot_update/api.py
+++ b/recipes/recipe_modules/bot_update/api.py
@@ -99,6 +99,7 @@
add_blamelists=False,
set_output_commit=False,
step_test_data=None,
+ enforce_fetch=False,
**kwargs):
"""
Args:
@@ -124,6 +125,8 @@
Requires falsy ignore_input_commit.
step_test_data: a null function that returns test bot_update.py output.
Use test_api.output_json to generate test data.
+ enforce_fetch: Enforce a new fetch to refresh the git cache, even if the
+ solution revision passed in already exists in the current git cache.
"""
assert use_site_config_creds is None, "use_site_config_creds is deprecated"
assert rietveld is None, "rietveld is deprecated"
diff --git a/recipes/recipe_modules/bot_update/resources/bot_update.py b/recipes/recipe_modules/bot_update/resources/bot_update.py
index 96b7248..f5bd23e 100755
--- a/recipes/recipe_modules/bot_update/resources/bot_update.py
+++ b/recipes/recipe_modules/bot_update/resources/bot_update.py
@@ -678,13 +678,13 @@
def git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
- cleanup_dir):
+ cleanup_dir, enforce_fetch):
build_dir = os.getcwd()
first_solution = True
for sln in solutions:
sln_dir = path.join(build_dir, sln['name'])
_git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
- cleanup_dir)
+ cleanup_dir, enforce_fetch)
if first_solution:
git_ref = git('log', '--format=%H', '--max-count=1',
cwd=path.join(build_dir, sln['name'])
@@ -694,7 +694,7 @@
def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
- cleanup_dir):
+ cleanup_dir, enforce_fetch):
name = sln['name']
url = sln['url']
populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
@@ -715,6 +715,9 @@
branch, revision = get_target_branch_and_revision(name, url, revisions)
pin = revision if COMMIT_HASH_RE.match(revision) else None
+ if enforce_fetch:
+ git(*populate_cmd, env=env)
+
# Step 1: populate/refresh cache, if necessary.
if not pin:
# Refresh only once.
@@ -864,14 +867,14 @@
def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
target_cpu, patch_root, patch_refs, gerrit_rebase_patch_ref,
no_fetch_tags, refs, git_cache_dir, cleanup_dir,
- gerrit_reset, disable_syntax_validation):
+ gerrit_reset, disable_syntax_validation, enforce_fetch):
# Get a checkout of each solution, without DEPS or hooks.
# Calling git directly because there is no way to run Gclient without
# invoking DEPS.
print('Fetching Git checkout')
git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
- cleanup_dir)
+ cleanup_dir, enforce_fetch)
# Ensure our build/ directory is set up with the correct .gclient file.
gclient_configure(solutions, target_os, target_os_only, target_cpu,
@@ -987,6 +990,12 @@
help=('Don\'t fetch tags from the server for the git checkout. '
'This can speed up fetch considerably when '
'there are many tags.'))
+ parse.add_option(
+ '--enforce_fetch',
+ help=('Enforce a new fetch to refresh the git cache, even if the '
+ 'solution revision passed in already exists in the current '
+ 'git cache.'))
+
# TODO(machenbach): Remove the flag when all uses have been removed.
parse.add_option('--output_manifest', action='store_true',
help=('Deprecated.'))
@@ -1130,6 +1139,7 @@
# Control how the fetch step will occur.
no_fetch_tags=options.no_fetch_tags,
+ enforce_fetch=options.enforce_fetch,
# Finally, extra configurations cleanup dir location.
refs=options.refs,
diff --git a/tests/bot_update_coverage_test.py b/tests/bot_update_coverage_test.py
index ecac91a..a3dbba6 100755
--- a/tests/bot_update_coverage_test.py
+++ b/tests/bot_update_coverage_test.py
@@ -157,6 +157,7 @@
'cleanup_dir': None,
'gerrit_reset': None,
'disable_syntax_validation': False,
+ 'enforce_fetch': False,
}
def setUp(self):