| # Copyright 2018 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Sync a git repository to a given manifest. |
| |
| This script is intended to define all of the ways that a source checkout can be |
| defined for a Chrome OS builder. |
| |
| If a sync completes successfully, the checked out code will exactly match |
| whatever manifest is defined, and no local git branches will remain. Extraneous |
| files not under version management will be ignored. |
| |
| It is NOT safe to assume that the checkout can be further updated with |
| a simple "repo sync", instead you should call this script again with |
| the same options. |
| """ |
| |
| import logging |
| |
| from chromite.cbuildbot import manifest_version |
| from chromite.cbuildbot import patch_series |
| from chromite.cbuildbot import repository |
| from chromite.lib import commandline |
| from chromite.lib import config_lib |
| from chromite.lib import gerrit |
| from chromite.lib import osutils |
| |
| |
| def GetParser(): |
| """Creates the argparse parser.""" |
| parser = commandline.ArgumentParser(description=__doc__) |
| |
| parser.add_argument( |
| "--repo-root", |
| type="str_path", |
| default=".", |
| help="Path to the repo root to sync.", |
| ) |
| |
| manifest_group = parser.add_argument_group( |
| "Manifest", description="What manifest do we sync?" |
| ) |
| |
| manifest_ex = manifest_group.add_mutually_exclusive_group() |
| manifest_ex.add_argument( |
| "--branch", default="master", help="Sync to top of given branch." |
| ) |
| manifest_ex.add_argument( |
| "--buildspec", |
| help="Path to manifest, relative to manifest-versions root.", |
| ) |
| manifest_ex.add_argument( |
| "--version", |
| help="Shorthand for an official release buildspec. e.g. 9799.0.0", |
| ) |
| manifest_ex.add_argument( |
| "--manifest-file", |
| type="str_path", |
| help="Sync to an existing local manifest file.", |
| ) |
| |
| manifest_group.add_argument("--groups", help="manifest groups to sync.") |
| |
| manifest_url_ex = manifest_group.add_mutually_exclusive_group() |
| manifest_url_ex.add_argument( |
| "--external", |
| action="store_true", |
| default=False, |
| help=( |
| "Sync to the external version of a manifest. Switch from " |
| "manifest-versions-internal to manifest-versions for buildspecs. " |
| "Not usable with --manifest." |
| ), |
| ) |
| manifest_url_ex.add_argument( |
| "--manifest-url", help="Manually set URL to fetch repo manifest from." |
| ) |
| |
| patch_group = parser.add_argument_group( |
| "Patch", description="Which patches should be included with the build?" |
| ) |
| patch_group.add_argument( |
| "-g", |
| "--gerrit-patches", |
| action="split_extend", |
| default=[], |
| metavar="Id1 *int_Id2...IdN", |
| help=( |
| "Space-separated list of short-form Gerrit " |
| "Change-Id's or change numbers to patch. " |
| "Please prepend '*' to internal Change-Id's" |
| ), |
| ) |
| |
| resources_group = parser.add_argument_group( |
| "Resources", description="External resources that might be needed." |
| ) |
| |
| resources_group.add_argument( |
| "--manifest-versions-int", |
| type="str_path", |
| help=( |
| "Directory for internal manifest versions checkout. " |
| "May be refreshed." |
| ), |
| ) |
| |
| resources_group.add_argument( |
| "--manifest-versions-ext", |
| type="str_path", |
| help=( |
| "Directory for internal manifest versions checkout. " |
| "May be refreshed." |
| ), |
| ) |
| |
| optimization_group = parser.add_argument_group( |
| "Optimization", |
| description="Hints provided to possibly speed up initial sync.", |
| ) |
| |
| optimization_group.add_argument( |
| "--copy-repo", |
| type="str_path", |
| help=( |
| "Path to an existing repo root. Used to preload the local " |
| "checkout if the local checkout doesn't exist." |
| ), |
| ) |
| optimization_group.add_argument( |
| "--git-cache-dir", type="str_path", help="Git cache directory to use." |
| ) |
| optimization_group.add_argument( |
| "--repo-url", help="Repo repository location." |
| ) |
| |
| return parser |
| |
| |
| def PrepareManifestVersions(options): |
| """Select manifest-versions checkout to use, and update it. |
| |
| Looks at command line options to decide which manifest-versions checkout to |
| use, and updates (or creates) it as needed. |
| |
| Args: |
| options: Parsed command line options. |
| |
| Returns: |
| Full path to manifest-versions directory to use for this sync. |
| |
| Raises: |
| AssertionError: If the needed manifest-versions path wasn't con the |
| command line. |
| """ |
| site_params = config_lib.GetSiteParams() |
| |
| if options.external: |
| assert ( |
| options.manifest_versions_ext |
| ), "--manifest-versions-ext required." |
| manifest_versions_url = site_params.MANIFEST_VERSIONS_GOB_URL |
| manifest_versions_path = options.manifest_versions_ext |
| else: |
| assert ( |
| options.manifest_versions_int |
| ), "--manifest-versions-int required." |
| manifest_versions_url = site_params.MANIFEST_VERSIONS_INT_GOB_URL |
| manifest_versions_path = options.manifest_versions_int |
| |
| # Resolve buildspecs against a current manifest versions value. |
| manifest_version.RefreshManifestCheckout( |
| manifest_versions_path, manifest_versions_url |
| ) |
| |
| return manifest_versions_path |
| |
| |
| def ResolveLocalManifestPath(options): |
| """Based on command line options, decide what local manifest file to use. |
| |
| Args: |
| options: Our parsed command line options. |
| |
| Returns: |
| Path to local manifest file to use, or None for no file. |
| """ |
| if options.manifest_file: |
| # If the user gives us an explicit local manifest file, use it. |
| return options.manifest_file |
| |
| elif options.buildspec: |
| # Buildspec builds use a manifest file from manifest_versions. We do NOT |
| # use manifest_versions as the manifest git repo, because it's so large |
| # that sync time would be a major performance problem. |
| manifest_versions_path = PrepareManifestVersions(options) |
| return manifest_version.ResolveBuildspec( |
| manifest_versions_path, options.buildspec |
| ) |
| |
| elif options.version: |
| # Versions are a short hand version of a buildspec. |
| manifest_versions_path = PrepareManifestVersions(options) |
| return manifest_version.ResolveBuildspecVersion( |
| manifest_versions_path, options.version |
| ) |
| |
| elif options.branch: |
| # Branch checkouts use our normal manifest repos, not a local manifest |
| # file. |
| return None |
| |
| else: |
| assert False, "No sync options specified. Should not be possible." |
| |
| |
| def main(argv): |
| parser = GetParser() |
| options = parser.parse_args(argv) |
| options.Freeze() |
| |
| local_manifest = ResolveLocalManifestPath(options) |
| |
| if local_manifest: |
| logging.info("Using local_manifest: %s", local_manifest) |
| |
| if options.manifest_url: |
| manifest_url = options.manifest_url |
| elif options.external: |
| manifest_url = config_lib.GetSiteParams().MANIFEST_URL |
| else: |
| manifest_url = config_lib.GetSiteParams().MANIFEST_INT_URL |
| |
| osutils.SafeMakedirs(options.repo_root) |
| repo = repository.RepoRepository( |
| manifest_repo_url=manifest_url, |
| directory=options.repo_root, |
| branch=options.branch, |
| git_cache_dir=options.git_cache_dir, |
| repo_url=options.repo_url, |
| groups=options.groups, |
| ) |
| |
| if options.copy_repo: |
| repo.PreLoad(options.copy_repo) |
| |
| repo.Sync(local_manifest=local_manifest, detach=True) |
| |
| if options.gerrit_patches: |
| patches = gerrit.GetGerritPatchInfo(options.gerrit_patches) |
| # TODO: Extract patches from manifest synced. |
| |
| helper_pool = patch_series.HelperPool.SimpleCreate( |
| cros_internal=not options.external, cros=True |
| ) |
| |
| series = patch_series.PatchSeries( |
| path=options.repo_root, |
| helper_pool=helper_pool, |
| forced_manifest=None, |
| ) |
| |
| _, failed_tot, failed_inflight = series.Apply(patches) |
| |
| failed = failed_tot + failed_inflight |
| if failed: |
| logging.error( |
| "Failed to apply: %s", ", ".join(str(p) for p in failed) |
| ) |
| return 1 |