| # -*- coding: utf-8 -*- |
| # Copyright 2018 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Sync a git repository to a given manifest. |
| |
| This script is intended to define all of the ways that a source checkout can be |
| defined for a Chrome OS builder. |
| |
| If a sync completes successfully, the checked out code will exactly match |
| whatever manifest is defined, and no local git branches will remain. Extraneous |
| files not under version management will be ignored. |
| |
| It is NOT safe to assume that the checkout can be further updated with |
| a simple "repo sync", instead you should call this script again with |
| the same options. |
| """ |
| |
| from __future__ import print_function |
| |
| from chromite.cbuildbot import manifest_version |
| from chromite.cbuildbot import patch_series |
| from chromite.cbuildbot import repository |
| from chromite.lib import commandline |
| from chromite.lib import cros_logging as logging |
| from chromite.lib import config_lib |
| from chromite.lib import gerrit |
| from chromite.lib import osutils |
| |
| |
| def GetParser(): |
| """Creates the argparse parser.""" |
| parser = commandline.ArgumentParser(description=__doc__) |
| |
| parser.add_argument('--repo-root', type='path', default='.', |
| help='Path to the repo root to sync.') |
| |
| manifest_group = parser.add_argument_group( |
| 'Manifest', |
| description='What manifest do we sync?') |
| |
| manifest_ex = manifest_group.add_mutually_exclusive_group() |
| manifest_ex.add_argument( |
| '--branch', |
| default='master', |
| help='Sync to top of given branch.') |
| manifest_ex.add_argument( |
| '--buildspec', |
| help='Path to manifest, relative to manifest-versions root.') |
| manifest_ex.add_argument( |
| '--version', |
| help='Shorthand for an official release buildspec. e.g. 9799.0.0') |
| manifest_ex.add_argument( |
| '--manifest-file', type='path', |
| help='Sync to an existing local manifest file.') |
| |
| manifest_group.add_argument( |
| '--groups', help='manifest groups to sync.') |
| |
| manifest_url_ex = manifest_group.add_mutually_exclusive_group() |
| manifest_url_ex.add_argument( |
| '--external', action='store_true', default=False, |
| help='Sync to the external version of a manifest. Switch from ' |
| 'manifest-versions-internal to manifest-versions for buildspecs. ' |
| 'Not usable with --manifest.') |
| manifest_url_ex.add_argument( |
| '--manifest-url', help='Manually set URL to fetch repo manifest from.') |
| |
| patch_group = parser.add_argument_group( |
| 'Patch', |
| description='Which patches should be included with the build?') |
| patch_group.add_argument( |
| '-g', '--gerrit-patches', action='split_extend', default=[], |
| metavar='Id1 *int_Id2...IdN', |
| help='Space-separated list of short-form Gerrit ' |
| "Change-Id's or change numbers to patch. " |
| "Please prepend '*' to internal Change-Id's") |
| |
| resources_group = parser.add_argument_group( |
| 'Resources', |
| description='External resources that might be needed.') |
| |
| resources_group.add_argument( |
| '--manifest-versions-int', type='path', |
| help='Directory for internal manifest versions checkout. ' |
| 'May be refreshed.') |
| |
| resources_group.add_argument( |
| '--manifest-versions-ext', type='path', |
| help='Directory for internal manifest versions checkout. ' |
| 'May be refreshed.') |
| |
| optimization_group = parser.add_argument_group( |
| 'Optimization', |
| description='Hints provided to possibly speed up initial sync.') |
| |
| optimization_group.add_argument( |
| '--copy-repo', type='path', |
| help='Path to an existing repo root. Used to preload the local ' |
| "checkout if the local checkout doesn't exist.") |
| optimization_group.add_argument( |
| '--git-cache-dir', type='path', |
| help='Git cache directory to use.') |
| optimization_group.add_argument( |
| '--repo-url', help='Repo repository location.') |
| |
| return parser |
| |
| |
| def PrepareManifestVersions(options): |
| """Select manifest-versions checkout to use, and update it. |
| |
| Looks at command line options to decide which manifest-versions checkout to |
| use, and updates (or creates) it as needed. |
| |
| Args: |
| options: Parsed command line options. |
| |
| Returns: |
| Full path to manifest-versions directory to use for this sync. |
| |
| Raises: |
| AssertionError: If the needed manifest-versions path wasn't con the |
| command line. |
| """ |
| site_params = config_lib.GetSiteParams() |
| |
| if options.external: |
| assert options.manifest_versions_ext, '--manifest-versions-ext required.' |
| manifest_versions_url = site_params.MANIFEST_VERSIONS_GOB_URL |
| manifest_versions_path = options.manifest_versions_ext |
| else: |
| assert options.manifest_versions_int, '--manifest-versions-int required.' |
| manifest_versions_url = site_params.MANIFEST_VERSIONS_INT_GOB_URL |
| manifest_versions_path = options.manifest_versions_int |
| |
| # Resolve buildspecs against a current manifest versions value. |
| manifest_version.RefreshManifestCheckout( |
| manifest_versions_path, manifest_versions_url) |
| |
| return manifest_versions_path |
| |
| |
| def ResolveLocalManifestPath(options): |
| """Based on command line options, decide what local manifest file to use. |
| |
| Args: |
| options: Our parsed command line options. |
| |
| Returns: |
| Path to local manifest file to use, or None for no file. |
| """ |
| if options.manifest_file: |
| # If the user gives us an explicit local manifest file, use it. |
| return options.manifest_file |
| |
| elif options.buildspec: |
| # Buildspec builds use a manifest file from manifest_versions. We do NOT |
| # use manifest_versions as the manifest git repo, because it's so large that |
| # sync time would be a major performance problem. |
| manifest_versions_path = PrepareManifestVersions(options) |
| return manifest_version.ResolveBuildspec( |
| manifest_versions_path, options.buildspec) |
| |
| elif options.version: |
| # Versions are a short hand version of a buildspec. |
| manifest_versions_path = PrepareManifestVersions(options) |
| return manifest_version.ResolveBuildspecVersion( |
| manifest_versions_path, options.version) |
| |
| elif options.branch: |
| # Branch checkouts use our normal manifest repos, not a local manifest file. |
| return None |
| |
| else: |
| assert False, 'No sync options specified. Should not be possible.' |
| |
| |
| def main(argv): |
| parser = GetParser() |
| options = parser.parse_args(argv) |
| options.Freeze() |
| |
| local_manifest = ResolveLocalManifestPath(options) |
| |
| if local_manifest: |
| logging.info('Using local_manifest: %s', local_manifest) |
| |
| if options.manifest_url: |
| manifest_url = options.manifest_url |
| elif options.external: |
| manifest_url = config_lib.GetSiteParams().MANIFEST_URL |
| else: |
| manifest_url = config_lib.GetSiteParams().MANIFEST_INT_URL |
| |
| osutils.SafeMakedirs(options.repo_root) |
| repo = repository.RepoRepository( |
| manifest_repo_url=manifest_url, |
| directory=options.repo_root, |
| branch=options.branch, |
| git_cache_dir=options.git_cache_dir, |
| repo_url=options.repo_url, |
| groups=options.groups) |
| |
| if options.copy_repo: |
| repo.PreLoad(options.copy_repo) |
| |
| if repository.IsARepoRoot(options.repo_root): |
| repo.BuildRootGitCleanup(prune_all=True) |
| |
| repo.Sync(local_manifest=local_manifest, detach=True) |
| |
| if options.gerrit_patches: |
| patches = gerrit.GetGerritPatchInfo(options.gerrit_patches) |
| # TODO: Extract patches from manifest synced. |
| |
| helper_pool = patch_series.HelperPool.SimpleCreate( |
| cros_internal=not options.external, cros=True) |
| |
| series = patch_series.PatchSeries( |
| path=options.repo_root, helper_pool=helper_pool, forced_manifest=None) |
| |
| _, failed_tot, failed_inflight = series.Apply(patches) |
| |
| failed = failed_tot + failed_inflight |
| if failed: |
| logging.error('Failed to apply: %s', ', '.join(str(p) for p in failed)) |
| return 1 |