| # -*- coding: utf-8 -*- |
| # Copyright 2015 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Library to use the androidbuild API to list and fetch builds.""" |
| |
| from __future__ import print_function |
| |
| import os |
| import pwd |
| |
| import googleapiclient.discovery |
| import googleapiclient.http |
| import httplib2 |
| import oauth2client.client |
| from six.moves import urllib |
| |
| |
| # Locations where to look for credentials JSON files, relative to the user's |
| # home directory. |
| HOMEDIR_JSON_CREDENTIALS_PATH = '.ab_creds.json' |
| |
| # Scope URL on which we need authorization. |
| DEFAULT_SCOPE_URL = 'https://www.googleapis.com/auth/androidbuild.internal' |
| |
| # Default service name and version to connect to. |
| DEFAULT_API_SERVICE_NAME = 'androidbuildinternal' |
| DEFAULT_API_VERSION = 'v2beta1' |
| |
| # Default chunk size to use when downloading files through androidbuild API. |
| DEFAULT_MEDIA_IO_CHUNKSIZE = 20 * 1024 * 1024 # 20MiB |
| |
| |
| class Error(Exception): |
| """Base exception on the androidbuild module.""" |
| |
| |
| class CredentialsNotFoundError(Error): |
| """Credentials file not found.""" |
| |
| |
| def FindCredentialsFile(override_json_credentials_path=None, |
| homedir_json_credentials_path=None): |
| """Find the path to an existing credentials file. |
| |
| Returns the path of the first one that is found. |
| |
| Args: |
| override_json_credentials_path: Path to always use, whenever specified. |
| This is meant for a file specified on a --json-key-file argument on the |
| command line. Whenever present, always use this value. |
| homedir_json_credentials_path: Optional override for the file to be looked |
| for in the user's home directory. Defaults to '.ab_creds.json'. |
| |
| Returns: |
| The resolved path to the file that was found. |
| |
| Raises: |
| CredentialsNotFoundError: If none of the files exist. |
| """ |
| if override_json_credentials_path is not None: |
| return override_json_credentials_path |
| |
| if homedir_json_credentials_path is None: |
| homedir_json_credentials_path = HOMEDIR_JSON_CREDENTIALS_PATH |
| |
| # Check for the file in the user's homedir: |
| user_homedir = os.path.expanduser('~') |
| if user_homedir: |
| json_path = os.path.join(user_homedir, homedir_json_credentials_path) |
| if os.path.exists(json_path): |
| return json_path |
| |
| # If not found, check at ~$PORTAGE_USERNAME. That might be the case if the |
| # tool is being used from within an ebuild script. |
| portage_username = os.environ.get('PORTAGE_USERNAME') |
| if portage_username: |
| try: |
| portage_homedir = pwd.getpwnam(portage_username).pw_dir |
| except KeyError: |
| # User $PORTAGE_USERNAME does not exist. |
| pass |
| else: |
| json_path = os.path.join(portage_homedir, homedir_json_credentials_path) |
| if os.path.exists(json_path): |
| return json_path |
| |
| raise CredentialsNotFoundError( |
| 'Could not find the JSON credentials at [%s] and no JSON file was ' |
| 'specified in command line arguments.' % homedir_json_credentials_path) |
| |
| |
| def LoadCredentials(json_credentials_path=None, scope_url=None): |
| """Load the credentials from a local file. |
| |
| Returns a scoped credentials object which can be used to .authorize() an |
| httlib2.Http() instance used by a googleapiclient. |
| |
| This method works both with service accounts (JSON generated from Pantheon's |
| API manager under Credentials section), or with authenticated users (using a |
| scheme similar to the one used by `gcloud auth login`.) |
| |
| Args: |
| json_credentials_path: Path to a JSON file with credentials for a service |
| account or for authenticated user. Defaults to looking for one using |
| FindCredentialsFile(). |
| scope_url: URL in which the credentials should be scoped. |
| |
| Returns: |
| A scoped oauth2client.client.Credentials object that can be used to |
| authorize an Http instance used by a googleapiclient object. |
| """ |
| json_credentials_path = FindCredentialsFile(json_credentials_path) |
| |
| # This is the way to support both service account credentials (JSON generated |
| # from Pantheon) or authenticated users (similar to `gcloud auth login`). |
| google_creds = oauth2client.client.GoogleCredentials.from_stream( |
| json_credentials_path) |
| |
| if scope_url is None: |
| scope_url = DEFAULT_SCOPE_URL |
| |
| # We need to rescope the credentials which are currently unscoped. |
| scoped_creds = google_creds.create_scoped(scope_url) |
| return scoped_creds |
| |
| |
| def GetApiClient(creds, api_service_name=None, api_version=None): |
| """Build an API client for androidbuild and authorize it. |
| |
| Args: |
| creds: The scoped oauth2client.client.Credentials to use for authorization. |
| api_service_name: Optional override for the API service name. |
| Defaults to 'androidbuildinternal' (from DEFAULT_API_SERVICE_NAME.) |
| api_version: Optional override for the API version. |
| Defaults to 'v2beta1' (from DEFAULT_API_VERSION.) |
| |
| Returns: |
| A googleapiclient.discovery.Resource that supports the androidbuild API |
| methods. |
| """ |
| if api_service_name is None: |
| api_service_name = DEFAULT_API_SERVICE_NAME |
| if api_version is None: |
| api_version = DEFAULT_API_VERSION |
| |
| base_http_client = httplib2.Http() |
| auth_http_client = creds.authorize(base_http_client) |
| ab_client = googleapiclient.discovery.build(api_service_name, api_version, |
| http=auth_http_client) |
| return ab_client |
| |
| |
| def FetchArtifact(ab_client, branch, target, build_id, filepath, output=None): |
| """Fetches an artifact using the API client. |
| |
| Args: |
| ab_client: The androidbuild API client. |
| branch: The name of the git branch. (Currently UNUSED!) |
| target: The name of the build target. |
| build_id: The id of the build. |
| filepath: Path to the file to download. |
| output: Path where to store the artifact. Defaults to filepath. |
| |
| Raises: |
| googleapiclient.errors.HttpError: If the requested artifact does not exist. |
| """ |
| # The "branch" is unused, so silent pylint warnings about it: |
| _ = branch |
| |
| # Get the media id to download. |
| # NOTE: For some reason the git branch is not needed here, which looks weird. |
| # That means in the ab:// URL the branch name will be essentially ignored. |
| media_id = ab_client.buildartifact().get_media( |
| target=target, |
| buildId=build_id, |
| attemptId='latest', |
| resourceId=filepath) |
| |
| if output is None: |
| output = filepath |
| |
| # Create directory structure, if needed. |
| outdir = os.path.dirname(output) |
| if outdir and not os.path.isdir(outdir): |
| os.makedirs(outdir) |
| |
| with open(output, 'wb') as f: |
| downloader = googleapiclient.http.MediaIoBaseDownload( |
| f, media_id, chunksize=DEFAULT_MEDIA_IO_CHUNKSIZE) |
| done = False |
| while not done: |
| _, done = downloader.next_chunk() |
| |
| |
| def FindRecentBuilds(ab_client, branch, target, |
| build_type='submitted', |
| build_attempt_status=None, |
| build_successful=None): |
| """Queries for the latest build_ids from androidbuild. |
| |
| Args: |
| ab_client: The androidbuild API client. |
| branch: The name of the git branch. |
| target: The name of the build target. |
| build_type: (Optional) The type of the build, defaults to 'submitted'. |
| build_attempt_status: (Optional) Status of attempt, use 'complete' to look |
| for completed builds only. |
| build_successful: (Optional) Whether to only return successful builds. |
| |
| Returns: |
| A list of numeric build_ids, sorted from most recent to oldest (in reverse |
| numerical order.) |
| """ |
| kwargs = { |
| 'branch': branch, |
| 'target': target, |
| } |
| if build_type is not None: |
| kwargs['buildType'] = build_type |
| if build_attempt_status is not None: |
| kwargs['buildAttemptStatus'] = build_attempt_status |
| if build_successful is not None: |
| kwargs['successful'] = build_successful |
| builds = ab_client.build().list(**kwargs).execute().get('builds') |
| |
| # Extract the build_ids, convert to int, arrange newest to oldest. |
| return sorted((int(build['buildId']) for build in builds), reverse=True) |
| |
| |
| def FindLatestGreenBuildId(ab_client, branch, target): |
| """Finds the latest build_id that has a green build. |
| |
| Args: |
| ab_client: The androidbuild API client. |
| branch: The name of the git branch. |
| target: The name of the build target. |
| |
| Returns: |
| A numeric build_id for the latest green build. |
| Returns None if no green builds were found for this branch and target. |
| """ |
| build_ids = FindRecentBuilds(ab_client, branch, target, |
| build_successful=True) |
| if build_ids: |
| return build_ids[0] |
| else: |
| return None |
| |
| |
| def SplitAbUrl(ab_url): |
| """Splits an ab://... URL into its fields. |
| |
| The URL has the following format: |
| ab://android-build/<branch>/<target>/<build_id>/<filepath> |
| |
| The "android-build" part is the <host> or <bucket> and for now is required to |
| be the literal "android-build" (we reserve it to extend the URL format in the |
| future.) |
| |
| <branch> is the git branch and <target> is the board name plus one of -user |
| or -userdebug or -eng or such. <build_id> is the numeric identifier of the |
| build. Finally, <filepath> is the path to the artifact itself. |
| |
| The two last components (<build_id> and <filepath>) may be absent from the |
| URL. An ab:// URL without a <branch> or <target> is invalid (for now.) |
| |
| Args: |
| ab_url: An ab://... URL. |
| |
| Returns: |
| A 4-tuple: branch, target, build_id, filepath. The two last components will |
| be set to None if they are absent from the URL. The returned <build_id> |
| component will be an integer, all others will be strings. |
| |
| Raises: |
| ValueError: If the URL is not a valid ab://... URL. |
| """ |
| o = urllib.parse.urlparse(ab_url) |
| |
| if o.scheme != 'ab': |
| raise ValueError('URL [%s] must start with ab:// protocol.' % ab_url) |
| |
| if o.hostname != 'android-build': |
| raise ValueError('URL [%s] must use "android-build" bucket.' % ab_url) |
| |
| # Split the remaining fields of the path. |
| parts = o.path.split('/', 4) |
| |
| if len(parts) < 3: |
| raise ValueError( |
| 'URL [%s] is too short and does not specify a target.' % ab_url) |
| |
| # First field will be empty. |
| assert parts[0] == '' |
| branch = urllib.parse.unquote(parts[1]) |
| target = urllib.parse.unquote(parts[2]) |
| |
| if not branch: |
| raise ValueError('URL [%s] has an empty branch.' % ab_url) |
| |
| if not target: |
| raise ValueError('URL [%s] has an empty target.' % ab_url) |
| |
| # Check if build_id is present. If present, it must be numeric. |
| if len(parts) > 3: |
| build_id_str = urllib.parse.unquote(parts[3]) |
| if not build_id_str.isdigit(): |
| raise ValueError('URL [%s] has a non-numeric build_id component [%s].' % |
| (ab_url, build_id_str)) |
| build_id = int(build_id_str) |
| else: |
| build_id = None |
| |
| # Last, use the remainder of the URL as the filepath. |
| if len(parts) > 4: |
| filepath = urllib.parse.unquote(parts[4]) |
| else: |
| filepath = None |
| |
| return (branch, target, build_id, filepath) |