blob: 2367e8259c58a0d0c611a3085eb8ed8e518036d5 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2021 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
from collections import defaultdict
import json
import os
import subprocess
import sys
from chromite.lib.cros_build_lib import IsInsideChroot
user_conf = '/etc/make.conf.user'
old_conf_line = 'source ${ROOT}/etc/make.conf.binhost'
conf_line = 'source /etc/binhost/${BOARD_USE}'
def run_prechecks():
if not IsInsideChroot():
print('This script must be run inside the CrOS chroot')
with open(user_conf, 'r') as f:
has_new_conf = False
has_old_conf = False
for line in'\n'):
if line == conf_line:
has_new_conf = True
if line == old_conf_line:
has_old_conf = True
if not has_new_conf:
print(f'Add "{conf_line}" to {user_conf} and rerun.')
if has_old_conf:
print(f'You have config from a previous version of this '
f'script in {user_conf}.')
print(f'Remove the line "{old_conf_line}" and rerun.')
if not has_new_conf or has_old_conf:
['sudo', 'mkdir', '-p', '/etc/binhost'],
def write_binhost(board, uri):
if uri:
data = f'PORTAGE_BINHOST="{uri}"'
# Make sure to clear the file if there's no BINHOST found.
data = ''
['sudo', 'tee', f'/etc/binhost/{board}'],
input=data.encode(), stdout=subprocess.DEVNULL, check=True)
def is_internal():
url =
['git', 'config', '--get', 'remote.origin.url'],
if '' in url:
return True
if '' in url:
return False
print(f'Unknown manifest source {url}')
def get_current_snapshot():
['git', 'rev-parse', 'HEAD'],
def internal_to_external_snapshot(rev):
['git', 'footers', '--key', 'Cr-External-Snapshot', rev],
def run_query(query):
query_str = json.dumps(query)
['bb', 'ls', '-json', '-fields', 'input,output',
'-predicate', query_str],
def parse_response(bb_ret):
# bb unhelpfully returns a bunch on concatenated JSON objects,
# rather then a single top-level list, so we need to have our own
# parsing loop rather then just doing json.loads().
decoder = json.JSONDecoder()
ret = []
while True:
obj, idx = decoder.raw_decode(bb_ret)
bb_ret = bb_ret[idx:].strip()
except json.decoder.JSONDecodeError:
return ret
def main():
parser = argparse.ArgumentParser(
description=f"""Find up-to-date binary package sources
Chrome OS tries to speed up builds by using binary packages taken from
CI builds. However, the configuration file that controls which build
these packages are taken from is itself a part of the git repo, which
means it is effectivly always out of date. See for
more details.
This script tries to solve this by looking up CI builds which match
the currently checked out manifest snapshot. Only exact matches are
used, so this is most useful if you sync to the "stable" manifest
branch. If you haven't used this script before, you must add the line
to the file {user_conf} in your chroot. This script also
relies on the buildbucket CLI tool. If you haven't previously used it,
you will need to run "bb auth-login".""")
'builder', nargs='?', default='', type=str,
help='By default, we try to make use of all builders. '
'However, for some boards, such as amd64-generic, there '
'are multiple builders with different profiles. By '
'default we assume the one named {board}-postsubmit is '
'the desired builder, if one exists, or the first '
'result otherwise, but if you have configured your '
'build with "setup_board --profile" or similar you may '
'need to set this option to a different builder. Note '
'that this is not persistent. If you run the default '
'form of this command any binhosts you set with this '
'will be overwritten.')
args = parser.parse_args()
query = {
'builder': {
'project': 'chromeos',
'bucket': 'postsubmit',
'tags': [{
'key': 'snapshot',
'value': get_current_snapshot(),
'status': 'SUCCESS',
if args.builder:
query['builder']['builder'] = args.builder
bb_ret = run_query(query)
# Some postsubmit builders use the external manifest, so if we
# just used the internal manifest try the external one too to get
# more builders.
if is_internal():
query['tags'][0]['value'] = (
bb_ret += run_query(query)
bb_ret = parse_response(bb_ret)
if not bb_ret:
print('Warning: buildbucket returned no builds.')
binhost_map = defaultdict(str)
# Files in /etc/binhost overwrite the normal BINHOST selection, so
# we don't want to leave stale data in there ever. Make sure
# binhost_map contains an entry for every file we manage. If we
# didn't get any results, we will clear the file in write_binhost.
# If we're only querying a single builder, don't worry about
# it. Not getting a result for a board doesn't mean that file is
# stale.
if not args.builder:
for path in os.listdir('/etc/binhost'):
binhost_map[path] = ''
for build in bb_ret:
if 'build_target' not in build['input']['properties']:
# The postsubmit-orchestrator task, skip
if 'prebuilts_uri' not in build['output']['properties']:
# Some builders don't create prebuilts, skip
name = build['builder']['builder']
board = build['input']['properties']['build_target']['name']
uri = build['output']['properties']['prebuilts_uri']
if not binhost_map[board] or name == f'{board}-postsubmit':
binhost_map[board] = uri
for board,uri in binhost_map.items():
write_binhost(board, uri)
binhosts_found = len([i for i in binhost_map.values() if i])
print(f'Found binhosts for {binhosts_found} boards')
if __name__ == '__main__':