get_binhost: Miscellaneous improvements
Move the binhost files out of the sysroot
Storing them in the sysroot leads to conflicts with setup_board:
before any emerging can take place all "source" references in
make.conf need to be resolved. But in a newly created sysroot the
make.conf.binhost file wouldn't exist, causing errors. You can work
around this by manually creating it beforehand, but it's a pain and
it doesn't interact nicely with things like "build_packages --cleanbuild".
Update all boards at once
Since we don't need the sysroot to already exist, we can search across
all postsumbit builders and create files for every available board at
once.
Unfortunately there doesn't seem to be any way to get the profile in
use from the builder attributes, or as a variable available in
make.conf, and portage handles multiple binhosts poorly, so for boards
with multiple profiles in use (such as amd64-generic-asan-fuzzer) we
just have to guess the right one. By default we use
{board}-postsubmit, but the user can specify a builder on the
commandline.
BUG=none
TEST=Manually tested
Change-Id: I4cddd52dd9d5390f459bb86ec6f86e1df622ba12
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/3092895
Commit-Queue: Fergus Dall <sidereal@google.com>
Tested-by: Fergus Dall <sidereal@google.com>
Reviewed-by: David Munro <davidmunro@google.com>
diff --git a/contrib/get_binhost.py b/contrib/get_binhost.py
index d1c2372..2367e82 100755
--- a/contrib/get_binhost.py
+++ b/contrib/get_binhost.py
@@ -4,14 +4,17 @@
# found in the LICENSE file.
import argparse
+from collections import defaultdict
import json
+import os
import subprocess
import sys
from chromite.lib.cros_build_lib import IsInsideChroot
user_conf = '/etc/make.conf.user'
-conf_line = 'source ${ROOT}/etc/make.conf.binhost'
+old_conf_line = 'source ${ROOT}/etc/make.conf.binhost'
+conf_line = 'source /etc/binhost/${BOARD_USE}'
def run_prechecks():
if not IsInsideChroot():
@@ -19,17 +22,40 @@
sys.exit(1)
with open(user_conf, 'r') as f:
+ has_new_conf = False
+ has_old_conf = False
for line in f.read().split('\n'):
if line == conf_line:
- break
- else:
- print(f'Add "{conf_line}" to {user_conf} and rerun')
+ has_new_conf = True
+ if line == old_conf_line:
+ has_old_conf = True
+
+ if not has_new_conf:
+ print(f'Add "{conf_line}" to {user_conf} and rerun.')
+ print('')
+ if has_old_conf:
+ print(f'You have config from a previous version of this '
+ f'script in {user_conf}.')
+ print(f'Remove the line "{old_conf_line}" and rerun.')
+ print('')
+
+ if not has_new_conf or has_old_conf:
sys.exit(1)
-def write_binhost(board, s):
subprocess.run(
- ['sudo', 'tee', f'/build/{board}/etc/make.conf.binhost'],
- input=s.encode(), check=True)
+ ['sudo', 'mkdir', '-p', '/etc/binhost'],
+ check=True)
+
+def write_binhost(board, uri):
+ if uri:
+ data = f'PORTAGE_BINHOST="{uri}"'
+ else:
+ # Make sure to clear the file if there's no BINHOST found.
+ data = ''
+
+ subprocess.run(
+ ['sudo', 'tee', f'/etc/binhost/{board}'],
+ input=data.encode(), stdout=subprocess.DEVNULL, check=True)
def is_internal():
url = subprocess.run(
@@ -65,10 +91,26 @@
query_str = json.dumps(query)
return subprocess.run(
- ['bb', 'ls', '-json', '-fields', 'output', '-predicate', query_str],
+ ['bb', 'ls', '-json', '-fields', 'input,output',
+ '-predicate', query_str],
stdout=subprocess.PIPE,
check=True).stdout.decode()
+def parse_response(bb_ret):
+ # bb unhelpfully returns a bunch on concatenated JSON objects,
+ # rather then a single top-level list, so we need to have our own
+ # parsing loop rather then just doing json.loads().
+
+ decoder = json.JSONDecoder()
+ ret = []
+ while True:
+ try:
+ obj, idx = decoder.raw_decode(bb_ret)
+ ret.append(obj)
+ bb_ret = bb_ret[idx:].strip()
+ except json.decoder.JSONDecodeError:
+ return ret
+
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -80,7 +122,7 @@
means it is effectivly always out of date. See crbug.com/1073565 for
more details.
-This script tries to solve this by looking up a CI build which matches
+This script tries to solve this by looking up CI builds which match
the currently checked out manifest snapshot. Only exact matches are
used, so this is most useful if you sync to the "stable" manifest
branch. If you haven't used this script before, you must add the line
@@ -88,7 +130,19 @@
to the file {user_conf} in your chroot. This script also
relies on the buildbucket CLI tool. If you haven't previously used it,
you will need to run "bb auth-login".""")
- parser.add_argument('board', type=str, help='The board to configure')
+ parser.add_argument(
+ 'builder', nargs='?', default='', type=str,
+ help='By default, we try to make use of all builders. '
+ 'However, for some boards, such as amd64-generic, there '
+ 'are multiple builders with different profiles. By '
+ 'default we assume the one named {board}-postsubmit is '
+ 'the desired builder, if one exists, or the first '
+ 'result otherwise, but if you have configured your '
+ 'build with "setup_board --profile" or similar you may '
+ 'need to set this option to a different builder. Note '
+ 'that this is not persistent. If you run the default '
+ 'form of this command any binhosts you set with this '
+ 'will be overwritten.')
args = parser.parse_args()
@@ -98,7 +152,6 @@
'builder': {
'project': 'chromeos',
'bucket': 'postsubmit',
- 'builder': f'{args.board}-postsubmit',
},
'tags': [{
@@ -109,28 +162,56 @@
'status': 'SUCCESS',
}
+ if args.builder:
+ query['builder']['builder'] = args.builder
+
bb_ret = run_query(query)
# Some postsubmit builders use the external manifest, so if we
- # didn't find anything and we were using the internal manifest,
- # try using the external one instead.
- if not bb_ret.strip() and is_internal():
+ # just used the internal manifest try the external one too to get
+ # more builders.
+ if is_internal():
query['tags'][0]['value'] = (
internal_to_external_snapshot(query['tags'][0]['value']))
- bb_ret = run_query(query)
+ bb_ret += run_query(query)
- # If we still didn't find anything, give up.
- if not bb_ret.strip():
- print('Failed to find an exact postsubmit match')
+ bb_ret = parse_response(bb_ret)
+ if not bb_ret:
+ print('Warning: buildbucket returned no builds.')
- # Clear the config file so builds will fallback to the
- # PORTAGE_BINHOST it would normally use.
- write_binhost(args.board, '')
- sys.exit(1)
+ binhost_map = defaultdict(str)
- bb_ret = json.loads(bb_ret)
- uri = bb_ret['output']['properties']['prebuilts_uri']
- write_binhost(args.board, f'PORTAGE_BINHOST={uri}\n')
+ # Files in /etc/binhost overwrite the normal BINHOST selection, so
+ # we don't want to leave stale data in there ever. Make sure
+ # binhost_map contains an entry for every file we manage. If we
+ # didn't get any results, we will clear the file in write_binhost.
+ #
+ # If we're only querying a single builder, don't worry about
+ # it. Not getting a result for a board doesn't mean that file is
+ # stale.
+ if not args.builder:
+ for path in os.listdir('/etc/binhost'):
+ binhost_map[path] = ''
+
+ for build in bb_ret:
+ if 'build_target' not in build['input']['properties']:
+ # The postsubmit-orchestrator task, skip
+ continue
+ if 'prebuilts_uri' not in build['output']['properties']:
+ # Some builders don't create prebuilts, skip
+ continue
+
+ name = build['builder']['builder']
+ board = build['input']['properties']['build_target']['name']
+ uri = build['output']['properties']['prebuilts_uri']
+ if not binhost_map[board] or name == f'{board}-postsubmit':
+ binhost_map[board] = uri
+
+ for board,uri in binhost_map.items():
+ write_binhost(board, uri)
+
+ binhosts_found = len([i for i in binhost_map.values() if i])
+ print(f'Found binhosts for {binhosts_found} boards')
if __name__ == '__main__':
main()
