llvm_tot: add an option for recipe builders.
Add an option to launch LLVM TOT builders with recipe builders to
update_packages_and_run_tests.py. The builders launched will not run
hwtests for now.
BUG=chromium:1072063
TEST=local tests.
Change-Id: I1c34090f2d2ea6d9b7efab0a0dccf795396c96d1
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2157624
Tested-by: Jian Cai <jiancai@google.com>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
diff --git a/llvm_tools/README.md b/llvm_tools/README.md
index ce094e9..b0fbea5 100644
--- a/llvm_tools/README.md
+++ b/llvm_tools/README.md
@@ -21,6 +21,8 @@
This script is used for updating a package's LLVM hash (sys-devel/llvm,
sys-libs/compiler-rt, sys-libs/libcxx, sys-libs/libcxxabi, and
sys-libs/llvm-libunwind) and then run tests after updating the git hash.
+There are three ways to test the change, including starting tryjobs,
+recipe builders or using cq+1.
An example when this script should be run is when certain boards would like
to be tested with the updated `LLVM_NEXT_HASH`.
@@ -56,15 +58,15 @@
```
Similarly as the previous example, but for updating `LLVM_NEXT_HASH` to
-the git hash of revision 367622 and test with tryobs:
+the git hash of revision 367622 and test with recipe builders:
```
$ ./update_packages_and_run_tests.py \
--is_llvm_next \
--llvm_version 367622 \
- tryjobs \
- --options nochromesdk latest-toolchain \
- --builders kevin-release-tryjob nocturne-release-tryjob
+ recipe \
+ --options -nocanary \
+ --builders chromeos/toolchain/kevin-llvm chromeos/toolchain/nocturne-llvm
```
## `update_chromeos_llvm_hash.py`
diff --git a/llvm_tools/update_packages_and_run_tests.py b/llvm_tools/update_packages_and_run_tests.py
index 9fe4fe0..f47b5cd 100755
--- a/llvm_tools/update_packages_and_run_tests.py
+++ b/llvm_tools/update_packages_and_run_tests.py
@@ -12,11 +12,11 @@
import datetime
import json
import os
+import subprocess
import chroot
import failure_modes
import get_llvm_hash
-import subprocess_helpers
import update_chromeos_llvm_hash
@@ -93,8 +93,10 @@
'(default: %(default)s)')
subparsers = parser.add_subparsers(dest='subparser_name')
+ subparser_names = []
# Testing with the tryjobs.
tryjob_subparser = subparsers.add_parser('tryjobs')
+ subparser_names.append('tryjobs')
tryjob_subparser.add_argument(
'--builders',
required=True,
@@ -110,12 +112,29 @@
default=[],
help='options to use for the tryjob testing')
+ # Testing with the recipe builders
+ recipe_subparser = subparsers.add_parser('recipe')
+ subparser_names.append('recipe')
+ recipe_subparser.add_argument(
+ '--options',
+ required=False,
+ nargs='+',
+ default=[],
+ help='options passed to the recipe builders')
+
+ recipe_subparser.add_argument(
+ '--builders',
+ required=True,
+ nargs='+',
+ default=[],
+ help='recipe builders to launch')
+
# Testing with CQ.
subparsers.add_parser('cq')
+ subparser_names.append('cq')
args_output = parser.parse_args()
- subparser_names = ['tryjobs', 'cq']
if args_output.subparser_name not in subparser_names:
parser.error('one of %s must be specified' % subparser_names)
@@ -157,11 +176,11 @@
for reviewer in reviewers:
cmd = [gerrit_abs_path, 'reviewers', str(cl), reviewer]
- subprocess_helpers.ExecCommandAndCaptureOutput(cmd)
+ subprocess.check_output(cmd)
-def AddTryjobLinkToCL(tryjobs, cl, chroot_path):
- """Adds the tryjob link(s) to the CL as a comment."""
+def AddLinksToCL(tests, cl, chroot_path):
+ """Adds the test link(s) to the CL as a comment."""
# NOTE: Invoking `cros_sdk` does not make each tryjob link appear on its own
# line, so invoking the `gerrit` command directly instead of using `cros_sdk`
@@ -171,15 +190,12 @@
# newline.
gerrit_abs_path = os.path.join(chroot_path, 'chromite/bin/gerrit')
- tryjob_links = ['Started the following tryjobs:']
- tryjob_links.extend(tryjob['link'] for tryjob in tryjobs)
+ links = ['Started the following tests:']
+ links.extend(test['link'] for test in tests)
- add_message_cmd = [
- gerrit_abs_path, 'message',
- str(cl), '\n'.join(tryjob_links)
- ]
+ add_message_cmd = [gerrit_abs_path, 'message', str(cl), '\n'.join(links)]
- subprocess_helpers.ExecCommandAndCaptureOutput(add_message_cmd)
+ subprocess.check_output(add_message_cmd)
# Testing with tryjobs
@@ -209,16 +225,15 @@
for extra_cl in extra_change_lists:
tryjob_cmd.extend(['-g', '%d' % extra_cl])
- tryjob_cmd.append(builder)
-
if options:
tryjob_cmd.extend('--%s' % option for option in options)
+ tryjob_cmd.append(builder)
+
return tryjob_cmd
-def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
- verbose):
+def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path):
"""Runs a tryjob/tryjobs.
Args:
@@ -228,7 +243,6 @@
options: Any options to be passed into the 'tryjob' command.
builders: All the builders to run the 'tryjob' with.
chroot_path: The absolute path to the chroot.
- verbose: Print command contents to `stdout`.
Returns:
A list that contains stdout contents of each tryjob, where stdout is
@@ -239,42 +253,91 @@
ValueError: Failed to submit a tryjob.
"""
- # Contains the results of each tryjob. The results are retrieved from 'out'
- # which is stdout of the command executer.
- tryjob_results = []
+ # Contains the results of each builder.
+ tests = []
- # For each builder passed into the command line:
- #
- # Run a tryjob with the change list number obtained from updating the
+ # Run tryjobs with the change list number obtained from updating the
# packages and append additional changes lists and options obtained from the
# command line.
- for cur_builder in builders:
- tryjob_cmd = GetTryJobCommand(cl_number, extra_change_lists, options,
- cur_builder)
+ for builder in builders:
+ cmd = GetTryJobCommand(cl_number, extra_change_lists, options, builder)
- out = subprocess_helpers.ChrootRunCommand(
- chroot_path, tryjob_cmd, verbose=verbose)
+ out = subprocess.check_output(cmd, cwd=chroot_path, encoding='utf-8')
- tryjob_launch_time = GetCurrentTimeInUTC()
+ test_output = json.loads(out)
- tryjob_contents = json.loads(out)
-
- buildbucket_id = int(tryjob_contents[0]['buildbucket_id'])
-
- new_tryjob = {
- 'launch_time': str(tryjob_launch_time),
- 'link': str(tryjob_contents[0]['url']),
- 'buildbucket_id': buildbucket_id,
+ tests.append({
+ 'launch_time': str(GetCurrentTimeInUTC()),
+ 'link': str(test_output[0]['url']),
+ 'buildbucket_id': int(test_output[0]['buildbucket_id']),
'extra_cls': extra_change_lists,
'options': options,
- 'builder': [cur_builder]
- }
+ 'builder': [builder]
+ })
- tryjob_results.append(new_tryjob)
+ AddLinksToCL(tests, cl_number, chroot_path)
- AddTryjobLinkToCL(tryjob_results, cl_number, chroot_path)
+ return tests
- return tryjob_results
+
+def StartRecipeBuilders(cl_number, extra_change_lists, options, builders,
+ chroot_path):
+ """Launch recipe builders.
+
+ Args:
+ cl_number: The CL created by updating the packages.
+ extra_change_lists: Any extra change lists that would run alongside the CL
+ that was created by updating the packages ('cl_number').
+ options: Any options to be passed into the 'tryjob' command.
+ builders: All the builders to run the 'tryjob' with.
+ chroot_path: The absolute path to the chroot.
+
+ Returns:
+ A list that contains stdout contents of each builder, where stdout is
+ information (a hashmap) about the tryjob. The hashmap also contains stderr
+ if there was an error when running a tryjob.
+
+ Raises:
+ ValueError: Failed to start a builder.
+ """
+
+ # Contains the results of each builder.
+ tests = []
+
+ # Launch a builders with the change list number obtained from updating the
+ # packages and append additional changes lists and options obtained from the
+ # command line.
+ for builder in builders:
+ cmd = ['bb', 'add', '-json']
+
+ if cl_number:
+ cmd.extend(['-cl', 'crrev.com/c/%d' % cl_number])
+
+ if extra_change_lists:
+ for cl in extra_change_lists:
+ cmd.extend(['-cl', 'crrev.com/c/%d' % cl])
+
+ if options:
+ cmd.extend(options)
+
+ cmd.append(builder)
+
+ out = subprocess.check_output(cmd, cwd=chroot_path, encoding='utf-8')
+
+ test_output = json.loads(out)
+
+ tests.append({
+ 'launch_time': test_output['createTime'],
+ 'link': 'http://ci.chromium.org/b/%s' % test_output['id'],
+ 'buildbucket_id': test_output['id'],
+ 'extra_cls': extra_change_lists,
+ 'options': options,
+ 'builder': [builder]
+ })
+
+ AddLinksToCL(tests, cl_number, chroot_path)
+
+ return tests
# Testing with CQ
@@ -299,7 +362,7 @@
for changes in cl_list:
cq_dry_run_cmd = [gerrit_abs_path, 'label-cq', str(changes), '1']
- subprocess_helpers.ExecCommandAndCaptureOutput(cq_dry_run_cmd)
+ subprocess.check_output(cq_dry_run_cmd)
def main():
@@ -338,7 +401,7 @@
'ebuilds': chroot_file_paths,
'extra_cls': args_output.extra_change_lists,
}
- if args_output.subparser_name == 'tryjobs':
+ if args_output.subparser_name in ('tryjobs', 'recipe'):
arg_dict['builders'] = args_output.builders
arg_dict['tryjob_options'] = args_output.options
if UnchangedSinceLastRun(args_output.last_tested, arg_dict):
@@ -373,13 +436,20 @@
print('Change list number: %d' % change_list.cl_number)
if args_output.subparser_name == 'tryjobs':
- tryjob_results = RunTryJobs(change_list.cl_number,
- args_output.extra_change_lists,
- args_output.options, args_output.builders,
- args_output.chroot_path, args_output.verbose)
- print('Tryjobs:')
- for tryjob in tryjob_results:
- print(tryjob)
+ tests = RunTryJobs(change_list.cl_number, args_output.extra_change_lists,
+ args_output.options, args_output.builders,
+ args_output.chroot_path)
+ print('Tests:')
+ for test in tests:
+ print(test)
+ elif args_output.subparser_name == 'recipe':
+ tests = StartRecipeBuilders(
+ change_list.cl_number, args_output.extra_change_lists,
+ args_output.options, args_output.builders, args_output.chroot_path)
+ print('Tests:')
+ for test in tests:
+ print(test)
+
else:
StartCQDryRun(change_list.cl_number, args_output.extra_change_lists,
args_output.chroot_path)
diff --git a/llvm_tools/update_packages_and_run_tests_unittest.py b/llvm_tools/update_packages_and_run_tests_unittest.py
index 70ecfff..25a8f83 100755
--- a/llvm_tools/update_packages_and_run_tests_unittest.py
+++ b/llvm_tools/update_packages_and_run_tests_unittest.py
@@ -9,13 +9,13 @@
from __future__ import print_function
import json
+import subprocess
import unittest
import unittest.mock as mock
import chroot
import get_llvm_hash
import git
-import subprocess_helpers
import test_helpers
import update_chromeos_llvm_hash
import update_packages_and_run_tests
@@ -68,112 +68,146 @@
last_tested_file, arg_dict), True)
def testGetTryJobCommandWithNoExtraInformation(self):
- test_change_list = 1234
+ change_list = 1234
- test_builder = 'nocturne'
+ builder = 'nocturne'
- expected_tryjob_cmd_list = [
+ expected_cmd = [
'cros', 'tryjob', '--yes', '--json', '-g',
- '%d' % test_change_list, test_builder
+ '%d' % change_list, builder
]
self.assertEqual(
- update_packages_and_run_tests.GetTryJobCommand(test_change_list, None,
- None, test_builder),
- expected_tryjob_cmd_list)
+ update_packages_and_run_tests.GetTryJobCommand(change_list, None, None,
+ builder), expected_cmd)
def testGetTryJobCommandWithExtraInformation(self):
- test_change_list = 4321
- test_extra_cls = [1000, 10]
- test_options = ['report_error', 'delete_tryjob']
- test_builder = 'kevin'
+ change_list = 4321
+ extra_cls = [1000, 10]
+ options = ['option1', 'option2']
+ builder = 'kevin'
- expected_tryjob_cmd_list = [
+ expected_cmd = [
'cros',
'tryjob',
'--yes',
'--json',
'-g',
- '%d' % test_change_list,
+ '%d' % change_list,
'-g',
- '%d' % test_extra_cls[0],
+ '%d' % extra_cls[0],
'-g',
- '%d' % test_extra_cls[1],
- test_builder,
- '--%s' % test_options[0],
- '--%s' % test_options[1],
+ '%d' % extra_cls[1],
+ '--%s' % options[0],
+ '--%s' % options[1],
+ builder,
]
self.assertEqual(
update_packages_and_run_tests.GetTryJobCommand(
- test_change_list, test_extra_cls, test_options, test_builder),
- expected_tryjob_cmd_list)
+ change_list, extra_cls, options, builder), expected_cmd)
@mock.patch.object(
update_packages_and_run_tests,
'GetCurrentTimeInUTC',
return_value='2019-09-09')
- @mock.patch.object(update_packages_and_run_tests, 'AddTryjobLinkToCL')
- @mock.patch.object(subprocess_helpers, 'ChrootRunCommand')
- def testSuccessfullySubmittedTryJob(
- self, mock_chroot_cmd, mock_add_tryjob_link_to_cl, mock_launch_time):
+ @mock.patch.object(update_packages_and_run_tests, 'AddLinksToCL')
+ @mock.patch.object(subprocess, 'check_output')
+ def testSuccessfullySubmittedTryJob(self, mock_cmd, mock_add_links_to_cl,
+ mock_launch_time):
- expected_tryjob_cmd_list = [
+ expected_cmd = [
'cros', 'tryjob', '--yes', '--json', '-g',
'%d' % 900, '-g',
- '%d' % 1200, 'builder1', '--some_option'
+ '%d' % 1200, '--some_option', 'builder1'
]
- buildbucket_id = '1234'
+ bb_id = '1234'
url = 'https://some_tryjob_url.com'
- tryjob_launch_contents = [{'buildbucket_id': buildbucket_id, 'url': url}]
+ mock_cmd.return_value = json.dumps([{'buildbucket_id': bb_id, 'url': url}])
- mock_chroot_cmd.return_value = json.dumps(tryjob_launch_contents)
-
- extra_cls = [1200]
- tryjob_options = ['some_option']
- builder_list = ['builder1']
chroot_path = '/some/path/to/chroot'
- cl_to_launch_tryjob = 900
- verbose = False
+ cl = 900
+ extra_cls = [1200]
+ options = ['some_option']
+ builders = ['builder1']
- tryjob_results_list = update_packages_and_run_tests.RunTryJobs(
- cl_to_launch_tryjob, extra_cls, tryjob_options, builder_list,
- chroot_path, verbose)
+ tests = update_packages_and_run_tests.RunTryJobs(cl, extra_cls, options,
+ builders, chroot_path)
- expected_tryjob_dict = {
+ expected_tests = [{
'launch_time': mock_launch_time.return_value,
'link': url,
- 'buildbucket_id': int(buildbucket_id),
+ 'buildbucket_id': int(bb_id),
'extra_cls': extra_cls,
- 'options': tryjob_options,
- 'builder': builder_list
- }
+ 'options': options,
+ 'builder': builders
+ }]
- self.assertEqual(tryjob_results_list, [expected_tryjob_dict])
+ self.assertEqual(tests, expected_tests)
- mock_chroot_cmd.assert_called_once_with(
- chroot_path, expected_tryjob_cmd_list, verbose=False)
+ mock_cmd.assert_called_once_with(
+ expected_cmd, cwd=chroot_path, encoding='utf-8')
- mock_add_tryjob_link_to_cl.assert_called_once()
+ mock_add_links_to_cl.assert_called_once()
- @mock.patch.object(
- subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
- def testSuccessfullyAddedTryjobLinkToCL(self, mock_exec_cmd):
+ @mock.patch.object(update_packages_and_run_tests, 'AddLinksToCL')
+ @mock.patch.object(subprocess, 'check_output')
+ def testSuccessfullySubmittedRecipeBuilders(self, mock_cmd,
+ mock_add_links_to_cl):
+
+ expected_cmd = [
+ 'bb', 'add', '-json', '-cl',
+ 'crrev.com/c/%s' % 900, '-cl',
+ 'crrev.com/c/%s' % 1200, 'some_option', 'builder1'
+ ]
+
+ bb_id = '1234'
+ create_time = '2020-04-18T00:03:53.978767Z'
+
+ mock_cmd.return_value = json.dumps({'id': bb_id, 'createTime': create_time})
+
+ chroot_path = '/some/path/to/chroot'
+ cl = 900
+ extra_cls = [1200]
+ options = ['some_option']
+ builders = ['builder1']
+
+ tests = update_packages_and_run_tests.StartRecipeBuilders(
+ cl, extra_cls, options, builders, chroot_path)
+
+ expected_tests = [{
+ 'launch_time': create_time,
+ 'link': 'http://ci.chromium.org/b/%s' % bb_id,
+ 'buildbucket_id': bb_id,
+ 'extra_cls': extra_cls,
+ 'options': options,
+ 'builder': builders
+ }]
+
+ self.assertEqual(tests, expected_tests)
+
+ mock_cmd.assert_called_once_with(
+ expected_cmd, cwd=chroot_path, encoding='utf-8')
+
+ mock_add_links_to_cl.assert_called_once()
+
+ @mock.patch.object(subprocess, 'check_output', return_value=None)
+ def testSuccessfullyAddedTestLinkToCL(self, mock_exec_cmd):
chroot_path = '/abs/path/to/chroot'
test_cl_number = 1000
- tryjob_result = [{'link': 'https://some_tryjob_link.com'}]
+ tests = [{'link': 'https://some_tryjob_link.com'}]
- update_packages_and_run_tests.AddTryjobLinkToCL(tryjob_result,
- test_cl_number, chroot_path)
+ update_packages_and_run_tests.AddLinksToCL(tests, test_cl_number,
+ chroot_path)
expected_gerrit_message = [
'%s/chromite/bin/gerrit' % chroot_path, 'message',
str(test_cl_number),
- 'Started the following tryjobs:\n%s' % tryjob_result[0]['link']
+ 'Started the following tests:\n%s' % tests[0]['link']
]
mock_exec_cmd.assert_called_once_with(expected_gerrit_message)
@@ -280,8 +314,7 @@
'\nCq-Depend: chromium:1234, chromium:5678')
# Mock ExecCommandAndCaptureOutput for the gerrit command execution.
- @mock.patch.object(
- subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+ @mock.patch.object(subprocess, 'check_output', return_value=None)
def testStartCQDryRunNoDeps(self, mock_exec_cmd):
chroot_path = '/abs/path/to/chroot'
test_cl_number = 1000
@@ -299,8 +332,7 @@
mock_exec_cmd.assert_called_once_with(expected_gerrit_message)
# Mock ExecCommandAndCaptureOutput for the gerrit command execution.
- @mock.patch.object(
- subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+ @mock.patch.object(subprocess, 'check_output', return_value=None)
# test with a single deps cl.
def testStartCQDryRunSingleDep(self, mock_exec_cmd):
chroot_path = '/abs/path/to/chroot'
@@ -326,8 +358,7 @@
mock.call(expected_gerrit_cmd_2))
# Mock ExecCommandAndCaptureOutput for the gerrit command execution.
- @mock.patch.object(
- subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+ @mock.patch.object(subprocess, 'check_output', return_value=None)
def testStartCQDryRunMultipleDep(self, mock_exec_cmd):
chroot_path = '/abs/path/to/chroot'
test_cl_number = 1000
@@ -359,8 +390,7 @@
mock.call(expected_gerrit_cmd_3))
# Mock ExecCommandAndCaptureOutput for the gerrit command execution.
- @mock.patch.object(
- subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+ @mock.patch.object(subprocess, 'check_output', return_value=None)
# test with no reviewers.
def testAddReviewersNone(self, mock_exec_cmd):
chroot_path = '/abs/path/to/chroot'
@@ -372,8 +402,7 @@
self.assertTrue(mock_exec_cmd.not_called)
# Mock ExecCommandAndCaptureOutput for the gerrit command execution.
- @mock.patch.object(
- subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+ @mock.patch.object(subprocess, 'check_output', return_value=None)
# test with multiple reviewers.
def testAddReviewersMultiple(self, mock_exec_cmd):
chroot_path = '/abs/path/to/chroot'
