| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # Copyright 2020 The ChromiumOS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Checks for various upstream events with the Rust toolchain. |
| |
| Sends an email if something interesting (probably) happened. |
| """ |
| |
| import argparse |
| import itertools |
| import json |
| import logging |
| import pathlib |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import time |
| from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple |
| |
| from cros_utils import bugs, email_sender, tiny_render |
| |
| |
| def gentoo_sha_to_link(sha: str) -> str: |
| """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" |
| return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}' |
| |
| |
| def send_email(subject: str, body: List[tiny_render.Piece]) -> None: |
| """Sends an email with the given title and body to... whoever cares.""" |
| email_sender.EmailSender().SendX20Email( |
| subject=subject, |
| identifier='rust-watch', |
| well_known_recipients=['cros-team'], |
| text_body=tiny_render.render_text_pieces(body), |
| html_body=tiny_render.render_html_pieces(body), |
| ) |
| |
| |
| class RustReleaseVersion(NamedTuple): |
| """Represents a version of Rust's stable compiler.""" |
| major: int |
| minor: int |
| patch: int |
| |
| @staticmethod |
| def from_string(version_string: str) -> 'RustReleaseVersion': |
| m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string) |
| if not m: |
| raise ValueError(f"{version_string!r} isn't a valid version string") |
| return RustReleaseVersion(*[int(x) for x in m.groups()]) |
| |
| def __str__(self) -> str: |
| return f'{self.major}.{self.minor}.{self.patch}' |
| |
| def to_json(self) -> str: |
| return str(self) |
| |
| @staticmethod |
| def from_json(s: str) -> 'RustReleaseVersion': |
| return RustReleaseVersion.from_string(s) |
| |
| |
| class State(NamedTuple): |
| """State that we keep around from run to run.""" |
| # The last Rust release tag that we've seen. |
| last_seen_release: RustReleaseVersion |
| |
| # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen |
| # that updates it. |
| last_gentoo_sha: str |
| |
| def to_json(self) -> Dict[str, Any]: |
| return { |
| 'last_seen_release': self.last_seen_release.to_json(), |
| 'last_gentoo_sha': self.last_gentoo_sha, |
| } |
| |
| @staticmethod |
| def from_json(s: Dict[str, Any]) -> 'State': |
| return State( |
| last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']), |
| last_gentoo_sha=s['last_gentoo_sha'], |
| ) |
| |
| |
| def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]: |
| """Parses `git ls-remote --tags` output into Rust stable release versions.""" |
| refs_tags = 'refs/tags/' |
| for line in lines: |
| _sha, tag = line.split(None, 1) |
| tag = tag.strip() |
| # Each tag has an associated 'refs/tags/name^{}', which is the actual |
| # object that the tag points to. That's irrelevant to us. |
| if tag.endswith('^{}'): |
| continue |
| |
| if not tag.startswith(refs_tags): |
| continue |
| |
| short_tag = tag[len(refs_tags):] |
| # There are a few old versioning schemes. Ignore them. |
| if short_tag.startswith('0.') or short_tag.startswith('release-'): |
| continue |
| yield RustReleaseVersion.from_string(short_tag) |
| |
| |
| def fetch_most_recent_release() -> RustReleaseVersion: |
| """Fetches the most recent stable `rustc` version.""" |
| result = subprocess.run( |
| ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'], |
| check=True, |
| stdin=None, |
| capture_output=True, |
| encoding='utf-8', |
| ) |
| tag_lines = result.stdout.strip().splitlines() |
| return max(parse_release_tags(tag_lines)) |
| |
| |
| class GitCommit(NamedTuple): |
| """Represents a single git commit.""" |
| sha: str |
| subject: str |
| |
| |
| def update_git_repo(git_dir: pathlib.Path) -> None: |
| """Updates the repo at `git_dir`, retrying a few times on failure.""" |
| for i in itertools.count(start=1): |
| result = subprocess.run( |
| ['git', 'fetch', 'origin'], |
| check=False, |
| cwd=str(git_dir), |
| stdin=None, |
| ) |
| |
| if not result.returncode: |
| break |
| |
| if i == 5: |
| # 5 attempts is too many. Something else may be wrong. |
| result.check_returncode() |
| |
| sleep_time = 60 * i |
| logging.error("Failed updating gentoo's repo; will try again in %ds...", |
| sleep_time) |
| time.sleep(sleep_time) |
| |
| |
| def get_new_gentoo_commits(git_dir: pathlib.Path, |
| most_recent_sha: str) -> List[GitCommit]: |
| """Gets commits to dev-lang/rust since `most_recent_sha`. |
| |
| Older commits come earlier in the returned list. |
| """ |
| commits = subprocess.run( |
| [ |
| 'git', |
| 'log', |
| '--format=%H %s', |
| f'{most_recent_sha}..origin/master', # nocheck |
| '--', |
| 'dev-lang/rust', |
| ], |
| capture_output=True, |
| check=False, |
| cwd=str(git_dir), |
| encoding='utf-8', |
| ) |
| |
| if commits.returncode: |
| logging.error('Error getting new gentoo commits; stderr:\n%s', |
| commits.stderr) |
| commits.check_returncode() |
| |
| results = [] |
| for line in commits.stdout.strip().splitlines(): |
| sha, subject = line.strip().split(None, 1) |
| results.append(GitCommit(sha=sha, subject=subject)) |
| |
| # `git log` outputs things in newest -> oldest order. |
| results.reverse() |
| return results |
| |
| |
| def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str: |
| """Sets up a gentoo git repo at the given directory. Returns HEAD.""" |
| subprocess.run( |
| [ |
| 'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git', |
| str(git_dir) |
| ], |
| stdin=None, |
| check=True, |
| ) |
| |
| head_rev = subprocess.run( |
| ['git', 'rev-parse', 'HEAD'], |
| cwd=str(git_dir), |
| check=True, |
| stdin=None, |
| capture_output=True, |
| encoding='utf-8', |
| ) |
| return head_rev.stdout.strip() |
| |
| |
| def read_state(state_file: pathlib.Path) -> State: |
| """Reads state from the given file.""" |
| with state_file.open(encoding='utf-8') as f: |
| return State.from_json(json.load(f)) |
| |
| |
| def atomically_write_state(state_file: pathlib.Path, state: State) -> None: |
| """Writes state to the given file.""" |
| temp_file = pathlib.Path(str(state_file) + '.new') |
| with temp_file.open('w', encoding='utf-8') as f: |
| json.dump(state.to_json(), f) |
| temp_file.rename(state_file) |
| |
| |
| def file_bug(title: str, body: str) -> None: |
| """Files a bug against gbiv@ with the given title/body.""" |
| bugs.CreateNewBug( |
| bugs.WellKnownComponents.CrOSToolchainPublic, |
| title, |
| body, |
| # To either take or reassign depending on the rotation. |
| assignee='gbiv@google.com', |
| ) |
| |
| |
| def maybe_compose_bug( |
| old_state: State, |
| newest_release: RustReleaseVersion, |
| ) -> Optional[Tuple[str, str]]: |
| """Creates a bug to file about the new release, if doing is desired.""" |
| if newest_release == old_state.last_seen_release: |
| return None |
| |
| title = f'[Rust] Update to {newest_release}' |
| body = ('A new release has been detected; we should probably roll to it. ' |
| "Please see go/crostc-rust-rotation for who's turn it is.") |
| return title, body |
| |
| |
| def maybe_compose_email( |
| new_gentoo_commits: List[GitCommit] |
| ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: |
| """Creates an email given our new state, if doing so is appropriate.""" |
| if not new_gentoo_commits: |
| return None |
| |
| subject_pieces = [] |
| body_pieces = [] |
| |
| # Separate the sections a bit for prettier output. |
| if body_pieces: |
| body_pieces += [tiny_render.line_break, tiny_render.line_break] |
| |
| if len(new_gentoo_commits) == 1: |
| subject_pieces.append('new rust ebuild commit detected') |
| body_pieces.append('commit:') |
| else: |
| subject_pieces.append('new rust ebuild commits detected') |
| body_pieces.append('commits (newest first):') |
| |
| commit_lines = [] |
| for commit in new_gentoo_commits: |
| commit_lines.append([ |
| tiny_render.Link( |
| gentoo_sha_to_link(commit.sha), |
| commit.sha[:12], |
| ), |
| f': {commit.subject}', |
| ]) |
| |
| body_pieces.append(tiny_render.UnorderedList(commit_lines)) |
| |
| subject = '[rust-watch] ' + '; '.join(subject_pieces) |
| return subject, body_pieces |
| |
| |
| def main(argv: List[str]) -> None: |
| logging.basicConfig(level=logging.INFO) |
| |
| parser = argparse.ArgumentParser( |
| description=__doc__, |
| formatter_class=argparse.RawDescriptionHelpFormatter) |
| parser.add_argument('--state_dir', |
| required=True, |
| help='Directory to store state in.') |
| parser.add_argument('--skip_side_effects', |
| action='store_true', |
| help="Don't send an email or file a bug.") |
| parser.add_argument( |
| '--skip_state_update', |
| action='store_true', |
| help="Don't update the state file. Doesn't apply to initial setup.") |
| opts = parser.parse_args(argv) |
| |
| state_dir = pathlib.Path(opts.state_dir) |
| state_file = state_dir / 'state.json' |
| gentoo_subdir = state_dir / 'upstream-gentoo' |
| if not state_file.exists(): |
| logging.info("state_dir isn't fully set up; doing that now.") |
| |
| # Could be in a partially set-up state. |
| if state_dir.exists(): |
| logging.info('incomplete state_dir detected; removing.') |
| shutil.rmtree(str(state_dir)) |
| |
| state_dir.mkdir(parents=True) |
| most_recent_release = fetch_most_recent_release() |
| most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) |
| atomically_write_state( |
| state_file, |
| State( |
| last_seen_release=most_recent_release, |
| last_gentoo_sha=most_recent_gentoo_commit, |
| ), |
| ) |
| # Running through this _should_ be a nop, but do it anyway. Should make any |
| # bugs more obvious on the first run of the script. |
| |
| prior_state = read_state(state_file) |
| logging.info('Last state was %r', prior_state) |
| |
| most_recent_release = fetch_most_recent_release() |
| logging.info('Most recent Rust release is %s', most_recent_release) |
| |
| logging.info('Fetching new commits from Gentoo') |
| update_git_repo(gentoo_subdir) |
| new_commits = get_new_gentoo_commits(gentoo_subdir, |
| prior_state.last_gentoo_sha) |
| logging.info('New commits: %r', new_commits) |
| |
| maybe_bug = maybe_compose_bug(prior_state, most_recent_release) |
| maybe_email = maybe_compose_email(new_commits) |
| |
| if maybe_bug is None: |
| logging.info('No bug to file') |
| else: |
| title, body = maybe_bug |
| if opts.skip_side_effects: |
| logging.info('Skipping sending bug with title %r and contents\n%s', |
| title, body) |
| else: |
| logging.info('Writing new bug') |
| file_bug(title, body) |
| |
| if maybe_email is None: |
| logging.info('No email to send') |
| else: |
| title, body = maybe_email |
| if opts.skip_side_effects: |
| logging.info('Skipping sending email with title %r and contents\n%s', |
| title, tiny_render.render_html_pieces(body)) |
| else: |
| logging.info('Sending email') |
| send_email(title, body) |
| |
| if opts.skip_state_update: |
| logging.info('Skipping state update, as requested') |
| return |
| |
| newest_sha = (new_commits[-1].sha |
| if new_commits else prior_state.last_gentoo_sha) |
| atomically_write_state( |
| state_file, |
| State( |
| last_seen_release=most_recent_release, |
| last_gentoo_sha=newest_sha, |
| ), |
| ) |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |