| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # Copyright 2020 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Checks for various upstream events with the Rust toolchain. |
| |
| Sends an email if something interesting (probably) happened. |
| """ |
| |
| import argparse |
| import itertools |
| import json |
| import logging |
| import pathlib |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import time |
| from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple |
| |
| from cros_utils import bugs |
| from cros_utils import email_sender |
| from cros_utils import tiny_render |
| |
| |
| def gentoo_sha_to_link(sha: str) -> str: |
| """Gets a URL to a webpage that shows the Gentoo commit at `sha`.""" |
| return f"https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}" |
| |
| |
| def send_email(subject: str, body: List[tiny_render.Piece]) -> None: |
| """Sends an email with the given title and body to... whoever cares.""" |
| email_sender.EmailSender().SendX20Email( |
| subject=subject, |
| identifier="rust-watch", |
| well_known_recipients=["cros-team"], |
| text_body=tiny_render.render_text_pieces(body), |
| html_body=tiny_render.render_html_pieces(body), |
| ) |
| |
| |
| class RustReleaseVersion(NamedTuple): |
| """Represents a version of Rust's stable compiler.""" |
| |
| major: int |
| minor: int |
| patch: int |
| |
| @staticmethod |
| def from_string(version_string: str) -> "RustReleaseVersion": |
| m = re.match(r"(\d+)\.(\d+)\.(\d+)", version_string) |
| if not m: |
| raise ValueError(f"{version_string!r} isn't a valid version string") |
| return RustReleaseVersion(*[int(x) for x in m.groups()]) |
| |
| def __str__(self) -> str: |
| return f"{self.major}.{self.minor}.{self.patch}" |
| |
| def to_json(self) -> str: |
| return str(self) |
| |
| @staticmethod |
| def from_json(s: str) -> "RustReleaseVersion": |
| return RustReleaseVersion.from_string(s) |
| |
| |
| class State(NamedTuple): |
| """State that we keep around from run to run.""" |
| |
| # The last Rust release tag that we've seen. |
| last_seen_release: RustReleaseVersion |
| |
| # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen |
| # that updates it. |
| last_gentoo_sha: str |
| |
| def to_json(self) -> Dict[str, Any]: |
| return { |
| "last_seen_release": self.last_seen_release.to_json(), |
| "last_gentoo_sha": self.last_gentoo_sha, |
| } |
| |
| @staticmethod |
| def from_json(s: Dict[str, Any]) -> "State": |
| return State( |
| last_seen_release=RustReleaseVersion.from_json( |
| s["last_seen_release"] |
| ), |
| last_gentoo_sha=s["last_gentoo_sha"], |
| ) |
| |
| |
| def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]: |
| """Parses `git ls-remote --tags` output into Rust stable release versions.""" |
| refs_tags = "refs/tags/" |
| for line in lines: |
| _sha, tag = line.split(None, 1) |
| tag = tag.strip() |
| # Each tag has an associated 'refs/tags/name^{}', which is the actual |
| # object that the tag points to. That's irrelevant to us. |
| if tag.endswith("^{}"): |
| continue |
| |
| if not tag.startswith(refs_tags): |
| continue |
| |
| short_tag = tag[len(refs_tags) :] |
| # There are a few old versioning schemes. Ignore them. |
| if short_tag.startswith("0.") or short_tag.startswith("release-"): |
| continue |
| yield RustReleaseVersion.from_string(short_tag) |
| |
| |
| def fetch_most_recent_release() -> RustReleaseVersion: |
| """Fetches the most recent stable `rustc` version.""" |
| result = subprocess.run( |
| ["git", "ls-remote", "--tags", "https://github.com/rust-lang/rust"], |
| check=True, |
| stdin=None, |
| capture_output=True, |
| encoding="utf-8", |
| ) |
| tag_lines = result.stdout.strip().splitlines() |
| return max(parse_release_tags(tag_lines)) |
| |
| |
| class GitCommit(NamedTuple): |
| """Represents a single git commit.""" |
| |
| sha: str |
| subject: str |
| |
| |
| def update_git_repo(git_dir: pathlib.Path) -> None: |
| """Updates the repo at `git_dir`, retrying a few times on failure.""" |
| for i in itertools.count(start=1): |
| result = subprocess.run( |
| ["git", "fetch", "origin"], |
| check=False, |
| cwd=str(git_dir), |
| stdin=None, |
| ) |
| |
| if not result.returncode: |
| break |
| |
| if i == 5: |
| # 5 attempts is too many. Something else may be wrong. |
| result.check_returncode() |
| |
| sleep_time = 60 * i |
| logging.error( |
| "Failed updating gentoo's repo; will try again in %ds...", |
| sleep_time, |
| ) |
| time.sleep(sleep_time) |
| |
| |
| def get_new_gentoo_commits( |
| git_dir: pathlib.Path, most_recent_sha: str |
| ) -> List[GitCommit]: |
| """Gets commits to dev-lang/rust since `most_recent_sha`. |
| |
| Older commits come earlier in the returned list. |
| """ |
| commits = subprocess.run( |
| [ |
| "git", |
| "log", |
| "--format=%H %s", |
| f"{most_recent_sha}..origin/master", # nocheck |
| "--", |
| "dev-lang/rust", |
| ], |
| capture_output=True, |
| check=False, |
| cwd=str(git_dir), |
| encoding="utf-8", |
| ) |
| |
| if commits.returncode: |
| logging.error( |
| "Error getting new gentoo commits; stderr:\n%s", commits.stderr |
| ) |
| commits.check_returncode() |
| |
| results = [] |
| for line in commits.stdout.strip().splitlines(): |
| sha, subject = line.strip().split(None, 1) |
| results.append(GitCommit(sha=sha, subject=subject)) |
| |
| # `git log` outputs things in newest -> oldest order. |
| results.reverse() |
| return results |
| |
| |
| def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str: |
| """Sets up a gentoo git repo at the given directory. Returns HEAD.""" |
| subprocess.run( |
| [ |
| "git", |
| "clone", |
| "https://anongit.gentoo.org/git/repo/gentoo.git", |
| str(git_dir), |
| ], |
| stdin=None, |
| check=True, |
| ) |
| |
| head_rev = subprocess.run( |
| ["git", "rev-parse", "HEAD"], |
| cwd=str(git_dir), |
| check=True, |
| stdin=None, |
| capture_output=True, |
| encoding="utf-8", |
| ) |
| return head_rev.stdout.strip() |
| |
| |
| def read_state(state_file: pathlib.Path) -> State: |
| """Reads state from the given file.""" |
| with state_file.open(encoding="utf-8") as f: |
| return State.from_json(json.load(f)) |
| |
| |
| def atomically_write_state(state_file: pathlib.Path, state: State) -> None: |
| """Writes state to the given file.""" |
| temp_file = pathlib.Path(str(state_file) + ".new") |
| with temp_file.open("w", encoding="utf-8") as f: |
| json.dump(state.to_json(), f) |
| temp_file.rename(state_file) |
| |
| |
| def file_bug(title: str, body: str) -> None: |
| """Files update bugs with the given title/body.""" |
| # (component, optional_assignee) |
| targets = ( |
| (bugs.WellKnownComponents.CrOSToolchainPublic, "gbiv@google.com"), |
| # b/269170429: Some Android folks said they wanted this before, and |
| # figuring out the correct way to apply permissions has been a pain. No |
| # one seems to be missing these notifications & the Android Rust folks |
| # are keeping on top of their toolchain, so ignore this for now. |
| # (bugs.WellKnownComponents.AndroidRustToolchain, None), |
| ) |
| for component, assignee in targets: |
| bugs.CreateNewBug(component, title, body, assignee) |
| |
| |
| def maybe_compose_bug( |
| old_state: State, |
| newest_release: RustReleaseVersion, |
| ) -> Optional[Tuple[str, str]]: |
| """Creates a bug to file about the new release, if doing is desired.""" |
| if newest_release == old_state.last_seen_release: |
| return None |
| |
| title = f"[Rust] Update to {newest_release}" |
| body = ( |
| "A new Rust stable release has been detected; we should probably roll " |
| "to it.\n" |
| "\n" |
| "The regression-from-stable-to-stable tag might be interesting to " |
| "keep an eye on: https://github.com/rust-lang/rust/labels/" |
| "regression-from-stable-to-stable\n" |
| "\n" |
| "If you notice any bugs or issues you'd like to share, please " |
| "also note them on go/shared-rust-update-notes.\n" |
| "\n" |
| "See go/crostc-rust-rotation for the current rotation schedule.\n" |
| "\n" |
| "For questions about this bot, please contact chromeos-toolchain@ and " |
| "CC gbiv@." |
| ) |
| return title, body |
| |
| |
| def maybe_compose_email( |
| new_gentoo_commits: List[GitCommit], |
| ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: |
| """Creates an email given our new state, if doing so is appropriate.""" |
| if not new_gentoo_commits: |
| return None |
| |
| subject_pieces = [] |
| body_pieces = [] |
| |
| # Separate the sections a bit for prettier output. |
| if body_pieces: |
| body_pieces += [tiny_render.line_break, tiny_render.line_break] |
| |
| if len(new_gentoo_commits) == 1: |
| subject_pieces.append("new rust ebuild commit detected") |
| body_pieces.append("commit:") |
| else: |
| subject_pieces.append("new rust ebuild commits detected") |
| body_pieces.append("commits (newest first):") |
| |
| commit_lines = [] |
| for commit in new_gentoo_commits: |
| commit_lines.append( |
| [ |
| tiny_render.Link( |
| gentoo_sha_to_link(commit.sha), |
| commit.sha[:12], |
| ), |
| f": {commit.subject}", |
| ] |
| ) |
| |
| body_pieces.append(tiny_render.UnorderedList(commit_lines)) |
| |
| subject = "[rust-watch] " + "; ".join(subject_pieces) |
| return subject, body_pieces |
| |
| |
| def main(argv: List[str]) -> None: |
| logging.basicConfig(level=logging.INFO) |
| |
| parser = argparse.ArgumentParser( |
| description=__doc__, |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| ) |
| parser.add_argument( |
| "--state_dir", required=True, help="Directory to store state in." |
| ) |
| parser.add_argument( |
| "--skip_side_effects", |
| action="store_true", |
| help="Don't send an email or file a bug.", |
| ) |
| parser.add_argument( |
| "--skip_state_update", |
| action="store_true", |
| help="Don't update the state file. Doesn't apply to initial setup.", |
| ) |
| opts = parser.parse_args(argv) |
| |
| state_dir = pathlib.Path(opts.state_dir) |
| state_file = state_dir / "state.json" |
| gentoo_subdir = state_dir / "upstream-gentoo" |
| if not state_file.exists(): |
| logging.info("state_dir isn't fully set up; doing that now.") |
| |
| # Could be in a partially set-up state. |
| if state_dir.exists(): |
| logging.info("incomplete state_dir detected; removing.") |
| shutil.rmtree(str(state_dir)) |
| |
| state_dir.mkdir(parents=True) |
| most_recent_release = fetch_most_recent_release() |
| most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir) |
| atomically_write_state( |
| state_file, |
| State( |
| last_seen_release=most_recent_release, |
| last_gentoo_sha=most_recent_gentoo_commit, |
| ), |
| ) |
| # Running through this _should_ be a nop, but do it anyway. Should make any |
| # bugs more obvious on the first run of the script. |
| |
| prior_state = read_state(state_file) |
| logging.info("Last state was %r", prior_state) |
| |
| most_recent_release = fetch_most_recent_release() |
| logging.info("Most recent Rust release is %s", most_recent_release) |
| |
| logging.info("Fetching new commits from Gentoo") |
| update_git_repo(gentoo_subdir) |
| new_commits = get_new_gentoo_commits( |
| gentoo_subdir, prior_state.last_gentoo_sha |
| ) |
| logging.info("New commits: %r", new_commits) |
| |
| maybe_bug = maybe_compose_bug(prior_state, most_recent_release) |
| maybe_email = maybe_compose_email(new_commits) |
| |
| if maybe_bug is None: |
| logging.info("No bug to file") |
| else: |
| title, body = maybe_bug |
| if opts.skip_side_effects: |
| logging.info( |
| "Skipping sending bug with title %r and contents\n%s", |
| title, |
| body, |
| ) |
| else: |
| logging.info("Writing new bug") |
| file_bug(title, body) |
| |
| if maybe_email is None: |
| logging.info("No email to send") |
| else: |
| title, body = maybe_email |
| if opts.skip_side_effects: |
| logging.info( |
| "Skipping sending email with title %r and contents\n%s", |
| title, |
| tiny_render.render_html_pieces(body), |
| ) |
| else: |
| logging.info("Sending email") |
| send_email(title, body) |
| |
| if opts.skip_state_update: |
| logging.info("Skipping state update, as requested") |
| return |
| |
| newest_sha = ( |
| new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha |
| ) |
| atomically_write_state( |
| state_file, |
| State( |
| last_seen_release=most_recent_release, |
| last_gentoo_sha=newest_sha, |
| ), |
| ) |
| |
| |
| if __name__ == "__main__": |
| sys.exit(main(sys.argv[1:])) |