blob: d92b89e9d8ec63eb36ce3f306e63eb95b7c33ee5 [file] [log] [blame]
#!/usr/bin/env python3
# -*- coding: utf-8 -*-"
# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Find missing stable and backported mainline fix patches in chromeos."""
from __future__ import print_function
import logging
import os
import subprocess
import sys
import MySQLdb
import common
import cloudsql_interface
import gerrit_interface
import git_interface
# Constant representing number CL's we want created on single new missing patch run
def get_subsequent_fixes(db, fixer_upstream_sha):
"""Recursively builds a Dictionary of fixes for a fixer upstream sha.
Table will be following format given fixer_upstream_sha A:
{'A': ['B','C', 'H'],
'B': ['D', 'E'],
'C': ['F','G'],
'D': [],
'E': ['H'],
'F': [],
'G': [],
'H': []
} where B Fixes A, C fixes A, D Fixes B, E Fixes B, etc.
This would end up return a list
['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H']
Note that H is a fix for A but is scheduled till after E since it also fixes E
TODO(*): modify this to use common table expressions (CTE) to avoid
building dependency table in python. Changing it to use CTE will
remove the use of this function.
Note: This change is blocked by infra: It requires mysql >= 8.0, but
CloudSQL currently only supports mysql version 5.7.
fixes = {}
fixes[fixer_upstream_sha] = 0
iteration = 0
new_shas = set()
while new_shas:
iteration += 1
seen_shas = set(fixes.keys())
fixes_for_new_shas = cloudsql_interface.upstream_fixes_for_shas(db, list(new_shas))
for sha in fixes_for_new_shas:
# if new fixes fix previous fixes then we want to grab them last
current_iteration = fixes[sha] if fixes.get(sha) else 0
fixes[sha] = max(current_iteration, iteration)
new_shas = set(fixes_for_new_shas) - set(seen_shas)
# sort fixes into list
fixing_shas = list(fixes.items())
fixing_shas.sort(key=lambda x: x[1])
# remove order information
fixing_shas = [sha[0] for sha in fixing_shas]
return fixing_shas
def upstream_sha_to_kernel_sha(db, chosen_table, branch, upstream_sha):
"""Retrieves chromeos/stable sha by indexing db.
Returns sha or None if upstream sha doesn't exist downstream.
c = db.cursor()
q = """SELECT sha
FROM {chosen_table}
WHERE branch = %s
AND (upstream_sha = %s
OR patch_id IN (
SELECT patch_id
FROM linux_upstream
WHERE sha = %s
c.execute(q, [branch, upstream_sha, upstream_sha])
row = c.fetchone()
return row[0] if row else None
def insert_by_patch_id(db, branch, fixedby_upstream_sha):
"""Handles case where fixedby_upstream_sha may have changed in kernels.
Returns True if successful patch_id insertion and False if patch_id not found.
c = db.cursor()
# Commit sha may have been modified in cherry-pick, backport, etc.
# Retrieve SHA in linux_chrome by patch-id by checking for fixedby_upstream_sha
# removes entries that are already tracked in chrome_fixes
q = """SELECT lc.sha
FROM linux_chrome AS lc
JOIN linux_upstream AS lu
ON lc.patch_id = lu.patch_id
JOIN upstream_fixes as uf
ON lc.upstream_sha = uf.upstream_sha
WHERE uf.fixedby_upstream_sha = %s AND branch = %s
AND (lc.sha, uf.fixedby_upstream_sha)
SELECT kernel_sha, fixedby_upstream_sha
FROM chrome_fixes
WHERE branch = %s
c.execute(q, [fixedby_upstream_sha, branch, branch])
chrome_shas = c.fetchall()
# fixedby_upstream_sha has already been merged into linux_chrome
# chrome shas represent kernel sha for the upstream_sha fixedby_upstream_sha
if chrome_shas:
for chrome_sha in chrome_shas:
entry_time = common.get_current_time()
cl_status =
reason = 'Already merged into linux_chrome [upstream sha %s]' % fixedby_upstream_sha
q = """INSERT INTO chrome_fixes
(kernel_sha, fixedby_upstream_sha, branch, entry_time,
close_time, initial_status, status, reason)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"""
c.execute(q, [chrome_sha, fixedby_upstream_sha, branch, entry_time,
entry_time, cl_status, cl_status, reason])
except MySQLdb.Error as e: # pylint: disable=no-member
'Failed to insert an already merged entry into chrome_fixes: error %d(%s)',
e.args[0], e.args[1])
return True
return False
def insert_fix_gerrit(db, chosen_table, chosen_fixes, branch, kernel_sha, fixedby_upstream_sha):
"""Inserts fix row by checking status of applying a fix change.
Return True if we create a new Gerrit CL, otherwise return False.
# Check if fix has been merged using it's patch-id since sha's might've changed
success = insert_by_patch_id(db, branch, fixedby_upstream_sha)
created_new_change = False
if success:
return created_new_change
c = db.cursor()
# Try applying patch and get status
status = git_interface.get_cherrypick_status(common.CHROMEOS_PATH,
'v%s' % branch,
'chromeos-%s' % branch,
cl_status =
entry_time = common.get_current_time()
close_time = fix_change_id = reason = None
q = """INSERT INTO {chosen_fixes}
(kernel_sha, fixedby_upstream_sha, branch, entry_time, close_time,
fix_change_id, initial_status, status, reason)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)""".format(chosen_fixes=chosen_fixes)
if status == common.Status.MERGED:
# Create a row for the merged CL (we don't need to track this), but can be stored
# to indicate that the changes of this patch are already merged
# entry_time and close_time are the same since we weren't tracking when it was merged
fixedby_kernel_sha = upstream_sha_to_kernel_sha(db, chosen_table,
branch, fixedby_upstream_sha)'%s SHA [%s] already merged bugfix patch [kernel: %s] [upstream: %s]',
chosen_fixes, kernel_sha, fixedby_kernel_sha, fixedby_upstream_sha)
reason = 'Patch applied to linux_chrome before this robot was run'
close_time = entry_time
# linux_chrome will have change-id's but stable merged fixes will not
# Correctly located fixedby_kernel_sha in linux_chrome
if chosen_table == 'linux_chrome' and fixedby_kernel_sha:
fix_change_id = git_interface.get_commit_changeid_linux_chrome(fixedby_kernel_sha)
elif status == common.Status.OPEN:
fix_change_id = gerrit_interface.create_change(kernel_sha, fixedby_upstream_sha, branch)
created_new_change = bool(fix_change_id)
# Checks if change was created successfully
if not created_new_change:
logging.error('Failed to create change for kernel_sha %s fixed by %s',
kernel_sha, fixedby_upstream_sha)
return False
elif status == common.Status.CONFLICT:
# Register conflict entry_time, do not create gerrit CL
# Requires engineer to manually explore why CL doesn't apply cleanly
c.execute(q, [kernel_sha, fixedby_upstream_sha, branch, entry_time,
close_time, fix_change_id, cl_status, cl_status, reason])'Inserted row into fixes table %s %s %s %s %s %s %s %s %s',
chosen_fixes, kernel_sha, fixedby_upstream_sha, branch,
entry_time, close_time, fix_change_id, cl_status, reason)
except MySQLdb.Error as e: # pylint: disable=no-member
'Error inserting fix CL into fixes table %s %s %s %s %s %s %s %s %s: error %d(%s)',
chosen_fixes, kernel_sha, fixedby_upstream_sha, branch,
entry_time, close_time, fix_change_id, cl_status, reason,
e.args[0], e.args[1])
return created_new_change
def fixup_unmerged_patches(db, branch, kernel_metadata):
"""Fixup script that attempts to reapply unmerged fixes to get latest status.
2 main actions performed by script include:
1) Handle case where a conflicting CL later can be applied cleanly without merge conflicts
2) Detect if the fix has been applied to linux_chrome externally
(i.e not merging through a fix created by this robot)
c = db.cursor()
fixes_table = kernel_metadata.kernel_fixes_table
q = """SELECT kernel_sha, fixedby_upstream_sha, status, fix_change_id
FROM {fixes_table}
WHERE status != 'MERGED'
AND branch = %s""".format(fixes_table=fixes_table)
c.execute(q, [branch])
rows = c.fetchall()
for row in rows:
kernel_sha, fixedby_upstream_sha, status, fix_change_id = row
new_status_enum = git_interface.get_cherrypick_status(common.CHROMEOS_PATH,
'v%s' % branch,
'chromeos-%s' % branch,
new_status =
if status == 'CONFLICT' and new_status == 'OPEN':
fix_change_id = gerrit_interface.create_change(kernel_sha, fixedby_upstream_sha, branch)
# Check if we successfully created the fix patch before performing update
if fix_change_id:
cloudsql_interface.update_conflict_to_open(db, fixes_table,
kernel_sha, fixedby_upstream_sha, fix_change_id)
elif new_status == 'MERGED':
reason = 'Fix was merged externally and detected by robot.'
if fix_change_id:
gerrit_interface.abandon_change(fix_change_id, branch, reason)
cloudsql_interface.update_change_merged(db, fixes_table,
kernel_sha, fixedby_upstream_sha, reason)
def update_fixes_in_branch(db, branch, kernel_metadata, limit):
"""Updates fix patch table row by determining if CL merged into linux_chrome."""
del limit # unused here
c = db.cursor()
chosen_fixes = kernel_metadata.kernel_fixes_table
# Old rows to Update
q = """UPDATE {chosen_fixes} AS fixes
JOIN linux_chrome AS lc
ON fixes.fixedby_upstream_sha = lc.upstream_sha
SET status = 'MERGED', close_time = %s, reason = %s
WHERE fixes.branch = %s
AND lc.branch = %s
AND (fixes.status = 'OPEN'
OR fixes.status = 'CONFLICT'
OR fixes.status = 'ABANDONED')""".format(chosen_fixes=chosen_fixes)
close_time = common.get_current_time()
reason = 'Patch has been applied to linux_chome'
c.execute(q, [close_time, reason, branch, branch])
'Updating rows that have been merged into linux_chrome in table %s / branch %s',
chosen_fixes, branch)
except MySQLdb.Error as e: # pylint: disable=no-member
logging.error('Error updating fixes table for merged commits %s %s %s %s: %d(%s)',
chosen_fixes, close_time, reason, branch, e.args[0], e.args[1])
# Sync status of unmerged patches in a branch
fixup_unmerged_patches(db, branch, kernel_metadata)
def create_new_fixes_in_branch(db, branch, kernel_metadata, limit):
"""Look for missing Fixup commits in provided chromeos or stable release."""
c = db.cursor()
branch_name = kernel_metadata.get_kernel_branch(branch)'Checking branch %s', branch_name)['git', 'checkout', branch_name], check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
# chosen_table is either linux_stable or linux_chrome
chosen_table = kernel_metadata.path
chosen_fixes = kernel_metadata.kernel_fixes_table
# New rows to insert
# Note: MySQLdb doesn't support inserting table names as parameters
# due to sql injection
q = """SELECT chosen_table.sha, uf.fixedby_upstream_sha
FROM {chosen_table} AS chosen_table
JOIN upstream_fixes AS uf
ON chosen_table.upstream_sha = uf.upstream_sha
WHERE branch = %s
AND (chosen_table.sha, uf.fixedby_upstream_sha)
SELECT chosen_fixes.kernel_sha, chosen_fixes.fixedby_upstream_sha
FROM {chosen_fixes} AS chosen_fixes
WHERE branch = %s
)""".format(chosen_table=chosen_table, chosen_fixes=chosen_fixes)
c.execute(q, [branch, branch])'Finding new rows to insert into fixes table %s %s %s',
chosen_table, chosen_fixes, branch)
except MySQLdb.Error as e: # pylint: disable=no-member
logging.error('Error finding new rows to insert %s %s %s: error %d(%s)',
chosen_table, chosen_fixes, branch, e.args[0], e.args[1])
count_new_changes = 0
# todo(hirthanan): Create an intermediate state in Status that allows us to
# create all the patches in chrome/stable fixes tables but does not add reviewers
# until quota is available. This should decouple the creation of gerrit CL's
# and adding reviewers to those CL's.
for (kernel_sha, fixedby_upstream_sha) in c.fetchall():
new_change = insert_fix_gerrit(db, chosen_table, chosen_fixes,
branch, kernel_sha, fixedby_upstream_sha)
if new_change:
count_new_changes += 1
if count_new_changes >= limit:
return count_new_changes
def missing_patches_sync(db, kernel_metadata, sync_branch_method, limit=None):
"""Helper to create or update fix patches in stable and chromeos releases."""
if len(sys.argv) > 1:
branches = sys.argv[1:]
branches = common.CHROMEOS_BRANCHES
for b in branches:
sync_branch_method(db, b, kernel_metadata, limit)
def new_missing_patches():
"""Rate limit calling create_new_fixes_in_branch."""
cloudsql_db = MySQLdb.Connect(user='linux_patches_robot', host='', db='linuxdb')
kernel_metadata = common.get_kernel_metadata(common.Kernel.linux_stable)
missing_patches_sync(cloudsql_db, kernel_metadata, create_new_fixes_in_branch,
kernel_metadata = common.get_kernel_metadata(common.Kernel.linux_chrome)
missing_patches_sync(cloudsql_db, kernel_metadata, create_new_fixes_in_branch,
def update_missing_patches():
"""Updates fixes table entries on regular basis."""
cloudsql_db = MySQLdb.Connect(user='linux_patches_robot', host='', db='linuxdb')
kernel_metadata = common.get_kernel_metadata(common.Kernel.linux_stable)
missing_patches_sync(cloudsql_db, kernel_metadata, update_fixes_in_branch)
kernel_metadata = common.get_kernel_metadata(common.Kernel.linux_chrome)
missing_patches_sync(cloudsql_db, kernel_metadata, update_fixes_in_branch)