findmissing: Extract decorator functions from main.py
We'll need the decorator functions from multiple places,
so move them into a new file, util.py. While at it, replace print
functions in moved code with logging.
BUG=None
TEST=Run on GCE
Change-Id: I8d45b4f8c1604ee2ef3d394a61c909c3cfb6f527
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/2200139
Reviewed-by: Curtis Malainey <cujomalainey@chromium.org>
Commit-Queue: Guenter Roeck <groeck@chromium.org>
Tested-by: Guenter Roeck <groeck@chromium.org>
diff --git a/contrib/findmissing/main.py b/contrib/findmissing/main.py
index 5f3e954..bd3ebaf 100755
--- a/contrib/findmissing/main.py
+++ b/contrib/findmissing/main.py
@@ -12,11 +12,7 @@
from __future__ import print_function
-import logging
-import os
-import subprocess
import sys
-import time
import MySQLdb
import cloudsql_interface
@@ -24,65 +20,7 @@
import gerrit_interface
import missing
import synchronize
-
-
-def check_service_key_secret_exists():
- """Raises an error if the service account secret key file doesn't exist
-
- This can be generated on GCP under service accounts (Generate service token)
- This file should automatically be generated when running the gce-startup.sh script.
- """
- secret_file_path = os.path.join(common.HOMEDIR, 'secrets/linux_patches_robot_key.json')
-
- if not os.path.exists(secret_file_path):
- raise FileNotFoundError('Service token secret file %s not found' % secret_file_path)
-
-
-def check_service_running(keyword):
- """Raises an error if there is no running process commands that match `keyword`."""
- process_grep = ['pgrep', '-f', keyword]
- try:
- subprocess.run(process_grep, check=True, stdout=subprocess.DEVNULL,
- encoding='utf-8', errors='ignore')
- except subprocess.CalledProcessError:
- raise ProcessLookupError('Service %s is not running.' % keyword)
-
-
-def check_cloud_sql_proxy_running():
- """Raises an error if cloud_sql_proxy service is not running."""
- check_service_running('cloud_sql_proxy')
-
-
-def check_git_cookie_authdaemon_running():
- """Raises an error if git-cookie-authdaemon service is not running."""
- check_service_running('git-cookie-authdaemon')
-
-
-def preliminary_check_decorator(is_gce):
- """Decorator for performing environment related checks."""
- def wrap_preliminary_check(f):
- """Inner function that wraps method with preliminary check."""
- def wrapped_preliminary_check(*args):
- """Sanity checks on state of environment before executing decorated function."""
- if is_gce:
- level = logging.INFO
- # Ensures we have service account credentials to connect to cloudsql (GCP)
- check_service_key_secret_exists()
- else:
- level = logging.WARNING
-
- logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
-
- # Ensure cloudsql proxy is running to allow connection
- check_cloud_sql_proxy_running()
-
- if is_gce:
- # Ensure we have token to allow service account to perform Gerrit API operations
- check_git_cookie_authdaemon_running()
-
- f(*args)
- return wrapped_preliminary_check
- return wrap_preliminary_check
+import util
def sync_repositories_and_databases():
@@ -101,7 +39,7 @@
missing.new_missing_patches()
-@preliminary_check_decorator(True)
+@util.preliminary_check_decorator(True)
def synchronize_and_create_patches(create_patches=False):
"""Synchronize repositories/databases + create new fixes."""
sync_repositories_and_databases()
@@ -110,43 +48,8 @@
create_new_patches()
-def set_gcloud_project_config():
- """Sets project settings to chromeos-missing-patches project."""
- set_project_cmd = ['gcloud', 'config', 'set', 'project', 'google.com:chromeos-missing-patches']
- subprocess.run(set_project_cmd, stderr=subprocess.DEVNULL, check=True)
-
-
-def cloud_sql_proxy_decorator(func):
- """Decorator for starting and stopping cloud_sql_proxy."""
- def cloud_sql_proxy_wrapper(*args, **kwargs):
- """Create cloud_sql_proxy process, run func, send signal.SIGKILL cloud_sql_proxy"""
- try:
- set_gcloud_project_config()
- sql_instance_cmd = ['gcloud', 'sql', 'instances', 'describe',
- 'linux-patches-sql', '--format=value[](connectionName)']
- sql_instance = subprocess.check_output(sql_instance_cmd, encoding='utf-8').rstrip()
-
- cloudsql_cmd = ['cloud_sql_proxy', '-instances=%s=tcp:3306' % sql_instance]
- cloud_sql_proxy_pid = subprocess.Popen(cloudsql_cmd, stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL)
-
- # Wait for cloud_sql_proxy to spin up
- # todo(hirthanan): read cloud_sql pipe to see when it starts up
- time.sleep(3)
-
- func(*args, **kwargs)
-
- cloud_sql_proxy_pid.kill()
- except subprocess.CalledProcessError:
- print('Failed to retrieve sql_instance from gcloud')
- print('User must be authenticated with Cloud SDK (run `gcloud auth login`)')
- print('User must also be added to GCP project chromeos-missing-patches.')
- raise
- return cloud_sql_proxy_wrapper
-
-
-@cloud_sql_proxy_decorator
-@preliminary_check_decorator(False)
+@util.cloud_sql_proxy_decorator
+@util.preliminary_check_decorator(False)
def abandon_fix_cl(fixes_table, kernel_sha, fixedby_upstream_sha, reason):
"""Abandons an fix CL + updates database fix table."""
cloudsql_db = MySQLdb.Connect(user='linux_patches_robot', host='127.0.0.1', db='linuxdb')
@@ -175,8 +78,8 @@
cloudsql_db.close()
-@cloud_sql_proxy_decorator
-@preliminary_check_decorator(False)
+@util.cloud_sql_proxy_decorator
+@util.preliminary_check_decorator(False)
def restore_fix_cl(fixes_table, kernel_sha, fixedby_upstream_sha, reason):
"""Restores an abandoned change + updates database fix table."""
cloudsql_db = MySQLdb.Connect(user='linux_patches_robot', host='127.0.0.1', db='linuxdb')
diff --git a/contrib/findmissing/util.py b/contrib/findmissing/util.py
new file mode 100755
index 0000000..ab21496
--- /dev/null
+++ b/contrib/findmissing/util.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Helper functions used by various commands
+
+preliminary_check_decorator():
+ Check if environment is set up correctly.
+
+cloud_sql_proxy_decorator():
+ Start and stop cloud_sql_proxy.
+"""
+
+from __future__ import print_function
+
+import logging
+import os
+import subprocess
+import time
+
+import common
+
+
+def check_service_key_secret_exists():
+ """Raises an error if the service account secret key file does not exist.
+
+ This can be generated on GCP under service accounts (Generate service token)
+ This file should automatically be generated when running the gce-startup.sh script.
+ """
+ secret_file_path = os.path.join(common.HOMEDIR, 'secrets/linux_patches_robot_key.json')
+
+ if not os.path.exists(secret_file_path):
+ raise FileNotFoundError('Service token secret file %s not found' % secret_file_path)
+
+
+def check_service_running(keyword):
+ """Raises an error if there is no running process commands that match `keyword`."""
+ process_grep = ['pgrep', '-f', keyword]
+ try:
+ subprocess.run(process_grep, check=True, stdout=subprocess.DEVNULL,
+ encoding='utf-8', errors='ignore')
+ except subprocess.CalledProcessError:
+ raise ProcessLookupError('Service %s is not running.' % keyword)
+
+
+def check_cloud_sql_proxy_running():
+ """Raises an error if cloud_sql_proxy service is not running."""
+ check_service_running('cloud_sql_proxy')
+
+
+def check_git_cookie_authdaemon_running():
+ """Raises an error if git-cookie-authdaemon service is not running."""
+ check_service_running('git-cookie-authdaemon')
+
+
+def preliminary_check_decorator(is_gce):
+ """Decorator for performing environment related checks."""
+ def wrap_preliminary_check(f):
+ """Inner function that wraps method with preliminary check."""
+ def wrapped_preliminary_check(*args):
+ """Sanity checks on state of environment before executing decorated function."""
+ if is_gce:
+ # Ensures we have service account credentials to connect to cloudsql (GCP)
+ check_service_key_secret_exists()
+
+ # Ensure cloudsql proxy is running to allow connection
+ check_cloud_sql_proxy_running()
+
+ if is_gce:
+ level = logging.INFO
+ # Ensure we have token to allow service account to perform Gerrit API operations
+ check_git_cookie_authdaemon_running()
+ else:
+ level = logging.WARNING
+
+ logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
+
+ f(*args)
+ return wrapped_preliminary_check
+ return wrap_preliminary_check
+
+
+def set_gcloud_project_config():
+ """Sets project settings to chromeos-missing-patches project."""
+ set_project_cmd = ['gcloud', 'config', 'set', 'project', 'google.com:chromeos-missing-patches']
+ subprocess.run(set_project_cmd, stderr=subprocess.DEVNULL, check=True)
+
+
+def cloud_sql_proxy_decorator(func):
+ """Decorator for starting and stopping cloud_sql_proxy."""
+ def cloud_sql_proxy_wrapper(*args, **kwargs):
+ """Create cloud_sql_proxy process, run func, send signal.SIGKILL cloud_sql_proxy"""
+ try:
+ set_gcloud_project_config()
+ sql_instance_cmd = ['gcloud', 'sql', 'instances', 'describe',
+ 'linux-patches-sql', '--format=value[](connectionName)']
+ sql_instance = subprocess.check_output(sql_instance_cmd, encoding='utf-8').rstrip()
+
+ cloudsql_cmd = ['cloud_sql_proxy', '-instances=%s=tcp:3306' % sql_instance]
+ cloud_sql_proxy_pid = subprocess.Popen(cloudsql_cmd, stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL)
+
+ # Wait for cloud_sql_proxy to spin up
+ # todo(hirthanan): read cloud_sql pipe to see when it starts up
+ time.sleep(3)
+
+ func(*args, **kwargs)
+
+ cloud_sql_proxy_pid.kill()
+ except subprocess.CalledProcessError:
+ logging.error('Failed to retrieve sql_instance from gcloud')
+ logging.error('User must be authenticated with Cloud SDK (run `gcloud auth login`)')
+ logging.error('User must also be added to GCP project chromeos-missing-patches.')
+ raise
+ return cloud_sql_proxy_wrapper
