| #!/usr/bin/env python |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from __future__ import print_function |
| |
| import ConfigParser |
| import functools |
| import json |
| import optparse |
| import os |
| import re |
| import socket |
| import sys |
| import subprocess |
| |
| from errors import (VerifyException, HookFailure, PrintErrorForProject, |
| PrintErrorsForCommit) |
| |
| # If repo imports us, the __name__ will be __builtin__, and the wrapper will |
| # be in $CHROMEOS_CHECKOUT/.repo/repo/main.py, so we need to go two directories |
| # up. The same logic also happens to work if we're executed directly. |
| if __name__ in ('__builtin__', '__main__'): |
| sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) |
| |
| from chromite.lib import patch |
| from chromite.licensing import licenses |
| |
| |
| COMMON_INCLUDED_PATHS = [ |
| # C++ and friends |
| r".*\.c$", r".*\.cc$", r".*\.cpp$", r".*\.h$", r".*\.m$", r".*\.mm$", |
| r".*\.inl$", r".*\.asm$", r".*\.hxx$", r".*\.hpp$", r".*\.s$", r".*\.S$", |
| # Scripts |
| r".*\.js$", r".*\.py$", r".*\.sh$", r".*\.rb$", r".*\.pl$", r".*\.pm$", |
| # No extension at all, note that ALL CAPS files are black listed in |
| # COMMON_EXCLUDED_LIST below. |
| r"(^|.*[\\\/])[^.]+$", |
| # Other |
| r".*\.java$", r".*\.mk$", r".*\.am$", |
| ] |
| |
| |
| COMMON_EXCLUDED_PATHS = [ |
| # avoid doing source file checks for kernel |
| r"/src/third_party/kernel/", |
| r"/src/third_party/kernel-next/", |
| r"/src/third_party/ktop/", |
| r"/src/third_party/punybench/", |
| r".*\bexperimental[\\\/].*", |
| r".*\b[A-Z0-9_]{2,}$", |
| r".*[\\\/]debian[\\\/]rules$", |
| # for ebuild trees, ignore any caches and manifest data |
| r".*/Manifest$", |
| r".*/metadata/[^/]*cache[^/]*/[^/]+/[^/]+$", |
| |
| # ignore profiles data (like overlay-tegra2/profiles) |
| r".*/overlay-.*/profiles/.*", |
| # ignore minified js and jquery |
| r".*\.min\.js", |
| r".*jquery.*\.js", |
| ] |
| |
| |
| _CONFIG_FILE = 'PRESUBMIT.cfg' |
| |
| |
| # Exceptions |
| |
| |
| class BadInvocation(Exception): |
| """An Exception indicating a bad invocation of the program.""" |
| pass |
| |
| |
| # General Helpers |
| |
| |
| def _run_command(cmd, cwd=None, stderr=None): |
| """Executes the passed in command and returns raw stdout output. |
| |
| Args: |
| cmd: The command to run; should be a list of strings. |
| cwd: The directory to switch to for running the command. |
| stderr: Can be one of None (print stderr to console), subprocess.STDOUT |
| (combine stderr with stdout), or subprocess.PIPE (ignore stderr). |
| |
| Returns: |
| The standard out from the process. |
| """ |
| p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, cwd=cwd) |
| return p.communicate()[0] |
| |
| |
| def _get_hooks_dir(): |
| """Returns the absolute path to the repohooks directory.""" |
| if __name__ == '__main__': |
| # Works when file is run on its own (__file__ is defined)... |
| return os.path.abspath(os.path.dirname(__file__)) |
| else: |
| # We need to do this when we're run through repo. Since repo executes |
| # us with execfile(), we don't get __file__ defined. |
| cmd = ['repo', 'forall', 'chromiumos/repohooks', '-c', 'pwd'] |
| return _run_command(cmd).strip() |
| |
| |
| def _match_regex_list(subject, expressions): |
| """Try to match a list of regular expressions to a string. |
| |
| Args: |
| subject: The string to match regexes on |
| expressions: A list of regular expressions to check for matches with. |
| |
| Returns: |
| Whether the passed in subject matches any of the passed in regexes. |
| """ |
| for expr in expressions: |
| if (re.search(expr, subject)): |
| return True |
| return False |
| |
| |
| def _filter_files(files, include_list, exclude_list=[]): |
| """Filter out files based on the conditions passed in. |
| |
| Args: |
| files: list of filepaths to filter |
| include_list: list of regex that when matched with a file path will cause it |
| to be added to the output list unless the file is also matched with a |
| regex in the exclude_list. |
| exclude_list: list of regex that when matched with a file will prevent it |
| from being added to the output list, even if it is also matched with a |
| regex in the include_list. |
| |
| Returns: |
| A list of filepaths that contain files matched in the include_list and not |
| in the exclude_list. |
| """ |
| filtered = [] |
| for f in files: |
| if (_match_regex_list(f, include_list) and |
| not _match_regex_list(f, exclude_list)): |
| filtered.append(f) |
| return filtered |
| |
| |
| def _verify_header_content(commit, content, fail_msg): |
| """Verify that file headers contain specified content. |
| |
| Args: |
| commit: the affected commit. |
| content: the content of the header to be verified. |
| fail_msg: the first message to display in case of failure. |
| |
| Returns: |
| The return value of HookFailure(). |
| """ |
| license_re = re.compile(content, re.MULTILINE) |
| bad_files = [] |
| files = _filter_files(_get_affected_files(commit), |
| COMMON_INCLUDED_PATHS, |
| COMMON_EXCLUDED_PATHS) |
| |
| for f in files: |
| if os.path.exists(f): # Ignore non-existant files |
| contents = open(f).read() |
| if len(contents) == 0: continue # Ignore empty files |
| if not license_re.search(contents): |
| bad_files.append(f) |
| if bad_files: |
| msg = "%s:\n%s\n%s" % (fail_msg, license_re.pattern, |
| "Found a bad header in these files:") |
| return HookFailure(msg, bad_files) |
| |
| |
| # Git Helpers |
| |
| |
| def _get_upstream_branch(): |
| """Returns the upstream tracking branch of the current branch. |
| |
| Raises: |
| Error if there is no tracking branch |
| """ |
| current_branch = _run_command(['git', 'symbolic-ref', 'HEAD']).strip() |
| current_branch = current_branch.replace('refs/heads/', '') |
| if not current_branch: |
| raise VerifyException('Need to be on a tracking branch') |
| |
| cfg_option = 'branch.' + current_branch + '.%s' |
| full_upstream = _run_command(['git', 'config', cfg_option % 'merge']).strip() |
| remote = _run_command(['git', 'config', cfg_option % 'remote']).strip() |
| if not remote or not full_upstream: |
| raise VerifyException('Need to be on a tracking branch') |
| |
| return full_upstream.replace('heads', 'remotes/' + remote) |
| |
| |
| def _get_patch(commit): |
| """Returns the patch for this commit.""" |
| return _run_command(['git', 'format-patch', '--stdout', '-1', commit]) |
| |
| |
| def _try_utf8_decode(data): |
| """Attempts to decode a string as UTF-8. |
| |
| Returns: |
| The decoded Unicode object, or the original string if parsing fails. |
| """ |
| try: |
| return unicode(data, 'utf-8', 'strict') |
| except UnicodeDecodeError: |
| return data |
| |
| |
| def _get_file_diff(file, commit): |
| """Returns a list of (linenum, lines) tuples that the commit touched.""" |
| output = _run_command(['git', 'show', '-p', '--no-ext-diff', commit, file]) |
| |
| new_lines = [] |
| line_num = 0 |
| for line in output.splitlines(): |
| m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line) |
| if m: |
| line_num = int(m.groups(1)[0]) |
| continue |
| if line.startswith('+') and not line.startswith('++'): |
| new_lines.append((line_num, _try_utf8_decode(line[1:]))) |
| if not line.startswith('-'): |
| line_num += 1 |
| return new_lines |
| |
| |
| def _get_affected_files(commit, include_deletes=False): |
| """Returns list of absolute filepaths that were modified/added. |
| |
| Args: |
| commit: The commit |
| include_deletes: If true we'll include delete in the list. |
| |
| Returns: |
| A list of modified/added (and perhaps deleted) files |
| """ |
| output = _run_command(['git', 'diff', '--name-status', commit + '^!']) |
| files = [] |
| for statusline in output.splitlines(): |
| m = re.match('^(\w)+\t(.+)$', statusline.rstrip()) |
| # Ignore deleted files, and return absolute paths of files |
| if (include_deletes or m.group(1)[0] != 'D'): |
| pwd = os.getcwd() |
| files.append(os.path.join(pwd, m.group(2))) |
| return files |
| |
| |
| def _get_commits(): |
| """Returns a list of commits for this review.""" |
| cmd = ['git', 'log', '%s..' % _get_upstream_branch(), '--format=%H'] |
| return _run_command(cmd).split() |
| |
| |
| def _get_commit_desc(commit): |
| """Returns the full commit message of a commit.""" |
| return _run_command(['git', 'log', '--format=%s%n%n%b', commit + '^!']) |
| |
| |
| # Common Hooks |
| |
| |
| def _check_no_long_lines(project, commit): |
| """Checks that there aren't any lines longer than maxlen characters in any of |
| the text files to be submitted. |
| """ |
| MAX_LEN = 80 |
| SKIP_REGEXP = re.compile('|'.join([ |
| r'https?://', |
| r'^#\s*(define|include|import|pragma|if|endif)\b'])) |
| |
| errors = [] |
| files = _filter_files(_get_affected_files(commit), |
| COMMON_INCLUDED_PATHS, |
| COMMON_EXCLUDED_PATHS) |
| |
| for afile in files: |
| for line_num, line in _get_file_diff(afile, commit): |
| # Allow certain lines to exceed the maxlen rule. |
| if (len(line) <= MAX_LEN or SKIP_REGEXP.search(line)): |
| continue |
| |
| errors.append('%s, line %s, %s chars' % (afile, line_num, len(line))) |
| if len(errors) == 5: # Just show the first 5 errors. |
| break |
| |
| if errors: |
| msg = 'Found lines longer than %s characters (first 5 shown):' % MAX_LEN |
| return HookFailure(msg, errors) |
| |
| |
| def _check_no_stray_whitespace(project, commit): |
| """Checks that there is no stray whitespace at source lines end.""" |
| errors = [] |
| files = _filter_files(_get_affected_files(commit), |
| COMMON_INCLUDED_PATHS, |
| COMMON_EXCLUDED_PATHS) |
| |
| for afile in files: |
| for line_num, line in _get_file_diff(afile, commit): |
| if line.rstrip() != line: |
| errors.append('%s, line %s' % (afile, line_num)) |
| if errors: |
| return HookFailure('Found line ending with white space in:', errors) |
| |
| |
| def _check_no_tabs(project, commit): |
| """Checks there are no unexpanded tabs.""" |
| TAB_OK_PATHS = [ |
| r"/src/third_party/u-boot/", |
| r".*\.ebuild$", |
| r".*\.eclass$", |
| r".*/[M|m]akefile$", |
| r".*\.mk$" |
| ] |
| |
| errors = [] |
| files = _filter_files(_get_affected_files(commit), |
| COMMON_INCLUDED_PATHS, |
| COMMON_EXCLUDED_PATHS + TAB_OK_PATHS) |
| |
| for afile in files: |
| for line_num, line in _get_file_diff(afile, commit): |
| if '\t' in line: |
| errors.append('%s, line %s' % (afile, line_num)) |
| if errors: |
| return HookFailure('Found a tab character in:', errors) |
| |
| |
| def _check_change_has_test_field(project, commit): |
| """Check for a non-empty 'TEST=' field in the commit message.""" |
| TEST_RE = r'\nTEST=\S+' |
| |
| if not re.search(TEST_RE, _get_commit_desc(commit)): |
| msg = 'Changelist description needs TEST field (after first line)' |
| return HookFailure(msg) |
| |
| |
| def _check_change_has_valid_cq_depend(project, commit): |
| """Check for a correctly formatted CQ-DEPEND field in the commit message.""" |
| msg = 'Changelist has invalid CQ-DEPEND target.' |
| example = 'Example: CQ-DEPEND=CL:1234, CL:2345' |
| try: |
| patch.GetPaladinDeps(_get_commit_desc(commit)) |
| except ValueError as ex: |
| return HookFailure(msg, [example, str(ex)]) |
| |
| |
| def _check_change_has_bug_field(project, commit): |
| """Check for a correctly formatted 'BUG=' field in the commit message.""" |
| OLD_BUG_RE = r'\nBUG=.*chromium-os' |
| if re.search(OLD_BUG_RE, _get_commit_desc(commit)): |
| msg = ('The chromium-os bug tracker is now deprecated. Please use\n' |
| 'the chromium tracker in your BUG= line now.') |
| return HookFailure(msg) |
| |
| BUG_RE = r'\nBUG=([Nn]one|(chrome-os-partner|chromium):\d+)' |
| if not re.search(BUG_RE, _get_commit_desc(commit)): |
| msg = ('Changelist description needs BUG field (after first line):\n' |
| 'BUG=chromium:9999 (for public tracker)\n' |
| 'BUG=chrome-os-partner:9999 (for partner tracker)\n' |
| 'BUG=None') |
| return HookFailure(msg) |
| |
| |
| def _check_for_uprev(project, commit): |
| """Check that we're not missing a revbump of an ebuild in the given commit. |
| |
| If the given commit touches files in a directory that has ebuilds somewhere |
| up the directory hierarchy, it's very likely that we need an ebuild revbump |
| in order for those changes to take effect. |
| |
| It's not totally trivial to detect a revbump, so at least detect that an |
| ebuild with a revision number in it was touched. This should handle the |
| common case where we use a symlink to do the revbump. |
| |
| TODO: it would be nice to enhance this hook to: |
| * Handle cases where people revbump with a slightly different syntax. I see |
| one ebuild (puppy) that revbumps with _pN. This is a false positive. |
| * Catches cases where people aren't using symlinks for revbumps. If they |
| edit a revisioned file directly (and are expected to rename it for revbump) |
| we'll miss that. Perhaps we could detect that the file touched is a |
| symlink? |
| |
| If a project doesn't use symlinks we'll potentially miss a revbump, but we're |
| still better off than without this check. |
| |
| Args: |
| project: The project to look at |
| commit: The commit to look at |
| |
| Returns: |
| A HookFailure or None. |
| """ |
| affected_paths = _get_affected_files(commit, include_deletes=True) |
| |
| # Don't yell about changes to whitelisted files... |
| whitelist = ['Manifest', 'metadata.xml'] |
| affected_paths = [path for path in affected_paths |
| if os.path.basename(path) not in whitelist] |
| if not affected_paths: |
| return None |
| |
| # If we've touched any file named with a -rN.ebuild then we'll say we're |
| # OK right away. See TODO above about enhancing this. |
| touched_revved_ebuild = any(re.search(r'-r\d*\.ebuild$', path) |
| for path in affected_paths) |
| if touched_revved_ebuild: |
| return None |
| |
| # We want to examine the current contents of all directories that are parents |
| # of files that were touched (up to the top of the project). |
| # |
| # ...note: we use the current directory contents even though it may have |
| # changed since the commit we're looking at. This is just a heuristic after |
| # all. Worst case we don't flag a missing revbump. |
| project_top = os.getcwd() |
| dirs_to_check = set([project_top]) |
| for path in affected_paths: |
| path = os.path.dirname(path) |
| while os.path.exists(path) and not os.path.samefile(path, project_top): |
| dirs_to_check.add(path) |
| path = os.path.dirname(path) |
| |
| # Look through each directory. If it's got an ebuild in it then we'll |
| # consider this as a case when we need a revbump. |
| for dir_path in dirs_to_check: |
| contents = os.listdir(dir_path) |
| ebuilds = [os.path.join(dir_path, path) |
| for path in contents if path.endswith('.ebuild')] |
| ebuilds_9999 = [path for path in ebuilds if path.endswith('-9999.ebuild')] |
| |
| # If the -9999.ebuild file was touched the bot will uprev for us. |
| # ...we'll use a simple intersection here as a heuristic... |
| if set(ebuilds_9999) & set(affected_paths): |
| continue |
| |
| if ebuilds: |
| return HookFailure('Changelist probably needs a revbump of an ebuild\n' |
| 'or a -r1.ebuild symlink if this is a new ebuild') |
| |
| return None |
| |
| |
| def _check_ebuild_licenses(_project, commit): |
| """Check if the LICENSE field in the ebuild is correct.""" |
| affected_paths = _get_affected_files(commit) |
| touched_ebuilds = [x for x in affected_paths if x.endswith('.ebuild')] |
| |
| # A list of licenses to ignore for now. |
| LICENSES_IGNORE = ['||', '(', ')', 'Proprietary', 'as-is'] |
| |
| for ebuild in touched_ebuilds: |
| # Skip virutal packages. |
| if ebuild.split('/')[-3] == 'virtual': |
| continue |
| |
| try: |
| license_types = licenses.GetLicenseTypesFromEbuild(ebuild) |
| except ValueError as e: |
| return HookFailure(e.message, [ebuild]) |
| |
| # Also ignore licenses ending with '?' |
| for license_type in [x for x in license_types |
| if x not in LICENSES_IGNORE and not x.endswith('?')]: |
| try: |
| licenses.Licensing.FindLicenseType(license_type) |
| except AssertionError as e: |
| return HookFailure(e.message, [ebuild]) |
| |
| |
| def _check_change_has_proper_changeid(project, commit): |
| """Verify that Change-ID is present in last paragraph of commit message.""" |
| desc = _get_commit_desc(commit) |
| loc = desc.rfind('\nChange-Id:') |
| if loc == -1 or re.search('\n\s*\n\s*\S+', desc[loc:]): |
| return HookFailure('Change-Id must be in last paragraph of description.') |
| |
| |
| def _check_license(project, commit): |
| """Verifies the license header.""" |
| LICENSE_HEADER = ( |
| r".* Copyright \(c\) 20[-0-9]{2,7} The Chromium OS Authors\. All rights " |
| r"reserved\." "\n" |
| r".* Use of this source code is governed by a BSD-style license that can " |
| "be\n" |
| r".* found in the LICENSE file\." |
| "\n" |
| ) |
| FAIL_MSG = "License must match" |
| |
| return _verify_header_content(commit, LICENSE_HEADER, FAIL_MSG) |
| |
| |
| def _check_google_copyright(project, commit): |
| """Verifies Google Inc. as copyright holder.""" |
| LICENSE_HEADER = ( |
| r".* Copyright 20[-0-9]{2,7} Google Inc\." |
| ) |
| FAIL_MSG = "Copyright must match" |
| |
| # Avoid blocking partners and external contributors. |
| fqdn = socket.getfqdn() |
| if not fqdn.endswith(".corp.google.com"): |
| return None |
| |
| return _verify_header_content(commit, LICENSE_HEADER, FAIL_MSG) |
| |
| |
| # Project-specific hooks |
| |
| |
| def _run_checkpatch(project, commit, options=[]): |
| """Runs checkpatch.pl on the given project""" |
| hooks_dir = _get_hooks_dir() |
| cmd = ['%s/checkpatch.pl' % hooks_dir] + options + ['-'] |
| p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) |
| output = p.communicate(_get_patch(commit))[0] |
| if p.returncode: |
| return HookFailure('checkpatch.pl errors/warnings\n\n' + output) |
| |
| |
| def _run_checkpatch_no_tree(project, commit): |
| return _run_checkpatch(project, commit, ['--no-tree']) |
| |
| def _run_checkpatch_ec(project, commit): |
| """Runs checkpatch with options for Chromium EC projects.""" |
| return _run_checkpatch(project, commit, ['--no-tree', |
| '--ignore=MSLEEP,VOLATILE']) |
| |
| def _kernel_configcheck(project, commit): |
| """Makes sure kernel config changes are not mixed with code changes""" |
| files = _get_affected_files(commit) |
| if not len(_filter_files(files, [r'chromeos/config'])) in [0, len(files)]: |
| return HookFailure('Changes to chromeos/config/ and regular files must ' |
| 'be in separate commits:\n%s' % '\n'.join(files)) |
| |
| def _run_json_check(project, commit): |
| """Checks that all JSON files are syntactically valid.""" |
| for f in _filter_files(_get_affected_files(commit), [r'.*\.json']): |
| try: |
| json.load(open(f)) |
| except Exception, e: |
| return HookFailure('Invalid JSON in %s: %s' % (f, e)) |
| |
| |
| def _check_manifests(project, commit): |
| """Make sure Manifest files only have DIST lines""" |
| paths = [] |
| |
| for path in _get_affected_files(commit): |
| if os.path.basename(path) != 'Manifest': |
| continue |
| if not os.path.exists(path): |
| continue |
| |
| with open(path, 'r') as f: |
| for line in f.readlines(): |
| if not line.startswith('DIST '): |
| paths.append(path) |
| break |
| |
| if paths: |
| return HookFailure('Please remove lines that do not start with DIST:\n%s' % |
| ('\n'.join(paths),)) |
| |
| |
| def _check_change_has_branch_field(project, commit): |
| """Check for a non-empty 'BRANCH=' field in the commit message.""" |
| BRANCH_RE = r'\nBRANCH=\S+' |
| |
| if not re.search(BRANCH_RE, _get_commit_desc(commit)): |
| msg = ('Changelist description needs BRANCH field (after first line)\n' |
| 'E.g. BRANCH=none or BRANCH=link,snow') |
| return HookFailure(msg) |
| |
| |
| def _check_change_has_signoff_field(project, commit): |
| """Check for a non-empty 'Signed-off-by:' field in the commit message.""" |
| SIGNOFF_RE = r'\nSigned-off-by: \S+' |
| |
| if not re.search(SIGNOFF_RE, _get_commit_desc(commit)): |
| msg = ('Changelist description needs Signed-off-by: field\n' |
| 'E.g. Signed-off-by: My Name <me@chromium.org>') |
| return HookFailure(msg) |
| |
| |
| def _run_project_hook_script(script, project, commit): |
| """Runs a project hook script. |
| |
| The script is run with the following environment variables set: |
| PRESUBMIT_PROJECT: The affected project |
| PRESUBMIT_COMMIT: The affected commit |
| PRESUBMIT_FILES: A newline-separated list of affected files |
| |
| The script is considered to fail if the exit code is non-zero. It should |
| write an error message to stdout. |
| """ |
| env = dict(os.environ) |
| env['PRESUBMIT_PROJECT'] = project |
| env['PRESUBMIT_COMMIT'] = commit |
| |
| # Put affected files in an environment variable |
| files = _get_affected_files(commit) |
| env['PRESUBMIT_FILES'] = '\n'.join(files) |
| |
| process = subprocess.Popen(script, env=env, shell=True, |
| stdin=open(os.devnull), |
| stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT) |
| stdout, _ = process.communicate() |
| if process.wait(): |
| if stdout: |
| stdout = re.sub('(?m)^', ' ', stdout) |
| return HookFailure('Hook script "%s" failed with code %d%s' % |
| (script, process.returncode, |
| ':\n' + stdout if stdout else '')) |
| |
| |
| # Base |
| |
| |
| # A list of hooks that are not project-specific |
| _COMMON_HOOKS = [ |
| _check_change_has_bug_field, |
| _check_change_has_valid_cq_depend, |
| _check_change_has_test_field, |
| _check_change_has_proper_changeid, |
| _check_ebuild_licenses, |
| _check_no_stray_whitespace, |
| _check_no_long_lines, |
| _check_license, |
| _check_no_tabs, |
| _check_for_uprev, |
| ] |
| |
| |
| # A dictionary of project-specific hooks(callbacks), indexed by project name. |
| # dict[project] = [callback1, callback2] |
| _PROJECT_SPECIFIC_HOOKS = { |
| "chromeos/autotest-tools": [_run_json_check], |
| "chromeos/overlays/chromeos-overlay": [_check_manifests], |
| "chromeos/overlays/chromeos-partner-overlay": [_check_manifests], |
| "chromeos/platform/ec-private": [_run_checkpatch_ec, |
| _check_change_has_branch_field], |
| "chromeos/third_party/intel-framework": [_check_change_has_branch_field], |
| "chromeos/vendor/kernel-exynos-staging": [_run_checkpatch, |
| _kernel_configcheck], |
| "chromeos/vendor/u-boot-exynos": [_run_checkpatch_no_tree], |
| "chromiumos/overlays/board-overlays": [_check_manifests], |
| "chromiumos/overlays/chromiumos-overlay": [_check_manifests], |
| "chromiumos/overlays/portage-stable": [_check_manifests], |
| "chromiumos/platform/ec": [_run_checkpatch_ec, |
| _check_change_has_branch_field], |
| "chromiumos/platform/mosys": [_check_change_has_branch_field], |
| "chromiumos/platform/vboot_reference": [_check_change_has_branch_field], |
| "chromiumos/third_party/coreboot": [_check_change_has_signoff_field], |
| "chromiumos/third_party/flashrom": [_check_change_has_branch_field], |
| "chromiumos/third_party/kernel": [_run_checkpatch, _kernel_configcheck], |
| "chromiumos/third_party/kernel-next": [_run_checkpatch, |
| _kernel_configcheck], |
| "chromiumos/third_party/u-boot": [_run_checkpatch_no_tree, |
| _check_change_has_branch_field], |
| } |
| |
| |
| # A dictionary of flags (keys) that can appear in the config file, and the hook |
| # that the flag disables (value) |
| _DISABLE_FLAGS = { |
| 'stray_whitespace_check': _check_no_stray_whitespace, |
| 'long_line_check': _check_no_long_lines, |
| 'cros_license_check': _check_license, |
| 'tab_check': _check_no_tabs, |
| 'branch_check': _check_change_has_branch_field, |
| 'signoff_check': _check_change_has_signoff_field, |
| } |
| |
| |
| def _get_disabled_hooks(config): |
| """Returns a set of hooks disabled by the current project's config file. |
| |
| Expects to be called within the project root. |
| |
| Args: |
| config: A ConfigParser for the project's config file. |
| """ |
| SECTION = 'Hook Overrides' |
| if not config.has_section(SECTION): |
| return set() |
| |
| disable_flags = [] |
| for flag in config.options(SECTION): |
| try: |
| if not config.getboolean(SECTION, flag): disable_flags.append(flag) |
| except ValueError as e: |
| msg = "Error parsing flag \'%s\' in %s file - " % (flag, _CONFIG_FILE) |
| print(msg + str(e)) |
| |
| disabled_keys = set(_DISABLE_FLAGS.iterkeys()).intersection(disable_flags) |
| return set([_DISABLE_FLAGS[key] for key in disabled_keys]) |
| |
| |
| def _get_project_hook_scripts(config): |
| """Returns a list of project-specific hook scripts. |
| |
| Args: |
| config: A ConfigParser for the project's config file. |
| """ |
| SECTION = 'Hook Scripts' |
| if not config.has_section(SECTION): |
| return [] |
| |
| hook_names_values = config.items(SECTION) |
| hook_names_values.sort(key=lambda x: x[0]) |
| return [x[1] for x in hook_names_values] |
| |
| |
| def _get_project_hooks(project): |
| """Returns a list of hooks that need to be run for a project. |
| |
| Expects to be called from within the project root. |
| """ |
| config = ConfigParser.RawConfigParser() |
| try: |
| config.read(_CONFIG_FILE) |
| except ConfigParser.Error: |
| # Just use an empty config file |
| config = ConfigParser.RawConfigParser() |
| |
| disabled_hooks = _get_disabled_hooks(config) |
| hooks = [hook for hook in _COMMON_HOOKS if hook not in disabled_hooks] |
| |
| if project in _PROJECT_SPECIFIC_HOOKS: |
| hooks.extend(hook for hook in _PROJECT_SPECIFIC_HOOKS[project] |
| if hook not in disabled_hooks) |
| |
| for script in _get_project_hook_scripts(config): |
| hooks.append(functools.partial(_run_project_hook_script, script)) |
| |
| return hooks |
| |
| |
| def _run_project_hooks(project, proj_dir=None, commit_list=None): |
| """For each project run its project specific hook from the hooks dictionary. |
| |
| Args: |
| project: The name of project to run hooks for. |
| proj_dir: If non-None, this is the directory the project is in. If None, |
| we'll ask repo. |
| commit_list: A list of commits to run hooks against. If None or empty list |
| then we'll automatically get the list of commits that would be uploaded. |
| |
| Returns: |
| Boolean value of whether any errors were ecountered while running the hooks. |
| """ |
| if proj_dir is None: |
| proj_dirs = _run_command(['repo', 'forall', project, '-c', 'pwd']).split() |
| if len(proj_dirs) == 0: |
| print('%s cannot be found.' % project, file=sys.stderr) |
| print('Please specify a valid project.', file=sys.stderr) |
| return True |
| if len(proj_dirs) > 1: |
| print('%s is associated with multiple directories.' % project, |
| file=sys.stderr) |
| print('Please specify a directory to help disambiguate.', file=sys.stderr) |
| return True |
| proj_dir = proj_dirs[0] |
| |
| pwd = os.getcwd() |
| # hooks assume they are run from the root of the project |
| os.chdir(proj_dir) |
| |
| if not commit_list: |
| try: |
| commit_list = _get_commits() |
| except VerifyException as e: |
| PrintErrorForProject(project, HookFailure(str(e))) |
| os.chdir(pwd) |
| return True |
| |
| hooks = _get_project_hooks(project) |
| error_found = False |
| for commit in commit_list: |
| error_list = [] |
| for hook in hooks: |
| hook_error = hook(project, commit) |
| if hook_error: |
| error_list.append(hook_error) |
| error_found = True |
| if error_list: |
| PrintErrorsForCommit(project, commit, _get_commit_desc(commit), |
| error_list) |
| |
| os.chdir(pwd) |
| return error_found |
| |
| # Main |
| |
| |
| def main(project_list, worktree_list=None, **kwargs): |
| """Main function invoked directly by repo. |
| |
| This function will exit directly upon error so that repo doesn't print some |
| obscure error message. |
| |
| Args: |
| project_list: List of projects to run on. |
| worktree_list: A list of directories. It should be the same length as |
| project_list, so that each entry in project_list matches with a directory |
| in worktree_list. If None, we will attempt to calculate the directories |
| automatically. |
| kwargs: Leave this here for forward-compatibility. |
| """ |
| found_error = False |
| if not worktree_list: |
| worktree_list = [None] * len(project_list) |
| for project, worktree in zip(project_list, worktree_list): |
| if _run_project_hooks(project, proj_dir=worktree): |
| found_error = True |
| |
| if (found_error): |
| msg = ('Preupload failed due to errors in project(s). HINTS:\n' |
| '- To disable some source style checks, and for other hints, see ' |
| '<checkout_dir>/src/repohooks/README\n' |
| '- To upload only current project, run \'repo upload .\'') |
| print(msg, file=sys.stderr) |
| sys.exit(1) |
| |
| |
| def _identify_project(path): |
| """Identify the repo project associated with the given path. |
| |
| Returns: |
| A string indicating what project is associated with the path passed in or |
| a blank string upon failure. |
| """ |
| return _run_command(['repo', 'forall', '.', '-c', 'echo ${REPO_PROJECT}'], |
| stderr=subprocess.PIPE, cwd=path).strip() |
| |
| |
| def direct_main(args, verbose=False): |
| """Run hooks directly (outside of the context of repo). |
| |
| # Setup for doctests below. |
| # ...note that some tests assume that running pre-upload on this CWD is fine. |
| # TODO: Use mock and actually mock out _run_project_hooks() for tests. |
| >>> mydir = os.path.dirname(os.path.abspath(__file__)) |
| >>> olddir = os.getcwd() |
| |
| # OK to run w/ no arugments; will run with CWD. |
| >>> os.chdir(mydir) |
| >>> direct_main(['prog_name'], verbose=True) |
| Running hooks on chromiumos/repohooks |
| 0 |
| >>> os.chdir(olddir) |
| |
| # Run specifying a dir |
| >>> direct_main(['prog_name', '--dir=%s' % mydir], verbose=True) |
| Running hooks on chromiumos/repohooks |
| 0 |
| |
| # Not a problem to use a bogus project; we'll just get default settings. |
| >>> direct_main(['prog_name', '--dir=%s' % mydir, '--project=X'],verbose=True) |
| Running hooks on X |
| 0 |
| |
| # Run with project but no dir |
| >>> os.chdir(mydir) |
| >>> direct_main(['prog_name', '--project=X'], verbose=True) |
| Running hooks on X |
| 0 |
| >>> os.chdir(olddir) |
| |
| # Try with a non-git CWD |
| >>> os.chdir('/tmp') |
| >>> direct_main(['prog_name']) |
| Traceback (most recent call last): |
| ... |
| BadInvocation: The current directory is not part of a git project. |
| |
| # Check various bad arguments... |
| >>> direct_main(['prog_name', 'bogus']) |
| Traceback (most recent call last): |
| ... |
| BadInvocation: Unexpected arguments: bogus |
| >>> direct_main(['prog_name', '--project=bogus', '--dir=bogusdir']) |
| Traceback (most recent call last): |
| ... |
| BadInvocation: Invalid dir: bogusdir |
| >>> direct_main(['prog_name', '--project=bogus', '--dir=/tmp']) |
| Traceback (most recent call last): |
| ... |
| BadInvocation: Not a git directory: /tmp |
| |
| Args: |
| args: The value of sys.argv |
| |
| Returns: |
| 0 if no pre-upload failures, 1 if failures. |
| |
| Raises: |
| BadInvocation: On some types of invocation errors. |
| """ |
| desc = 'Run Chromium OS pre-upload hooks on changes compared to upstream.' |
| parser = optparse.OptionParser(description=desc) |
| |
| parser.add_option('--dir', default=None, |
| help='The directory that the project lives in. If not ' |
| 'specified, use the git project root based on the cwd.') |
| parser.add_option('--project', default=None, |
| help='The project repo path; this can affect how the hooks ' |
| 'get run, since some hooks are project-specific. For ' |
| 'chromite this is chromiumos/chromite. If not specified, ' |
| 'the repo tool will be used to figure this out based on ' |
| 'the dir.') |
| parser.add_option('--rerun-since', default=None, |
| help='Rerun hooks on old commits since the given date. ' |
| 'The date should match git log\'s concept of a date. ' |
| 'e.g. 2012-06-20') |
| |
| parser.usage = "pre-upload.py [options] [commits]" |
| |
| opts, args = parser.parse_args(args[1:]) |
| |
| if opts.rerun_since: |
| if args: |
| raise BadInvocation('Can\'t pass commits and use rerun-since: %s' % |
| ' '.join(args)) |
| |
| cmd = ['git', 'log', '--since="%s"' % opts.rerun_since, '--pretty=%H'] |
| all_commits = _run_command(cmd).splitlines() |
| bot_commits = _run_command(cmd + ['--author=chrome-bot']).splitlines() |
| |
| # Eliminate chrome-bot commits but keep ordering the same... |
| bot_commits = set(bot_commits) |
| args = [c for c in all_commits if c not in bot_commits] |
| |
| |
| # Check/normlaize git dir; if unspecified, we'll use the root of the git |
| # project from CWD |
| if opts.dir is None: |
| git_dir = _run_command(['git', 'rev-parse', '--git-dir'], |
| stderr=subprocess.PIPE).strip() |
| if not git_dir: |
| raise BadInvocation('The current directory is not part of a git project.') |
| opts.dir = os.path.dirname(os.path.abspath(git_dir)) |
| elif not os.path.isdir(opts.dir): |
| raise BadInvocation('Invalid dir: %s' % opts.dir) |
| elif not os.path.isdir(os.path.join(opts.dir, '.git')): |
| raise BadInvocation('Not a git directory: %s' % opts.dir) |
| |
| # Identify the project if it wasn't specified; this _requires_ the repo |
| # tool to be installed and for the project to be part of a repo checkout. |
| if not opts.project: |
| opts.project = _identify_project(opts.dir) |
| if not opts.project: |
| raise BadInvocation("Repo couldn't identify the project of %s" % opts.dir) |
| |
| if verbose: |
| print("Running hooks on %s" % (opts.project)) |
| |
| found_error = _run_project_hooks(opts.project, proj_dir=opts.dir, |
| commit_list=args) |
| if found_error: |
| return 1 |
| return 0 |
| |
| |
| def _test(): |
| """Run any built-in tests.""" |
| import doctest |
| doctest.testmod() |
| |
| |
| if __name__ == '__main__': |
| if sys.argv[1:2] == ["--test"]: |
| _test() |
| exit_code = 0 |
| else: |
| prog_name = os.path.basename(sys.argv[0]) |
| try: |
| exit_code = direct_main(sys.argv) |
| except BadInvocation, e: |
| print("%s: %s" % (prog_name, str(e)), file=sys.stderr) |
| exit_code = 1 |
| sys.exit(exit_code) |