blob: fd670219d21f4b374c00108ba1ee10d5fb546d46 [file] [log] [blame]
# -*- coding: utf-8 -*-
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Run lint checks on the specified files."""
from __future__ import print_function
import functools
import json
import multiprocessing
import os
import re
import sys
import urllib
from chromite.lib import constants
from chromite.cli import command
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import git
from chromite.lib import osutils
from chromite.lib import parallel
# Extract a script's shebang.
SHEBANG_RE = re.compile(r'^#!\s*([^\s]+)(\s+([^\s]+))?')
def _GetProjectPath(path):
"""Find the absolute path of the git checkout that contains |path|."""
if git.FindRepoCheckoutRoot(path):
manifest = git.ManifestCheckout.Cached(path)
return manifest.FindCheckoutFromPath(path).GetPath(absolute=True)
# Maybe they're running on a file outside of a checkout.
# e.g. cros lint ~/ /tmp/
return os.path.dirname(path)
def _GetPylintrc(path):
"""Locate the pylintrc file that applies to |path|."""
path = os.path.realpath(path)
project_path = _GetProjectPath(path)
parent = os.path.dirname(path)
while project_path and parent.startswith(project_path):
pylintrc = os.path.join(parent, 'pylintrc')
if os.path.isfile(pylintrc):
parent = os.path.dirname(parent)
if project_path is None or not os.path.isfile(pylintrc):
pylintrc = os.path.join(constants.SOURCE_ROOT, 'chromite', 'pylintrc')
return pylintrc
def _GetPylintGroups(paths):
"""Return a dictionary mapping pylintrc files to lists of paths."""
groups = {}
for path in paths:
pylintrc = _GetPylintrc(path)
if pylintrc:
groups.setdefault(pylintrc, []).append(path)
return groups
def _GetPythonPath(paths):
"""Return the set of Python library paths to use."""
return sys.path + [
# Add the Portage installation inside the chroot to the Python path.
# This ensures that scripts that need to import portage can do so.
os.path.join(constants.SOURCE_ROOT, 'chroot', 'usr', 'lib', 'portage',
# Scripts outside of chromite expect the scripts in src/scripts/lib to
# be importable.
os.path.join(constants.CROSUTILS_DIR, 'lib'),
# Allow platform projects to be imported by name (e.g. crostestutils).
os.path.join(constants.SOURCE_ROOT, 'src', 'platform'),
# Ideally we'd modify meta_path in pylint to handle our virtual chromite
# module, but that's not possible currently. We'll have to deal with
# that at some point if we want `cros lint` to work when the dir is not
# named 'chromite'.
# Also allow scripts to import from their current directory.
] + list(set(os.path.dirname(x) for x in paths))
# The mapping between the "cros lint" --output-format flag and
# --output flag.
'colorized': 'emacs',
'msvs': 'vs7',
'parseable': 'emacs',
# The mapping between the "cros lint" --output-format flag and shellcheck
# flags.
# Note that the msvs mapping here isn't quite VS format, but it's closer than
# the default output.
'colorized': ['--color=always'],
'msvs': ['--format=gcc'],
'parseable': ['--format=gcc'],
def _LinterRunCommand(cmd, debug, **kwargs):
"""Run the linter with common RunCommand args set as higher levels expect."""
return cros_build_lib.RunCommand(cmd, error_code_ok=True, print_cmd=debug,
debug_level=logging.NOTICE, **kwargs)
def _WhiteSpaceLintData(path, data):
"""Run basic whitespace checks on |data|.
path: The name of the file (for diagnostics).
data: The file content to lint.
True if everything passed.
ret = True
# Make sure files all have a trailing newline.
if not data.endswith('\n'):
ret = False
logging.warn('%s: file needs a trailing newline', path)
# Disallow leading & trailing blank lines.
if data.startswith('\n'):
ret = False
logging.warn('%s: delete leading blank lines', path)
if data.endswith('\n\n'):
ret = False
logging.warn('%s: delete trailing blank lines', path)
for i, line in enumerate(data.splitlines(), start=1):
if line.rstrip() != line:
ret = False
logging.warn('%s:%i: trim trailing whitespace: %s', path, i, line)
return ret
def _CpplintFile(path, output_format, debug):
"""Returns result of running cpplint on |path|."""
cmd = [os.path.join(constants.DEPOT_TOOLS_DIR, '')]
if output_format != 'default':
cmd.append('--output=%s' % CPPLINT_OUTPUT_FORMAT_MAP[output_format])
return _LinterRunCommand(cmd, debug)
def _PylintFile(path, output_format, debug):
"""Returns result of running pylint on |path|."""
pylint = os.path.join(constants.DEPOT_TOOLS_DIR, 'pylint')
pylintrc = _GetPylintrc(path)
cmd = [pylint, '--rcfile=%s' % pylintrc]
if output_format != 'default':
cmd.append('--output-format=%s' % output_format)
extra_env = {'PYTHONPATH': ':'.join(_GetPythonPath([path]))}
return _LinterRunCommand(cmd, debug, extra_env=extra_env)
def _JsonLintFile(path, _output_format, _debug):
"""Returns result of running json lint checks on |path|."""
result = cros_build_lib.CommandResult('python -mjson.tool "%s"' % path,
data = osutils.ReadFile(path)
# Strip off leading UTF-8 BOM if it exists.
if data.startswith(b'\xef\xbb\xbf'):
data = data[3:]
# Strip out comments for JSON parsing.
stripped_data = re.sub(r'^\s*#.*', '', data, flags=re.M)
# See if it validates.
except ValueError as e:
result.returncode = 1
logging.notice('%s: %s', path, e)
# Check whitespace.
if not _WhiteSpaceLintData(path, data):
result.returncode = 1
return result
def _MarkdownLintFile(path, _output_format, _debug):
"""Returns result of running lint checks on |path|."""
result = cros_build_lib.CommandResult('mdlint(internal) "%s"' % path,
data = osutils.ReadFile(path)
# Check whitespace.
if not _WhiteSpaceLintData(path, data):
result.returncode = 1
return result
def _ShellLintFile(path, output_format, debug, gentoo_format=False):
"""Returns result of running lint checks on |path|."""
# TODO: Try using `checkbashisms`.
syntax_check = _LinterRunCommand(['bash', '-n', path], debug)
if syntax_check.returncode != 0:
return syntax_check
# Try using shellcheck if it exists.
shellcheck = osutils.Which('shellcheck')
if not shellcheck:
logging.notice('Install shellcheck for additional shell linting.')
return syntax_check
cmd = [shellcheck]
if output_format != 'default':
if gentoo_format:
# ebuilds don't explicitly export variables or contain a shebang.
# ebuilds always use bash.
lint_result = _LinterRunCommand(cmd, debug)
# During testing, we don't want to fail the linter for shellcheck errors,
# so override the return code.
if lint_result.returncode != 0:
bug_url = (
'' +
'Defect report from Developer',
'Bad shellcheck warnings for %s' % os.path.basename(path),
'Shellcheck output from file:\n%s\n\n<paste output here>\n\n'
"What is wrong with shellcheck's findings?\n" % path,
logging.warn('Shellcheck found problems. These will eventually become '
'errors. If the shellcheck findings are not useful, '
'please file a bug at:\n%s', bug_url)
lint_result.returncode = 0
return lint_result
def _GentooShellLintFile(path, output_format, debug):
"""Run shell checks with Gentoo rules."""
return _ShellLintFile(path, output_format, debug, gentoo_format=True)
def _BreakoutDataByLinter(map_to_return, path):
"""Maps a linter method to the content of the |path|."""
# Detect by content of the file itself.
with open(path) as fp:
# We read 128 bytes because that's the Linux kernel's current limit.
# Look for BINPRM_BUF_SIZE in fs/binfmt_script.c.
data =
if not data.startswith('#!'):
# If the file doesn't have a shebang, nothing to do.
m = SHEBANG_RE.match(data)
if m:
prog =
if prog == '/usr/bin/env':
prog =
basename = os.path.basename(prog)
if basename.startswith('python'):
pylint_list = map_to_return.setdefault(_PylintFile, [])
elif basename in ('sh', 'dash', 'bash'):
shlint_list = map_to_return.setdefault(_ShellLintFile, [])
except IOError as e:
logging.debug('%s: reading initial data failed: %s', path, e)
# Map file extensions to a linter function.
# Note these are defined to keep in line with Technically, we
# could include additional ones, but would just filter them out.
frozenset({'.cc', '.cpp', '.h'}): _CpplintFile,
frozenset({'.json'}): _JsonLintFile,
frozenset({'.py'}): _PylintFile,
frozenset({'.sh'}): _ShellLintFile,
frozenset({'.ebuild', '.eclass', '.bashrc'}): _GentooShellLintFile,
frozenset({'.md'}): _MarkdownLintFile,
def _BreakoutFilesByLinter(files):
"""Maps a linter method to the list of files to lint."""
map_to_return = {}
for f in files:
extension = os.path.splitext(f)[1]
for extensions, linter in _EXT_TO_LINTER_MAP.items():
if extension in extensions:
todo = map_to_return.setdefault(linter, [])
if os.path.isfile(f):
_BreakoutDataByLinter(map_to_return, f)
return map_to_return
def _Dispatcher(errors, output_format, debug, linter, path):
"""Call |linter| on |path| and take care of coalescing exit codes/output."""
result = linter(path, output_format, debug)
if result.returncode:
with errors.get_lock():
errors.value += 1
class LintCommand(command.CliCommand):
"""Run lint checks on the specified files."""
EPILOG = """
Right now, only supports cpplint and pylint. We may also in the future
run other checks (e.g. pyflakes, etc.)
# The output formats supported by cros lint.
OUTPUT_FORMATS = ('default', 'colorized', 'msvs', 'parseable')
def AddParser(cls, parser):
super(LintCommand, cls).AddParser(parser)
parser.add_argument('files', help='Files to lint', nargs='*')
parser.add_argument('--output', default='default',
help='Output format to pass to the linters. Supported '
'formats are: default (no option is passed to the '
'linter), colorized, msvs (Visual Studio) and '
def Run(self):
files = self.options.files
if not files:
# Running with no arguments is allowed to make the repo upload hook
# simple, but print a warning so that if someone runs this manually
# they are aware that nothing was linted.
logging.warning('No files provided to lint. Doing nothing.')
errors = multiprocessing.Value('i')
linter_map = _BreakoutFilesByLinter(files)
dispatcher = functools.partial(_Dispatcher, errors,
self.options.output, self.options.debug)
# Special case one file as it's common -- faster to avoid parallel startup.
if sum([len(x) for _, x in linter_map.iteritems()]) == 1:
linter, files = linter_map.items()[0]
dispatcher(linter, files[0])
# Run the linter in parallel on the files.
with parallel.BackgroundTaskRunner(dispatcher) as q:
for linter, files in linter_map.iteritems():
for path in files:
q.put([linter, path])
if errors.value:
logging.error("I found lint errors in %i files.", errors.value)