common-mk/gyplint.py

#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Linter for checking GYP files used in platform2 projects."""

from __future__ import print_function

import collections
import difflib
import os
import re
import sys

import gyp_compiler

# Find chromite!
sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                '..', '..', '..'))

from chromite.lib import commandline
from chromite.lib import cros_logging as logging


# Object holding the result of a lint check.
LintResult = collections.namedtuple('LintResult', (
    # The name of the linter checking.
    'linter',
    # The file the issue was found in.
    'file',
    # The message for this check.
    'msg',
    # The type of result -- error, warning, etc...
    'type',
))


def WalkGyp(functor, gypdata):
  """Walk |gypdata| and call |functor| on each node."""
  def WalkNode(node):
    ret = []
    if isinstance(node, dict):
      for k, v in node.items():
        ret += functor(k, v)
        ret += WalkNode(v)
    elif isinstance(node, (tuple, list)):
      for v in node:
        ret += WalkNode(v)
    return ret

  return WalkNode(gypdata)


def GypLintLibFlags(gypdata):
  """-lfoo flags belong in 'libraries' and not 'ldflags'."""
  def CheckNode(key, value):
    ret = []
    if key.startswith('ldflags'):
      for flag in value:
        if flag.startswith('-l'):
          ret.append('-l flags should be in "libraries", not "ldflags": %s' %
                     (flag,))
    return ret

  return WalkGyp(CheckNode, gypdata)


def GypLintVisibilityFlags(gypdata):
  """Packages should not change -fvisibility settings."""
  def CheckNode(key, value):
    ret = []
    if key.startswith('cflags'):
      for flag in value:
        if flag.startswith('-fvisibility'):
          ret.append('do not use -fvisibility; to export symbols, use '
                     'brillo/brillo_export.h instead')
    return ret

  return WalkGyp(CheckNode, gypdata)


def GypLintDefineFlags(gypdata):
  """-D flags should be in 'defines', not cflags."""
  def CheckNode(key, value):
    ret = []
    if key.startswith('cflags'):
      for flag in value:
        if flag.startswith('-D'):
          ret.append('-D flags should be in "defines", not "%s": %s' %
                     (key, flag))
    return ret

  return WalkGyp(CheckNode, gypdata)


def GypLintCommonTesting(gypdata):
  """Packages should use common_test.gypi instead of -lgtest/-lgmock."""
  def CheckNode(key, value):
    ret = []
    if key.startswith('ldflags') or key.startswith('libraries'):
      if '-lgtest' in value or '-lgmock' in value:
        ret.append('use common-mk/common_test.gypi for tests instead of '
                   'linking against -lgtest/-lgmock directly')
    return ret

  return WalkGyp(CheckNode, gypdata)


def GypLintOrderedFiles(gypdata):
  """Files should be kept sorted.

  Python's sorted() function relies on __cmp__ of the objects it is sorting.
  Since we're feeding it ASCII strings, it will do byte-wise comparing.  This
  matches clang-format behavior when it sorts include files.
  """
  def CheckNode(key, value):
    ret = []
    if key == 'sources':
      lines = list(difflib.unified_diff(value, sorted(value), lineterm=''))
      if lines:
        # Skip the first three entries as those are the diff header.
        ret.append('source files should be kept sorted:\n%s' %
                   ('\n'.join(lines[3:]),))
    return ret

  return WalkGyp(CheckNode, gypdata)


# It's not easy to auto-discover pkg-config files as we don't require a chroot
# or a fully installed sysroot to run this linter.  Plus, there's no clean way
# to correlate -lfoo names with pkg-config .pc file names.  List the packages
# that we tend to use in platform2 projects.
KNOWN_PC_FILES = {
    '-lblkid': 'blkid',
    '-lcap': 'libcap',
    '-lcrypto': 'libcrypto',
    '-ldbus-1': 'dbus-1',
    '-ldbus-c++-1': 'dbus-c++-1',
    '-ldbus-glib-1': 'dbus-glib-1',
    '-lexpat': 'expat',
    '-lfuse': 'fuse',
    '-lglib-2.0': 'glib-2.0',
    '-lgobject-2.0': 'gobject-2.0',
    '-lgthread-2.0': 'gthread-2.0',
    '-lminijail': 'libminijail',
    '-lpcre': 'libpcre',
    '-lpcrecpp': 'libpcrecpp',
    '-lpcreposix': 'libpcreposix',
    '-lprotobuf': 'protobuf',
    '-lprotobuf-lite': 'protobuf-lite',
    '-lssl': 'libssl',
    '-ludev': 'libudev',
    '-lusb-1.0': 'libusb-1.0',
    '-luuid': 'uuid',
    '-lz': 'zlib',
}
KNOWN_PC_LIBS = frozenset(KNOWN_PC_FILES.keys())

def GypLintPkgConfigs(gypdata):
  """Use pkg-config files for known libs instead of manual -lfoo linkage."""
  def CheckNode(key, value):
    ret = []
    if key.startswith('ldflags') or key.startswith('libraries'):
      for v in KNOWN_PC_LIBS & set(value):
        ret.append(('use pkg-config instead: delete "%s" from "%s" and add '
                    '"%s" to "deps"') % (v, key, KNOWN_PC_FILES[v]))
    return ret

  return WalkGyp(CheckNode, gypdata)


def RawLintWhitespace(data):
  """Make sure whitespace is sane."""
  ret = []
  if not data.endswith('\n'):
    ret.append('missing newline at end of file')
  return ret


def LineIsComment(line):
  """Whether this entire line is a comment (and whitespace).

  Note: This doesn't handle inline comments like:
    [  # Blah blah.
  But we discourage those in general, so shouldn't be a problem.
  """
  return line.lstrip().startswith('#')


def LinesLintWhitespace(lines):
  """Make sure whitespace is sane."""
  ret = []

  for i, line in enumerate(lines, 1):
    if '\t' in line:
      ret.append('use spaces, not tabs: line %i: %s' % (i, line))

    if line.rstrip() != line:
      ret.append('delete trailing whitespace: line %i: %s' % (i, line))

  if lines:
    if not lines[0]:
      ret.append('delete leading blanklines')

    if not lines[-1]:
      ret.append('delete trailing blanklines')

  return ret


def LinesLintDanglingCommas(lines):
  """Check for missing dangling commas."""
  ret = []
  for i, line in enumerate(lines, 1):
    if not LineIsComment(line):
      if len(line) > 1 and line.endswith(('}', ']', "'")):
        ret.append('add a dangling comma: line %i: %s' % (i, line))
  return ret


def LinesLintSingleQuotes(lines):
  """Check for double quote usage."""
  ret = []
  for i, line in enumerate(lines, 1):
    if not LineIsComment(line):
      if line.endswith('"'):
        ret.append('use single quotes instead of double quotes: line %i: %s' %
                   (i, line))
  return ret


def LinesLintCuddled(lines):
  """Allow for cuddled braces only with one liners.

  This is OK:
    'dependencies': ['libwimax_manager'],
  But not this:
    'dependencies': ['libwimax_manager',
                     'libfoo'],
  """
  ret = []

  named_element = re.compile(r"^'[^']+': ([[{].*)$")

  for i, line in enumerate(lines, 1):
    if LineIsComment(line):
      continue

    check = line.lstrip()

    # Handle named elements like:
    # 'foo': [
    m = named_element.match(check)
    if m:
      check = m.group(1)

    # Skip lines that are just one char -- can't be dangling!
    if len(check) <= 1:
      continue

    # Skip lines that don't have a list or dict.
    if check[0] not in ('[', '{'):
      continue

    # Ignore conditional forms:
    # ['deps != []', {
    if check.startswith("['") and check.endswith(', {'):
      continue

    # Ignore lists of objects form:
    # 'foo': [{
    if check == '[{':
      continue

    # Ignore one-line forms:
    # ['foo', 'bar'],
    #
    # Note: This will not catch lists of lists like:
    # 'item': [ ['foo', 'bar'],
    #           ['hungry', 'cat'] ],
    # But that's OK because those will usually get caught by the indentation
    # checker.  We'd need to have the AST available here to support it.
    if check[-1] == ',' and (check[0], check[-2]) == ('[', ']'):
      continue

    # If we're still here, the line is bad.
    ret.append('uncuddle lists/dicts with multiple elements: line %i: %s' %
               (i, line))

  return ret


def LinesLintIndent(lines):
  """Make sure indentation levels are correct."""
  ret = []

  indent = 0
  for i, line in enumerate(lines, 1):
    # Always allow blank lines.
    if not line:
      continue

    # Allow indent to increase or decrease or stay the same.
    # Note: We don't currently check overall syntax to make sure the change
    # is a valid change.  This just checks within common bounds.
    num_spaces = len(line) - len(line.lstrip())
    if num_spaces in (indent - 2, indent, indent + 2):
      indent = num_spaces
      continue

    ret.append('indent is incorrect: %i: %s' % (i, line))

  return ret


# The regex used to find gyplint options in the file.
# This matches the regex pylint uses.
OPTIONS_RE = re.compile(r'^\s*#.*\bgyplint:\s*([^\n;]+)', flags=re.MULTILINE)

# Object holding linter settings.
LintSettings = collections.namedtuple('LinterSettings', (
    # Linters to skip.
    'skip',
))

def ParseOptions(options, name=None):
  """Parse out the linter settings from |options|.

  Currently we support:
    disable=<linter name>

  Args:
    options: A list of linter options (e.g. ['foo=bar']).
    name: The file we're parsing.

  Returns:
    A LintSettings object.
  """
  skip = set()

  for option in options:
    key, value = option.split('=', 1)
    key = key.strip()
    value = value.strip()

    if key == 'disable':
      skip.update(x.strip() for x in value.split(','))
    else:
      raise ValueError('%s: unknown gyplint option: %s' % (name, key))

  return LintSettings(skip)


def RunLinters(prefix, name, data, settings=None):
  """Run linters starting with |prefix| against |data|."""
  if settings is None:
    settings = ParseOptions([])

  linters = [x for x in globals()
             if x.startswith(prefix) and x not in settings.skip]

  ret = []
  for linter in linters:
    functor = globals().get(linter)
    for result in functor(data):
      ret.append(LintResult(linter, name, result, logging.ERROR))
  return ret


def CheckGyp(name, gypdata, settings=None):
  """Check |gypdata| for common mistakes."""
  return RunLinters('GypLint', name, gypdata, settings=settings)


def CheckGypData(name, data):
  """Check |data| (gyp file as a string) for common mistakes."""
  try:
    gypdata = gyp_compiler.CheckedEval(data)
  except Exception as e:
    return [LintResult('gyp.input.CheckedEval', name, 'invalid format: %s' % e,
                       logging.ERROR)]

  # Parse the gyplint flags in the file.
  m = OPTIONS_RE.findall(data)
  settings = ParseOptions(m, name=name)

  lines = data.splitlines()
  ret = []

  # Run linters on the raw data first (for style/syntax).
  ret += RunLinters('RawLint', name, data, settings=settings)
  ret += RunLinters('LinesLint', name, lines, settings=settings)
  # Then run linters against the parsed AST.
  ret += CheckGyp(name, gypdata, settings)

  return ret


def CheckGypFile(gypfile):
  """Check |gypfile| for common mistakes."""
  with open(gypfile) as fp:
    return CheckGypData(gypfile, fp.read())


def FilterFiles(files, extensions):
  """Filter out |files| based on |extensions|."""
  for f in files:
    # Chop of the leading period as we'll get back ".bin".
    extension = os.path.splitext(f)[1][1:]
    if extension in extensions and not os.path.basename(f).startswith('.'):
      logging.debug('Checking %s', f)
      yield f
    else:
      logging.debug('Skipping %s', f)


def FilterPaths(paths, extensions):
  """Walk |paths| recursively and filter out content in it."""
  files = []
  dirs = []
  for path in paths:
    if os.path.isdir(path):
      dirs.append(path)
    else:
      files.append(path)

  for gypfile in FilterFiles(files, extensions):
    yield gypfile

  for gypdir in dirs:
    for root, _, files in os.walk(gypdir):
      for gypfile in FilterFiles(files, extensions):
        yield os.path.join(root, gypfile)


def GetParser():
  """Return an argument parser."""
  parser = commandline.ArgumentParser(description=__doc__)
  parser.add_argument('--extensions', default='gyp,gypi',
                      help='Comma delimited file extensions to check. '
                           '(default: %(default)s)')
  parser.add_argument('files', nargs='*',
                      help='Files to run lint.')
  return parser


def main(argv):
  parser = GetParser()
  opts = parser.parse_args(argv)

  if not opts.files:
    logging.warning('No files provided to lint.  Doing nothing.')
    return 0

  extensions = set(opts.extensions.split(','))
  num_files = 0
  for gypfile in FilterPaths(opts.files, extensions):
    results = CheckGypFile(gypfile)
    if results:
      logging.error('**** %s: found %i issue(s)', gypfile, len(results))
      for result in results:
        logging.log(result.type, '%s: %s', result.linter, result.msg)
      num_files += 1
  if num_files:
    logging.error('%i file(s) failed linting', num_files)
  return 1 if num_files else 0


if __name__ == '__main__':
  commandline.ScriptWrapperMain(lambda _: main)