# -*- coding:utf-8 -*-
|
# Copyright 2016 The Android Open Source Project
|
#
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# Unless required by applicable law or agreed to in writing, software
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# See the License for the specific language governing permissions and
|
# limitations under the License.
|
|
"""Functions that implement the actual checks."""
|
|
from __future__ import print_function
|
|
import json
|
import os
|
import platform
|
import re
|
import sys
|
|
_path = os.path.realpath(__file__ + '/../..')
|
if sys.path[0] != _path:
|
sys.path.insert(0, _path)
|
del _path
|
|
# pylint: disable=wrong-import-position
|
import rh.results
|
import rh.git
|
import rh.utils
|
|
|
class Placeholders(object):
|
"""Holder class for replacing ${vars} in arg lists.
|
|
To add a new variable to replace in config files, just add it as a @property
|
to this class using the form. So to add support for BIRD:
|
@property
|
def var_BIRD(self):
|
return <whatever this is>
|
|
You can return either a string or an iterable (e.g. a list or tuple).
|
"""
|
|
def __init__(self, diff=()):
|
"""Initialize.
|
|
Args:
|
diff: The list of files that changed.
|
"""
|
self.diff = diff
|
|
def expand_vars(self, args):
|
"""Perform place holder expansion on all of |args|.
|
|
Args:
|
args: The args to perform expansion on.
|
|
Returns:
|
The updated |args| list.
|
"""
|
all_vars = set(self.vars())
|
replacements = dict((var, self.get(var)) for var in all_vars)
|
|
ret = []
|
for arg in args:
|
# First scan for exact matches
|
for key, val in replacements.items():
|
var = '${%s}' % (key,)
|
if arg == var:
|
if isinstance(val, str):
|
ret.append(val)
|
else:
|
ret.extend(val)
|
# We break on first hit to avoid double expansion.
|
break
|
else:
|
# If no exact matches, do an inline replacement.
|
def replace(m):
|
val = self.get(m.group(1))
|
if isinstance(val, str):
|
return val
|
else:
|
return ' '.join(val)
|
ret.append(re.sub(r'\$\{(%s)\}' % ('|'.join(all_vars),),
|
replace, arg))
|
|
return ret
|
|
@classmethod
|
def vars(cls):
|
"""Yield all replacement variable names."""
|
for key in dir(cls):
|
if key.startswith('var_'):
|
yield key[4:]
|
|
def get(self, var):
|
"""Helper function to get the replacement |var| value."""
|
return getattr(self, 'var_%s' % (var,))
|
|
@property
|
def var_PREUPLOAD_COMMIT_MESSAGE(self):
|
"""The git commit message."""
|
return os.environ.get('PREUPLOAD_COMMIT_MESSAGE', '')
|
|
@property
|
def var_PREUPLOAD_COMMIT(self):
|
"""The git commit sha1."""
|
return os.environ.get('PREUPLOAD_COMMIT', '')
|
|
@property
|
def var_PREUPLOAD_FILES(self):
|
"""List of files modified in this git commit."""
|
return [x.file for x in self.diff if x.status != 'D']
|
|
@property
|
def var_REPO_ROOT(self):
|
"""The root of the repo checkout."""
|
return rh.git.find_repo_root()
|
|
@property
|
def var_BUILD_OS(self):
|
"""The build OS (see _get_build_os_name for details)."""
|
return _get_build_os_name()
|
|
|
class HookOptions(object):
|
"""Holder class for hook options."""
|
|
def __init__(self, name, args, tool_paths):
|
"""Initialize.
|
|
Args:
|
name: The name of the hook.
|
args: The override commandline arguments for the hook.
|
tool_paths: A dictionary with tool names to paths.
|
"""
|
self.name = name
|
self._args = args
|
self._tool_paths = tool_paths
|
|
@staticmethod
|
def expand_vars(args, diff=()):
|
"""Perform place holder expansion on all of |args|."""
|
replacer = Placeholders(diff=diff)
|
return replacer.expand_vars(args)
|
|
def args(self, default_args=(), diff=()):
|
"""Gets the hook arguments, after performing place holder expansion.
|
|
Args:
|
default_args: The list to return if |self._args| is empty.
|
diff: The list of files that changed in the current commit.
|
|
Returns:
|
A list with arguments.
|
"""
|
args = self._args
|
if not args:
|
args = default_args
|
|
return self.expand_vars(args, diff=diff)
|
|
def tool_path(self, tool_name):
|
"""Gets the path in which the |tool_name| executable can be found.
|
|
This function performs expansion for some place holders. If the tool
|
does not exist in the overridden |self._tool_paths| dictionary, the tool
|
name will be returned and will be run from the user's $PATH.
|
|
Args:
|
tool_name: The name of the executable.
|
|
Returns:
|
The path of the tool with all optional place holders expanded.
|
"""
|
assert tool_name in TOOL_PATHS
|
if tool_name not in self._tool_paths:
|
return TOOL_PATHS[tool_name]
|
|
tool_path = os.path.normpath(self._tool_paths[tool_name])
|
return self.expand_vars([tool_path])[0]
|
|
|
def _run_command(cmd, **kwargs):
|
"""Helper command for checks that tend to gather output."""
|
kwargs.setdefault('redirect_stderr', True)
|
kwargs.setdefault('combine_stdout_stderr', True)
|
kwargs.setdefault('capture_output', True)
|
kwargs.setdefault('error_code_ok', True)
|
return rh.utils.run_command(cmd, **kwargs)
|
|
|
def _match_regex_list(subject, expressions):
|
"""Try to match a list of regular expressions to a string.
|
|
Args:
|
subject: The string to match regexes on.
|
expressions: An iterable of regular expressions to check for matches with.
|
|
Returns:
|
Whether the passed in subject matches any of the passed in regexes.
|
"""
|
for expr in expressions:
|
if re.search(expr, subject):
|
return True
|
return False
|
|
|
def _filter_diff(diff, include_list, exclude_list=()):
|
"""Filter out files based on the conditions passed in.
|
|
Args:
|
diff: list of diff objects to filter.
|
include_list: list of regex that when matched with a file path will cause
|
it to be added to the output list unless the file is also matched with
|
a regex in the exclude_list.
|
exclude_list: list of regex that when matched with a file will prevent it
|
from being added to the output list, even if it is also matched with a
|
regex in the include_list.
|
|
Returns:
|
A list of filepaths that contain files matched in the include_list and not
|
in the exclude_list.
|
"""
|
filtered = []
|
for d in diff:
|
if (d.status != 'D' and
|
_match_regex_list(d.file, include_list) and
|
not _match_regex_list(d.file, exclude_list)):
|
# We've got a match!
|
filtered.append(d)
|
return filtered
|
|
|
def _get_build_os_name():
|
"""Gets the build OS name.
|
|
Returns:
|
A string in a format usable to get prebuilt tool paths.
|
"""
|
system = platform.system()
|
if 'Darwin' in system or 'Macintosh' in system:
|
return 'darwin-x86'
|
else:
|
# TODO: Add more values if needed.
|
return 'linux-x86'
|
|
|
def _fixup_func_caller(cmd, **kwargs):
|
"""Wraps |cmd| around a callable automated fixup.
|
|
For hooks that support automatically fixing errors after running (e.g. code
|
formatters), this function provides a way to run |cmd| as the |fixup_func|
|
parameter in HookCommandResult.
|
"""
|
def wrapper():
|
result = _run_command(cmd, **kwargs)
|
if result.returncode not in (None, 0):
|
return result.output
|
return None
|
return wrapper
|
|
|
def _check_cmd(hook_name, project, commit, cmd, fixup_func=None, **kwargs):
|
"""Runs |cmd| and returns its result as a HookCommandResult."""
|
return [rh.results.HookCommandResult(hook_name, project, commit,
|
_run_command(cmd, **kwargs),
|
fixup_func=fixup_func)]
|
|
|
# Where helper programs exist.
|
TOOLS_DIR = os.path.realpath(__file__ + '/../../tools')
|
|
def get_helper_path(tool):
|
"""Return the full path to the helper |tool|."""
|
return os.path.join(TOOLS_DIR, tool)
|
|
|
def check_custom(project, commit, _desc, diff, options=None, **kwargs):
|
"""Run a custom hook."""
|
return _check_cmd(options.name, project, commit, options.args((), diff),
|
**kwargs)
|
|
|
def check_checkpatch(project, commit, _desc, diff, options=None):
|
"""Run |diff| through the kernel's checkpatch.pl tool."""
|
tool = get_helper_path('checkpatch.pl')
|
cmd = ([tool, '-', '--root', project.dir] +
|
options.args(('--ignore=GERRIT_CHANGE_ID',), diff))
|
return _check_cmd('checkpatch.pl', project, commit, cmd,
|
input=rh.git.get_patch(commit))
|
|
|
def check_clang_format(project, commit, _desc, diff, options=None):
|
"""Run git clang-format on the commit."""
|
tool = get_helper_path('clang-format.py')
|
clang_format = options.tool_path('clang-format')
|
git_clang_format = options.tool_path('git-clang-format')
|
tool_args = (['--clang-format', clang_format, '--git-clang-format',
|
git_clang_format] +
|
options.args(('--style', 'file', '--commit', commit), diff))
|
cmd = [tool] + tool_args
|
fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args)
|
return _check_cmd('clang-format', project, commit, cmd,
|
fixup_func=fixup_func)
|
|
|
def check_google_java_format(project, commit, _desc, _diff, options=None):
|
"""Run google-java-format on the commit."""
|
|
tool = get_helper_path('google-java-format.py')
|
google_java_format = options.tool_path('google-java-format')
|
google_java_format_diff = options.tool_path('google-java-format-diff')
|
tool_args = ['--google-java-format', google_java_format,
|
'--google-java-format-diff', google_java_format_diff,
|
'--commit', commit] + options.args()
|
cmd = [tool] + tool_args
|
fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args)
|
return _check_cmd('google-java-format', project, commit, cmd,
|
fixup_func=fixup_func)
|
|
|
def check_commit_msg_bug_field(project, commit, desc, _diff, options=None):
|
"""Check the commit message for a 'Bug:' line."""
|
field = 'Bug'
|
regex = r'^%s: (None|[0-9]+(, [0-9]+)*)$' % (field,)
|
check_re = re.compile(regex)
|
|
if options.args():
|
raise ValueError('commit msg %s check takes no options' % (field,))
|
|
found = []
|
for line in desc.splitlines():
|
if check_re.match(line):
|
found.append(line)
|
|
if not found:
|
error = ('Commit message is missing a "%s:" line. It must match the\n'
|
'following case-sensitive regex:\n\n %s') % (field, regex)
|
else:
|
return
|
|
return [rh.results.HookResult('commit msg: "%s:" check' % (field,),
|
project, commit, error=error)]
|
|
|
def check_commit_msg_changeid_field(project, commit, desc, _diff, options=None):
|
"""Check the commit message for a 'Change-Id:' line."""
|
field = 'Change-Id'
|
regex = r'^%s: I[a-f0-9]+$' % (field,)
|
check_re = re.compile(regex)
|
|
if options.args():
|
raise ValueError('commit msg %s check takes no options' % (field,))
|
|
found = []
|
for line in desc.splitlines():
|
if check_re.match(line):
|
found.append(line)
|
|
if len(found) == 0:
|
error = ('Commit message is missing a "%s:" line. It must match the\n'
|
'following case-sensitive regex:\n\n %s') % (field, regex)
|
elif len(found) > 1:
|
error = ('Commit message has too many "%s:" lines. There can be only '
|
'one.') % (field,)
|
else:
|
return
|
|
return [rh.results.HookResult('commit msg: "%s:" check' % (field,),
|
project, commit, error=error)]
|
|
|
PREBUILT_APK_MSG = """Commit message is missing required prebuilt APK
|
information. To generate the information, use the aapt tool to dump badging
|
information of the APKs being uploaded, specify where the APK was built, and
|
specify whether the APKs are suitable for release:
|
|
for apk in $(find . -name '*.apk' | sort); do
|
echo "${apk}"
|
${AAPT} dump badging "${apk}" |
|
grep -iE "(package: |sdkVersion:|targetSdkVersion:)" |
|
sed -e "s/' /'\\n/g"
|
echo
|
done
|
|
It must match the following case-sensitive multiline regex searches:
|
|
%s
|
|
For more information, see go/platform-prebuilt and go/android-prebuilt.
|
|
"""
|
|
|
def check_commit_msg_prebuilt_apk_fields(project, commit, desc, diff,
|
options=None):
|
"""Check that prebuilt APK commits contain the required lines."""
|
|
if options.args():
|
raise ValueError('prebuilt apk check takes no options')
|
|
filtered = _filter_diff(diff, [r'\.apk$'])
|
if not filtered:
|
return
|
|
regexes = [
|
r'^package: .*$',
|
r'^sdkVersion:.*$',
|
r'^targetSdkVersion:.*$',
|
r'^Built here:.*$',
|
(r'^This build IS( NOT)? suitable for'
|
r'( preview|( preview or)? public) release'
|
r'( but IS NOT suitable for public release)?\.$')
|
]
|
|
missing = []
|
for regex in regexes:
|
if not re.search(regex, desc, re.MULTILINE):
|
missing.append(regex)
|
|
if missing:
|
error = PREBUILT_APK_MSG % '\n '.join(missing)
|
else:
|
return
|
|
return [rh.results.HookResult('commit msg: "prebuilt apk:" check',
|
project, commit, error=error)]
|
|
|
TEST_MSG = """Commit message is missing a "Test:" line. It must match the
|
following case-sensitive regex:
|
|
%s
|
|
The Test: stanza is free-form and should describe how you tested your change.
|
As a CL author, you'll have a consistent place to describe the testing strategy
|
you use for your work. As a CL reviewer, you'll be reminded to discuss testing
|
as part of your code review, and you'll more easily replicate testing when you
|
patch in CLs locally.
|
|
Some examples below:
|
|
Test: make WITH_TIDY=1 mmma art
|
Test: make test-art
|
Test: manual - took a photo
|
Test: refactoring CL. Existing unit tests still pass.
|
|
Check the git history for more examples. It's a free-form field, so we urge
|
you to develop conventions that make sense for your project. Note that many
|
projects use exact test commands, which are perfectly fine.
|
|
Adding good automated tests with new code is critical to our goals of keeping
|
the system stable and constantly improving quality. Please use Test: to
|
highlight this area of your development. And reviewers, please insist on
|
high-quality Test: descriptions.
|
"""
|
|
|
def check_commit_msg_test_field(project, commit, desc, _diff, options=None):
|
"""Check the commit message for a 'Test:' line."""
|
field = 'Test'
|
regex = r'^%s: .*$' % (field,)
|
check_re = re.compile(regex)
|
|
if options.args():
|
raise ValueError('commit msg %s check takes no options' % (field,))
|
|
found = []
|
for line in desc.splitlines():
|
if check_re.match(line):
|
found.append(line)
|
|
if not found:
|
error = TEST_MSG % (regex)
|
else:
|
return
|
|
return [rh.results.HookResult('commit msg: "%s:" check' % (field,),
|
project, commit, error=error)]
|
|
|
def check_cpplint(project, commit, _desc, diff, options=None):
|
"""Run cpplint."""
|
# This list matches what cpplint expects. We could run on more (like .cxx),
|
# but cpplint would just ignore them.
|
filtered = _filter_diff(diff, [r'\.(cc|h|cpp|cu|cuh)$'])
|
if not filtered:
|
return
|
|
cpplint = options.tool_path('cpplint')
|
cmd = [cpplint] + options.args(('${PREUPLOAD_FILES}',), filtered)
|
return _check_cmd('cpplint', project, commit, cmd)
|
|
|
def check_gofmt(project, commit, _desc, diff, options=None):
|
"""Checks that Go files are formatted with gofmt."""
|
filtered = _filter_diff(diff, [r'\.go$'])
|
if not filtered:
|
return
|
|
gofmt = options.tool_path('gofmt')
|
cmd = [gofmt, '-l'] + options.args((), filtered)
|
ret = []
|
for d in filtered:
|
data = rh.git.get_file_content(commit, d.file)
|
result = _run_command(cmd, input=data)
|
if result.output:
|
ret.append(rh.results.HookResult(
|
'gofmt', project, commit, error=result.output,
|
files=(d.file,)))
|
return ret
|
|
|
def check_json(project, commit, _desc, diff, options=None):
|
"""Verify json files are valid."""
|
if options.args():
|
raise ValueError('json check takes no options')
|
|
filtered = _filter_diff(diff, [r'\.json$'])
|
if not filtered:
|
return
|
|
ret = []
|
for d in filtered:
|
data = rh.git.get_file_content(commit, d.file)
|
try:
|
json.loads(data)
|
except ValueError as e:
|
ret.append(rh.results.HookResult(
|
'json', project, commit, error=str(e),
|
files=(d.file,)))
|
return ret
|
|
|
def check_pylint(project, commit, _desc, diff, options=None):
|
"""Run pylint."""
|
filtered = _filter_diff(diff, [r'\.py$'])
|
if not filtered:
|
return
|
|
pylint = options.tool_path('pylint')
|
cmd = [
|
get_helper_path('pylint.py'),
|
'--executable-path', pylint,
|
] + options.args(('${PREUPLOAD_FILES}',), filtered)
|
return _check_cmd('pylint', project, commit, cmd)
|
|
|
def check_xmllint(project, commit, _desc, diff, options=None):
|
"""Run xmllint."""
|
# XXX: Should we drop most of these and probe for <?xml> tags?
|
extensions = frozenset((
|
'dbus-xml', # Generated DBUS interface.
|
'dia', # File format for Dia.
|
'dtd', # Document Type Definition.
|
'fml', # Fuzzy markup language.
|
'form', # Forms created by IntelliJ GUI Designer.
|
'fxml', # JavaFX user interfaces.
|
'glade', # Glade user interface design.
|
'grd', # GRIT translation files.
|
'iml', # Android build modules?
|
'kml', # Keyhole Markup Language.
|
'mxml', # Macromedia user interface markup language.
|
'nib', # OS X Cocoa Interface Builder.
|
'plist', # Property list (for OS X).
|
'pom', # Project Object Model (for Apache Maven).
|
'rng', # RELAX NG schemas.
|
'sgml', # Standard Generalized Markup Language.
|
'svg', # Scalable Vector Graphics.
|
'uml', # Unified Modeling Language.
|
'vcproj', # Microsoft Visual Studio project.
|
'vcxproj', # Microsoft Visual Studio project.
|
'wxs', # WiX Transform File.
|
'xhtml', # XML HTML.
|
'xib', # OS X Cocoa Interface Builder.
|
'xlb', # Android locale bundle.
|
'xml', # Extensible Markup Language.
|
'xsd', # XML Schema Definition.
|
'xsl', # Extensible Stylesheet Language.
|
))
|
|
filtered = _filter_diff(diff, [r'\.(%s)$' % '|'.join(extensions)])
|
if not filtered:
|
return
|
|
# TODO: Figure out how to integrate schema validation.
|
# XXX: Should we use python's XML libs instead?
|
cmd = ['xmllint'] + options.args(('${PREUPLOAD_FILES}',), filtered)
|
|
return _check_cmd('xmllint', project, commit, cmd)
|
|
|
def check_android_test_mapping(project, commit, _desc, diff, options=None):
|
"""Verify Android TEST_MAPPING files are valid."""
|
if options.args():
|
raise ValueError('Android TEST_MAPPING check takes no options')
|
filtered = _filter_diff(diff, [r'(^|.*/)TEST_MAPPING$'])
|
if not filtered:
|
return
|
|
testmapping_format = options.tool_path('android-test-mapping-format')
|
cmd = [testmapping_format] + options.args(
|
(project.dir, '${PREUPLOAD_FILES}',), filtered)
|
return _check_cmd('android-test-mapping-format', project, commit, cmd)
|
|
|
# Hooks that projects can opt into.
|
# Note: Make sure to keep the top level README.md up to date when adding more!
|
BUILTIN_HOOKS = {
|
'android_test_mapping_format': check_android_test_mapping,
|
'checkpatch': check_checkpatch,
|
'clang_format': check_clang_format,
|
'commit_msg_bug_field': check_commit_msg_bug_field,
|
'commit_msg_changeid_field': check_commit_msg_changeid_field,
|
'commit_msg_prebuilt_apk_fields': check_commit_msg_prebuilt_apk_fields,
|
'commit_msg_test_field': check_commit_msg_test_field,
|
'cpplint': check_cpplint,
|
'gofmt': check_gofmt,
|
'google_java_format': check_google_java_format,
|
'jsonlint': check_json,
|
'pylint': check_pylint,
|
'xmllint': check_xmllint,
|
}
|
|
# Additional tools that the hooks can call with their default values.
|
# Note: Make sure to keep the top level README.md up to date when adding more!
|
TOOL_PATHS = {
|
'android-test-mapping-format':
|
os.path.join(TOOLS_DIR, 'android_test_mapping_format.py'),
|
'clang-format': 'clang-format',
|
'cpplint': os.path.join(TOOLS_DIR, 'cpplint.py'),
|
'git-clang-format': 'git-clang-format',
|
'gofmt': 'gofmt',
|
'google-java-format': 'google-java-format',
|
'google-java-format-diff': 'google-java-format-diff.py',
|
'pylint': 'pylint',
|
}
|
