#
|
# Copyright 2015 Google Inc.
|
#
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# Unless required by applicable law or agreed to in writing, software
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# See the License for the specific language governing permissions and
|
# limitations under the License.
|
|
"""Custom script to run PyLint on apitools codebase.
|
|
"Inspired" by the similar script in gcloud-python.
|
|
This runs pylint as a script via subprocess in two different
|
subprocesses. The first lints the production/library code
|
using the default rc file (PRODUCTION_RC). The second lints the
|
demo/test code using an rc file (TEST_RC) which allows more style
|
violations (hence it has a reduced number of style checks).
|
"""
|
|
import ConfigParser
|
import copy
|
import os
|
import subprocess
|
import sys
|
|
|
IGNORED_DIRECTORIES = [
|
'apitools/gen/testdata',
|
'samples/bigquery_sample/bigquery_v2',
|
'samples/dns_sample/dns_v1',
|
'samples/fusiontables_sample/fusiontables_v1',
|
'samples/iam_sample/iam_v1',
|
'samples/servicemanagement_sample/servicemanagement_v1',
|
'samples/storage_sample/storage_v1',
|
'venv',
|
]
|
IGNORED_FILES = [
|
'ez_setup.py',
|
'run_pylint.py',
|
'setup.py',
|
]
|
PRODUCTION_RC = 'default.pylintrc'
|
TEST_RC = 'reduced.pylintrc'
|
TEST_DISABLED_MESSAGES = [
|
'exec-used',
|
'invalid-name',
|
'missing-docstring',
|
'protected-access',
|
]
|
TEST_RC_ADDITIONS = {
|
'MESSAGES CONTROL': {
|
'disable': ',\n'.join(TEST_DISABLED_MESSAGES),
|
},
|
}
|
|
|
def read_config(filename):
|
"""Reads pylintrc config onto native ConfigParser object."""
|
config = ConfigParser.ConfigParser()
|
with open(filename, 'r') as file_obj:
|
config.readfp(file_obj)
|
return config
|
|
|
def make_test_rc(base_rc_filename, additions_dict, target_filename):
|
"""Combines a base rc and test additions into single file."""
|
main_cfg = read_config(base_rc_filename)
|
|
# Create fresh config for test, which must extend production.
|
test_cfg = ConfigParser.ConfigParser()
|
test_cfg._sections = copy.deepcopy(main_cfg._sections)
|
|
for section, opts in additions_dict.items():
|
curr_section = test_cfg._sections.setdefault(
|
section, test_cfg._dict())
|
for opt, opt_val in opts.items():
|
curr_val = curr_section.get(opt)
|
if curr_val is None:
|
raise KeyError('Expected to be adding to existing option.')
|
curr_section[opt] = '%s\n%s' % (curr_val, opt_val)
|
|
with open(target_filename, 'w') as file_obj:
|
test_cfg.write(file_obj)
|
|
|
def valid_filename(filename):
|
"""Checks if a file is a Python file and is not ignored."""
|
for directory in IGNORED_DIRECTORIES:
|
if filename.startswith(directory):
|
return False
|
return (filename.endswith('.py') and
|
filename not in IGNORED_FILES)
|
|
|
def is_production_filename(filename):
|
"""Checks if the file contains production code.
|
|
:rtype: boolean
|
:returns: Boolean indicating production status.
|
"""
|
return not ('demo' in filename or 'test' in filename or
|
filename.startswith('regression'))
|
|
|
def get_files_for_linting(allow_limited=True, diff_base=None):
|
"""Gets a list of files in the repository.
|
|
By default, returns all files via ``git ls-files``. However, in some cases
|
uses a specific commit or branch (a so-called diff base) to compare
|
against for changed files. (This requires ``allow_limited=True``.)
|
|
To speed up linting on Travis pull requests against master, we manually
|
set the diff base to origin/master. We don't do this on non-pull requests
|
since origin/master will be equivalent to the currently checked out code.
|
One could potentially use ${TRAVIS_COMMIT_RANGE} to find a diff base but
|
this value is not dependable.
|
|
:type allow_limited: boolean
|
:param allow_limited: Boolean indicating if a reduced set of files can
|
be used.
|
|
:rtype: pair
|
:returns: Tuple of the diff base using the the list of filenames to be
|
linted.
|
"""
|
if os.getenv('TRAVIS') == 'true':
|
# In travis, don't default to master.
|
diff_base = None
|
|
if (os.getenv('TRAVIS_BRANCH') == 'master' and
|
os.getenv('TRAVIS_PULL_REQUEST') != 'false'):
|
# In the case of a pull request into master, we want to
|
# diff against HEAD in master.
|
diff_base = 'origin/master'
|
|
if diff_base is not None and allow_limited:
|
result = subprocess.check_output(['git', 'diff', '--name-only',
|
diff_base])
|
print 'Using files changed relative to %s:' % (diff_base,)
|
print '-' * 60
|
print result.rstrip('\n') # Don't print trailing newlines.
|
print '-' * 60
|
else:
|
print 'Diff base not specified, listing all files in repository.'
|
result = subprocess.check_output(['git', 'ls-files'])
|
|
return result.rstrip('\n').split('\n'), diff_base
|
|
|
def get_python_files(all_files=None, diff_base=None):
|
"""Gets a list of all Python files in the repository that need linting.
|
|
Relies on :func:`get_files_for_linting()` to determine which files should
|
be considered.
|
|
NOTE: This requires ``git`` to be installed and requires that this
|
is run within the ``git`` repository.
|
|
:type all_files: list or ``NoneType``
|
:param all_files: Optional list of files to be linted.
|
|
:rtype: tuple
|
:returns: A tuple containing two lists and a boolean. The first list
|
contains all production files, the next all test/demo files and
|
the boolean indicates if a restricted fileset was used.
|
"""
|
using_restricted = False
|
if all_files is None:
|
all_files, diff_base = get_files_for_linting(diff_base=diff_base)
|
using_restricted = diff_base is not None
|
|
library_files = []
|
non_library_files = []
|
for filename in all_files:
|
if valid_filename(filename):
|
if is_production_filename(filename):
|
library_files.append(filename)
|
else:
|
non_library_files.append(filename)
|
|
return library_files, non_library_files, using_restricted
|
|
|
def lint_fileset(filenames, rcfile, description):
|
"""Lints a group of files using a given rcfile."""
|
# Only lint filenames that exist. For example, 'git diff --name-only'
|
# could spit out deleted / renamed files. Another alternative could
|
# be to use 'git diff --name-status' and filter out files with a
|
# status of 'D'.
|
filenames = [filename for filename in filenames
|
if os.path.exists(filename)]
|
if filenames:
|
rc_flag = '--rcfile=%s' % (rcfile,)
|
pylint_shell_command = ['pylint', rc_flag] + filenames
|
status_code = subprocess.call(pylint_shell_command)
|
if status_code != 0:
|
error_message = ('Pylint failed on %s with '
|
'status %d.' % (description, status_code))
|
print >> sys.stderr, error_message
|
sys.exit(status_code)
|
else:
|
print 'Skipping %s, no files to lint.' % (description,)
|
|
|
def main(argv):
|
"""Script entry point. Lints both sets of files."""
|
diff_base = argv[1] if len(argv) > 1 else None
|
make_test_rc(PRODUCTION_RC, TEST_RC_ADDITIONS, TEST_RC)
|
library_files, non_library_files, using_restricted = get_python_files(
|
diff_base=diff_base)
|
try:
|
lint_fileset(library_files, PRODUCTION_RC, 'library code')
|
lint_fileset(non_library_files, TEST_RC, 'test and demo code')
|
except SystemExit:
|
if not using_restricted:
|
raise
|
|
message = 'Restricted lint failed, expanding to full fileset.'
|
print >> sys.stderr, message
|
all_files, _ = get_files_for_linting(allow_limited=False)
|
library_files, non_library_files, _ = get_python_files(
|
all_files=all_files)
|
lint_fileset(library_files, PRODUCTION_RC, 'library code')
|
lint_fileset(non_library_files, TEST_RC, 'test and demo code')
|
|
|
if __name__ == '__main__':
|
main(sys.argv)
|
