Commit 59606b88 authored by Marshall Greenblatt's avatar Marshall Greenblatt

Update tooling to use yapf for Python file formatting (issue #2171)

parent d4f06e38
# Copyright (c) 2017 The Chromium Embedded Framework Authors. All rights
# reserved. Use of this source code is governed by a BSD-style license that
# can be found in the LICENSE file
# Configuration settings for tools/fix_style.py
{
# Directories containing these path components will be ignored.
'ignore_directories': ['yapf'],
}
[style]
based_on_style = chromium
......@@ -5,13 +5,22 @@
import os, re, sys
from clang_util import clang_format
from file_util import *
from file_util import eval_file, get_files, read_file, write_file
from git_util import get_changed_files
from yapf_util import yapf_format
# Valid extensions for files we want to clang-format.
DEFAULT_LINT_WHITELIST_REGEX = r"(.*\.cpp|.*\.cc|.*\.h|.*\.mm)$"
# File extensions that can be formatted.
DEFAULT_LINT_WHITELIST_REGEX = r"(.*\.cpp|.*\.cc|.*\.h|.*\.java|.*\.mm|.*\.py)$"
DEFAULT_LINT_BLACKLIST_REGEX = r"$^"
# Directories containing these path components will be ignored.
IGNORE_DIRECTORIES = []
# Script directory.
script_dir = os.path.dirname(__file__)
root_dir = os.path.join(script_dir, os.pardir)
def msg(filename, status):
if sys.platform == 'win32':
# Use Unix path separator.
......@@ -27,14 +36,32 @@ def msg(filename, status):
print "%-60s %s" % (filename, status)
updatect = 0
def read_config():
style_cfg = os.path.join(root_dir, ".style.cfg")
if os.path.exists(style_cfg):
config = eval_file(style_cfg)
if 'ignore_directories' in config:
global IGNORE_DIRECTORIES
IGNORE_DIRECTORIES = config['ignore_directories']
def update_file(filename):
oldcontents = read_file(filename)
if len(oldcontents) == 0:
msg(filename, "empty")
return;
return
if os.path.splitext(filename)[1] == ".py":
# Format Python files using YAPF.
newcontents = yapf_format(filename, oldcontents)
else:
# Format C/C++/ObjC/Java files using clang-format.
newcontents = clang_format(filename, oldcontents)
newcontents = clang_format(filename, oldcontents)
if newcontents is None:
raise Exception("Failed to process %s" % filename)
......@@ -47,7 +74,8 @@ def update_file(filename):
msg(filename, "ok")
return
def fix_style(filenames, white_list = None, black_list = None):
def fix_style(filenames, white_list=None, black_list=None):
""" Execute clang-format with the specified arguments. """
if not white_list:
white_list = DEFAULT_LINT_WHITELIST_REGEX
......@@ -57,6 +85,16 @@ def fix_style(filenames, white_list = None, black_list = None):
black_regex = re.compile(black_list)
for filename in filenames:
# Ignore files from specific directories.
ignore = False
for dir_part in filename.split(os.sep):
if dir_part in IGNORE_DIRECTORIES:
msg(filename, "ignored")
ignore = True
break
if ignore:
continue
if filename.find('*') > 0:
# Expand wildcards.
filenames.extend(get_files(filename))
......@@ -83,6 +121,7 @@ def fix_style(filenames, white_list = None, black_list = None):
else:
msg(filename, "skipped")
if __name__ == "__main__":
if len(sys.argv) == 1:
print "Usage: %s [file-path|git-hash|unstaged|staged] ..." % sys.argv[0]
......@@ -96,6 +135,9 @@ if __name__ == "__main__":
print " staged\t\tProcess all staged files in the Git repo."
sys.exit(1)
# Read the configuration file.
read_config()
# Process anything passed on the command-line.
fix_style(sys.argv[1:])
print 'Done - Wrote %d files.' % updatect
This diff is collapsed.
Name: yapf
Short Name: yapf
URL: https://github.com/google/yapf
Date: 28 May 2017
Version: 0.16.2
Revision: 9f168a12
License: Apache 2.0
License File: LICENSE
Description:
A formatter for Python files.
Local Modifications:
None
# Copyright 2015-2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import yapf
yapf.run_main()
This diff is collapsed.
# Copyright 2015-2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright 2015-2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Calculate the number of blank lines between top-level entities.
Calculates how many blank lines we need between classes, functions, and other
entities at the same level.
CalculateBlankLines(): the main function exported by this module.
Annotations:
newlines: The number of newlines required before the node.
"""
from lib2to3 import pytree
from yapf.yapflib import py3compat
from yapf.yapflib import pytree_utils
from yapf.yapflib import pytree_visitor
_NO_BLANK_LINES = 1
_ONE_BLANK_LINE = 2
_TWO_BLANK_LINES = 3
_PYTHON_STATEMENTS = frozenset({
'small_stmt', 'expr_stmt', 'print_stmt', 'del_stmt', 'pass_stmt',
'break_stmt', 'continue_stmt', 'return_stmt', 'raise_stmt', 'yield_stmt',
'import_stmt', 'global_stmt', 'exec_stmt', 'assert_stmt', 'if_stmt',
'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt', 'nonlocal_stmt',
'async_stmt', 'simple_stmt'
})
def CalculateBlankLines(tree):
"""Run the blank line calculator visitor over the tree.
This modifies the tree in place.
Arguments:
tree: the top-level pytree node to annotate with subtypes.
"""
blank_line_calculator = _BlankLineCalculator()
blank_line_calculator.Visit(tree)
class _BlankLineCalculator(pytree_visitor.PyTreeVisitor):
"""_BlankLineCalculator - see file-level docstring for a description."""
def __init__(self):
self.class_level = 0
self.function_level = 0
self.last_comment_lineno = 0
self.last_was_decorator = False
self.last_was_class_or_function = False
def Visit_simple_stmt(self, node): # pylint: disable=invalid-name
self.DefaultNodeVisit(node)
if pytree_utils.NodeName(node.children[0]) == 'COMMENT':
self.last_comment_lineno = node.children[0].lineno
def Visit_decorator(self, node): # pylint: disable=invalid-name
if (self.last_comment_lineno and
self.last_comment_lineno == node.children[0].lineno - 1):
self._SetNumNewlines(node.children[0], _NO_BLANK_LINES)
else:
self._SetNumNewlines(node.children[0], self._GetNumNewlines(node))
for child in node.children:
self.Visit(child)
self.last_was_decorator = True
def Visit_classdef(self, node): # pylint: disable=invalid-name
self.last_was_class_or_function = False
index = self._SetBlankLinesBetweenCommentAndClassFunc(node)
self.last_was_decorator = False
self.class_level += 1
for child in node.children[index:]:
self.Visit(child)
self.class_level -= 1
self.last_was_class_or_function = True
def Visit_funcdef(self, node): # pylint: disable=invalid-name
self.last_was_class_or_function = False
index = self._SetBlankLinesBetweenCommentAndClassFunc(node)
if _AsyncFunction(node):
index = self._SetBlankLinesBetweenCommentAndClassFunc(
node.prev_sibling.parent)
self._SetNumNewlines(node.children[0], None)
else:
index = self._SetBlankLinesBetweenCommentAndClassFunc(node)
self.last_was_decorator = False
self.function_level += 1
for child in node.children[index:]:
self.Visit(child)
self.function_level -= 1
self.last_was_class_or_function = True
def DefaultNodeVisit(self, node):
"""Override the default visitor for Node.
This will set the blank lines required if the last entity was a class or
function.
Arguments:
node: (pytree.Node) The node to visit.
"""
if self.last_was_class_or_function:
if pytree_utils.NodeName(node) in _PYTHON_STATEMENTS:
leaf = _GetFirstChildLeaf(node)
self._SetNumNewlines(leaf, self._GetNumNewlines(leaf))
self.last_was_class_or_function = False
super(_BlankLineCalculator, self).DefaultNodeVisit(node)
def _SetBlankLinesBetweenCommentAndClassFunc(self, node):
"""Set the number of blanks between a comment and class or func definition.
Class and function definitions have leading comments as children of the
classdef and functdef nodes.
Arguments:
node: (pytree.Node) The classdef or funcdef node.
Returns:
The index of the first child past the comment nodes.
"""
index = 0
while pytree_utils.IsCommentStatement(node.children[index]):
# Standalone comments are wrapped in a simple_stmt node with the comment
# node as its only child.
self.Visit(node.children[index].children[0])
if not self.last_was_decorator:
self._SetNumNewlines(node.children[index].children[0], _ONE_BLANK_LINE)
index += 1
if (index and node.children[index].lineno -
1 == node.children[index - 1].children[0].lineno):
self._SetNumNewlines(node.children[index], _NO_BLANK_LINES)
else:
if self.last_comment_lineno + 1 == node.children[index].lineno:
num_newlines = _NO_BLANK_LINES
else:
num_newlines = self._GetNumNewlines(node)
self._SetNumNewlines(node.children[index], num_newlines)
return index
def _GetNumNewlines(self, node):
if self.last_was_decorator:
return _NO_BLANK_LINES
elif self._IsTopLevel(node):
return _TWO_BLANK_LINES
return _ONE_BLANK_LINE
def _SetNumNewlines(self, node, num_newlines):
pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.NEWLINES,
num_newlines)
def _IsTopLevel(self, node):
return (not (self.class_level or self.function_level) and
_StartsInZerothColumn(node))
def _StartsInZerothColumn(node):
return (_GetFirstChildLeaf(node).column == 0 or
(_AsyncFunction(node) and node.prev_sibling.column == 0))
def _AsyncFunction(node):
return (py3compat.PY3 and node.prev_sibling and
pytree_utils.NodeName(node.prev_sibling) == 'ASYNC')
def _GetFirstChildLeaf(node):
if isinstance(node, pytree.Leaf):
return node
return _GetFirstChildLeaf(node.children[0])
This diff is collapsed.
# Copyright 2015-2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Insert "continuation" nodes into lib2to3 tree.
The "backslash-newline" continuation marker is shoved into the node's prefix.
Pull them out and make it into nodes of their own.
SpliceContinuations(): the main funciton exported by this module.
"""
from lib2to3 import pytree
from yapf.yapflib import format_token
def SpliceContinuations(tree):
"""Given a pytree, splice the continuation marker into nodes.
Arguments:
tree: (pytree.Node) The tree to work on. The tree is modified by this
function.
"""
def RecSplicer(node):
"""Inserts a continuation marker into the node."""
if isinstance(node, pytree.Leaf):
if node.prefix.lstrip().startswith('\\\n'):
new_lineno = node.lineno - node.prefix.count('\n')
return pytree.Leaf(
type=format_token.CONTINUATION,
value=node.prefix,
context=('', (new_lineno, 0)))
return None
num_inserted = 0
for index, child in enumerate(node.children[:]):
continuation_node = RecSplicer(child)
if continuation_node:
node.children.insert(index + num_inserted, continuation_node)
num_inserted += 1
RecSplicer(tree)
# Copyright 2015-2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""YAPF error object."""
class YapfError(Exception):
"""Parent class for user errors or input errors.
Exceptions of this type are handled by the command line tool
and result in clear error messages, as opposed to backtraces.
"""
pass
# Copyright 2015-2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Interface to file resources.
This module provides functions for interfacing with files: opening, writing, and
querying.
"""
import fnmatch
import os
import re
from lib2to3.pgen2 import tokenize
from yapf.yapflib import errors
from yapf.yapflib import py3compat
from yapf.yapflib import style
CR = '\r'
LF = '\n'
CRLF = '\r\n'
def GetDefaultStyleForDir(dirname):
"""Return default style name for a given directory.
Looks for .style.yapf or setup.cfg in the parent directories.
Arguments:
dirname: (unicode) The name of the directory.
Returns:
The filename if found, otherwise return the global default (pep8).
"""
dirname = os.path.abspath(dirname)
while True:
# See if we have a .style.yapf file.
style_file = os.path.join(dirname, style.LOCAL_STYLE)
if os.path.exists(style_file):
return style_file
# See if we have a setup.cfg file with a '[yapf]' section.
config_file = os.path.join(dirname, style.SETUP_CONFIG)
if os.path.exists(config_file):
with open(config_file) as fd:
config = py3compat.ConfigParser()
config.read_file(fd)
if config.has_section('yapf'):
return config_file
dirname = os.path.dirname(dirname)
if (not dirname or not os.path.basename(dirname) or
dirname == os.path.abspath(os.path.sep)):
break
global_file = os.path.expanduser(style.GLOBAL_STYLE)
if os.path.exists(global_file):
return global_file
return style.DEFAULT_STYLE
def GetCommandLineFiles(command_line_file_list, recursive, exclude):
"""Return the list of files specified on the command line."""
return _FindPythonFiles(command_line_file_list, recursive, exclude)
def WriteReformattedCode(filename,
reformatted_code,
in_place=False,
encoding=''):
"""Emit the reformatted code.
Write the reformatted code into the file, if in_place is True. Otherwise,
write to stdout.
Arguments:
filename: (unicode) The name of the unformatted file.
reformatted_code: (unicode) The reformatted code.
in_place: (bool) If True, then write the reformatted code to the file.
encoding: (unicode) The encoding of the file.
"""
if in_place:
with py3compat.open_with_encoding(
filename, mode='w', encoding=encoding, newline='') as fd:
fd.write(reformatted_code)
else:
py3compat.EncodeAndWriteToStdout(reformatted_code)
def LineEnding(lines):
"""Retrieve the line ending of the original source."""
endings = {CRLF: 0, CR: 0, LF: 0}
for line in lines:
if line.endswith(CRLF):
endings[CRLF] += 1
elif line.endswith(CR):
endings[CR] += 1
elif line.endswith(LF):
endings[LF] += 1
return (sorted(endings, key=endings.get, reverse=True) or [LF])[0]
def _FindPythonFiles(filenames, recursive, exclude):
"""Find all Python files."""
python_files = []
for filename in filenames:
if os.path.isdir(filename):
if recursive:
# TODO(morbo): Look into a version of os.walk that can handle recursion.
python_files.extend(
os.path.join(dirpath, f)
for dirpath, _, filelist in os.walk(filename) for f in filelist
if IsPythonFile(os.path.join(dirpath, f)))
else:
raise errors.YapfError(
"directory specified without '--recursive' flag: %s" % filename)
elif os.path.isfile(filename):
python_files.append(filename)
if exclude:
return [
f for f in python_files
if not any(fnmatch.fnmatch(f, p) for p in exclude)
]
return python_files
def IsPythonFile(filename):
"""Return True if filename is a Python file."""
if os.path.splitext(filename)[1] == '.py':
return True
try:
with open(filename, 'rb') as fd:
encoding = tokenize.detect_encoding(fd.readline)[0]
# Check for correctness of encoding.
with py3compat.open_with_encoding(
filename, mode='r', encoding=encoding) as fd:
fd.read()
except UnicodeDecodeError:
encoding = 'latin-1'
except (IOError, SyntaxError):
# If we fail to detect encoding (or the encoding cookie is incorrect - which
# will make detect_encoding raise SyntaxError), assume it's not a Python
# file.
return False
try:
with py3compat.open_with_encoding(
filename, mode='r', encoding=encoding) as fd:
first_line = fd.readlines()[0]
except (IOError, IndexError):
return False
return re.match(r'^#!.*\bpython[23]?\b', first_line)
This diff is collapsed.
# Copyright 2015-2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Pytree nodes with extra formatting information.
This is a thin wrapper around a pytree.Leaf node.
"""
import keyword
import re
from lib2to3.pgen2 import token
from yapf.yapflib import py3compat
from yapf.yapflib import pytree_utils
from yapf.yapflib import style
CONTINUATION = token.N_TOKENS
token.N_TOKENS += 1
class Subtype(object):