Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use pathlib #226

Merged
merged 36 commits into from
Oct 6, 2020
Merged
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
4327b51
change core._match()
ju-sh Aug 20, 2020
548d306
in between
ju-sh Aug 20, 2020
be56ce6
add some type annotations
ju-sh Aug 20, 2020
9bdd333
more type annotations
ju-sh Aug 20, 2020
d8203ff
add update() method to LoggingSet
ju-sh Aug 20, 2020
06929f3
very minor change
ju-sh Aug 20, 2020
9385471
change get_modules()
ju-sh Aug 20, 2020
7cd8f02
in between again
ju-sh Aug 21, 2020
e867f41
finish scavenge()
ju-sh Aug 22, 2020
45e1bd2
modify some tests
ju-sh Aug 23, 2020
b9e6f8f
finish test_encoding.py
ju-sh Aug 23, 2020
11cbd43
use pathlib in test init file
ju-sh Aug 27, 2020
5944dcc
fix all non-script tests
ju-sh Aug 29, 2020
46d8df7
fix for more tests
ju-sh Aug 29, 2020
536db5b
fix for script tests
ju-sh Aug 29, 2020
94d2892
fix mypy
ju-sh Aug 29, 2020
3c34c3b
purge type annotations
ju-sh Sep 8, 2020
3ea3dc9
remove explicit update method of LoggingSet
ju-sh Sep 8, 2020
46e7493
sync with master
ju-sh Sep 8, 2020
69c41e1
remove type hints
ju-sh Sep 11, 2020
ab23dc2
Revert "sync with master"
ju-sh Sep 12, 2020
907f658
resolve conflicts
ju-sh Sep 12, 2020
ff86348
Merge branch 'master' into ju-sh-test_trial
jendrikseipp Sep 12, 2020
6523807
use relative_to() in format_path()
ju-sh Sep 12, 2020
474e4a9
convert args to pathlib inside function
ju-sh Sep 14, 2020
480b51e
fix windows error
ju-sh Oct 6, 2020
f5ac8fb
fix style and convert whitelist paths to str
ju-sh Oct 6, 2020
d870596
Simplify get_modules().
jendrikseipp Oct 6, 2020
4f1c70a
Polish code a bit.
jendrikseipp Oct 6, 2020
ae66d58
Use _match() function.
jendrikseipp Oct 6, 2020
7ea0faf
Remove some patterns.
jendrikseipp Oct 6, 2020
5683de6
Support Windows in patterns.
jendrikseipp Oct 6, 2020
b164619
Add changelog entry.
jendrikseipp Oct 6, 2020
f34dd55
Add debug output.
jendrikseipp Oct 6, 2020
307ea50
Change debug output.
jendrikseipp Oct 6, 2020
eb761cd
Remove debug output.
jendrikseipp Oct 6, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions tests/__init__.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
import glob
import os.path
import pathlib
import subprocess
import sys

import pytest

from vulture import core

DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(DIR)
WHITELISTS = glob.glob(os.path.join(REPO, "vulture", "whitelists", "*.py"))
REPO = pathlib.Path(__file__).resolve().parents[1]
WHITELISTS = [
str(path) for path in (REPO / "vulture" / "whitelists").glob("*.py")
]


def call_vulture(args, **kwargs):
Expand Down
15 changes: 7 additions & 8 deletions tests/test_encoding.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,21 @@ def test_encoding2(v):
assert not v.found_dead_code_or_error


def test_non_utf8_encoding(v, tmpdir):
def test_non_utf8_encoding(v, tmp_path):
jendrikseipp marked this conversation as resolved.
Show resolved Hide resolved
code = ""
name = "non_utf8"
non_utf_8_file = str(tmpdir.mkdir(name).join(name + ".py"))
non_utf_8_file = tmp_path / (name + ".py")
with open(non_utf_8_file, mode="wb") as f:
f.write(codecs.BOM_UTF16_LE)
f.write(code.encode("utf_16_le"))
v.scavenge([f.name])
v.scavenge([non_utf_8_file])
assert v.found_dead_code_or_error


def test_utf8_with_bom(v, tmpdir):
def test_utf8_with_bom(v, tmp_path):
name = "utf8_bom"
filename = str(tmpdir.mkdir(name).join(name + ".py"))
filepath = tmp_path / (name + ".py")
# utf8_sig prepends the BOM to the file.
with open(filename, mode="w", encoding="utf-8-sig") as f:
f.write("")
v.scavenge([f.name])
filepath.write_text("", encoding="utf-8-sig")
v.scavenge([filepath])
assert not v.found_dead_code_or_error
4 changes: 3 additions & 1 deletion tests/test_imports.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import pathlib

from . import check, v

assert v # Silence pyflakes.
Expand Down Expand Up @@ -218,7 +220,7 @@ def test_ignore_init_py_files(v):

unused_var = 'monty'
""",
filename="nested/project/__init__.py",
filename=pathlib.Path("nested/project/__init__.py"),
)
check(v.unused_imports, [])
check(v.unused_vars, ["unused_var"])
4 changes: 3 additions & 1 deletion tests/test_report.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import pathlib

import pytest

from . import v
Expand Down Expand Up @@ -29,7 +31,7 @@ def myfunc():
@pytest.fixture
def check_report(v, capsys):
def test_report(code, expected, make_whitelist=False):
filename = "foo.py"
filename = pathlib.Path("foo.py")
v.scan(code, filename=filename)
capsys.readouterr()
v.report(make_whitelist=make_whitelist)
Expand Down
7 changes: 4 additions & 3 deletions tests/test_scavenging.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import sys
import pathlib

from . import check, v

Expand Down Expand Up @@ -489,7 +490,7 @@ class BasicTestCase:
class OtherClass:
pass
""",
filename="test_function_names.py",
filename=pathlib.Path("test_function_names.py"),
)
check(v.defined_attrs, [])
check(v.defined_classes, ["OtherClass"])
Expand All @@ -510,7 +511,7 @@ async def test_func():
async def other_func():
pass
""",
filename="test_function_names.py",
filename=pathlib.Path("test_function_names.py"),
)
check(v.defined_funcs, ["other_func"])
check(v.unused_funcs, ["other_func"])
Expand All @@ -525,7 +526,7 @@ async def test_func():
async def other_func():
pass
""",
filename="function_names.py",
filename=pathlib.Path("function_names.py"),
)
check(v.defined_funcs, ["test_func", "other_func"])
check(v.unused_funcs, ["other_func", "test_func"])
Expand Down
12 changes: 6 additions & 6 deletions vulture/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
command-line arguments or the pyproject.toml file.
"""
import argparse
import pathlib
import sys
from os.path import abspath, exists

import toml

Expand Down Expand Up @@ -183,11 +183,11 @@ def make_config(argv=None, tomlfile=None):
config = _parse_toml(tomlfile)
detected_toml_path = str(tomlfile)
else:
toml_path = abspath("pyproject.toml")
if exists(toml_path):
with open(toml_path) as config:
config = _parse_toml(config)
detected_toml_path = toml_path
toml_path = pathlib.Path("pyproject.toml").resolve()
if toml_path.is_file():
with open(toml_path) as fconfig:
config = _parse_toml(fconfig)
detected_toml_path = str(toml_path)
else:
config = {}

Expand Down
43 changes: 28 additions & 15 deletions vulture/core.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import ast
from fnmatch import fnmatch, fnmatchcase
import os.path
import pathlib
import pkgutil
import re
import string
Expand All @@ -11,7 +11,6 @@
from vulture import utils
from vulture.config import make_config

__version__ = "2.0"

DEFAULT_CONFIDENCE = 60

Expand Down Expand Up @@ -48,8 +47,17 @@ def _match(name, patterns, case=True):

def _is_test_file(filename):
return _match(
os.path.abspath(filename),
["*/test/*", "*/tests/*", "*/test*.py", "*/*_test.py", "*/*-test.py"],
filename.resolve(),
[
"*/test/*",
jendrikseipp marked this conversation as resolved.
Show resolved Hide resolved
"*/tests/*",
"*/test*.py",
"*/*_test.py",
"*/*-test.py",
"test*.py",
"*_test.py",
"*-test.py",
],
case=False,
)

Expand All @@ -64,7 +72,7 @@ def _ignore_import(filename, import_name):
Ignore imports from __init__.py files since they're commonly used to
collect objects from a package.
"""
return os.path.basename(filename) == "__init__.py" or import_name == "*"
return filename.name == "__init__.py" or import_name == "*"


def _ignore_function(filename, function_name):
Expand Down Expand Up @@ -170,7 +178,7 @@ class Vulture(ast.NodeVisitor):
"""Find dead code."""

def __init__(
self, verbose=False, ignore_names=None, ignore_decorators=None
self, verbose=False, ignore_names=None, ignore_decorators=None,
):
self.verbose = verbose

Expand All @@ -191,11 +199,12 @@ def get_list(typ):
self.ignore_names = ignore_names or []
self.ignore_decorators = ignore_decorators or []

self.filename = ""
self.filename = pathlib.Path()
self.code = []
self.found_dead_code_or_error = False

def scan(self, code, filename=""):
filename = pathlib.Path(filename)
self.code = code.splitlines()
self.noqa_lines = noqa.parse_noqa(self.code)
self.filename = filename
Expand All @@ -210,9 +219,11 @@ def handle_syntax_error(e):

try:
node = (
ast.parse(code, filename=self.filename, type_comments=True)
ast.parse(
code, filename=str(self.filename), type_comments=True
)
if sys.version_info >= (3, 8) # type_comments requires 3.8+
else ast.parse(code, filename=self.filename)
else ast.parse(code, filename=str(self.filename))
)
except SyntaxError as err:
handle_syntax_error(err)
Expand All @@ -232,14 +243,16 @@ def handle_syntax_error(e):

def scavenge(self, paths, exclude=None):
jendrikseipp marked this conversation as resolved.
Show resolved Hide resolved
def prepare_pattern(pattern):
if not any(char in pattern for char in ["*", "?", "["]):
if not any(char in pattern for char in "*?["):
pattern = f"*{pattern}*"
return pattern

exclude = [prepare_pattern(pattern) for pattern in (exclude or [])]

def exclude_file(name):
return any(fnmatch(name, pattern) for pattern in exclude)
return any(fnmatch(str(name), pattern) for pattern in exclude)

paths = [pathlib.Path(path) for path in paths]

for module in utils.get_modules(paths):
if exclude_file(module):
Expand All @@ -261,12 +274,12 @@ def exclude_file(name):

unique_imports = {item.name for item in self.defined_imports}
for import_name in unique_imports:
path = os.path.join("whitelists", import_name) + "_whitelist.py"
path = pathlib.Path("whitelists") / (import_name + "_whitelist.py")
if exclude_file(path):
self._log("Excluded whitelist:", path)
else:
try:
module_data = pkgutil.get_data("vulture", path)
module_data = pkgutil.get_data("vulture", str(path))
self._log("Included whitelist:", path)
except OSError:
# Most imported modules don't have a whitelist.
Expand All @@ -282,7 +295,7 @@ def get_unused_code(self, min_confidence=0, sort_by_size=False):
raise ValueError("min_confidence must be between 0 and 100.")

def by_name(item):
return (item.filename.lower(), item.first_lineno)
return (str(item.filename).lower(), item.first_lineno)

def by_size(item):
return (item.size,) + by_name(item)
Expand All @@ -307,7 +320,7 @@ def by_size(item):
)

def report(
self, min_confidence=0, sort_by_size=False, make_whitelist=False
self, min_confidence=0, sort_by_size=False, make_whitelist=False,
):
"""
Print ordered list of Item objects to stdout.
Expand Down
42 changes: 26 additions & 16 deletions vulture/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,11 @@ def condition_is_always_true(condition):


def format_path(path):
if not path:
try:
return path.relative_to(os.curdir)
except ValueError:
# Path is not below the current directory.
return path
ju-sh marked this conversation as resolved.
Show resolved Hide resolved
relpath = os.path.relpath(path)
return relpath if not relpath.startswith("..") else path


def get_decorator_name(decorator):
Expand All @@ -63,23 +64,32 @@ def get_decorator_name(decorator):
return "@" + ".".join(reversed(parts))


def get_modules(paths, toplevel=True):
def get_modules(paths):
jendrikseipp marked this conversation as resolved.
Show resolved Hide resolved
"""Take files from the command line even if they don't end with .py."""
modules = []
for path in paths:
path = os.path.abspath(path)
if toplevel and path.endswith(".pyc"):
sys.exit(f".pyc files are not supported: {path}")
if os.path.isfile(path) and (path.endswith(".py") or toplevel):
modules.append(path)
elif os.path.isdir(path):
subpaths = [
os.path.join(path, filename)
for filename in sorted(os.listdir(path))
]
modules.extend(get_modules(subpaths, toplevel=False))
elif toplevel:
path = path.resolve()

if not path.exists():
sys.exit(f"Error: {path} could not be found.")

if path.is_file():
top_paths = (_ for _ in [path])
else:
top_paths = path.glob("*")
jendrikseipp marked this conversation as resolved.
Show resolved Hide resolved

for top_path in top_paths:
if top_path.is_file():
if top_path.suffix == ".pyc":
sys.exit(f".pyc files are not supported: {top_path}")
else:
modules.append(top_path)
elif not top_path.is_dir():
sys.exit(f"Error: {top_path} could not be found.")
sub_paths = path.rglob("*.py")
for sub_path in sub_paths:
if sub_path.is_file():
modules.append(sub_path)
jendrikseipp marked this conversation as resolved.
Show resolved Hide resolved
return modules


Expand Down