Bump pycodestyle from 2.6.0 to 2.7.0

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ exclude: "^cloudformation/|^docs/|^log_processing/kibana/|^wiki/"
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v3.4.0
     hooks:
       - id: check-added-large-files
       - id: check-symlinks
@@ -17,7 +17,7 @@ repos:
     hooks:
       - id: yamllint
   - repo: https://github.com/aws-cloudformation/cfn-python-lint
-    rev: v0.45.0
+    rev: v0.47.2
     hooks:
       - id: cfn-python-lint
         files: cloudformation/.*\.(json|yml|yaml)$
@@ -26,16 +26,16 @@ repos:
     hooks:
       - id: blocklint
   - repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.7.1
+    rev: v1.8.0
     hooks:
       - id: python-check-blanket-noqa
       - id: python-use-type-annotations
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.10.0
+    rev: v2.11.0
     hooks:
       - id: pyupgrade
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.7.0
+    rev: 5.8.0
     hooks:
       - id: isort
   - repo: https://github.com/psf/black

README.md

@@ -335,9 +335,8 @@ Please format your code using:
 * [isort](https://github.com/timothycrosley/isort)
 
 Also, we want to run code through:
-* [pycodestyle](https://www.python.org/dev/peps/pep-0008/) (see [local config](setup.cfg))
 * [flake8](https://flake8.pycqa.org/en/latest/)
-* [type checker](http://mypy-lang.org/)
+* [mypy](http://mypy-lang.org/)
 
 #### Adding a pre-commit hook
 

etc/arthur_completion.sh

@@ -64,9 +64,6 @@ _arthur_completion()
         "auto_design"|"bootstrap_transformations")
             COMPREPLY=( $(compgen -W "CTAS VIEW" -- "$cur") )
             ;;
-        "selftest")
-            COMPREPLY=( $(compgen -W "all doctest type-check" -- "$cur") )
-            ;;
         *)
             case "$cur" in
                 "@"*)

etc/future_setup.cfg

@@ -28,9 +28,3 @@ warn_redundant_casts = True
 # warn_unused_ignores = True
 # warn_return_any = True
 warn_unused_configs = True
-
-[pycodestyle]
-# Settings are compatible with black, see https://github.com/psf/black/issues/354
-ignore = E203,W503
-# This is the current setting for this project. We may want to drop this to 100.
-max-line-length = 100

python/etl/commands.py

@@ -1532,21 +1532,9 @@ def add_arguments(self, parser):
         # For self-tests, dial logging back to (almost) nothing so that logging in console
         # doesn't mix with test output.
         parser.set_defaults(log_level="CRITICAL")
-        parser.add_argument(
-            "test_family",
-            help="select which family of tests to run",
-            nargs="?",
-            choices=["pep8", "doctest", "type-check", "all"],
-            default="all",
-        )
 
     def callback(self, args, config):
-        if args.test_family in ("pep8", "all"):
-            etl.selftest.run_pep8("etl", args.log_level)
-        if args.test_family in ("doctest", "all"):
-            etl.selftest.run_doctest("etl", args.log_level)
-        if args.test_family in ("type-check", "all"):
-            etl.selftest.run_type_checker()
+        etl.selftest.run_doctest("etl", args.log_level)
 
 
 if __name__ == "__main__":

python/etl/selftest.py

@@ -1,27 +1,20 @@
-"""
-Provide "self-test" feature of Arthur.
-
-We can run
-* all the doctests from the source code
-* static type checking against source code
-"""
+"""Provide "self-test" feature of Arthur, which executes all doctests."""
 
 import doctest
 import logging
-import os.path
 import sys
 import unittest
 from typing import Optional
 
-import pycodestyle
-
 # Skip etl.commands to avoid circular dependency
 import etl.config
+import etl.config.env
 import etl.data_warehouse
 import etl.db
 import etl.design
 import etl.design.bootstrap
 import etl.design.load
+import etl.dialect.redshift
 import etl.errors
 import etl.explain
 import etl.extract
@@ -38,26 +31,13 @@
 import etl.text
 import etl.timer
 import etl.unload
+import etl.util
 import etl.validate
 
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.NullHandler())
 
 
-def run_pep8(module_: Optional[str] = None, log_level: str = "INFO") -> None:
-    print("Running PEP8 check...", flush=True)
-    if module_ is None:
-        module_ = __name__
-    quiet = log_level not in ("DEBUG", "INFO")
-    style_guide = pycodestyle.StyleGuide(parse_argv=False, config_file="setup.cfg", quiet=quiet)
-    report = style_guide.check_files(["python"])
-    if report.total_errors > 0:
-        raise etl.errors.SelfTestError(
-            "Unsuccessful (warning=%d, errors=%d)" % (report.get_count("W"), report.get_count("E"))
-        )
-    print("OK")
-
-
 def load_tests(loader, tests, pattern):
     """
     Add tests within doctests so that the unittest runner picks them up.
@@ -90,40 +70,9 @@ def run_doctest(module_: Optional[str] = None, log_level: str = "INFO") -> None:
         )
 
 
-def run_type_checker() -> None:
-    print("Running type checker...", flush=True)
-    if not os.path.isdir("python"):
-        raise etl.errors.ETLRuntimeError("Cannot find source directory: 'python'")
-
-    # We wait with this import so that commands can be invoked in an environment where mypy is
-    # not installed.
-    import mypy.api
-
-    normal_report, error_report, exit_status = mypy.api.run(
-        ["python", "--strict-optional", "--ignore-missing-imports"]  # Should match setup.py's package_dir
-    )
-    if normal_report:
-        print("Type checking report:\n")
-        print(normal_report)
-    if error_report:
-        print("Error report:\n")
-        print(error_report)
-    if exit_status != 0:
-        raise etl.errors.SelfTestError("Unsuccessful (exit status = %d)" % exit_status)
-    print("OK")
-
-
-def run_tests() -> None:
+if __name__ == "__main__":
     try:
-        run_pep8()
         run_doctest()
-        run_type_checker()
     except Exception as exc:
         print(exc)
         sys.exit(1)
-
-
-if __name__ == "__main__":
-    # Running "python3 -m etl.selftest" will only run doc tests.
-    # Use "run_tests.py" to run all of the tests.
-    run_doctest()

requirements.txt

@@ -5,7 +5,6 @@ jmespath==0.10.0
 jsonschema==3.2.0
 pgpasslib==1.1.0
 psycopg2-binary==2.8.6
-pycodestyle==2.6.0
 PyYAML==5.4.1
 simplejson==3.17.2
 tabulate==0.8.9

setup.cfg

@@ -32,9 +32,3 @@ warn_redundant_casts = True
 # warn_unused_ignores = True
 # warn_return_any = True
 warn_unused_configs = True
-
-[pycodestyle]
-# Settings are compatible with black, see https://github.com/psf/black/issues/354
-ignore = E203,W503
-# This is the current setting for this project. We may want to drop this to 100.
-max-line-length = 120