feat(templates): Use Ruff to lint projects generated with Cookiecutte…

…r templates (#1648) * feat(templates): Use Ruff to lint Cookiecutter templates * Address import order issue in target template * Add end-to-end test for SQL target --------- Co-authored-by: Will Da Silva <[email protected]>
meltano · May 2, 2023 · dc307b6 · dc307b6
1 parent 3609320
commit dc307b6
Show file tree

Hide file tree

Showing 25 changed files with 226 additions and 269 deletions.
diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,4 +1,5 @@
 pip==23.1.2
 poetry==1.4.2
+pre-commit==3.2.2
 nox==2023.4.22
 nox-poetry==1.0.2
diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml
@@ -30,13 +30,18 @@ jobs:
     - name: Check out the repository
       uses: actions/[email protected]
 
-    - name: Install Poetry & Tox
+    - name: Upgrade pip
       env:
         PIP_CONSTRAINT: .github/workflows/constraints.txt
+      run: |
+        pip install pip
+        pip --version
+
+    - name: Install Poetry
       run: |
         pipx install poetry
         poetry --version
-        pipx install tox
+
     - name: Setup Python ${{ matrix.python-version }}
       uses: actions/[email protected]
       with:
@@ -45,12 +50,10 @@ jobs:
         cache: 'pip'
         cache-dependency-path: 'poetry.lock'
 
-    - name: Upgrade pip
-      env:
-        PIP_CONSTRAINT: .github/workflows/constraints.txt
+    - name: Install pre-commit
       run: |
-        pip install pip
-        pip --version
+        pipx install pre-commit
+        pre-commit --version
 
     - name: Install Nox
       env:
@@ -63,6 +66,7 @@ jobs:
     - name: Run Nox
       run: |
         nox --python=${{ matrix.python-version }} --session=test_cookiecutter
+
     - name: Upload build artifacts
       if: always()
       uses: actions/upload-artifact@v3

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,7 +16,8 @@ repos:
   - id: check-yaml
     exclude: |
         (?x)^(
-            cookiecutter/.*/meltano.yml
+            cookiecutter/.*/meltano.yml|
+            cookiecutter/.*/.pre-commit-config.yaml
         )$
   - id: end-of-file-fixer
     exclude: |

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml
@@ -0,0 +1,36 @@
+ci:
+  autofix_prs: true
+  autoupdate_schedule: weekly
+  autoupdate_commit_msg: 'chore: pre-commit autoupdate'
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: check-json
+  - id: check-toml
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+
+- repo: https://github.com/charliermarsh/ruff-pre-commit
+  rev: v0.0.263
+  hooks:
+  - id: ruff
+    args: [--fix, --exit-non-zero-on-fix]
+
+- repo: https://github.com/psf/black
+  rev: 23.3.0
+  hooks:
+  - id: black
+
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.2.0
+  hooks:
+  - id: mypy
+    additional_dependencies:
+    {%- if cookiecutter.stream_type == "SQL" %}
+    - sqlalchemy-stubs
+    {%- else %}
+    - types-requests
+    {%- endif %}
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -32,35 +32,37 @@ cached-property = "^1" # Remove after Python 3.7 support is dropped
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.1"
-flake8 = "^5.0.4"
-darglint = "^1.8.1"
-black = "^23.1.0"
-pyupgrade = "^3.3.1"
-mypy = "^1.0.0"
-isort = "^5.11.5"
 singer-sdk = { version="^0.26.0", extras = ["testing"] }
-{%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
-types-requests = "^2.28.11.12"
-{%- endif %}
-{%- if cookiecutter.stream_type == 'SQL' %}
-sqlalchemy-stubs = "^0.4"
-{%- endif %}
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]
 
-[tool.isort]
-profile = "black"
-multi_line_output = 3 # Vertical Hanging Indent
-src_paths = "{{cookiecutter.library_name}}"
-
 [tool.mypy]
 python_version = "3.9"
 warn_unused_configs = true
 {%- if cookiecutter.stream_type == 'SQL' %}
 plugins = "sqlmypy"
 {%- endif %}
 
+[tool.ruff]
+ignore = [
+    "ANN101",  # missing-type-self
+    "ANN102",  # missing-type-cls
+]
+select = ["ALL"]
+src = ["{{cookiecutter.library_name}}"]
+target-version = "py37"
+
+
+[tool.ruff.flake8-annotations]
+allow-star-arg-any = true
+
+[tool.ruff.isort]
+known-first-party = ["{{cookiecutter.library_name}}"]
+
+[tool.ruff.pydocstyle]
+convention = "google"
+
 [build-system]
 requires = ["poetry-core>=1.0.8"]
 build-backend = "poetry.core.masonry.api"

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/{{ 'test' }}_core.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/{{ 'test' }}_core.py
@@ -6,17 +6,16 @@
 
 from {{ cookiecutter.library_name }}.tap import Tap{{ cookiecutter.source_name }}
 
-
 SAMPLE_CONFIG = {
-    "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
+    "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"),
     # TODO: Initialize minimal tap config
 }
 
 
 # Run standard built-in tap tests from the SDK:
 TestTap{{ cookiecutter.source_name }} = get_tap_test_class(
     tap_class=Tap{{ cookiecutter.source_name }},
-    config=SAMPLE_CONFIG
+    config=SAMPLE_CONFIG,
 )
 
 

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini
@@ -1,18 +1,14 @@
 # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy
 
 [tox]
-envlist = py39
+envlist = py37, py38, py39, py310, py311
 isolated_build = true
 
 [testenv]
 allowlist_externals = poetry
 commands =
     poetry install -v
     poetry run pytest
-    poetry run black --check {{cookiecutter.library_name}}/
-    poetry run flake8 {{cookiecutter.library_name}}
-    poetry run pydocstyle {{cookiecutter.library_name}}
-    poetry run mypy {{cookiecutter.library_name}} --exclude='{{cookiecutter.library_name}}/tests'
 
 [testenv:pytest]
 # Run the python tests.
@@ -21,31 +17,3 @@ envlist = py37, py38, py39, py310, py311
 commands =
     poetry install -v
     poetry run pytest
-
-[testenv:format]
-# Attempt to auto-resolve lint errors before they are raised.
-# To execute, run `tox -e format`
-commands =
-    poetry install -v
-    poetry run black {{cookiecutter.library_name}}/
-    poetry run isort {{cookiecutter.library_name}}
-
-[testenv:lint]
-# Raise an error if lint and style standards are not met.
-# To execute, run `tox -e lint`
-commands =
-    poetry install -v
-    poetry run black --check --diff {{cookiecutter.library_name}}/
-    poetry run isort --check {{cookiecutter.library_name}}
-    poetry run flake8 {{cookiecutter.library_name}}
-    # refer to mypy.ini for specific settings
-    poetry run mypy . --exclude='tests'
-
-[flake8]
-docstring-convention = google
-ignore = W503
-max-line-length = 88
-max-complexity = 10
-
-[pydocstyle]
-ignore = D105,D203,D213
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py
@@ -7,7 +7,7 @@
 
 {%- if cookiecutter.stream_type == "SQL" %}
 
-# from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream
+from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream
 {%- else %}
 
 # TODO: Import your custom stream types here:
@@ -20,6 +20,10 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type ==
 
     name = "{{ cookiecutter.tap_id }}"
 
+    {%- if cookiecutter.stream_type == "SQL" %}
+    default_stream_class = {{ cookiecutter.source_name }}Stream
+    {%- endif %}
+
     # TODO: Update this section with the actual config values you expect:
     config_jsonschema = th.PropertiesList(
         th.Property(

diff --git a/...okiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} b/...okiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%}
@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from typing import Iterable
 
-import requests
+import requests  # noqa: TCH002
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
 
 {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
@@ -16,20 +16,14 @@ from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}A
 class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream):
     """{{ cookiecutter.source_name }} stream class."""
 
-    # TODO: Set the API's base URL here:
     @property
     def url_base(self) -> str:
-        """Return the API URL root, configurable via tap settings.
-
-        Returns:
-            The base URL for all requests.
-        """
-        return self.config["api_url"]
-
-    # Alternatively, use a static string for url_base:
-    # url_base = "https://api.mysample.com"
+        """Return the API URL root, configurable via tap settings."""
+        # TODO: hardcode a value here, or retrieve it from self.config
+        return "https://api.mysample.com"
 
 {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
+
     @property
     def authenticator(self) -> {{ cookiecutter.source_name }}Authenticator:
         """Return a new authenticator object.
@@ -68,10 +62,13 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream)
         """
         # TODO: Parse response body and return a set of records.
         resp_json = response.json()
-        for record in resp_json.get("<TODO>"):
-            yield record
+        yield from resp_json.get("<TODO>")
 
-    def post_process(self, row: dict, context: dict | None = None) -> dict | None:
+    def post_process(
+        self,
+        row: dict,
+        context: dict | None = None,  # noqa: ARG002
+    ) -> dict | None:
         """As needed, append or transform raw data to match expected structure.
 
         Args:

diff --git a/...cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} b/...cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%}
@@ -10,7 +10,10 @@ from singer_sdk.streams import Stream
 class {{ cookiecutter.source_name }}Stream(Stream):
     """Stream class for {{ cookiecutter.source_name }} streams."""
 
-    def get_records(self, context: dict | None) -> Iterable[dict]:
+    def get_records(
+        self,
+        context: dict | None,  # noqa: ARG002
+    ) -> Iterable[dict]:
         """Return a generator of record-type dictionary objects.
 
         The optional `context` argument is used to identify a specific slice of the
@@ -24,7 +27,8 @@ class {{ cookiecutter.source_name }}Stream(Stream):
             NotImplementedError: If the implementation is TODO
         """
         # TODO: Write logic to extract data from the upstream source.
-        # records = mysource.getall()
+        # records = mysource.getall()  # noqa: ERA001
         # for record in records:
-        #     yield record.to_dict()
-        raise NotImplementedError("The method is not yet implemented (TODO)")
+        #     yield record.to_dict()  # noqa: ERA001
+        errmsg = "The method is not yet implemented (TODO)"
+        raise NotImplementedError(errmsg)