feat: improve included tap and target tests in singer_sdk.testing

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = W503, C901, ANN101
+ignore = W503, C901, ANN101, ANN102, ANN401
 max-line-length = 88
 exclude = cookiecutter
 per-file-ignores =

.pre-commit-config.yaml

@@ -70,7 +70,8 @@ repos:
     exclude: |
         (?x)^(
             singer_sdk/helpers/_simpleeval.py|
-            tests/core/test_simpleeval.py
+            tests/core/test_simpleeval.py|
+            singer_sdk/testing/runners.py
         )$
 
 - repo: https://github.com/python-poetry/poetry

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tests/{{ 'test' }}_core.py

@@ -2,7 +2,16 @@
 
 import datetime
 
-from singer_sdk.testing import get_standard_tap_tests
+from singer_sdk.testing import (
+    TapTestRunner,
+    get_test_class,
+    pytest_generate_tests  # pytest hook function, required for standard tests
+)
+from singer_sdk.testing.suites import (
+    tap_stream_attribute_tests,
+    tap_stream_tests,
+    tap_tests,
+)
 
 from {{ cookiecutter.library_name }}.tap import Tap{{ cookiecutter.source_name }}
 
@@ -13,14 +22,13 @@
 
 
 # Run standard built-in tap tests from the SDK:
-def test_standard_tap_tests():
-    """Run standard tap tests from the SDK."""
-    tests = get_standard_tap_tests(
-        Tap{{ cookiecutter.source_name }},
+TestTap{{ cookiecutter.source_name }} = get_test_class(
+    test_runner=TapTestRunner(
+        tap_class=Tap{{ cookiecutter.source_name }},
         config=SAMPLE_CONFIG
-    )
-    for test in tests:
-        test()
+    ),
+    test_suites=[tap_tests, tap_stream_tests, tap_stream_attribute_tests],
+)
 
 
 # TODO: Create additional tests as appropriate for your tap.

cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/tests/{{ 'test' }}_core.py

@@ -1,27 +1,40 @@
 """Tests standard target features using the built-in SDK tests library."""
 
-import datetime
-
+import pytest
 from typing import Dict, Any
 
-from singer_sdk.testing import get_standard_target_tests
+from singer_sdk.testing import TargetTestRunner, get_test_class
+from singer_sdk.testing.suites import target_tests
 
 from {{ cookiecutter.library_name }}.target import Target{{ cookiecutter.destination_name }}
 
 SAMPLE_CONFIG: Dict[str, Any] = {
     # TODO: Initialize minimal target config
 }
 
-
 # Run standard built-in target tests from the SDK:
-def test_standard_target_tests():
-    """Run standard target tests from the SDK."""
-    tests = get_standard_target_tests(
-        Target{{ cookiecutter.destination_name }},
-        config=SAMPLE_CONFIG,
-    )
-    for test in tests:
-        test()
+StandardTargetTests = get_test_class(
+    test_runner=TargetTestRunner(
+        target_class=Target{{ cookiecutter.destination_name }}, config=SAMPLE_CONFIG
+    ),
+    test_suites=[target_tests],
+)
+
+
+class TestTarget{{ cookiecutter.destination_name }}(StandardTargetTests):
+    """Standard Target Tests."""
+
+    @pytest.fixture(scope="class")
+    def resource(self):
+        """Generic external resource.
+
+        This fixture is useful for setup and teardown of external resources,
+        such output folders, tables, buckets etc. for use during testing.
+
+        Example usage can be found in the SDK samples test suite:
+        https://github.com/meltano/sdk/tree/main/tests/samples
+        """
+        yield "resource"
 
 
 # TODO: Create additional tests as appropriate for your target.

docs/index.rst

@@ -71,6 +71,7 @@ Advanced Topics
    CONTRIBUTING
    implementation/index
    typing
+   testing
 
 .. _Singer: https://singer.io
 .. _Singer Spec: https://hub.meltano.com/singer/spec

docs/testing.md

@@ -0,0 +1,115 @@
+# Testing Taps & Targets
+
+The Meltano SDK includes suits of standard tests for both Taps and Targets to help you get started.
+These suites cover most common cases out-of-the-box, and tests are added to the standard suites as new errors are encountered by users in their deployments.
+
+## Test Framework
+
+The Meltano SDK test framwrok consists of 4 main components:
+
+1. A runner class (`TapTestRunner` and `TargetTestRunner`), responsible for executing Taps/Targets and capturing their output.
+1. A suite dataclass, containing a list of tests.
+1. A test template classes (`TapTestTemplate`, `StreamTestTemplate`, `AttributeTestTemplate` and `TargetTestTemplate`), with methods to `.setup()`, `.test()`, `.validate()` and `.teardown()` (called in that order using `.run()`).
+1. A `get_test_class` factory method, to take a runner and a list of suits and return a `pytest` test class.
+
+## Example Usage
+
+If you created your Tap/Target using the provided cookiecutter templates, you will find the following snippets in `<library_name>/tests/<library_name>_core.py`.
+
+### Testing Taps
+
+```python
+import datetime
+
+from singer_sdk.testing import (
+    TapTestRunner,
+    get_test_class,
+    pytest_generate_tests # pytest hook function, required for standard tests
+)
+from singer_sdk.testing.suites import (
+    tap_stream_attribute_tests,
+    tap_stream_tests,
+    tap_tests,
+)
+
+from example.tap import TapExample
+
+SAMPLE_CONFIG = {
+    "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
+}
+
+
+# Run standard built-in tap tests from the SDK:
+TestTapExample = get_test_class(
+    test_runner=TapTestRunner(tap_class=TapExample, config=SAMPLE_CONFIG),
+    test_suites=[tap_tests, tap_stream_tests, tap_stream_attribute_tests],
+)
+```
+
+### Testing Targets
+
+```python
+import pytest
+from typing import Dict, Any
+
+from singer_sdk.testing import TargetTestRunner, get_test_class
+from singer_sdk.testing.suites import target_tests
+
+from example.target import TargetExample
+
+SAMPLE_CONFIG: Dict[str, Any] = {
+    # TODO: Initialize minimal target config
+}
+
+# Run standard built-in target tests from the SDK:
+StandardTargetTests = get_test_class(
+    test_runner=TargetTestRunner(
+        target_class=TargetTargetExample, config=SAMPLE_CONFIG
+    ),
+    test_suites=[target_tests],
+)
+
+
+class TestTargetExample(StandardTargetTests):
+    """Standard Target Tests."""
+
+    @pytest.fixture(scope="class")
+    def resource(self):
+        """Generic external resource.
+
+        This fixture is useful for setup and teardown of external resources,
+        such output folders, tables, buckets etc. for use during testing.
+
+        Example usage can be found in the SDK samples test suite:
+        https://github.com/meltano/sdk/tree/main/tests/samples
+        """
+        yield "resource"
+```
+
+## Writing New Tests
+
+Writing new tests is as easy as subclassing the appropriate class.
+Check out [`singer_sdk/testing/tap_tests.py`](https://github.com/meltano/sdk/tree/main/singer_sdk/testing/tap_tests.py)` and [`singer_sdk/testing/target_tests.py`](https://github.com/meltano/sdk/tree/main/singer_sdk/testing/target_tests.py) for inspiration.
+
+```python
+class TapCLIPrintsTest(TapTestTemplate):
+    "Test that the tap is able to print standard metadata."
+    name = "cli_prints"
+
+    def test(self):
+        self.tap.print_version()
+        self.tap.print_about()
+        self.tap.print_about(format="json")
+```
+
+Once you have created some tests, add them to a suite:
+
+```python
+my_custom_tap_tests = TestSuite(
+    type="tap", tests=[TapCLIPrintsTest]
+)
+```
+
+This suite can now be passed to `get_test_class` along with any other suites, to generate your custom test class.
+
+If your new test covers a common or general case, consider contributing to the standard test library via a pull request to [meltano/sdk](https://github.com/meltano/sdk).

noxfile.py

@@ -86,7 +86,6 @@ def tests(session: Session) -> None:
             "--parallel",
             "-m",
             "pytest",
-            "-x",
             "-v",
             *session.posargs,
         )

pyproject.toml

@@ -57,6 +57,7 @@ python-dotenv = ">=0.20,<0.22"
 typing-extensions = "^4.2.0"
 simplejson = "^3.17.6"
 jsonschema = "^4.16.0"
+pytest = "^7.2.0"
 pytz = "^2022.2.1"
 PyYAML = "^6.0"
 

samples/sample_tap_gitlab/gitlab_rest_streams.py

@@ -100,7 +100,7 @@ class ReleasesStream(ProjectBasedStream):
 
     name = "releases"
     path = "/projects/{project_id}/releases"
-    primary_keys = ["project_id", "commit_id", "tag_name"]
+    primary_keys = ["project_id", "tag_name"]
     replication_key = None
     schema_filepath = SCHEMAS_DIR / "releases.json"
 
@@ -112,7 +112,7 @@ class IssuesStream(ProjectBasedStream):
     path = "/projects/{project_id}/issues?scope=all&updated_after={start_date}"
     primary_keys = ["id"]
     replication_key = "updated_at"
-    is_sorted = True
+    is_sorted = False
     schema_filepath = SCHEMAS_DIR / "issues.json"