🎉 Refactor Pokeapi Source to CDK (#8432)

* Refactor Pokeapi Source to CDK

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -491,7 +491,7 @@
 - name: PokeAPI
   sourceDefinitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968
   dockerRepository: airbyte/source-pokeapi
-  dockerImageTag: 0.1.1
+  dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.io/integrations/sources/pokeapi
   icon: pokeapi.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -4727,7 +4727,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-pokeapi:0.1.1"
+- dockerImage: "airbyte/source-pokeapi:0.1.3"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/pokeapi"
     connectionSpecification:

airbyte-integrations/connectors/source-pokeapi/.dockerignore

@@ -1,6 +1,6 @@
 *
 !Dockerfile
-!Dockerfile.test
+!main.py
 !source_pokeapi
 !setup.py
 !secrets

airbyte-integrations/connectors/source-pokeapi/Dockerfile

@@ -1,18 +1,17 @@
-FROM airbyte/integration-base-python:0.1.7
+FROM python:3.7-slim
 
 # Bash is installed for more convenient debugging.
 RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
-ENV CODE_PATH="source_pokeapi"
-ENV AIRBYTE_IMPL_MODULE="source_pokeapi"
-ENV AIRBYTE_IMPL_PATH="SourcePokeapi"
 
 WORKDIR /airbyte/integration_code
-COPY $CODE_PATH ./$CODE_PATH
+COPY source_pokeapi ./source_pokeapi
+COPY main.py ./
 COPY setup.py ./
 RUN pip install .
 
-ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-pokeapi

airbyte-integrations/connectors/source-pokeapi/README.md

@@ -85,10 +85,45 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pokeapi:dev discover -
 docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-pokeapi:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json
 ```
 
+## Testing
+Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+First install test dependencies into your virtual environment:
+```
+pip install .[tests]
+```
+### Unit Tests
+To run unit tests locally, from the connector directory run:
+```
+python -m pytest unit_tests
+```
+
 ### Integration Tests
-1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-pokeapi:integrationTest` to run the standard integration test suite.
-1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`.
-   Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector).
+
+#### Custom Integration tests
+Place custom tests inside `integration_tests/` folder, then, from the connector root, run
+```
+python -m pytest integration_tests
+```
+#### Acceptance Tests
+Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information.
+If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
+To run your integration tests with acceptance tests, from the connector root, run
+```
+python -m pytest integration_tests -p integration_tests.acceptance
+```
+To run your integration tests with docker
+
+### Using gradle to run tests
+All commands should be run from airbyte project root.
+To run unittest run:
+```
+./gradlew :airbyte-integrations:connectors:source-pokeapi:unitTest
+```
+To run acceptance and custom integration tests run:
+```
+./gradlew :airbyte-integrations:connectors:source-pokeapi:IntegrationTest
+```
 
 ## Dependency Management
 All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.

airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml

@@ -0,0 +1,15 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-pokeapi:dev
+tests:
+  connection:
+    - config_path: "integration_tests/config.json"
+      status: "succeed"
+  discovery:
+    - config_path: "integration_tests/config.json"
+  basic_read:
+    - config_path: "integration_tests/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+  full_refresh:
+    - config_path: "integration_tests/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-pokeapi/acceptance-test-docker.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-)
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp:/tmp \
+    -v $(pwd):/test_input \
+    airbyte/source-acceptance-test \
+    --acceptance-test-config /test_input
+

airbyte-integrations/connectors/source-pokeapi/build.gradle

@@ -1,30 +1,9 @@
 plugins {
     id 'airbyte-python'
     id 'airbyte-docker'
-    id 'airbyte-standard-source-test-file'
+    id 'airbyte-source-acceptance-test'
 }
 
 airbytePython {
     moduleDirectory 'source_pokeapi'
 }
-
-airbyteStandardSourceTestFile {
-    // For more information on standard source tests, see https://docs.airbyte.io/connector-development/testing-connectors
-
-    // All these input paths must live inside this connector's directory (or subdirectories)
-    specPath = "source_pokeapi/spec.json"
-
-    // configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file
-    // there (in case it contains any credentials)
-    configPath = "sample_files/config.json"
-
-    // Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured
-    // for incremental syncing (e.g. include cursor fields, etc).
-    configuredCatalogPath = "sample_files/configured_catalog.json"
-}
-
-
-dependencies {
-    implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
-    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
-}

airbyte-integrations/connectors/source-pokeapi/integration_tests/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#

airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """This fixture is a placeholder for external resources that acceptance test might require."""
+    yield

airbyte-integrations/connectors/source-pokeapi/main.py

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+import sys
+
+from airbyte_cdk.entrypoint import launch
+from source_pokeapi import SourcePokeapi
+
+if __name__ == "__main__":
+    source = SourcePokeapi()
+    launch(source, sys.argv[1:])

airbyte-integrations/connectors/source-pokeapi/requirements.txt

@@ -1,4 +1,2 @@
-# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies.
--e ../../bases/airbyte-protocol
--e ../../bases/base-python
+-e ../../bases/source-acceptance-test
 -e .

airbyte-integrations/connectors/source-pokeapi/setup.py

@@ -3,38 +3,21 @@
 #
 
 
-"""
-MIT License
-
-Copyright (c) 2020 Airbyte
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+from setuptools import find_packages, setup
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-"""
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"]
 
-from setuptools import find_packages, setup
+TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test"]
 
 setup(
     name="source_pokeapi",
     description="Source implementation for Pokeapi.",
     author="Airbyte",
     author_email="[email protected]",
     packages=find_packages(),
-    install_requires=["airbyte-protocol", "base-python", "pytest==6.1.2"],
+    install_requires=MAIN_REQUIREMENTS,
     package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]},
+    extras_require={
+        "tests": TEST_REQUIREMENTS,
+    },
 )

airbyte-integrations/connectors/source-pokeapi/source_pokeapi/__init__.py

@@ -1,26 +1,6 @@
-"""
-MIT License
-
-Copyright (c) 2020 Airbyte
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-"""
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
 
 from .source import SourcePokeapi
 

airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json

@@ -117,7 +117,7 @@
                   }
                 },
                 "rarity": {
-                  "type": ["null", "string"]
+                  "type": ["null", "integer"]
                 }
               }
             }

airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py

@@ -6,7 +6,9 @@
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
 
 import requests
-from base_python import AbstractSource, HttpStream, Stream
+from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http import HttpStream
 
 from . import pokemon_list
 

airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py

@@ -3,32 +3,5 @@
 #
 
 
-"""
-MIT License
-
-Copyright (c) 2020 Airbyte
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-"""
-
-# format anchor
-
-
 def test_example_method():
     assert True

docs/integrations/sources/pokeapi.md

@@ -36,6 +36,7 @@ The PokéAPI uses the same [JSONSchema](https://json-schema.org/understanding-js
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.3 | 2021-12-03 | [8432](https://github.com/airbytehq/airbyte/pull/8432) | Migrate from base_python to CDK, add SAT tests. |
 | 0.1.1 | 2020-06-29 | [1046](https://github.com/airbytehq/airbyte/pull/4410) | Fix runtime UI error from GitHub store path. |
 | 0.1.0 | 2020-05-04 | [1046](https://github.com/airbytehq/airbyte/pull/3149) | Add source for PokeAPI. |