diff --git a/airbyte-commons-docker/build.gradle b/airbyte-commons-docker/build.gradle new file mode 100644 index 000000000000..4c2ec5099bfc --- /dev/null +++ b/airbyte-commons-docker/build.gradle @@ -0,0 +1,11 @@ +plugins { + id "java-library" +} + +dependencies { + implementation 'org.apache.commons:commons-compress:1.20' + implementation 'com.github.docker-java:docker-java:3.2.8' + implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.8' + + testImplementation 'org.apache.commons:commons-lang3:3.11' +} diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtils.java b/airbyte-commons-docker/src/main/java/io/airbyte/commons/docker/DockerUtils.java similarity index 96% rename from airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtils.java rename to airbyte-commons-docker/src/main/java/io/airbyte/commons/docker/DockerUtils.java index e7d83439ee77..26f015f2f691 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtils.java +++ b/airbyte-commons-docker/src/main/java/io/airbyte/commons/docker/DockerUtils.java @@ -34,7 +34,6 @@ import java.io.File; import java.util.Set; -// TODO(Davin): This should be split out into an Docker commons to minimise dependency pollution. public class DockerUtils { private static final DockerClientConfig CONFIG = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java b/airbyte-commons-docker/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java rename to airbyte-commons-docker/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java diff --git a/airbyte-commons/build.gradle b/airbyte-commons/build.gradle index 4c2ec5099bfc..7ea1d6aa92f9 100644 --- a/airbyte-commons/build.gradle +++ b/airbyte-commons/build.gradle @@ -4,8 +4,5 @@ plugins { dependencies { implementation 'org.apache.commons:commons-compress:1.20' - implementation 'com.github.docker-java:docker-java:3.2.8' - implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.8' - - testImplementation 'org.apache.commons:commons-lang3:3.11' + implementation 'org.apache.commons:commons-lang3:3.11' } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java b/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java index 248f0dc81ced..9c6067b9d351 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java @@ -26,7 +26,7 @@ import com.google.common.collect.Streams; import java.util.stream.Collectors; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; public class Strings { diff --git a/airbyte-integrations/connectors/source-always-works/.dockerignore b/airbyte-integrations/connectors/source-always-works/.dockerignore deleted file mode 100644 index cdada38cab87..000000000000 --- a/airbyte-integrations/connectors/source-always-works/.dockerignore +++ /dev/null @@ -1,7 +0,0 @@ -* -!Dockerfile -!Dockerfile.test -!main.py -!source_always_works -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-always-works/Dockerfile b/airbyte-integrations/connectors/source-always-works/Dockerfile deleted file mode 100644 index 52f9db22a3d1..000000000000 --- a/airbyte-integrations/connectors/source-always-works/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.7-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash socat && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_always_works ./source_always_works -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-always-works diff --git a/airbyte-integrations/connectors/source-always-works/README.md b/airbyte-integrations/connectors/source-always-works/README.md deleted file mode 100644 index a82392d4f21e..000000000000 --- a/airbyte-integrations/connectors/source-always-works/README.md +++ /dev/null @@ -1,129 +0,0 @@ -# Always Works Source - -This is the repository for the Always Works source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/always-works). - -## Local development - -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-always-works:build -``` - -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/always-works) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_always_works/spec.json` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source always-works test creds` -and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - -### Locally running the connector docker image - -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-always-works:dev -``` - -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-always-works:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-always-works:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-always-works:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-always-works:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-always-works:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` -## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-always-works:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-always-works:integrationTest -``` - -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-always-works/acceptance-test-config.yml b/airbyte-integrations/connectors/source-always-works/acceptance-test-config.yml deleted file mode 100644 index 21baf0487cce..000000000000 --- a/airbyte-integrations/connectors/source-always-works/acceptance-test-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md) -# for more information about how to configure these tests -connector_image: airbyte/source-always-works:dev -tests: - spec: - - spec_path: "source_always_works/spec.json" - connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "exception" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - validate_output_from_all_streams: yes -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - state_path: "integration_tests/abnormal_state.json" - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-always-works/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-always-works/acceptance-test-docker.sh deleted file mode 100644 index 1425ff74f151..000000000000 --- a/airbyte-integrations/connectors/source-always-works/acceptance-test-docker.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env sh -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-always-works/build.gradle b/airbyte-integrations/connectors/source-always-works/build.gradle deleted file mode 100644 index f96068f38d70..000000000000 --- a/airbyte-integrations/connectors/source-always-works/build.gradle +++ /dev/null @@ -1,14 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-source-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_always_works' -} - -dependencies { - implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) - implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-always-works/integration_tests/__init__.py b/airbyte-integrations/connectors/source-always-works/integration_tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/airbyte-integrations/connectors/source-always-works/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-always-works/integration_tests/abnormal_state.json deleted file mode 100644 index 52b0f2c2118f..000000000000 --- a/airbyte-integrations/connectors/source-always-works/integration_tests/abnormal_state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" - } -} diff --git a/airbyte-integrations/connectors/source-always-works/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-always-works/integration_tests/acceptance.py deleted file mode 100644 index eeb4a2d3e02e..000000000000 --- a/airbyte-integrations/connectors/source-always-works/integration_tests/acceptance.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - - -import pytest - -pytest_plugins = ("source_acceptance_test.plugin",) - - -@pytest.fixture(scope="session", autouse=True) -def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments - yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-always-works/integration_tests/catalog.json b/airbyte-integrations/connectors/source-always-works/integration_tests/catalog.json deleted file mode 100644 index 23da72ab1a5c..000000000000 --- a/airbyte-integrations/connectors/source-always-works/integration_tests/catalog.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "streams": [ - { - "name": "always_works_stream", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": "stub", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "count": { - "type": "integer" - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-always-works/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-always-works/integration_tests/configured_catalog.json deleted file mode 100644 index 0822a4ac8caa..000000000000 --- a/airbyte-integrations/connectors/source-always-works/integration_tests/configured_catalog.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "always_works_stream", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": ["stub"], - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "count": { - "type": "integer" - } - } - } - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-always-works/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-always-works/integration_tests/invalid_config.json deleted file mode 100644 index f3732995784f..000000000000 --- a/airbyte-integrations/connectors/source-always-works/integration_tests/invalid_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" -} diff --git a/airbyte-integrations/connectors/source-always-works/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-always-works/integration_tests/sample_config.json deleted file mode 100644 index ecc4913b84c7..000000000000 --- a/airbyte-integrations/connectors/source-always-works/integration_tests/sample_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "fix-me": "TODO" -} diff --git a/airbyte-integrations/connectors/source-always-works/main.py b/airbyte-integrations/connectors/source-always-works/main.py deleted file mode 100644 index 9ecc637c6884..000000000000 --- a/airbyte-integrations/connectors/source-always-works/main.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - - -import sys - -from airbyte_cdk.entrypoint import launch -from source_always_works import SourceAlwaysWorks - -if __name__ == "__main__": - source = SourceAlwaysWorks() - launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-always-works/requirements.txt b/airbyte-integrations/connectors/source-always-works/requirements.txt deleted file mode 100644 index 0411042aa091..000000000000 --- a/airbyte-integrations/connectors/source-always-works/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ --e ../../bases/source-acceptance-test --e . diff --git a/airbyte-integrations/connectors/source-always-works/setup.py b/airbyte-integrations/connectors/source-always-works/setup.py deleted file mode 100644 index 52a13a5ad172..000000000000 --- a/airbyte-integrations/connectors/source-always-works/setup.py +++ /dev/null @@ -1,48 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] - -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "source-acceptance-test", -] - -setup( - name="source_always_works", - description="Source implementation for Always Works.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-always-works/source_always_works/__init__.py b/airbyte-integrations/connectors/source-always-works/source_always_works/__init__.py deleted file mode 100644 index e5b3b92da7de..000000000000 --- a/airbyte-integrations/connectors/source-always-works/source_always_works/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .source import SourceAlwaysWorks - -__all__ = ["SourceAlwaysWorks"] diff --git a/airbyte-integrations/connectors/source-always-works/source_always_works/schemas/always_works_stream.json b/airbyte-integrations/connectors/source-always-works/source_always_works/schemas/always_works_stream.json deleted file mode 100644 index c450eda71eea..000000000000 --- a/airbyte-integrations/connectors/source-always-works/source_always_works/schemas/always_works_stream.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "count": { - "type": "integer" - } - } -} diff --git a/airbyte-integrations/connectors/source-always-works/source_always_works/source.py b/airbyte-integrations/connectors/source-always-works/source_always_works/source.py deleted file mode 100644 index 0a5b81b1706e..000000000000 --- a/airbyte-integrations/connectors/source-always-works/source_always_works/source.py +++ /dev/null @@ -1,101 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - - -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple - -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream - - -class AlwaysWorksStream(HttpStream): - - current_counter = 0 # Counter for current response number. - url_base = "stub_base" - primary_key = "stub_key" - - def __init__(self, limit: int): - super().__init__() - self._limit = limit - - # Ignored Functions - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "unused" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {} - - # Used Functions - def read_records( - self, - sync_mode: SyncMode, - stream_state: Mapping[str, Any] = None, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - """ - Override this to only call the parse_response function. - """ - return self.parse_response(requests.Response()) - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - """ - Return the self._limit number of slices, so the same number of read_record - calls are made. - """ - for i in range(self._limit): - yield i - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - Returns an incrementing number. - """ - self.current_counter += 1 - yield {"count": self.current_counter} - - -class SourceAlwaysWorks(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - Always works is always working; return True - """ - return True, None - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - return [AlwaysWorksStream(config["limit"])] diff --git a/airbyte-integrations/connectors/source-always-works/source_always_works/spec.json b/airbyte-integrations/connectors/source-always-works/source_always_works/spec.json deleted file mode 100644 index 17c8f21a26fc..000000000000 --- a/airbyte-integrations/connectors/source-always-works/source_always_works/spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "documentationUrl": "http://www.stubstub.stub", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Always Works Spec", - "type": "object", - "required": ["limit"], - "additionalProperties": false, - "properties": { - "limit": { - "type": "number", - "description": "Number of records the Always On Stream will return." - } - } - } -} diff --git a/airbyte-integrations/connectors/source-always-works/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-always-works/unit_tests/unit_test.py deleted file mode 100644 index b8a8150b507f..000000000000 --- a/airbyte-integrations/connectors/source-always-works/unit_tests/unit_test.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - - -def test_example_method(): - assert True diff --git a/airbyte-scheduler/persistence/build.gradle b/airbyte-scheduler/persistence/build.gradle index 6d8bed7cf08d..4025f0552f98 100644 --- a/airbyte-scheduler/persistence/build.gradle +++ b/airbyte-scheduler/persistence/build.gradle @@ -4,6 +4,7 @@ plugins { dependencies { implementation project(':airbyte-analytics') + implementation project(':airbyte-commons-docker') implementation project(':airbyte-config:models') implementation project(':airbyte-config:persistence') implementation project(':airbyte-db') diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index dfb935359b4d..45cc73bfb3b1 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -20,6 +20,7 @@ dependencies { implementation project(':airbyte-analytics') implementation project(':airbyte-api') + implementation project(':airbyte-commons-docker') implementation project(':airbyte-config:models') implementation project(':airbyte-config:persistence') implementation project(':airbyte-config:init') diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile deleted file mode 100644 index e435ea69234e..000000000000 --- a/airbyte-workers/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM openjdk:14.0.2-slim - -WORKDIR /airbyte - -COPY build/distributions/airbyte-workers*.tar airbyte-workers.tar - -RUN tar xf airbyte-workers.tar --strip-components=1 - -ENTRYPOINT ["./bin/airbyte-workers"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/worker diff --git a/airbyte-workers/README.md b/airbyte-workers/README.md deleted file mode 100644 index 1fcb03e6920c..000000000000 --- a/airbyte-workers/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Kube Queueing POC - -To build sync worker: -``` -cd ~/code/airbyte -./gradlew :airbyte-workers:airbyteDocker -cd ~/code/airbyte/airbyte-workers/docker-shim-mvp/destination-listen-and-echo -docker build -t airbyte/destination-listen-and-echo:dev . -``` - -To run the sync worker -``` -cd ~/code/airbyte/airbyte-workers/src/main/resources/kube_queue_poc -kubectl apply -f kube-sync-workers.yaml -``` diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 7cfde5c566a7..6140d0a9d63b 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -1,22 +1,8 @@ import org.jsonschema2pojo.SourceType plugins { - id 'application' + id 'java-library' id 'com.github.eirnym.js2p' version '1.0' - id 'airbyte-docker' -} - -application { - mainClass = 'io.airbyte.workers.process.KubeProcessBuilderFactoryPOC' - applicationDefaultJvmArgs = ['-Xmx500m', - '-XX:NativeMemoryTracking=detail', - "-Djava.rmi.server.hostname=localhost", - '-Dcom.sun.management.jmxremote=true', - '-Dcom.sun.management.jmxremote.port=6000', - "-Dcom.sun.management.jmxremote.rmi.port=6000", - '-Dcom.sun.management.jmxremote.local.only=false', - '-Dcom.sun.management.jmxremote.authenticate=false', - '-Dcom.sun.management.jmxremote.ssl=false'] } configurations { @@ -41,6 +27,8 @@ dependencies { testImplementation 'org.testcontainers:testcontainers:1.15.3' testImplementation 'org.testcontainers:postgresql:1.15.1' testImplementation 'org.postgresql:postgresql:42.2.18' + + testImplementation project(':airbyte-commons-docker') } jsonSchema2Pojo { diff --git a/airbyte-workers/docker-shim-mvp/README.md b/airbyte-workers/docker-shim-mvp/README.md deleted file mode 100644 index ab6ba11b53b0..000000000000 --- a/airbyte-workers/docker-shim-mvp/README.md +++ /dev/null @@ -1,16 +0,0 @@ -## Introduction -MVP for shim over our current entrypoint using `socat`. - -1. From the `~/code/airbyte` directory run `./gradlew :airbyte-integrations:connectors:source-always-works:airbyteDocker` - -2. From `~/code/airbyte/airbyte-integrations/connectors/source-always-works/docker-shim-mvp/docker-shim-base` run `docker build -t airbyte/docker-shim-base:dev .` - -Note: This only handle the simpler CDK entrypoint now. This does not handle the more complex -Java entrypoints. - -3. Run `echo "{\"limit\": 1000}" > ~/code/airbyte/airbyte-integrations/connectors/source-always-works/secrets/config.json` - -4. Listen on localhost: - `socat -d -d -d TCP-LISTEN:9000,bind=127.0.0.1 stdout` - -5. From `~/code/airbyte/airbyte-integrations/connectors/source-always-works`, run `docker run -it --rm --network=host -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests -e SRC_IP=host.docker.internal -e SRC_PORT=9000 airbyte/docker-shim-base:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json` diff --git a/airbyte-workers/docker-shim-mvp/destination-listen-and-echo/Dockerfile b/airbyte-workers/docker-shim-mvp/destination-listen-and-echo/Dockerfile deleted file mode 100644 index 201ee5bc0245..000000000000 --- a/airbyte-workers/docker-shim-mvp/destination-listen-and-echo/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -# Use the same java base to simulate a destination image. -#FROM airbyte/integration-base-java:dev -FROM debian:latest -ENV AIRBYTE_ENTRYPOINT="./listen-and-echo.sh" - -RUN apt-get update && apt-get install -y bash socat && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code - -COPY listen-and-echo.sh ./ - -ENTRYPOINT ["./listen-and-echo.sh"] diff --git a/airbyte-workers/docker-shim-mvp/destination-listen-and-echo/listen-and-echo.sh b/airbyte-workers/docker-shim-mvp/destination-listen-and-echo/listen-and-echo.sh deleted file mode 100755 index 324a30a667f7..000000000000 --- a/airbyte-workers/docker-shim-mvp/destination-listen-and-echo/listen-and-echo.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -socat -d -d -d TCP-LISTEN:"${DEST_PORT}",bind="${DEST_IP}" stdout diff --git a/airbyte-workers/docker-shim-mvp/docker-shim-base/Dockerfile b/airbyte-workers/docker-shim-mvp/docker-shim-base/Dockerfile deleted file mode 100644 index bf595b601a35..000000000000 --- a/airbyte-workers/docker-shim-mvp/docker-shim-base/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Replace this with an image of any Python source that uses the new CDK entrypoint. -FROM airbyte/source-always-works:dev - -WORKDIR /airbyte/integration_code - -COPY socat.sh ./ - -ENTRYPOINT ["./socat.sh"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-always-works-socat-shim diff --git a/airbyte-workers/docker-shim-mvp/docker-shim-base/socat.sh b/airbyte-workers/docker-shim-mvp/docker-shim-base/socat.sh deleted file mode 100755 index a6409ef94340..000000000000 --- a/airbyte-workers/docker-shim-mvp/docker-shim-base/socat.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -echo "ip: ${SRC_IP}", "port: ${SRC_PORT}" -echo "$@" -python "/airbyte/integration_code/main.py" "$@" | socat -d -d -d - TCP:${SRC_IP}:${SRC_PORT} diff --git a/airbyte-workers/named_pipes/README.md b/airbyte-workers/named_pipes/README.md deleted file mode 100644 index 932b4ce8226c..000000000000 --- a/airbyte-workers/named_pipes/README.md +++ /dev/null @@ -1,32 +0,0 @@ -Try to use a mounted named pipe instead of using socat. - -For builds: -``` -cd ~/code/airbyte/airbyte-workers/named_pipes/np_source -docker build -t np_source:dev . -cd ~/code/airbyte/airbyte-workers/named_pipes/np_dest -docker build -t np_dest:dev . -``` - -Then run the "init container" equivalent: -``` -docker run -it --rm -v /tmp/np_file:/tmp/np_file --entrypoint "sh" np_source:dev -c "rm /tmp/np_file/pipe && mkfifo /tmp/np_file/pipe" -``` - -Then run the source and destination at the same time (you can do either first): -``` -docker run -it --rm -v /tmp/np_file:/tmp/np_file --entrypoint "sh" np_source:dev -c "/tmp/run.sh > /tmp/np_file/pipe" -docker run -it --rm -v /tmp/np_file:/tmp/np_file --entrypoint "sh" np_dest:dev -c "cat /tmp/np_file/pipe | /tmp/run.sh" -``` - -Notes: -- The source blocks until something starts reading from the named pipe. -- `cat` detects when the source reaches its EOF -- You can start the destination before the source and it terminates as expected. - -Todo: -- Set this up for Kube -- Use an init container to create the initial named pipe -- Use a shared mount -- Test message sizes / etc -- Error code propagation diff --git a/airbyte-workers/named_pipes/np_dest/Dockerfile b/airbyte-workers/named_pipes/np_dest/Dockerfile deleted file mode 100644 index b8c641c6cefe..000000000000 --- a/airbyte-workers/named_pipes/np_dest/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM debian:latest -COPY run.sh /tmp/run.sh -ENV AIRBYTE_ENTRYPOINT="/tmp/run.sh" -ENTRYPOINT /tmp/run.sh diff --git a/airbyte-workers/named_pipes/np_dest/run.sh b/airbyte-workers/named_pipes/np_dest/run.sh deleted file mode 100755 index 57b4f9695dfe..000000000000 --- a/airbyte-workers/named_pipes/np_dest/run.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -while IFS='$\n' read -r line; do - echo "received $line" -done diff --git a/airbyte-workers/named_pipes/np_source/Dockerfile b/airbyte-workers/named_pipes/np_source/Dockerfile deleted file mode 100644 index 7fe38d098405..000000000000 --- a/airbyte-workers/named_pipes/np_source/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM alpine:latest -COPY run.sh /tmp/run.sh -ENV AIRBYTE_ENTRYPOINT="/tmp/run.sh" -ENTRYPOINT ["/tmp/run.sh"] diff --git a/airbyte-workers/named_pipes/np_source/run.sh b/airbyte-workers/named_pipes/np_source/run.sh deleted file mode 100755 index 2ff0116d5b23..000000000000 --- a/airbyte-workers/named_pipes/np_source/run.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -echo "starting..." - -while IFS= read -r line -do - echo "{\"msg\": \"$line\"}" - sleep 3 -done < "/pipes/file.txt" - -echo "ended." diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessBuilderFactoryPOC.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessBuilderFactoryPOC.java deleted file mode 100644 index 9aef753eef4b..000000000000 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessBuilderFactoryPOC.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.workers.process; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.io.IOs; -import io.fabric8.kubernetes.client.DefaultKubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClient; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.util.Collections; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class KubeProcessBuilderFactoryPOC { - - private static final Logger LOGGER = LoggerFactory.getLogger(KubeProcessBuilderFactoryPOC.class); - private static final KubernetesClient KUBE_CLIENT = new DefaultKubernetesClient(); - - public static void testSyncWorkflow() throws IOException, InterruptedException { - LOGGER.info("Launching source process..."); - Process src = new KubePodProcess(KUBE_CLIENT, - "src", - "default", - "np_source:dev", - 9002, - 9003, - false, - ImmutableMap.of("file.txt", "testing\none\ntwo\nthree\nfour\nfive\nsix\nseven"), - null); - - LOGGER.info("Launching destination process..."); - Process dest = new KubePodProcess(KUBE_CLIENT, - "dest", - "default", - "np_dest:dev", - 9005, - 9006, - true, - Collections.emptyMap(), - null); - - LOGGER.info("Launching background thread to read destination lines..."); - ExecutorService executor = Executors.newSingleThreadExecutor(); - var listenTask = executor.submit(() -> { - BufferedReader reader = new BufferedReader(new InputStreamReader(dest.getInputStream())); - try { - String line; - while ((line = reader.readLine()) != null) { - LOGGER.info("Destination sent: {}", line); - } - } catch (IOException e) { - e.printStackTrace(); - } - }); - - LOGGER.info("Copying source stdout to destination stdin..."); - - try (BufferedReader reader = IOs.newBufferedReader(src.getInputStream())) { - try (PrintWriter writer = new PrintWriter(dest.getOutputStream(), true)) { - String line; - while ((line = reader.readLine()) != null) { - LOGGER.info("relaying: " + line); - writer.println(line); - } - } - } - - LOGGER.info("Waiting for source process to terminate..."); - src.waitFor(); - LOGGER.info("Waiting for destination process to terminate..."); - dest.waitFor(); - - LOGGER.info("Closing sync worker resources..."); - listenTask.cancel(true); - executor.shutdownNow(); - // TODO(Davin, issue-3611): Figure out why these commands are not effectively shutting down OkHTTP - // even though documentation suggests so. See - // https://square.github.io/okhttp/4.x/okhttp/okhttp3/-ok-http-client/#shutdown-isnt-necessary - // Instead, the pod shuts down after 5 minutes as the pool reaps the remaining idle connection after - // 5 minutes of inactivity, as per the default configuration. - // OK_HTTP_CLIENT.dispatcher().executorService().shutdownNow(); - // OK_HTTP_CLIENT.connectionPool().evictAll(); - // The Kube client has issues with closing the client. Since manually injecting the OkHttp client - // also doesn't work, it is not clear whether it's OkHTTP or the Fabric client at fault. - // See https://github.com/fabric8io/kubernetes-client/issues/2403. - KUBE_CLIENT.close(); - LOGGER.info("Done!"); - // Manually exit for the time being. - System.exit(0); - } - - /* - * To run: cd ~/code/airbyte/airbyte-workers/named_pipes/np_source docker build -t np_source:dev . - * cd ~/code/airbyte/airbyte-workers/named_pipes/np_dest docker build -t np_dest:dev . cd - * ~/code/airbyte ./gradlew :airbyte-workers:airbyteDocker kubectl apply -f - * ~/code/airbyte/airbyte-workers/src/main/resources/kube_queue_poc/launch - */ - public static void main(String[] args) throws InterruptedException, IOException { - testSyncWorkflow(); - } - -} diff --git a/airbyte-workers/src/main/resources/kube_queue_poc/launch/airbyte-worker.yaml b/airbyte-workers/src/main/resources/kube_queue_poc/launch/airbyte-worker.yaml deleted file mode 100644 index 687bc425d1a9..000000000000 --- a/airbyte-workers/src/main/resources/kube_queue_poc/launch/airbyte-worker.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: sync-worker -spec: - restartPolicy: Never - containers: - - name: sync-worker - image: airbyte/worker:dev - # imagePullPolicy: Always -> prevents local images from working - ports: - - containerPort: 9002 - name: src-stdout - - containerPort: 9003 - name: src-stderr - - containerPort: 9005 - name: dest-stdout - - containerPort: 9006 - name: dest-stderr diff --git a/airbyte-workers/src/main/resources/kube_queue_poc/launch/default_service_account_perms.yaml b/airbyte-workers/src/main/resources/kube_queue_poc/launch/default_service_account_perms.yaml deleted file mode 100644 index 776010ddfb45..000000000000 --- a/airbyte-workers/src/main/resources/kube_queue_poc/launch/default_service_account_perms.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - creationTimestamp: null - name: default-role - namespace: default -rules: - - apiGroups: - - "" - resources: - - services - - pods - verbs: - - get - - list - - create - - update - - delete - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - creationTimestamp: null - name: default-sa-binding - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: default-role -subjects: - - kind: ServiceAccount - name: default - namespace: default diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java index 2b84da32201f..78d83a6f9bbd 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java @@ -88,7 +88,7 @@ void discover() throws WorkerException { void read() throws WorkerException { launcher.read(JOB_ROOT, "config", "{}", "catalog", "{}", "state", "{}"); - Mockito.verify(processFactory).create(JOB_ID, JOB_ATTEMPT, JOB_ROOT, FAKE_IMAGE, false, CONFIG_CATALOG_FILES, null, + Mockito.verify(processFactory).create(JOB_ID, JOB_ATTEMPT, JOB_ROOT, FAKE_IMAGE, false, CONFIG_CATALOG_STATE_FILES, null, Lists.newArrayList( "read", "--config", "config", diff --git a/settings.gradle b/settings.gradle index f781d0d65417..5c759460eb71 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,6 +18,7 @@ include ':airbyte-api' include ':airbyte-cli' include ':airbyte-cdk:python' include ':airbyte-commons' +include ':airbyte-commons-docker' include ':airbyte-config:models' include ':airbyte-config:init' include ':airbyte-config:persistence'