From 97c26293f7247e62f8a05e76b133923156dfe3a5 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 10:47:29 -0800 Subject: [PATCH 01/20] fix python install order --- airbyte-integrations/bases/airbyte-protocol/build.gradle | 1 + buildSrc/src/main/groovy/airbyte-python.gradle | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/airbyte-integrations/bases/airbyte-protocol/build.gradle b/airbyte-integrations/bases/airbyte-protocol/build.gradle index 71c89d74ea73e..9d524a9066a7b 100644 --- a/airbyte-integrations/bases/airbyte-protocol/build.gradle +++ b/airbyte-integrations/bases/airbyte-protocol/build.gradle @@ -17,3 +17,4 @@ task generateProtocolClassFilesWithoutLicense(type: Exec) { task generateProtocolClassFiles(dependsOn: [generateProtocolClassFilesWithoutLicense, rootProject.spotlessPython]) rootProject.spotlessPython.mustRunAfter(generateProtocolClassFilesWithoutLicense) rootProject.build.dependsOn generateProtocolClassFiles +airbytePythonApply.dependsOn(generateProtocolClassFiles) diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 27c3d12db8be9..9f3b17dd3cd53 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -49,6 +49,11 @@ class AirbytePythonPlugin implements Plugin { inputs.file('requirements.txt') outputs.file('build/installedlocalreqs.txt') outputs.cacheIf { true } + + // HACK: makes all integrations depend on installing requirements for bases. long term we should resolve deps and install in order. + if(project.getPath().startsWith(":airbyte-integrations:connectors")) { + dependsOn project.rootProject.getTasksByName("airbytePythonApply", true).findAll { it.project.getPath().startsWith(":airbyte-integrations:bases")} + } } project.task('installReqs', type: PythonTask, dependsOn: project.installLocalReqs) { From 8fec8ce854b442e34e5281823c152d226d0ee8c5 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 11:10:55 -0800 Subject: [PATCH 02/20] fix base install reqs dependencies --- airbyte-integrations/bases/base-normalization/build.gradle | 2 ++ airbyte-integrations/bases/base-python-test/build.gradle | 2 ++ airbyte-integrations/bases/base-python/build.gradle | 2 ++ airbyte-integrations/bases/base-singer/build.gradle | 2 ++ 4 files changed, 8 insertions(+) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index d3b397d445ae8..4e350cba759cc 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -10,3 +10,5 @@ airbytePython { dependencies { implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs) } + +installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs") diff --git a/airbyte-integrations/bases/base-python-test/build.gradle b/airbyte-integrations/bases/base-python-test/build.gradle index f3d0d6e9b812a..0ede89d115683 100644 --- a/airbyte-integrations/bases/base-python-test/build.gradle +++ b/airbyte-integrations/bases/base-python-test/build.gradle @@ -10,3 +10,5 @@ airbytePython { dependencies { implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } + +installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs") diff --git a/airbyte-integrations/bases/base-python/build.gradle b/airbyte-integrations/bases/base-python/build.gradle index f37c0d27978af..8734f5f61a7fd 100644 --- a/airbyte-integrations/bases/base-python/build.gradle +++ b/airbyte-integrations/bases/base-python/build.gradle @@ -11,3 +11,5 @@ dependencies { implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs) } + +airbytePythonApply.dependsOn(":airbyte-integrations:bases:airbyte-protocol:airbytePythonApply") diff --git a/airbyte-integrations/bases/base-singer/build.gradle b/airbyte-integrations/bases/base-singer/build.gradle index 491458e7d9a1f..d9314256612c4 100644 --- a/airbyte-integrations/bases/base-singer/build.gradle +++ b/airbyte-integrations/bases/base-singer/build.gradle @@ -10,3 +10,5 @@ airbytePython { dependencies { implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } + +installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs", ":airbyte-integrations:bases:base-python:installReqs") From 9f8355f28c996184cc8baeec3fb75545d0b94086 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 11:20:30 -0800 Subject: [PATCH 03/20] don't clobber dependencies --- airbyte-integrations/bases/airbyte-protocol/build.gradle | 2 +- airbyte-integrations/bases/base-normalization/build.gradle | 2 +- airbyte-integrations/bases/base-python-test/build.gradle | 2 +- airbyte-integrations/bases/base-python/build.gradle | 2 +- airbyte-integrations/bases/base-singer/build.gradle | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/bases/airbyte-protocol/build.gradle b/airbyte-integrations/bases/airbyte-protocol/build.gradle index 9d524a9066a7b..914a05b22a4cc 100644 --- a/airbyte-integrations/bases/airbyte-protocol/build.gradle +++ b/airbyte-integrations/bases/airbyte-protocol/build.gradle @@ -17,4 +17,4 @@ task generateProtocolClassFilesWithoutLicense(type: Exec) { task generateProtocolClassFiles(dependsOn: [generateProtocolClassFilesWithoutLicense, rootProject.spotlessPython]) rootProject.spotlessPython.mustRunAfter(generateProtocolClassFilesWithoutLicense) rootProject.build.dependsOn generateProtocolClassFiles -airbytePythonApply.dependsOn(generateProtocolClassFiles) +installReqs.dependsOn(installLocalReqs, generateProtocolClassFiles) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 4e350cba759cc..b1e30fb3cb507 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -11,4 +11,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs) } -installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs") +installReqs.dependsOn(installLocalReqs, ":airbyte-integrations:bases:airbyte-protocol:installReqs") diff --git a/airbyte-integrations/bases/base-python-test/build.gradle b/airbyte-integrations/bases/base-python-test/build.gradle index 0ede89d115683..a3c1055e636bf 100644 --- a/airbyte-integrations/bases/base-python-test/build.gradle +++ b/airbyte-integrations/bases/base-python-test/build.gradle @@ -11,4 +11,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } -installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs") +installReqs.dependsOn(installLocalReqs, ":airbyte-integrations:bases:airbyte-protocol:installReqs") diff --git a/airbyte-integrations/bases/base-python/build.gradle b/airbyte-integrations/bases/base-python/build.gradle index 8734f5f61a7fd..b9f670eb1555c 100644 --- a/airbyte-integrations/bases/base-python/build.gradle +++ b/airbyte-integrations/bases/base-python/build.gradle @@ -12,4 +12,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs) } -airbytePythonApply.dependsOn(":airbyte-integrations:bases:airbyte-protocol:airbytePythonApply") +installReqs.dependsOn(installLocalReqs, ":airbyte-integrations:bases:airbyte-protocol:installReqs") diff --git a/airbyte-integrations/bases/base-singer/build.gradle b/airbyte-integrations/bases/base-singer/build.gradle index d9314256612c4..1597a8b3c7bba 100644 --- a/airbyte-integrations/bases/base-singer/build.gradle +++ b/airbyte-integrations/bases/base-singer/build.gradle @@ -11,4 +11,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } -installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs", ":airbyte-integrations:bases:base-python:installReqs") +installReqs.dependsOn(installLocalReqs, ":airbyte-integrations:bases:airbyte-protocol:installReqs", ":airbyte-integrations:bases:base-python:installReqs") From a23606d9527017c8a3d22084f3dd883024213d99 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 11:37:05 -0800 Subject: [PATCH 04/20] remove cacheIf forcing --- buildSrc/src/main/groovy/airbyte-python.gradle | 3 --- 1 file changed, 3 deletions(-) diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 9f3b17dd3cd53..f7017f234dfe2 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -48,7 +48,6 @@ class AirbytePythonPlugin implements Plugin { command = "install -r requirements.txt" inputs.file('requirements.txt') outputs.file('build/installedlocalreqs.txt') - outputs.cacheIf { true } // HACK: makes all integrations depend on installing requirements for bases. long term we should resolve deps and install in order. if(project.getPath().startsWith(":airbyte-integrations:connectors")) { @@ -61,7 +60,6 @@ class AirbytePythonPlugin implements Plugin { command = "install .[main]" inputs.file('setup.py') outputs.file('build/installedreqs.txt') - outputs.cacheIf { true } } project.task('installTestReqs', type: PythonTask, dependsOn: project.installReqs) { @@ -69,7 +67,6 @@ class AirbytePythonPlugin implements Plugin { command = "install .[tests]" inputs.file('setup.py') outputs.file('build/installedtestreqs.txt') - outputs.cacheIf { true } } if(project.file('unit_tests').exists()) { From 8095db325b7ce169228ab27d8591fcb352d0dbb6 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 11:51:10 -0800 Subject: [PATCH 05/20] it wasn't clobbering actually --- airbyte-integrations/bases/airbyte-protocol/build.gradle | 2 +- airbyte-integrations/bases/base-normalization/build.gradle | 2 +- airbyte-integrations/bases/base-python-test/build.gradle | 2 +- airbyte-integrations/bases/base-python/build.gradle | 2 +- airbyte-integrations/bases/base-singer/build.gradle | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/bases/airbyte-protocol/build.gradle b/airbyte-integrations/bases/airbyte-protocol/build.gradle index 914a05b22a4cc..983907a46c89d 100644 --- a/airbyte-integrations/bases/airbyte-protocol/build.gradle +++ b/airbyte-integrations/bases/airbyte-protocol/build.gradle @@ -17,4 +17,4 @@ task generateProtocolClassFilesWithoutLicense(type: Exec) { task generateProtocolClassFiles(dependsOn: [generateProtocolClassFilesWithoutLicense, rootProject.spotlessPython]) rootProject.spotlessPython.mustRunAfter(generateProtocolClassFilesWithoutLicense) rootProject.build.dependsOn generateProtocolClassFiles -installReqs.dependsOn(installLocalReqs, generateProtocolClassFiles) +installReqs.dependsOn(generateProtocolClassFiles) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index b1e30fb3cb507..4e350cba759cc 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -11,4 +11,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs) } -installReqs.dependsOn(installLocalReqs, ":airbyte-integrations:bases:airbyte-protocol:installReqs") +installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs") diff --git a/airbyte-integrations/bases/base-python-test/build.gradle b/airbyte-integrations/bases/base-python-test/build.gradle index a3c1055e636bf..0ede89d115683 100644 --- a/airbyte-integrations/bases/base-python-test/build.gradle +++ b/airbyte-integrations/bases/base-python-test/build.gradle @@ -11,4 +11,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } -installReqs.dependsOn(installLocalReqs, ":airbyte-integrations:bases:airbyte-protocol:installReqs") +installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs") diff --git a/airbyte-integrations/bases/base-python/build.gradle b/airbyte-integrations/bases/base-python/build.gradle index b9f670eb1555c..8e2555cf9e76c 100644 --- a/airbyte-integrations/bases/base-python/build.gradle +++ b/airbyte-integrations/bases/base-python/build.gradle @@ -12,4 +12,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs) } -installReqs.dependsOn(installLocalReqs, ":airbyte-integrations:bases:airbyte-protocol:installReqs") +installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs") diff --git a/airbyte-integrations/bases/base-singer/build.gradle b/airbyte-integrations/bases/base-singer/build.gradle index 1597a8b3c7bba..d9314256612c4 100644 --- a/airbyte-integrations/bases/base-singer/build.gradle +++ b/airbyte-integrations/bases/base-singer/build.gradle @@ -11,4 +11,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } -installReqs.dependsOn(installLocalReqs, ":airbyte-integrations:bases:airbyte-protocol:installReqs", ":airbyte-integrations:bases:base-python:installReqs") +installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs", ":airbyte-integrations:bases:base-python:installReqs") From a6c147d6f7fdc15fe8e372e6ad015de1fb8df221 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 12:21:58 -0800 Subject: [PATCH 06/20] fix pip version --- buildSrc/src/main/groovy/airbyte-python.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index f7017f234dfe2..030ca2c0837a6 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -23,6 +23,7 @@ class AirbytePythonPlugin implements Plugin { pip 'mypy:0.790' pip 'isort:5.6.4' pip 'pytest:6.1.2' + pip 'pip:20.3' } project.task('blackFormat', type: PythonTask) { From 5a4eb3bc08ace6748636d53b696fac1f6cffec49 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 12:47:55 -0800 Subject: [PATCH 07/20] loosen pyyaml version --- airbyte-integrations/bases/airbyte-protocol/setup.py | 2 +- airbyte-integrations/bases/base-python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/airbyte-protocol/setup.py b/airbyte-integrations/bases/airbyte-protocol/setup.py index 0a1239d5623d2..93727f9ac5eff 100644 --- a/airbyte-integrations/bases/airbyte-protocol/setup.py +++ b/airbyte-integrations/bases/airbyte-protocol/setup.py @@ -32,5 +32,5 @@ url="https://github.com/airbytehq/airbyte", packages=setuptools.find_packages(), package_data={"": ["models/yaml/*.yaml"]}, - install_requires=["PyYAML==5.3.1", "pydantic==1.6.1"], + install_requires=["PyYAML>=4.2b1", "pydantic==1.6.1"], ) diff --git a/airbyte-integrations/bases/base-python/setup.py b/airbyte-integrations/bases/base-python/setup.py index aecdc63eb7c65..1fb1751ff5edc 100644 --- a/airbyte-integrations/bases/base-python/setup.py +++ b/airbyte-integrations/bases/base-python/setup.py @@ -32,7 +32,7 @@ url="https://github.com/airbytehq/airbyte", packages=setuptools.find_packages(), package_data={"": ["models/yaml/*.yaml"]}, - install_requires=["PyYAML==5.3.1", "pydantic==1.6.1", "airbyte-protocol"], + install_requires=["PyYAML>=4.2b1", "pydantic==1.6.1", "airbyte-protocol"], entry_points={ "console_scripts": ["base-python=base_python.entrypoint:main"], }, From 2652600fe2eeff9f67eb68440a1e0e1e1d2ef885 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 12:48:37 -0800 Subject: [PATCH 08/20] Revert "loosen pyyaml version" This reverts commit 5a4eb3bc08ace6748636d53b696fac1f6cffec49. --- airbyte-integrations/bases/airbyte-protocol/setup.py | 2 +- airbyte-integrations/bases/base-python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/airbyte-protocol/setup.py b/airbyte-integrations/bases/airbyte-protocol/setup.py index 93727f9ac5eff..0a1239d5623d2 100644 --- a/airbyte-integrations/bases/airbyte-protocol/setup.py +++ b/airbyte-integrations/bases/airbyte-protocol/setup.py @@ -32,5 +32,5 @@ url="https://github.com/airbytehq/airbyte", packages=setuptools.find_packages(), package_data={"": ["models/yaml/*.yaml"]}, - install_requires=["PyYAML>=4.2b1", "pydantic==1.6.1"], + install_requires=["PyYAML==5.3.1", "pydantic==1.6.1"], ) diff --git a/airbyte-integrations/bases/base-python/setup.py b/airbyte-integrations/bases/base-python/setup.py index 1fb1751ff5edc..aecdc63eb7c65 100644 --- a/airbyte-integrations/bases/base-python/setup.py +++ b/airbyte-integrations/bases/base-python/setup.py @@ -32,7 +32,7 @@ url="https://github.com/airbytehq/airbyte", packages=setuptools.find_packages(), package_data={"": ["models/yaml/*.yaml"]}, - install_requires=["PyYAML>=4.2b1", "pydantic==1.6.1", "airbyte-protocol"], + install_requires=["PyYAML==5.3.1", "pydantic==1.6.1", "airbyte-protocol"], entry_points={ "console_scripts": ["base-python=base_python.entrypoint:main"], }, From da3ab27259deb3400a662bf355da08537a4fb4f2 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 13:17:16 -0800 Subject: [PATCH 09/20] move tap-adwords dependency --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-google-adwords-singer/Dockerfile | 3 ++- .../connectors/source-google-adwords-singer/build.gradle | 7 +++++++ .../connectors/source-google-adwords-singer/setup.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 26bb14f8c34ca..e8e1683ddc136 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -11,7 +11,7 @@ - sourceDefinitionId: fdc8b827-3257-4b33-83cc-106d234c34d4 name: Google Adwords dockerRepository: airbyte/source-google-adwords-singer - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://hub.docker.com/r/airbyte/source-google-adwords - sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e name: Github diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile index 3e27ea5f0a13e..0b86e60b5abdb 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile @@ -9,7 +9,8 @@ ENV AIRBYTE_IMPL_PATH="SourceGoogleAdwordsSinger" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ +RUN pip install tap-adwords==1.12.0 RUN pip install ".[main]" -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-google-adwords-singer diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle index 6106950749cf3..91ff396b7d6e7 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle +++ b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle @@ -12,3 +12,10 @@ airbytePython { dependencies { implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } + +// used to allow local iteration to work +task('installSingerTap', type: PythonTask) { + module = "pip" + command = "install tap-adwords==1.12.0" +} +installReqs.dependsOn installSingerTap diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/setup.py b/airbyte-integrations/connectors/source-google-adwords-singer/setup.py index 897acedd30b5e..5eecc2bd7999f 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/setup.py +++ b/airbyte-integrations/connectors/source-google-adwords-singer/setup.py @@ -38,7 +38,7 @@ # Dependencies required by the main package but not integration tests should go in main. Deps required by # integration tests but not the main package go in integration_tests. Deps required by both should go in # install_requires. - "main": ["base-singer", "base-python", "tap-adwords==1.12.0"], + "main": ["base-singer", "base-python"], "tests": ["airbyte-python-test", "pytest"], }, ) From 256cf555e856a3f2f67e907d249736e76bf01a61 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 13:36:35 -0800 Subject: [PATCH 10/20] downgrade shopify api version used to match --- airbyte-integrations/connectors/source-shopify-singer/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-shopify-singer/setup.py b/airbyte-integrations/connectors/source-shopify-singer/setup.py index 5ea1aa41b8b77..77ecac80441dd 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/setup.py +++ b/airbyte-integrations/connectors/source-shopify-singer/setup.py @@ -30,7 +30,7 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "tap-shopify==1.2.6", "ShopifyAPI==8.0.4"], + install_requires=["airbyte-protocol", "tap-shopify==1.2.6", "ShopifyAPI==8.0.1"], package_data={"": ["*.json"]}, setup_requires=["pytest-runner"], tests_require=["pytest"], From c06482fb2a6038320592821fe6e15a0f1910d8dd Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 14:03:17 -0800 Subject: [PATCH 11/20] run seed init --- .../932e6363-d006-4464-a9f5-102b82e07c06.json | 7 ------- .../ec4b9503-13cb-48ab-a4ab-6ade4be46567.json | 7 ------- .../fdc8b827-3257-4b33-83cc-106d234c34d4.json | 2 +- 3 files changed, 1 insertion(+), 15 deletions(-) delete mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json delete mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json deleted file mode 100644 index f3d4e29804038..0000000000000 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "sourceDefinitionId": "932e6363-d006-4464-a9f5-102b82e07c06", - "name": "Twilio", - "dockerRepository": "airbyte/source-twilio-singer", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://hub.docker.com/r/airbyte/source-twilio-singer" -} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json deleted file mode 100644 index 6026bda249167..0000000000000 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "sourceDefinitionId": "ec4b9503-13cb-48ab-a4ab-6ade4be46567", - "name": "Freshdesk", - "dockerRepository": "airbyte/source-freshdesk", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://hub.docker.com/r/airbyte/source-freshdesk" -} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json index a8e281c3c2ffe..383d92f827fbb 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "fdc8b827-3257-4b33-83cc-106d234c34d4", "name": "Google Adwords", "dockerRepository": "airbyte/source-google-adwords-singer", - "dockerImageTag": "0.1.3", + "dockerImageTag": "0.1.4", "documentationUrl": "https://hub.docker.com/r/airbyte/source-google-adwords" } From caee9ee9eefd623716cb1269aa72372e4baca5de Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 14:45:12 -0800 Subject: [PATCH 12/20] add test curl --- .github/workflows/gradle.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index dc991db3d1787..7a7ca9d55f1e1 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -66,6 +66,9 @@ jobs: with: python-version: '3.7' + - name: check connectivity + run: curl https://repository.mulesoft.org/nexus/content/repositories/public/com/amazon/redshift/redshift-jdbc42/1.2.43.1067/redshift-jdbc42-1.2.43.1067.pom + - name: Start local Docker registry run: docker run -d -p 5000:5000 --restart=always --name registry -v /tmp/docker-registry:/var/lib/registry registry:2 && npx wait-on tcp:5000 From 84f58be0482e688ef55bc24e52e5bb942dcac9cc Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 14:48:07 -0800 Subject: [PATCH 13/20] Revert "add test curl" This reverts commit caee9ee9eefd623716cb1269aa72372e4baca5de. --- .github/workflows/gradle.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 7a7ca9d55f1e1..dc991db3d1787 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -66,9 +66,6 @@ jobs: with: python-version: '3.7' - - name: check connectivity - run: curl https://repository.mulesoft.org/nexus/content/repositories/public/com/amazon/redshift/redshift-jdbc42/1.2.43.1067/redshift-jdbc42-1.2.43.1067.pom - - name: Start local Docker registry run: docker run -d -p 5000:5000 --restart=always --name registry -v /tmp/docker-registry:/var/lib/registry registry:2 && npx wait-on tcp:5000 From 64071fc5ea48b6d201ab0c1317272c32c21eaab3 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 15:19:23 -0800 Subject: [PATCH 14/20] revert to simpler docker times --- .github/workflows/gradle.yml | 14 -------------- tools/bin/build_image.sh | 22 ++++------------------ 2 files changed, 4 insertions(+), 32 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index dc991db3d1787..df4132f16bff4 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -15,17 +15,6 @@ jobs: - name: Check images exist run: ./tools/bin/check_images_exist.sh -# # docker should use restore-keys since these are just used for layer caching, not gradle task caching -# - name: Docker Caching -# uses: actions/cache@v2 -# with: -# path: | -# /tmp/docker-registry -# key: ${{ secrets.CACHE_VERSION }}-docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }}-${{ github.sha }} -# restore-keys: | -# ${{ secrets.CACHE_VERSION }}-docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }}- -# ${{ secrets.CACHE_VERSION }}-docker-${{ runner.os }}- - - name: Pip Caching uses: actions/cache@v2 with: @@ -66,9 +55,6 @@ jobs: with: python-version: '3.7' - - name: Start local Docker registry - run: docker run -d -p 5000:5000 --restart=always --name registry -v /tmp/docker-registry:/var/lib/registry registry:2 && npx wait-on tcp:5000 - - name: Build run: ./gradlew --no-daemon build --scan diff --git a/tools/bin/build_image.sh b/tools/bin/build_image.sh index 712f6914b0a7f..0812efacdf0a4 100755 --- a/tools/bin/build_image.sh +++ b/tools/bin/build_image.sh @@ -14,21 +14,7 @@ assert_root cd "$PROJECT_DIR" -if [[ -z "$CI" ]]; then - # run standard build locally (not on CI) - DOCKER_BUILDKIT=1 docker build \ - -f "$DOCKERFILE" . \ - -t "$TAGGED_IMAGE" \ - --iidfile "$ID_FILE" -else - # run build with local docker registery for CI - docker pull localhost:5000/"$TAGGED_IMAGE" || true - DOCKER_BUILDKIT=1 docker build \ - -f "$DOCKERFILE" . \ - -t "$TAGGED_IMAGE" \ - --iidfile "$ID_FILE" \ - --cache-from localhost:5000/"$TAGGED_IMAGE" \ - --build-arg BUILDKIT_INLINE_CACHE=1 - docker tag "$TAGGED_IMAGE" localhost:5000/"$TAGGED_IMAGE" - docker push localhost:5000/"$TAGGED_IMAGE" -fi +DOCKER_BUILDKIT=1 docker build \ + -f "$DOCKERFILE" . \ + -t "$TAGGED_IMAGE" \ + --iidfile "$ID_FILE" From 81f1fc1f561e365c75d2738e053e297c4bd871d7 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 15:22:19 -0800 Subject: [PATCH 15/20] rename task function so it doesn't match the task name --- buildSrc/src/main/groovy/airbyte-docker.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/airbyte-docker.gradle b/buildSrc/src/main/groovy/airbyte-docker.gradle index 908c4e0ccf03f..5d69ab86e96fc 100644 --- a/buildSrc/src/main/groovy/airbyte-docker.gradle +++ b/buildSrc/src/main/groovy/airbyte-docker.gradle @@ -44,7 +44,7 @@ abstract class AirbyteDockerTask extends DefaultTask { } @TaskAction - def airbyteDocker() { + def dockerTask() { def scriptPath = Paths.get(rootDir.absolutePath, 'tools/bin/build_image.sh').toString() buildDockerfile(scriptPath, dockerfileName) } From f08fd370be55dd4726c8c3f07c9e550525fc8386 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 15:50:15 -0800 Subject: [PATCH 16/20] turn off buildkit --- tools/bin/build_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/build_image.sh b/tools/bin/build_image.sh index 0812efacdf0a4..fd8df42390815 100755 --- a/tools/bin/build_image.sh +++ b/tools/bin/build_image.sh @@ -14,7 +14,7 @@ assert_root cd "$PROJECT_DIR" -DOCKER_BUILDKIT=1 docker build \ +docker build \ -f "$DOCKERFILE" . \ -t "$TAGGED_IMAGE" \ --iidfile "$ID_FILE" From f4bfe21f4d9658f47ab709aa6eb84cbb6199793e Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 16:17:03 -0800 Subject: [PATCH 17/20] fix redshift dir for credentials --- tools/bin/ci_credentials.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index ebd5ef6c40651..6c434f25e1691 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -49,7 +49,7 @@ echo "$BIGQUERY_INTEGRATION_TEST_CREDS" > "${SOURCEFILE_DIR}/gcs.json" echo "$AWS_S3_INTEGRATION_TEST_CREDS" > "${SOURCEFILE_DIR}/aws.json" REDSHIFT_DIR=airbyte-integrations/connectors/source-redshift/secrets -mkdir REDSHIFT_DIR +mkdir $REDSHIFT_DIR echo "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" > "${REDSHIFT_DIR}/config.json" MAILCHIMP_SECRETS_DIR=airbyte-integrations/connectors/source-mailchimp/secrets From dbc779d4e2d1dd2c50ea86764b293b29c3fd2dbb Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 16:23:30 -0800 Subject: [PATCH 18/20] add twilio and freshdesk to seed yaml --- .../932e6363-d006-4464-a9f5-102b82e07c0.json | 7 +++++++ .../ec4b9503-13cb-48ab-a4ab-6ade4be46567.json | 7 +++++++ .../src/main/resources/seed/source_definitions.yaml | 10 ++++++++++ 3 files changed, 24 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c0.json create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c0.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c0.json new file mode 100644 index 0000000000000..e9841b28906d2 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c0.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "932e6363-d006-4464-a9f5-102b82e07c0", + "name": "Twilio", + "dockerRepository": "airbyte/source-twilio-singer", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-twilio-singer" +} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json new file mode 100644 index 0000000000000..6026bda249167 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "ec4b9503-13cb-48ab-a4ab-6ade4be46567", + "name": "Freshdesk", + "dockerRepository": "airbyte/source-freshdesk", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-freshdesk" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e8e1683ddc136..0f805f8a80670 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -98,3 +98,13 @@ dockerRepository: airbyte/source-redshift dockerImageTag: 0.1.0 documentationUrl: https://hub.docker.com/repository/docker/airbyte/source-redshift +- sourceDefinitionId: 932e6363-d006-4464-a9f5-102b82e07c0 + name: Twilio + dockerRepository: airbyte/source-twilio-singer + dockerImageTag: 0.1.0 + documentationUrl: https://hub.docker.com/r/airbyte/source-twilio-singer +- sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 + name: Freshdesk + dockerRepository: airbyte/source-freshdesk + dockerImageTag: 0.1.0 + documentationUrl: https://hub.docker.com/r/airbyte/source-freshdesk From afe7299656b75d232de8c5754e7719ac9ab783c7 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 16:24:32 -0800 Subject: [PATCH 19/20] fix typo --- ...2b82e07c0.json => 932e6363-d006-4464-a9f5-102b82e07c06.json} | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/{932e6363-d006-4464-a9f5-102b82e07c0.json => 932e6363-d006-4464-a9f5-102b82e07c06.json} (74%) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c0.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json similarity index 74% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c0.json rename to airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json index e9841b28906d2..f3d4e29804038 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c0.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json @@ -1,5 +1,5 @@ { - "sourceDefinitionId": "932e6363-d006-4464-a9f5-102b82e07c0", + "sourceDefinitionId": "932e6363-d006-4464-a9f5-102b82e07c06", "name": "Twilio", "dockerRepository": "airbyte/source-twilio-singer", "dockerImageTag": "0.1.0", diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0f805f8a80670..b183a8ef90e16 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -98,7 +98,7 @@ dockerRepository: airbyte/source-redshift dockerImageTag: 0.1.0 documentationUrl: https://hub.docker.com/repository/docker/airbyte/source-redshift -- sourceDefinitionId: 932e6363-d006-4464-a9f5-102b82e07c0 +- sourceDefinitionId: 932e6363-d006-4464-a9f5-102b82e07c06 name: Twilio dockerRepository: airbyte/source-twilio-singer dockerImageTag: 0.1.0 From 011a911d664a1673d15096dc2acf713f839c0d71 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 1 Dec 2020 16:41:56 -0800 Subject: [PATCH 20/20] get shopify passing similar to the adwords fix --- .../connectors/source-shopify-singer/Dockerfile | 2 ++ .../connectors/source-shopify-singer/Dockerfile.test | 2 ++ .../connectors/source-shopify-singer/build.gradle | 7 +++++++ .../connectors/source-shopify-singer/setup.py | 2 +- 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile index 07430e0ef774f..d622580ae2173 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile @@ -10,6 +10,8 @@ ENV AIRBYTE_IMPL_PATH="SourceShopifySinger" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ +RUN pip install pip==20.3 +RUN pip install tap-shopify==1.2.6 RUN pip install ".[main]" LABEL io.airbyte.version=0.1.4 diff --git a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test index f35afaa184ee3..f7ba3a37e5fd6 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test +++ b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test @@ -18,6 +18,8 @@ COPY secrets/config.json $CODE_PATH/config.json COPY $MODULE_NAME/*.json $CODE_PATH/ COPY setup.py ./ +RUN pip install pip==20.3 +RUN pip install tap-shopify==1.2.6 RUN pip install ".[tests]" WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-shopify-singer/build.gradle b/airbyte-integrations/connectors/source-shopify-singer/build.gradle index a8e72915bd8e3..9e8473d2f7cf6 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/build.gradle +++ b/airbyte-integrations/connectors/source-shopify-singer/build.gradle @@ -11,3 +11,10 @@ airbytePython { dependencies { implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } + +// used to allow local iteration to work +task('installSingerTap', type: PythonTask) { + module = "pip" + command = "install tap-shopify==1.2.6" +} +installReqs.dependsOn installSingerTap diff --git a/airbyte-integrations/connectors/source-shopify-singer/setup.py b/airbyte-integrations/connectors/source-shopify-singer/setup.py index 77ecac80441dd..0606711670f34 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/setup.py +++ b/airbyte-integrations/connectors/source-shopify-singer/setup.py @@ -30,7 +30,7 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "tap-shopify==1.2.6", "ShopifyAPI==8.0.1"], + install_requires=["airbyte-protocol", "ShopifyAPI==8.0.4"], package_data={"": ["*.json"]}, setup_requires=["pytest-runner"], tests_require=["pytest"],