enforce order of installing python dependencies (#1149)

* fix python install order * fix base install reqs dependencies * don't clobber dependencies * remove cacheIf forcing * it wasn't clobbering actually * fix pip version * loosen pyyaml version * Revert "loosen pyyaml version" This reverts commit 5a4eb3b. * move tap-adwords dependency * downgrade shopify api version used to match * run seed init * add test curl * Revert "add test curl" This reverts commit caee9ee. * revert to simpler docker times * rename task function so it doesn't match the task name * turn off buildkit * fix redshift dir for credentials * add twilio and freshdesk to seed yaml * fix typo * get shopify passing similar to the adwords fix
airbytehq · Dec 2, 2020 · 674ee2c · 674ee2c
1 parent 0017675
commit 674ee2c
Show file tree

Hide file tree

Showing 19 changed files with 55 additions and 42 deletions.
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -15,17 +15,6 @@ jobs:
       - name: Check images exist
         run: ./tools/bin/check_images_exist.sh
 
-#      # docker should use restore-keys since these are just used for layer caching, not gradle task caching
-#      - name: Docker Caching
-#        uses: actions/cache@v2
-#        with:
-#          path: |
-#            /tmp/docker-registry
-#          key: ${{ secrets.CACHE_VERSION }}-docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }}-${{ github.sha }}
-#          restore-keys: |
-#            ${{ secrets.CACHE_VERSION }}-docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }}-
-#            ${{ secrets.CACHE_VERSION }}-docker-${{ runner.os }}-
-
       - name: Pip Caching
         uses: actions/cache@v2
         with:
@@ -66,9 +55,6 @@ jobs:
         with:
           python-version: '3.7'
 
-      - name: Start local Docker registry
-        run: docker run -d -p 5000:5000 --restart=always --name registry -v /tmp/docker-registry:/var/lib/registry registry:2 && npx wait-on tcp:5000
-
       - name: Build
         run: ./gradlew --no-daemon build --scan
 

diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "fdc8b827-3257-4b33-83cc-106d234c34d4",
   "name": "Google Adwords",
   "dockerRepository": "airbyte/source-google-adwords-singer",
-  "dockerImageTag": "0.1.3",
+  "dockerImageTag": "0.1.4",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-google-adwords"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -11,7 +11,7 @@
 - sourceDefinitionId: fdc8b827-3257-4b33-83cc-106d234c34d4
   name: Google Adwords
   dockerRepository: airbyte/source-google-adwords-singer
-  dockerImageTag: 0.1.3
+  dockerImageTag: 0.1.4
   documentationUrl: https://hub.docker.com/r/airbyte/source-google-adwords
 - sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   name: Github
@@ -98,3 +98,13 @@
   dockerRepository: airbyte/source-redshift
   dockerImageTag: 0.1.0
   documentationUrl: https://hub.docker.com/repository/docker/airbyte/source-redshift
+- sourceDefinitionId: 932e6363-d006-4464-a9f5-102b82e07c06
+  name: Twilio
+  dockerRepository: airbyte/source-twilio-singer
+  dockerImageTag: 0.1.0
+  documentationUrl: https://hub.docker.com/r/airbyte/source-twilio-singer
+- sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567
+  name: Freshdesk
+  dockerRepository: airbyte/source-freshdesk
+  dockerImageTag: 0.1.0
+  documentationUrl: https://hub.docker.com/r/airbyte/source-freshdesk
diff --git a/airbyte-integrations/bases/airbyte-protocol/build.gradle b/airbyte-integrations/bases/airbyte-protocol/build.gradle
@@ -17,3 +17,4 @@ task generateProtocolClassFilesWithoutLicense(type: Exec) {
 task generateProtocolClassFiles(dependsOn: [generateProtocolClassFilesWithoutLicense, rootProject.spotlessPython])
 rootProject.spotlessPython.mustRunAfter(generateProtocolClassFilesWithoutLicense)
 rootProject.build.dependsOn generateProtocolClassFiles
+installReqs.dependsOn(generateProtocolClassFiles)
diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle
@@ -10,3 +10,5 @@ airbytePython {
 dependencies {
     implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs)
 }
+
+installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs")
diff --git a/airbyte-integrations/bases/base-python-test/build.gradle b/airbyte-integrations/bases/base-python-test/build.gradle
@@ -10,3 +10,5 @@ airbytePython {
 dependencies {
     implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
 }
+
+installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs")
diff --git a/airbyte-integrations/bases/base-python/build.gradle b/airbyte-integrations/bases/base-python/build.gradle
@@ -11,3 +11,5 @@ dependencies {
     implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs)
     implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs)
 }
+
+installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs")
diff --git a/airbyte-integrations/bases/base-singer/build.gradle b/airbyte-integrations/bases/base-singer/build.gradle
@@ -10,3 +10,5 @@ airbytePython {
 dependencies {
     implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
 }
+
+installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs", ":airbyte-integrations:bases:base-python:installReqs")
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile
@@ -9,7 +9,8 @@ ENV AIRBYTE_IMPL_PATH="SourceGoogleAdwordsSinger"
 WORKDIR /airbyte/integration_code
 COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
+RUN pip install tap-adwords==1.12.0
 RUN pip install ".[main]"
 
-LABEL io.airbyte.version=0.1.3
+LABEL io.airbyte.version=0.1.4
 LABEL io.airbyte.name=airbyte/source-google-adwords-singer
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle
@@ -12,3 +12,10 @@ airbytePython {
 dependencies {
     implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs)
 }
+
+// used to allow local iteration to work
+task('installSingerTap', type: PythonTask) {
+    module = "pip"
+    command = "install tap-adwords==1.12.0"
+}
+installReqs.dependsOn installSingerTap
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/setup.py b/airbyte-integrations/connectors/source-google-adwords-singer/setup.py
@@ -38,7 +38,7 @@
         # Dependencies required by the main package but not integration tests should go in main. Deps required by
         # integration tests but not the main package go in integration_tests. Deps required by both should go in
         # install_requires.
-        "main": ["base-singer", "base-python", "tap-adwords==1.12.0"],
+        "main": ["base-singer", "base-python"],
         "tests": ["airbyte-python-test", "pytest"],
     },
 )
diff --git a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile
@@ -10,6 +10,8 @@ ENV AIRBYTE_IMPL_PATH="SourceShopifySinger"
 WORKDIR /airbyte/integration_code
 COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
+RUN pip install pip==20.3
+RUN pip install tap-shopify==1.2.6
 RUN pip install ".[main]"
 
 LABEL io.airbyte.version=0.1.4

diff --git a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test
@@ -18,6 +18,8 @@ COPY secrets/config.json $CODE_PATH/config.json
 COPY $MODULE_NAME/*.json $CODE_PATH/
 COPY setup.py ./
 
+RUN pip install pip==20.3
+RUN pip install tap-shopify==1.2.6
 RUN pip install ".[tests]"
 
 WORKDIR /airbyte
diff --git a/airbyte-integrations/connectors/source-shopify-singer/build.gradle b/airbyte-integrations/connectors/source-shopify-singer/build.gradle
@@ -11,3 +11,10 @@ airbytePython {
 dependencies {
     implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs)
 }
+
+// used to allow local iteration to work
+task('installSingerTap', type: PythonTask) {
+    module = "pip"
+    command = "install tap-shopify==1.2.6"
+}
+installReqs.dependsOn installSingerTap
diff --git a/airbyte-integrations/connectors/source-shopify-singer/setup.py b/airbyte-integrations/connectors/source-shopify-singer/setup.py
@@ -30,7 +30,7 @@
     author="Airbyte",
     author_email="[email protected]",
     packages=find_packages(),
-    install_requires=["airbyte-protocol", "tap-shopify==1.2.6", "ShopifyAPI==8.0.4"],
+    install_requires=["airbyte-protocol", "ShopifyAPI==8.0.4"],
     package_data={"": ["*.json"]},
     setup_requires=["pytest-runner"],
     tests_require=["pytest"],

diff --git a/buildSrc/src/main/groovy/airbyte-docker.gradle b/buildSrc/src/main/groovy/airbyte-docker.gradle
@@ -44,7 +44,7 @@ abstract class AirbyteDockerTask extends DefaultTask {
     }
 
     @TaskAction
-    def airbyteDocker() {
+    def dockerTask() {
         def scriptPath = Paths.get(rootDir.absolutePath, 'tools/bin/build_image.sh').toString()
         buildDockerfile(scriptPath, dockerfileName)
     }

diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle
@@ -23,6 +23,7 @@ class AirbytePythonPlugin implements Plugin<Project> {
             pip 'mypy:0.790'
             pip 'isort:5.6.4'
             pip 'pytest:6.1.2'
+            pip 'pip:20.3'
         }
 
         project.task('blackFormat', type: PythonTask) {
@@ -48,23 +49,25 @@ class AirbytePythonPlugin implements Plugin<Project> {
             command = "install -r requirements.txt"
             inputs.file('requirements.txt')
             outputs.file('build/installedlocalreqs.txt')
-            outputs.cacheIf { true }
+
+            // HACK: makes all integrations depend on installing requirements for bases. long term we should resolve deps and install in order.
+            if(project.getPath().startsWith(":airbyte-integrations:connectors")) {
+                dependsOn project.rootProject.getTasksByName("airbytePythonApply", true).findAll { it.project.getPath().startsWith(":airbyte-integrations:bases")}
+            }
         }
 
         project.task('installReqs', type: PythonTask, dependsOn: project.installLocalReqs) {
             module = "pip"
             command = "install .[main]"
             inputs.file('setup.py')
             outputs.file('build/installedreqs.txt')
-            outputs.cacheIf { true }
         }
 
         project.task('installTestReqs', type: PythonTask, dependsOn: project.installReqs) {
             module = "pip"
             command = "install .[tests]"
             inputs.file('setup.py')
             outputs.file('build/installedtestreqs.txt')
-            outputs.cacheIf { true }
         }
 
         if(project.file('unit_tests').exists()) {

diff --git a/tools/bin/build_image.sh b/tools/bin/build_image.sh
@@ -14,21 +14,7 @@ assert_root
 
 cd "$PROJECT_DIR"
 
-if [[ -z "$CI" ]]; then
-  # run standard build locally (not on CI)
-  DOCKER_BUILDKIT=1 docker build \
-    -f "$DOCKERFILE" . \
-    -t "$TAGGED_IMAGE" \
-    --iidfile "$ID_FILE"
-else
-  # run build with local docker registery for CI
-  docker pull localhost:5000/"$TAGGED_IMAGE" || true
-  DOCKER_BUILDKIT=1 docker build \
-    -f "$DOCKERFILE" . \
-    -t "$TAGGED_IMAGE" \
-    --iidfile "$ID_FILE" \
-    --cache-from localhost:5000/"$TAGGED_IMAGE" \
-    --build-arg BUILDKIT_INLINE_CACHE=1
-  docker tag "$TAGGED_IMAGE" localhost:5000/"$TAGGED_IMAGE"
-  docker push localhost:5000/"$TAGGED_IMAGE"
-fi
+docker build \
+  -f "$DOCKERFILE" . \
+  -t "$TAGGED_IMAGE" \
+  --iidfile "$ID_FILE"
diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh
@@ -49,7 +49,7 @@ echo "$BIGQUERY_INTEGRATION_TEST_CREDS" > "${SOURCEFILE_DIR}/gcs.json"
 echo "$AWS_S3_INTEGRATION_TEST_CREDS" > "${SOURCEFILE_DIR}/aws.json"
 
 REDSHIFT_DIR=airbyte-integrations/connectors/source-redshift/secrets
-mkdir REDSHIFT_DIR
+mkdir $REDSHIFT_DIR
 echo "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" > "${REDSHIFT_DIR}/config.json"
 
 MAILCHIMP_SECRETS_DIR=airbyte-integrations/connectors/source-mailchimp/secrets