From 0f06d95adad7b4153572b3ac46d6a01dd3cd2a05 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 29 Oct 2021 14:21:09 -0700 Subject: [PATCH 01/38] Remove the use of docker compose to build --- airbyte-config/init/build.gradle | 11 +++++ airbyte-db/lib/build.gradle | 11 +++++ airbyte-migration/build.gradle | 9 ++++ airbyte-scheduler/app/build.gradle | 11 +++++ airbyte-server/build.gradle | 11 +++++ airbyte-webapp/build.gradle | 10 +++++ airbyte-workers/build.gradle | 10 +++++ build.gradle | 70 +++++++++++++----------------- 8 files changed, 103 insertions(+), 40 deletions(-) diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 945051223ea9f..110b308655a03 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -1,5 +1,8 @@ +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage + plugins { id 'java' + id 'com.bmuschko.docker-remote-api' } dependencies { @@ -7,3 +10,11 @@ dependencies { implementation project(':airbyte-config:models') } + +def buildTag = System.getenv('VERSION') ?: 'dev' +// Use task types +task buildMyAppImage(type: DockerBuildImage) { + inputDir = file('.') + images.add("airbyte/init:$buildTag") +} +build.dependsOn(buildMyAppImage) diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 1d3d5dd2d3312..f2b0e895bcde2 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -1,5 +1,8 @@ +import com.bmuschko.gradle.docker.tasks.image.* + plugins { id 'java-library' + id 'com.bmuschko.docker-remote-api' } dependencies { @@ -69,3 +72,11 @@ task(dumpJobsSchema, dependsOn: 'classes', type: JavaExec) { classpath = sourceSets.main.runtimeClasspath args 'jobs', 'dump_schema' } + +def buildTag = System.getenv('VERSION') ?: 'dev' +// Use task types +task buildMyAppImage(type: DockerBuildImage) { + inputDir = file('.') + images.add("airbyte/db:$buildTag") +} +build.dependsOn(buildMyAppImage) diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index ac752cf739c7d..556a3255ad6d2 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -1,5 +1,8 @@ +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage + plugins { id 'application' + id 'com.bmuschko.docker-remote-api' } dependencies { @@ -15,3 +18,9 @@ application { mainClass = 'io.airbyte.migrate.MigrationRunner' } +// Use task types +task buildMyAppImage(type: DockerBuildImage) { + inputDir = file('.') + images.add("airbyte/migration:$version") +} +build.dependsOn(buildMyAppImage) diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index ddd6a0cd06996..840f90ea39410 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -1,5 +1,8 @@ +import com.bmuschko.gradle.docker.tasks.image.* + plugins { id 'application' + id 'com.bmuschko.docker-remote-api' } dependencies { @@ -57,3 +60,11 @@ run { environment "TEMPORAL_HOST", "localhost:7233" } + +def buildTag = System.getenv('VERSION') ?: 'dev' +// Use task types +task buildMyAppImage(type: DockerBuildImage) { + inputDir = file('.') + images.add("airbyte/scheduler:$buildTag") +} +build.dependsOn(buildMyAppImage) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index edc7c55fb2506..d883b1e06bb20 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -1,7 +1,10 @@ +import com.bmuschko.gradle.docker.tasks.image.* + plugins { id 'application' id 'maven-publish' id 'com.github.johnrengelman.shadow' version '7.1.0' + id 'com.bmuschko.docker-remote-api' } shadowJar { @@ -126,3 +129,11 @@ run { environment "AIRBYTE_ROLE", System.getenv('AIRBYTE_ROLE') environment "TEMPORAL_HOST", "localhost:7233" } + +def buildTag = System.getenv('VERSION') ?: 'dev' +// Use task types +task buildMyAppImage(type: DockerBuildImage) { + inputDir = file('.') + images.add("airbyte/server:$buildTag") +} +build.dependsOn(buildMyAppImage) diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index 1ea5a0ed0dbc8..7b6d349e13453 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -1,6 +1,9 @@ +import com.bmuschko.gradle.docker.tasks.image.* + plugins { id "base" id "com.github.node-gradle.node" version "3.1.1" + id 'com.bmuschko.docker-remote-api' } def nodeVersion = System.getenv('NODE_VERSION') ?: '14.11.0' @@ -40,3 +43,10 @@ task copyDocs(type: Copy) { copyDocs.dependsOn npm_run_build assemble.dependsOn copyDocs + +// Use task types +task buildMyAppImage(type: DockerBuildImage) { + inputDir = file('.') + images.add("airbyte/webapp:$version") +} +build.dependsOn(buildMyAppImage) diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index b1a2f9a38afd5..7c177213573f0 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -1,9 +1,11 @@ import org.jsonschema2pojo.SourceType +import com.bmuschko.gradle.docker.tasks.image.* plugins { id 'application' id 'com.github.eirnym.js2p' version '1.0' id 'airbyte-integration-test-java' + id 'com.bmuschko.docker-remote-api' } configurations { @@ -62,3 +64,11 @@ application { mainClass = mainClassName applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } + +def buildTag = System.getenv('VERSION') ?: 'dev' +// Use task types +task buildMyAppImage(type: DockerBuildImage) { + inputDir = file('.') + images.add("airbyte/worker:$buildTag") +} +build.dependsOn(buildMyAppImage) diff --git a/build.gradle b/build.gradle index 271b27c23feb4..5b0a96620600b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,14 @@ +buildscript { + repositories { + maven { + url "https://plugins.gradle.org/m2/" + } + } + dependencies { + classpath 'com.bmuschko:gradle-docker-plugin:7.1.0' + } +} + plugins { id 'base' id 'pmd' @@ -58,22 +69,22 @@ def createJavaLicenseWith = { license -> // monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable. def createSpotlessTarget = { pattern -> def excludes = [ - '.gradle', - 'node_modules', - '.eggs', - '.mypy_cache', - '.venv', - '*.egg-info', - 'build', - 'dbt-project-template', - 'dbt-project-template-mssql', - 'dbt-project-template-mysql', - 'dbt-project-template-oracle', - 'dbt_test_config', - 'normalization_test_output', - 'tools', - 'secrets', - 'charts' // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. + '.gradle', + 'node_modules', + '.eggs', + '.mypy_cache', + '.venv', + '*.egg-info', + 'build', + 'dbt-project-template', + 'dbt-project-template-mssql', + 'dbt-project-template-mysql', + 'dbt-project-template-oracle', + 'dbt_test_config', + 'normalization_test_output', + 'tools', + 'secrets', + 'charts' // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. ] if (System.getenv().containsKey("SUB_BUILD")) { @@ -125,9 +136,9 @@ allprojects { ":airbyte-webapp", ].toSet().asImmutable() - if (project.getPath() in composeDeps) { + /*if (project.getPath() in composeDeps) { composeBuild.dependsOn(project.getPath() + ':assemble') - } + }*/ } } @@ -233,6 +244,7 @@ subprojects { testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2' testImplementation 'org.mockito:mockito-junit-jupiter:3.12.4' testImplementation 'org.assertj:assertj-core:3.21.0' + } tasks.withType(Tar) { @@ -244,28 +256,6 @@ subprojects { } } -task composeBuild { - def buildTag = System.getenv('VERSION') ?: 'dev' - def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64' - def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64' - def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2' - def dockerComposeFile = buildArch == 'arm64' ? 'docker-compose.build-m1.yaml' : 'docker-compose.build.yaml' - doFirst { - exec { - workingDir rootDir - commandLine 'docker-compose', '-f', dockerComposeFile, 'build', '--parallel', '--quiet' - environment 'VERSION', buildTag - environment 'DOCKER_BUILD_PLATFORM', buildPlatform - environment 'DOCKER_BUILD_ARCH', buildArch - environment 'JDK_VERSION', jdkVersion - } - } -} - -if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { - build.dependsOn(composeBuild) -} - task('generate') { dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) } dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } From e85f1a9023b45b1c6c783fefe8e22015612ca502 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 29 Oct 2021 14:23:29 -0700 Subject: [PATCH 02/38] Update .dockerignore --- airbyte-config/init/.dockerignore | 1 + airbyte-db/lib/.dockerignore | 1 + 2 files changed, 2 insertions(+) diff --git a/airbyte-config/init/.dockerignore b/airbyte-config/init/.dockerignore index 5ad9d43099b67..8c942441de7dc 100644 --- a/airbyte-config/init/.dockerignore +++ b/airbyte-config/init/.dockerignore @@ -1,3 +1,4 @@ * !src !scripts +!Dockerfile diff --git a/airbyte-db/lib/.dockerignore b/airbyte-db/lib/.dockerignore index 7a1eba35d5be1..859d61d1d02aa 100644 --- a/airbyte-db/lib/.dockerignore +++ b/airbyte-db/lib/.dockerignore @@ -1,2 +1,3 @@ * !src +!Dockerfile From dbbb2987d56bc9fb10902c4ae42e6c54a632a3de Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 29 Oct 2021 15:06:20 -0700 Subject: [PATCH 03/38] Add explicit dependencies --- airbyte-config/init/build.gradle | 3 +++ airbyte-db/lib/build.gradle | 3 +++ airbyte-migration/build.gradle | 6 ++++++ airbyte-scheduler/app/build.gradle | 6 ++++++ airbyte-server/build.gradle | 11 +++++++++++ airbyte-webapp/build.gradle | 4 ++++ airbyte-workers/build.gradle | 6 ++++++ 7 files changed, 39 insertions(+) diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 110b308655a03..efe14044ff497 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -14,6 +14,9 @@ dependencies { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn compileTestJava + dependsOn jar + dependsOn processTestResources inputDir = file('.') images.add("airbyte/init:$buildTag") } diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index f2b0e895bcde2..5656114afbdbb 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -76,6 +76,9 @@ task(dumpJobsSchema, dependsOn: 'classes', type: JavaExec) { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn compileTestJava + dependsOn jar + dependsOn processTestResources inputDir = file('.') images.add("airbyte/db:$buildTag") } diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index 556a3255ad6d2..cf214ce2a0ff1 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -20,6 +20,12 @@ application { // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn compileTestJava + dependsOn distTar + dependsOn distZip + dependsOn jar + dependsOn processTestResources + dependsOn startScripts inputDir = file('.') images.add("airbyte/migration:$version") } diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index 840f90ea39410..94df3b630297d 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -64,6 +64,12 @@ run { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn compileTestJava + dependsOn distTar + dependsOn distZip + dependsOn jar + dependsOn processTestResources + dependsOn startScripts inputDir = file('.') images.add("airbyte/scheduler:$buildTag") } diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index d883b1e06bb20..fc2c915efdcce 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -133,6 +133,17 @@ run { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn shadowJar + dependsOn compileTestJava + dependsOn copySeed + dependsOn distTar + dependsOn distZip + dependsOn jar + dependsOn processTestResources + dependsOn shadowDistTar + dependsOn shadowDistZip + dependsOn startScripts + dependsOn startShadowScripts inputDir = file('.') images.add("airbyte/server:$buildTag") } diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index 7b6d349e13453..1c2d721f8d6bd 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -46,6 +46,10 @@ assemble.dependsOn copyDocs // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyDocs + dependsOn nodeSetup + dependsOn npmInstall + dependsOn npm_run_build inputDir = file('.') images.add("airbyte/webapp:$version") } diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 7c177213573f0..7ca35e2bc3e59 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -68,6 +68,12 @@ application { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn compileTestJava + dependsOn distTar + dependsOn distZip + dependsOn jar + dependsOn processTestResources + dependsOn startScripts inputDir = file('.') images.add("airbyte/worker:$buildTag") } From 9b00dc5a4c023035184dad05deb7ed50d4b9ba25 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 29 Oct 2021 16:30:19 -0700 Subject: [PATCH 04/38] Allign .dockerfileingnore, rm implicit dependencies --- airbyte-config/init/.dockerignore | 1 + airbyte-db/lib/.dockerignore | 1 + airbyte-migration/.dockerignore | 1 + airbyte-scheduler/app/.dockerignore | 1 + airbyte-server/.dockerignore | 1 + airbyte-workers/.dockerignore | 4 ++++ airbyte-workers/build.gradle | 5 ++++- 7 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 airbyte-workers/.dockerignore diff --git a/airbyte-config/init/.dockerignore b/airbyte-config/init/.dockerignore index 8c942441de7dc..ad9cd62e34bfe 100644 --- a/airbyte-config/init/.dockerignore +++ b/airbyte-config/init/.dockerignore @@ -1,4 +1,5 @@ * !src !scripts +!build !Dockerfile diff --git a/airbyte-db/lib/.dockerignore b/airbyte-db/lib/.dockerignore index 859d61d1d02aa..bfe0a3f7813c5 100644 --- a/airbyte-db/lib/.dockerignore +++ b/airbyte-db/lib/.dockerignore @@ -1,3 +1,4 @@ * !src !Dockerfile +!build diff --git a/airbyte-migration/.dockerignore b/airbyte-migration/.dockerignore index 65c7d0ad3e73c..bfe0a3f7813c5 100644 --- a/airbyte-migration/.dockerignore +++ b/airbyte-migration/.dockerignore @@ -1,3 +1,4 @@ * +!src !Dockerfile !build diff --git a/airbyte-scheduler/app/.dockerignore b/airbyte-scheduler/app/.dockerignore index 65c7d0ad3e73c..bfe0a3f7813c5 100644 --- a/airbyte-scheduler/app/.dockerignore +++ b/airbyte-scheduler/app/.dockerignore @@ -1,3 +1,4 @@ * +!src !Dockerfile !build diff --git a/airbyte-server/.dockerignore b/airbyte-server/.dockerignore index 65c7d0ad3e73c..bfe0a3f7813c5 100644 --- a/airbyte-server/.dockerignore +++ b/airbyte-server/.dockerignore @@ -1,3 +1,4 @@ * +!src !Dockerfile !build diff --git a/airbyte-workers/.dockerignore b/airbyte-workers/.dockerignore new file mode 100644 index 0000000000000..bfe0a3f7813c5 --- /dev/null +++ b/airbyte-workers/.dockerignore @@ -0,0 +1,4 @@ +* +!src +!Dockerfile +!build diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 7ca35e2bc3e59..73ebd689166b2 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -74,7 +74,10 @@ task buildMyAppImage(type: DockerBuildImage) { dependsOn jar dependsOn processTestResources dependsOn startScripts + dependsOn compileIntegrationTestJavaJava + dependsOn test + dependsOn jacocoTestReport inputDir = file('.') images.add("airbyte/worker:$buildTag") } -build.dependsOn(buildMyAppImage) +assemble.dependsOn(buildMyAppImage) From 0f049e7fd03d43207fbac675b92e0696691be483 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 09:15:25 -0700 Subject: [PATCH 05/38] improve scheduler gradle --- airbyte-scheduler/app/build.gradle | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index 94df3b630297d..d265bad75a85e 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -64,13 +64,19 @@ run { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn processResources + dependsOn processTestResources + dependsOn compileJava dependsOn compileTestJava - dependsOn distTar - dependsOn distZip + dependsOn test dependsOn jar - dependsOn processTestResources dependsOn startScripts + dependsOn distTar + dependsOn distZip + dependsOn jacocoTestReport inputDir = file('.') images.add("airbyte/scheduler:$buildTag") + noCache = false + quiet = false } -build.dependsOn(buildMyAppImage) +assemble.dependsOn(buildMyAppImage) From 8957cd7d9ef5dd1814cf60d9d2ed1f622b99069d Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 09:24:57 -0700 Subject: [PATCH 06/38] Clean up migration gradle file --- airbyte-migration/build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index cf214ce2a0ff1..572d5d69d2613 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -21,10 +21,13 @@ application { // Use task types task buildMyAppImage(type: DockerBuildImage) { dependsOn compileTestJava + dependsOn processResources + dependsOn processTestResources + dependsOn test + dependsOn jacocoTestReport dependsOn distTar dependsOn distZip dependsOn jar - dependsOn processTestResources dependsOn startScripts inputDir = file('.') images.add("airbyte/migration:$version") From e3dd24698dc7fd549221f1f3a7f4a103a072d148 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 10:09:34 -0700 Subject: [PATCH 07/38] Clean up server gradle --- airbyte-server/build.gradle | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index fc2c915efdcce..0cf42803961db 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -133,18 +133,27 @@ run { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copySeed + dependsOn processResources + dependsOn processTestResources + dependsOn jar dependsOn shadowJar dependsOn compileTestJava - dependsOn copySeed dependsOn distTar dependsOn distZip - dependsOn jar - dependsOn processTestResources dependsOn shadowDistTar dependsOn shadowDistZip dependsOn startScripts dependsOn startShadowScripts + dependsOn test + dependsOn jacocoTestReport inputDir = file('.') images.add("airbyte/server:$buildTag") } build.dependsOn(buildMyAppImage) +startScripts.dependsOn(shadowJar) +distTar.dependsOn(shadowJar) +distZip.dependsOn(shadowJar) +startShadowScripts.dependsOn(jar) +shadowDistTar.dependsOn(jar) +shadowDistZip.dependsOn(jar) From 5ab190cb490434678ab31f91f61a48f01b65c7a3 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 11:17:40 -0700 Subject: [PATCH 08/38] tmp --- airbyte-db/lib/.dockerignore | 2 +- airbyte-db/lib/build.gradle | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-db/lib/.dockerignore b/airbyte-db/lib/.dockerignore index bfe0a3f7813c5..d22de23543cd6 100644 --- a/airbyte-db/lib/.dockerignore +++ b/airbyte-db/lib/.dockerignore @@ -1,4 +1,4 @@ * !src +!docker !Dockerfile -!build diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 5656114afbdbb..a83bf3c1d57a9 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -79,7 +79,9 @@ task buildMyAppImage(type: DockerBuildImage) { dependsOn compileTestJava dependsOn jar dependsOn processTestResources + dependsOn jacocoTestReport inputDir = file('.') +// dockerFile = file('Dockerfile') images.add("airbyte/db:$buildTag") } build.dependsOn(buildMyAppImage) From 7d32b90b56f9a207135c1d680c98bed38a1425aa Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 13:54:19 -0700 Subject: [PATCH 09/38] Look at diff --- airbyte-db/lib/Dockerfile | 3 --- airbyte-db/lib/build.gradle | 13 ++++++++++--- airbyte-db/lib/docker/Dockerfile | 3 +++ airbyte-db/lib/docker/init.sql | 7 +++++++ airbyte-scheduler/app/build.gradle | 13 ++++++++++--- airbyte-scheduler/app/{ => docker}/Dockerfile | 2 +- 6 files changed, 31 insertions(+), 10 deletions(-) delete mode 100644 airbyte-db/lib/Dockerfile create mode 100644 airbyte-db/lib/docker/Dockerfile create mode 100644 airbyte-db/lib/docker/init.sql rename airbyte-scheduler/app/{ => docker}/Dockerfile (82%) diff --git a/airbyte-db/lib/Dockerfile b/airbyte-db/lib/Dockerfile deleted file mode 100644 index a2feafc85de83..0000000000000 --- a/airbyte-db/lib/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM postgres:13-alpine - -COPY src/main/resources/init.sql /docker-entrypoint-initdb.d/000_init.sql diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index a83bf3c1d57a9..406ac6afe2dfb 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -73,15 +73,22 @@ task(dumpJobsSchema, dependsOn: 'classes', type: JavaExec) { args 'jobs', 'dump_schema' } +task copyGeneratedXml(type: Copy) { + from('src/main/resources') { + include 'init.sql' + } + into 'docker' +} + def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyGeneratedXml dependsOn compileTestJava dependsOn jar dependsOn processTestResources dependsOn jacocoTestReport - inputDir = file('.') -// dockerFile = file('Dockerfile') + inputDir = file('docker') images.add("airbyte/db:$buildTag") } -build.dependsOn(buildMyAppImage) +assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-db/lib/docker/Dockerfile b/airbyte-db/lib/docker/Dockerfile new file mode 100644 index 0000000000000..66b86d877b013 --- /dev/null +++ b/airbyte-db/lib/docker/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:13-alpine + +COPY init.sql /docker-entrypoint-initdb.d/000_init.sql diff --git a/airbyte-db/lib/docker/init.sql b/airbyte-db/lib/docker/init.sql new file mode 100644 index 0000000000000..f32874a59ba2d --- /dev/null +++ b/airbyte-db/lib/docker/init.sql @@ -0,0 +1,7 @@ +CREATE + DATABASE airbyte; + +\connect airbyte; + +GRANT ALL ON +DATABASE airbyte TO docker; diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index d265bad75a85e..28974da8bff53 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -61,9 +61,18 @@ run { } +task copyGeneratedTar(type: Copy) { + from('build/distributions') { + include 'airbyte-scheduler-*.tar' + } + into 'docker' +} + + def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyGeneratedTar dependsOn processResources dependsOn processTestResources dependsOn compileJava @@ -74,9 +83,7 @@ task buildMyAppImage(type: DockerBuildImage) { dependsOn distTar dependsOn distZip dependsOn jacocoTestReport - inputDir = file('.') + inputDir = file('docker') images.add("airbyte/scheduler:$buildTag") - noCache = false - quiet = false } assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-scheduler/app/Dockerfile b/airbyte-scheduler/app/docker/Dockerfile similarity index 82% rename from airbyte-scheduler/app/Dockerfile rename to airbyte-scheduler/app/docker/Dockerfile index 8984019c79a17..ad5b893d5c404 100644 --- a/airbyte-scheduler/app/Dockerfile +++ b/airbyte-scheduler/app/docker/Dockerfile @@ -5,7 +5,7 @@ ENV APPLICATION airbyte-scheduler WORKDIR /app -COPY build/distributions/${APPLICATION}-0*.tar ${APPLICATION}.tar +COPY ${APPLICATION}-0*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 From d693c5c7172239fa6f0454e26d4bfe0c29d1f30d Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 13:58:42 -0700 Subject: [PATCH 10/38] rm unwanted files --- airbyte-db/lib/docker/init.sql | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 airbyte-db/lib/docker/init.sql diff --git a/airbyte-db/lib/docker/init.sql b/airbyte-db/lib/docker/init.sql deleted file mode 100644 index f32874a59ba2d..0000000000000 --- a/airbyte-db/lib/docker/init.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE - DATABASE airbyte; - -\connect airbyte; - -GRANT ALL ON -DATABASE airbyte TO docker; From 80840315936daf29946d422aa655876fbecc72ca Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 14:02:55 -0700 Subject: [PATCH 11/38] Make sure that we don't commit file that will be copied to the docker container --- .gitignore | 1 + airbyte-db/lib/build.gradle | 2 +- airbyte-db/lib/docker/Dockerfile | 2 +- airbyte-scheduler/app/build.gradle | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 45316ca315b54..b200d1f25029f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ data .classpath .project .settings +bin # Logs acceptance_tests_logs/ diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 406ac6afe2dfb..9dac35d141685 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -77,7 +77,7 @@ task copyGeneratedXml(type: Copy) { from('src/main/resources') { include 'init.sql' } - into 'docker' + into 'docker/bin' } def buildTag = System.getenv('VERSION') ?: 'dev' diff --git a/airbyte-db/lib/docker/Dockerfile b/airbyte-db/lib/docker/Dockerfile index 66b86d877b013..a9f42ce521ba1 100644 --- a/airbyte-db/lib/docker/Dockerfile +++ b/airbyte-db/lib/docker/Dockerfile @@ -1,3 +1,3 @@ FROM postgres:13-alpine -COPY init.sql /docker-entrypoint-initdb.d/000_init.sql +COPY bin/init.sql /docker-entrypoint-initdb.d/000_init.sql diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index 28974da8bff53..310eb653a8a09 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -68,7 +68,6 @@ task copyGeneratedTar(type: Copy) { into 'docker' } - def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { From e63c0f2c3f365bd1cbffad439dcdd996247cb5c4 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 14:10:06 -0700 Subject: [PATCH 12/38] Make the scheduler docker build to be increamental --- airbyte-scheduler/app/build.gradle | 4 +++- airbyte-scheduler/app/docker/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index 310eb653a8a09..9bf43c75b9f4c 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -62,10 +62,12 @@ run { } task copyGeneratedTar(type: Copy) { + dependsOn distTar + from('build/distributions') { include 'airbyte-scheduler-*.tar' } - into 'docker' + into 'docker/bin' } def buildTag = System.getenv('VERSION') ?: 'dev' diff --git a/airbyte-scheduler/app/docker/Dockerfile b/airbyte-scheduler/app/docker/Dockerfile index ad5b893d5c404..203a18f1728a3 100644 --- a/airbyte-scheduler/app/docker/Dockerfile +++ b/airbyte-scheduler/app/docker/Dockerfile @@ -5,7 +5,7 @@ ENV APPLICATION airbyte-scheduler WORKDIR /app -COPY ${APPLICATION}-0*.tar ${APPLICATION}.tar +COPY "bin/${APPLICATION}-0*.tar" ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 From 1b6bc9bdaf7b5bc9d92d0eedb7c2aea350b35ecf Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 14:24:37 -0700 Subject: [PATCH 13/38] Make the worker docker build to be incremental --- airbyte-workers/build.gradle | 12 +++++++++++- airbyte-workers/{ => docker}/Dockerfile | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) rename airbyte-workers/{ => docker}/Dockerfile (95%) diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 73ebd689166b2..f38faab9c4850 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -65,9 +65,19 @@ application { applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } +task copyGeneratedTar(type: Copy) { + dependsOn distTar + + from('build/distributions') { + include 'airbyte-workers-*.tar' + } + into 'docker/bin' +} + def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyGeneratedTar dependsOn compileTestJava dependsOn distTar dependsOn distZip @@ -77,7 +87,7 @@ task buildMyAppImage(type: DockerBuildImage) { dependsOn compileIntegrationTestJavaJava dependsOn test dependsOn jacocoTestReport - inputDir = file('.') + inputDir = file('docker') images.add("airbyte/worker:$buildTag") } assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/docker/Dockerfile similarity index 95% rename from airbyte-workers/Dockerfile rename to airbyte-workers/docker/Dockerfile index 513bfc194030d..4c3a1b52293db 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/docker/Dockerfile @@ -28,7 +28,7 @@ RUN chmod +x ./kubectl RUN mv ./kubectl /usr/local/bin # Move and run worker -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar +COPY bin/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 From 3b9efe96965d70330270a9d4347d9e657c67c2d3 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 14:36:03 -0700 Subject: [PATCH 14/38] make a the server docker build to be incremental --- airbyte-server/build.gradle | 15 +++++++++++++-- airbyte-server/{ => docker}/Dockerfile | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) rename airbyte-server/{ => docker}/Dockerfile (82%) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 0cf42803961db..6aad126dd6890 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -130,9 +130,19 @@ run { environment "TEMPORAL_HOST", "localhost:7233" } +task copyGeneratedTar(type: Copy) { + dependsOn distTar + + from('build/distributions') { + include 'airbyte-server-*.tar' + } + into 'docker/bin' +} + def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyGeneratedTar dependsOn copySeed dependsOn processResources dependsOn processTestResources @@ -147,13 +157,14 @@ task buildMyAppImage(type: DockerBuildImage) { dependsOn startShadowScripts dependsOn test dependsOn jacocoTestReport - inputDir = file('.') + inputDir = file('docker') images.add("airbyte/server:$buildTag") } -build.dependsOn(buildMyAppImage) +assemble.dependsOn(buildMyAppImage) startScripts.dependsOn(shadowJar) distTar.dependsOn(shadowJar) distZip.dependsOn(shadowJar) startShadowScripts.dependsOn(jar) shadowDistTar.dependsOn(jar) shadowDistZip.dependsOn(jar) +copyGeneratedTar.dependsOn(shadowDistTar) diff --git a/airbyte-server/Dockerfile b/airbyte-server/docker/Dockerfile similarity index 82% rename from airbyte-server/Dockerfile rename to airbyte-server/docker/Dockerfile index 57d0dbd262cbc..fb9180a64c4f8 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/docker/Dockerfile @@ -7,7 +7,7 @@ ENV APPLICATION airbyte-server WORKDIR /app -COPY build/distributions/${APPLICATION}-0*.tar ${APPLICATION}.tar +COPY bin/${APPLICATION}-0*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 From 5b3f566602a005bc98f239fb3418d271a9f1551e Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 14:54:35 -0700 Subject: [PATCH 15/38] Make the airbyte-migration to be incremental --- airbyte-migration/build.gradle | 13 ++++++++++++- airbyte-migration/{ => docker}/Dockerfile | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) rename airbyte-migration/{ => docker}/Dockerfile (83%) diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index 572d5d69d2613..a70f04751bf5e 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -18,8 +18,19 @@ application { mainClass = 'io.airbyte.migrate.MigrationRunner' } +task copyGeneratedTar(type: Copy) { + dependsOn distTar + + from('build/distributions') { + include 'airbyte-migration-*.tar' + } + into 'docker/bin' +} + + // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyGeneratedTar dependsOn compileTestJava dependsOn processResources dependsOn processTestResources @@ -29,7 +40,7 @@ task buildMyAppImage(type: DockerBuildImage) { dependsOn distZip dependsOn jar dependsOn startScripts - inputDir = file('.') + inputDir = file('docker') images.add("airbyte/migration:$version") } build.dependsOn(buildMyAppImage) diff --git a/airbyte-migration/Dockerfile b/airbyte-migration/docker/Dockerfile similarity index 83% rename from airbyte-migration/Dockerfile rename to airbyte-migration/docker/Dockerfile index 8a657e07ee5bb..c17ca492065d8 100644 --- a/airbyte-migration/Dockerfile +++ b/airbyte-migration/docker/Dockerfile @@ -6,7 +6,7 @@ ENV APPLICATION airbyte-migration WORKDIR /app # Move and run scheduler -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar +COPY bin/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 From dee7b079a8f2e96dcc080325ec3d5997d4673b62 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 15:49:09 -0700 Subject: [PATCH 16/38] Make the webapp docker build incremental --- airbyte-webapp/Dockerfile | 9 --------- airbyte-webapp/build.gradle | 23 +++++++++++++++++++---- airbyte-webapp/docker/Dockerfile | 7 +++++++ 3 files changed, 26 insertions(+), 13 deletions(-) delete mode 100644 airbyte-webapp/Dockerfile create mode 100644 airbyte-webapp/docker/Dockerfile diff --git a/airbyte-webapp/Dockerfile b/airbyte-webapp/Dockerfile deleted file mode 100644 index e1054ff5154fa..0000000000000 --- a/airbyte-webapp/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM nginx:1.19-alpine as webapp - -EXPOSE 80 - -COPY build/docs docs/ -# docs get copied twice because npm gradle plugin ignores output dir. -COPY build /usr/share/nginx/html -RUN rm -rf /usr/share/nginx/html/docs -COPY nginx/default.conf.template /etc/nginx/templates/default.conf.template diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index 1c2d721f8d6bd..883e99734f893 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -36,21 +36,36 @@ assemble.dependsOn npm_run_build build.finalizedBy test task copyDocs(type: Copy) { - from "${System.getProperty("user.dir")}/docs/integrations" - into "${buildDir}/docs/integrations" + from "${project.rootProject.projectDir}/docs/integrations" + into "docker/bin/docs/integratbimions" duplicatesStrategy DuplicatesStrategy.INCLUDE } +task copyBuild(type: Copy) { + from "${project.projectDir}/build" + into "docker/bin/build" + exclude ".docker" +} + +task copyNginx(type: Copy) { + from "${project.projectDir}/nginx" + into "docker/bin/nginx" +} + +copyBuild.dependsOn npm_run_build +copyNginx.dependsOn npm_run_build copyDocs.dependsOn npm_run_build assemble.dependsOn copyDocs // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyBuild + dependsOn copyNginx dependsOn copyDocs dependsOn nodeSetup dependsOn npmInstall dependsOn npm_run_build - inputDir = file('.') + inputDir = file('docker') images.add("airbyte/webapp:$version") } -build.dependsOn(buildMyAppImage) +assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-webapp/docker/Dockerfile b/airbyte-webapp/docker/Dockerfile new file mode 100644 index 0000000000000..1eead5631b6fb --- /dev/null +++ b/airbyte-webapp/docker/Dockerfile @@ -0,0 +1,7 @@ +FROM nginx:1.19-alpine as webapp + +EXPOSE 80 + +COPY bin/docs docs/ +COPY bin/build /usr/share/nginx/html +COPY bin/nginx/default.conf.template /etc/nginx/templates/default.conf.template From 2976178df73b38e62782ddc745e7b3f291ef498b Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 15:56:57 -0700 Subject: [PATCH 17/38] Make the config init docker build to be incremental --- airbyte-config/init/build.gradle | 10 ++++++++-- airbyte-config/init/{ => docker}/Dockerfile | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) rename airbyte-config/init/{ => docker}/Dockerfile (87%) diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index efe14044ff497..07e5568d1e654 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -11,13 +11,19 @@ dependencies { implementation project(':airbyte-config:models') } +task copyScripts(type: Copy) { + from('scripts') + into 'docker/bin/scripts' +} + def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyScripts dependsOn compileTestJava dependsOn jar dependsOn processTestResources - inputDir = file('.') + inputDir = file('docker') images.add("airbyte/init:$buildTag") } -build.dependsOn(buildMyAppImage) +assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-config/init/Dockerfile b/airbyte-config/init/docker/Dockerfile similarity index 87% rename from airbyte-config/init/Dockerfile rename to airbyte-config/init/docker/Dockerfile index afba972dedf19..3144a772ef4ef 100644 --- a/airbyte-config/init/Dockerfile +++ b/airbyte-config/init/docker/Dockerfile @@ -4,4 +4,4 @@ WORKDIR /app # the sole purpose of this image is to seed the data volume with the default data # that the app should have when it is first installed. -COPY scripts scripts +COPY bin/scripts scripts From cb538dbb6a89589e6aa01bf3c4853c67751644b5 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 16:01:17 -0700 Subject: [PATCH 18/38] Remove the dockerignorfile --- airbyte-config/init/.dockerignore | 5 ----- airbyte-db/lib/.dockerignore | 4 ---- airbyte-migration/.dockerignore | 4 ---- airbyte-scheduler/app/.dockerignore | 4 ---- airbyte-server/.dockerignore | 4 ---- airbyte-webapp/.dockerignore | 4 ---- airbyte-workers/.dockerignore | 4 ---- 7 files changed, 29 deletions(-) delete mode 100644 airbyte-config/init/.dockerignore delete mode 100644 airbyte-db/lib/.dockerignore delete mode 100644 airbyte-migration/.dockerignore delete mode 100644 airbyte-scheduler/app/.dockerignore delete mode 100644 airbyte-server/.dockerignore delete mode 100644 airbyte-webapp/.dockerignore delete mode 100644 airbyte-workers/.dockerignore diff --git a/airbyte-config/init/.dockerignore b/airbyte-config/init/.dockerignore deleted file mode 100644 index ad9cd62e34bfe..0000000000000 --- a/airbyte-config/init/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -* -!src -!scripts -!build -!Dockerfile diff --git a/airbyte-db/lib/.dockerignore b/airbyte-db/lib/.dockerignore deleted file mode 100644 index d22de23543cd6..0000000000000 --- a/airbyte-db/lib/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!src -!docker -!Dockerfile diff --git a/airbyte-migration/.dockerignore b/airbyte-migration/.dockerignore deleted file mode 100644 index bfe0a3f7813c5..0000000000000 --- a/airbyte-migration/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!src -!Dockerfile -!build diff --git a/airbyte-scheduler/app/.dockerignore b/airbyte-scheduler/app/.dockerignore deleted file mode 100644 index bfe0a3f7813c5..0000000000000 --- a/airbyte-scheduler/app/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!src -!Dockerfile -!build diff --git a/airbyte-server/.dockerignore b/airbyte-server/.dockerignore deleted file mode 100644 index bfe0a3f7813c5..0000000000000 --- a/airbyte-server/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!src -!Dockerfile -!build diff --git a/airbyte-webapp/.dockerignore b/airbyte-webapp/.dockerignore deleted file mode 100644 index b284b9daeb6c5..0000000000000 --- a/airbyte-webapp/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!Dockerfile -!build -!nginx diff --git a/airbyte-workers/.dockerignore b/airbyte-workers/.dockerignore deleted file mode 100644 index bfe0a3f7813c5..0000000000000 --- a/airbyte-workers/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!src -!Dockerfile -!build From fecb366defdd258b329fa080ae7bf29660fb599d Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 16:05:25 -0700 Subject: [PATCH 19/38] remove unused gradle block --- build.gradle | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/build.gradle b/build.gradle index 5b0a96620600b..006c048ed8c0a 100644 --- a/build.gradle +++ b/build.gradle @@ -125,24 +125,6 @@ check.dependsOn 'spotlessApply' allprojects { apply plugin: 'base' - afterEvaluate { project -> - def composeDeps = [ - ":airbyte-config:init", - ":airbyte-db:lib", - ":airbyte-migration", - ":airbyte-scheduler:app", - ":airbyte-workers", - ":airbyte-server", - ":airbyte-webapp", - ].toSet().asImmutable() - - /*if (project.getPath() in composeDeps) { - composeBuild.dependsOn(project.getPath() + ':assemble') - }*/ - } -} - -allprojects { // by default gradle uses directory as the project name. That works very well in a single project environment but // projects clobber each other in an environments with subprojects when projects are in directories named identically. def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.') From 71e79daa58a28b5a12c57bc4c76a98f8573ad503 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 16:38:41 -0700 Subject: [PATCH 20/38] Make the airbyte cli docker build incremental --- airbyte-cli/build.gradle | 12 +++++++++++- airbyte-cli/{ => docker}/Dockerfile | 0 2 files changed, 11 insertions(+), 1 deletion(-) rename airbyte-cli/{ => docker}/Dockerfile (100%) diff --git a/airbyte-cli/build.gradle b/airbyte-cli/build.gradle index 3388338f60a7a..207a4e1bec742 100644 --- a/airbyte-cli/build.gradle +++ b/airbyte-cli/build.gradle @@ -1,3 +1,13 @@ +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage + plugins { - id "airbyte-docker" + id 'com.bmuschko.docker-remote-api' } + +def buildTag = System.getenv('VERSION') ?: 'dev' +// Use task types +task buildMyAppImage(type: DockerBuildImage) { + inputDir = file('docker') + images.add("airbyte/db:$buildTag") +} +assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-cli/Dockerfile b/airbyte-cli/docker/Dockerfile similarity index 100% rename from airbyte-cli/Dockerfile rename to airbyte-cli/docker/Dockerfile From 24a7cf8fc04b0d01071af570dbd9e0acde5e8294 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Nov 2021 16:51:47 -0700 Subject: [PATCH 21/38] Rm the use of composeBuild --- .github/workflows/gradle.yml | 4 ++-- docs/project-overview/changelog/README.md | 3 ++- .../bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh | 2 +- tools/bin/release_version.sh | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index f485acc781ace..a2eb7970b4163 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -339,7 +339,7 @@ jobs: EOF - name: Build Platform Docker Images - run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan + run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan - name: Run End-to-End Frontend Tests run: ./tools/bin/e2e_test.sh @@ -457,7 +457,7 @@ jobs: HOME: /home/runner - name: Build Platform Docker Images - run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan + run: SUB_BUILD=PLATFORM ./gradlew assemble --scan - name: Run Logging Tests run: ./tools/bin/cloud_storage_logging_test.sh diff --git a/docs/project-overview/changelog/README.md b/docs/project-overview/changelog/README.md index 86c8266c239b0..6729332c0004b 100644 --- a/docs/project-overview/changelog/README.md +++ b/docs/project-overview/changelog/README.md @@ -257,7 +257,8 @@ Airbyte is comprised of 2 parts: The "production" version of Airbyte is the version of the app specified in `.env`. With each production release, we update the version in the `.env` file. This version will always be available for download on DockerHub. It is the version of the app that runs when a user runs `docker-compose up`. -The "development" version of Airbyte is the head of master branch. It is the version of the app that runs when a user runs `./gradlew composeBuild && VERSION=dev docker compose up`. +The "development" version of Airbyte is the head of master branch. It is the version of the app that runs when a user runs `./gradlew build && +VERSION=dev docker compose up`. ### Production Release Schedule diff --git a/tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh b/tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh index f796e94bb5fa8..85ee8e4c517bd 100755 --- a/tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh +++ b/tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh @@ -14,7 +14,7 @@ TAG=$(openssl rand -hex 12) echo "Tag" $TAG docker login -u airbytebot -p $DOCKER_PASSWORD -VERSION=$TAG ./gradlew composeBuild +VERSION=$TAG ./gradlew build VERSION=$TAG docker-compose -f docker-compose.build.yaml push # For running on Mac diff --git a/tools/bin/release_version.sh b/tools/bin/release_version.sh index 5cc3d245fcd1c..1ae49cddacc13 100755 --- a/tools/bin/release_version.sh +++ b/tools/bin/release_version.sh @@ -38,7 +38,7 @@ GIT_REVISION=$(git rev-parse HEAD) echo "Bumped version from ${PREV_VERSION} to ${NEW_VERSION}" echo "Building and publishing version $NEW_VERSION for git revision $GIT_REVISION..." -SUB_BUILD=PLATFORM ./gradlew clean composeBuild +SUB_BUILD=PLATFORM ./gradlew clean build SUB_BUILD=PLATFORM ./gradlew publish VERSION=$NEW_VERSION GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose.build.yaml build VERSION=$NEW_VERSION GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose.build.yaml push From c9251d98a4a4a56f5ebbf7099c104819865c311b Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 2 Nov 2021 10:18:59 -0700 Subject: [PATCH 22/38] Fix cli docker image name --- airbyte-cli/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cli/build.gradle b/airbyte-cli/build.gradle index 207a4e1bec742..f725efcc002ab 100644 --- a/airbyte-cli/build.gradle +++ b/airbyte-cli/build.gradle @@ -8,6 +8,6 @@ def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { inputDir = file('docker') - images.add("airbyte/db:$buildTag") + images.add("airbyte/cli:$buildTag") } assemble.dependsOn(buildMyAppImage) From 07d67f6161d970b37916a9d635408a8e0591cc9c Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 2 Nov 2021 10:46:29 -0700 Subject: [PATCH 23/38] Fix migration build --- .bumpversion.cfg | 2 ++ airbyte-migration/build.gradle | 3 ++- airbyte-migration/docker/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fd833ca80ac61..36449b09b1d0a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -10,6 +10,8 @@ serialize = [bumpversion:file:.env] +[bumpversion:file:airbyte-migration/docker/Dockerfile] + [bumpversion:file:airbyte-server/docker/Dockerfile] [bumpversion:file:airbyte-workers/docker/Dockerfile] diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index a70f04751bf5e..e8cc7aa30b67d 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -21,13 +21,14 @@ application { task copyGeneratedTar(type: Copy) { dependsOn distTar + delete 'docker/bin' + from('build/distributions') { include 'airbyte-migration-*.tar' } into 'docker/bin' } - // Use task types task buildMyAppImage(type: DockerBuildImage) { dependsOn copyGeneratedTar diff --git a/airbyte-migration/docker/Dockerfile b/airbyte-migration/docker/Dockerfile index c17ca492065d8..c4c06cf04015a 100644 --- a/airbyte-migration/docker/Dockerfile +++ b/airbyte-migration/docker/Dockerfile @@ -6,7 +6,7 @@ ENV APPLICATION airbyte-migration WORKDIR /app # Move and run scheduler -COPY bin/${APPLICATION}*.tar ${APPLICATION}.tar +COPY bin/${APPLICATION}-0.30.23-alpha.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 From 23ed511c2d5c68bb1e35585003ae1b820969eebf Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 2 Nov 2021 13:28:37 -0700 Subject: [PATCH 24/38] Use the same version than prod --- airbyte-db/lib/build.gradle | 7 +++---- airbyte-migration/build.gradle | 9 --------- airbyte-migration/docker/Dockerfile | 2 +- airbyte-server/build.gradle | 16 ++-------------- airbyte-server/docker/Dockerfile | 4 ++-- airbyte-webapp/build.gradle | 3 --- airbyte-workers/build.gradle | 11 ++--------- airbyte-workers/docker/Dockerfile | 4 ++-- 8 files changed, 12 insertions(+), 44 deletions(-) diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 9dac35d141685..0a0e882eb9d65 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -74,6 +74,9 @@ task(dumpJobsSchema, dependsOn: 'classes', type: JavaExec) { } task copyGeneratedXml(type: Copy) { + + delete 'docker/bin' + from('src/main/resources') { include 'init.sql' } @@ -84,10 +87,6 @@ def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { dependsOn copyGeneratedXml - dependsOn compileTestJava - dependsOn jar - dependsOn processTestResources - dependsOn jacocoTestReport inputDir = file('docker') images.add("airbyte/db:$buildTag") } diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index e8cc7aa30b67d..da043aecb4a2a 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -32,15 +32,6 @@ task copyGeneratedTar(type: Copy) { // Use task types task buildMyAppImage(type: DockerBuildImage) { dependsOn copyGeneratedTar - dependsOn compileTestJava - dependsOn processResources - dependsOn processTestResources - dependsOn test - dependsOn jacocoTestReport - dependsOn distTar - dependsOn distZip - dependsOn jar - dependsOn startScripts inputDir = file('docker') images.add("airbyte/migration:$version") } diff --git a/airbyte-migration/docker/Dockerfile b/airbyte-migration/docker/Dockerfile index c4c06cf04015a..8251f78778c4f 100644 --- a/airbyte-migration/docker/Dockerfile +++ b/airbyte-migration/docker/Dockerfile @@ -6,7 +6,7 @@ ENV APPLICATION airbyte-migration WORKDIR /app # Move and run scheduler -COPY bin/${APPLICATION}-0.30.23-alpha.tar ${APPLICATION}.tar +COPY bin/${APPLICATION}-0.30.25-alpha.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 6aad126dd6890..67be098590c3c 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -133,6 +133,8 @@ run { task copyGeneratedTar(type: Copy) { dependsOn distTar + delete 'docker/bin' + from('build/distributions') { include 'airbyte-server-*.tar' } @@ -143,20 +145,6 @@ def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { dependsOn copyGeneratedTar - dependsOn copySeed - dependsOn processResources - dependsOn processTestResources - dependsOn jar - dependsOn shadowJar - dependsOn compileTestJava - dependsOn distTar - dependsOn distZip - dependsOn shadowDistTar - dependsOn shadowDistZip - dependsOn startScripts - dependsOn startShadowScripts - dependsOn test - dependsOn jacocoTestReport inputDir = file('docker') images.add("airbyte/server:$buildTag") } diff --git a/airbyte-server/docker/Dockerfile b/airbyte-server/docker/Dockerfile index 623960322edb4..d0c47b99860c0 100644 --- a/airbyte-server/docker/Dockerfile +++ b/airbyte-server/docker/Dockerfile @@ -7,7 +7,7 @@ ENV APPLICATION airbyte-server WORKDIR /app -ADD bin/${APPLICATION}-0.30.23-alpha.tar /app +ADD bin/${APPLICATION}-0.30.25-alpha.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.23-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.25-alpha/bin/${APPLICATION}"] diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index 883e99734f893..b650f5780ff24 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -62,9 +62,6 @@ task buildMyAppImage(type: DockerBuildImage) { dependsOn copyBuild dependsOn copyNginx dependsOn copyDocs - dependsOn nodeSetup - dependsOn npmInstall - dependsOn npm_run_build inputDir = file('docker') images.add("airbyte/webapp:$version") } diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index f38faab9c4850..43c6e8087b7b4 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -68,6 +68,8 @@ application { task copyGeneratedTar(type: Copy) { dependsOn distTar + delete 'docker/bin' + from('build/distributions') { include 'airbyte-workers-*.tar' } @@ -78,15 +80,6 @@ def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { dependsOn copyGeneratedTar - dependsOn compileTestJava - dependsOn distTar - dependsOn distZip - dependsOn jar - dependsOn processTestResources - dependsOn startScripts - dependsOn compileIntegrationTestJavaJava - dependsOn test - dependsOn jacocoTestReport inputDir = file('docker') images.add("airbyte/worker:$buildTag") } diff --git a/airbyte-workers/docker/Dockerfile b/airbyte-workers/docker/Dockerfile index 62f14d0b57021..72d9601c8a650 100644 --- a/airbyte-workers/docker/Dockerfile +++ b/airbyte-workers/docker/Dockerfile @@ -28,7 +28,7 @@ RUN chmod +x ./kubectl RUN mv ./kubectl /usr/local/bin # Move worker app -ADD bin/${APPLICATION}-0.30.23-alpha.tar /app +ADD bin/${APPLICATION}-0.30.25-alpha.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.23-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.25-alpha/bin/${APPLICATION}"] From 3acd34e68e3c47d449b9064052853e8bb5e150fe Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 2 Nov 2021 13:39:21 -0700 Subject: [PATCH 25/38] Add missing version update in the airbyte scheduler --- airbyte-scheduler/app/build.gradle | 2 ++ airbyte-scheduler/app/docker/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index 9bf43c75b9f4c..b3cbdc3d9ca21 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -64,6 +64,8 @@ run { task copyGeneratedTar(type: Copy) { dependsOn distTar + delete 'docker/bin' + from('build/distributions') { include 'airbyte-scheduler-*.tar' } diff --git a/airbyte-scheduler/app/docker/Dockerfile b/airbyte-scheduler/app/docker/Dockerfile index 551e17ad52200..f002d00c091ca 100644 --- a/airbyte-scheduler/app/docker/Dockerfile +++ b/airbyte-scheduler/app/docker/Dockerfile @@ -5,7 +5,7 @@ ENV APPLICATION airbyte-scheduler WORKDIR /app -ADD bin/${APPLICATION}-0.30.23-alpha.tar /app +ADD bin/${APPLICATION}-0.30.25-alpha.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.23-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.25-alpha/bin/${APPLICATION}"] From d279ca2a276a5d97a21e14ab6706a25c1845d19a Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 2 Nov 2021 15:02:13 -0700 Subject: [PATCH 26/38] Move all the docker build related files to the build folder --- airbyte-cli/{docker => }/Dockerfile | 0 airbyte-cli/build.gradle | 11 ++++++- airbyte-db/lib/{docker => }/Dockerfile | 0 airbyte-db/lib/build.gradle | 15 +++++++--- airbyte-migration/{docker => }/Dockerfile | 0 airbyte-migration/build.gradle | 15 +++++++--- airbyte-scheduler/app/{docker => }/Dockerfile | 0 airbyte-scheduler/app/build.gradle | 25 +++++++--------- airbyte-server/{docker => }/Dockerfile | 0 airbyte-server/build.gradle | 16 ++++++---- airbyte-webapp/{docker => }/Dockerfile | 0 airbyte-webapp/build.gradle | 29 ++++++++++++++----- airbyte-workers/{docker => }/Dockerfile | 0 airbyte-workers/build.gradle | 16 ++++++---- 14 files changed, 87 insertions(+), 40 deletions(-) rename airbyte-cli/{docker => }/Dockerfile (100%) rename airbyte-db/lib/{docker => }/Dockerfile (100%) rename airbyte-migration/{docker => }/Dockerfile (100%) rename airbyte-scheduler/app/{docker => }/Dockerfile (100%) rename airbyte-server/{docker => }/Dockerfile (100%) rename airbyte-webapp/{docker => }/Dockerfile (100%) rename airbyte-workers/{docker => }/Dockerfile (100%) diff --git a/airbyte-cli/docker/Dockerfile b/airbyte-cli/Dockerfile similarity index 100% rename from airbyte-cli/docker/Dockerfile rename to airbyte-cli/Dockerfile diff --git a/airbyte-cli/build.gradle b/airbyte-cli/build.gradle index f725efcc002ab..f62e01425b510 100644 --- a/airbyte-cli/build.gradle +++ b/airbyte-cli/build.gradle @@ -4,10 +4,19 @@ plugins { id 'com.bmuschko.docker-remote-api' } +task copyDocker(type: Copy) { + delete "build/docker" + + from "${project.projectDir}/Dockerfile" + into "build/docker/" +} + def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { - inputDir = file('docker') + dependsOn copyDocker + + inputDir = file('build/docker') images.add("airbyte/cli:$buildTag") } assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-db/lib/docker/Dockerfile b/airbyte-db/lib/Dockerfile similarity index 100% rename from airbyte-db/lib/docker/Dockerfile rename to airbyte-db/lib/Dockerfile diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 0a0e882eb9d65..6390819118c38 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -73,21 +73,28 @@ task(dumpJobsSchema, dependsOn: 'classes', type: JavaExec) { args 'jobs', 'dump_schema' } -task copyGeneratedXml(type: Copy) { +task copyDocker(type: Copy) { + delete "build/docker" + + from "${project.projectDir}/Dockerfile" + into "build/docker/" +} - delete 'docker/bin' +task copyGeneratedXml(type: Copy) { + dependsOn copyDocker from('src/main/resources') { include 'init.sql' } - into 'docker/bin' + into 'build/docker/bin' } def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyDocker dependsOn copyGeneratedXml - inputDir = file('docker') + inputDir = file('build/docker') images.add("airbyte/db:$buildTag") } assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-migration/docker/Dockerfile b/airbyte-migration/Dockerfile similarity index 100% rename from airbyte-migration/docker/Dockerfile rename to airbyte-migration/Dockerfile diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index da043aecb4a2a..2c51855e112d8 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -18,21 +18,28 @@ application { mainClass = 'io.airbyte.migrate.MigrationRunner' } +task copyDocker(type: Copy) { + delete "build/docker" + + from "${project.projectDir}/Dockerfile" + into "build/docker/" +} + task copyGeneratedTar(type: Copy) { dependsOn distTar - - delete 'docker/bin' + dependsOn copyDocker from('build/distributions') { include 'airbyte-migration-*.tar' } - into 'docker/bin' + into 'build/docker/bin' } // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyDocker dependsOn copyGeneratedTar - inputDir = file('docker') + inputDir = file('build/docker') images.add("airbyte/migration:$version") } build.dependsOn(buildMyAppImage) diff --git a/airbyte-scheduler/app/docker/Dockerfile b/airbyte-scheduler/app/Dockerfile similarity index 100% rename from airbyte-scheduler/app/docker/Dockerfile rename to airbyte-scheduler/app/Dockerfile diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index b3cbdc3d9ca21..cd6131214198b 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -61,32 +61,29 @@ run { } +task copyDocker(type: Copy) { + delete "build/docker" + + from "${project.projectDir}/Dockerfile" + into "build/docker/" +} + task copyGeneratedTar(type: Copy) { + dependsOn copyDocker dependsOn distTar - delete 'docker/bin' - from('build/distributions') { include 'airbyte-scheduler-*.tar' } - into 'docker/bin' + into 'build/docker/bin' } def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { dependsOn copyGeneratedTar - dependsOn processResources - dependsOn processTestResources - dependsOn compileJava - dependsOn compileTestJava - dependsOn test - dependsOn jar - dependsOn startScripts - dependsOn distTar - dependsOn distZip - dependsOn jacocoTestReport - inputDir = file('docker') + dependsOn copyDocker + inputDir = file('build/docker') images.add("airbyte/scheduler:$buildTag") } assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-server/docker/Dockerfile b/airbyte-server/Dockerfile similarity index 100% rename from airbyte-server/docker/Dockerfile rename to airbyte-server/Dockerfile diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 67be098590c3c..ae0c0f1de3ef6 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -130,22 +130,28 @@ run { environment "TEMPORAL_HOST", "localhost:7233" } -task copyGeneratedTar(type: Copy) { - dependsOn distTar +task copyDocker(type: Copy) { + delete "build/docker" + + from "${project.projectDir}/Dockerfile" + into "build/docker/" +} - delete 'docker/bin' +task copyGeneratedTar(type: Copy) { + dependsOn copyDocker from('build/distributions') { include 'airbyte-server-*.tar' } - into 'docker/bin' + into 'build/docker/bin' } def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyDocker dependsOn copyGeneratedTar - inputDir = file('docker') + inputDir = file('build/docker') images.add("airbyte/server:$buildTag") } assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-webapp/docker/Dockerfile b/airbyte-webapp/Dockerfile similarity index 100% rename from airbyte-webapp/docker/Dockerfile rename to airbyte-webapp/Dockerfile diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index b650f5780ff24..a1718a701712e 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -35,21 +35,35 @@ task test(type: NpmTask) { assemble.dependsOn npm_run_build build.finalizedBy test -task copyDocs(type: Copy) { - from "${project.rootProject.projectDir}/docs/integrations" - into "docker/bin/docs/integratbimions" - duplicatesStrategy DuplicatesStrategy.INCLUDE +task copyDocker(type: Copy) { + delete "build/docker" + + from "${project.projectDir}/Dockerfile" + into "build/docker/" } task copyBuild(type: Copy) { + dependsOn copyDocker + from "${project.projectDir}/build" - into "docker/bin/build" + into "build/docker/bin/build" exclude ".docker" + exclude "docker" +} + +task copyDocs(type: Copy) { + dependsOn copyDocker + + from "${project.rootProject.projectDir}/docs/integrations" + into "build/docker/bin/docs/integratbimions" + duplicatesStrategy DuplicatesStrategy.INCLUDE } task copyNginx(type: Copy) { + dependsOn copyDocker + from "${project.projectDir}/nginx" - into "docker/bin/nginx" + into "build/docker/bin/nginx" } copyBuild.dependsOn npm_run_build @@ -59,10 +73,11 @@ assemble.dependsOn copyDocs // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyDocker dependsOn copyBuild dependsOn copyNginx dependsOn copyDocs - inputDir = file('docker') + inputDir = file('build/docker') images.add("airbyte/webapp:$version") } assemble.dependsOn(buildMyAppImage) diff --git a/airbyte-workers/docker/Dockerfile b/airbyte-workers/Dockerfile similarity index 100% rename from airbyte-workers/docker/Dockerfile rename to airbyte-workers/Dockerfile diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 43c6e8087b7b4..16059d5291847 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -65,22 +65,28 @@ application { applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } -task copyGeneratedTar(type: Copy) { - dependsOn distTar +task copyDocker(type: Copy) { + delete "build/docker" + + from "${project.projectDir}/Dockerfile" + into "build/docker/" +} - delete 'docker/bin' +task copyGeneratedTar(type: Copy) { + dependsOn copyDocker from('build/distributions') { include 'airbyte-workers-*.tar' } - into 'docker/bin' + into 'build/docker/bin' } def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyDocker dependsOn copyGeneratedTar - inputDir = file('docker') + inputDir = file('build/docker') images.add("airbyte/worker:$buildTag") } assemble.dependsOn(buildMyAppImage) From d53150c8e2d607b17c4eec5da1506c7ebe79dad6 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 2 Nov 2021 15:07:03 -0700 Subject: [PATCH 27/38] Remove outdated file updates --- .bumpversion.cfg | 8 ++++---- .gitignore | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 36449b09b1d0a..4115418d01aaf 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -10,13 +10,13 @@ serialize = [bumpversion:file:.env] -[bumpversion:file:airbyte-migration/docker/Dockerfile] +[bumpversion:file:airbyte-migration/Dockerfile] -[bumpversion:file:airbyte-server/docker/Dockerfile] +[bumpversion:file:airbyte-server/Dockerfile] -[bumpversion:file:airbyte-workers/docker/Dockerfile] +[bumpversion:file:airbyte-workers/Dockerfile] -[bumpversion:file:airbyte-scheduler/app/docker/Dockerfile] +[bumpversion:file:airbyte-scheduler/app/Dockerfile] [bumpversion:file:airbyte-webapp/package.json] diff --git a/.gitignore b/.gitignore index f7c2adc1e1ac7..c20c705ffe50e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ data .classpath .project .settings -bin # Logs acceptance_tests_logs/ From 0c47eb7de6d339fab0fea05738ba7b50fad46345 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 2 Nov 2021 15:21:04 -0700 Subject: [PATCH 28/38] Fix workers project --- airbyte-workers/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 16059d5291847..887b655ed291d 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -74,6 +74,7 @@ task copyDocker(type: Copy) { task copyGeneratedTar(type: Copy) { dependsOn copyDocker + dependsOn distTar from('build/distributions') { include 'airbyte-workers-*.tar' @@ -86,6 +87,7 @@ def buildTag = System.getenv('VERSION') ?: 'dev' task buildMyAppImage(type: DockerBuildImage) { dependsOn copyDocker dependsOn copyGeneratedTar + dependsOn distTar inputDir = file('build/docker') images.add("airbyte/worker:$buildTag") } From cba2bced8d6a1029cfcb8ca248cf4d161d54caf5 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 2 Nov 2021 15:38:35 -0700 Subject: [PATCH 29/38] update config docker build --- airbyte-config/init/{docker => }/Dockerfile | 0 airbyte-config/init/build.gradle | 15 ++++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) rename airbyte-config/init/{docker => }/Dockerfile (100%) diff --git a/airbyte-config/init/docker/Dockerfile b/airbyte-config/init/Dockerfile similarity index 100% rename from airbyte-config/init/docker/Dockerfile rename to airbyte-config/init/Dockerfile diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 07e5568d1e654..9ad0f994c65ce 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -11,19 +11,24 @@ dependencies { implementation project(':airbyte-config:models') } +task copyDocker(type: Copy) { + delete "build/docker" + + from "${project.projectDir}/Dockerfile" + into "build/docker/" +} + task copyScripts(type: Copy) { from('scripts') - into 'docker/bin/scripts' + into 'build/docker/bin/scripts' } def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { + dependsOn copyDocker dependsOn copyScripts - dependsOn compileTestJava - dependsOn jar - dependsOn processTestResources - inputDir = file('docker') + inputDir = file('build/docker') images.add("airbyte/init:$buildTag") } assemble.dependsOn(buildMyAppImage) From fa899f9a4577184876118e480ce5df7a99b9c623 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 2 Nov 2021 16:57:37 -0700 Subject: [PATCH 30/38] Pr comments and factorization example --- airbyte-cli/build.gradle | 23 +---------------------- airbyte-config/init/build.gradle | 2 +- airbyte-db/lib/build.gradle | 10 +++++----- airbyte-migration/build.gradle | 4 ++-- airbyte-scheduler/app/build.gradle | 6 +++--- airbyte-server/build.gradle | 13 +++---------- airbyte-webapp/build.gradle | 8 ++++---- airbyte-workers/build.gradle | 6 +++--- build.gradle | 29 +++++++++++++++++++++++++++++ 9 files changed, 51 insertions(+), 50 deletions(-) diff --git a/airbyte-cli/build.gradle b/airbyte-cli/build.gradle index f62e01425b510..10889e3ffdea9 100644 --- a/airbyte-cli/build.gradle +++ b/airbyte-cli/build.gradle @@ -1,22 +1 @@ -import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage - -plugins { - id 'com.bmuschko.docker-remote-api' -} - -task copyDocker(type: Copy) { - delete "build/docker" - - from "${project.projectDir}/Dockerfile" - into "build/docker/" -} - -def buildTag = System.getenv('VERSION') ?: 'dev' -// Use task types -task buildMyAppImage(type: DockerBuildImage) { - dependsOn copyDocker - - inputDir = file('build/docker') - images.add("airbyte/cli:$buildTag") -} -assemble.dependsOn(buildMyAppImage) +assemble.dependsOn(buildDockerImageTest) diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 9ad0f994c65ce..cb7a464ac0ca3 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -26,8 +26,8 @@ task copyScripts(type: Copy) { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types task buildMyAppImage(type: DockerBuildImage) { - dependsOn copyDocker dependsOn copyScripts + dependsOn copyDocker inputDir = file('build/docker') images.add("airbyte/init:$buildTag") } diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 6390819118c38..feacc9c1f9460 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -1,4 +1,4 @@ -import com.bmuschko.gradle.docker.tasks.image.* +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage plugins { id 'java-library' @@ -80,7 +80,7 @@ task copyDocker(type: Copy) { into "build/docker/" } -task copyGeneratedXml(type: Copy) { +task copyInitSql(type: Copy) { dependsOn copyDocker from('src/main/resources') { @@ -91,10 +91,10 @@ task copyGeneratedXml(type: Copy) { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types -task buildMyAppImage(type: DockerBuildImage) { +task buildDockerImage(type: DockerBuildImage) { dependsOn copyDocker - dependsOn copyGeneratedXml + dependsOn copyInitSql inputDir = file('build/docker') images.add("airbyte/db:$buildTag") } -assemble.dependsOn(buildMyAppImage) +assemble.dependsOn(buildDockerImage) diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index 2c51855e112d8..dbfaab8844594 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -36,10 +36,10 @@ task copyGeneratedTar(type: Copy) { } // Use task types -task buildMyAppImage(type: DockerBuildImage) { +task buildDockerImage(type: DockerBuildImage) { dependsOn copyDocker dependsOn copyGeneratedTar inputDir = file('build/docker') images.add("airbyte/migration:$version") } -build.dependsOn(buildMyAppImage) +build.dependsOn(buildDockerImage) diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index cd6131214198b..8fe8c4533f5f4 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -1,4 +1,4 @@ -import com.bmuschko.gradle.docker.tasks.image.* +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage plugins { id 'application' @@ -80,10 +80,10 @@ task copyGeneratedTar(type: Copy) { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types -task buildMyAppImage(type: DockerBuildImage) { +task buildDockerImage(type: DockerBuildImage) { dependsOn copyGeneratedTar dependsOn copyDocker inputDir = file('build/docker') images.add("airbyte/scheduler:$buildTag") } -assemble.dependsOn(buildMyAppImage) +assemble.dependsOn(buildDockerImage) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index ae0c0f1de3ef6..882904fc6583e 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -1,4 +1,4 @@ -import com.bmuschko.gradle.docker.tasks.image.* +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage plugins { id 'application' @@ -148,17 +148,10 @@ task copyGeneratedTar(type: Copy) { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types -task buildMyAppImage(type: DockerBuildImage) { +task buildDockerImage(type: DockerBuildImage) { dependsOn copyDocker dependsOn copyGeneratedTar inputDir = file('build/docker') images.add("airbyte/server:$buildTag") } -assemble.dependsOn(buildMyAppImage) -startScripts.dependsOn(shadowJar) -distTar.dependsOn(shadowJar) -distZip.dependsOn(shadowJar) -startShadowScripts.dependsOn(jar) -shadowDistTar.dependsOn(jar) -shadowDistZip.dependsOn(jar) -copyGeneratedTar.dependsOn(shadowDistTar) +assemble.dependsOn(buildDockerImage) diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index a1718a701712e..8518a75d29118 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -1,4 +1,4 @@ -import com.bmuschko.gradle.docker.tasks.image.* +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage plugins { id "base" @@ -55,7 +55,7 @@ task copyDocs(type: Copy) { dependsOn copyDocker from "${project.rootProject.projectDir}/docs/integrations" - into "build/docker/bin/docs/integratbimions" + into "build/docker/bin/docs/integrations" duplicatesStrategy DuplicatesStrategy.INCLUDE } @@ -72,7 +72,7 @@ copyDocs.dependsOn npm_run_build assemble.dependsOn copyDocs // Use task types -task buildMyAppImage(type: DockerBuildImage) { +task buildDockerImage(type: DockerBuildImage) { dependsOn copyDocker dependsOn copyBuild dependsOn copyNginx @@ -80,4 +80,4 @@ task buildMyAppImage(type: DockerBuildImage) { inputDir = file('build/docker') images.add("airbyte/webapp:$version") } -assemble.dependsOn(buildMyAppImage) +assemble.dependsOn(buildDockerImage) diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 887b655ed291d..73cb1556bbf41 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -1,5 +1,5 @@ import org.jsonschema2pojo.SourceType -import com.bmuschko.gradle.docker.tasks.image.* +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage plugins { id 'application' @@ -84,11 +84,11 @@ task copyGeneratedTar(type: Copy) { def buildTag = System.getenv('VERSION') ?: 'dev' // Use task types -task buildMyAppImage(type: DockerBuildImage) { +task buildDockerImage(type: DockerBuildImage) { dependsOn copyDocker dependsOn copyGeneratedTar dependsOn distTar inputDir = file('build/docker') images.add("airbyte/worker:$buildTag") } -assemble.dependsOn(buildMyAppImage) +assemble.dependsOn(buildDockerImage) diff --git a/build.gradle b/build.gradle index 006c048ed8c0a..ebd071db1dcfa 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,5 @@ +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage + buildscript { repositories { maven { @@ -122,6 +124,33 @@ spotless { } check.dependsOn 'spotlessApply' +allprojects { + task copyDockerTest(type: Copy) { + delete "build/docker" + + from "${project.projectDir}/Dockerfile" + into "build/docker/" + } + + // Use task types + task buildDockerImageTest(type: DockerBuildImage) { + dependsOn copyDockerTest + + def buildTag = System.getenv('VERSION') ?: 'dev' + def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64' + def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2' + def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64' + + def artifactName = project.name.replaceFirst("airbyte-", "") + + inputDir = file('build/docker') + platform = buildPlatform + images.add("airbyte/$artifactName:$buildTag") + buildArgs.put('JDK_VERSION', jdkVersion) + buildArgs.put('DOCKER_BUILD_ARCH', buildArch) + } +} + allprojects { apply plugin: 'base' From 43bb8b0cfcaac7d83df22a8fed34e6527ce03561 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 3 Nov 2021 09:43:09 -0700 Subject: [PATCH 31/38] Single definition of the incremental docker tasks --- airbyte-cli/build.gradle | 3 ++- airbyte-config/init/build.gradle | 19 +++---------------- airbyte-db/lib/build.gradle | 17 ++--------------- airbyte-migration/build.gradle | 18 +++--------------- airbyte-scheduler/app/build.gradle | 17 ++--------------- airbyte-server/build.gradle | 18 +++--------------- airbyte-webapp/build.gradle | 21 +++++---------------- airbyte-workers/build.gradle | 18 ++---------------- build.gradle | 10 ++++++---- 9 files changed, 28 insertions(+), 113 deletions(-) diff --git a/airbyte-cli/build.gradle b/airbyte-cli/build.gradle index 10889e3ffdea9..b994d660933fb 100644 --- a/airbyte-cli/build.gradle +++ b/airbyte-cli/build.gradle @@ -1 +1,2 @@ -assemble.dependsOn(buildDockerImageTest) +project.properties.put("artifactName" , "cli") +assemble.dependsOn(buildDockerImage) diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index cb7a464ac0ca3..2224d5353b2a0 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -11,24 +11,11 @@ dependencies { implementation project(':airbyte-config:models') } -task copyDocker(type: Copy) { - delete "build/docker" - - from "${project.projectDir}/Dockerfile" - into "build/docker/" -} - task copyScripts(type: Copy) { from('scripts') into 'build/docker/bin/scripts' } -def buildTag = System.getenv('VERSION') ?: 'dev' -// Use task types -task buildMyAppImage(type: DockerBuildImage) { - dependsOn copyScripts - dependsOn copyDocker - inputDir = file('build/docker') - images.add("airbyte/init:$buildTag") -} -assemble.dependsOn(buildMyAppImage) +project.properties.put("artifactName" , "worker") +buildDockerImage.dependsOn(copyScripts) +assemble.dependsOn(buildDockerImage) diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index feacc9c1f9460..6a2eae42cb0ff 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -73,13 +73,6 @@ task(dumpJobsSchema, dependsOn: 'classes', type: JavaExec) { args 'jobs', 'dump_schema' } -task copyDocker(type: Copy) { - delete "build/docker" - - from "${project.projectDir}/Dockerfile" - into "build/docker/" -} - task copyInitSql(type: Copy) { dependsOn copyDocker @@ -89,12 +82,6 @@ task copyInitSql(type: Copy) { into 'build/docker/bin' } -def buildTag = System.getenv('VERSION') ?: 'dev' -// Use task types -task buildDockerImage(type: DockerBuildImage) { - dependsOn copyDocker - dependsOn copyInitSql - inputDir = file('build/docker') - images.add("airbyte/db:$buildTag") -} +project.properties.put("artifactName" , "db") +buildDockerImage.dependsOn(copyInitSql) assemble.dependsOn(buildDockerImage) diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index dbfaab8844594..e0f9657f8b37c 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -18,13 +18,6 @@ application { mainClass = 'io.airbyte.migrate.MigrationRunner' } -task copyDocker(type: Copy) { - delete "build/docker" - - from "${project.projectDir}/Dockerfile" - into "build/docker/" -} - task copyGeneratedTar(type: Copy) { dependsOn distTar dependsOn copyDocker @@ -35,11 +28,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -// Use task types -task buildDockerImage(type: DockerBuildImage) { - dependsOn copyDocker - dependsOn copyGeneratedTar - inputDir = file('build/docker') - images.add("airbyte/migration:$version") -} -build.dependsOn(buildDockerImage) +project.properties.put("artifactName" , "migration") +buildDockerImage.dependsOn(copyGeneratedTar) +assemble.dependsOn(buildDockerImage) diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index 8fe8c4533f5f4..8712567603aa2 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -61,13 +61,6 @@ run { } -task copyDocker(type: Copy) { - delete "build/docker" - - from "${project.projectDir}/Dockerfile" - into "build/docker/" -} - task copyGeneratedTar(type: Copy) { dependsOn copyDocker dependsOn distTar @@ -78,12 +71,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -def buildTag = System.getenv('VERSION') ?: 'dev' -// Use task types -task buildDockerImage(type: DockerBuildImage) { - dependsOn copyGeneratedTar - dependsOn copyDocker - inputDir = file('build/docker') - images.add("airbyte/scheduler:$buildTag") -} +project.properties.put("artifactName" , "scheduler") +buildDockerImage.dependsOn(copyGeneratedTar) assemble.dependsOn(buildDockerImage) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 882904fc6583e..6ef92bfffe0e4 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -130,15 +130,9 @@ run { environment "TEMPORAL_HOST", "localhost:7233" } -task copyDocker(type: Copy) { - delete "build/docker" - - from "${project.projectDir}/Dockerfile" - into "build/docker/" -} - task copyGeneratedTar(type: Copy) { dependsOn copyDocker + dependsOn distTar from('build/distributions') { include 'airbyte-server-*.tar' @@ -146,12 +140,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -def buildTag = System.getenv('VERSION') ?: 'dev' -// Use task types -task buildDockerImage(type: DockerBuildImage) { - dependsOn copyDocker - dependsOn copyGeneratedTar - inputDir = file('build/docker') - images.add("airbyte/server:$buildTag") -} +project.properties.put("artifactName" , "server") +buildDockerImage.dependsOn(copyGeneratedTar) assemble.dependsOn(buildDockerImage) diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index 8518a75d29118..d97c073918daa 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -35,13 +35,6 @@ task test(type: NpmTask) { assemble.dependsOn npm_run_build build.finalizedBy test -task copyDocker(type: Copy) { - delete "build/docker" - - from "${project.projectDir}/Dockerfile" - into "build/docker/" -} - task copyBuild(type: Copy) { dependsOn copyDocker @@ -71,13 +64,9 @@ copyNginx.dependsOn npm_run_build copyDocs.dependsOn npm_run_build assemble.dependsOn copyDocs -// Use task types -task buildDockerImage(type: DockerBuildImage) { - dependsOn copyDocker - dependsOn copyBuild - dependsOn copyNginx - dependsOn copyDocs - inputDir = file('build/docker') - images.add("airbyte/webapp:$version") -} +project.properties.put("artifactName" , "webapp") +buildDockerImage.dependsOn(copyBuild) +buildDockerImage.dependsOn(copyNginx) +buildDockerImage.dependsOn(copyDocs) +copyDocker.dependsOn(npm_run_build) assemble.dependsOn(buildDockerImage) diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 73cb1556bbf41..e6d7dd511ca1a 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -65,13 +65,6 @@ application { applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } -task copyDocker(type: Copy) { - delete "build/docker" - - from "${project.projectDir}/Dockerfile" - into "build/docker/" -} - task copyGeneratedTar(type: Copy) { dependsOn copyDocker dependsOn distTar @@ -82,13 +75,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -def buildTag = System.getenv('VERSION') ?: 'dev' -// Use task types -task buildDockerImage(type: DockerBuildImage) { - dependsOn copyDocker - dependsOn copyGeneratedTar - dependsOn distTar - inputDir = file('build/docker') - images.add("airbyte/worker:$buildTag") -} +project.properties.put("artifactName" , "worker") +buildDockerImage.dependsOn(copyGeneratedTar) assemble.dependsOn(buildDockerImage) diff --git a/build.gradle b/build.gradle index ebd071db1dcfa..3982a3f0cd748 100644 --- a/build.gradle +++ b/build.gradle @@ -125,7 +125,9 @@ spotless { check.dependsOn 'spotlessApply' allprojects { - task copyDockerTest(type: Copy) { + apply plugin: 'com.bmuschko.docker-remote-api' + + task copyDocker(type: Copy) { delete "build/docker" from "${project.projectDir}/Dockerfile" @@ -133,15 +135,15 @@ allprojects { } // Use task types - task buildDockerImageTest(type: DockerBuildImage) { - dependsOn copyDockerTest + task buildDockerImage(type: DockerBuildImage) { + dependsOn copyDocker def buildTag = System.getenv('VERSION') ?: 'dev' def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64' def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2' def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64' - def artifactName = project.name.replaceFirst("airbyte-", "") + def artifactName = project.getProperties().get("artifactName") inputDir = file('build/docker') platform = buildPlatform From a52643fbb07e48b0e129ab606e143a4512b2c12b Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 3 Nov 2021 10:06:25 -0700 Subject: [PATCH 32/38] Add documentation --- .../developing-on-docker.md | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 docs/contributing-to-airbyte/developing-on-docker.md diff --git a/docs/contributing-to-airbyte/developing-on-docker.md b/docs/contributing-to-airbyte/developing-on-docker.md new file mode 100644 index 0000000000000..37da8602ff7bb --- /dev/null +++ b/docs/contributing-to-airbyte/developing-on-docker.md @@ -0,0 +1,44 @@ +# Developing on docker + +## Version + +This doc is relevant starting from the version 0.30.26-alpha. Prior to that, it was another design. + +## Incrementality + +The docker build is fully incremental, which means that it will only build an image if it is needed. We need to keep it that way. +A task, `buildDockerImage`, is available for building a docker image for any given module. Behind the scene, it will create a dedicated folder in +the `build` folder of a module and copy the `Dockerfile` in it. The goal is to make sure that we have an isolated context which helps with +incrementality. All files that need to be present in the docker image will need to be copy into this folder. + +## Adding a new docker build + +Once you have a `Dockerfile`, generating the docker image is done in the following way: +- specify the artifact name, +- make the build docker task to depend on the `assemble` task. + +For example: +```groovy +project.properties.put("artifactName" , "cli") +assemble.dependsOn(buildDockerImage) +``` + +If you need to add files in your image you need to copy them in `build/docker/bin` first. The need to happen after the `copyDocker` task. +The `copyDocker` task clean up the `build/docker` folder. + +For example: +```groovy +task copyGeneratedTar(type: Copy) { + dependsOn copyDocker + dependsOn distTar + + from('build/distributions') { + include 'airbyte-workers-*.tar' + } + into 'build/docker/bin' +} + +project.properties.put("artifactName" , "worker") +buildDockerImage.dependsOn(copyGeneratedTar) +assemble.dependsOn(buildDockerImage) +``` From 8a3b1fc5dc18b4b8f542fe0dedfe8bf54e94436b Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 3 Nov 2021 14:58:14 -0700 Subject: [PATCH 33/38] test --- airbyte-cli/build.gradle | 2 -- build.gradle | 31 ++++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/airbyte-cli/build.gradle b/airbyte-cli/build.gradle index b994d660933fb..e69de29bb2d1d 100644 --- a/airbyte-cli/build.gradle +++ b/airbyte-cli/build.gradle @@ -1,2 +0,0 @@ -project.properties.put("artifactName" , "cli") -assemble.dependsOn(buildDockerImage) diff --git a/build.gradle b/build.gradle index 3982a3f0cd748..64c3cddc69c95 100644 --- a/build.gradle +++ b/build.gradle @@ -143,13 +143,30 @@ allprojects { def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2' def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64' - def artifactName = project.getProperties().get("artifactName") - - inputDir = file('build/docker') - platform = buildPlatform - images.add("airbyte/$artifactName:$buildTag") - buildArgs.put('JDK_VERSION', jdkVersion) - buildArgs.put('DOCKER_BUILD_ARCH', buildArch) + def projectPathToArtifactName = [ + ":airbyte-config:init": "config", + ":airbyte-db:lib": "db", + ":airbyte-migration": "migration", + ":airbyte-scheduler:app": "scheduler", + ":airbyte-workers": "worker", + ":airbyte-server": "server", + ":airbyte-webapp": "webapp", + ] + + if (projectPathToArtifactName.keySet().contains(project.getPath())) { + assemble.dependsOn buildDockerImage + def artifactName = projectPathToArtifactName[project.getPath()] + + println(project.getPath()) + println(artifactName) + println(projectPathToArtifactName) + + inputDir = file('build/docker') + platform = buildPlatform + images.add("airbyte/$artifactName:$buildTag") + buildArgs.put('JDK_VERSION', jdkVersion) + buildArgs.put('DOCKER_BUILD_ARCH', buildArch) + } } } From 317ae8775b47d2e33a5d4de6ba08d63754085346 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 3 Nov 2021 15:49:53 -0700 Subject: [PATCH 34/38] Make the factorize incremental to work --- airbyte-cli/build.gradle | 3 ++ airbyte-config/init/build.gradle | 8 +++-- airbyte-db/lib/build.gradle | 6 ++-- airbyte-migration/build.gradle | 6 ++-- airbyte-scheduler/app/build.gradle | 6 ++-- airbyte-server/build.gradle | 6 ++-- airbyte-webapp/build.gradle | 12 +++---- airbyte-workers/build.gradle | 6 ++-- build.gradle | 50 +++++++++--------------------- 9 files changed, 44 insertions(+), 59 deletions(-) diff --git a/airbyte-cli/build.gradle b/airbyte-cli/build.gradle index e69de29bb2d1d..6071a62565da6 100644 --- a/airbyte-cli/build.gradle +++ b/airbyte-cli/build.gradle @@ -0,0 +1,3 @@ +Task dockerBuildTask = getDockerBuildTask("cli", "${project.projectDir}/build/docker") +dockerBuildTask.dependsOn(copyDocker) +assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 2224d5353b2a0..de289d90aa536 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -12,10 +12,12 @@ dependencies { } task copyScripts(type: Copy) { + dependsOn copyDocker + from('scripts') into 'build/docker/bin/scripts' } -project.properties.put("artifactName" , "worker") -buildDockerImage.dependsOn(copyScripts) -assemble.dependsOn(buildDockerImage) +Task dockerBuildTask = getDockerBuildTask("init", "${project.projectDir}/build/docker") +dockerBuildTask.dependsOn(copyScripts) +assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 6a2eae42cb0ff..2879a8b896de0 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -82,6 +82,6 @@ task copyInitSql(type: Copy) { into 'build/docker/bin' } -project.properties.put("artifactName" , "db") -buildDockerImage.dependsOn(copyInitSql) -assemble.dependsOn(buildDockerImage) +Task dockerBuildTask = getDockerBuildTask("db", "${project.projectDir}/build/docker") +dockerBuildTask.dependsOn(copyInitSql) +assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index e0f9657f8b37c..9241ed30d614c 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -28,6 +28,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -project.properties.put("artifactName" , "migration") -buildDockerImage.dependsOn(copyGeneratedTar) -assemble.dependsOn(buildDockerImage) +Task dockerBuildTask = getDockerBuildTask("migration", "${project.projectDir}/build/docker") +dockerBuildTask.dependsOn(copyGeneratedTar) +assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index cd6692cc9784b..ed0d2f3c2f5b9 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -71,6 +71,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -project.properties.put("artifactName" , "scheduler") -buildDockerImage.dependsOn(copyGeneratedTar) -assemble.dependsOn(buildDockerImage) +Task dockerBuildTask = getDockerBuildTask("scheduler", "${project.projectDir}/build/docker") +dockerBuildTask.dependsOn(copyGeneratedTar) +assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 6ef92bfffe0e4..e9dc04be236bb 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -140,6 +140,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -project.properties.put("artifactName" , "server") -buildDockerImage.dependsOn(copyGeneratedTar) -assemble.dependsOn(buildDockerImage) +Task dockerBuildTask = getDockerBuildTask("server", "${project.projectDir}/build/docker") +dockerBuildTask.dependsOn(copyGeneratedTar) +assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index d97c073918daa..0946599fcbe06 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -63,10 +63,10 @@ copyBuild.dependsOn npm_run_build copyNginx.dependsOn npm_run_build copyDocs.dependsOn npm_run_build assemble.dependsOn copyDocs - -project.properties.put("artifactName" , "webapp") -buildDockerImage.dependsOn(copyBuild) -buildDockerImage.dependsOn(copyNginx) -buildDockerImage.dependsOn(copyDocs) copyDocker.dependsOn(npm_run_build) -assemble.dependsOn(buildDockerImage) + +Task dockerBuildTask = getDockerBuildTask("webapp", "${project.projectDir}/build/docker") +dockerBuildTask.dependsOn(copyBuild) +dockerBuildTask.dependsOn(copyNginx) +dockerBuildTask.dependsOn(copyDocs) +assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index e6d7dd511ca1a..dec2b915baf92 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -75,6 +75,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -project.properties.put("artifactName" , "worker") -buildDockerImage.dependsOn(copyGeneratedTar) -assemble.dependsOn(buildDockerImage) +Task dockerBuildTask = getDockerBuildTask("worker", "${project.projectDir}/build/docker") +dockerBuildTask.dependsOn(copyGeneratedTar) +assemble.dependsOn(dockerBuildTask) diff --git a/build.gradle b/build.gradle index 64c3cddc69c95..8fc8bffafd1ed 100644 --- a/build.gradle +++ b/build.gradle @@ -124,6 +124,21 @@ spotless { } check.dependsOn 'spotlessApply' +def Task getDockerBuildTask(String artifactName, String dockerBuildDir) { + return task ("buildDockerImage-$artifactName" (type: DockerBuildImage) { + def buildTag = System.getenv('VERSION') ?: 'dev' + def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64' + def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2' + def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64' + + inputDir = file(dockerBuildDir) + platform = buildPlatform + images.add("airbyte/$artifactName:$buildTag") + buildArgs.put('JDK_VERSION', jdkVersion) + buildArgs.put('DOCKER_BUILD_ARCH', buildArch) + }) +} + allprojects { apply plugin: 'com.bmuschko.docker-remote-api' @@ -133,41 +148,6 @@ allprojects { from "${project.projectDir}/Dockerfile" into "build/docker/" } - - // Use task types - task buildDockerImage(type: DockerBuildImage) { - dependsOn copyDocker - - def buildTag = System.getenv('VERSION') ?: 'dev' - def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64' - def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2' - def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64' - - def projectPathToArtifactName = [ - ":airbyte-config:init": "config", - ":airbyte-db:lib": "db", - ":airbyte-migration": "migration", - ":airbyte-scheduler:app": "scheduler", - ":airbyte-workers": "worker", - ":airbyte-server": "server", - ":airbyte-webapp": "webapp", - ] - - if (projectPathToArtifactName.keySet().contains(project.getPath())) { - assemble.dependsOn buildDockerImage - def artifactName = projectPathToArtifactName[project.getPath()] - - println(project.getPath()) - println(artifactName) - println(projectPathToArtifactName) - - inputDir = file('build/docker') - platform = buildPlatform - images.add("airbyte/$artifactName:$buildTag") - buildArgs.put('JDK_VERSION', jdkVersion) - buildArgs.put('DOCKER_BUILD_ARCH', buildArch) - } - } } allprojects { From d64750a7dcac41b9a1004d8df1b2c1f5f9c0bb91 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 3 Nov 2021 16:26:11 -0700 Subject: [PATCH 35/38] Update docs and remove unused import --- airbyte-cli/build.gradle | 2 +- airbyte-config/init/build.gradle | 2 +- airbyte-db/lib/build.gradle | 2 +- airbyte-migration/build.gradle | 2 +- airbyte-scheduler/app/build.gradle | 2 +- airbyte-server/build.gradle | 2 +- airbyte-webapp/build.gradle | 2 +- airbyte-workers/build.gradle | 2 +- build.gradle | 5 +-- .../developing-on-docker.md | 36 ++++++++++--------- 10 files changed, 30 insertions(+), 27 deletions(-) diff --git a/airbyte-cli/build.gradle b/airbyte-cli/build.gradle index 6071a62565da6..83d7285df9b85 100644 --- a/airbyte-cli/build.gradle +++ b/airbyte-cli/build.gradle @@ -1,3 +1,3 @@ -Task dockerBuildTask = getDockerBuildTask("cli", "${project.projectDir}/build/docker") +Task dockerBuildTask = getDockerBuildTask("cli", project.projectDir) dockerBuildTask.dependsOn(copyDocker) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 259a43a720056..06461a765f158 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -16,6 +16,6 @@ task copyScripts(type: Copy) { into 'build/docker/bin/scripts' } -Task dockerBuildTask = getDockerBuildTask("init", "${project.projectDir}/build/docker") +Task dockerBuildTask = getDockerBuildTask("init", project.projectDir) dockerBuildTask.dependsOn(copyScripts) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 13ddcb1640efb..508bfb0bacb12 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -80,6 +80,6 @@ task copyInitSql(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("db", "${project.projectDir}/build/docker") +Task dockerBuildTask = getDockerBuildTask("db", project.projectDir) dockerBuildTask.dependsOn(copyInitSql) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index a055489121355..3b8c097ac842e 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -26,6 +26,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("migration", "${project.projectDir}/build/docker") +Task dockerBuildTask = getDockerBuildTask("migration", project.projectDir) dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index f649bcba37bb5..f003f408c2919 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -69,6 +69,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("scheduler", "${project.projectDir}/build/docker") +Task dockerBuildTask = getDockerBuildTask("scheduler", project.projectDir) dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 2c77a8bd4e9d7..de525f3df9ea2 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -138,6 +138,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("server", "${project.projectDir}/build/docker") +Task dockerBuildTask = getDockerBuildTask("server", project.projectDir) dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index 5c57148de0593..7fa0f76d4f685 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -63,7 +63,7 @@ copyDocs.dependsOn npm_run_build assemble.dependsOn copyDocs copyDocker.dependsOn(npm_run_build) -Task dockerBuildTask = getDockerBuildTask("webapp", "${project.projectDir}/build/docker") +Task dockerBuildTask = getDockerBuildTask("webapp", project.projectDir) dockerBuildTask.dependsOn(copyBuild) dockerBuildTask.dependsOn(copyNginx) dockerBuildTask.dependsOn(copyDocs) diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index a606a2bcd1d63..06bd5e06c47a4 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -74,6 +74,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("worker", "${project.projectDir}/build/docker") +Task dockerBuildTask = getDockerBuildTask("worker", project.projectDir) dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/build.gradle b/build.gradle index 8fc8bffafd1ed..cdb78c1b99b07 100644 --- a/build.gradle +++ b/build.gradle @@ -124,14 +124,15 @@ spotless { } check.dependsOn 'spotlessApply' -def Task getDockerBuildTask(String artifactName, String dockerBuildDir) { +@SuppressWarnings('GroovyAssignabilityCheck') +def Task getDockerBuildTask(String artifactName, String projectDir) { return task ("buildDockerImage-$artifactName" (type: DockerBuildImage) { def buildTag = System.getenv('VERSION') ?: 'dev' def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64' def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2' def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64' - inputDir = file(dockerBuildDir) + inputDir = file("$projectDir/build/docker") platform = buildPlatform images.add("airbyte/$artifactName:$buildTag") buildArgs.put('JDK_VERSION', jdkVersion) diff --git a/docs/contributing-to-airbyte/developing-on-docker.md b/docs/contributing-to-airbyte/developing-on-docker.md index 37da8602ff7bb..09d28c8626923 100644 --- a/docs/contributing-to-airbyte/developing-on-docker.md +++ b/docs/contributing-to-airbyte/developing-on-docker.md @@ -2,43 +2,45 @@ ## Version -This doc is relevant starting from the version 0.30.26-alpha. Prior to that, it was another design. +This doc is relevant starting from the version 0.30.28-alpha. Prior to that, it was another design. ## Incrementality The docker build is fully incremental, which means that it will only build an image if it is needed. We need to keep it that way. -A task, `buildDockerImage`, is available for building a docker image for any given module. Behind the scene, it will create a dedicated folder in -the `build` folder of a module and copy the `Dockerfile` in it. The goal is to make sure that we have an isolated context which helps with -incrementality. All files that need to be present in the docker image will need to be copy into this folder. +A task generator, `getDockerBuildTask`, is available for building a docker image for any given module. Behind the scene, it will generate a +task which will run the build of a docker image in a specific folder. The goal is to make sure that we have an isolated +context which helps with incrementality. All files that need to be present in the docker image will need to be copy into this folder. The generate +method takes 2 arguments: +- The image name, for example if `foo` is given as an image name, the image `airbyte/foo` will be created +- The project directory folder. It is needed because the `getDockerBuildTask` is declared in the rootProject ## Adding a new docker build Once you have a `Dockerfile`, generating the docker image is done in the following way: -- specify the artifact name, +- specify the artifact name and the project directory, +- make sure that the Dockerfile is properly copied to the docker context dir before building the image - make the build docker task to depend on the `assemble` task. For example: ```groovy -project.properties.put("artifactName" , "cli") -assemble.dependsOn(buildDockerImage) +Task dockerBuildTask = getDockerBuildTask("cli", project.projectDir) +dockerBuildTask.dependsOn(copyDocker) +assemble.dependsOn(dockerBuildTask) ``` If you need to add files in your image you need to copy them in `build/docker/bin` first. The need to happen after the `copyDocker` task. -The `copyDocker` task clean up the `build/docker` folder. +The `copyDocker` task clean up the `build/docker` folder as a first step. For example: ```groovy -task copyGeneratedTar(type: Copy) { +task copyScripts(type: Copy) { dependsOn copyDocker - dependsOn distTar - from('build/distributions') { - include 'airbyte-workers-*.tar' - } - into 'build/docker/bin' + from('scripts') + into 'build/docker/bin/scripts' } -project.properties.put("artifactName" , "worker") -buildDockerImage.dependsOn(copyGeneratedTar) -assemble.dependsOn(buildDockerImage) +Task dockerBuildTask = getDockerBuildTask("init", project.projectDir) +dockerBuildTask.dependsOn(copyScripts) +assemble.dependsOn(dockerBuildTask) ``` From d42857acbd0e73145c27637feac81389701cdded Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 3 Nov 2021 16:47:39 -0700 Subject: [PATCH 36/38] Fix build and remove unused pluggins --- airbyte-cli/build.gradle | 2 +- airbyte-config/init/build.gradle | 3 +-- airbyte-db/lib/build.gradle | 3 +-- airbyte-migration/build.gradle | 3 +-- airbyte-scheduler/app/build.gradle | 3 +-- airbyte-server/build.gradle | 3 +-- airbyte-webapp/build.gradle | 3 +-- airbyte-workers/build.gradle | 3 +-- 8 files changed, 8 insertions(+), 15 deletions(-) diff --git a/airbyte-cli/build.gradle b/airbyte-cli/build.gradle index 83d7285df9b85..4cccd9d4f4018 100644 --- a/airbyte-cli/build.gradle +++ b/airbyte-cli/build.gradle @@ -1,3 +1,3 @@ -Task dockerBuildTask = getDockerBuildTask("cli", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir") dockerBuildTask.dependsOn(copyDocker) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 06461a765f158..fbe103703417b 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java' - id 'com.bmuschko.docker-remote-api' } dependencies { @@ -16,6 +15,6 @@ task copyScripts(type: Copy) { into 'build/docker/bin/scripts' } -Task dockerBuildTask = getDockerBuildTask("init", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("init", "$project.projectDir") dockerBuildTask.dependsOn(copyScripts) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 508bfb0bacb12..5be6ca9a1e8e9 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java-library' - id 'com.bmuschko.docker-remote-api' } dependencies { @@ -80,6 +79,6 @@ task copyInitSql(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("db", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("db", "$project.projectDir") dockerBuildTask.dependsOn(copyInitSql) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle index 3b8c097ac842e..1a9504dec522b 100644 --- a/airbyte-migration/build.gradle +++ b/airbyte-migration/build.gradle @@ -1,6 +1,5 @@ plugins { id 'application' - id 'com.bmuschko.docker-remote-api' } dependencies { @@ -26,6 +25,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("migration", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("migration", "$project.projectDir") dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index f003f408c2919..371137316379c 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -1,6 +1,5 @@ plugins { id 'application' - id 'com.bmuschko.docker-remote-api' } dependencies { @@ -69,6 +68,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("scheduler", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("scheduler", "$project.projectDir") dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index de525f3df9ea2..65bf114b06eb0 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -2,7 +2,6 @@ plugins { id 'application' id 'maven-publish' id 'com.github.johnrengelman.shadow' version '7.1.0' - id 'com.bmuschko.docker-remote-api' } shadowJar { @@ -138,6 +137,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("server", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("server", "$project.projectDir") dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index 7fa0f76d4f685..886827b844947 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -1,7 +1,6 @@ plugins { id "base" id "com.github.node-gradle.node" version "3.1.1" - id 'com.bmuschko.docker-remote-api' } def nodeVersion = System.getenv('NODE_VERSION') ?: '14.11.0' @@ -63,7 +62,7 @@ copyDocs.dependsOn npm_run_build assemble.dependsOn copyDocs copyDocker.dependsOn(npm_run_build) -Task dockerBuildTask = getDockerBuildTask("webapp", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("webapp", "$project.projectDir") dockerBuildTask.dependsOn(copyBuild) dockerBuildTask.dependsOn(copyNginx) dockerBuildTask.dependsOn(copyDocs) diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 06bd5e06c47a4..f461116ce2d70 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -4,7 +4,6 @@ plugins { id 'application' id 'com.github.eirnym.js2p' version '1.0' id 'airbyte-integration-test-java' - id 'com.bmuschko.docker-remote-api' } configurations { @@ -74,6 +73,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("worker", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("worker", "$project.projectDir") dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) From 24fd33c71ade99b34b62823d2d8e5bd7434e8077 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 3 Nov 2021 17:13:30 -0700 Subject: [PATCH 37/38] Update doc based on PR comments --- docs/SUMMARY.md | 1 + docs/contributing-to-airbyte/developing-on-docker.md | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 44cfa4f835ba0..b8040b0007076 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -197,6 +197,7 @@ * [Contributing to Airbyte](contributing-to-airbyte/README.md) * [Code of Conduct](contributing-to-airbyte/code-of-conduct.md) * [Developing Locally](contributing-to-airbyte/developing-locally.md) + * [Developing on Docker](contributing-to-airbyte/developing-on-docker.md) * [Developing on Kubernetes](contributing-to-airbyte/developing-on-kubernetes.md) * [Monorepo Python Development](contributing-to-airbyte/monorepo-python-development.md) * [Code Style](contributing-to-airbyte/code-style.md) diff --git a/docs/contributing-to-airbyte/developing-on-docker.md b/docs/contributing-to-airbyte/developing-on-docker.md index 09d28c8626923..34f225c8b0172 100644 --- a/docs/contributing-to-airbyte/developing-on-docker.md +++ b/docs/contributing-to-airbyte/developing-on-docker.md @@ -1,12 +1,9 @@ # Developing on docker -## Version - -This doc is relevant starting from the version 0.30.28-alpha. Prior to that, it was another design. - ## Incrementality -The docker build is fully incremental, which means that it will only build an image if it is needed. We need to keep it that way. +The docker build is fully incremental for the platform build, which means that it will only build an image if it is needed. We need to keep it that +way. A task generator, `getDockerBuildTask`, is available for building a docker image for any given module. Behind the scene, it will generate a task which will run the build of a docker image in a specific folder. The goal is to make sure that we have an isolated context which helps with incrementality. All files that need to be present in the docker image will need to be copy into this folder. The generate From 1d3d22f2d85f3834ba1d31e3ab68abc386e3dc32 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 5 Nov 2021 11:25:48 -0700 Subject: [PATCH 38/38] format --- airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java index ebf3346b2c0e5..9fb0a22a76250 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java @@ -26,7 +26,6 @@ import java.util.function.Function; import java.util.function.Supplier; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory;