diff --git a/airbyte-secrets-migration/Dockerfile b/airbyte-secrets-migration/Dockerfile deleted file mode 100644 index 84e6f6ad17e2e..0000000000000 --- a/airbyte-secrets-migration/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG JDK_VERSION=14.0.2 -FROM openjdk:${JDK_VERSION}-slim - -ENV APPLICATION airbyte-secrets-migration -WORKDIR /airbyte -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar -RUN tar xf ${APPLICATION}.tar --strip-components=1 - -# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/secrets-migration - -CMD ["/airbyte/bin/airbyte-secrets-migration"] \ No newline at end of file diff --git a/airbyte-secrets-migration/README.md b/airbyte-secrets-migration/README.md deleted file mode 100644 index ca21bcb5d74ab..0000000000000 --- a/airbyte-secrets-migration/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Airbyte Secrets Migration - -This is the repository for the Secrets Migration helper pod. It enables migrating -secrets from a DatabaseConfigPersistence to a GoogleSecretsManagerConfigPersistence -as a one-time bulk. - -## Local development - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-secrets-migrationsbuild -``` - -#### Build -Build the connector image via Gradle: -``` -./gradlew :airbyte-secrets-migration:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run - -## Testing -We use `JUnit` for Java tests. - -### Unit and Integration Tests -Place unit tests under `src/test/...` -Place integration tests in `src/test-integration/...` - diff --git a/airbyte-secrets-migration/build.gradle b/airbyte-secrets-migration/build.gradle deleted file mode 100644 index 615839ba480f9..0000000000000 --- a/airbyte-secrets-migration/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.secretsmigration.SecretsMigration' -} - -dependencies { - implementation project(':airbyte-db:lib') - implementation project(':airbyte-scheduler:client') - implementation project(':airbyte-config:models') - implementation project(':airbyte-protocol:models') - implementation project(':airbyte-config:persistence') - testImplementation project(':airbyte-json-validation') - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation "org.testcontainers:postgresql:1.15.3" - integrationTestJavaImplementation project(':airbyte-secrets-migration') -} - diff --git a/airbyte-secrets-migration/src/main/java/io/airbyte/secretsmigration/SecretsMigration.java b/airbyte-secrets-migration/src/main/java/io/airbyte/secretsmigration/SecretsMigration.java deleted file mode 100644 index 0220481630be9..0000000000000 --- a/airbyte-secrets-migration/src/main/java/io/airbyte/secretsmigration/SecretsMigration.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.secretsmigration; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.cloud.storage.StorageOptions; -import io.airbyte.config.Configs; -import io.airbyte.config.EnvConfigs; -import io.airbyte.config.persistence.ConfigPersistence; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.DatabaseConfigPersistence; -import io.airbyte.config.persistence.split_secrets.LocalTestingSecretPersistence; -import io.airbyte.config.persistence.split_secrets.RealSecretsHydrator; -import io.airbyte.db.Database; -import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; -import io.airbyte.scheduler.client.BucketSpecCacheSchedulerClient; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SecretsMigration { - - private static final Path TEST_ROOT = Path.of("/tmp/airbyte_tests"); - private static final Logger LOGGER = LoggerFactory.getLogger(SecretsMigration.class); - final boolean dryRun; - final ConfigRepository readFrom; - final ConfigRepository writeTo; - - public SecretsMigration(final ConfigRepository readFrom, final ConfigRepository writeTo, final boolean dryRun) { - this.readFrom = readFrom; - this.writeTo = writeTo; - this.dryRun = dryRun; - } - - public void run() throws IOException { - LOGGER.info("Starting migration run."); - - LOGGER.info("... Dry Run: deserializing configurations and writing to the new store..."); - Map> configurations = readFrom.dumpConfigs(); - writeTo.replaceAllConfigsDeserializing(configurations, true); - - LOGGER.info("... With dryRun=" + dryRun + ": deserializing configurations and writing to the new store..."); - configurations = readFrom.dumpConfigs(); - writeTo.replaceAllConfigsDeserializing(configurations, dryRun); - - LOGGER.info("Migration run complete."); - } - - public static void main(final String[] args) throws Exception { - final Configs configs = new EnvConfigs(); - - final Database database = new ConfigsDatabaseInstance( - "docker", // configs.getConfigDatabaseUser(), - "docker", // configs.getConfigDatabasePassword(), - "jdbc:postgresql://localhost:8011/airbyte") // configs.getConfigDatabaseUrl()) - .getInitialized(); - - final ConfigPersistence configPersistence = new DatabaseConfigPersistence(database).withValidation(); - - final ConfigRepository configRepository = - new ConfigRepository( - configPersistence, - new RealSecretsHydrator(new LocalTestingSecretPersistence(database)), - Optional.of(new LocalTestingSecretPersistence(database)), - Optional.of(new LocalTestingSecretPersistence(database))); - - configRepository.setSpecFetcher(dockerImage -> BucketSpecCacheSchedulerClient - .attemptToFetchSpecFromBucket(StorageOptions.getDefaultInstance().getService(), "io-airbyte-cloud-spec-cache", dockerImage).get()); - - final SecretsMigration migration = new SecretsMigration(configRepository, configRepository, false); - LOGGER.info("starting: {}", SecretsMigration.class); - migration.run(); - LOGGER.info("completed: {}", SecretsMigration.class); - } - -} diff --git a/airbyte-secrets-migration/src/test/resources/file-source-config.json b/airbyte-secrets-migration/src/test/resources/file-source-config.json deleted file mode 100644 index d772181f39a3d..0000000000000 --- a/airbyte-secrets-migration/src/test/resources/file-source-config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "provider": { - "storage": "HTTPS" - }, - "url": "http://www.example.com/foobadurltest", - "format": "csv", - "dataset_name": "foo" -} diff --git a/airbyte-secrets-migration/src/test/resources/postgres-source-config-nossl.json b/airbyte-secrets-migration/src/test/resources/postgres-source-config-nossl.json deleted file mode 100644 index 606f5fe960d74..0000000000000 --- a/airbyte-secrets-migration/src/test/resources/postgres-source-config-nossl.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "ssl": false, - "host": "localhost", - "port": 5001, - "database": "test", - "password": "passwordredacted", - "username": "testcaseuser", - "tunnel_method": { - "tunnel_method": "NONE" - }, - "replication_method": { - "method": "Standard" - } -} diff --git a/airbyte-secrets-migration/src/test/resources/postgres-source-config-ssl.json b/airbyte-secrets-migration/src/test/resources/postgres-source-config-ssl.json deleted file mode 100644 index 3a2c456977ba8..0000000000000 --- a/airbyte-secrets-migration/src/test/resources/postgres-source-config-ssl.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "ssl": true, - "host": "localhost", - "port": 5001, - "database": "test", - "password": "passwordredacted", - "username": "testcaseuser", - "tunnel_method": { - "tunnel_method": "NONE" - }, - "replication_method": { - "method": "Standard" - } -} diff --git a/airbyte-secrets-migration/src/test/resources/s3-dest-config.json b/airbyte-secrets-migration/src/test/resources/s3-dest-config.json deleted file mode 100644 index 8f9327d4dd94c..0000000000000 --- a/airbyte-secrets-migration/src/test/resources/s3-dest-config.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "format": { - "compression_codec": { - "codec": "no compression" - }, - "format_type": "Avro" - }, - "secret_access_key": "abcdef", - "access_key_id": "abcdefghi", - "s3_bucket_region": "us-east-1", - "s3_bucket_path": "data_sync/test", - "s3_bucket_name": "airbyte_sync", - "s3_endpoint": "" -} diff --git a/settings.gradle b/settings.gradle index 6fbfd9eeaa2eb..6532fb0a03725 100644 --- a/settings.gradle +++ b/settings.gradle @@ -57,7 +57,6 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") include ':airbyte-server' include ':airbyte-tests' include ':airbyte-webapp' - include ':airbyte-secrets-migration' } // connectors base