Secrets Migration utility (#6489)

* Created skeleton of a migration utility module.
* Secrets migration hello world functional
* Added dependencies for secrets migration
* Create Secrets store migration utility and related tests.
* Make secrets migration work in kube
* Make pod for secrets migration give right health result to kube

airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java

@@ -5,6 +5,7 @@
 package io.airbyte.config.persistence;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.lang.MoreBooleans;
 import io.airbyte.config.AirbyteConfig;
 import io.airbyte.config.ConfigSchema;
@@ -22,6 +23,7 @@
 import io.airbyte.validation.json.JsonValidationException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -256,6 +258,26 @@ public List<DestinationOAuthParameter> listDestinationOAuthParam() throws JsonVa
     return persistence.listConfigs(ConfigSchema.DESTINATION_OAUTH_PARAM, DestinationOAuthParameter.class);
   }
 
+  /**
+   * Converts between a dumpConfig() output and a replaceAllConfigs() input, by deserializing the
+   * string/jsonnode into the AirbyteConfig, Stream<Object<AirbyteConfig.getClassName()>
+   *
+   * @param configurations from dumpConfig()
+   * @return input suitable for replaceAllConfigs()
+   */
+  public static Map<AirbyteConfig, Stream<?>> deserialize(Map<String, Stream<JsonNode>> configurations) {
+    Map<AirbyteConfig, Stream<?>> deserialized = new LinkedHashMap<AirbyteConfig, Stream<?>>();
+    for (String configSchemaName : configurations.keySet()) {
+      deserialized.put(ConfigSchema.valueOf(configSchemaName),
+          configurations.get(configSchemaName).map(jsonNode -> Jsons.object(jsonNode, ConfigSchema.valueOf(configSchemaName).getClassName())));
+    }
+    return deserialized;
+  }
+
+  public void replaceAllConfigsDeserializing(final Map<String, Stream<JsonNode>> configs, final boolean dryRun) throws IOException {
+    replaceAllConfigs(deserialize(configs), dryRun);
+  }
+
   public void replaceAllConfigs(final Map<AirbyteConfig, Stream<?>> configs, final boolean dryRun) throws IOException {
     persistence.replaceAllConfigs(configs, dryRun);
   }

airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/FileSystemConfigPersistence.java

@@ -14,6 +14,7 @@
 import java.io.UnsupportedEncodingException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -109,6 +110,9 @@ public Map<String, Stream<JsonNode>> dumpConfigs() throws IOException {
   }
 
   private List<String> listDirectories() throws IOException {
+    if (!configRoot.toFile().exists()) {
+      return new ArrayList<String>();
+    }
     try (Stream<Path> files = Files.list(configRoot)) {
       return files.map(c -> c.getFileName().toString()).collect(Collectors.toList());
     }
@@ -168,9 +172,10 @@ public void replaceAllConfigs(Map<AirbyteConfig, Stream<?>> configs, boolean dry
       FileUtils.deleteDirectory(rootOverride.toFile());
       return;
     }
-
-    FileUtils.moveDirectory(configRoot.toFile(), storageRoot.resolve(oldConfigsDir).toFile());
-    LOGGER.info("Renamed config to {} successfully", oldConfigsDir);
+    if (configRoot.toFile().exists()) {
+      FileUtils.moveDirectory(configRoot.toFile(), storageRoot.resolve(oldConfigsDir).toFile());
+      LOGGER.info("Renamed config to {} successfully", oldConfigsDir);
+    }
 
     FileUtils.moveDirectory(rootOverride.toFile(), configRoot.toFile());
     LOGGER.info("Renamed " + importDirectory + " to config successfully");

airbyte-secrets-migration/Dockerfile

@@ -0,0 +1,12 @@
+FROM openjdk:14.0.2-slim
+
+ENV APPLICATION airbyte-secrets-migration
+WORKDIR /airbyte
+COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+RUN tar xf ${APPLICATION}.tar --strip-components=1
+
+# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile.
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/secrets-migration
+
+CMD ["/airbyte/bin/airbyte-secrets-migration"]

airbyte-secrets-migration/README.md

@@ -0,0 +1,31 @@
+# Airbyte Secrets Migration
+
+This is the repository for the Secrets Migration helper pod.  It enables migrating
+secrets from a DatabaseConfigPersistence to a GoogleSecretsManagerConfigPersistence 
+as a one-time bulk.
+
+## Local development
+
+#### Building via Gradle
+From the Airbyte repository root, run:
+```
+./gradlew :airbyte-secrets-migrationsbuild
+```
+
+#### Build
+Build the connector image via Gradle:
+```
+./gradlew :airbyte-secrets-migration:airbyteDocker
+```
+When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
+the Dockerfile.
+
+#### Run
+
+## Testing
+We use `JUnit` for Java tests.
+
+### Unit and Integration Tests
+Place unit tests under `src/test/...`
+Place integration tests in `src/test-integration/...` 
+

airbyte-secrets-migration/build.gradle

@@ -0,0 +1,20 @@
+plugins {
+    id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
+}
+
+application {
+    mainClass = 'io.airbyte.secretsmigration.SecretsMigration'
+}
+
+dependencies {
+    implementation project(':airbyte-db:lib')
+    implementation project(':airbyte-config:models')
+    implementation project(':airbyte-config:persistence')
+    testImplementation project(':airbyte-json-validation')
+    testImplementation 'org.apache.commons:commons-lang3:3.11'
+    testImplementation "org.testcontainers:postgresql:1.15.1"
+    integrationTestJavaImplementation project(':airbyte-secrets-migration')
+}
+

airbyte-secrets-migration/src/main/java/io/airbyte/secretsmigration/SecretsMigration.java

@@ -0,0 +1,89 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.secretsmigration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.config.Configs;
+import io.airbyte.config.EnvConfigs;
+import io.airbyte.config.persistence.ConfigPersistence;
+import io.airbyte.config.persistence.ConfigRepository;
+import io.airbyte.config.persistence.DatabaseConfigPersistence;
+import io.airbyte.config.persistence.FileSystemConfigPersistence;
+import io.airbyte.db.instance.configs.ConfigsDatabaseInstance;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.stream.Stream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SecretsMigration {
+
+  private static final Path TEST_ROOT = Path.of("/tmp/airbyte_tests");
+  private static final Logger LOGGER = LoggerFactory.getLogger(SecretsMigration.class);
+  final Configs configs;
+  final boolean dryRun;
+  final ConfigPersistence readFromPersistence;
+  final ConfigPersistence writeToPersistence;
+
+  public SecretsMigration(Configs envConfigs, ConfigPersistence readFromPersistence, ConfigPersistence writeToPersistence, boolean dryRun) {
+    this.configs = envConfigs;
+    this.readFromPersistence = readFromPersistence;
+    this.writeToPersistence = writeToPersistence;
+    this.dryRun = dryRun;
+  }
+
+  public void run() throws IOException {
+    LOGGER.info("Starting migration run.");
+
+    final ConfigRepository readFromConfigRepository = new ConfigRepository(readFromPersistence);
+    final ConfigRepository writeToConfigRepository = new ConfigRepository(writeToPersistence);
+
+    LOGGER.info("... Dry Run: deserializing configurations and writing to the new store...");
+    Map<String, Stream<JsonNode>> configurations = readFromConfigRepository.dumpConfigs();
+    writeToConfigRepository.replaceAllConfigsDeserializing(configurations, true);
+
+    LOGGER.info("... With dryRun=" + dryRun + ": deserializing configurations and writing to the new store...");
+    configurations = readFromConfigRepository.dumpConfigs();
+    writeToConfigRepository.replaceAllConfigsDeserializing(configurations, dryRun);
+
+    LOGGER.info("Migration run complete.");
+  }
+
+  public static void main(String[] args) throws Exception {
+    final Configs configs = new EnvConfigs();
+    final ConfigPersistence readFromPersistence = new DatabaseConfigPersistence(new ConfigsDatabaseInstance(
+        configs.getConfigDatabaseUser(),
+        configs.getConfigDatabasePassword(),
+        configs.getConfigDatabaseUrl())
+            .getInitialized()).withValidation();
+    final ConfigPersistence writeToPersistence = new FileSystemConfigPersistence(TEST_ROOT);
+    final SecretsMigration migration = new SecretsMigration(configs, readFromPersistence, writeToPersistence, false);
+    LOGGER.info("starting: {}", SecretsMigration.class);
+    migration.run();
+    LOGGER.info("completed: {}", SecretsMigration.class);
+  }
+
+}