control-service: secrets service implementation

projects/control-service/projects/base/src/main/java/com/vmware/taurus/base/FeatureFlags.java

@@ -31,4 +31,7 @@ public class FeatureFlags {
 
   @Value("${datajobs.security.oauth2.enabled:false}")
   boolean oAuth2Enabled = true;
+
+  @Value("${featureflag.vault.integration.enabled:false}")
+  boolean vaultIntegrationEnabled = false;
 }

projects/control-service/projects/pipelines_control_service/build.gradle

@@ -42,6 +42,9 @@ dependencies { // Implementation dependencies are found on compile classpath of
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
     implementation 'org.springframework.retry:spring-retry'
 
+    // Hashicorp Vault client libraries
+    implementation 'org.springframework.vault:spring-vault-core'
+
     // Janino is used for conditional processing in the logback-spring.xml file
     implementation 'org.codehaus.janino:janino'
 

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/ExternalSystemError.java

@@ -51,7 +51,10 @@ public enum MainExternalSystem {
     WEBHOOK_SERVER("Webhook server"),
 
     /** The Git repository that contains the data job code */
-    GIT("Git");
+    GIT("Git"),
+
+    /** A secrets storage/repository used to store data jobs secrets. */
+    SECRETS("Secrets");
 
     private final String userFacingName;
 

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/SecretStorageNotConfiguredException.java

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.exception;
+
+/** Exception thrown, when a secret storage has not been configured */
+public class SecretStorageNotConfiguredException extends ExternalSystemError {
+
+  public SecretStorageNotConfiguredException() {
+    super(MainExternalSystem.SECRETS, "Not Configured");
+  }
+}

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/controller/DataJobsSecretsController.java

@@ -23,19 +23,59 @@
 public class DataJobsSecretsController implements DataJobsSecretsApi {
   static Logger log = LoggerFactory.getLogger(DataJobsSecretsController.class);
 
+  //  private final FeatureFlags featureFlags;
+
+  //  private final JobSecretsService secretsService;
+
+  //  @Autowired
+  //  public DataJobsSecretsController(FeatureFlags featureFlags, JobSecretsService secretsService)
+  // {
+  //  public DataJobsSecretsController(FeatureFlags featureFlags, JobSecretsService secretsService)
+  // {
+  //    this.featureFlags = featureFlags;
+  //    this.secretsService = secretsService;
+  //  }
+
   @Override
   public ResponseEntity<Void> dataJobSecretsUpdate(
       String teamName, String jobName, String deploymentId, Map<String, Object> requestBody) {
     log.debug("Updating secrets for job: {}", jobName);
+
+    //    TODO: Remove after adding tests
     throw new ResponseStatusException(
         HttpStatus.NOT_IMPLEMENTED, "Secrets service is not implemented");
+
+    //    TODO: Working implementation. Uncomment after adding tests
+    //    if (featureFlags.isVaultIntegrationEnabled()) {
+    //      secretsService.updateJobSecrets(jobName, requestBody);
+    //      return ResponseEntity.noContent().build();
+    //    }
+    //
+    //        throw new SecretStorageNotConfiguredException();
   }
 
   @Override
   public ResponseEntity<Map<String, Object>> dataJobSecretsRead(
       String teamName, String jobName, String deploymentId) {
     log.debug("Reading secrets for job: {}", jobName);
+
+    //    TODO: Remove after adding tests
     throw new ResponseStatusException(
         HttpStatus.NOT_IMPLEMENTED, "Secrets service is not implemented");
+
+    //    TODO: Working implementation. Uncomment after adding tests
+    //    if (featureFlags.isVaultIntegrationEnabled()) {
+    //      try {
+    //        return ResponseEntity.ok(secretsService.readJobSecrets(jobName));
+    //      } catch (JsonProcessingException e) {
+    //        log.error("Error while parsing secrets for job: " + jobName, e);
+    //
+    //        throw new ResponseStatusException(
+    //                HttpStatus.INTERNAL_SERVER_ERROR, "Error while parsing secrets for job: " +
+    // jobName);
+    //      }
+    //    }
+    //
+    //        throw new SecretStorageNotConfiguredException();
   }
 }

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/JobSecrets.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.secrets.service;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.springframework.data.annotation.Id;
+import org.springframework.vault.repository.mapping.Secret;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Secret
+public class JobSecrets {
+
+  @Id private String jobName;
+
+  private String secretsJson;
+}

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/JobSecretsService.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.secrets.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.vmware.taurus.base.FeatureFlags;
+import lombok.extern.slf4j.Slf4j;
+import org.json.JSONObject;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.vault.authentication.TokenAuthentication;
+import org.springframework.vault.client.VaultEndpoint;
+import org.springframework.vault.core.VaultTemplate;
+import org.springframework.vault.support.Versioned;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Service
+public class JobSecretsService {
+
+  private static final String SECRET = "secret";
+
+  private final ObjectMapper objectMapper = new ObjectMapper();
+  private VaultTemplate vaultTemplate;
+
+  public JobSecretsService(
+      @Value("${vdk.vault.uri:}") String vaultUri,
+      @Value("${vdk.vault.token:}") String vaultToken,
+      FeatureFlags featureFlags)
+      throws URISyntaxException {
+
+    if (featureFlags.isVaultIntegrationEnabled()) {
+      VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri));
+      TokenAuthentication clientAuthentication = new TokenAuthentication(vaultToken);
+
+      this.vaultTemplate = new VaultTemplate(vaultEndpoint, clientAuthentication);
+    }
+  }
+
+  public void updateJobSecrets(String jobName, Map<String, Object> secrets) {
+    Versioned<JobSecrets> readResponse =
+        vaultTemplate.opsForVersionedKeyValue(SECRET).get(jobName, JobSecrets.class);
+
+    JobSecrets jobSecrets;
+
+    if (readResponse != null && readResponse.hasData()) {
+      jobSecrets = readResponse.getData();
+    } else {
+      jobSecrets = new JobSecrets(jobName, null);
+    }
+
+    var updatedSecrets =
+        secrets.entrySet().stream()
+            .collect(
+                Collectors.toMap(
+                    Map.Entry::getKey,
+                    entry -> {
+                      if (entry.getValue() == null) {
+                        entry.setValue(JSONObject.NULL);
+                      }
+                      return entry.getValue();
+                    }));
+
+    jobSecrets.setSecretsJson(new JSONObject(updatedSecrets).toString());
+
+    vaultTemplate.opsForVersionedKeyValue(SECRET).put(jobName, jobSecrets);
+  }
+
+  public Map<String, Object> readJobSecrets(String jobName) throws JsonProcessingException {
+    Versioned<JobSecrets> readResponse =
+        vaultTemplate.opsForVersionedKeyValue(SECRET).get(jobName, JobSecrets.class);
+
+    JobSecrets jobSecrets;
+
+    if (readResponse != null && readResponse.hasData()) {
+      jobSecrets = readResponse.getData();
+      return objectMapper.readValue(jobSecrets.getSecretsJson(), Map.class);
+    } else {
+      return Collections.emptyMap();
+    }
+  }
+}

projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties

@@ -244,3 +244,9 @@ datajobs.aws.roleArn=${DATAJOBS_AWS_ROLE_ARN:}
 # datajobs.aws.defaultSessionDurationSeconds is the default credential expiration for a job builder
 # instance. Default value is 30 minutes.
 datajobs.aws.defaultSessionDurationSeconds=${DATAJOBS_AWS_DEFAULT_SESSION_DURATION_SECONDS:1800}
+
+# Hashicorp Vault Integration settings
+# When disabled/not configured the Secrets functionality won't work
+featureflag.vault.integration.enabled=false
+vdk.vault.uri=http://localhost:8200
+vdk.vault.token=root

projects/control-service/projects/versions-of-external-dependencies.gradle

@@ -45,6 +45,7 @@ project.ext {
             'com.amazonaws:aws-java-sdk-core'                                    : 'com.amazonaws:aws-java-sdk-core:1.12.485',
             'com.amazonaws:aws-java-sdk-sts'                                     : 'com.amazonaws:aws-java-sdk-sts:1.12.483',
             'com.amazonaws:aws-java-sdk-ecr'                                     : 'com.amazonaws:aws-java-sdk-ecr:1.12.485',
+            'org.springframework.vault:spring-vault-core'                        : 'org.springframework.vault:spring-vault-core:2.3.3',
 
             // transitive dependencies version force (freeze)
             // on next upgrade, revise if those still need to be set explicitly
@@ -55,5 +56,6 @@ project.ext {
             'com.squareup.okio:okio'                                             : 'com.squareup.okio:okio:3.3.0',
             'org.apache.commons:commons-compress'                                : 'org.apache.commons:commons-compress:1.23.0',
             'org.hibernate:hibernate-jpamodelgen'                                : 'org.hibernate:hibernate-jpamodelgen:5.6.15.Final'
+
     ]
 }