control-service: secrets service implementation

projects/control-service/projects/base/src/main/java/com/vmware/taurus/base/FeatureFlags.java

@@ -31,4 +31,7 @@ public class FeatureFlags {
 
   @Value("${datajobs.security.oauth2.enabled:false}")
   boolean oAuth2Enabled = true;
+
+  @Value("${featureflag.vault.integration.enabled:false}")
+  boolean vaultIntegrationEnabled = false;
 }

projects/control-service/projects/pipelines_control_service/build.gradle

@@ -42,6 +42,9 @@ dependencies { // Implementation dependencies are found on compile classpath of
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
     implementation 'org.springframework.retry:spring-retry'
 
+    // Hashicorp Vault client libraries
+    implementation versions. 'org.springframework.vault:spring-vault-core'
+
     // Janino is used for conditional processing in the logback-spring.xml file
     implementation 'org.codehaus.janino:janino'
 

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/DataJobPropertiesException.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.exception;
+
+import org.springframework.http.HttpStatus;
+
+public class DataJobPropertiesException extends DomainError implements UserFacingError {
+
+  public DataJobPropertiesException(String jobName, String why) {
+    super(
+        String.format("Error processing properties for '%s' data job.", jobName),
+        why,
+        "Unable to process data job properties.",
+        "Try re-creating the data job properties or contract the service operator",
+        null);
+  }
+
+  @Override
+  public HttpStatus getHttpStatus() {
+    return HttpStatus.BAD_REQUEST;
+  }
+}

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/DataJobSecretsException.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.exception;
+
+import org.springframework.http.HttpStatus;
+
+public class DataJobSecretsException extends DomainError implements UserFacingError {
+
+  public DataJobSecretsException(String jobName, String why) {
+    super(
+        String.format("Error processing secrets for '%s' data job.", jobName),
+        why,
+        "Unable to process data job secrets.",
+        "Try re-creating the data job secrets or contract the service operator",
+        null);
+  }
+
+  @Override
+  public HttpStatus getHttpStatus() {
+    return HttpStatus.BAD_REQUEST;
+  }
+}

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/ExternalSystemError.java

@@ -51,7 +51,10 @@ public enum MainExternalSystem {
     WEBHOOK_SERVER("Webhook server"),
 
     /** The Git repository that contains the data job code */
-    GIT("Git");
+    GIT("Git"),
+
+    /** A secrets storage/repository used to store data jobs secrets. */
+    SECRETS("Secrets");
 
     private final String userFacingName;
 

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/SecretStorageNotConfiguredException.java

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.exception;
+
+/** Exception thrown, when a secret storage has not been configured */
+public class SecretStorageNotConfiguredException extends ExternalSystemError {
+
+  public SecretStorageNotConfiguredException() {
+    super(MainExternalSystem.SECRETS, "Not Configured");
+  }
+}

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/properties/controller/DataJobsPropertiesController.java

@@ -7,15 +7,14 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.vmware.taurus.controlplane.model.api.DataJobsPropertiesApi;
+import com.vmware.taurus.exception.DataJobPropertiesException;
 import com.vmware.taurus.properties.service.PropertiesService;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.ComponentScan;
-import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.RestController;
-import org.springframework.web.server.ResponseStatusException;
 
 import java.util.Map;
 
@@ -50,8 +49,7 @@ public ResponseEntity<Map<String, Object>> dataJobPropertiesRead(
     } catch (JsonProcessingException e) {
       log.error("Error while parsing properties for job: " + jobName, e);
 
-      throw new ResponseStatusException(
-          HttpStatus.INTERNAL_SERVER_ERROR, "Error while parsing properties for job: " + jobName);
+      throw new DataJobPropertiesException(jobName, "Error while parsing properties.");
     }
   }
 }

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/controller/DataJobsSecretsController.java

@@ -23,19 +23,55 @@
 public class DataJobsSecretsController implements DataJobsSecretsApi {
   static Logger log = LoggerFactory.getLogger(DataJobsSecretsController.class);
 
+  //    private final FeatureFlags featureFlags;
+  //
+  //    private final JobSecretsService secretsService;
+
+  //    @Autowired
+  //    public DataJobsSecretsController(FeatureFlags featureFlags, JobSecretsService
+  // secretsService) {
+  //        this.featureFlags = featureFlags;
+  //        this.secretsService = secretsService;
+  //    }
+
   @Override
   public ResponseEntity<Void> dataJobSecretsUpdate(
       String teamName, String jobName, String deploymentId, Map<String, Object> requestBody) {
     log.debug("Updating secrets for job: {}", jobName);
+
+    //    TODO: Remove after adding tests
     throw new ResponseStatusException(
         HttpStatus.NOT_IMPLEMENTED, "Secrets service is not implemented");
+
+    //    TODO: Working implementation. Uncomment after adding tests
+    //    if (featureFlags.isVaultIntegrationEnabled()) {
+    //      secretsService.updateJobSecrets(jobName, requestBody);
+    //      return ResponseEntity.noContent().build();
+    //    }
+    //
+    //        throw new SecretStorageNotConfiguredException();
   }
 
   @Override
   public ResponseEntity<Map<String, Object>> dataJobSecretsRead(
       String teamName, String jobName, String deploymentId) {
     log.debug("Reading secrets for job: {}", jobName);
+
+    //    TODO: Remove after adding tests
     throw new ResponseStatusException(
         HttpStatus.NOT_IMPLEMENTED, "Secrets service is not implemented");
+
+    //    TODO: Working implementation. Uncomment after adding tests
+    //        if (featureFlags.isVaultIntegrationEnabled()) {
+    //            try {
+    //                return ResponseEntity.ok(secretsService.readJobSecrets(jobName));
+    //            } catch (JsonProcessingException e) {
+    //                log.error("Error while parsing secrets for job: " + jobName, e);
+    //                throw new DataJobSecretsException(jobName, "Error while parsing secrets for
+    // job");
+    //            }
+    //        }
+    //
+    //        throw new SecretStorageNotConfiguredException();
   }
 }

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/JobSecrets.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.secrets.service;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.springframework.data.annotation.Id;
+import org.springframework.vault.repository.mapping.Secret;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Secret
+public class JobSecrets {
+
+  @Id private String jobName;
+
+  private String secretsJson;
+}

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/JobSecretsService.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.secrets.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.vmware.taurus.base.FeatureFlags;
+import lombok.extern.slf4j.Slf4j;
+import org.json.JSONObject;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.vault.authentication.TokenAuthentication;
+import org.springframework.vault.client.VaultEndpoint;
+import org.springframework.vault.core.VaultTemplate;
+import org.springframework.vault.support.Versioned;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Service
+public class JobSecretsService {
+
+  private static final String SECRET = "secret";
+
+  private final ObjectMapper objectMapper = new ObjectMapper();
+  private VaultTemplate vaultTemplate;
+
+  public JobSecretsService(
+      @Value("${vdk.vault.uri:}") String vaultUri,
+      @Value("${vdk.vault.token:}") String vaultToken,
+      FeatureFlags featureFlags)
+      throws URISyntaxException {
+
+    if (featureFlags.isVaultIntegrationEnabled()) {
+      VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri));
+      TokenAuthentication clientAuthentication = new TokenAuthentication(vaultToken);
+
+      this.vaultTemplate = new VaultTemplate(vaultEndpoint, clientAuthentication);
+    }
+  }
+
+  public void updateJobSecrets(String jobName, Map<String, Object> secrets) {
+    Versioned<JobSecrets> readResponse =
+        vaultTemplate.opsForVersionedKeyValue(SECRET).get(jobName, JobSecrets.class);
+
+    JobSecrets jobSecrets;
+
+    if (readResponse != null && readResponse.hasData()) {
+      jobSecrets = readResponse.getData();
+    } else {
+      jobSecrets = new JobSecrets(jobName, null);
+    }
+
+    var updatedSecrets =
+        secrets.entrySet().stream()
+            .collect(
+                Collectors.toMap(
+                    Map.Entry::getKey,
+                    entry -> {
+                      if (entry.getValue() == null) {
+                        entry.setValue(JSONObject.NULL);
+                      }
+                      return entry.getValue();
+                    }));
+
+    jobSecrets.setSecretsJson(new JSONObject(updatedSecrets).toString());
+
+    vaultTemplate.opsForVersionedKeyValue(SECRET).put(jobName, jobSecrets);
+  }
+
+  public Map<String, Object> readJobSecrets(String jobName) throws JsonProcessingException {
+    Versioned<JobSecrets> readResponse =
+        vaultTemplate.opsForVersionedKeyValue(SECRET).get(jobName, JobSecrets.class);
+
+    JobSecrets jobSecrets;
+
+    if (readResponse != null && readResponse.hasData()) {
+      jobSecrets = readResponse.getData();
+      return objectMapper.readValue(jobSecrets.getSecretsJson(), Map.class);
+    } else {
+      return Collections.emptyMap();
+    }
+  }
+}

projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties

@@ -244,3 +244,9 @@ datajobs.aws.roleArn=${DATAJOBS_AWS_ROLE_ARN:}
 # datajobs.aws.defaultSessionDurationSeconds is the default credential expiration for a job builder
 # instance. Default value is 30 minutes.
 datajobs.aws.defaultSessionDurationSeconds=${DATAJOBS_AWS_DEFAULT_SESSION_DURATION_SECONDS:1800}
+
+# Hashicorp Vault Integration settings
+# When disabled/not configured the Secrets functionality won't work
+featureflag.vault.integration.enabled=false
+vdk.vault.uri=http://localhost:8200
+vdk.vault.token=root

projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/properties/DataJobsPropertiesControllerTest.java

@@ -6,6 +6,7 @@
 package com.vmware.taurus.properties;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.vmware.taurus.exception.DataJobPropertiesException;
 import com.vmware.taurus.properties.controller.DataJobsPropertiesController;
 import com.vmware.taurus.properties.service.PropertiesService;
 import org.junit.jupiter.api.Test;
@@ -15,7 +16,6 @@
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
-import org.springframework.web.server.ResponseStatusException;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -72,15 +72,15 @@ void testDataJobPropertiesRead_JsonProcessingException() throws JsonProcessingEx
     when(propertiesService.readJobProperties(jobName)).thenThrow(JsonProcessingException.class);
 
     ResponseEntity<Map<String, Object>> expectedResponse =
-        ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null);
+        ResponseEntity.status(HttpStatus.BAD_REQUEST).body(null);
 
-    ResponseStatusException thrownException =
+    DataJobPropertiesException thrownException =
         org.junit.jupiter.api.Assertions.assertThrows(
-            ResponseStatusException.class,
+            DataJobPropertiesException.class,
             () -> controller.dataJobPropertiesRead(null, jobName, null));
 
     verify(propertiesService, times(1)).readJobProperties(jobName);
 
-    assertEquals(expectedResponse.getStatusCode(), thrownException.getStatus());
+    assertEquals(expectedResponse.getStatusCode(), thrownException.getHttpStatus());
   }
 }

projects/control-service/projects/versions-of-external-dependencies.gradle

@@ -45,6 +45,7 @@ project.ext {
             'com.amazonaws:aws-java-sdk-core'                                    : 'com.amazonaws:aws-java-sdk-core:1.12.485',
             'com.amazonaws:aws-java-sdk-sts'                                     : 'com.amazonaws:aws-java-sdk-sts:1.12.483',
             'com.amazonaws:aws-java-sdk-ecr'                                     : 'com.amazonaws:aws-java-sdk-ecr:1.12.485',
+            'org.springframework.vault:spring-vault-core'                        : 'org.springframework.vault:spring-vault-core:2.3.3',
 
             // transitive dependencies version force (freeze)
             // on next upgrade, revise if those still need to be set explicitly
@@ -55,5 +56,6 @@ project.ext {
             'com.squareup.okio:okio'                                             : 'com.squareup.okio:okio:3.3.0',
             'org.apache.commons:commons-compress'                                : 'org.apache.commons:commons-compress:1.23.0',
             'org.hibernate:hibernate-jpamodelgen'                                : 'org.hibernate:hibernate-jpamodelgen:5.6.15.Final'
+
     ]
 }