diff --git a/projects/control-service/projects/base/src/main/java/com/vmware/taurus/base/FeatureFlags.java b/projects/control-service/projects/base/src/main/java/com/vmware/taurus/base/FeatureFlags.java index 197b5ab2d4..46481ec124 100644 --- a/projects/control-service/projects/base/src/main/java/com/vmware/taurus/base/FeatureFlags.java +++ b/projects/control-service/projects/base/src/main/java/com/vmware/taurus/base/FeatureFlags.java @@ -31,4 +31,7 @@ public class FeatureFlags { @Value("${datajobs.security.oauth2.enabled:false}") boolean oAuth2Enabled = true; + + @Value("${featureflag.vault.integration.enabled:false}") + boolean vaultIntegrationEnabled = false; } diff --git a/projects/control-service/projects/pipelines_control_service/build.gradle b/projects/control-service/projects/pipelines_control_service/build.gradle index 9f23825ff7..3b6a0b95ad 100644 --- a/projects/control-service/projects/pipelines_control_service/build.gradle +++ b/projects/control-service/projects/pipelines_control_service/build.gradle @@ -42,6 +42,9 @@ dependencies { // Implementation dependencies are found on compile classpath of implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.retry:spring-retry' + // Hashicorp Vault client libraries + implementation versions.'org.springframework.vault:spring-vault-core' + // Janino is used for conditional processing in the logback-spring.xml file implementation 'org.codehaus.janino:janino' diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/DataJobPropertiesException.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/DataJobPropertiesException.java new file mode 100644 index 0000000000..7ac359fb23 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/DataJobPropertiesException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021-2023 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.exception; + +import org.springframework.http.HttpStatus; + +public class DataJobPropertiesException extends DomainError implements UserFacingError { + + public DataJobPropertiesException(String jobName, String why) { + super( + String.format("Exception processing properties for '%s' data job.", jobName), + why, + "Unable to process data job properties.", + "Try re-creating the data job properties or contract the service operator", + null); + } + + @Override + public HttpStatus getHttpStatus() { + return HttpStatus.BAD_REQUEST; + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/DataJobSecretsException.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/DataJobSecretsException.java new file mode 100644 index 0000000000..770570438c --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/DataJobSecretsException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021-2023 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.exception; + +import org.springframework.http.HttpStatus; + +public class DataJobSecretsException extends DomainError implements UserFacingError { + + public DataJobSecretsException(String jobName, String why) { + super( + String.format("Exception processing secrets for '%s' data job.", jobName), + why, + "Unable to process data job secrets.", + "Try re-creating the data job secrets or contract the service operator", + null); + } + + @Override + public HttpStatus getHttpStatus() { + return HttpStatus.BAD_REQUEST; + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/ExternalSystemError.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/ExternalSystemError.java index 4939b42618..451270178a 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/ExternalSystemError.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/ExternalSystemError.java @@ -51,7 +51,10 @@ public enum MainExternalSystem { WEBHOOK_SERVER("Webhook server"), /** The Git repository that contains the data job code */ - GIT("Git"); + GIT("Git"), + + /** A secrets storage/repository used to store data jobs secrets. */ + SECRETS("Secrets"); private final String userFacingName; diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/SecretStorageNotConfiguredException.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/SecretStorageNotConfiguredException.java new file mode 100644 index 0000000000..5563e8d0c0 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/SecretStorageNotConfiguredException.java @@ -0,0 +1,14 @@ +/* + * Copyright 2021-2023 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.exception; + +/** Exception thrown, when a secret storage has not been configured */ +public class SecretStorageNotConfiguredException extends ExternalSystemError { + + public SecretStorageNotConfiguredException() { + super(MainExternalSystem.SECRETS, "Not Configured"); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/properties/controller/DataJobsPropertiesController.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/properties/controller/DataJobsPropertiesController.java index f11af077a2..193b6b798a 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/properties/controller/DataJobsPropertiesController.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/properties/controller/DataJobsPropertiesController.java @@ -7,15 +7,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.vmware.taurus.controlplane.model.api.DataJobsPropertiesApi; +import com.vmware.taurus.exception.DataJobPropertiesException; import com.vmware.taurus.properties.service.PropertiesService; import io.swagger.v3.oas.annotations.tags.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.ComponentScan; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.server.ResponseStatusException; import java.util.Map; @@ -50,8 +49,7 @@ public ResponseEntity> dataJobPropertiesRead( } catch (JsonProcessingException e) { log.error("Error while parsing properties for job: " + jobName, e); - throw new ResponseStatusException( - HttpStatus.INTERNAL_SERVER_ERROR, "Error while parsing properties for job: " + jobName); + throw new DataJobPropertiesException(jobName, "Error while parsing properties."); } } } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/controller/DataJobsSecretsController.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/controller/DataJobsSecretsController.java index 5ac91b0b3b..5822382a55 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/controller/DataJobsSecretsController.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/controller/DataJobsSecretsController.java @@ -5,37 +5,69 @@ package com.vmware.taurus.secrets.controller; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.vmware.taurus.base.FeatureFlags; import com.vmware.taurus.controlplane.model.api.DataJobsSecretsApi; +import com.vmware.taurus.exception.DataJobSecretsException; +import com.vmware.taurus.exception.SecretStorageNotConfiguredException; +import com.vmware.taurus.secrets.service.JobSecretsService; import io.swagger.v3.oas.annotations.tags.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.ComponentScan; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.lang.Nullable; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.server.ResponseStatusException; import java.util.Map; @RestController @ComponentScan(basePackages = "com.vmware.taurus.secrets") @Tag(name = "Data Jobs Secrets") +@ConditionalOnProperty(value = "featureflag.vault.integration.enabled") public class DataJobsSecretsController implements DataJobsSecretsApi { static Logger log = LoggerFactory.getLogger(DataJobsSecretsController.class); + private final FeatureFlags featureFlags; + + private final JobSecretsService secretsService; + + @Autowired + public DataJobsSecretsController( + FeatureFlags featureFlags, @Nullable JobSecretsService secretsService) { + this.featureFlags = featureFlags; + this.secretsService = secretsService; + } + @Override public ResponseEntity dataJobSecretsUpdate( String teamName, String jobName, String deploymentId, Map requestBody) { log.debug("Updating secrets for job: {}", jobName); - throw new ResponseStatusException( - HttpStatus.NOT_IMPLEMENTED, "Secrets service is not implemented"); + + if (featureFlags.isVaultIntegrationEnabled()) { + secretsService.updateJobSecrets(jobName, requestBody); + return ResponseEntity.noContent().build(); + } + + throw new SecretStorageNotConfiguredException(); } @Override public ResponseEntity> dataJobSecretsRead( String teamName, String jobName, String deploymentId) { log.debug("Reading secrets for job: {}", jobName); - throw new ResponseStatusException( - HttpStatus.NOT_IMPLEMENTED, "Secrets service is not implemented"); + + if (featureFlags.isVaultIntegrationEnabled()) { + try { + return ResponseEntity.ok(secretsService.readJobSecrets(jobName)); + } catch (JsonProcessingException e) { + log.error("Error while parsing secrets for job: " + jobName, e); + throw new DataJobSecretsException(jobName, "Error while parsing secrets for job"); + } + } + + throw new SecretStorageNotConfiguredException(); } } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/controller/NoOpDataJobsSecretsController.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/controller/NoOpDataJobsSecretsController.java new file mode 100644 index 0000000000..b5b62fc7e1 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/controller/NoOpDataJobsSecretsController.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021-2023 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.secrets.controller; + +import com.vmware.taurus.controlplane.model.api.DataJobsSecretsApi; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import java.util.Map; + +/** + * This class is used to provide the secrets endpoint in swagger and rest even when secrets support + * is disabled. + */ +@RestController +@ComponentScan(basePackages = "com.vmware.taurus.secrets") +@Tag(name = "Data Jobs Secrets") +@ConditionalOnProperty(value = "featureflag.vault.integration.enabled", havingValue = "false") +public class NoOpDataJobsSecretsController implements DataJobsSecretsApi { + @Override + public ResponseEntity dataJobSecretsUpdate( + String teamName, String jobName, String deploymentId, Map requestBody) { + throw new ResponseStatusException( + HttpStatus.NOT_IMPLEMENTED, "Secrets service is not implemented"); + } + + @Override + public ResponseEntity> dataJobSecretsRead( + String teamName, String jobName, String deploymentId) { + throw new ResponseStatusException( + HttpStatus.NOT_IMPLEMENTED, "Secrets service is not implemented"); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/JobSecretsService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/JobSecretsService.java new file mode 100644 index 0000000000..7fc7fa614e --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/JobSecretsService.java @@ -0,0 +1,16 @@ +/* + * Copyright 2021-2023 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.secrets.service; + +import com.fasterxml.jackson.core.JsonProcessingException; + +import java.util.Map; + +public interface JobSecretsService { + void updateJobSecrets(String jobName, Map secrets); + + Map readJobSecrets(String jobName) throws JsonProcessingException; +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultConfiguration.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultConfiguration.java new file mode 100644 index 0000000000..4a35225bd0 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultConfiguration.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021-2023 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.secrets.service.vault; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.vault.authentication.TokenAuthentication; +import org.springframework.vault.client.VaultEndpoint; +import org.springframework.vault.core.VaultOperations; +import org.springframework.vault.core.VaultTemplate; + +import java.net.URI; +import java.net.URISyntaxException; + +@Configuration +public class VaultConfiguration { + + @Bean + public VaultOperations vaultOperations( + @Value("${vdk.vault.uri:}") String vaultUri, @Value("${vdk.vault.token:}") String vaultToken) + throws URISyntaxException { + VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri)); + TokenAuthentication clientAuthentication = new TokenAuthentication(vaultToken); + + return new VaultTemplate(vaultEndpoint, clientAuthentication); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultJobSecrets.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultJobSecrets.java new file mode 100644 index 0000000000..aa48026893 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultJobSecrets.java @@ -0,0 +1,23 @@ +/* + * Copyright 2021-2023 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.secrets.service.vault; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.data.annotation.Id; +import org.springframework.vault.repository.mapping.Secret; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Secret +public class VaultJobSecrets { + + @Id private String jobName; + + private String secretsJson; +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultJobSecretsService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultJobSecretsService.java new file mode 100644 index 0000000000..3c49a17fda --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultJobSecretsService.java @@ -0,0 +1,79 @@ +/* + * Copyright 2021-2023 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.secrets.service.vault; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.json.JSONObject; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.springframework.vault.core.VaultOperations; +import org.springframework.vault.support.Versioned; + +import java.util.Collections; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +@Service +@ConditionalOnProperty(value = "featureflag.vault.integration.enabled") +public class VaultJobSecretsService implements com.vmware.taurus.secrets.service.JobSecretsService { + + private static final String SECRET = "secret"; + + private final ObjectMapper objectMapper = new ObjectMapper(); + private final VaultOperations vaultOperations; + + public VaultJobSecretsService(VaultOperations vaultOperations) { + this.vaultOperations = vaultOperations; + } + + @Override + public void updateJobSecrets(String jobName, Map secrets) { + Versioned readResponse = + vaultOperations.opsForVersionedKeyValue(SECRET).get(jobName, VaultJobSecrets.class); + + VaultJobSecrets vaultJobSecrets; + + if (readResponse != null && readResponse.hasData()) { + vaultJobSecrets = readResponse.getData(); + } else { + vaultJobSecrets = new VaultJobSecrets(jobName, null); + } + + var updatedSecrets = + secrets.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> { + if (entry.getValue() == null) { + entry.setValue(JSONObject.NULL); + } + return entry.getValue(); + })); + + vaultJobSecrets.setSecretsJson(new JSONObject(updatedSecrets).toString()); + + vaultOperations.opsForVersionedKeyValue(SECRET).put(jobName, vaultJobSecrets); + } + + @Override + public Map readJobSecrets(String jobName) throws JsonProcessingException { + Versioned readResponse = + vaultOperations.opsForVersionedKeyValue(SECRET).get(jobName, VaultJobSecrets.class); + + VaultJobSecrets vaultJobSecrets; + + if (readResponse != null && readResponse.hasData()) { + vaultJobSecrets = readResponse.getData(); + return objectMapper.readValue(vaultJobSecrets.getSecretsJson(), Map.class); + } else { + return Collections.emptyMap(); + } + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties b/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties index d596aa0dea..33f2754ddf 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties +++ b/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties @@ -244,3 +244,9 @@ datajobs.aws.roleArn=${DATAJOBS_AWS_ROLE_ARN:} # datajobs.aws.defaultSessionDurationSeconds is the default credential expiration for a job builder # instance. Default value is 30 minutes. datajobs.aws.defaultSessionDurationSeconds=${DATAJOBS_AWS_DEFAULT_SESSION_DURATION_SECONDS:1800} + +# Hashicorp Vault Integration settings +# When disabled/not configured the Secrets functionality won't work +featureflag.vault.integration.enabled=false +vdk.vault.uri=http://localhost:8200 +vdk.vault.token=root diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/properties/DataJobsPropertiesControllerTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/properties/DataJobsPropertiesControllerTest.java index 750d25312f..cc873a75b5 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/properties/DataJobsPropertiesControllerTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/properties/DataJobsPropertiesControllerTest.java @@ -6,6 +6,7 @@ package com.vmware.taurus.properties; import com.fasterxml.jackson.core.JsonProcessingException; +import com.vmware.taurus.exception.DataJobPropertiesException; import com.vmware.taurus.properties.controller.DataJobsPropertiesController; import com.vmware.taurus.properties.service.PropertiesService; import org.junit.jupiter.api.Test; @@ -15,7 +16,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.server.ResponseStatusException; import java.util.HashMap; import java.util.Map; @@ -72,15 +72,15 @@ void testDataJobPropertiesRead_JsonProcessingException() throws JsonProcessingEx when(propertiesService.readJobProperties(jobName)).thenThrow(JsonProcessingException.class); ResponseEntity> expectedResponse = - ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null); + ResponseEntity.status(HttpStatus.BAD_REQUEST).body(null); - ResponseStatusException thrownException = + DataJobPropertiesException thrownException = org.junit.jupiter.api.Assertions.assertThrows( - ResponseStatusException.class, + DataJobPropertiesException.class, () -> controller.dataJobPropertiesRead(null, jobName, null)); verify(propertiesService, times(1)).readJobProperties(jobName); - assertEquals(expectedResponse.getStatusCode(), thrownException.getStatus()); + assertEquals(expectedResponse.getStatusCode(), thrownException.getHttpStatus()); } } diff --git a/projects/control-service/projects/versions-of-external-dependencies.gradle b/projects/control-service/projects/versions-of-external-dependencies.gradle index 2d462f1c9e..229ff8bc25 100644 --- a/projects/control-service/projects/versions-of-external-dependencies.gradle +++ b/projects/control-service/projects/versions-of-external-dependencies.gradle @@ -45,6 +45,7 @@ project.ext { 'com.amazonaws:aws-java-sdk-core' : 'com.amazonaws:aws-java-sdk-core:1.12.487', 'com.amazonaws:aws-java-sdk-sts' : 'com.amazonaws:aws-java-sdk-sts:1.12.486', 'com.amazonaws:aws-java-sdk-ecr' : 'com.amazonaws:aws-java-sdk-ecr:1.12.487', + 'org.springframework.vault:spring-vault-core' : 'org.springframework.vault:spring-vault-core:2.3.3', // transitive dependencies version force (freeze) // on next upgrade, revise if those still need to be set explicitly @@ -55,5 +56,6 @@ project.ext { 'com.squareup.okio:okio' : 'com.squareup.okio:okio:3.3.0', 'org.apache.commons:commons-compress' : 'org.apache.commons:commons-compress:1.23.0', 'org.hibernate:hibernate-jpamodelgen' : 'org.hibernate:hibernate-jpamodelgen:5.6.15.Final' + ] }