Skip to content

Commit

Permalink
control-service: switch to Approle Vault authentication (#2435)
Browse files Browse the repository at this point in the history
Switch to using Approle for Vault Authentication rather than Tokens.

Approle authentication is more suitable for long running applications,
like the Control Service. 

https://developer.hashicorp.com/vault/docs/auth/approle 
https://developer.hashicorp.com/vault/tutorials/auth-methods/approle

Update service code, VEP, Tests and helm charts.

---------

Signed-off-by: Dako Dakov <[email protected]>
Co-authored-by: github-actions <>
  • Loading branch information
dakodakov authored Jul 21, 2023
1 parent d471dd4 commit 0cbeb9f
Show file tree
Hide file tree
Showing 8 changed files with 222 additions and 35 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ Generate default Vault configuration.
*/}}
{{- define "pipelines-control-service.vaultSecret" -}}
URI: {{ default "http://localhost:8200" .Values.secrets.vault.uri | b64enc | quote }}
TOKEN: {{ default "root" .Values.secrets.vault.token | b64enc | quote }}
ROLEID: {{ default "root" .Values.secrets.vault.approle.roleid | b64enc | quote }}
SECRETID: {{ default "root" .Values.secrets.vault.approle.secretid | b64enc | quote }}
{{- end -}}

{{/*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -307,11 +307,16 @@ spec:
secretKeyRef:
name: { { .Values.secrets.vault.externalSecretName | default (include "pipelines-control-service.vaultSecretName" . ) } }
key: URI
- name: VDK_VAULT_TOKEN
- name: VDK_VAULT_APPROLE_ROLEID
valueFrom:
secretKeyRef:
name: { { .Values.secrets.vault.externalSecretName | default (include "pipelines-control-service.vaultSecretName" . ) } }
key: TOKEN
key: ROLEID
- name: VDK_VAULT_APPROLE_SECRETID
valueFrom:
secretKeyRef:
name: { { .Values.secrets.vault.externalSecretName | default (include "pipelines-control-service.vaultSecretName" . ) } }
key: SECRETID
- name: DATAJOBS_VAULT_SIZE_LIMIT_BYTES
value: "{{ .Values.secrets.vault.sizeLimitBytes }}"
{{- end }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1093,10 +1093,13 @@ alertmanager:
secrets:
vault:
enabled: false
## Name of the secret which holds Vault URI and Token. The chart will not attempt to create this, but will use it as is.
## The secret should contain keys: URI, TOKEN
## Name of the secret which holds Vault URI and Approle RoleId and SecretId. The chart will not attempt to
## create this, but will use it as is.
## The secret should contain keys: URI, ROLEID, SECRETID
externalSecretName: ""
## Alternatively provide the uri and token here. externalSecretName takes precedence if both are set.
## Alternatively provide the uri and Approle Settings here. externalSecretName takes precedence if both are set.
uri: "http://localhost:8200"
token: "root"
approle:
roleid: foo
secretid: foo
sizeLimitBytes: 1048576
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,18 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.vault.authentication.TokenAuthentication;
import org.springframework.vault.client.VaultEndpoint;
import org.springframework.vault.core.VaultTemplate;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.vault.VaultContainer;

import java.net.URI;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import static com.vmware.taurus.secrets.service.vault.VaultTestSetup.setupVaultTemplate;
import static org.junit.jupiter.api.Assertions.assertThrows;

@SpringBootTest(
Expand All @@ -37,18 +36,18 @@ public class TestVaultJobSecretsServiceIT extends BaseIT {

@Container
private static final VaultContainer vaultContainer =
new VaultContainer<>("vault:1.0.2").withVaultToken("root");
new VaultContainer<>("vault:1.13.3").withVaultToken("root");

private static VaultJobSecretsService vaultJobSecretService;

@BeforeAll
public static void init() throws URISyntaxException {
public static void init() throws URISyntaxException, IOException, InterruptedException {
String vaultUri = vaultContainer.getHttpHostAddress();

VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri));
TokenAuthentication clientAuthentication = new TokenAuthentication("root");
// Setup vault app roles authentication
// https://developer.hashicorp.com/vault/tutorials/auth-methods/approle

VaultTemplate vaultTemplate = new VaultTemplate(vaultEndpoint, clientAuthentication);
VaultTemplate vaultTemplate = setupVaultTemplate(vaultUri, vaultContainer);

vaultJobSecretService = new VaultJobSecretsService(vaultTemplate);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Copyright 2021-2023 VMware, Inc.
* SPDX-License-Identifier: Apache-2.0
*/

package com.vmware.taurus.secrets.service.vault;

import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClients;
import org.jetbrains.annotations.NotNull;
import org.springframework.vault.authentication.AppRoleAuthentication;
import org.springframework.vault.authentication.AppRoleAuthenticationOptions;
import org.springframework.vault.client.VaultEndpoint;
import org.springframework.vault.core.VaultTemplate;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.DefaultUriBuilderFactory;
import org.testcontainers.containers.ContainerState;
import org.testcontainers.vault.VaultContainer;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class VaultTestSetup {

@NotNull
static VaultTemplate setupVaultTemplate(String vaultUri, VaultContainer vaultContainer)
throws IOException, InterruptedException, URISyntaxException {
setupAppRole(vaultUri, vaultContainer);
String roleId = getRoleId(vaultContainer);
String secretId = getSecretId(vaultContainer);
// create the authentication
AppRoleAuthentication clientAuthentication =
getAppRoleAuthentication(vaultUri, roleId, secretId);
VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri + "/v1/"));
VaultTemplate vaultTemplate = new VaultTemplate(vaultEndpoint, clientAuthentication);
return vaultTemplate;
}

private static void setupAppRole(String vaultUri, ContainerState vaultContainer)
throws IOException, InterruptedException {
vaultContainer.execInContainer("vault", "auth", "enable", "approle");

// Create a new test policy via rest as there's no good way to do it via the command mechanism
HttpClient httpClient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(vaultUri + "/v1/sys/policies/acl/testpolicy");
httpPost.setHeader("X-Vault-Token", "root");
StringEntity requestBody =
new StringEntity(
"{\n"
+ " \"policy\": \"path \\\"secret/*\\\" {\\n"
+ " capabilities = [ \\\"create\\\", \\\"read\\\",\\\"update\\\", \\\"patch\\\","
+ " \\\"delete\\\",\\\"list\\\" ]\\n"
+ "}\"\n"
+ "}");
httpPost.setEntity(requestBody);
httpClient.execute(httpPost);

// create "test" role with the policy
vaultContainer.execInContainer(
"vault",
"write",
"auth/approle/role/test",
"token_policies=testpolicy",
"token_ttl=1h",
"token_max_ttl=4h");
}

@NotNull
private static String getRoleId(ContainerState vaultContainer)
throws IOException, InterruptedException {
org.testcontainers.containers.Container.ExecResult execResult =
vaultContainer.execInContainer(
"vault", "read", "auth/approle/role/test/role-id"); // read the role-id
String output = execResult.getStdout();
String roleId = output.substring(output.lastIndexOf(" ")).trim();
return roleId;
}

@NotNull
private static String getSecretId(ContainerState vaultContainer)
throws IOException, InterruptedException {
org.testcontainers.containers.Container.ExecResult execResult =
vaultContainer.execInContainer(
"vault", "write", "-force", "auth/approle/role/test/secret-id"); // read the secret-id
String output = execResult.getStdout();
String secretId =
output
.substring(output.indexOf("secret_id") + 9, output.indexOf("secret_id_accessor"))
.trim();
return secretId;
}

@NotNull
private static AppRoleAuthentication getAppRoleAuthentication(
String vaultUri, String roleId, String secretId) {
AppRoleAuthenticationOptions.AppRoleAuthenticationOptionsBuilder builder =
AppRoleAuthenticationOptions.builder()
.roleId(AppRoleAuthenticationOptions.RoleId.provided(roleId))
.secretId(AppRoleAuthenticationOptions.SecretId.provided(secretId));

RestTemplate restTemplate = new RestTemplate();
restTemplate.setUriTemplateHandler(new DefaultUriBuilderFactory(vaultUri + "/v1/"));

AppRoleAuthentication clientAuthentication =
new AppRoleAuthentication(builder.build(), restTemplate);
return clientAuthentication;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,83 @@

package com.vmware.taurus.secrets.service.vault;

import com.vmware.taurus.exception.SecretStorageNotConfiguredException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.vault.authentication.TokenAuthentication;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.lang.Nullable;
import org.springframework.vault.authentication.*;
import org.springframework.vault.client.VaultEndpoint;
import org.springframework.vault.config.AbstractVaultConfiguration;
import org.springframework.vault.core.VaultOperations;
import org.springframework.vault.core.VaultTemplate;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.DefaultUriBuilderFactory;

import java.net.URI;
import java.net.URISyntaxException;

@Slf4j
@Configuration
public class VaultConfiguration {
public class VaultConfiguration extends AbstractVaultConfiguration {

@Value("${vdk.vault.uri:}")
String vaultUri;

@Value("${vdk.vault.approle.roleid:}")
String roleId;

@Value("${vdk.vault.approle.secretid:}")
String secretId;

@Value("${vdk.vault.token:}")
@Nullable
String vaultToken;

@Bean
public VaultOperations vaultOperations(
@Value("${vdk.vault.uri:}") String vaultUri, @Value("${vdk.vault.token:}") String vaultToken)
throws URISyntaxException {
VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri));
TokenAuthentication clientAuthentication = new TokenAuthentication(vaultToken);
VaultEndpoint vaultEndpoint, SessionManager sessionManager) {

SimpleClientHttpRequestFactory clientHttpRequestFactory = new SimpleClientHttpRequestFactory();
return new VaultTemplate(vaultEndpoint, clientHttpRequestFactory, sessionManager);
}

@NotNull
@Override
@Bean
public VaultEndpoint vaultEndpoint() {
try {
return VaultEndpoint.from(new URI(this.vaultUri));
} catch (URISyntaxException e) {
throw new SecretStorageNotConfiguredException();
}
}

@NotNull
@Override
@Bean
public ClientAuthentication clientAuthentication() {
// Token authentication should only be used for development purposes. If the token expires, the
// secrets
// functionality will stop working
if (StringUtils.isNotBlank(vaultToken)) {
log.warn("Initializing vault integration with Token Authentication.");
return new TokenAuthentication(vaultToken);
} else {
log.info("Initializing vault integration with AppRole Authentication.");
AppRoleAuthenticationOptions.AppRoleAuthenticationOptionsBuilder builder =
AppRoleAuthenticationOptions.builder()
.roleId(AppRoleAuthenticationOptions.RoleId.provided(this.roleId))
.secretId(AppRoleAuthenticationOptions.SecretId.provided(this.secretId));

RestTemplate restTemplate = new RestTemplate();
restTemplate.setUriTemplateHandler(new DefaultUriBuilderFactory(vaultUri));

return new VaultTemplate(vaultEndpoint, clientAuthentication);
return new AppRoleAuthentication(builder.build(), restTemplate);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,18 @@ datajobs.aws.defaultSessionDurationSeconds=${DATAJOBS_AWS_DEFAULT_SESSION_DURATI

# Hashicorp Vault Integration settings
# When disabled/not configured the Secrets functionality won't work
# In production, we would only use AppRole Authentication which require setup in vault:
# https://developer.hashicorp.com/vault/docs/auth/approle
# https://developer.hashicorp.com/vault/tutorials/auth-methods/approle
# and should provide roleid and secretid to the service.
#
# For local development you can start a vault server with the following command:
# vault server -dev -dev-root-token-id="root"
# and configure only the uri and the token
featureflag.vault.integration.enabled=false
vdk.vault.uri=http://localhost:8200
# If you get 404 errors related to vault operations, double-check the vault URI, it should usually end with "/v1/"
vdk.vault.uri=http://localhost:8200/v1/
vdk.vault.approle.roleid=
vdk.vault.approle.secretid=
vdk.vault.token=root
datajobs.vault.size.limit.bytes=1048576
24 changes: 12 additions & 12 deletions specs/vep-1493-vault-integration/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -258,19 +258,19 @@ Changes to the properties cli command:
We are going to enhance the VDK-CS configuration with an optional Spring Vault Configuration.

```yaml
spring.cloud.vault:
host: localhost
port: 8200
scheme: https
uri: https://localhost:8200
connection-timeout: 5000
read-timeout: 15000
config:
token: 19aefa97-cccc-bbbb-aaaa-225940e63d76
# Hashicorp Vault Integration settings
# When disabled/not configured the Secrets functionality won't work
featureflag.vault.integration.enabled=true
vdk.vault.uri=https://localhost:8200/v1/
vdk.vault.approle.roleid=z2d59142-9b53-d163-1ae9-d6286d3dfe22
vdk.vault.approle.secretid=c336f9e9-1555-8c29-9a7f-283608d06fbd
datajobs.vault.size.limit.bytes=1048576
```
The configuration can be marked optional as outlined in the
[documentation](https://docs.spring.io/spring-cloud-vault/docs/current/reference/html/#vault.configdata.location.optional)
which allows users who are not interested in using a secret storage, to simply disable this feature.
The feature flag allows users who are not interested in using a secret storage, to simply disable this feature.

The VDK Control Service is going to authenticate using Vault's [AppRole mechanism](https://developer.hashicorp.com/vault/tutorials/auth-methods/approle).
THe Service Operator should setup the App Role authentication based on their needs in Vault, and provide the Vault URI
the App Role's Role ID and Secret ID.

### Secrets service

Expand Down

0 comments on commit 0cbeb9f

Please sign in to comment.