From faa8e413d04d762e9dceec22f090d5d9ef8c8660 Mon Sep 17 00:00:00 2001 From: Bence Csati <113284287+csatib02@users.noreply.github.com> Date: Thu, 22 Feb 2024 19:10:18 +0100 Subject: [PATCH] fix: handle different vault injection cases (#75) * fix: handle different vault injection cases Signed-off-by: Bence Csati * chore(multi-provider-test): adjust Signed-off-by: Bence Csati * feat(multi-provider-test): add testcase Signed-off-by: Bence Csati * chore: minor fix Signed-off-by: Bence Csati * chore: minor fix Signed-off-by: Bence Csati * chore: minor fix Signed-off-by: Bence Csati * fix: minor fix Signed-off-by: Bence Csati * fix(env_store_test.go): fix tests Signed-off-by: Bence Csati * fix(env_store_test.go): minor fix Signed-off-by: Bence Csati --------- Signed-off-by: Bence Csati --- common/config.go | 1 - e2e/multi-provider.bats | 57 +++++++++++++++++++++++++------- e2e/vault-provider.bats | 54 +++++++++++++++++++++++------- env_store.go | 6 +++- env_store_test.go | 62 ++++++++++++++++++++++++++--------- provider/file/config_test.go | 4 ++- provider/vault/config_test.go | 4 ++- 7 files changed, 144 insertions(+), 44 deletions(-) diff --git a/common/config.go b/common/config.go index 9d3e6fd..8b0f7d8 100644 --- a/common/config.go +++ b/common/config.go @@ -27,7 +27,6 @@ const ( LogServerEnv = "SECRET_INIT_LOG_SERVER" DaemonEnv = "SECRET_INIT_DAEMON" DelayEnv = "SECRET_INIT_DELAY" - ProviderEnv = "SECRET_INIT_PROVIDER" ) type Config struct { diff --git a/e2e/multi-provider.bats b/e2e/multi-provider.bats index 8af59ca..fe4e82b 100644 --- a/e2e/multi-provider.bats +++ b/e2e/multi-provider.bats @@ -28,9 +28,9 @@ setup_vault_provider() { export VAULT_ADDR="http://127.0.0.1:8200" export VAULT_TOKEN_FILE="$TMPFILE_TOKEN" - export MYSQL_PASSWORD=vault:secret/data/test/mysql#MYSQL_PASSWORD - export AWS_SECRET_ACCESS_KEY=vault:secret/data/test/aws#AWS_SECRET_ACCESS_KEY - export AWS_ACCESS_KEY_ID=vault:secret/data/test/aws#AWS_ACCESS_KEY_ID + export MYSQL_PASSWORD="vault:secret/data/test/mysql#MYSQL_PASSWORD" + export AWS_SECRET_ACCESS_KEY="vault:secret/data/test/aws#AWS_SECRET_ACCESS_KEY" + export AWS_ACCESS_KEY_ID="vault:secret/data/test/aws#AWS_ACCESS_KEY_ID" start_vault } @@ -63,24 +63,26 @@ add_secrets_to_vault() { docker exec "$vault_container_name" vault kv put secret/test/aws AWS_ACCESS_KEY_ID=secretId AWS_SECRET_ACCESS_KEY=s3cr3t } -remove_secrets_from_vault() { - docker exec "$vault_container_name" vault kv delete secret/test/mysql - docker exec "$vault_container_name" vault kv delete secret/test/aws +add_custom_secret_to_vault() { + local path="$1" + shift + local data=() + + for secret in "$@"; do + data+=("$secret") + done + + vault kv put "$path" "${data[@]}" } teardown() { - stop_vault + docker compose down rm -f "$TMPFILE_SECRET" rm -f "$TMPFILE_TOKEN" rm -f secret-init } -stop_vault() { - remove_secrets_from_vault - docker compose down -} - assert_output_contains() { local expected=$1 local output=$2 @@ -172,3 +174,34 @@ check_process_status() { assert_output_contains "AWS_ACCESS_KEY_ID=secretId" "$run_output" assert_output_contains "FILE_SECRET=secret-value" "$run_output" } + +@test "secrets successfully loaded using different injection cases" { + setup_file_provider + + setup_vault_provider + set_vault_token 227e1cce-6bf7-30bb-2d2a-acc854318caf + add_secrets_to_vault + + # Secret with version + add_custom_secret_to_vault "secret/test/mysql" "MYSQL_PASSWORD=modify3d3xtr3ms3cr3t" + export MYSQL_PASSWORD="vault:secret/data/test/mysql#MYSQL_PASSWORD#2" + + # Inline secrets with scheme + add_custom_secret_to_vault "secret/test/scheme" "SCHEME_SECRET1=sch3m3s3cr3tONE" "SCHEME_SECRET2=sch3m3s3cr3tTWO" + export SCHEME_SECRET="scheme://\${vault:secret/data/test/scheme#SCHEME_SECRET1}:\${vault:secret/data/test/scheme#SCHEME_SECRET2}@$VAULT_ADDR" + + # Enable pki secrets engine and generate root certificates + vault secrets enable -path=pki pki + export ROOT_CERT=">>vault:pki/root/generate/internal#certificate" + export ROOT_CERT_CACHED=">>vault:pki/root/generate/internal#certificate" + + run_output=$(./secret-init env | grep 'FILE_SECRET\|MYSQL_PASSWORD\|SCHEME_SECRET\|ROOT_CERT\|ROOT_CERT_CACHED') + assert_success + + assert_output_contains "FILE_SECRET=secret-value" "$run_output" + assert_output_contains "MYSQL_PASSWORD=modify3d3xtr3ms3cr3t" "$run_output" + assert_output_contains "SCHEME_SECRET=scheme://sch3m3s3cr3tONE:sch3m3s3cr3tTWO@$VAULT_ADDR" "$run_output" + + [ $ROOT_CERT == $ROOT_CERT_CACHED ] + assert_success "ROOT_CERT and ROOT_CERT_CACHED are not the same" +} diff --git a/e2e/vault-provider.bats b/e2e/vault-provider.bats index 8a882f2..3f246bf 100644 --- a/e2e/vault-provider.bats +++ b/e2e/vault-provider.bats @@ -15,9 +15,9 @@ setup_vault_provider() { export VAULT_ADDR="http://127.0.0.1:8200" export VAULT_TOKEN_FILE="$TMPFILE_TOKEN" - export MYSQL_PASSWORD=vault:secret/data/test/mysql#MYSQL_PASSWORD - export AWS_SECRET_ACCESS_KEY=vault:secret/data/test/aws#AWS_SECRET_ACCESS_KEY - export AWS_ACCESS_KEY_ID=vault:secret/data/test/aws#AWS_ACCESS_KEY_ID + export MYSQL_PASSWORD="vault:secret/data/test/mysql#MYSQL_PASSWORD" + export AWS_SECRET_ACCESS_KEY="vault:secret/data/test/aws#AWS_SECRET_ACCESS_KEY" + export AWS_ACCESS_KEY_ID="vault:secret/data/test/aws#AWS_ACCESS_KEY_ID" start_vault } @@ -50,23 +50,25 @@ add_secrets_to_vault() { docker exec "$vault_container_name" vault kv put secret/test/aws AWS_ACCESS_KEY_ID=secretId AWS_SECRET_ACCESS_KEY=s3cr3t } -remove_secrets_from_vault() { - docker exec "$vault_container_name" vault kv delete secret/test/mysql - docker exec "$vault_container_name" vault kv delete secret/test/aws +add_custom_secret_to_vault() { + local path="$1" + shift + local data=() + + for secret in "$@"; do + data+=("$secret") + done + + vault kv put "$path" "${data[@]}" } teardown() { - stop_vault + docker compose down rm -f "$TMPFILE_TOKEN" rm -f secret-init } -stop_vault() { - remove_secrets_from_vault - docker compose down -} - assert_output_contains() { local expected=$1 local output=$2 @@ -147,3 +149,31 @@ check_process_status() { assert_output_contains "AWS_SECRET_ACCESS_KEY=s3cr3t" "$run_output" assert_output_contains "AWS_ACCESS_KEY_ID=secretId" "$run_output" } + +@test "secrets sucessfully loaded from vault using different injection cases" { + setup_vault_provider + set_vault_token 227e1cce-6bf7-30bb-2d2a-acc854318caf + add_secrets_to_vault + + # Secret with version + add_custom_secret_to_vault "secret/test/mysql" "MYSQL_PASSWORD=modify3d3xtr3ms3cr3t" + export MYSQL_PASSWORD="vault:secret/data/test/mysql#MYSQL_PASSWORD#2" + + # Inline secrets with scheme + add_custom_secret_to_vault "secret/test/scheme" "SCHEME_SECRET1=sch3m3s3cr3tONE" "SCHEME_SECRET2=sch3m3s3cr3tTWO" + export SCHEME_SECRET="scheme://\${vault:secret/data/test/scheme#SCHEME_SECRET1}:\${vault:secret/data/test/scheme#SCHEME_SECRET2}@$VAULT_ADDR" + + # Enable pki secrets engine and generate root certificates + vault secrets enable -path=pki pki + export ROOT_CERT=">>vault:pki/root/generate/internal#certificate" + export ROOT_CERT_CACHED=">>vault:pki/root/generate/internal#certificate" + + run_output=$(./secret-init env | grep 'MYSQL_PASSWORD\|SCHEME_SECRET\|ROOT_CERT\|ROOT_CERT_CACHED') + assert_success + + assert_output_contains "MYSQL_PASSWORD=modify3d3xtr3ms3cr3t" "$run_output" + assert_output_contains "SCHEME_SECRET=scheme://sch3m3s3cr3tONE:sch3m3s3cr3tTWO@$VAULT_ADDR" "$run_output" + + [ $ROOT_CERT == $ROOT_CERT_CACHED ] + assert_success "ROOT_CERT and ROOT_CERT_CACHED are not the same" +} diff --git a/env_store.go b/env_store.go index 64d9b4d..d0414d6 100644 --- a/env_store.go +++ b/env_store.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "os" + "regexp" "strings" "sync" @@ -157,7 +158,10 @@ func getProviderPath(path string) (string, string) { var fileProviderName = file.ProviderName return fileProviderName, strings.TrimPrefix(path, "file:") } - if strings.HasPrefix(path, "vault:") { + // If the path contains some string formatted as "vault:{STR}#{STR}" + // it is most probably a vault path + re := regexp.MustCompile(`(vault:)(.*)#(.*)`) + if re.MatchString(path) { var vaultProviderName = vault.ProviderName // Do not remove the prefix since it will be processed during injection return vaultProviderName, path diff --git a/env_store_test.go b/env_store_test.go index c77dace..4def902 100644 --- a/env_store_test.go +++ b/env_store_test.go @@ -25,44 +25,76 @@ import ( ) func TestEnvStore_GetProviderPaths(t *testing.T) { - secretFile := newSecretFile(t, "secretId") - defer os.Remove(secretFile) - tests := []struct { name string + envs map[string]string wantPaths map[string][]string - addvault bool }{ { - name: "single provider", + name: "file provider", + envs: map[string]string{ + "AWS_SECRET_ACCESS_KEY_ID": "file:secret/data/test/aws", + }, wantPaths: map[string][]string{ "file": { - secretFile, + "secret/data/test/aws", + }, + }, + }, + { + name: "vault provider", + envs: map[string]string{ + "ACCOUNT_PASSWORD_1": "vault:secret/data/account#password#1", + "ACCOUNT_PASSWORD": "vault:secret/data/account#password", + "ROOT_CERT": ">>vault:pki/root/generate/internal#certificate", + "ROOT_CERT_CACHED": ">>vault:pki/root/generate/internal#certificate", + "INLINE_SECRET": "scheme://${vault:secret/data/account#username}:${vault:secret/data/account#password}@127.0.0.1:8080", + "INLINE_SECRET_EMBEDDED_TEMPLATE": "scheme://${vault:secret/data/account#username}:${vault:secret/data/account#${.password | urlquery}}@127.0.0.1:8080", + "INLINE_DYNAMIC_SECRET": "${>>vault:pki/root/generate/internal#certificate}__${>>vault:pki/root/generate/internal#certificate}", + }, + wantPaths: map[string][]string{ + "vault": { + "ACCOUNT_PASSWORD_1=vault:secret/data/account#password#1", + "ACCOUNT_PASSWORD=vault:secret/data/account#password", + "ROOT_CERT=>>vault:pki/root/generate/internal#certificate", + "ROOT_CERT_CACHED=>>vault:pki/root/generate/internal#certificate", + "INLINE_SECRET=scheme://${vault:secret/data/account#username}:${vault:secret/data/account#password}@127.0.0.1:8080", + "INLINE_SECRET_EMBEDDED_TEMPLATE=scheme://${vault:secret/data/account#username}:${vault:secret/data/account#${.password | urlquery}}@127.0.0.1:8080", + "INLINE_DYNAMIC_SECRET=${>>vault:pki/root/generate/internal#certificate}__${>>vault:pki/root/generate/internal#certificate}", }, }, - addvault: false, }, { name: "multi provider", + envs: map[string]string{ + "AWS_SECRET_ACCESS_KEY_ID": "file:secret/data/test/aws", + "MYSQL_PASSWORD": "vault:secret/data/test/mysql#MYSQL_PASSWORD", + "AWS_SECRET_ACCESS_KEY": "vault:secret/data/test/aws#AWS_SECRET_ACCESS_KEY", + }, wantPaths: map[string][]string{ "vault": { "MYSQL_PASSWORD=vault:secret/data/test/mysql#MYSQL_PASSWORD", "AWS_SECRET_ACCESS_KEY=vault:secret/data/test/aws#AWS_SECRET_ACCESS_KEY", }, "file": { - secretFile, + "secret/data/test/aws", }, }, - addvault: true, }, } for _, tt := range tests { ttp := tt t.Run(ttp.name, func(t *testing.T) { - createEnvsForProvider(ttp.addvault, secretFile) - envStore := NewEnvStore() - paths := envStore.GetProviderPaths() + // prepare envs + for envKey, envVal := range ttp.envs { + os.Setenv(envKey, envVal) + } + t.Cleanup(func() { + os.Clearenv() + }) + + paths := NewEnvStore().GetProviderPaths() for key, expectedSlice := range ttp.wantPaths { actualSlice, ok := paths[key] @@ -126,9 +158,8 @@ func TestEnvStore_GetProviderSecrets(t *testing.T) { ttp := tt t.Run(ttp.name, func(t *testing.T) { createEnvsForProvider(ttp.addvault, secretFile) - envStore := NewEnvStore() - providerSecrets, err := envStore.LoadProviderSecrets(ttp.providerPaths) + providerSecrets, err := NewEnvStore().LoadProviderSecrets(ttp.providerPaths) if err != nil { assert.EqualError(t, ttp.err, err.Error(), "Unexpected error message") } @@ -184,9 +215,8 @@ func TestEnvStore_ConvertProviderSecrets(t *testing.T) { ttp := tt t.Run(ttp.name, func(t *testing.T) { createEnvsForProvider(ttp.addvault, secretFile) - envStore := NewEnvStore() - secretsEnv, err := envStore.ConvertProviderSecrets(ttp.providerSecrets) + secretsEnv, err := NewEnvStore().ConvertProviderSecrets(ttp.providerSecrets) if err != nil { assert.EqualError(t, ttp.err, err.Error(), "Unexpected error message") } diff --git a/provider/file/config_test.go b/provider/file/config_test.go index feda721..ced85d4 100644 --- a/provider/file/config_test.go +++ b/provider/file/config_test.go @@ -47,7 +47,9 @@ func TestConfig(t *testing.T) { for envKey, envVal := range ttp.env { os.Setenv(envKey, envVal) } - defer os.Clearenv() + t.Cleanup(func() { + os.Clearenv() + }) config := LoadConfig() diff --git a/provider/vault/config_test.go b/provider/vault/config_test.go index 73002e3..e6ba67c 100644 --- a/provider/vault/config_test.go +++ b/provider/vault/config_test.go @@ -112,7 +112,9 @@ func TestConfig(t *testing.T) { for envKey, envVal := range ttp.env { os.Setenv(envKey, envVal) } - defer os.Clearenv() + t.Cleanup(func() { + os.Clearenv() + }) config, err := LoadConfig() if err != nil {