diff --git a/e2e/consul/consul.go b/e2e/consul/consul.go index f09813ad0c7..71bccdc6b6d 100644 --- a/e2e/consul/consul.go +++ b/e2e/consul/consul.go @@ -23,9 +23,6 @@ const ( consulJobRegisterOnUpdatePart1 = "consul/input/services_empty.nomad" consulJobRegisterOnUpdatePart2 = "consul/input/services_present.nomad" - - consulPolicyServiceInput = "/consul/input/consul-policy-for-nomad.hcl" - consulPolicyTaskInput = "/consul/input/consul-policy-for-tasks.hcl" ) const ( @@ -55,12 +52,6 @@ func init() { func (tc *ConsulE2ETest) BeforeAll(f *framework.F) { e2eutil.WaitForLeader(f.T(), tc.Nomad()) e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1) - - // setup consul ACL's for WI auth - e2eutil.SetupConsulACLsForServices(f.T(), tc.Consul(), consulPolicyServiceInput) - e2eutil.SetupConsulServiceIntentions(f.T(), tc.Consul()) - e2eutil.SetupConsulACLsForTasks(f.T(), tc.Consul(), "nomad-default", consulPolicyTaskInput) - e2eutil.SetupConsulJWTAuth(f.T(), tc.Consul(), tc.Nomad().Address(), nil) } func (tc *ConsulE2ETest) AfterEach(f *framework.F) { diff --git a/e2e/consul/consul_test.go b/e2e/consul/consul_test.go index 70a08b598a1..502762204a1 100644 --- a/e2e/consul/consul_test.go +++ b/e2e/consul/consul_test.go @@ -13,17 +13,10 @@ func TestConsul(t *testing.T) { // todo: migrate the remaining consul tests nomad := e2eutil.NomadClient(t) - consul := e2eutil.ConsulClient(t) e2eutil.WaitForLeader(t, nomad) e2eutil.WaitForNodesReady(t, nomad, 1) - // setup consul ACL's for WI auth - e2eutil.SetupConsulACLsForServices(t, consul, consulPolicyServiceInput) - e2eutil.SetupConsulServiceIntentions(t, consul) - e2eutil.SetupConsulACLsForTasks(t, consul, "nomad-default", consulPolicyTaskInput) - e2eutil.SetupConsulJWTAuth(t, consul, nomad.Address(), nil) - t.Run("testServiceReversion", testServiceReversion) t.Run("testAllocRestart", testAllocRestart) } diff --git a/e2e/consul/input/consul-policy-for-nomad.hcl b/e2e/consul/input/consul-policy-for-nomad.hcl deleted file mode 100755 index ddf9de81f36..00000000000 --- a/e2e/consul/input/consul-policy-for-nomad.hcl +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -# Policy for the Nomad agent. Note that with this policy we must use Workload -# Identity for Connect jobs, or we'll get "failed to derive SI token" errors -# from the client because the Nomad agent's token doesn't have "acl:write" - -# The operator:write permission is required for creating config entries for -# connect ingress gateways. operator ACLs are not namespaced, though the -# config entries they can generate are. -operator = "write" - -agent_prefix "" { - policy = "read" -} - -node_prefix "" { - policy = "read" -} - -service_prefix "nomad" { - policy = "write" -} - -service_prefix "" { - policy = "read" -} - -# for use with Consul ENT -namespace_prefix "prod" { - - node_prefix "" { - policy = "read" - } - - service_prefix "nomad" { - policy = "write" - } - - service_prefix "" { - policy = "read" - } - -} diff --git a/e2e/consul/input/consul-policy-for-tasks.hcl b/e2e/consul/input/consul-policy-for-tasks.hcl deleted file mode 100755 index 28d31883bf4..00000000000 --- a/e2e/consul/input/consul-policy-for-tasks.hcl +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -// policy without namespaces, for Consul CE. This policy is for Nomad tasks -// using WI so they can read services and KV from Consul when rendering templates. - -key_prefix "" { - policy = "read" -} - -service_prefix "" { - policy = "read" -} diff --git a/e2e/consulcompat/run_ce_test.go b/e2e/consulcompat/run_ce_test.go index 7477ee31513..ab8caa73996 100644 --- a/e2e/consulcompat/run_ce_test.go +++ b/e2e/consulcompat/run_ce_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/hashicorp/go-version" - "github.com/hashicorp/nomad/e2e/e2eutil" "github.com/hashicorp/nomad/testutil" ) @@ -39,14 +38,14 @@ func testConsulBuild(t *testing.T, b build, baseDir string) { // Note that with this policy we must use Workload Identity for Connect // jobs, or we'll get "failed to derive SI token" errors from the client // because the Nomad agent's token doesn't have "acl:write" - token := e2eutil.SetupConsulACLsForServices(t, consulAPI, + token := setupConsulACLsForServices(t, consulAPI, "./input/consul-policy-for-nomad.hcl") // we need service intentions so Connect apps can reach each other, and // an ACL role and policy that tasks will be able to use to render // templates - e2eutil.SetupConsulServiceIntentions(t, consulAPI) - e2eutil.SetupConsulACLsForTasks(t, consulAPI, + setupConsulServiceIntentions(t, consulAPI) + setupConsulACLsForTasks(t, consulAPI, "nomad-default", "./input/consul-policy-for-tasks.hcl") // note: Nomad needs to be live before we can setup Consul auth methods @@ -72,7 +71,7 @@ func testConsulBuild(t *testing.T, b build, baseDir string) { nc := startNomad(t, consulCfg) // configure authentication for WI to Consul - e2eutil.SetupConsulJWTAuth(t, consulAPI, nc.Address(), nil) + setupConsulJWTAuth(t, consulAPI, nc.Address(), nil) verifyConsulFingerprint(t, nc, b.Version, "default") runConnectJob(t, nc, "default", "./input/connect.nomad.hcl") diff --git a/e2e/consulcompat/shared_run_test.go b/e2e/consulcompat/shared_run_test.go index 52f355d9c6a..cbf7a2a51f1 100644 --- a/e2e/consulcompat/shared_run_test.go +++ b/e2e/consulcompat/shared_run_test.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/go-version" "github.com/hashicorp/nomad/api" nomadapi "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper/uuid" "github.com/shoenig/test/must" "github.com/shoenig/test/wait" ) @@ -177,3 +178,122 @@ func runConnectJob(t *testing.T, nc *nomadapi.Client, ns, filePath string) { _, _, err = nc.AllocFS().Stat(alloc, "dashboard/local/count-api.txt", nil) must.NoError(t, err) } + +// setupConsulACLsForServices installs a base set of ACL policies and returns a +// token that the Nomad agent can use +func setupConsulACLsForServices(t *testing.T, consulAPI *consulapi.Client, policyFilePath string) string { + + d, err := os.Getwd() + must.NoError(t, err) + t.Log(d) + policyRules, err := os.ReadFile(policyFilePath) + must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath)) + + policy := &consulapi.ACLPolicy{ + Name: "nomad-cluster-" + uuid.Short(), + Description: "policy for nomad agent", + Rules: string(policyRules), + } + + policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil) + must.NoError(t, err, must.Sprint("could not write policy to Consul")) + + token := &consulapi.ACLToken{ + Description: "token for Nomad agent", + Policies: []*consulapi.ACLLink{{ + ID: policy.ID, + Name: policy.Name, + }}, + } + token, _, err = consulAPI.ACL().TokenCreate(token, nil) + must.NoError(t, err, must.Sprint("could not create token in Consul")) + + return token.SecretID +} + +func setupConsulServiceIntentions(t *testing.T, consulAPI *consulapi.Client) { + ixn := &consulapi.Intention{ + SourceName: "count-dashboard", + DestinationName: "count-api", + Action: "allow", + } + _, err := consulAPI.Connect().IntentionUpsert(ixn, nil) + must.NoError(t, err, must.Sprint("could not create intention")) +} + +// setupConsulACLsForTasks installs a base set of ACL policies and returns a +// token that the Nomad agent can use +func setupConsulACLsForTasks(t *testing.T, consulAPI *consulapi.Client, roleName, policyFilePath string) { + + policyRules, err := os.ReadFile(policyFilePath) + must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath)) + + policy := &consulapi.ACLPolicy{ + Name: "nomad-tasks-" + uuid.Short(), + Description: "policy for nomad tasks", + Rules: string(policyRules), + } + + policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil) + must.NoError(t, err, must.Sprint("could not write policy to Consul")) + + role := &consulapi.ACLRole{ + Name: roleName, // note: must match "prod-${nomad_namespace}" + Description: "role for nomad tasks", + Policies: []*consulapi.ACLLink{{ + ID: policy.ID, + Name: policy.Name, + }}, + } + _, _, err = consulAPI.ACL().RoleCreate(role, nil) + must.NoError(t, err, must.Sprint("could not create token in Consul")) +} + +func setupConsulJWTAuth(t *testing.T, consulAPI *consulapi.Client, address string, namespaceRules []*consulapi.ACLAuthMethodNamespaceRule) { + + authConfig := map[string]any{ + "JWKSURL": fmt.Sprintf("%s/.well-known/jwks.json", address), + "JWTSupportedAlgs": []string{"RS256"}, + "BoundAudiences": "consul.io", + "ClaimMappings": map[string]string{ + "nomad_namespace": "nomad_namespace", + "nomad_job_id": "nomad_job_id", + "nomad_task": "nomad_task", + "nomad_service": "nomad_service", + }, + } + + _, _, err := consulAPI.ACL().AuthMethodCreate(&consulapi.ACLAuthMethod{ + Name: "nomad-workloads", + Type: "jwt", + DisplayName: "nomad-workloads", + Description: "login method for Nomad tasks with workload identity (WI)", + MaxTokenTTL: time.Hour, + TokenLocality: "local", + Config: authConfig, + NamespaceRules: namespaceRules, + }, nil) + must.NoError(t, err, must.Sprint("could not create Consul auth method for Nomad workloads")) + + rule := &consulapi.ACLBindingRule{ + ID: "", + Description: "binding rule for Nomad workload identities (WI) for tasks", + AuthMethod: "nomad-workloads", + Selector: `"nomad_service" not in value`, + BindType: "role", + BindName: "nomad-${value.nomad_namespace}", + } + _, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil) + must.NoError(t, err, must.Sprint("could not create Consul binding rule")) + + rule = &consulapi.ACLBindingRule{ + ID: "", + Description: "binding rule for Nomad workload identities (WI) for services", + AuthMethod: "nomad-workloads", + Selector: `"nomad_service" in value`, + BindType: "service", + BindName: "${value.nomad_service}", + } + _, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil) + must.NoError(t, err, must.Sprint("could not create Consul binding rule")) +} diff --git a/e2e/e2eutil/consul.go b/e2e/e2eutil/consul.go index 18ef0949bdc..e156ecb635d 100644 --- a/e2e/e2eutil/consul.go +++ b/e2e/e2eutil/consul.go @@ -232,128 +232,3 @@ func DeleteConsulTokens(t *testing.T, client *capi.Client, tokens map[string][]s } } } - -// SetupConsulACLsForServices installs a base set of ACL policies and returns a -// token that the Nomad agent can use -func SetupConsulACLsForServices(t *testing.T, consulAPI *capi.Client, policyFilePath string) string { - - d, err := os.Getwd() - must.NoError(t, err) - t.Log(d) - policyRules, err := os.ReadFile(policyFilePath) - must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath)) - - policy := &capi.ACLPolicy{ - Name: "nomad-cluster-" + uuid.Short(), - Description: "policy for nomad agent", - Rules: string(policyRules), - } - - policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil) - must.NoError(t, err, must.Sprint("could not write policy to Consul")) - - token := &capi.ACLToken{ - Description: "token for Nomad agent", - Policies: []*capi.ACLLink{{ - ID: policy.ID, - Name: policy.Name, - }}, - } - token, _, err = consulAPI.ACL().TokenCreate(token, nil) - must.NoError(t, err, must.Sprint("could not create token in Consul")) - - return token.SecretID -} - -func SetupConsulServiceIntentions(t *testing.T, consulAPI *capi.Client) { - ixn := &capi.Intention{ - SourceName: "count-dashboard", - DestinationName: "count-api", - Action: "allow", - } - _, err := consulAPI.Connect().IntentionUpsert(ixn, nil) - must.NoError(t, err, must.Sprint("could not create intention")) -} - -// SetupConsulACLsForTasks installs a base set of ACL policies and returns a -// token that the Nomad agent can use -func SetupConsulACLsForTasks(t *testing.T, consulAPI *capi.Client, roleName, policyFilePath string) { - - policyRules, err := os.ReadFile(policyFilePath) - must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath)) - - policy := &capi.ACLPolicy{ - Name: "nomad-tasks-" + uuid.Short(), - Description: "policy for nomad tasks", - Rules: string(policyRules), - } - - policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil) - must.NoError(t, err, must.Sprint("could not write policy to Consul")) - - role := &capi.ACLRole{ - Name: roleName, // note: must match "prod-${nomad_namespace}" - Description: "role for nomad tasks", - Policies: []*capi.ACLLink{{ - ID: policy.ID, - Name: policy.Name, - }}, - } - _, _, err = consulAPI.ACL().RoleCreate(role, nil) - if err != nil { - // TODO: because we run two types of Consul E2E tests, these setup - // functions will run twice. This is okay for all except when - // creating roles. So if the role already exists, just continue. - // When old framework tests are migrated, this should be removed. - must.ErrorContains(t, err, "already exists") - } -} - -func SetupConsulJWTAuth(t *testing.T, consulAPI *capi.Client, address string, namespaceRules []*capi.ACLAuthMethodNamespaceRule) { - - authConfig := map[string]any{ - "JWKSURL": fmt.Sprintf("%s/.well-known/jwks.json", address), - "JWTSupportedAlgs": []string{"RS256"}, - "BoundAudiences": "consul.io", - "ClaimMappings": map[string]string{ - "nomad_namespace": "nomad_namespace", - "nomad_job_id": "nomad_job_id", - "nomad_task": "nomad_task", - "nomad_service": "nomad_service", - }, - } - - _, _, err := consulAPI.ACL().AuthMethodCreate(&capi.ACLAuthMethod{ - Name: "nomad-workloads", - Type: "jwt", - DisplayName: "nomad-workloads", - Description: "login method for Nomad tasks with workload identity (WI)", - MaxTokenTTL: time.Hour, - TokenLocality: "local", - Config: authConfig, - NamespaceRules: namespaceRules, - }, nil) - must.NoError(t, err, must.Sprint("could not create Consul auth method for Nomad workloads")) - - rule := &capi.ACLBindingRule{ - ID: "", - Description: "binding rule for Nomad workload identities (WI) for tasks", - AuthMethod: "nomad-workloads", - Selector: `"nomad_service" not in value`, - BindType: "role", - BindName: "nomad-${value.nomad_namespace}", - } - _, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil) - must.NoError(t, err, must.Sprint("could not create Consul binding rule")) - - rule = &capi.ACLBindingRule{ - ID: "", - Description: "binding rule for Nomad workload identities (WI) for services", - AuthMethod: "nomad-workloads", - Selector: `"nomad_service" in value`, - BindType: "service", - BindName: "${value.nomad_service}", - } - _, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil) - must.NoError(t, err, must.Sprint("could not create Consul binding rule")) -} diff --git a/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-client-policy.hcl b/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-client-policy.hcl deleted file mode 100644 index c07dc09b03a..00000000000 --- a/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-client-policy.hcl +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -// The Nomad Client will be registering things into its buddy Consul Client. -// Note: because we also test the use of Consul namespaces, this token must be -// able to register services, read the keystore, and read node data for any -// namespace. -// The operator=write permission is required for creating config entries for -// connect ingress gateways. operator ACLs are not namespaced, though the -// config entries they can generate are. -operator = "write" - -agent_prefix "" { - policy = "read" -} - -namespace_prefix "" { - // The acl=write permission is required for generating Consul Service Identity - // tokens for consul connect services. Those services could be configured for - // any Consul namespace the job-submitter has access to. - acl = "write" - - key_prefix "" { - policy = "read" - } - - node_prefix "" { - policy = "read" - } - - service_prefix "" { - policy = "write" - } -} diff --git a/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-server-policy.hcl b/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-server-policy.hcl deleted file mode 100644 index 5df4224668d..00000000000 --- a/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-server-policy.hcl +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -// The operator=write permission is required for creating config entries for -// connect ingress gateways. operator ACLs are not namespaced, though the -// config entries they can generate are. -operator = "write" - -agent_prefix "" { - policy = "read" -} - -namespace_prefix "" { - // The acl=write permission is required for generating Consul Service Identity - // tokens for consul connect services. Those services could be configured for - // any Consul namespace the job-submitter has access to. - acl = "write" -} - -service_prefix "" { - policy = "write" -} - -agent_prefix "" { - policy = "read" -} - -node_prefix "" { - policy = "read" -} diff --git a/e2e/terraform/provision-infra/scripts/bootstrap-consul.sh b/e2e/terraform/provision-infra/scripts/bootstrap-consul.sh index 3c688a7a736..13c52821577 100755 --- a/e2e/terraform/provision-infra/scripts/bootstrap-consul.sh +++ b/e2e/terraform/provision-infra/scripts/bootstrap-consul.sh @@ -31,4 +31,36 @@ echo "writing Consul cluster policy and token" consul acl policy create -name consul-agents -rules @${DIR}/consul-agents-policy.hcl consul acl token create -policy-name=consul-agents -secret "$CONSUL_AGENT_TOKEN" -echo "Consul successfully bootstraped!" \ No newline at end of file +# The following ACL's are used so Nomad services and tasks can register +# via Workload Identity +echo "writing ACLs for Nomad Workload Identity integration..." + +echo "writing Consul auth-method" +consul acl auth-method create \ + -name 'nomad-workloads' \ + -type 'jwt' \ + -config @${DIR}/consul-workload-identity/auth-method.json \ + -namespace-rule-selector '"consul_namespace" in value' \ + -namespace-rule-bind-namespace '${value.consul_namespace}' + +echo "writing binding-rule for Nomad services" +consul acl binding-rule create \ + -method 'nomad-workloads' \ + -bind-type 'service' \ + -bind-name '${value.nomad_service}' \ + -selector '"nomad_service" in value' + +echo "writing binding-rule for Nomad tasks" +consul acl binding-rule create \ + -method 'nomad-workloads' \ + -bind-type 'role' \ + -bind-name 'nomad-tasks-${value.nomad_namespace}' \ + -selector '"nomad_service" not in value' + +echo "writing policy for Nomad tasks" +consul acl policy create -name policy-nomad-tasks -rules @${DIR}/consul-workload-identity/nomad-task-policy.hcl + +echo "creating role for Nomad tasks using previously created policy" +consul acl role create -name nomad-default-tasks -policy-name policy-nomad-tasks + +echo "Consul successfully bootstraped!" diff --git a/e2e/terraform/provision-infra/scripts/consul-workload-identity/auth-method.json b/e2e/terraform/provision-infra/scripts/consul-workload-identity/auth-method.json new file mode 100644 index 00000000000..121f585d40c --- /dev/null +++ b/e2e/terraform/provision-infra/scripts/consul-workload-identity/auth-method.json @@ -0,0 +1,25 @@ +{ + "Name": "nomad-workloads", + "Type": "jwt", + "DisplayName": "nomad-workloads", + "Description": "Login method for Nomad workloads using workload identities", + "TokenLocality": "local", + "Config": { + "BoundAudiences": [ + "consul.io" + ], + "ClaimMappings": { + "consul_namespace": "consul_namespace", + "nomad_job_id": "nomad_job_id", + "nomad_namespace": "nomad_namespace", + "nomad_service": "nomad_service", + "nomad_task": "nomad_task" + }, + "JWKSURL": "http://localhost:4646/.well-known/jwks.json", + "JWTSupportedAlgs": [ + "RS256" + ] + }, + "CreateIndex": 0, + "ModifyIndex": 0 +} diff --git a/e2e/terraform/provision-infra/scripts/consul-workload-identity/nomad-task-policy.hcl b/e2e/terraform/provision-infra/scripts/consul-workload-identity/nomad-task-policy.hcl new file mode 100644 index 00000000000..247ecb28088 --- /dev/null +++ b/e2e/terraform/provision-infra/scripts/consul-workload-identity/nomad-task-policy.hcl @@ -0,0 +1,11 @@ +// A policy used by the Consul role associated +// with nomad tasks. +// Used for Workload Identity integration. + +service_prefix "" { + policy = "read" +} + +key_prefix "" { + policy = "read" +}