change to use terraform for consul WI ACL bootstrapping

e2e/consul/consul.go

@@ -23,9 +23,6 @@ const (
 
 	consulJobRegisterOnUpdatePart1 = "consul/input/services_empty.nomad"
 	consulJobRegisterOnUpdatePart2 = "consul/input/services_present.nomad"
-
-	consulPolicyServiceInput = "/consul/input/consul-policy-for-nomad.hcl"
-	consulPolicyTaskInput    = "/consul/input/consul-policy-for-tasks.hcl"
 )
 
 const (
@@ -55,12 +52,6 @@ func init() {
 func (tc *ConsulE2ETest) BeforeAll(f *framework.F) {
 	e2eutil.WaitForLeader(f.T(), tc.Nomad())
 	e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1)
-
-	// setup consul ACL's for WI auth
-	e2eutil.SetupConsulACLsForServices(f.T(), tc.Consul(), consulPolicyServiceInput)
-	e2eutil.SetupConsulServiceIntentions(f.T(), tc.Consul())
-	e2eutil.SetupConsulACLsForTasks(f.T(), tc.Consul(), "nomad-default", consulPolicyTaskInput)
-	e2eutil.SetupConsulJWTAuth(f.T(), tc.Consul(), tc.Nomad().Address(), nil)
 }
 
 func (tc *ConsulE2ETest) AfterEach(f *framework.F) {

e2e/consul/consul_test.go

@@ -13,17 +13,10 @@ func TestConsul(t *testing.T) {
 	// todo: migrate the remaining consul tests
 
 	nomad := e2eutil.NomadClient(t)
-	consul := e2eutil.ConsulClient(t)
 
 	e2eutil.WaitForLeader(t, nomad)
 	e2eutil.WaitForNodesReady(t, nomad, 1)
 
-	// setup consul ACL's for WI auth
-	e2eutil.SetupConsulACLsForServices(t, consul, consulPolicyServiceInput)
-	e2eutil.SetupConsulServiceIntentions(t, consul)
-	e2eutil.SetupConsulACLsForTasks(t, consul, "nomad-default", consulPolicyTaskInput)
-	e2eutil.SetupConsulJWTAuth(t, consul, nomad.Address(), nil)
-
 	t.Run("testServiceReversion", testServiceReversion)
 	t.Run("testAllocRestart", testAllocRestart)
 }

e2e/consulcompat/run_ce_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 
 	"github.com/hashicorp/go-version"
-	"github.com/hashicorp/nomad/e2e/e2eutil"
 	"github.com/hashicorp/nomad/testutil"
 )
 
@@ -39,14 +38,14 @@ func testConsulBuild(t *testing.T, b build, baseDir string) {
 		// Note that with this policy we must use Workload Identity for Connect
 		// jobs, or we'll get "failed to derive SI token" errors from the client
 		// because the Nomad agent's token doesn't have "acl:write"
-		token := e2eutil.SetupConsulACLsForServices(t, consulAPI,
+		token := setupConsulACLsForServices(t, consulAPI,
 			"./input/consul-policy-for-nomad.hcl")
 
 		// we need service intentions so Connect apps can reach each other, and
 		// an ACL role and policy that tasks will be able to use to render
 		// templates
-		e2eutil.SetupConsulServiceIntentions(t, consulAPI)
-		e2eutil.SetupConsulACLsForTasks(t, consulAPI,
+		setupConsulServiceIntentions(t, consulAPI)
+		setupConsulACLsForTasks(t, consulAPI,
 			"nomad-default", "./input/consul-policy-for-tasks.hcl")
 
 		// note: Nomad needs to be live before we can setup Consul auth methods
@@ -72,7 +71,7 @@ func testConsulBuild(t *testing.T, b build, baseDir string) {
 		nc := startNomad(t, consulCfg)
 
 		// configure authentication for WI to Consul
-		e2eutil.SetupConsulJWTAuth(t, consulAPI, nc.Address(), nil)
+		setupConsulJWTAuth(t, consulAPI, nc.Address(), nil)
 
 		verifyConsulFingerprint(t, nc, b.Version, "default")
 		runConnectJob(t, nc, "default", "./input/connect.nomad.hcl")

e2e/consulcompat/shared_run_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/nomad/api"
 	nomadapi "github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/shoenig/test/must"
 	"github.com/shoenig/test/wait"
 )
@@ -177,3 +178,122 @@ func runConnectJob(t *testing.T, nc *nomadapi.Client, ns, filePath string) {
 	_, _, err = nc.AllocFS().Stat(alloc, "dashboard/local/count-api.txt", nil)
 	must.NoError(t, err)
 }
+
+// setupConsulACLsForServices installs a base set of ACL policies and returns a
+// token that the Nomad agent can use
+func setupConsulACLsForServices(t *testing.T, consulAPI *consulapi.Client, policyFilePath string) string {
+
+	d, err := os.Getwd()
+	must.NoError(t, err)
+	t.Log(d)
+	policyRules, err := os.ReadFile(policyFilePath)
+	must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath))
+
+	policy := &consulapi.ACLPolicy{
+		Name:        "nomad-cluster-" + uuid.Short(),
+		Description: "policy for nomad agent",
+		Rules:       string(policyRules),
+	}
+
+	policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil)
+	must.NoError(t, err, must.Sprint("could not write policy to Consul"))
+
+	token := &consulapi.ACLToken{
+		Description: "token for Nomad agent",
+		Policies: []*consulapi.ACLLink{{
+			ID:   policy.ID,
+			Name: policy.Name,
+		}},
+	}
+	token, _, err = consulAPI.ACL().TokenCreate(token, nil)
+	must.NoError(t, err, must.Sprint("could not create token in Consul"))
+
+	return token.SecretID
+}
+
+func setupConsulServiceIntentions(t *testing.T, consulAPI *consulapi.Client) {
+	ixn := &consulapi.Intention{
+		SourceName:      "count-dashboard",
+		DestinationName: "count-api",
+		Action:          "allow",
+	}
+	_, err := consulAPI.Connect().IntentionUpsert(ixn, nil)
+	must.NoError(t, err, must.Sprint("could not create intention"))
+}
+
+// setupConsulACLsForTasks installs a base set of ACL policies and returns a
+// token that the Nomad agent can use
+func setupConsulACLsForTasks(t *testing.T, consulAPI *consulapi.Client, roleName, policyFilePath string) {
+
+	policyRules, err := os.ReadFile(policyFilePath)
+	must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath))
+
+	policy := &consulapi.ACLPolicy{
+		Name:        "nomad-tasks-" + uuid.Short(),
+		Description: "policy for nomad tasks",
+		Rules:       string(policyRules),
+	}
+
+	policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil)
+	must.NoError(t, err, must.Sprint("could not write policy to Consul"))
+
+	role := &consulapi.ACLRole{
+		Name:        roleName, // note: must match "prod-${nomad_namespace}"
+		Description: "role for nomad tasks",
+		Policies: []*consulapi.ACLLink{{
+			ID:   policy.ID,
+			Name: policy.Name,
+		}},
+	}
+	_, _, err = consulAPI.ACL().RoleCreate(role, nil)
+	must.NoError(t, err, must.Sprint("could not create token in Consul"))
+}
+
+func setupConsulJWTAuth(t *testing.T, consulAPI *consulapi.Client, address string, namespaceRules []*consulapi.ACLAuthMethodNamespaceRule) {
+
+	authConfig := map[string]any{
+		"JWKSURL":          fmt.Sprintf("%s/.well-known/jwks.json", address),
+		"JWTSupportedAlgs": []string{"RS256"},
+		"BoundAudiences":   "consul.io",
+		"ClaimMappings": map[string]string{
+			"nomad_namespace": "nomad_namespace",
+			"nomad_job_id":    "nomad_job_id",
+			"nomad_task":      "nomad_task",
+			"nomad_service":   "nomad_service",
+		},
+	}
+
+	_, _, err := consulAPI.ACL().AuthMethodCreate(&consulapi.ACLAuthMethod{
+		Name:           "nomad-workloads",
+		Type:           "jwt",
+		DisplayName:    "nomad-workloads",
+		Description:    "login method for Nomad tasks with workload identity (WI)",
+		MaxTokenTTL:    time.Hour,
+		TokenLocality:  "local",
+		Config:         authConfig,
+		NamespaceRules: namespaceRules,
+	}, nil)
+	must.NoError(t, err, must.Sprint("could not create Consul auth method for Nomad workloads"))
+
+	rule := &consulapi.ACLBindingRule{
+		ID:          "",
+		Description: "binding rule for Nomad workload identities (WI) for tasks",
+		AuthMethod:  "nomad-workloads",
+		Selector:    `"nomad_service" not in value`,
+		BindType:    "role",
+		BindName:    "nomad-${value.nomad_namespace}",
+	}
+	_, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil)
+	must.NoError(t, err, must.Sprint("could not create Consul binding rule"))
+
+	rule = &consulapi.ACLBindingRule{
+		ID:          "",
+		Description: "binding rule for Nomad workload identities (WI) for services",
+		AuthMethod:  "nomad-workloads",
+		Selector:    `"nomad_service" in value`,
+		BindType:    "service",
+		BindName:    "${value.nomad_service}",
+	}
+	_, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil)
+	must.NoError(t, err, must.Sprint("could not create Consul binding rule"))
+}

e2e/e2eutil/consul.go

@@ -232,128 +232,3 @@ func DeleteConsulTokens(t *testing.T, client *capi.Client, tokens map[string][]s
 		}
 	}
 }
-
-// SetupConsulACLsForServices installs a base set of ACL policies and returns a
-// token that the Nomad agent can use
-func SetupConsulACLsForServices(t *testing.T, consulAPI *capi.Client, policyFilePath string) string {
-
-	d, err := os.Getwd()
-	must.NoError(t, err)
-	t.Log(d)
-	policyRules, err := os.ReadFile(policyFilePath)
-	must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath))
-
-	policy := &capi.ACLPolicy{
-		Name:        "nomad-cluster-" + uuid.Short(),
-		Description: "policy for nomad agent",
-		Rules:       string(policyRules),
-	}
-
-	policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil)
-	must.NoError(t, err, must.Sprint("could not write policy to Consul"))
-
-	token := &capi.ACLToken{
-		Description: "token for Nomad agent",
-		Policies: []*capi.ACLLink{{
-			ID:   policy.ID,
-			Name: policy.Name,
-		}},
-	}
-	token, _, err = consulAPI.ACL().TokenCreate(token, nil)
-	must.NoError(t, err, must.Sprint("could not create token in Consul"))
-
-	return token.SecretID
-}
-
-func SetupConsulServiceIntentions(t *testing.T, consulAPI *capi.Client) {
-	ixn := &capi.Intention{
-		SourceName:      "count-dashboard",
-		DestinationName: "count-api",
-		Action:          "allow",
-	}
-	_, err := consulAPI.Connect().IntentionUpsert(ixn, nil)
-	must.NoError(t, err, must.Sprint("could not create intention"))
-}
-
-// SetupConsulACLsForTasks installs a base set of ACL policies and returns a
-// token that the Nomad agent can use
-func SetupConsulACLsForTasks(t *testing.T, consulAPI *capi.Client, roleName, policyFilePath string) {
-
-	policyRules, err := os.ReadFile(policyFilePath)
-	must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath))
-
-	policy := &capi.ACLPolicy{
-		Name:        "nomad-tasks-" + uuid.Short(),
-		Description: "policy for nomad tasks",
-		Rules:       string(policyRules),
-	}
-
-	policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil)
-	must.NoError(t, err, must.Sprint("could not write policy to Consul"))
-
-	role := &capi.ACLRole{
-		Name:        roleName, // note: must match "prod-${nomad_namespace}"
-		Description: "role for nomad tasks",
-		Policies: []*capi.ACLLink{{
-			ID:   policy.ID,
-			Name: policy.Name,
-		}},
-	}
-	_, _, err = consulAPI.ACL().RoleCreate(role, nil)
-	if err != nil {
-		// TODO: because we run two types of Consul E2E tests, these setup
-		// functions will run twice. This is okay for all except when
-		// creating roles. So if the role already exists, just continue.
-		// When old framework tests are migrated, this should be removed.
-		must.ErrorContains(t, err, "already exists")
-	}
-}
-
-func SetupConsulJWTAuth(t *testing.T, consulAPI *capi.Client, address string, namespaceRules []*capi.ACLAuthMethodNamespaceRule) {
-
-	authConfig := map[string]any{
-		"JWKSURL":          fmt.Sprintf("%s/.well-known/jwks.json", address),
-		"JWTSupportedAlgs": []string{"RS256"},
-		"BoundAudiences":   "consul.io",
-		"ClaimMappings": map[string]string{
-			"nomad_namespace": "nomad_namespace",
-			"nomad_job_id":    "nomad_job_id",
-			"nomad_task":      "nomad_task",
-			"nomad_service":   "nomad_service",
-		},
-	}
-
-	_, _, err := consulAPI.ACL().AuthMethodCreate(&capi.ACLAuthMethod{
-		Name:           "nomad-workloads",
-		Type:           "jwt",
-		DisplayName:    "nomad-workloads",
-		Description:    "login method for Nomad tasks with workload identity (WI)",
-		MaxTokenTTL:    time.Hour,
-		TokenLocality:  "local",
-		Config:         authConfig,
-		NamespaceRules: namespaceRules,
-	}, nil)
-	must.NoError(t, err, must.Sprint("could not create Consul auth method for Nomad workloads"))
-
-	rule := &capi.ACLBindingRule{
-		ID:          "",
-		Description: "binding rule for Nomad workload identities (WI) for tasks",
-		AuthMethod:  "nomad-workloads",
-		Selector:    `"nomad_service" not in value`,
-		BindType:    "role",
-		BindName:    "nomad-${value.nomad_namespace}",
-	}
-	_, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil)
-	must.NoError(t, err, must.Sprint("could not create Consul binding rule"))
-
-	rule = &capi.ACLBindingRule{
-		ID:          "",
-		Description: "binding rule for Nomad workload identities (WI) for services",
-		AuthMethod:  "nomad-workloads",
-		Selector:    `"nomad_service" in value`,
-		BindType:    "service",
-		BindName:    "${value.nomad_service}",
-	}
-	_, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil)
-	must.NoError(t, err, must.Sprint("could not create Consul binding rule"))
-}