From bb6ac363b0a08b7d863c184e7ad2931f9c6bb841 Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 21 Sep 2023 12:30:23 -0600 Subject: [PATCH 01/22] Explicit container test --- Makefile | 2 + .../assets/Dockerfile-consul-dataplane | 5 + .../libs/cluster/dataplane.go | 122 ++++++++ .../libs/topology/peering_topology.go | 30 +- .../multiport/explicit_destination_test.go | 289 ++++++++++++++++++ .../catalog/static-client-service.json | 33 ++ .../catalog/static-client-workload.json | 34 +++ .../catalog/static-server-service.json | 33 ++ .../catalog/static-server-workload.json | 34 +++ .../catalog/upstreams.json | 45 +++ 10 files changed, 613 insertions(+), 14 deletions(-) create mode 100644 test/integration/consul-container/assets/Dockerfile-consul-dataplane create mode 100644 test/integration/consul-container/libs/cluster/dataplane.go create mode 100644 test/integration/consul-container/test/multiport/explicit_destination_test.go create mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-service.json create mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-workload.json create mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-service.json create mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-workload.json create mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/upstreams.json diff --git a/Makefile b/Makefile index 73d381cff5df..24bc72fdbfff 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ UI_BUILD_TAG?=consul-build-ui BUILD_CONTAINER_NAME?=consul-builder CONSUL_IMAGE_VERSION?=latest ENVOY_VERSION?='1.25.4' +CONSUL_DATAPLANE_IMAGE := $(or $(CONSUL_DATAPLANE_IMAGE),$(hashicorp/consul-dataplane:1.2.2)) CONSUL_VERSION?=$(shell cat version/VERSION) @@ -347,6 +348,7 @@ test-compat-integ-setup: dev-docker @docker run --rm -t $(CONSUL_COMPAT_TEST_IMAGE):local consul version @# 'consul-envoy:target-version' is needed by compatibility integ test @docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=$(CONSUL_COMPAT_TEST_IMAGE):local --build-arg ENVOY_VERSION=${ENVOY_VERSION} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + @docker build -t consul-dataplane:local --build-arg CONSUL_DATAPLANE_IMAGE=${CONSUL_DATAPLANE_IMAGE} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets .PHONY: test-compat-integ test-compat-integ: test-compat-integ-setup ## Test compat integ diff --git a/test/integration/consul-container/assets/Dockerfile-consul-dataplane b/test/integration/consul-container/assets/Dockerfile-consul-dataplane new file mode 100644 index 000000000000..ac43822e0b47 --- /dev/null +++ b/test/integration/consul-container/assets/Dockerfile-consul-dataplane @@ -0,0 +1,5 @@ +ARG CONSUL_DATAPLANE_IMAGE + +FROM ${CONSUL_DATAPLANE_IMAGE} as consuldataplane +COPY --from=busybox:uclibc /bin/sh /bin/sh +COPY --from=ghcr.io/tarampampam/curl:latest /bin/curl /bin/curl \ No newline at end of file diff --git a/test/integration/consul-container/libs/cluster/dataplane.go b/test/integration/consul-container/libs/cluster/dataplane.go new file mode 100644 index 000000000000..fbad790919d7 --- /dev/null +++ b/test/integration/consul-container/libs/cluster/dataplane.go @@ -0,0 +1,122 @@ +package cluster + +import ( + "context" + "fmt" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "strconv" + "time" +) + +type ConsulDataplaneContainer struct { + ctx context.Context + container testcontainers.Container + ip string + appPort []int + serviceName string + externalAdminPort int + internalAdminPort int +} + +func (g ConsulDataplaneContainer) GetAddr() (string, int) { + return g.ip, g.appPort[0] +} + +// GetAdminAddr returns the external admin port +func (g ConsulDataplaneContainer) GetAdminAddr() (string, int) { + return "localhost", g.externalAdminPort +} + +func (c ConsulDataplaneContainer) Terminate() error { + return TerminateContainer(c.ctx, c.container, true) +} + +func (g ConsulDataplaneContainer) GetStatus() (string, error) { + state, err := g.container.State(g.ctx) + return state.Status, err +} + +func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses string, grpcPort int, serviceBindPorts []int, + node Agent, containerArgs ...string) (*ConsulDataplaneContainer, error) { + namePrefix := fmt.Sprintf("%s-consul-dataplane-%s", node.GetDatacenter(), proxyID) + containerName := utils.RandName(namePrefix) + + internalAdminPort, err := node.ClaimAdminPort() + if err != nil { + return nil, err + } + + pod := node.GetPod() + if pod == nil { + return nil, fmt.Errorf("node Pod is required") + } + + var ( + appPortStrs []string + adminPortStr = strconv.Itoa(internalAdminPort) + ) + + for _, port := range serviceBindPorts { + appPortStrs = append(appPortStrs, strconv.Itoa(port)) + } + + // expose the app ports and the envoy adminPortStr on the agent container + exposedPorts := make([]string, len(appPortStrs)) + copy(exposedPorts, appPortStrs) + exposedPorts = append(exposedPorts, adminPortStr) + + command := []string{ + "-addresses", serverAddresses, + fmt.Sprintf("-grpc-port=%d", grpcPort), + fmt.Sprintf("-proxy-id=%s", proxyID), + "-proxy-namespace=default", + "-proxy-partition=default", + //fmt.Sprintf("-service-node-name=%s", node.GetName()), + "-log-level=info", + "-log-json=false", + //"-envoy-concurrency=2", + "-tls-disabled", + //"-consul-dns-bind-port=8601", + fmt.Sprintf("-envoy-admin-bind-port=%d", internalAdminPort), + } + + command = append(command, containerArgs...) + + req := testcontainers.ContainerRequest{ + Image: "consul-dataplane:local", + WaitingFor: wait.ForLog("").WithStartupTimeout(60 * time.Second), + AutoRemove: false, + Name: containerName, + Cmd: command, + Env: map[string]string{}, + } + + info, err := LaunchContainerOnNode(ctx, node, req, exposedPorts) + if err != nil { + return nil, err + } + out := &ConsulDataplaneContainer{ + ctx: ctx, + container: info.Container, + ip: info.IP, + serviceName: containerName, + externalAdminPort: info.MappedPorts[adminPortStr].Int(), + internalAdminPort: internalAdminPort, + } + + for _, port := range appPortStrs { + out.appPort = append(out.appPort, info.MappedPorts[port].Int()) + } + + fmt.Printf("NewConsulDataplane: proxyID %s, mapped App Port %d, service bind port %v\n", + proxyID, out.appPort, serviceBindPorts) + fmt.Printf("NewConsulDataplane: proxyID %s, , mapped admin port %d, admin port %d\n", + proxyID, out.externalAdminPort, internalAdminPort) + + fmt.Printf("NewConsulDataplane out: %+v", out) + fmt.Printf("NewConsulDataplane info: %+v", info) + + return out, nil +} diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index df72598082cd..d3b47afa15d8 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -43,7 +43,7 @@ type PeeringClusterSize struct { // // - an accepting cluster with 3 servers and 1 client agent. The client should be used to // host a service for export: staticServerSvc. -// - a dialing cluster with 1 server and 1 client. The client should be used to host a +// - an dialing cluster with 1 server and 1 client. The client should be used to host a // service connecting to staticServerSvc. // - Create the peering, export the service from accepting cluster, and verify service // connectivity. @@ -120,7 +120,7 @@ func BasicPeeringTwoClustersSetup( libassert.PeeringStatus(t, acceptingClient, AcceptingPeerName, api.PeeringStateActive) // libassert.PeeringExports(t, acceptingClient, acceptingPeerName, 1) - // Register a static-server service in acceptingCluster and export to dialing cluster + // Register an static-server service in acceptingCluster and export to dialing cluster var serverService, serverSidecarService libservice.Service { clientNode := acceptingCluster.Clients()[0] @@ -144,7 +144,7 @@ func BasicPeeringTwoClustersSetup( require.NoError(t, serverService.Export("default", AcceptingPeerName, acceptingClient)) } - // Register a static-client service in dialing cluster and set upstream to static-server service + // Register an static-client service in dialing cluster and set upstream to static-server service var clientSidecarService *libservice.ConnectContainer { clientNode := dialingCluster.Clients()[0] @@ -267,18 +267,20 @@ func NewClusterWithConfig( retryJoin = append(retryJoin, fmt.Sprintf("agent-%d", i)) } - // Add numClients static clients to register the service - configBuilder := libcluster.NewConfigBuilder(ctx). - Client(). - Peering(true). - RetryJoin(retryJoin...) - clientConf := configBuilder.ToAgentConfig(t) - t.Logf("%s client config: \n%s", opts.Datacenter, clientConf.JSON) - if clientHclConfig != "" { - clientConf.MutatebyAgentConfig(clientHclConfig) - } + if config.NumClients > 0 { + // Add numClients static clients to register the service + configbuiilder := libcluster.NewConfigBuilder(ctx). + Client(). + Peering(true). + RetryJoin(retryJoin...) + clientConf := configbuiilder.ToAgentConfig(t) + t.Logf("%s client config: \n%s", opts.Datacenter, clientConf.JSON) + if clientHclConfig != "" { + clientConf.MutatebyAgentConfig(clientHclConfig) + } - require.NoError(t, cluster.AddN(*clientConf, config.NumClients, true)) + require.NoError(t, cluster.AddN(*clientConf, config.NumClients, true)) + } // Use the client agent as the HTTP endpoint since we will not rotate it in many tests. var client *api.Client diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go new file mode 100644 index 000000000000..e0a82bd86264 --- /dev/null +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -0,0 +1,289 @@ +package multiport + +import ( + "context" + "embed" + "fmt" + "github.com/hashicorp/consul/internal/catalog" + "github.com/hashicorp/consul/internal/mesh" + "github.com/hashicorp/consul/internal/resource" + pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v1alpha1" + pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1" + libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + "github.com/stretchr/testify/require" + "testing" + "time" + + rtest "github.com/hashicorp/consul/internal/resource/resourcetest" + "github.com/hashicorp/consul/proto-public/pbresource" + "github.com/hashicorp/consul/sdk/testutil/retry" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" +) + +var ( + //go:embed integration_test_data + testData embed.FS + requestRetryTimer = &retry.Timer{Timeout: 120 * time.Second, Wait: 500 * time.Millisecond} +) + +// TestMultiportService_Explicit makes sure two services in the same datacenter have connectivity +// with transparent proxy enabled. +// +// Steps: +// - Create a single server cluster. +// - Create the example static-server and sidecar containers, then register them both with Consul +// - Create an example static-client sidecar, then register both the service and sidecar with Consul +// - Make sure a request from static-client to the virtual address (.virtual.consul) returns a +// response from the upstream. +func TestMultiportService_Explicit(t *testing.T) { + t.Parallel() + + cluster := createCluster(t) // 2 client agent pods + followers, err := cluster.Followers() + require.NoError(t, err) + client := pbresource.NewResourceServiceClient(followers[0].GetGRPCConn()) + resourceClient := rtest.NewClient(client) + + serverService := createServerServicesAndWorkloads(t, resourceClient) + createClientServicesAndWorkloads(t, resourceClient, serverService) + + clientDataplane := createServices(t, cluster) + //_, adminPort := clientDataplane.GetAdminAddr() + _, port := clientDataplane.GetAddr() + + createClientUpstreams(t, resourceClient, serverService, port) + + //libassert.AssertUpstreamEndpointStatus(t, adminPort, "static-server.default", "HEALTHY", 1) + //libassert.GetEnvoyListenerTCPFilters(t, adminPort) + + assertDataplaneContainerState(t, clientDataplane, "running") + libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-service", "") + +} + +// createServices creates the static-client and static-server services with +// transparent proxy enabled. It returns a Service for the static-client. +func createServices(t *testing.T, cluster *libcluster.Cluster) *libcluster.ConsulDataplaneContainer { + { + node := cluster.Agents[1] + //client := node.GetClient() + + // Create a service and dataplane + _, err := createServiceAndDataplane(t, node, "static-server-workload", "static-server", 8080, 8079) + require.NoError(t, err) + + //libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy", nil) + //libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil) + } + + { + node := cluster.Agents[2] + // Create a service and dataplane + clientDataplane, err := createServiceAndDataplane(t, node, "static-client-workload", "static-client", 8080, 8079) + require.NoError(t, err) + + //libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil) + return clientDataplane + } +} + +func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, serviceName string, httpPort, grpcPort int) (*libcluster.ConsulDataplaneContainer, error) { + // Do some trickery to ensure that partial completion is correctly torn + // down, but successful execution is not. + var deferClean utils.ResettableDefer + defer deferClean.Execute() + + // Create a service and proxy instance + svc, err := libservice.NewExampleService(context.Background(), serviceName, httpPort, grpcPort, node) + if err != nil { + return nil, err + } + deferClean.Add(func() { + _ = svc.Terminate() + }) + + // Create Consul Dataplane + dp, err := libcluster.NewConsulDataplane(context.Background(), proxyID, "0.0.0.0", 8502, node) + require.NoError(t, err) + deferClean.Add(func() { + _ = dp.Terminate() + }) + + // disable cleanup functions now that we have an object with a Terminate() function + deferClean.Reset() + + return dp, nil +} + +func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client) *pbresource.Resource { + serverService := rtest.ResourceID(&pbresource.ID{ + Name: "static-server-service", + Type: catalog.ServiceType, + Tenancy: resource.DefaultNamespacedTenancy(), + }).WithData(t, &pbcatalog.Service{ + Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-server"}}, + Ports: []*pbcatalog.ServicePort{ + {TargetPort: "tcp", Protocol: pbcatalog.Protocol_PROTOCOL_TCP}, + {TargetPort: "mesh", Protocol: pbcatalog.Protocol_PROTOCOL_MESH}, + }, + }).Write(t, resourceClient) + + workloadPortMap := make(map[string]*pbcatalog.WorkloadPort, 2) + workloadPortMap["tcp"] = &pbcatalog.WorkloadPort{ + Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, + } + workloadPortMap["mesh"] = &pbcatalog.WorkloadPort{ + Port: 20001, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + } + + rtest.ResourceID(&pbresource.ID{ + Name: "static-server-workload", + Type: catalog.WorkloadType, + Tenancy: resource.DefaultNamespacedTenancy(), + }). + WithData(t, &pbcatalog.Workload{ + Addresses: []*pbcatalog.WorkloadAddress{ + {Host: "127.0.0.1"}, + }, + Ports: workloadPortMap, + Identity: "static-server-identity", + }). + Write(t, resourceClient) + return serverService +} + +func createClientServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client, staticServerRef *pbresource.Resource) { + rtest.ResourceID(&pbresource.ID{ + Name: "static-client-service", + Type: catalog.ServiceType, + Tenancy: resource.DefaultNamespacedTenancy(), + }).WithData(t, &pbcatalog.Service{ + Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-client"}}, + Ports: []*pbcatalog.ServicePort{ + {TargetPort: "tcp", Protocol: pbcatalog.Protocol_PROTOCOL_TCP}, + {TargetPort: "mesh", Protocol: pbcatalog.Protocol_PROTOCOL_MESH}, + }, + }).Write(t, resourceClient) + + workloadPortMap := make(map[string]*pbcatalog.WorkloadPort, 2) + workloadPortMap["tcp"] = &pbcatalog.WorkloadPort{ + Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, + } + workloadPortMap["mesh"] = &pbcatalog.WorkloadPort{ + Port: 20001, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + } + + rtest.ResourceID(&pbresource.ID{ + Name: "static-client-workload", + Type: catalog.WorkloadType, + Tenancy: resource.DefaultNamespacedTenancy(), + }). + WithData(t, &pbcatalog.Workload{ + Addresses: []*pbcatalog.WorkloadAddress{ + {Host: "127.0.0.1"}, + }, + Ports: workloadPortMap, + Identity: "static-client-identity", + }). + Write(t, resourceClient) + + rtest.ResourceID(&pbresource.ID{ + Name: "static-client-upstreams", + Type: mesh.UpstreamsType, + Tenancy: resource.DefaultNamespacedTenancy(), + }). + WithData(t, &pbmesh.Upstreams{ + Upstreams: []*pbmesh.Upstream{ + { + DestinationRef: resource.Reference(staticServerRef.GetId(), ""), + DestinationPort: "tcp", + ListenAddr: &pbmesh.Upstream_IpPort{ + IpPort: &pbmesh.IPPortAddress{ + Ip: "127.0.0.1", + Port: 1234, + }, + }, + }, + }, + Workloads: &pbcatalog.WorkloadSelector{ + Prefixes: []string{"static-client"}, + }, + }). + Write(t, resourceClient) +} + +func createClientUpstreams(t *testing.T, resourceClient *rtest.Client, staticServerRef *pbresource.Resource, portNumber int) { + rtest.ResourceID(&pbresource.ID{ + Name: "static-client-upstreams", + Type: mesh.UpstreamsType, + Tenancy: resource.DefaultNamespacedTenancy(), + }). + WithData(t, &pbmesh.Upstreams{ + Upstreams: []*pbmesh.Upstream{ + { + DestinationRef: resource.Reference(staticServerRef.GetId(), ""), + DestinationPort: "tcp", + ListenAddr: &pbmesh.Upstream_IpPort{ + IpPort: &pbmesh.IPPortAddress{ + Ip: "127.0.0.1", + Port: uint32(portNumber), + }, + }, + }, + }, + Workloads: &pbcatalog.WorkloadSelector{ + Prefixes: []string{"static-client"}, + }, + }). + Write(t, resourceClient) +} + +func createCluster(t *testing.T) *libcluster.Cluster { + cluster, _, _ := topology.NewCluster(t, &topology.ClusterConfig{ + NumServers: 3, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + InjectGossipEncryption: true, + AllowHTTPAnyway: true, + }, + Cmd: `-hcl=experiments=["resource-apis"] log_level="TRACE"`, + }) + + return cluster +} + +// assertDataplaneContainerState validates service container status +func assertDataplaneContainerState(t *testing.T, dataplane *libcluster.ConsulDataplaneContainer, state string) { + containerStatus, err := dataplane.GetStatus() + require.NoError(t, err) + require.Equal(t, containerStatus, state, fmt.Sprintf("Expected: %s. Got %s", state, containerStatus)) +} + +// assertHTTPRequestToServiceAddress checks the result of a request from the +// given `client` container to the given `server` container. If expSuccess is +// true, this checks for a successful request and otherwise it checks for the +// error we expect when traffic is rejected by mTLS. +// +// This assumes the destination service is running Fortio. It makes the request +// to `:8080/debug?env=dump` and checks for `FORTIO_NAME=` +// in the response. +func assertHTTPRequestToServiceAddress(t *testing.T, client, server libcluster.Agent, expServiceName string, expSuccess bool) { + upstreamURL := fmt.Sprintf("http://%s:8080/debug?env=dump", server.GetIP()) + retry.RunWith(requestRetryTimer, t, func(r *retry.R) { + out, err := client.Exec(context.Background(), []string{"curl", "-s", upstreamURL}) + t.Logf("curl request to upstream service address: url=%s\nerr = %v\nout = %s", upstreamURL, err, out) + + if expSuccess { + require.NoError(r, err) + require.Contains(r, out, fmt.Sprintf("FORTIO_NAME=%s", expServiceName)) + } else { + require.Error(r, err) + require.Contains(r, err.Error(), "exit code 52") + } + }) +} diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-service.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-service.json new file mode 100644 index 000000000000..a90e5dd3fe8c --- /dev/null +++ b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-service.json @@ -0,0 +1,33 @@ +{ + "id": { + "type": { + "group": "catalog", + "group_version": "v1alpha1", + "kind": "Service" + }, + "tenancy": { + "partition": "default", + "namespace": "default", + "peerName": "local" + }, + "name": "static-client-service" + }, + "data": { + "@type": "hashicorp.consul.catalog.v1alpha1.Service", + "workloads": { + "prefixes": [ + "static-client" + ] + }, + "ports": [ + { + "target_port": "tcp", + "protocol": "PROTOCOL_TCP" + }, + { + "target_port": "mesh", + "protocol": "PROTOCOL_MESH" + } + ] + } +} \ No newline at end of file diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-workload.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-workload.json new file mode 100644 index 000000000000..ef0f517242d4 --- /dev/null +++ b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-workload.json @@ -0,0 +1,34 @@ +{ + "id": { + "type": { + "group": "catalog", + "group_version": "v1alpha1", + "kind": "Workload" + }, + "tenancy": { + "partition": "default", + "namespace": "default", + "peerName": "local" + }, + "name": "static-client-workload" + }, + "data": { + "@type": "hashicorp.consul.catalog.v1alpha1.Workload", + "addresses": [ + { + "host": "127.0.0.1" + } + ], + "ports": { + "tcp": { + "port": 19090, + "protocol": "PROTOCOL_TCP" + }, + "mesh": { + "port": 20000, + "protocol": "PROTOCOL_MESH" + } + }, + "identity": "static-client-identity" + } +} \ No newline at end of file diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-service.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-service.json new file mode 100644 index 000000000000..dd7aea052487 --- /dev/null +++ b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-service.json @@ -0,0 +1,33 @@ +{ + "id": { + "type": { + "group": "catalog", + "group_version": "v1alpha1", + "kind": "Service" + }, + "tenancy": { + "partition": "default", + "namespace": "default", + "peerName": "local" + }, + "name": "static-server-service" + }, + "data": { + "@type": "hashicorp.consul.catalog.v1alpha1.Service", + "workloads": { + "prefixes": [ + "static-server" + ] + }, + "ports": [ + { + "target_port": "tcp", + "protocol": "PROTOCOL_TCP" + }, + { + "target_port": "mesh", + "protocol": "PROTOCOL_MESH" + } + ] + } +} \ No newline at end of file diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-workload.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-workload.json new file mode 100644 index 000000000000..0b875f2dac4a --- /dev/null +++ b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-workload.json @@ -0,0 +1,34 @@ +{ + "id": { + "type": { + "group": "catalog", + "group_version": "v1alpha1", + "kind": "Workload" + }, + "tenancy": { + "partition": "default", + "namespace": "default", + "peerName": "local" + }, + "name": "static-server-workload" + }, + "data": { + "@type": "hashicorp.consul.catalog.v1alpha1.Workload", + "addresses": [ + { + "host": "127.0.0.1" + } + ], + "ports": { + "tcp": { + "port": 19091, + "protocol": "PROTOCOL_TCP" + }, + "mesh": { + "port": 20000, + "protocol": "PROTOCOL_MESH" + } + }, + "identity": "static-server-identity" + } +} \ No newline at end of file diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/upstreams.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/upstreams.json new file mode 100644 index 000000000000..c38345c56b59 --- /dev/null +++ b/test/integration/consul-container/test/multiport/integration_test_data/catalog/upstreams.json @@ -0,0 +1,45 @@ +{ + "id": { + "type": { + "group": "mesh", + "group_version": "v1alpha1", + "kind": "Upstreams" + }, + "tenancy": { + "partition": "default", + "namespace": "default", + "peerName": "local" + }, + "name": "static-client-upstreams" + }, + "data": { + "@type": "hashicorp.consul.mesh.v1alpha1.Upstreams", + "workloads": { + "prefixes": [ + "static-client" + ] + }, + "upstreams": [ + { + "destination_ref": { + "type": { + "group": "catalog", + "group_version": "v1alpha1", + "kind": "Service" + }, + "name": "static-server-service", + "tenancy": { + "partition": "default", + "namespace": "default", + "peerName": "local" + } + }, + "destination_port": "tcp", + "ip_port": { + "ip": "127.0.0.1", + "port": 1234 + } + } + ] + } +} \ No newline at end of file From d066de4c4cabec5c24e65111873bef4db4972a95 Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 21 Sep 2023 12:41:51 -0600 Subject: [PATCH 02/22] remove static resources --- .../multiport/explicit_destination_test.go | 3 -- .../catalog/static-client-service.json | 33 -------------- .../catalog/static-client-workload.json | 34 -------------- .../catalog/static-server-service.json | 33 -------------- .../catalog/static-server-workload.json | 34 -------------- .../catalog/upstreams.json | 45 ------------------- 6 files changed, 182 deletions(-) delete mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-service.json delete mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-workload.json delete mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-service.json delete mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-workload.json delete mode 100644 test/integration/consul-container/test/multiport/integration_test_data/catalog/upstreams.json diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index e0a82bd86264..77d46c948055 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -2,7 +2,6 @@ package multiport import ( "context" - "embed" "fmt" "github.com/hashicorp/consul/internal/catalog" "github.com/hashicorp/consul/internal/mesh" @@ -24,8 +23,6 @@ import ( ) var ( - //go:embed integration_test_data - testData embed.FS requestRetryTimer = &retry.Timer{Timeout: 120 * time.Second, Wait: 500 * time.Millisecond} ) diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-service.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-service.json deleted file mode 100644 index a90e5dd3fe8c..000000000000 --- a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-service.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": { - "type": { - "group": "catalog", - "group_version": "v1alpha1", - "kind": "Service" - }, - "tenancy": { - "partition": "default", - "namespace": "default", - "peerName": "local" - }, - "name": "static-client-service" - }, - "data": { - "@type": "hashicorp.consul.catalog.v1alpha1.Service", - "workloads": { - "prefixes": [ - "static-client" - ] - }, - "ports": [ - { - "target_port": "tcp", - "protocol": "PROTOCOL_TCP" - }, - { - "target_port": "mesh", - "protocol": "PROTOCOL_MESH" - } - ] - } -} \ No newline at end of file diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-workload.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-workload.json deleted file mode 100644 index ef0f517242d4..000000000000 --- a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-client-workload.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "id": { - "type": { - "group": "catalog", - "group_version": "v1alpha1", - "kind": "Workload" - }, - "tenancy": { - "partition": "default", - "namespace": "default", - "peerName": "local" - }, - "name": "static-client-workload" - }, - "data": { - "@type": "hashicorp.consul.catalog.v1alpha1.Workload", - "addresses": [ - { - "host": "127.0.0.1" - } - ], - "ports": { - "tcp": { - "port": 19090, - "protocol": "PROTOCOL_TCP" - }, - "mesh": { - "port": 20000, - "protocol": "PROTOCOL_MESH" - } - }, - "identity": "static-client-identity" - } -} \ No newline at end of file diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-service.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-service.json deleted file mode 100644 index dd7aea052487..000000000000 --- a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-service.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": { - "type": { - "group": "catalog", - "group_version": "v1alpha1", - "kind": "Service" - }, - "tenancy": { - "partition": "default", - "namespace": "default", - "peerName": "local" - }, - "name": "static-server-service" - }, - "data": { - "@type": "hashicorp.consul.catalog.v1alpha1.Service", - "workloads": { - "prefixes": [ - "static-server" - ] - }, - "ports": [ - { - "target_port": "tcp", - "protocol": "PROTOCOL_TCP" - }, - { - "target_port": "mesh", - "protocol": "PROTOCOL_MESH" - } - ] - } -} \ No newline at end of file diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-workload.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-workload.json deleted file mode 100644 index 0b875f2dac4a..000000000000 --- a/test/integration/consul-container/test/multiport/integration_test_data/catalog/static-server-workload.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "id": { - "type": { - "group": "catalog", - "group_version": "v1alpha1", - "kind": "Workload" - }, - "tenancy": { - "partition": "default", - "namespace": "default", - "peerName": "local" - }, - "name": "static-server-workload" - }, - "data": { - "@type": "hashicorp.consul.catalog.v1alpha1.Workload", - "addresses": [ - { - "host": "127.0.0.1" - } - ], - "ports": { - "tcp": { - "port": 19091, - "protocol": "PROTOCOL_TCP" - }, - "mesh": { - "port": 20000, - "protocol": "PROTOCOL_MESH" - } - }, - "identity": "static-server-identity" - } -} \ No newline at end of file diff --git a/test/integration/consul-container/test/multiport/integration_test_data/catalog/upstreams.json b/test/integration/consul-container/test/multiport/integration_test_data/catalog/upstreams.json deleted file mode 100644 index c38345c56b59..000000000000 --- a/test/integration/consul-container/test/multiport/integration_test_data/catalog/upstreams.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "id": { - "type": { - "group": "mesh", - "group_version": "v1alpha1", - "kind": "Upstreams" - }, - "tenancy": { - "partition": "default", - "namespace": "default", - "peerName": "local" - }, - "name": "static-client-upstreams" - }, - "data": { - "@type": "hashicorp.consul.mesh.v1alpha1.Upstreams", - "workloads": { - "prefixes": [ - "static-client" - ] - }, - "upstreams": [ - { - "destination_ref": { - "type": { - "group": "catalog", - "group_version": "v1alpha1", - "kind": "Service" - }, - "name": "static-server-service", - "tenancy": { - "partition": "default", - "namespace": "default", - "peerName": "local" - } - }, - "destination_port": "tcp", - "ip_port": { - "ip": "127.0.0.1", - "port": 1234 - } - } - ] - } -} \ No newline at end of file From 63c6ae10fff7d3afcf1c8523d9ebdf41d4a49cdf Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 21 Sep 2023 13:15:52 -0600 Subject: [PATCH 03/22] fix passing serviceBindPorts --- .../test/multiport/explicit_destination_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 77d46c948055..6dff104346d8 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -70,7 +70,7 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) *libcluster.Consu //client := node.GetClient() // Create a service and dataplane - _, err := createServiceAndDataplane(t, node, "static-server-workload", "static-server", 8080, 8079) + _, err := createServiceAndDataplane(t, node, "static-server-workload", "static-server", 8080, 8079, []int{8080}) require.NoError(t, err) //libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy", nil) @@ -80,7 +80,7 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) *libcluster.Consu { node := cluster.Agents[2] // Create a service and dataplane - clientDataplane, err := createServiceAndDataplane(t, node, "static-client-workload", "static-client", 8080, 8079) + clientDataplane, err := createServiceAndDataplane(t, node, "static-client-workload", "static-client", 8080, 8079, []int{libcluster.ServiceUpstreamLocalBindPort}) require.NoError(t, err) //libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil) @@ -88,7 +88,7 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) *libcluster.Consu } } -func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, serviceName string, httpPort, grpcPort int) (*libcluster.ConsulDataplaneContainer, error) { +func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, serviceName string, httpPort, grpcPort int, serviceBindPorts []int) (*libcluster.ConsulDataplaneContainer, error) { // Do some trickery to ensure that partial completion is correctly torn // down, but successful execution is not. var deferClean utils.ResettableDefer @@ -104,7 +104,7 @@ func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, ser }) // Create Consul Dataplane - dp, err := libcluster.NewConsulDataplane(context.Background(), proxyID, "0.0.0.0", 8502, node) + dp, err := libcluster.NewConsulDataplane(context.Background(), proxyID, "0.0.0.0", 8502, serviceBindPorts, node) require.NoError(t, err) deferClean.Add(func() { _ = dp.Terminate() From 73694ee106d3e6a7fa10bb145f163144417b126a Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 21 Sep 2023 15:54:45 -0600 Subject: [PATCH 04/22] WIP --- .../libs/cluster/dataplane.go | 3 +- .../multiport/explicit_destination_test.go | 54 +++++++++---------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/test/integration/consul-container/libs/cluster/dataplane.go b/test/integration/consul-container/libs/cluster/dataplane.go index fbad790919d7..a6177d72e92d 100644 --- a/test/integration/consul-container/libs/cluster/dataplane.go +++ b/test/integration/consul-container/libs/cluster/dataplane.go @@ -79,6 +79,7 @@ func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses str //"-envoy-concurrency=2", "-tls-disabled", //"-consul-dns-bind-port=8601", + //fmt.Sprintf("-xds-bind-port=%d", xdsBindPort), fmt.Sprintf("-envoy-admin-bind-port=%d", internalAdminPort), } @@ -100,7 +101,7 @@ func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses str out := &ConsulDataplaneContainer{ ctx: ctx, container: info.Container, - ip: info.IP, + ip: node.GetIP(), serviceName: containerName, externalAdminPort: info.MappedPorts[adminPortStr].Int(), internalAdminPort: internalAdminPort, diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 6dff104346d8..3eaa943e7586 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -44,10 +44,14 @@ func TestMultiportService_Explicit(t *testing.T) { client := pbresource.NewResourceServiceClient(followers[0].GetGRPCConn()) resourceClient := rtest.NewClient(client) - serverService := createServerServicesAndWorkloads(t, resourceClient) - createClientServicesAndWorkloads(t, resourceClient, serverService) + serverIP := cluster.Agents[1].GetIP() + clientIP := cluster.Agents[2].GetIP() + + serverService := createServerServicesAndWorkloads(t, resourceClient, serverIP) + createClientServicesAndWorkloads(t, resourceClient, serverService, clientIP) + + _, clientDataplane := createServices(t, cluster) - clientDataplane := createServices(t, cluster) //_, adminPort := clientDataplane.GetAdminAddr() _, port := clientDataplane.GetAddr() @@ -64,28 +68,24 @@ func TestMultiportService_Explicit(t *testing.T) { // createServices creates the static-client and static-server services with // transparent proxy enabled. It returns a Service for the static-client. -func createServices(t *testing.T, cluster *libcluster.Cluster) *libcluster.ConsulDataplaneContainer { - { - node := cluster.Agents[1] - //client := node.GetClient() +func createServices(t *testing.T, cluster *libcluster.Cluster) (*libcluster.ConsulDataplaneContainer, *libcluster.ConsulDataplaneContainer) { + node := cluster.Agents[1] + //client := node.GetClient() - // Create a service and dataplane - _, err := createServiceAndDataplane(t, node, "static-server-workload", "static-server", 8080, 8079, []int{8080}) - require.NoError(t, err) + // Create a service and dataplane + serverDataplane, err := createServiceAndDataplane(t, node, "static-server-workload", "static-server", 8080, 8079, []int{8080}) + require.NoError(t, err) - //libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy", nil) - //libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil) - } + //libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy", nil) + //libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil) - { - node := cluster.Agents[2] - // Create a service and dataplane - clientDataplane, err := createServiceAndDataplane(t, node, "static-client-workload", "static-client", 8080, 8079, []int{libcluster.ServiceUpstreamLocalBindPort}) - require.NoError(t, err) + node = cluster.Agents[2] + // Create a service and dataplane + clientDataplane, err := createServiceAndDataplane(t, node, "static-client-workload", "static-client", 8080, 8079, []int{libcluster.ServiceUpstreamLocalBindPort}) + require.NoError(t, err) - //libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil) - return clientDataplane - } + //libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil) + return serverDataplane, clientDataplane } func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, serviceName string, httpPort, grpcPort int, serviceBindPorts []int) (*libcluster.ConsulDataplaneContainer, error) { @@ -116,7 +116,7 @@ func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, ser return dp, nil } -func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client) *pbresource.Resource { +func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client, ipAddress string) *pbresource.Resource { serverService := rtest.ResourceID(&pbresource.ID{ Name: "static-server-service", Type: catalog.ServiceType, @@ -134,7 +134,7 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, } workloadPortMap["mesh"] = &pbcatalog.WorkloadPort{ - Port: 20001, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, } rtest.ResourceID(&pbresource.ID{ @@ -144,7 +144,7 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client }). WithData(t, &pbcatalog.Workload{ Addresses: []*pbcatalog.WorkloadAddress{ - {Host: "127.0.0.1"}, + {Host: ipAddress}, }, Ports: workloadPortMap, Identity: "static-server-identity", @@ -153,7 +153,7 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client return serverService } -func createClientServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client, staticServerRef *pbresource.Resource) { +func createClientServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client, staticServerRef *pbresource.Resource, ipAddress string) { rtest.ResourceID(&pbresource.ID{ Name: "static-client-service", Type: catalog.ServiceType, @@ -171,7 +171,7 @@ func createClientServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, } workloadPortMap["mesh"] = &pbcatalog.WorkloadPort{ - Port: 20001, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, } rtest.ResourceID(&pbresource.ID{ @@ -181,7 +181,7 @@ func createClientServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client }). WithData(t, &pbcatalog.Workload{ Addresses: []*pbcatalog.WorkloadAddress{ - {Host: "127.0.0.1"}, + {Host: ipAddress}, }, Ports: workloadPortMap, Identity: "static-client-identity", From 5dcf1dda8452cc333fa1a767e1407699321f2f0d Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 22 Sep 2023 13:03:15 -0400 Subject: [PATCH 05/22] fix explicit upstream test --- .../libs/cluster/dataplane.go | 2 +- .../multiport/explicit_destination_test.go | 103 ++++-------------- 2 files changed, 25 insertions(+), 80 deletions(-) diff --git a/test/integration/consul-container/libs/cluster/dataplane.go b/test/integration/consul-container/libs/cluster/dataplane.go index a6177d72e92d..713637c09a1a 100644 --- a/test/integration/consul-container/libs/cluster/dataplane.go +++ b/test/integration/consul-container/libs/cluster/dataplane.go @@ -101,7 +101,7 @@ func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses str out := &ConsulDataplaneContainer{ ctx: ctx, container: info.Container, - ip: node.GetIP(), + ip: info.IP, serviceName: containerName, externalAdminPort: info.MappedPorts[adminPortStr].Int(), internalAdminPort: internalAdminPort, diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 3eaa943e7586..c659ffe22e2f 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -38,7 +38,7 @@ var ( func TestMultiportService_Explicit(t *testing.T) { t.Parallel() - cluster := createCluster(t) // 2 client agent pods + cluster := createCluster(t) followers, err := cluster.Followers() require.NoError(t, err) client := pbresource.NewResourceServiceClient(followers[0].GetGRPCConn()) @@ -48,40 +48,33 @@ func TestMultiportService_Explicit(t *testing.T) { clientIP := cluster.Agents[2].GetIP() serverService := createServerServicesAndWorkloads(t, resourceClient, serverIP) - createClientServicesAndWorkloads(t, resourceClient, serverService, clientIP) + createClientResources(t, resourceClient, serverService, clientIP) _, clientDataplane := createServices(t, cluster) - //_, adminPort := clientDataplane.GetAdminAddr() _, port := clientDataplane.GetAddr() - createClientUpstreams(t, resourceClient, serverService, port) - - //libassert.AssertUpstreamEndpointStatus(t, adminPort, "static-server.default", "HEALTHY", 1) - //libassert.GetEnvoyListenerTCPFilters(t, adminPort) - assertDataplaneContainerState(t, clientDataplane, "running") libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-service", "") - + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") } // createServices creates the static-client and static-server services with // transparent proxy enabled. It returns a Service for the static-client. func createServices(t *testing.T, cluster *libcluster.Cluster) (*libcluster.ConsulDataplaneContainer, *libcluster.ConsulDataplaneContainer) { - node := cluster.Agents[1] + n1 := cluster.Agents[1] //client := node.GetClient() // Create a service and dataplane - serverDataplane, err := createServiceAndDataplane(t, node, "static-server-workload", "static-server", 8080, 8079, []int{8080}) + serverDataplane, err := createServiceAndDataplane(t, n1, "static-server-workload", "static-server", 8080, 8079, []int{}) require.NoError(t, err) //libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy", nil) //libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil) - node = cluster.Agents[2] + n2 := cluster.Agents[2] // Create a service and dataplane - clientDataplane, err := createServiceAndDataplane(t, node, "static-client-workload", "static-client", 8080, 8079, []int{libcluster.ServiceUpstreamLocalBindPort}) + clientDataplane, err := createServiceAndDataplane(t, n2, "static-client-workload", "static-client", 8080, 8079, []int{libcluster.ServiceUpstreamLocalBindPort}) require.NoError(t, err) //libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil) @@ -129,12 +122,13 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client }, }).Write(t, resourceClient) - workloadPortMap := make(map[string]*pbcatalog.WorkloadPort, 2) - workloadPortMap["tcp"] = &pbcatalog.WorkloadPort{ - Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, - } - workloadPortMap["mesh"] = &pbcatalog.WorkloadPort{ - Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + workloadPortMap := map[string]*pbcatalog.WorkloadPort{ + "tcp": &pbcatalog.WorkloadPort{ + Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, + }, + "mesh": &pbcatalog.WorkloadPort{ + Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + }, } rtest.ResourceID(&pbresource.ID{ @@ -153,7 +147,7 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client return serverService } -func createClientServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client, staticServerRef *pbresource.Resource, ipAddress string) { +func createClientResources(t *testing.T, resourceClient *rtest.Client, staticServerRef *pbresource.Resource, ipAddress string) { rtest.ResourceID(&pbresource.ID{ Name: "static-client-service", Type: catalog.ServiceType, @@ -166,12 +160,13 @@ func createClientServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client }, }).Write(t, resourceClient) - workloadPortMap := make(map[string]*pbcatalog.WorkloadPort, 2) - workloadPortMap["tcp"] = &pbcatalog.WorkloadPort{ - Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, - } - workloadPortMap["mesh"] = &pbcatalog.WorkloadPort{ - Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + workloadPortMap := map[string]*pbcatalog.WorkloadPort{ + "tcp": { + Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, + }, + "mesh": { + Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + }, } rtest.ResourceID(&pbresource.ID{ @@ -200,34 +195,8 @@ func createClientServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client DestinationPort: "tcp", ListenAddr: &pbmesh.Upstream_IpPort{ IpPort: &pbmesh.IPPortAddress{ - Ip: "127.0.0.1", - Port: 1234, - }, - }, - }, - }, - Workloads: &pbcatalog.WorkloadSelector{ - Prefixes: []string{"static-client"}, - }, - }). - Write(t, resourceClient) -} - -func createClientUpstreams(t *testing.T, resourceClient *rtest.Client, staticServerRef *pbresource.Resource, portNumber int) { - rtest.ResourceID(&pbresource.ID{ - Name: "static-client-upstreams", - Type: mesh.UpstreamsType, - Tenancy: resource.DefaultNamespacedTenancy(), - }). - WithData(t, &pbmesh.Upstreams{ - Upstreams: []*pbmesh.Upstream{ - { - DestinationRef: resource.Reference(staticServerRef.GetId(), ""), - DestinationPort: "tcp", - ListenAddr: &pbmesh.Upstream_IpPort{ - IpPort: &pbmesh.IPPortAddress{ - Ip: "127.0.0.1", - Port: uint32(portNumber), + Ip: "0.0.0.0", + Port: libcluster.ServiceUpstreamLocalBindPort, }, }, }, @@ -260,27 +229,3 @@ func assertDataplaneContainerState(t *testing.T, dataplane *libcluster.ConsulDat require.NoError(t, err) require.Equal(t, containerStatus, state, fmt.Sprintf("Expected: %s. Got %s", state, containerStatus)) } - -// assertHTTPRequestToServiceAddress checks the result of a request from the -// given `client` container to the given `server` container. If expSuccess is -// true, this checks for a successful request and otherwise it checks for the -// error we expect when traffic is rejected by mTLS. -// -// This assumes the destination service is running Fortio. It makes the request -// to `:8080/debug?env=dump` and checks for `FORTIO_NAME=` -// in the response. -func assertHTTPRequestToServiceAddress(t *testing.T, client, server libcluster.Agent, expServiceName string, expSuccess bool) { - upstreamURL := fmt.Sprintf("http://%s:8080/debug?env=dump", server.GetIP()) - retry.RunWith(requestRetryTimer, t, func(r *retry.R) { - out, err := client.Exec(context.Background(), []string{"curl", "-s", upstreamURL}) - t.Logf("curl request to upstream service address: url=%s\nerr = %v\nout = %s", upstreamURL, err, out) - - if expSuccess { - require.NoError(r, err) - require.Contains(r, out, fmt.Sprintf("FORTIO_NAME=%s", expServiceName)) - } else { - require.Error(r, err) - require.Contains(r, err.Error(), "exit code 52") - } - }) -} From 6fda2a43b227a62587f1a3a5699d8f1070ebd942 Mon Sep 17 00:00:00 2001 From: John Murret Date: Fri, 22 Sep 2023 11:20:11 -0700 Subject: [PATCH 06/22] use my image in CI until dataplane is fixed. --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 24bc72fdbfff..c272779ccdea 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,8 @@ UI_BUILD_TAG?=consul-build-ui BUILD_CONTAINER_NAME?=consul-builder CONSUL_IMAGE_VERSION?=latest ENVOY_VERSION?='1.25.4' -CONSUL_DATAPLANE_IMAGE := $(or $(CONSUL_DATAPLANE_IMAGE),$(hashicorp/consul-dataplane:1.2.2)) +# TODO(jm): fix this when consul-dataplane is multi-arch +CONSUL_DATAPLANE_IMAGE := $(or $(CONSUL_DATAPLANE_IMAGE),"jmurrethc/consul-dataplane-dbg") CONSUL_VERSION?=$(shell cat version/VERSION) From 08a5a4dcad5bf46eb23155d324f4f2d46bd4a11d Mon Sep 17 00:00:00 2001 From: John Murret Date: Fri, 22 Sep 2023 11:32:56 -0700 Subject: [PATCH 07/22] gofmt --- .../test/multiport/explicit_destination_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index c659ffe22e2f..8d8b275ad8e8 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -123,10 +123,10 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client }).Write(t, resourceClient) workloadPortMap := map[string]*pbcatalog.WorkloadPort{ - "tcp": &pbcatalog.WorkloadPort{ + "tcp": { Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, }, - "mesh": &pbcatalog.WorkloadPort{ + "mesh": { Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, }, } From 90f4563798d9171397498f12f5f2f11f9c7e587f Mon Sep 17 00:00:00 2001 From: John Murret Date: Fri, 22 Sep 2023 11:50:43 -0700 Subject: [PATCH 08/22] fixing reference to v2beta1 in test-containers --- test/integration/consul-container/go.mod | 2 ++ .../test/multiport/explicit_destination_test.go | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod index 8917a5c5a29f..4447e11d06e4 100644 --- a/test/integration/consul-container/go.mod +++ b/test/integration/consul-container/go.mod @@ -2,6 +2,8 @@ module github.com/hashicorp/consul/test/integration/consul-container go 1.20 +replace github.com/hashicorp/consul/proto-public => ../../../proto-public + require ( fortio.org/fortio v1.54.0 github.com/avast/retry-go v3.0.0+incompatible diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 8d8b275ad8e8..473c71b0c10b 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package multiport import ( @@ -6,8 +9,8 @@ import ( "github.com/hashicorp/consul/internal/catalog" "github.com/hashicorp/consul/internal/mesh" "github.com/hashicorp/consul/internal/resource" - pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v1alpha1" - pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1" + pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v2beta1" + pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1" libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" "github.com/stretchr/testify/require" "testing" From 7936b2a2a5e7bbcaa0427b0c11f830fe747c080c Mon Sep 17 00:00:00 2001 From: John Murret Date: Fri, 22 Sep 2023 12:10:51 -0700 Subject: [PATCH 09/22] WIP --- .../multiport/explicit_destination_test.go | 87 ++++++++++++++----- 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 473c71b0c10b..0b8a1997800f 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -6,9 +6,6 @@ package multiport import ( "context" "fmt" - "github.com/hashicorp/consul/internal/catalog" - "github.com/hashicorp/consul/internal/mesh" - "github.com/hashicorp/consul/internal/resource" pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v2beta1" pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1" libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" @@ -66,21 +63,16 @@ func TestMultiportService_Explicit(t *testing.T) { // transparent proxy enabled. It returns a Service for the static-client. func createServices(t *testing.T, cluster *libcluster.Cluster) (*libcluster.ConsulDataplaneContainer, *libcluster.ConsulDataplaneContainer) { n1 := cluster.Agents[1] - //client := node.GetClient() // Create a service and dataplane serverDataplane, err := createServiceAndDataplane(t, n1, "static-server-workload", "static-server", 8080, 8079, []int{}) require.NoError(t, err) - //libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy", nil) - //libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil) - n2 := cluster.Agents[2] // Create a service and dataplane clientDataplane, err := createServiceAndDataplane(t, n2, "static-client-workload", "static-client", 8080, 8079, []int{libcluster.ServiceUpstreamLocalBindPort}) require.NoError(t, err) - //libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil) return serverDataplane, clientDataplane } @@ -114,9 +106,17 @@ func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, ser func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client, ipAddress string) *pbresource.Resource { serverService := rtest.ResourceID(&pbresource.ID{ - Name: "static-server-service", - Type: catalog.ServiceType, - Tenancy: resource.DefaultNamespacedTenancy(), + Name: "static-server-service", + Type: &pbresource.Type{ + Group: "catalog", + GroupVersion: "v2beta1", + Kind: "Service", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "default", + Namespace: "default", + PeerName: "local", + }, }).WithData(t, &pbcatalog.Service{ Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-server"}}, Ports: []*pbcatalog.ServicePort{ @@ -135,9 +135,17 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client } rtest.ResourceID(&pbresource.ID{ - Name: "static-server-workload", - Type: catalog.WorkloadType, - Tenancy: resource.DefaultNamespacedTenancy(), + Name: "static-server-workload", + Type: &pbresource.Type{ + Group: "catalog", + GroupVersion: "v2beta1", + Kind: "Workload", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "default", + Namespace: "default", + PeerName: "local", + }, }). WithData(t, &pbcatalog.Workload{ Addresses: []*pbcatalog.WorkloadAddress{ @@ -152,9 +160,17 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client func createClientResources(t *testing.T, resourceClient *rtest.Client, staticServerRef *pbresource.Resource, ipAddress string) { rtest.ResourceID(&pbresource.ID{ - Name: "static-client-service", - Type: catalog.ServiceType, - Tenancy: resource.DefaultNamespacedTenancy(), + Name: "static-client-service", + Type: &pbresource.Type{ + Group: "catalog", + GroupVersion: "v2beta1", + Kind: "Service", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "default", + Namespace: "default", + PeerName: "local", + }, }).WithData(t, &pbcatalog.Service{ Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-client"}}, Ports: []*pbcatalog.ServicePort{ @@ -173,9 +189,17 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer } rtest.ResourceID(&pbresource.ID{ - Name: "static-client-workload", - Type: catalog.WorkloadType, - Tenancy: resource.DefaultNamespacedTenancy(), + Name: "static-client-workload", + Type: &pbresource.Type{ + Group: "catalog", + GroupVersion: "v2beta1", + Kind: "Workload", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "default", + Namespace: "default", + PeerName: "local", + }, }). WithData(t, &pbcatalog.Workload{ Addresses: []*pbcatalog.WorkloadAddress{ @@ -186,15 +210,30 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer }). Write(t, resourceClient) + destId := staticServerRef.GetId() + destRef := &pbresource.Reference{ + Type: destId.Type, + Tenancy: destId.Tenancy, + Name: destId.Name, + Section: "", + } rtest.ResourceID(&pbresource.ID{ - Name: "static-client-upstreams", - Type: mesh.UpstreamsType, - Tenancy: resource.DefaultNamespacedTenancy(), + Name: "static-client-upstreams", + Type: &pbresource.Type{ + Group: "catalog", + GroupVersion: "v2beta1", + Kind: "Upstreams", + }, + Tenancy: &pbresource.Tenancy{ + Partition: "default", + Namespace: "default", + PeerName: "local", + }, }). WithData(t, &pbmesh.Upstreams{ Upstreams: []*pbmesh.Upstream{ { - DestinationRef: resource.Reference(staticServerRef.GetId(), ""), + DestinationRef: destRef, DestinationPort: "tcp", ListenAddr: &pbmesh.Upstream_IpPort{ IpPort: &pbmesh.IPPortAddress{ From b3c4e00b37acd0185255ce61f15cbddf33924989 Mon Sep 17 00:00:00 2001 From: John Murret Date: Fri, 22 Sep 2023 12:33:54 -0700 Subject: [PATCH 10/22] remove bad references --- test/integration/consul-container/go.mod | 2 -- .../test/multiport/explicit_destination_test.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod index 4447e11d06e4..8917a5c5a29f 100644 --- a/test/integration/consul-container/go.mod +++ b/test/integration/consul-container/go.mod @@ -2,8 +2,6 @@ module github.com/hashicorp/consul/test/integration/consul-container go 1.20 -replace github.com/hashicorp/consul/proto-public => ../../../proto-public - require ( fortio.org/fortio v1.54.0 github.com/avast/retry-go v3.0.0+incompatible diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 0b8a1997800f..3d1b0c83e8b3 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -220,7 +220,7 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer rtest.ResourceID(&pbresource.ID{ Name: "static-client-upstreams", Type: &pbresource.Type{ - Group: "catalog", + Group: "mesh", GroupVersion: "v2beta1", Kind: "Upstreams", }, From 9789e96791c528d001d4cc191c45e3b67e46a280 Mon Sep 17 00:00:00 2001 From: John Murret Date: Fri, 22 Sep 2023 12:40:08 -0700 Subject: [PATCH 11/22] add missing license headers --- .../consul-container/assets/Dockerfile-consul-dataplane | 3 +++ test/integration/consul-container/libs/cluster/dataplane.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/test/integration/consul-container/assets/Dockerfile-consul-dataplane b/test/integration/consul-container/assets/Dockerfile-consul-dataplane index ac43822e0b47..508ac1e96bb9 100644 --- a/test/integration/consul-container/assets/Dockerfile-consul-dataplane +++ b/test/integration/consul-container/assets/Dockerfile-consul-dataplane @@ -1,3 +1,6 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + ARG CONSUL_DATAPLANE_IMAGE FROM ${CONSUL_DATAPLANE_IMAGE} as consuldataplane diff --git a/test/integration/consul-container/libs/cluster/dataplane.go b/test/integration/consul-container/libs/cluster/dataplane.go index 713637c09a1a..1bf0ed6beb84 100644 --- a/test/integration/consul-container/libs/cluster/dataplane.go +++ b/test/integration/consul-container/libs/cluster/dataplane.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package cluster import ( From c1b40e5a5b8e2a9d3fe31f18e5a1da6396de5ae1 Mon Sep 17 00:00:00 2001 From: John Murret Date: Fri, 22 Sep 2023 12:55:05 -0700 Subject: [PATCH 12/22] allow access internal/resource/resourcetest --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c272779ccdea..219ef2700aa9 100644 --- a/Makefile +++ b/Makefile @@ -265,7 +265,8 @@ lint-container-test-deps: ## Check that the test-container module only imports a @cd test/integration/consul-container && \ $(CURDIR)/build-support/scripts/check-allowed-imports.sh \ github.com/hashicorp/consul \ - internal/catalog/catalogtest + internal/catalog/catalogtest \ + internal/resource/resourcetest ##@ Testing From f464e17a39b66df3c3e7e1cf6c8e4aa5c22158b9 Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 25 Sep 2023 10:00:43 -0700 Subject: [PATCH 13/22] fix check-allowed-imports to append array items --- Makefile | 4 ++-- build-support/scripts/check-allowed-imports.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 219ef2700aa9..7a1548939870 100644 --- a/Makefile +++ b/Makefile @@ -265,8 +265,8 @@ lint-container-test-deps: ## Check that the test-container module only imports a @cd test/integration/consul-container && \ $(CURDIR)/build-support/scripts/check-allowed-imports.sh \ github.com/hashicorp/consul \ - internal/catalog/catalogtest \ - internal/resource/resourcetest + "internal/catalog/catalogtest" \ + "internal/resource/resourcetest" ##@ Testing diff --git a/build-support/scripts/check-allowed-imports.sh b/build-support/scripts/check-allowed-imports.sh index efba156c7937..02cd8ffdee8b 100755 --- a/build-support/scripts/check-allowed-imports.sh +++ b/build-support/scripts/check-allowed-imports.sh @@ -46,7 +46,7 @@ function main { then module_root="$1" else - allowed_packages+="$1" + allowed_packages+=("$1") fi shift esac From d9a37c72b1d2610903ca842ac020a3ee34559c45 Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 25 Sep 2023 10:25:45 -0700 Subject: [PATCH 14/22] use preview image for dataplane --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7a1548939870..386088e7b664 100644 --- a/Makefile +++ b/Makefile @@ -66,8 +66,7 @@ UI_BUILD_TAG?=consul-build-ui BUILD_CONTAINER_NAME?=consul-builder CONSUL_IMAGE_VERSION?=latest ENVOY_VERSION?='1.25.4' -# TODO(jm): fix this when consul-dataplane is multi-arch -CONSUL_DATAPLANE_IMAGE := $(or $(CONSUL_DATAPLANE_IMAGE),"jmurrethc/consul-dataplane-dbg") +CONSUL_DATAPLANE_IMAGE := $(or $(CONSUL_DATAPLANE_IMAGE),"docker.io/hashicorppreview/consul-dataplane:1.3-dev") CONSUL_VERSION?=$(shell cat version/VERSION) From baf23c2cf247abf8e7ae0e6295196e274cca0d6a Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 25 Sep 2023 10:36:16 -0700 Subject: [PATCH 15/22] revert some inadverntent comment updates in peering_topology --- .../consul-container/libs/topology/peering_topology.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index d3b47afa15d8..15f68c46d9a2 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -43,7 +43,7 @@ type PeeringClusterSize struct { // // - an accepting cluster with 3 servers and 1 client agent. The client should be used to // host a service for export: staticServerSvc. -// - an dialing cluster with 1 server and 1 client. The client should be used to host a +// - a dialing cluster with 1 server and 1 client. The client should be used to host a // service connecting to staticServerSvc. // - Create the peering, export the service from accepting cluster, and verify service // connectivity. @@ -120,7 +120,7 @@ func BasicPeeringTwoClustersSetup( libassert.PeeringStatus(t, acceptingClient, AcceptingPeerName, api.PeeringStateActive) // libassert.PeeringExports(t, acceptingClient, acceptingPeerName, 1) - // Register an static-server service in acceptingCluster and export to dialing cluster + // Register a static-server service in acceptingCluster and export to dialing cluster var serverService, serverSidecarService libservice.Service { clientNode := acceptingCluster.Clients()[0] @@ -144,7 +144,7 @@ func BasicPeeringTwoClustersSetup( require.NoError(t, serverService.Export("default", AcceptingPeerName, acceptingClient)) } - // Register an static-client service in dialing cluster and set upstream to static-server service + // Register a static-client service in dialing cluster and set upstream to static-server service var clientSidecarService *libservice.ConnectContainer { clientNode := dialingCluster.Clients()[0] From a9f2aa6fed1597a94362823bdd40992906cc2d4a Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 25 Sep 2023 10:52:43 -0700 Subject: [PATCH 16/22] add building local consul-dataplane image to compatibility-tests CI --- .github/workflows/test-integrations.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 40e75ac3ede2..18b13b3888cf 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -384,6 +384,7 @@ jobs: contents: read env: ENVOY_VERSION: "1.25.4" + CONSUL_DATAPLANE_IMAGE: docker.io/hashicorppreview/consul-dataplane:1.3-dev steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. @@ -415,6 +416,8 @@ jobs: - name: Retry Build consul-envoy:target-version image if: steps.buildConsulEnvoyImage.outcome == 'failure' run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Build consul-dataplane:local image + run: docker build -t consul-dataplane:local --build-arg CONSUL_DATAPLANE_IMAGE=${env.CONSUL_DATAPLANE_IMAGE} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets - name: Configure GH workaround for ipv6 loopback if: ${{ !endsWith(github.repository, '-enterprise') }} run: | From 56963e469649ab67cf4e04745889e51d91277c33 Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 25 Sep 2023 11:07:27 -0700 Subject: [PATCH 17/22] fix substitution in CI --- .github/workflows/test-integrations.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 18b13b3888cf..9dd7d3a3a35b 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -384,7 +384,7 @@ jobs: contents: read env: ENVOY_VERSION: "1.25.4" - CONSUL_DATAPLANE_IMAGE: docker.io/hashicorppreview/consul-dataplane:1.3-dev + CONSUL_DATAPLANE_IMAGE: "docker.io/hashicorppreview/consul-dataplane:1.3-dev" steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. @@ -417,7 +417,7 @@ jobs: if: steps.buildConsulEnvoyImage.outcome == 'failure' run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets - name: Build consul-dataplane:local image - run: docker build -t consul-dataplane:local --build-arg CONSUL_DATAPLANE_IMAGE=${env.CONSUL_DATAPLANE_IMAGE} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets + run: docker build -t consul-dataplane:local --build-arg CONSUL_DATAPLANE_IMAGE=${{ env.CONSUL_DATAPLANE_IMAGE }} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets - name: Configure GH workaround for ipv6 loopback if: ${{ !endsWith(github.repository, '-enterprise') }} run: | From d75d50a3574626d265e1c9aabf5b0e00e6d0608d Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 25 Sep 2023 11:19:41 -0700 Subject: [PATCH 18/22] change upstreams to destinations based on incoming change --- .../test/multiport/explicit_destination_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 3d1b0c83e8b3..5a93ac66a57d 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -230,12 +230,12 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer PeerName: "local", }, }). - WithData(t, &pbmesh.Upstreams{ - Upstreams: []*pbmesh.Upstream{ + WithData(t, &pbmesh.Destinations{ + Destinations: []*pbmesh.Destination{ { DestinationRef: destRef, DestinationPort: "tcp", - ListenAddr: &pbmesh.Upstream_IpPort{ + ListenAddr: &pbmesh.Destination_IpPort{ IpPort: &pbmesh.IPPortAddress{ Ip: "0.0.0.0", Port: libcluster.ServiceUpstreamLocalBindPort, From 85b1229be45d832e5677f2d52feb39047de92abf Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 25 Sep 2023 11:33:36 -0700 Subject: [PATCH 19/22] fixing use of upstreams in resource update --- .../test/multiport/explicit_destination_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 5a93ac66a57d..6feed6eab9ff 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -222,7 +222,7 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer Type: &pbresource.Type{ Group: "mesh", GroupVersion: "v2beta1", - Kind: "Upstreams", + Kind: "Destinations", }, Tenancy: &pbresource.Tenancy{ Partition: "default", From 63c6d57e7aead3ae96d0a27d0dea683fe99f3da3 Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 25 Sep 2023 13:13:27 -0700 Subject: [PATCH 20/22] remove commented out lines and enable envoy concurrency on dataplane. --- .../consul-container/libs/cluster/dataplane.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/integration/consul-container/libs/cluster/dataplane.go b/test/integration/consul-container/libs/cluster/dataplane.go index 1bf0ed6beb84..3e43c290cb1b 100644 --- a/test/integration/consul-container/libs/cluster/dataplane.go +++ b/test/integration/consul-container/libs/cluster/dataplane.go @@ -76,13 +76,10 @@ func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses str fmt.Sprintf("-proxy-id=%s", proxyID), "-proxy-namespace=default", "-proxy-partition=default", - //fmt.Sprintf("-service-node-name=%s", node.GetName()), "-log-level=info", "-log-json=false", - //"-envoy-concurrency=2", + "-envoy-concurrency=2", "-tls-disabled", - //"-consul-dns-bind-port=8601", - //fmt.Sprintf("-xds-bind-port=%d", xdsBindPort), fmt.Sprintf("-envoy-admin-bind-port=%d", internalAdminPort), } @@ -119,8 +116,5 @@ func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses str fmt.Printf("NewConsulDataplane: proxyID %s, , mapped admin port %d, admin port %d\n", proxyID, out.externalAdminPort, internalAdminPort) - fmt.Printf("NewConsulDataplane out: %+v", out) - fmt.Printf("NewConsulDataplane info: %+v", info) - return out, nil } From 4893908d568cc66fd254a9639657bc0e2a745bfd Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 25 Sep 2023 13:43:54 -0700 Subject: [PATCH 21/22] changes to addess PR feedback --- .../libs/topology/peering_topology.go | 24 +++++++-------- .../multiport/explicit_destination_test.go | 29 ++----------------- 2 files changed, 13 insertions(+), 40 deletions(-) diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index 15f68c46d9a2..947bd065cab5 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -267,21 +267,19 @@ func NewClusterWithConfig( retryJoin = append(retryJoin, fmt.Sprintf("agent-%d", i)) } - if config.NumClients > 0 { - // Add numClients static clients to register the service - configbuiilder := libcluster.NewConfigBuilder(ctx). - Client(). - Peering(true). - RetryJoin(retryJoin...) - clientConf := configbuiilder.ToAgentConfig(t) - t.Logf("%s client config: \n%s", opts.Datacenter, clientConf.JSON) - if clientHclConfig != "" { - clientConf.MutatebyAgentConfig(clientHclConfig) - } - - require.NoError(t, cluster.AddN(*clientConf, config.NumClients, true)) + // Add numClients static clients to register the service + configbuiilder := libcluster.NewConfigBuilder(ctx). + Client(). + Peering(true). + RetryJoin(retryJoin...) + clientConf := configbuiilder.ToAgentConfig(t) + t.Logf("%s client config: \n%s", opts.Datacenter, clientConf.JSON) + if clientHclConfig != "" { + clientConf.MutatebyAgentConfig(clientHclConfig) } + require.NoError(t, cluster.AddN(*clientConf, config.NumClients, true)) + // Use the client agent as the HTTP endpoint since we will not rotate it in many tests. var client *api.Client if config.NumClients > 0 { diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 6feed6eab9ff..659c76dd24d5 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -112,11 +112,6 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client GroupVersion: "v2beta1", Kind: "Service", }, - Tenancy: &pbresource.Tenancy{ - Partition: "default", - Namespace: "default", - PeerName: "local", - }, }).WithData(t, &pbcatalog.Service{ Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-server"}}, Ports: []*pbcatalog.ServicePort{ @@ -141,11 +136,6 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client GroupVersion: "v2beta1", Kind: "Workload", }, - Tenancy: &pbresource.Tenancy{ - Partition: "default", - Namespace: "default", - PeerName: "local", - }, }). WithData(t, &pbcatalog.Workload{ Addresses: []*pbcatalog.WorkloadAddress{ @@ -158,7 +148,7 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client return serverService } -func createClientResources(t *testing.T, resourceClient *rtest.Client, staticServerRef *pbresource.Resource, ipAddress string) { +func createClientResources(t *testing.T, resourceClient *rtest.Client, staticServerResource *pbresource.Resource, ipAddress string) { rtest.ResourceID(&pbresource.ID{ Name: "static-client-service", Type: &pbresource.Type{ @@ -166,11 +156,6 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer GroupVersion: "v2beta1", Kind: "Service", }, - Tenancy: &pbresource.Tenancy{ - Partition: "default", - Namespace: "default", - PeerName: "local", - }, }).WithData(t, &pbcatalog.Service{ Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-client"}}, Ports: []*pbcatalog.ServicePort{ @@ -195,11 +180,6 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer GroupVersion: "v2beta1", Kind: "Workload", }, - Tenancy: &pbresource.Tenancy{ - Partition: "default", - Namespace: "default", - PeerName: "local", - }, }). WithData(t, &pbcatalog.Workload{ Addresses: []*pbcatalog.WorkloadAddress{ @@ -210,7 +190,7 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer }). Write(t, resourceClient) - destId := staticServerRef.GetId() + destId := staticServerResource.GetId() destRef := &pbresource.Reference{ Type: destId.Type, Tenancy: destId.Tenancy, @@ -224,11 +204,6 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer GroupVersion: "v2beta1", Kind: "Destinations", }, - Tenancy: &pbresource.Tenancy{ - Partition: "default", - Namespace: "default", - PeerName: "local", - }, }). WithData(t, &pbmesh.Destinations{ Destinations: []*pbmesh.Destination{ From 0c8bd89dbf32a064f5d8b81bea0fbb8249fb0a7b Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 26 Sep 2023 15:57:06 -0400 Subject: [PATCH 22/22] small fixes --- .../libs/topology/peering_topology.go | 4 +-- .../multiport/explicit_destination_test.go | 36 +++---------------- 2 files changed, 7 insertions(+), 33 deletions(-) diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index 947bd065cab5..df72598082cd 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -268,11 +268,11 @@ func NewClusterWithConfig( } // Add numClients static clients to register the service - configbuiilder := libcluster.NewConfigBuilder(ctx). + configBuilder := libcluster.NewConfigBuilder(ctx). Client(). Peering(true). RetryJoin(retryJoin...) - clientConf := configbuiilder.ToAgentConfig(t) + clientConf := configBuilder.ToAgentConfig(t) t.Logf("%s client config: \n%s", opts.Datacenter, clientConf.JSON) if clientHclConfig != "" { clientConf.MutatebyAgentConfig(clientHclConfig) diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go index 659c76dd24d5..d8b02d065470 100644 --- a/test/integration/consul-container/test/multiport/explicit_destination_test.go +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -11,21 +11,15 @@ import ( libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" "github.com/stretchr/testify/require" "testing" - "time" rtest "github.com/hashicorp/consul/internal/resource/resourcetest" "github.com/hashicorp/consul/proto-public/pbresource" - "github.com/hashicorp/consul/sdk/testutil/retry" libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) -var ( - requestRetryTimer = &retry.Timer{Timeout: 120 * time.Second, Wait: 500 * time.Millisecond} -) - // TestMultiportService_Explicit makes sure two services in the same datacenter have connectivity // with transparent proxy enabled. // @@ -107,11 +101,7 @@ func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, ser func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client, ipAddress string) *pbresource.Resource { serverService := rtest.ResourceID(&pbresource.ID{ Name: "static-server-service", - Type: &pbresource.Type{ - Group: "catalog", - GroupVersion: "v2beta1", - Kind: "Service", - }, + Type: pbcatalog.ServiceType, }).WithData(t, &pbcatalog.Service{ Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-server"}}, Ports: []*pbcatalog.ServicePort{ @@ -131,11 +121,7 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client rtest.ResourceID(&pbresource.ID{ Name: "static-server-workload", - Type: &pbresource.Type{ - Group: "catalog", - GroupVersion: "v2beta1", - Kind: "Workload", - }, + Type: pbcatalog.WorkloadType, }). WithData(t, &pbcatalog.Workload{ Addresses: []*pbcatalog.WorkloadAddress{ @@ -151,11 +137,7 @@ func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client func createClientResources(t *testing.T, resourceClient *rtest.Client, staticServerResource *pbresource.Resource, ipAddress string) { rtest.ResourceID(&pbresource.ID{ Name: "static-client-service", - Type: &pbresource.Type{ - Group: "catalog", - GroupVersion: "v2beta1", - Kind: "Service", - }, + Type: pbcatalog.ServiceType, }).WithData(t, &pbcatalog.Service{ Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-client"}}, Ports: []*pbcatalog.ServicePort{ @@ -175,11 +157,7 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer rtest.ResourceID(&pbresource.ID{ Name: "static-client-workload", - Type: &pbresource.Type{ - Group: "catalog", - GroupVersion: "v2beta1", - Kind: "Workload", - }, + Type: pbcatalog.WorkloadType, }). WithData(t, &pbcatalog.Workload{ Addresses: []*pbcatalog.WorkloadAddress{ @@ -199,11 +177,7 @@ func createClientResources(t *testing.T, resourceClient *rtest.Client, staticSer } rtest.ResourceID(&pbresource.ID{ Name: "static-client-upstreams", - Type: &pbresource.Type{ - Group: "mesh", - GroupVersion: "v2beta1", - Kind: "Destinations", - }, + Type: pbmesh.DestinationsType, }). WithData(t, &pbmesh.Destinations{ Destinations: []*pbmesh.Destination{