fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/li…

…veness probes (#7909)

Put a specified admin address to proxy metadata and use it when generating an admin cluster.

Signed-off-by: Ilya Lobkov <[email protected]>

pkg/core/xds/metadata.go

@@ -19,13 +19,24 @@ var metadataLog = core.Log.WithName("xds-server").WithName("metadata-tracker")
 
 const (
 	// Supported Envoy node metadata fields.
+<<<<<<< HEAD
 	fieldDataplaneAdminPort         = "dataplane.admin.port"
 	fieldDataplaneDNSPort           = "dataplane.dns.port"
 	fieldDataplaneDNSEmptyPort      = "dataplane.dns.empty.port"
 	fieldDataplaneDataplaneResource = "dataplane.resource"
 	fieldDynamicMetadata            = "dynamicMetadata"
 	fieldDataplaneProxyType         = "dataplane.proxyType"
 	fieldVersion                    = "version"
+=======
+	FieldDataplaneAdminPort         = "dataplane.admin.port"
+	FieldDataplaneAdminAddress      = "dataplane.admin.address"
+	FieldDataplaneDNSPort           = "dataplane.dns.port"
+	FieldDataplaneDNSEmptyPort      = "dataplane.dns.empty.port"
+	FieldDataplaneDataplaneResource = "dataplane.resource"
+	FieldDynamicMetadata            = "dynamicMetadata"
+	FieldDataplaneProxyType         = "dataplane.proxyType"
+	FieldVersion                    = "version"
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
 	FieldPrefixDependenciesVersion  = "version.dependencies"
 	fieldFeatures                   = "features"
 )
@@ -45,6 +56,7 @@ const (
 // This way, xDS server will be able to use Envoy node metadata
 // to generate xDS resources that depend on environment-specific configuration.
 type DataplaneMetadata struct {
+<<<<<<< HEAD
 	Resource        model.Resource
 	AdminPort       uint32
 	DNSPort         uint32
@@ -53,6 +65,22 @@ type DataplaneMetadata struct {
 	ProxyType       mesh_proto.ProxyType
 	Version         *mesh_proto.Version
 	Features        Features
+=======
+	Resource            model.Resource
+	AdminPort           uint32
+	AdminAddress        string
+	DNSPort             uint32
+	EmptyDNSPort        uint32
+	DynamicMetadata     map[string]string
+	ProxyType           mesh_proto.ProxyType
+	Version             *mesh_proto.Version
+	Features            Features
+	SocketDir           string
+	AccessLogSocketPath string
+	MetricsSocketPath   string
+	MetricsCertPath     string
+	MetricsKeyPath      string
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
 }
 
 // GetDataplaneResource returns the underlying DataplaneResource, if present.
@@ -105,6 +133,13 @@ func (m *DataplaneMetadata) GetAdminPort() uint32 {
 	return m.AdminPort
 }
 
+func (m *DataplaneMetadata) GetAdminAddress() string {
+	if m == nil {
+		return ""
+	}
+	return m.AdminAddress
+}
+
 func (m *DataplaneMetadata) GetDNSPort() uint32 {
 	if m == nil {
 		return 0
@@ -144,10 +179,18 @@ func DataplaneMetadataFromXdsMetadata(xdsMetadata *structpb.Struct) *DataplaneMe
 	if field := xdsMetadata.Fields[fieldDataplaneProxyType]; field != nil {
 		metadata.ProxyType = mesh_proto.ProxyType(field.GetStringValue())
 	}
+<<<<<<< HEAD
 	metadata.AdminPort = uint32Metadata(xdsMetadata, fieldDataplaneAdminPort)
 	metadata.DNSPort = uint32Metadata(xdsMetadata, fieldDataplaneDNSPort)
 	metadata.EmptyDNSPort = uint32Metadata(xdsMetadata, fieldDataplaneDNSEmptyPort)
 	if value := xdsMetadata.Fields[fieldDataplaneDataplaneResource]; value != nil {
+=======
+	metadata.AdminPort = uint32Metadata(xdsMetadata, FieldDataplaneAdminPort)
+	metadata.AdminAddress = xdsMetadata.Fields[FieldDataplaneAdminAddress].GetStringValue()
+	metadata.DNSPort = uint32Metadata(xdsMetadata, FieldDataplaneDNSPort)
+	metadata.EmptyDNSPort = uint32Metadata(xdsMetadata, FieldDataplaneDNSEmptyPort)
+	if value := xdsMetadata.Fields[FieldDataplaneDataplaneResource]; value != nil {
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
 		res, err := rest.YAML.UnmarshalCore([]byte(value.GetStringValue()))
 		if err != nil {
 			metadataLog.Error(err, "invalid value in dataplane metadata", "field", fieldDataplaneDataplaneResource, "value", value)

pkg/xds/bootstrap/template_v3.go

@@ -292,7 +292,12 @@ func genConfig(parameters configParameters, proxyConfig xds.Proxy, useTokenPath
 		res.Node.Metadata.Fields["dataplane.resource"] = util_proto.MustNewValueForStruct(parameters.DataplaneResource)
 	}
 	if parameters.AdminPort != 0 {
+<<<<<<< HEAD
 		res.Node.Metadata.Fields["dataplane.admin.port"] = util_proto.MustNewValueForStruct(strconv.Itoa(int(parameters.AdminPort)))
+=======
+		res.Node.Metadata.Fields[core_xds.FieldDataplaneAdminPort] = util_proto.MustNewValueForStruct(strconv.Itoa(int(parameters.AdminPort)))
+		res.Node.Metadata.Fields[core_xds.FieldDataplaneAdminAddress] = util_proto.MustNewValueForStruct(parameters.AdminAddress)
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
 		res.Admin = &envoy_bootstrap_v3.Admin{
 			Address: &envoy_core_v3.Address{
 				Address: &envoy_core_v3.Address_SocketAddress{

pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml

@@ -46,6 +46,11 @@ node:
   cluster: backend
   id: default.dp-1.default
   metadata:
+<<<<<<< HEAD
+=======
+    accessLogSocketPath: /tmp/kuma-al-dp-1.default-default.sock
+    dataplane.admin.address: 127.0.0.1
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
     dataplane.admin.port: "1234"
     dataplane.proxyType: dataplane
     features: []

pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml

@@ -40,6 +40,11 @@ node:
   cluster: backend
   id: mesh.name.namespace
   metadata:
+<<<<<<< HEAD
+=======
+    accessLogSocketPath: /tmp/kuma-al-name.namespace-mesh.sock
+    dataplane.admin.address: 192.168.0.1
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
     dataplane.admin.port: "9902"
     dataplane.proxyType: dataplane
     features: []

pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml

@@ -46,6 +46,11 @@ node:
   cluster: backend
   id: mesh.name.namespace
   metadata:
+<<<<<<< HEAD
+=======
+    accessLogSocketPath: /tmp/kuma-al-name.namespace-mesh.sock
+    dataplane.admin.address: 192.168.0.1
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
     dataplane.admin.port: "1234"
     dataplane.proxyType: dataplane
     dataplane.resource: |2-

pkg/xds/bootstrap/testdata/generator.default-config-token-path.golden.yaml

@@ -66,6 +66,11 @@ node:
   cluster: backend
   id: mesh.name.namespace
   metadata:
+<<<<<<< HEAD
+=======
+    accessLogSocketPath: /tmp/kuma-al-name.namespace-mesh.sock
+    dataplane.admin.address: 127.0.0.1
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
     dataplane.admin.port: "1234"
     dataplane.dns.empty.port: "53002"
     dataplane.dns.port: "53001"

pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml

@@ -46,6 +46,11 @@ node:
   cluster: backend
   id: mesh.name.namespace
   metadata:
+<<<<<<< HEAD
+=======
+    accessLogSocketPath: /tmp/kuma-al-name.namespace-mesh.sock
+    dataplane.admin.address: 127.0.0.1
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
     dataplane.admin.port: "1234"
     dataplane.dns.empty.port: "53002"
     dataplane.dns.port: "53001"

pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.golden.yaml

@@ -36,6 +36,11 @@ node:
   cluster: backend
   id: mesh.name.namespace
   metadata:
+<<<<<<< HEAD
+=======
+    accessLogSocketPath: /tmp/kuma-al-name.namespace-mesh.sock
+    dataplane.admin.address: 127.0.0.1
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
     dataplane.admin.port: "1234"
     dataplane.proxyType: dataplane
     features: []

pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.ipv6.golden.yaml

@@ -36,6 +36,11 @@ node:
   cluster: backend
   id: mesh.name.namespace
   metadata:
+<<<<<<< HEAD
+=======
+    accessLogSocketPath: /tmp/kuma-al-name.namespace-mesh.sock
+    dataplane.admin.address: 127.0.0.1
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
     dataplane.admin.port: "1234"
     dataplane.proxyType: dataplane
     features: []

pkg/xds/bootstrap/testdata/generator.metrics-config.kubernetes.golden.yaml

@@ -36,6 +36,11 @@ node:
   cluster: backend
   id: mesh.name.namespace
   metadata:
+<<<<<<< HEAD
+=======
+    accessLogSocketPath: /tmp/kuma-al-name.namespace-mesh.sock
+    dataplane.admin.address: 127.0.0.1
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
     dataplane.admin.port: "1234"
     dataplane.proxyType: dataplane
     features: []

pkg/xds/generator/admin_proxy_generator.go

@@ -3,6 +3,10 @@ package generator
 import (
 	"context"
 
+	"github.com/asaskevich/govalidator"
+	"github.com/pkg/errors"
+	"golang.org/x/exp/maps"
+
 	core_xds "github.com/kumahq/kuma/pkg/core/xds"
 	xds_context "github.com/kumahq/kuma/pkg/xds/context"
 	envoy_common "github.com/kumahq/kuma/pkg/xds/envoy"
@@ -33,6 +37,14 @@ var staticTlsEndpointPaths = []*envoy_common.StaticEndpointPath{
 type AdminProxyGenerator struct {
 }
 
+var adminAddressAllowedValues = map[string]struct{}{
+	"127.0.0.1": {},
+	"0.0.0.0":   {},
+	"::1":       {},
+	"::":        {},
+	"":          {},
+}
+
 func (g AdminProxyGenerator) Generate(ctx context.Context, xdsCtx xds_context.Context, proxy *core_xds.Proxy) (*core_xds.ResourceSet, error) {
 	if proxy.Metadata.GetAdminPort() == 0 {
 		// It's not possible to export Admin endpoints if Envoy Admin API has not been enabled on that dataplane.
@@ -42,14 +54,30 @@ func (g AdminProxyGenerator) Generate(ctx context.Context, xdsCtx xds_context.Co
 	adminPort := proxy.Metadata.GetAdminPort()
 	// We assume that Admin API must be available on a loopback interface (while users
 	// can override the default value `127.0.0.1` in the Bootstrap Server section of `kuma-cp` config,
-	// the only reasonable alternative is `0.0.0.0`).
+	// the only reasonable alternatives are `::1`, `0.0.0.0` or `::`).
 	// In contrast to `AdminPort`, we shouldn't trust `AdminAddress` from the Envoy node metadata
 	// since it would allow a malicious user to manipulate that value and use Prometheus endpoint
 	// as a gateway to another host.
-	adminAddress := "127.0.0.1"
 	envoyAdminClusterName := envoy_names.GetEnvoyAdminClusterName()
+<<<<<<< HEAD
 	cluster, err := envoy_clusters.NewClusterBuilder(proxy.APIVersion).
 		Configure(envoy_clusters.ProvidedEndpointCluster(envoyAdminClusterName, false, core_xds.Endpoint{Target: adminAddress, Port: adminPort})).
+=======
+	adminAddress := proxy.Metadata.GetAdminAddress()
+	if _, ok := adminAddressAllowedValues[adminAddress]; !ok {
+		return nil, errors.Errorf("envoy admin cluster is not allowed to have addresses other than %v", maps.Keys(adminAddressAllowedValues))
+	}
+	switch adminAddress {
+	case "", "0.0.0.0":
+		adminAddress = "127.0.0.1"
+	case "::":
+		adminAddress = "::1"
+	}
+	cluster, err := envoy_clusters.NewClusterBuilder(proxy.APIVersion, envoyAdminClusterName).
+		Configure(envoy_clusters.ProvidedEndpointCluster(
+			govalidator.IsIPv6(adminAddress),
+			core_xds.Endpoint{Target: adminAddress, Port: adminPort})).
+>>>>>>> 8481ec2ba (fix(kuma-cp): specifying IPv6 Envoy Admin address breaks readiness/liveness probes (#7909))
 		Configure(envoy_clusters.DefaultTimeout()).
 		Build()
 	if err != nil {
@@ -63,7 +91,7 @@ func (g AdminProxyGenerator) Generate(ctx context.Context, xdsCtx xds_context.Co
 	}
 
 	// We bind admin to 127.0.0.1 by default, creating another listener with same address and port will result in error.
-	if g.getAddress(proxy) != "127.0.0.1" {
+	if g.getAddress(proxy) != adminAddress {
 		filterChains := []envoy_listeners.ListenerBuilderOpt{
 			envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(proxy.APIVersion).
 				Configure(envoy_listeners.StaticEndpoints(envoy_names.GetAdminListenerName(), staticEndpointPaths)),

pkg/xds/generator/admin_proxy_generator_test.go

@@ -25,6 +25,7 @@ var _ = Describe("AdminProxyGenerator", func() {
 	type testCase struct {
 		dataplaneFile string
 		expected      string
+		adminAddress  string
 	}
 
 	DescribeTable("should generate envoy config",
@@ -49,7 +50,8 @@ var _ = Describe("AdminProxyGenerator", func() {
 
 			proxy := &xds.Proxy{
 				Metadata: &xds.DataplaneMetadata{
-					AdminPort: 9901,
+					AdminPort:    9901,
+					AdminAddress: given.adminAddress,
 				},
 				EnvoyAdminMTLSCerts: xds.ServerSideMTLSCerts{
 					CaPEM: []byte("caPEM"),
@@ -76,9 +78,63 @@ var _ = Describe("AdminProxyGenerator", func() {
 			// and output matches golden files
 			Expect(actual).To(MatchGoldenYAML(filepath.Join("testdata", "admin", given.expected)))
 		},
-		Entry("should generate admin resources", testCase{
+		Entry("should generate admin resources, empty admin address", testCase{
 			dataplaneFile: "01.dataplane.input.yaml",
 			expected:      "01.envoy-config.golden.yaml",
+			adminAddress:  "",
+		}),
+		Entry("should generate admin resources, IPv4 loopback", testCase{
+			dataplaneFile: "02.dataplane.input.yaml",
+			expected:      "02.envoy-config.golden.yaml",
+			adminAddress:  "127.0.0.1",
+		}),
+		Entry("should generate admin resources, IPv6 loopback", testCase{
+			dataplaneFile: "03.dataplane.input.yaml",
+			expected:      "03.envoy-config.golden.yaml",
+			adminAddress:  "::1",
+		}),
+		Entry("should generate admin resources, unspecified IPv4", testCase{
+			dataplaneFile: "04.dataplane.input.yaml",
+			expected:      "04.envoy-config.golden.yaml",
+			adminAddress:  "0.0.0.0",
+		}),
+		Entry("should generate admin resources, unspecified IPv6", testCase{
+			dataplaneFile: "05.dataplane.input.yaml",
+			expected:      "05.envoy-config.golden.yaml",
+			adminAddress:  "::",
 		}),
 	)
+
+	It("should return error when admin address is not allowed", func() {
+		ctx := xds_context.Context{
+			Mesh: xds_context.MeshContext{
+				Resource: &core_mesh.MeshResource{
+					Meta: &test_model.ResourceMeta{
+						Name: "default",
+					},
+				},
+			},
+		}
+
+		proxy := &xds.Proxy{
+			Metadata: &xds.DataplaneMetadata{
+				AdminPort:    9901,
+				AdminAddress: "192.168.0.1", // it's not allowed to use such address
+			},
+			EnvoyAdminMTLSCerts: xds.ServerSideMTLSCerts{
+				CaPEM: []byte("caPEM"),
+				ServerPair: tls.KeyPair{
+					CertPEM: []byte("certPEM"),
+					KeyPEM:  []byte("keyPEM"),
+				},
+			},
+			Dataplane:  core_mesh.NewDataplaneResource(),
+			APIVersion: envoy_common.APIV3,
+		}
+
+		// when
+		_, err := generator.Generate(context.Background(), ctx, proxy)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(Equal(`envoy admin cluster is not allowed to have addresses other than [127.0.0.1 0.0.0.0 ::1 :: ]`))
+	})
 })

pkg/xds/generator/testdata/admin/02.dataplane.input.yaml

@@ -0,0 +1,9 @@
+type: Dataplane
+name: web-1
+mesh: default
+networking:
+  address: 192.168.0.1
+  inbound:
+    - port: 1234
+      tags:
+        kuma.io/service: web