diff --git a/.travis.yml b/.travis.yml index ea9bf1c4..bc5520a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go go: - - 1.11.1 + - 1.11.12 before_install: - sudo add-apt-repository -y ppa:jonathonf/python-3.6 diff --git a/CHANGELOG.md b/CHANGELOG.md index f0655234..4a83886b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# v2.0.0 +* Added support for multiple feed-ingress controllers per cluster +* Feed-ingress invocation split into subcommands, using double-dashed arguments +* Remove support for deprecated ingress resource annotations: + `sky.uk/frontend-elb-scheme` (replacement `sky.uk/frontend-scheme`), + `sky.uk/backend-keepalive-seconds` (replacement `sky.uk/backend-timeout-seconds`) +* Remove deprecated feed-ingress command-line argument `--nginx-default-backend-keepalive-seconds` (replacement `--nginx-default-backend-timeout-seconds`) + +This is a breaking change. Follow the instructions to [upgrade from v1 to v2](https://github.com/sky-uk/feed#upgrade-from-v1-to-v2) + # v1.14.2 * Skip ingress when http and/or path are not defined diff --git a/Gopkg.lock b/Gopkg.lock index 4ce1b82b..21d003f7 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -248,6 +248,14 @@ pruneopts = "UT" revision = "50d4dbd4eb0e84778abe37cefef140271d96fade" +[[projects]] + digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + pruneopts = "UT" + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + [[projects]] digest = "1:e22af8c7518e1eab6f2eab2b7d7558927f816262586cd6ed9f349c97a6c285c4" name = "github.com/jmespath/go-jmespath" @@ -406,11 +414,20 @@ version = "0.1.0" [[projects]] - digest = "1:ce8848a6fa8a2014d5fd57b5caa9c3c8000e29fb4cfe1b5e3c3d340ce766d56e" + digest = "1:e096613fb7cf34743d49af87d197663cfccd61876e2219853005a57baedfa562" + name = "github.com/spf13/cobra" + packages = ["."] + pruneopts = "UT" + revision = "f2b07da1e2c38d5f12845a4f607e2e1018cbb1f5" + version = "v0.0.5" + +[[projects]] + digest = "1:c1b1102241e7f645bc8e0c22ae352e8f0dc6484b6cb4d132fa9f24174e0119e2" name = "github.com/spf13/pflag" packages = ["."] pruneopts = "UT" - revision = "5ccb023bc27df288a957c5e994cd44fd19619465" + revision = "298182f68c66c05229eb03ac171abe6e309ee79a" + version = "v1.0.3" [[projects]] digest = "1:9bf0f8c6684cd6cb34d838c6f8a07043d6c48be79ffe271cae99633990f8d786" @@ -757,6 +774,8 @@ "github.com/sethgrid/pester", "github.com/sirupsen/logrus", "github.com/sky-uk/merlin/types", + "github.com/spf13/cobra", + "github.com/spf13/pflag", "github.com/stretchr/testify/assert", "github.com/stretchr/testify/mock", "github.com/vishvananda/netlink", @@ -767,6 +786,7 @@ "k8s.io/client-go/kubernetes", "k8s.io/client-go/pkg/api/v1", "k8s.io/client-go/pkg/apis/extensions/v1beta1", + "k8s.io/client-go/pkg/apis/meta/v1", "k8s.io/client-go/pkg/fields", "k8s.io/client-go/pkg/util/intstr", "k8s.io/client-go/tools/cache", diff --git a/Gopkg.toml b/Gopkg.toml index 025df866..d9252884 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -32,3 +32,7 @@ [[constraint]] branch = "master" name = "github.com/onrik/logrus" + +[[constraint]] + name = "github.com/spf13/pflag" + version = "1.0.3" diff --git a/Makefile b/Makefile index 6e0f84e8..449db8e1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,15 @@ +ifdef VERSION + version := $(VERSION) +else + git_rev := $(shell git rev-parse --short HEAD) + git_tag := $(shell git tag --points-at=$(git_rev)) + version := $(if $(git_tag),$(git_tag),dev-$(git_rev)) +endif + pkgs := $(shell go list ./... | grep -v /vendor/) files := $(shell find . -path ./vendor -prune -o -name '*.go' -print) +build_time := $(shell date -u) +ldflags := -X "github.com/sky-uk/feed/feed-ingress/cmd.version=$(version)" -X "github.com/sky-uk/feed/feed-ingress/cmd.buildTime=$(build_time)" .PHONY: all format test build vet lint copy docker release checkformat check clean @@ -9,9 +19,9 @@ travis : checkformat check docker setup: @echo "== setup" - go get github.com/golang/lint/golint - go get golang.org/x/tools/cmd/goimports - go get github.com/golang/dep/cmd/dep + go get -u golang.org/x/lint/golint + go get -u golang.org/x/tools/cmd/goimports + go get -u github.com/golang/dep/cmd/dep dep ensure format : @@ -21,7 +31,7 @@ format : build : @echo "== build" - @go install -v ./cmd/... + @go install -v ./feed-ingress/... ./feed-dns/... unformatted = $(shell goimports -l $(files)) diff --git a/README.md b/README.md index 5b39cb0c..be07b955 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This project contains Kubernetes controllers for managing external ingress with AWS or [IPVS](https://github.com/sky-uk/merlin). -There are two controllers provided, `feed-ingress` which runs an nginx instance, and `feed-dns` which manages route53 entries. +There are two controllers provided, `feed-ingress` which runs an NGINX instance, and `feed-dns` which manages Amazon Route 53 entries. They can be run independently as needed, or together to provide a full ingress solution. `feed-ingress` can be arbitrarily scaled up to support any traffic load. Feed is actively used in production and should be stable enough for general usage. We can scale to many thousands of @@ -12,38 +12,89 @@ requests per second with only a handful of replicas. # Using -Docker images are released using semantic versioning. See the [examples](examples/) for deployment yaml files that +Docker images are released using semantic versioning. See the [examples](examples/) for deployment YAML files that can be applied to a cluster. ## Requirements -* An internal and internet-facing ELB has been created and can reach your kubernetes cluster. The ELBs should be tagged with `sky.uk/KubernetesClusterFrontend=` which is used by feed to discover them. -* A Route53 hosted zone has been created to match your ingress resources. +* An internal and internet-facing ELB exists which can reach your Kubernetes cluster. +The ELBs should be tagged with `sky.uk/KubernetesClusterFrontend=` which is used by feed to discover them. +If you are using v2 of `feed-ingress` the ELBs should also be tagged with `sky.uk/KubernetesClusterIngressClass=`. +See [upgrade from v1 to v2](#upgrade-from-v1-to-v2) for more information. +* A Route 53 hosted zone has been created to match your ingress resources. + +## RBAC permissions + +The following RBAC permissions are required by the service account under which feed runs: + +```yaml +rules: +- apiGroups: + - "" + - extensions + resources: + - ingresses + - namespaces + - services + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses/status + verbs: + - update +``` ## Known Limitations -* nginx reloads can be disruptive. On reload, nginx will finish in-flight requests, then abruptly - close all server connections. This is a limitation of nginx, and affects all nginx solutions. We mitigate this by: +* NGINX reloads can be disruptive. On reload, NGINX will finish in-flight requests, then abruptly + close all server connections. This is a limitation of NGINX, and affects all NGINX solutions. We mitigate this by: * Rate limiting reloads. This is user configurable. * Using service IPs, which are stable. Reloads will only happen if an ingress or service changes, which is rare compared to pod changes. * feed-dns only supports a single hosted zone at this time, but this should be straightforward to add support for. - PRs are welcome. +PRs are welcome. + +# Upgrading + +## Upgrade from v1 to v2 + +This is a breaking change to support [multiple ingress controllers per cluster](#multiple-ingress-controllers-per-cluster). +The feed-ingress command-line structure has also changed. There are subcommands for the various load balancer types and +arguments use double dashes to be POSIX-compliant. + +To upgrade, follow these steps: +1. Tag the ELBs with `sky.uk/KubernetesClusterIngressClass=` to indicate which feed-ingress controllers should attach to them +1. Annotate all ingresses with `kubernetes.io/ingress.class=` +1. Replace deprecated ingress resource annotations with their replacements: + `sky.uk/frontend-elb-scheme` becomes `sky.uk/frontend-scheme`, + `sky.uk/backend-keepalive-seconds` becomes `sky.uk/backend-timeout-seconds` +1. Use double dashes for all arguments +1. Provide the mandatory argument `--ingress-class=` to feed-ingress with a value matching the ELB tag. + For migrating existing deployments, you may provide the new (but deprecated) flag `--include-classless-ingresses` + which instructs feed-ingress to additionally consider ingress resources that have no `kubernetes.io/ingress.class` annotation +1. Instead of using the argument `-registration-frontend-type`, use instead a subcommand of `feed-ingress` + (for example `feed-ingress -registration-frontend-type=elb ` becomes `feed-ingress elb `) +1. Rename the argument `-elb-label-value` to `--elb-frontend-tag-value` +1. Rename the argument `-nginx-default-backend-keepalive-seconds` to `--nginx-default-backend-timeout-seconds` # Overview ## feed-ingress -`feed-ingress` manages an nginx instance, updating its configuration dynamically for ingress resources. It attaches to +`feed-ingress` manages an NGINX instance, updating its configuration dynamically for ingress resources. It attaches to ELBs which are intended to be the frontend for all traffic. See the command line options with: - docker run skycirrus/feed-ingress:v1.1.0 -h + docker run skycirrus/feed-ingress:v2.0.0 -h ### SSL termination on ELB -SSL termination could be done on ELBS, and we believe that this is the safest and best performing +SSL termination could be done on ELBs, and we believe that this is the safest and best performing approach for production usage. Unfortunately, ELBs don't support SNI at this time, so this limits SSL usage to a single domain. One workaround is to use a wildcard certificate for the entire zone that `feed-dns` manages. Another is to place an SSL termination EC2 instance in front of the ELBs. @@ -52,18 +103,19 @@ Another is to place an SSL termination EC2 instance in front of the ELBs. SSL termination can be done on feed-ingress. This approach still requires a layer 4 load balancer, eg. ELB or IPVS, in front. -For the moment you can setup a default wildcard ssl: -``` - # Set default ssl path + name file without extension. Feed expects two files: one ending in .crt (the CA) and the other in .key (the private key), for example: - -ssl-path=/etc/ssl/default-ssl/default-ssl - # ... will cause feed to look for /etc/ssl/default-ssl.crt and /etc/ssl/default.ssl.key +For the moment you can setup a default wildcard SSL: + +```bash +# Set default SSL path + name file without extension. Feed expects two files: one ending in .crt (the CA) +# and the other in .key (the private key), for example: --ssl-path=/etc/ssl/default-ssl/default-ssl +# will cause feed to look for /etc/ssl/default-ssl.crt and /etc/ssl/default.ssl.key ``` You can mount the `.key` and `.crt` though a Kubernetes Secret see [feed-ingress-deployment-ssl](examples/feed-ingress-deployment-ssl.yml). ### Merlin support -Merlin is a distributed loadbalancer based on IPVS, with a gRPC based API. feed supports attaching to merlin +Merlin is a distributed load balancer based on IPVS, with a gRPC based API. Feed supports attaching to merlin as a frontend for ingress. See the [example](examples/feed-ingress-deployment-merlin.yml) for details. @@ -72,7 +124,7 @@ See the [example](examples/feed-ingress-deployment-merlin.yml) for details. _Gorb support is deprecated, and will be removed at some point. Use merlin instead._ -feed has support for configuring IPVS via [gorb](https://github.com/sky-uk/gorb). +Feed has support for configuring IPVS via [gorb](https://github.com/sky-uk/gorb). Gorb exposes a REST api to interrogate and modify the IPVS configuration such as virtual services and backends. The configuration can be stored in a distributed key/value store. @@ -80,7 +132,7 @@ Although IPVS supports multiple packet-forwarding methods, feed currently only s It provides the ability to manage the loopback interface so the ingress instance can pretend to be IPVS at the IP level. feed-ingress pod will need to define the `NET_ADMIN` Linux capability to be able to manage the loopback interface. -``` +```yaml securityContext: capabilities: add: @@ -95,11 +147,12 @@ The build now includes support for OpenTracing, and the default Docker image inc implementation. To enable OpenTracing, you will need to provide the following options: -``` - # Define the path to the OpenTracing vendor plugin - -nginx-opentracing-plugin-path=/usr/local/lib/libjaegertracing_plugin.linux_amd64.so - # Define the path to the config for the vendor plugin - -nginx-opentracing-config-path=/etc/jaeger-nginx-config.json + +```bash +# Define the path to the OpenTracing vendor plugin +--nginx-opentracing-plugin-path=/usr/local/lib/libjaegertracing_plugin.linux_amd64.so +# Define the path to the config for the vendor plugin +--nginx-opentracing-config-path=/etc/jaeger-nginx-config.json ``` Note that the status and metrics endpoints will *not* have OpenTracing applied. @@ -112,58 +165,75 @@ Note that the status and metrics endpoints will *not* have OpenTracing applied. client_header_buffer_size 16k; client_body_buffer_size 16k; large_client_header_buffers 4 16k -``` +``` They can be overridden by passing the following arguments during startup. -``` - -nginx-client-header-buffer-size-in-kb=16 - -nginx-client-body-buffer-size-in-kb=16 - - # Set the maximum number and size of buffers used for reading large client request header. If this value is set, - # -nginx-client-header-buffer-size-in-kb should be passed in as well. Otherwise, this value will be ignored. - -nginx-large-client-header-buffer-blocks=4 +```bash +--nginx-client-header-buffer-size-in-kb=16 +--nginx-client-body-buffer-size-in-kb=16 + +# Set the maximum number and size of buffers used for reading large client request header. If this value is set, +# --nginx-client-header-buffer-size-in-kb should be passed in as well. Otherwise, this value will be ignored. +--nginx-large-client-header-buffer-blocks=4 ``` ### Ingress status When using either the [elb](#elb) or [Merlin](#merlin) updater, the ingress status will be updated with relevant -loadbalancer information. This can then be used with other controllers such a `external-dns` which can set DNS for any +load balancer information. This can then be used with other controllers such a `external-dns` which can set DNS for any given ingress using the ingress status. -#### elb +#### ELB -feed will automatically discover all of your elb's and then use the `sky.uk/frontend-scheme` annotation to match an elb -label to an ingress. The updater will then set the ingress status to the elb's DNS name. +Feed will automatically discover all of your ELBs and then use the `sky.uk/frontend-scheme` annotation to match an ELB +label to an ingress. The updater will then set the ingress status to the ELBs DNS name. #### Merlin -The Merlin updater is currently unable to auto-discover all hosted loadbalancers on a Merlin server; instead the status -updater supports two different types: `internal` and `internet-facing`. These two loadbalancers are set using the +The Merlin updater is currently unable to auto-discover all hosted load balancers on a Merlin server; instead the status +updater supports two different types: `internal` and `internet-facing`. These two load balancers are set using the `merlin-internal-hostname` and `merlin-internet-facing-hostname` flags respectively. -An ingress can select which loadbalancer it wants to be associated with by setting the `sky.uk/frontend-scheme` +An ingress can select which load balancer it wants to be associated with by setting the `sky.uk/frontend-scheme` annotation to either `internal` or `internet-facing`. ### Running feed-ingress on privileged ports feed-ingress can be run on privileged ports by defining the `NET_BIND_SERVICE` Linux capability. -``` + +```yaml securityContext: capabilities: add: - NET_BIND_SERVICE - ``` +### Multiple ingress controllers per cluster + +Multiple feed-ingress controllers can be created per cluster. ELBs should be tagged with `sky.uk/KubernetesClusterIngressClass=` +and feed instances started with `--ingress-class=`. Feed instances will attach to ELBs with matching ingress class names. + +A feed ingress controller will adopt ingress resources with a matching `kubernetes.io/ingress.class=` annotation. +Ingress resources with no annotation will normally not be adopted and will have no traffic sent to their associated services. +However, see the deprecated flag `--include-classless-ingresses` which instructs feed-ingress to additionally consider +ingress resources with no `kubernetes.io/ingress.class` annotation. + +Use the script `classless-ingresses.sh` to find ingresses without this annotation. + +#### Support + +This feature is supported by `feed-ingress` and the `elb` load balancer. It is currently not supported by `feed-dns` +or any other load balancer type. PRs are welcome. + ## feed-dns -`feed-dns` manages a Route53 hosted zone, updating entries to point to ELBs or arbitrary hostnames. It is designed to +`feed-dns` manages a Route 53 hosted zone, updating entries to point to ELBs or arbitrary hostnames. It is designed to be run as a single instance per zone in your cluster. See the command line options with: - docker run skycirrus/feed-dns:v1.1.0 -h + docker run skycirrus/feed-dns:v2.0.0 -h ### DNS records @@ -176,10 +246,11 @@ Any records pointing to one of the endpoints associated with this controller tha entry are deleted. For any new ingress entry, a record is created to point to the correct endpoint. Existing records which do not meet these conditions remain untouched. -Each ingress must have the following tag `sky.uk/frontend-scheme` (`sky.uk/frontend-elb-scheme` is **deprecated**) set to `internal` or `internet-facing` so the -record can be set to the correct endpoint. +Each ingress must have the following be annotated with `sky.uk/frontend-scheme` set to `internal` or `internet-facing` +so the record can be set to the correct endpoint. -If you're using ELBs then ALIAS (A) records will be created. If you've explicitly provided CNAMEs of your load-balancers then CNAMEs will be created. +If you're using ELBs then ALIAS (A) records will be created. If you've explicitly provided CNAMEs of your +load balancers then CNAMEs will be created. ## Ingress annotations @@ -187,19 +258,25 @@ The controllers support several annotations on ingress resources. See the [examp ## ALB Support -feed has support for ALBs. Unfortunately, ALBs have a bug that prevents non-disruptive deployments of feed (specifically, +Feed has support for ALBs. Unfortunately, ALBs have a bug that prevents non-disruptive deployments of feed (specifically, they don't respect the deregistration delay). As a result, we don't recommend using ALBs at this time. -# Comparison to official nginx ingress controller - -feed was started before the [official nginx ingress controller](https://github.com/kubernetes/ingress-nginx) became production ready. The main differences that exist now are: -* feed has fewer features, as we only built it for our needs. -* feed pods attach directly to ELB/ALBs or IPVS nodes. The official controller relies on the `LoadBalancer` service type, which generally forwards traffic to every node in your cluster (`service.spec.externalTrafficPolicy` can be set in some providers to mitigate this). We found this problematic: - * It increases the amount of traffic flowing through your cluster, as traffic is routed through every node unnecessarily. - * ELB health checks don't work - the ELBs will disable arbitrary nodes, rather than a broken ingress pod. -* feed uses services, while the official controller uses endpoints: - * Primarily to reduce the number of nginx reloads that occur, which are problematic in busy environments. It may be possible to mitigate this though with a dynamic update of nginx (via plugin), and is something we've discussed doing for service updates. - * It's debateable whether using endpoints directly is a good idea conceptually, as it bypasses kube-proxy and any service mesh in place. +# Comparison to official NGINX ingress controller + +Feed was started before the [official NGINX ingress controller](https://github.com/kubernetes/ingress-nginx) became production ready. +The main differences that exist now are: +* Feed has fewer features, as we only built it for our needs. +* Feed pods attach directly to ELB/ALBs or IPVS nodes. The official controller relies on the `LoadBalancer` service type, + which generally forwards traffic to every node in your cluster (`service.spec.externalTrafficPolicy` can be set in + some providers to mitigate this). We found this problematic: + * It increases the amount of traffic flowing through your cluster, as traffic is routed through every node unnecessarily. + * ELB health checks don't work - the ELBs will disable arbitrary nodes, rather than a broken ingress pod. +* Feed uses services, while the official controller uses endpoints: + * Primarily to reduce the number of NGINX reloads that occur, which are problematic in busy environments. + It may be possible to mitigate this though with a dynamic update of NGINX (via plugin), and is something + we've discussed doing for service updates. + * It's debatable whether using endpoints directly is a good idea conceptually, as it bypasses kube-proxy + and any service mesh in place. # Development diff --git a/classless-ingresses.sh b/classless-ingresses.sh new file mode 100755 index 00000000..fd6ae457 --- /dev/null +++ b/classless-ingresses.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e + +ingress_class="kubernetes.io/ingress.class" + +if [[ $# -eq 0 ]]; then + cat >&2 < +EOF + exit 1 +fi + +context="${1}" + +kubectl --context "${context}" get ingress --all-namespaces -o json | \ + jq --raw-output ' + ["NAMESPACE","INGRESS"], + ( .items[] + | select(.metadata.annotations["'${ingress_class}'"] | length == 0) + | [.metadata.namespace, .metadata.name] + ) | @tsv' diff --git a/cmd/feed-ingress/main.go b/cmd/feed-ingress/main.go deleted file mode 100644 index ff96630c..00000000 --- a/cmd/feed-ingress/main.go +++ /dev/null @@ -1,488 +0,0 @@ -package main - -import ( - "flag" - "fmt" - _ "net/http/pprof" - "strconv" - "strings" - "time" - - log "github.com/sirupsen/logrus" - "github.com/sky-uk/feed/alb" - "github.com/sky-uk/feed/controller" - "github.com/sky-uk/feed/elb" - elb_status "github.com/sky-uk/feed/elb/status" - "github.com/sky-uk/feed/gorb" - "github.com/sky-uk/feed/k8s" - "github.com/sky-uk/feed/merlin" - merlin_status "github.com/sky-uk/feed/merlin/status" - "github.com/sky-uk/feed/nginx" - "github.com/sky-uk/feed/util/cmd" - "github.com/sky-uk/feed/util/metrics" -) - -var ( - debug bool - kubeconfig string - resyncPeriod time.Duration - ingressPort int - ingressHTTPSPort int - ingressHealthPort int - healthPort int - region string - elbLabelValue string - elbExpectedNumber int - drainDelay time.Duration - targetGroupNames cmd.CommaSeparatedValues - targetGroupDeregistrationDelay time.Duration - pushgatewayURL string - pushgatewayIntervalSeconds int - pushgatewayLabels cmd.KeyValues - controllerConfig controller.Config - nginxConfig nginx.Conf - nginxLogHeaders cmd.CommaSeparatedValues - nginxTrustedFrontends cmd.CommaSeparatedValues - nginxSSLPath string - nginxVhostStatsSharedMemory int - nginxOpenTracingPluginPath string - nginxOpenTracingConfigPath string - legacyBackendKeepaliveSeconds int - registrationFrontendType string - gorbIngressInstanceIP string - gorbEndpoint string - gorbServicesDefinition string - gorbBackendMethod string - gorbBackendWeight int - gorbVipLoadbalancer string - gorbManageLoopback bool - gorbBackendHealthcheckInterval string - gorbBackendHealthcheckType string - gorbInterfaceProcFsPath string - merlinEndpoint string - merlinRequestTimeout time.Duration - merlinServiceID string - merlinHTTPSServiceID string - merlinInstanceIP string - merlinForwardMethod string - merlinDrainDelay time.Duration - merlinHealthUpThreshold uint - merlinHealthDownThreshold uint - merlinHealthPeriod time.Duration - merlinHealthTimeout time.Duration - merlinVIP string - merlinVIPInterface string - merlinInternalHostname string - merlinInternetFacingHostname string -) - -const ( - unset = -1 - defaultResyncPeriod = time.Minute * 15 - defaultIngressPort = 8080 - defaultIngressHTTPSPort = unset - defaultIngressAllow = "0.0.0.0/0" - defaultIngressHealthPort = 8081 - defaultIngressStripPath = true - defaultIngressExactPath = false - defaultHealthPort = 12082 - defaultNginxBinary = "/usr/sbin/nginx" - defaultNginxWorkingDir = "/nginx" - defaultNginxWorkers = 1 - defaultNginxWorkerConnections = 1024 - defaultNginxWorkerShutdownTimeoutSeconds = 0 - defaultNginxKeepAliveSeconds = 60 - defaultNginxBackendKeepalives = 512 - defaultNginxBackendTimeoutSeconds = 60 - defaultNginxBackendConnectTimeoutSeconds = 1 - defaultNginxBackendMaxConnections = 0 - defaultNginxProxyBufferSize = 16 - defaultNginxProxyBufferBlocks = 4 - defaultNginxLogLevel = "warn" - defaultNginxServerNamesHashBucketSize = unset - defaultNginxServerNamesHashMaxSize = unset - defaultNginxProxyProtocol = false - defaultNginxUpdatePeriod = time.Second * 30 - defaultNginxSSLPath = "/etc/ssl/default-ssl/default-ssl" - defaultNginxVhostStatsSharedMemory = 1 - defaultNginxOpenTracingPluginPath = "" - defaultNginxOpenTracingConfigPath = "" - defaultElbLabelValue = "" - defaultDrainDelay = time.Second * 60 - defaultTargetGroupDeregistrationDelay = time.Second * 300 - defaultRegion = "eu-west-1" - defaultElbExpectedNumber = 0 - defaultPushgatewayIntervalSeconds = 60 - defaultAccessLogDir = "/var/log/nginx" - defaultRegistrationFrontendType = "elb" - defaultGorbIngressInstanceIP = "127.0.0.1" - defaultGorbEndpoint = "http://127.0.0.1:80" - defaultGorbBackendMethod = "dr" - defaultGorbBackendWeight = 1000 - defaultGorbServicesDefinition = "http-proxy:80,https-proxy:443" - defaultGorbVipLoadbalancer = "127.0.0.1" - defaultGorbManageLoopback = true - defaultGorbInterfaceProcFsPath = "/host-ipv4-proc/" - defaultGorbBackendHealthcheckInterval = "1s" - defaultGorbBackendHealthcheckType = "http" - defaultMerlinForwardMethod = "route" - defaultMerlinHealthUpThreshold = 3 - defaultMerlinHealthDownThreshold = 2 - defaultMerlinHealthPeriod = 10 * time.Second - defaultMerlinHealthTimeout = time.Second - defaultMerlinVIPInterface = "lo" - defaultClientHeaderBufferSize = 16 - defaultClientBodyBufferSize = 16 - defaultLargeClientHeaderBufferBlocks = 4 -) - -func init() { - // general flags - flag.BoolVar(&debug, "debug", false, - "Enable debug logging.") - flag.StringVar(&kubeconfig, "kubeconfig", "", - "Path to kubeconfig for connecting to the apiserver. Leave blank to connect inside a cluster.") - flag.DurationVar(&resyncPeriod, "resync-period", defaultResyncPeriod, - "Resync with the apiserver periodically to handle missed updates.") - flag.IntVar(&ingressPort, "ingress-port", defaultIngressPort, - "Port to serve ingress traffic to backend services.") - flag.IntVar(&ingressHTTPSPort, "ingress-https-port", defaultIngressHTTPSPort, - "Port to serve ingress https traffic to backend services.") - flag.IntVar(&ingressHealthPort, "ingress-health-port", defaultIngressHealthPort, - "Port for ingress /health and /status pages. Should be used by frontends to determine if ingress is available.") - flag.StringVar(&controllerConfig.DefaultAllow, "ingress-allow", defaultIngressAllow, - "Source IP or CIDR to allow ingress access by default. This is overridden by the sky.uk/allow "+ - "annotation on ingress resources. Leave empty to deny all access by default.") - flag.BoolVar(&controllerConfig.DefaultStripPath, "ingress-strip-path", defaultIngressStripPath, - "Whether to strip the ingress path from the URL before passing to backend services. For example, "+ - "if enabled 'myhost/myapp/health' would be passed as '/health' to the backend service. If disabled, "+ - "it would be passed as '/myapp/health'. Enabling this requires nginx to process the URL, which has some "+ - "limitations. URL encoded characters will not work correctly in some cases, and backend services will "+ - "need to take care to properly construct URLs, such as by using the 'X-Original-URI' header."+ - "Can be overridden with the sky.uk/strip-path annotation per ingress") - flag.BoolVar(&controllerConfig.DefaultExactPath, "ingress-exact-path", defaultIngressExactPath, - "Whether to consider the ingress path to be an exact match rather than as a prefix. For example, "+ - "if enabled 'myhost/myapp/health' would match 'myhost/myapp/health' but not 'myhost/myapp/health/x'."+ - " If disabled, it would match both (and redirect requests from 'myhost/myapp/health' to "+ - " '/myhost/myapp/health/'. Can be overridden with the sky.uk/exact-path annotation per ingress") - flag.IntVar(&healthPort, "health-port", defaultHealthPort, - "Port for checking the health of the ingress controller on /health. Also provides /debug/pprof.") - - // nginx flags - flag.StringVar(&nginxConfig.BinaryLocation, "nginx-binary", defaultNginxBinary, - "Location of nginx binary.") - flag.StringVar(&nginxConfig.WorkingDir, "nginx-workdir", defaultNginxWorkingDir, - "Directory to store nginx files. Also the location of the nginx.tmpl file.") - flag.IntVar(&nginxConfig.WorkerProcesses, "nginx-workers", defaultNginxWorkers, - "Number of nginx worker processes.") - flag.IntVar(&nginxConfig.WorkerConnections, "nginx-worker-connections", defaultNginxWorkerConnections, - "Max number of connections per nginx worker. Includes both client and proxy connections.") - flag.IntVar(&nginxConfig.WorkerShutdownTimeoutSeconds, "nginx-worker-shutdown-timeout-seconds", defaultNginxWorkerShutdownTimeoutSeconds, - "Timeout for a graceful shutdown of worker processes.") - flag.IntVar(&nginxConfig.KeepaliveSeconds, "nginx-keepalive-seconds", defaultNginxKeepAliveSeconds, - "Keep alive time for persistent client connections to nginx. Should generally be set larger than frontend "+ - "keep alive times to prevent stale connections.") - flag.IntVar(&nginxConfig.BackendKeepalives, "nginx-backend-keepalive-count", defaultNginxBackendKeepalives, - "Maximum number of keepalive connections per backend service. Keepalive connections count against"+ - " nginx-worker-connections limit, and will be restricted by that global limit as well.") - flag.IntVar(&legacyBackendKeepaliveSeconds, "nginx-default-backend-keepalive-seconds", unset, - "Deprecated. Use -nginx-default-backend-timeout-seconds instead.") - flag.IntVar(&controllerConfig.DefaultBackendTimeoutSeconds, "nginx-default-backend-timeout-seconds", - defaultNginxBackendTimeoutSeconds, - "Timeout for requests to backends. Can be overridden per ingress with the sky.uk/backend-timeout-seconds annotation.") - flag.IntVar(&nginxConfig.BackendConnectTimeoutSeconds, "nginx-backend-connect-timeout-seconds", - defaultNginxBackendConnectTimeoutSeconds, - "Connect timeout to backend services.") - flag.IntVar(&controllerConfig.DefaultBackendMaxConnections, "nginx-default-backend-max-connections", - defaultNginxBackendMaxConnections, - "Maximum number of connections to a single backend. Can be overridden per ingress with the sky.uk/backend-max-connections annotation.") - flag.IntVar(&controllerConfig.DefaultProxyBufferSize, "nginx-default-proxy-buffer-size", - defaultNginxProxyBufferSize, - "Proxy buffer size for response. Can be overridden per ingress with the sky.uk/proxy-buffer-size-in-kb annotation.") - flag.IntVar(&controllerConfig.DefaultProxyBufferBlocks, "nginx-default-proxy-buffer-blocks", - defaultNginxProxyBufferBlocks, - "Proxy buffer blocks for response. Can be overridden per ingress with the sky.uk/proxy-buffer-blocks annotation.") - flag.StringVar(&nginxConfig.LogLevel, "nginx-loglevel", defaultNginxLogLevel, - "Log level for nginx. See http://nginx.org/en/docs/ngx_core_module.html#error_log for levels.") - flag.IntVar(&nginxConfig.ServerNamesHashBucketSize, "nginx-server-names-hash-bucket-size", defaultNginxServerNamesHashBucketSize, - "Sets the bucket size for the server names hash tables. Setting this to 0 or less will exclude this "+ - "config from the nginx conf file. The details of setting up hash tables are provided "+ - "in a separate document. http://nginx.org/en/docs/hash.html") - flag.IntVar(&nginxConfig.ServerNamesHashMaxSize, "nginx-server-names-hash-max-size", defaultNginxServerNamesHashMaxSize, - "Sets the maximum size of the server names hash tables. Setting this to 0 or less will exclude this "+ - "config from the nginx conf file. The details of setting up hash tables are provided "+ - "in a separate document. http://nginx.org/en/docs/hash.html") - flag.BoolVar(&nginxConfig.ProxyProtocol, "nginx-proxy-protocol", defaultNginxProxyProtocol, - "Enable PROXY protocol for nginx listeners.") - flag.DurationVar(&nginxConfig.UpdatePeriod, "nginx-update-period", defaultNginxUpdatePeriod, - "How often nginx reloads can occur. Too frequent will result in many nginx worker processes alive at the same time.") - flag.StringVar(&nginxConfig.AccessLogDir, "access-log-dir", defaultAccessLogDir, "Access logs direcoty.") - flag.BoolVar(&nginxConfig.AccessLog, "access-log", false, "Enable access logs directive.") - flag.Var(&nginxLogHeaders, "nginx-log-headers", "Comma separated list of headers to be logged in access logs") - flag.Var(&nginxTrustedFrontends, "nginx-trusted-frontends", - "Comma separated list of CIDRs to trust when determining the client's real IP from "+ - "frontends. The client IP is used for allowing or denying ingress access. "+ - "This will typically be the ELB subnet.") - flag.StringVar(&nginxSSLPath, "ssl-path", defaultNginxSSLPath, - "Set default ssl path + name file without extension. Feed expects two files: one ending in .crt (the CA) and the other in .key (the private key).") - flag.IntVar(&nginxVhostStatsSharedMemory, "nginx-vhost-stats-shared-memory", defaultNginxVhostStatsSharedMemory, - "Memory (in MiB) which should be allocated for use by the vhost statistics module") - flag.StringVar(&nginxOpenTracingPluginPath, "nginx-opentracing-plugin-path", defaultNginxOpenTracingPluginPath, - "Path to OpenTracing plugin on disk (eg. /usr/local/lib/libjaegertracing_plugin.so)") - flag.StringVar(&nginxOpenTracingConfigPath, "nginx-opentracing-config-path", defaultNginxOpenTracingConfigPath, - "Path to OpenTracing config on disk (eg. /etc/jaeger-nginx-config.json)") - flag.IntVar(&nginxConfig.ClientHeaderBufferSize, "nginx-client-header-buffer-size-in-kb", defaultClientHeaderBufferSize, "Sets buffer size for reading client request header") - flag.IntVar(&nginxConfig.ClientBodyBufferSize, "nginx-client-body-buffer-size-in-kb", defaultClientBodyBufferSize, "Sets buffer size for reading client request body") - flag.IntVar(&nginxConfig.LargeClientHeaderBufferBlocks, "nginx-large-client-header-buffer-blocks", defaultLargeClientHeaderBufferBlocks, "Sets the maximum number of buffers used for reading large client request header") - - // elb/alb flags - flag.StringVar(®ion, "region", defaultRegion, - "AWS region for frontend attachment.") - flag.StringVar(&elbLabelValue, "elb-label-value", defaultElbLabelValue, - "Attach to ELBs tagged with "+elb.ElbTag+"=value. Leave empty to not attach.") - flag.StringVar(®istrationFrontendType, "registration-frontend-type", defaultRegistrationFrontendType, - "Define the registration frontend type. Must be merlin, gorb, elb or alb.") - flag.IntVar(&elbExpectedNumber, "elb-expected-number", defaultElbExpectedNumber, - "Expected number of ELBs to attach to. If 0 the controller will not check,"+ - " otherwise it fails to start if it can't attach to this number.") - flag.DurationVar(&drainDelay, "drain-delay", defaultDrainDelay, "Delay to wait"+ - " for feed-ingress to drain from the registration component on shutdown. Should match the ELB's drain time.") - flag.Var(&targetGroupNames, "alb-target-group-names", - "Names of ALB target groups to attach to, separated by commas.") - flag.DurationVar(&targetGroupDeregistrationDelay, "alb-target-group-deregistration-delay", - defaultTargetGroupDeregistrationDelay, - "Delay to wait for feed-ingress to deregister from the ALB target group on shutdown. Should match"+ - " the target group setting in AWS.") - - // prometheus flags - flag.StringVar(&pushgatewayURL, "pushgateway", "", - "Prometheus pushgateway URL for pushing metrics. Leave blank to not push metrics.") - flag.IntVar(&pushgatewayIntervalSeconds, "pushgateway-interval", defaultPushgatewayIntervalSeconds, - "Interval in seconds for pushing metrics.") - flag.Var(&pushgatewayLabels, "pushgateway-label", - "A label=value pair to attach to metrics pushed to prometheus. Specify multiple times for multiple labels.") - - // gorb flags - flag.StringVar(&gorbEndpoint, "gorb-endpoint", defaultGorbEndpoint, "Define the endpoint to talk to gorb for registration.") - flag.StringVar(&gorbIngressInstanceIP, "gorb-ingress-instance-ip", defaultGorbIngressInstanceIP, - "Define the ingress instance ip, the ip of the node where feed-ingress is running.") - flag.StringVar(&gorbServicesDefinition, "gorb-services-definition", defaultGorbServicesDefinition, - "Comma separated list of Service Definition (e.g. 'http-proxy:80,https-proxy:443') to register via Gorb") - flag.StringVar(&gorbBackendMethod, "gorb-backend-method", defaultGorbBackendMethod, - "Define the backend method (e.g. nat, dr, tunnel) to register via Gorb ") - flag.IntVar(&gorbBackendWeight, "gorb-backend-weight", defaultGorbBackendWeight, - "Define the backend weight to register via Gorb") - flag.StringVar(&gorbVipLoadbalancer, "gorb-vip-loadbalancer", defaultGorbVipLoadbalancer, - "Define the vip loadbalancer to set the loopback. Only necessary when Direct Return is enabled.") - flag.BoolVar(&gorbManageLoopback, "gorb-management-loopback", defaultGorbManageLoopback, - "Enable loopback creation. Only necessary when Direct Return is enabled") - flag.StringVar(&gorbInterfaceProcFsPath, "gorb-interface-proc-fs-path", defaultGorbInterfaceProcFsPath, - "Path to the interface proc file system. Only necessary when Direct Return is enabled") - flag.StringVar(&gorbBackendHealthcheckInterval, "gorb-backend-healthcheck-interval", defaultGorbBackendHealthcheckInterval, - "Define the gorb healthcheck interval for the backend") - flag.StringVar(&gorbBackendHealthcheckType, "gorb-backend-healthcheck-type", defaultGorbBackendHealthcheckType, - "Define the gorb healthcheck type for the backend. Must be either 'tcp', 'http' or 'none'") - - // merlin flags - flag.StringVar(&merlinEndpoint, "merlin-endpoint", "", - "Merlin gRPC endpoint to connect to. Expected format is scheme://authority/endpoint_name (see "+ - "https://github.com/grpc/grpc/blob/master/doc/naming.md). Will load balance between all available servers.") - flag.DurationVar(&merlinRequestTimeout, "merlin-request-timeout", time.Second*10, - "Timeout for any requests to merlin.") - flag.StringVar(&merlinServiceID, "merlin-service-id", "", "Merlin http virtual service ID to attach to.") - flag.StringVar(&merlinHTTPSServiceID, "merlin-https-service-id", "", "Merlin https virtual service ID to attach to.") - flag.StringVar(&merlinInstanceIP, "merlin-instance-ip", "", "Ingress IP to register with merlin") - flag.StringVar(&merlinForwardMethod, "merlin-forward-method", defaultMerlinForwardMethod, "IPVS forwarding method,"+ - " must be one of route, tunnel, or masq.") - flag.DurationVar(&merlinDrainDelay, "merlin-drain-delay", defaultDrainDelay, "Delay to wait after for connections"+ - " to bleed off when deregistering from merlin. Real server weight is set to 0 during this delay.") - flag.UintVar(&merlinHealthUpThreshold, "merlin-health-up-threshold", defaultMerlinHealthUpThreshold, - "Number of checks before merlin will consider this instance healthy.") - flag.UintVar(&merlinHealthDownThreshold, "merlin-health-down-threshold", defaultMerlinHealthDownThreshold, - "Number of checks before merlin will consider this instance unhealthy.") - flag.DurationVar(&merlinHealthPeriod, "merlin-health-period", defaultMerlinHealthPeriod, - "The time between health checks.") - flag.DurationVar(&merlinHealthTimeout, "merlin-health-timeout", defaultMerlinHealthTimeout, - "The timeout for health checks.") - flag.StringVar(&merlinVIP, "merlin-vip", "", "VIP to assign to loopback to support direct route and tunnel.") - flag.StringVar(&merlinVIPInterface, "merlin-vip-interface", defaultMerlinVIPInterface, - "VIP interface to assign the VIP.") - flag.StringVar(&merlinInternalHostname, "merlin-internal-hostname", "", - "Hostname of the internal facing load-balancer.") - flag.StringVar(&merlinInternetFacingHostname, "merlin-internet-facing-hostname", "", - "Hostname of the internet facing load-balancer") -} - -func main() { - flag.Parse() - - cmd.ConfigureLogging(debug) - cmd.ConfigureMetrics("feed-ingress", pushgatewayLabels, pushgatewayURL, pushgatewayIntervalSeconds) - - client, err := k8s.New(kubeconfig, resyncPeriod) - if err != nil { - log.Fatal("Unable to create k8s client: ", err) - } - controllerConfig.KubernetesClient = client - - controllerConfig.Updaters, err = createIngressUpdaters(client) - if err != nil { - log.Fatal("Unable to create ingress updaters: ", err) - } - - // If the legacy setting is set, use it instead to preserve backwards compatibility. - if legacyBackendKeepaliveSeconds != unset { - controllerConfig.DefaultBackendTimeoutSeconds = legacyBackendKeepaliveSeconds - } - - feedController := controller.New(controllerConfig) - - cmd.AddHealthMetrics(feedController, metrics.PrometheusIngressSubsystem) - cmd.AddHealthPort(feedController, healthPort) - cmd.AddSignalHandler(feedController) - - if err = feedController.Start(); err != nil { - log.Fatal("Error while starting controller: ", err) - } - log.Info("Controller started") - - select {} -} - -func createIngressUpdaters(kubernetesClient k8s.Client) ([]controller.Updater, error) { - nginxConfig.Ports = []nginx.Port{{Name: "http", Port: ingressPort}} - if ingressHTTPSPort != unset { - nginxConfig.Ports = append(nginxConfig.Ports, nginx.Port{Name: "https", Port: ingressHTTPSPort}) - } - - nginxConfig.HealthPort = ingressHealthPort - nginxConfig.SSLPath = nginxSSLPath - nginxConfig.TrustedFrontends = nginxTrustedFrontends - nginxConfig.LogHeaders = nginxLogHeaders - nginxConfig.VhostStatsSharedMemory = nginxVhostStatsSharedMemory - nginxConfig.OpenTracingPlugin = nginxOpenTracingPluginPath - nginxConfig.OpenTracingConfig = nginxOpenTracingConfigPath - nginxUpdater := nginx.New(nginxConfig) - - updaters := []controller.Updater{nginxUpdater} - - switch registrationFrontendType { - - case "elb": - elbUpdater, err := elb.New(region, elbLabelValue, elbExpectedNumber, drainDelay) - if err != nil { - return updaters, err - } - updaters = append(updaters, elbUpdater) - - statusConfig := elb_status.Config{ - Region: region, - LabelValue: elbLabelValue, - KubernetesClient: kubernetesClient, - } - elbStatusUpdater, err := elb_status.New(statusConfig) - if err != nil { - return updaters, err - } - updaters = append(updaters, elbStatusUpdater) - - case "alb": - albUpdater, err := alb.New(region, targetGroupNames, targetGroupDeregistrationDelay) - if err != nil { - return updaters, err - } - updaters = append(updaters, albUpdater) - - case "gorb": - virtualServices, err := toVirtualServices(gorbServicesDefinition) - if err != nil { - return nil, fmt.Errorf("invalid gorb services definition. Must be a comma separated list - e.g. 'http-proxy:80,https-proxy:443', but was %s", gorbServicesDefinition) - } - - if gorbBackendHealthcheckType != "tcp" && gorbBackendHealthcheckType != "http" && gorbBackendHealthcheckType != "none" { - return nil, fmt.Errorf("invalid gorb backend healthcheck type. Must be either 'tcp', 'http' or 'none', but was %s", gorbBackendHealthcheckType) - } - - config := gorb.Config{ - ServerBaseURL: gorbEndpoint, - InstanceIP: gorbIngressInstanceIP, - DrainDelay: drainDelay, - ServicesDefinition: virtualServices, - BackendMethod: gorbBackendMethod, - BackendWeight: gorbBackendWeight, - VipLoadbalancer: gorbVipLoadbalancer, - ManageLoopback: gorbManageLoopback, - BackendHealthcheckInterval: gorbBackendHealthcheckInterval, - BackendHealthcheckType: gorbBackendHealthcheckType, - InterfaceProcFsPath: gorbInterfaceProcFsPath, - } - gorbUpdater, err := gorb.New(&config) - if err != nil { - return updaters, err - } - updaters = append(updaters, gorbUpdater) - - case "merlin": - config := merlin.Config{ - Endpoint: merlinEndpoint, - Timeout: merlinRequestTimeout, - ServiceID: merlinServiceID, - HTTPSServiceID: merlinHTTPSServiceID, - InstanceIP: merlinInstanceIP, - InstancePort: uint16(ingressPort), - InstanceHTTPSPort: uint16(ingressHTTPSPort), - ForwardMethod: merlinForwardMethod, - DrainDelay: merlinDrainDelay, - HealthPort: uint16(ingressHealthPort), - // This value is hardcoded into the nginx template. - HealthPath: "health", - HealthUpThreshold: uint32(merlinHealthUpThreshold), - HealthDownThreshold: uint32(merlinHealthDownThreshold), - HealthPeriod: merlinHealthPeriod, - HealthTimeout: merlinHealthTimeout, - VIP: merlinVIP, - VIPInterface: merlinVIPInterface, - } - merlinUpdater, err := merlin.New(config) - if err != nil { - return updaters, err - } - updaters = append(updaters, merlinUpdater) - - if merlinInternalHostname != "" || merlinInternetFacingHostname != "" { - statusConfig := merlin_status.Config{ - InternalHostname: merlinInternalHostname, - InternetFacingHostname: merlinInternetFacingHostname, - KubernetesClient: kubernetesClient, - } - merlinStatusUpdater, err := merlin_status.New(statusConfig) - if err != nil { - return updaters, err - } - updaters = append(updaters, merlinStatusUpdater) - } - - default: - return nil, fmt.Errorf("invalid registration frontend type. Must be either gorb, elb, alb, merlin but"+ - "was %s", registrationFrontendType) - } - - return updaters, nil -} - -func toVirtualServices(servicesCsv string) ([]gorb.VirtualService, error) { - virtualServices := make([]gorb.VirtualService, 0) - servicesDefinitionArr := strings.Split(servicesCsv, ",") - for _, service := range servicesDefinitionArr { - servicesArr := strings.Split(service, ":") - if len(servicesArr) != 2 { - return nil, fmt.Errorf("unable to convert %s to servicename:port combination", servicesArr) - } - port, err := strconv.Atoi(servicesArr[1]) - if err != nil { - return nil, fmt.Errorf("unable to convert port %s to int", servicesArr[1]) - } - virtualServices = append(virtualServices, gorb.VirtualService{Name: servicesArr[0], Port: port}) - } - return virtualServices, nil -} diff --git a/controller/controller.go b/controller/controller.go index f61b02f4..f3c7b77e 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -16,28 +16,27 @@ import ( "github.com/sky-uk/feed/k8s" "github.com/sky-uk/feed/util" v1 "k8s.io/client-go/pkg/api/v1" + "k8s.io/client-go/pkg/apis/extensions/v1beta1" ) const ( ingressAllowAnnotation = "sky.uk/allow" frontendSchemeAnnotation = "sky.uk/frontend-scheme" - // Deprecated: retained to maintain backwards compatibility. - frontendElbSchemeAnnotation = "sky.uk/frontend-elb-scheme" - stripPathAnnotation = "sky.uk/strip-path" - exactPathAnnotation = "sky.uk/exact-path" + stripPathAnnotation = "sky.uk/strip-path" + exactPathAnnotation = "sky.uk/exact-path" - // Old annotation - still supported to maintain backwards compatibility. - legacyBackendKeepAliveSeconds = "sky.uk/backend-keepalive-seconds" - backendTimeoutSeconds = "sky.uk/backend-timeout-seconds" - proxyBufferSizeAnnotation = "sky.uk/proxy-buffer-size-in-kb" - proxyBufferBlocksAnnotation = "sky.uk/proxy-buffer-blocks" + backendTimeoutSeconds = "sky.uk/backend-timeout-seconds" + proxyBufferSizeAnnotation = "sky.uk/proxy-buffer-size-in-kb" + proxyBufferBlocksAnnotation = "sky.uk/proxy-buffer-blocks" maxAllowedProxyBufferSize = 32 maxAllowedProxyBufferBlocks = 8 // sets Nginx (http://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_conns) backendMaxConnections = "sky.uk/backend-max-connections" + + ingressClassAnnotation = "kubernetes.io/ingress.class" ) // Controller operates on ingress resources, listening for updates and notifying its Updaters. @@ -66,6 +65,9 @@ type controller struct { started bool updatesHealth util.SafeError sync.Mutex + name string + includeClasslessIngresses bool + namespaceSelector *k8s.NamespaceSelector } // Config for creating a new ingress controller. @@ -79,6 +81,9 @@ type Config struct { DefaultBackendMaxConnections int DefaultProxyBufferSize int DefaultProxyBufferBlocks int + Name string + IncludeClasslessIngresses bool + NamespaceSelector *k8s.NamespaceSelector } // New creates an ingress controller. @@ -94,6 +99,9 @@ func New(conf Config) Controller { defaultProxyBufferSize: conf.DefaultProxyBufferSize, defaultProxyBufferBlocks: conf.DefaultProxyBufferBlocks, doneCh: make(chan struct{}), + name: conf.Name, + includeClasslessIngresses: conf.IncludeClasslessIngresses, + namespaceSelector: conf.NamespaceSelector, } } @@ -132,7 +140,8 @@ func (c *controller) Start() error { func (c *controller) watchForUpdates() { ingressWatcher := c.client.WatchIngresses() serviceWatcher := c.client.WatchServices() - c.watcher = k8s.CombineWatchers(ingressWatcher, serviceWatcher) + namespaceWatcher := c.client.WatchNamespaces() + c.watcher = k8s.CombineWatchers(ingressWatcher, serviceWatcher, namespaceWatcher) c.watcherDone.Add(1) go c.handleUpdates() } @@ -162,7 +171,13 @@ func (c *controller) updateIngresses() (err error) { err = fmt.Errorf("unexpected error: %v: %v", value, string(debug.Stack())) } }() - ingresses, err := c.client.GetIngresses() + + var ingresses []*v1beta1.Ingress + if c.namespaceSelector == nil { + ingresses, err = c.client.GetAllIngresses() + } else { + ingresses, err = c.client.GetIngresses(c.namespaceSelector) + } log.Infof("Found %d ingresses", len(ingresses)) if err != nil { return err @@ -184,31 +199,34 @@ func (c *controller) updateIngresses() (err error) { serviceName := serviceName{namespace: ingress.Namespace, name: path.Backend.ServiceName} - if address := serviceMap[serviceName]; address != "" { + if address := serviceMap[serviceName]; address == "" { + skipped = append(skipped, fmt.Sprintf("%s/%s (service doesn't exist)", ingress.Namespace, ingress.Name)) + } else if !c.ingressClassSupported(ingress) { + skipped = append(skipped, fmt.Sprintf("%s/%s (ingress requests class [%s]; this instance is [%s])", + ingress.Namespace, ingress.Name, ingress.Annotations[ingressClassAnnotation], c.name)) + } else { entry := IngressEntry{ - Namespace: ingress.Namespace, - Name: ingress.Name, - Host: rule.Host, - Path: path.Path, - ServiceAddress: address, - ServicePort: int32(path.Backend.ServicePort.IntValue()), - Allow: c.defaultAllow, - StripPaths: c.defaultStripPath, - ExactPath: c.defaultExactPath, - BackendTimeoutSeconds: c.defaultBackendTimeout, + Namespace: ingress.Namespace, + Name: ingress.Name, + Host: rule.Host, + Path: path.Path, + ServiceAddress: address, + ServicePort: int32(path.Backend.ServicePort.IntValue()), + Allow: c.defaultAllow, + StripPaths: c.defaultStripPath, + ExactPath: c.defaultExactPath, BackendTimeoutSeconds: c.defaultBackendTimeout, BackendMaxConnections: c.defaultBackendMaxConnections, ProxyBufferSize: c.defaultProxyBufferSize, ProxyBufferBlocks: c.defaultProxyBufferBlocks, CreationTimestamp: ingress.CreationTimestamp.Time, Ingress: ingress, + IngressClass: ingress.Annotations[ingressClassAnnotation], } log.Debugf("Found ingress to update: %s", ingress.Name) if lbScheme, ok := ingress.Annotations[frontendSchemeAnnotation]; ok { entry.LbScheme = lbScheme - } else { - entry.LbScheme = ingress.Annotations[frontendElbSchemeAnnotation] } if allow, ok := ingress.Annotations[ingressAllowAnnotation]; ok { @@ -225,7 +243,8 @@ func (c *controller) updateIngresses() (err error) { } else if stripPath == "false" { entry.StripPaths = false } else { - log.Warnf("Ingress %s has an invalid strip path annotation: %s. Using default", ingress.Name, stripPath) + log.Warnf("Ingress %s/%s has an invalid strip path annotation [%s]. Using default", + ingress.Namespace, ingress.Name, stripPath) } } @@ -235,15 +254,11 @@ func (c *controller) updateIngresses() (err error) { } else if exactPath == "false" { entry.ExactPath = false } else { - log.Warnf("Ingress %s has an invalid exact path annotation: %s. Using default", ingress.Name, exactPath) + log.Warnf("Ingress %s/%s has an invalid exact path annotation [%s]. Using default", + ingress.Namespace, ingress.Name, exactPath) } } - if backendKeepAlive, ok := ingress.Annotations[legacyBackendKeepAliveSeconds]; ok { - tmp, _ := strconv.Atoi(backendKeepAlive) - entry.BackendTimeoutSeconds = tmp - } - if timeout, ok := ingress.Annotations[backendTimeoutSeconds]; ok { tmp, _ := strconv.Atoi(timeout) entry.BackendTimeoutSeconds = tmp @@ -277,8 +292,6 @@ func (c *controller) updateIngresses() (err error) { } else { skipped = append(skipped, fmt.Sprintf("%s (%v)", entry.NamespaceName(), err)) } - } else { - skipped = append(skipped, fmt.Sprintf("%s/%s (service doesn't exist)", ingress.Namespace, ingress.Name)) } } @@ -302,6 +315,19 @@ func (c *controller) updateIngresses() (err error) { return nil } +func (c *controller) ingressClassSupported(ingress *v1beta1.Ingress) bool { + + isValid := false + + if ingressClass, ok := ingress.Annotations[ingressClassAnnotation]; ok { + isValid = ingressClass == c.name + } else { + isValid = c.includeClasslessIngresses + } + + return isValid +} + type serviceName struct { namespace string name string diff --git a/controller/controller_test.go b/controller/controller_test.go index 24672d89..05389864 100644 --- a/controller/controller_test.go +++ b/controller/controller_test.go @@ -13,10 +13,12 @@ import ( "github.com/stretchr/testify/mock" v1 "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/pkg/apis/extensions/v1beta1" + metav1 "k8s.io/client-go/pkg/apis/meta/v1" "k8s.io/client-go/pkg/util/intstr" ) const smallWaitTime = time.Millisecond * 50 +const defaultIngressClass = "main" type fakeUpdater struct { mock.Mock @@ -73,11 +75,14 @@ func createDefaultStubs() (*fakeUpdater, *fake.FakeClient) { client := new(fake.FakeClient) ingressWatcher, _ := createFakeWatcher() serviceWatcher, _ := createFakeWatcher() + namespaceWatcher, _ := createFakeWatcher() - client.On("GetIngresses").Return([]*v1beta1.Ingress{}, nil) + client.On("GetAllIngresses").Return([]*v1beta1.Ingress{}, nil) + client.On("GetIngresses", mock.Anything).Return([]*v1beta1.Ingress{}, nil) client.On("GetServices").Return([]*v1.Service{}, nil) client.On("WatchIngresses").Return(ingressWatcher) client.On("WatchServices").Return(serviceWatcher) + client.On("WatchNamespaces").Return(namespaceWatcher) updater.On("Start").Return(nil) updater.On("Stop").Return(nil) updater.On("Update", mock.Anything).Return(nil) @@ -223,7 +228,7 @@ func TestControllerIsUnhealthyIfUpdaterIsUnhealthy(t *testing.T) { assert.NoError(controller.Health()) assert.Equal(lbErr, controller.Health()) - controller.Stop() + _ = controller.Stop() } func TestControllerReturnsErrorIfUpdaterFails(t *testing.T) { @@ -247,6 +252,7 @@ func TestUnhealthyIfUpdaterFails(t *testing.T) { ingressWatcher, updateCh := createFakeWatcher() serviceWatcher, _ := createFakeWatcher() + namespaceWatcher, _ := createFakeWatcher() updater.On("Start").Return(nil) updater.On("Stop").Return(nil) @@ -254,10 +260,11 @@ func TestUnhealthyIfUpdaterFails(t *testing.T) { updater.On("Update", mock.Anything).Return(fmt.Errorf("kaboom, update failed :(")).Once() updater.On("Health").Return(nil) - client.On("GetIngresses").Return([]*v1beta1.Ingress{}, nil) + client.On("GetAllIngresses").Return([]*v1beta1.Ingress{}, nil) client.On("GetServices").Return([]*v1.Service{}, nil) client.On("WatchIngresses").Return(ingressWatcher) client.On("WatchServices").Return(serviceWatcher) + client.On("WatchNamespaces").Return(namespaceWatcher) assert.NoError(controller.Start()) // expect @@ -270,508 +277,869 @@ func TestUnhealthyIfUpdaterFails(t *testing.T) { assert.Error(controller.Health()) // cleanup - controller.Stop() + _ = controller.Stop() } func defaultConfig() Config { return Config{ DefaultAllow: ingressDefaultAllow, DefaultBackendTimeoutSeconds: backendTimeout, + Name: defaultIngressClass, } } -func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) { - //given - assert := assert.New(t) +type testSpec struct { + description string + ingresses []*v1beta1.Ingress + services []*v1.Service + namespaces []*v1.Namespace + entries IngressEntries + config Config +} - var tests = []struct { - description string - ingresses []*v1beta1.Ingress - services []*v1.Service - entries IngressEntries - config Config - }{ - { - "ingress tagged with sky.uk/frontend-scheme", - createIngressesFromNonELBAnnotation(), - createDefaultServices(), - createLbEntriesFixture(), - defaultConfig(), - }, - { - "ingress with corresponding service", - createDefaultIngresses(), - createDefaultServices(), - createLbEntriesFixture(), - defaultConfig(), - }, - { - "ingress with extra services", - createDefaultIngresses(), - append(createDefaultServices(), - createServiceFixture("another one", ingressNamespace, serviceIP)...), - createLbEntriesFixture(), - defaultConfig(), - }, - { - "ingress without corresponding service", - createDefaultIngresses(), - []*v1.Service{}, - nil, - defaultConfig(), - }, - { - "ingress with service with non-matching namespace", - createDefaultIngresses(), - createServiceFixture(ingressSvcName, "lalala land", serviceIP), - nil, - defaultConfig(), - }, - { - "ingress with service with non-matching name", - createDefaultIngresses(), - createServiceFixture("lalala service", ingressNamespace, serviceIP), - nil, - defaultConfig(), - }, - { - "ingress with missing host name", - createIngressesFixture("", ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: ingressAllow, - backendTimeoutSeconds: "10", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - nil, - defaultConfig(), - }, - { - "ingress with missing service name", - createIngressesFixture(ingressHost, "", ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: ingressAllow, - backendTimeoutSeconds: "10", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - nil, - defaultConfig(), - }, - { - "ingress with missing service port", - createIngressesFixture(ingressHost, ingressSvcName, 0, - map[string]string{ - ingressAllowAnnotation: ingressAllow, - backendTimeoutSeconds: "10", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - nil, - defaultConfig(), - }, - { - "ingress with missing service IP", - createDefaultIngresses(), - createServiceFixture(ingressSvcName, ingressNamespace, ""), - nil, - defaultConfig(), - }, - { - "ingress with 'None' as service IP", - createDefaultIngresses(), - createServiceFixture(ingressSvcName, ingressNamespace, "None"), - nil, - defaultConfig(), - }, - { - "ingress with default allow", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - backendTimeoutSeconds: "10", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - Allow: strings.Split(ingressDefaultAllow, ","), - BackendTimeoutSeconds: backendTimeout, - }}, - defaultConfig(), - }, - { - "ingress with empty allow", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - backendTimeoutSeconds: "10", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "internal", - Allow: []string{}, - BackendTimeoutSeconds: backendTimeout, - }}, - defaultConfig(), - }, - { - "ingress with strip paths set to true", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - stripPathAnnotation: "true", - backendTimeoutSeconds: "10", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "internal", - Allow: []string{}, - StripPaths: true, - BackendTimeoutSeconds: backendTimeout, - }}, - defaultConfig(), - }, - { - "ingress with strip paths set to false", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - stripPathAnnotation: "false", - backendTimeoutSeconds: "10", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "internal", - Allow: []string{}, - StripPaths: false, - BackendTimeoutSeconds: backendTimeout, - }}, - defaultConfig(), - }, - { - "ingress with exact path set to true", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - exactPathAnnotation: "true", - backendTimeoutSeconds: "10", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "internal", - Allow: []string{}, - ExactPath: true, - BackendTimeoutSeconds: backendTimeout, - }}, - defaultConfig(), - }, - { - "ingress with exact path set to false", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - exactPathAnnotation: "false", - backendTimeoutSeconds: "10", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "internal", - Allow: []string{}, - ExactPath: false, - BackendTimeoutSeconds: backendTimeout, - }}, - defaultConfig(), - }, - { - "ingress with overridden backend timeout", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - stripPathAnnotation: "false", - backendTimeoutSeconds: "20", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "internal", - Allow: []string{}, - StripPaths: false, - BackendTimeoutSeconds: 20, - }}, - defaultConfig(), - }, - { - "ingress with default backend timeout", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - stripPathAnnotation: "false", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "internal", - Allow: []string{}, - StripPaths: false, - BackendTimeoutSeconds: backendTimeout, - }}, - defaultConfig(), +func TestUpdaterIsUpdatedForIngressTaggedWithSkyFrontendScheme(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress tagged with sky.uk/frontend-scheme", + createIngressesFromNonELBAnnotation(), + createDefaultServices(), + createDefaultNamespaces(), + createLbEntriesFixture(), + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithCorrespondingService(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with corresponding service", + createDefaultIngresses(), + createDefaultServices(), + createDefaultNamespaces(), + createLbEntriesFixture(), + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithExtraServices(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with extra services", + createDefaultIngresses(), + append(createDefaultServices(), + createServiceFixture("another one", ingressNamespace, serviceIP)...), + createDefaultNamespaces(), + createLbEntriesFixture(), + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithoutCorrespondingService(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress without corresponding service", + createDefaultIngresses(), + []*v1.Service{}, + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForServiceWithNonMatchingNamespace(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with service with non-matching namespace", + createDefaultIngresses(), + createServiceFixture(ingressSvcName, "lalala land", serviceIP), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForServiceWithNonMatchingName(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with service with non-matching name", + createDefaultIngresses(), + createServiceFixture("lalala service", ingressNamespace, serviceIP), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithMissingHostName(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with missing host name", + createIngressesFixture(ingressNamespace, "", ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: ingressAllow, + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithMissingServiceName(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with missing service name", + createIngressesFixture(ingressNamespace, ingressHost, "", ingressSvcPort, map[string]string{ + ingressAllowAnnotation: ingressAllow, + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithMissingServicePort(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with missing service port", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, 0, map[string]string{ + ingressAllowAnnotation: ingressAllow, + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithMissingServiceIP(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with missing service IP", + createDefaultIngresses(), + createServiceFixture(ingressSvcName, ingressNamespace, ""), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithNoneAsServiceIP(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with 'None' as service IP", + createDefaultIngresses(), + createServiceFixture(ingressSvcName, ingressNamespace, "None"), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithDefaultAllow(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with default allow", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + backendTimeoutSeconds: "10", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + Allow: strings.Split(ingressDefaultAllow, ","), + IngressClass: defaultIngressClass, + BackendTimeoutSeconds: backendTimeout, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithEmptyAllow(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with empty allow", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + BackendTimeoutSeconds: backendTimeout, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithStripPathsTrue(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with strip paths set to true", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "true", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: true, + BackendTimeoutSeconds: backendTimeout, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithStripPathsFalse(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with strip paths set to false", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: backendTimeout, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithExactPathTrue(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with exact path set to true", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + exactPathAnnotation: "true", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + ExactPath: true, + BackendTimeoutSeconds: backendTimeout, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithExactPathFalse(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with exact path set to false", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + exactPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + ExactPath: false, + BackendTimeoutSeconds: backendTimeout, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithOverriddenBackendTimeout(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with overridden backend timeout", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "20", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: 20, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithDefaultBackendTimeout(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with default backend timeout", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: backendTimeout, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithOverriddenBackendMaxConnections(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with overridden backend max connections", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "20", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + backendMaxConnections: "512", + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: 20, + BackendMaxConnections: 512, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithDefaultBackendMaxConnections(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with default backend max connections", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "20", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: 20, + BackendMaxConnections: defaultMaxConnections, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressWithDefaultProxyBufferValuesWhenNotOverridden(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with default proxy buffer values when not overridden by the ingress definition", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: 10, + BackendMaxConnections: defaultMaxConnections, + ProxyBufferSize: 2, + ProxyBufferBlocks: 3, + }}, + Config{ + DefaultBackendTimeoutSeconds: backendTimeout, + DefaultProxyBufferSize: 2, + DefaultProxyBufferBlocks: 3, + Name: defaultIngressClass, }, - { - "ingress with overridden backend max connections", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - stripPathAnnotation: "false", - backendTimeoutSeconds: "20", - frontendElbSchemeAnnotation: "internal", - backendMaxConnections: "512", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "internal", - Allow: []string{}, - StripPaths: false, - BackendTimeoutSeconds: 20, - BackendMaxConnections: 512, - }}, - defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressOverridesDefaultProxyBufferValues(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress definition overrides default proxy buffer values", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + proxyBufferSizeAnnotation: "6", + proxyBufferBlocksAnnotation: "4", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: 10, + BackendMaxConnections: defaultMaxConnections, + ProxyBufferSize: 6, + ProxyBufferBlocks: 4, + }}, + Config{ + DefaultBackendTimeoutSeconds: backendTimeout, + DefaultProxyBufferSize: 2, + DefaultProxyBufferBlocks: 3, + Name: defaultIngressClass, }, - { - "ingress with default backend max connections", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - stripPathAnnotation: "false", - backendTimeoutSeconds: "20", - frontendElbSchemeAnnotation: "internal", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "internal", - Allow: []string{}, - StripPaths: false, - BackendTimeoutSeconds: 20, - BackendMaxConnections: defaultMaxConnections, - }}, - defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressDefinitionResetsToMacWhenProxyBufferValuesExceedMax(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress definition resets to max when proxy buffer values exceed max allowed values", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + proxyBufferSizeAnnotation: "64", + proxyBufferBlocksAnnotation: "12", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: 10, + BackendMaxConnections: defaultMaxConnections, + ProxyBufferSize: 32, + ProxyBufferBlocks: 8, + }}, + Config{ + DefaultBackendTimeoutSeconds: backendTimeout, + DefaultProxyBufferSize: 2, + DefaultProxyBufferBlocks: 3, + Name: defaultIngressClass, }, - { - "ingress with default proxy buffer values when not overridden by the ingress definition", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "", - Allow: []string{}, - StripPaths: false, - BackendTimeoutSeconds: 10, - BackendMaxConnections: defaultMaxConnections, - ProxyBufferSize: 2, - ProxyBufferBlocks: 3, - }}, - Config{ - DefaultBackendTimeoutSeconds: backendTimeout, - DefaultProxyBufferSize: 2, - DefaultProxyBufferBlocks: 3, - }, + }) +} + +func TestUpdaterIsUpdatedForIngressClassNotSetInIngress(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress without ingress class set", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + IngressClass: defaultIngressClass, + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: backendTimeout, + }}, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressClassSetToDefaultInIngress(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress with class set to default value", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: backendTimeout, + IngressClass: defaultIngressClass, + }}, + Config{ + DefaultAllow: ingressDefaultAllow, + DefaultBackendTimeoutSeconds: backendTimeout, + Name: defaultIngressClass, }, - { - "ingress definition overrides default proxy buffer values", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - proxyBufferSizeAnnotation: "6", - proxyBufferBlocksAnnotation: "4", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "", - Allow: []string{}, - StripPaths: false, - BackendTimeoutSeconds: 10, - BackendMaxConnections: defaultMaxConnections, - ProxyBufferSize: 6, - ProxyBufferBlocks: 4, - }}, - Config{ - DefaultBackendTimeoutSeconds: backendTimeout, - DefaultProxyBufferSize: 2, - DefaultProxyBufferBlocks: 3, - }, + }) +} + +func TestUpdaterIsNotUpdatedForIngressClassSetToTestAndConfigSetToDefault(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress requesting class==test; feed instance has class " + defaultIngressClass, + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: "test", + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedWhenIncludingClasslessIngresses(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress has no class annotation; feed-ingress is including classless ingresses", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: backendTimeout, + IngressClass: "", + }}, + Config{ + DefaultAllow: ingressDefaultAllow, + DefaultBackendTimeoutSeconds: backendTimeout, + Name: defaultIngressClass, + IncludeClasslessIngresses: true, }, - { - "ingress definition resets to max when proxy buffer values exceed max allowed values", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: "", - proxyBufferSizeAnnotation: "64", - proxyBufferBlocksAnnotation: "12", - }, ingressPath), - createDefaultServices(), - []IngressEntry{{ - Namespace: ingressNamespace, - Name: ingressName, - Host: ingressHost, - Path: ingressPath, - ServiceAddress: serviceIP, - ServicePort: ingressSvcPort, - LbScheme: "", - Allow: []string{}, - StripPaths: false, - BackendTimeoutSeconds: 10, - BackendMaxConnections: defaultMaxConnections, - ProxyBufferSize: 32, - ProxyBufferBlocks: 8, - }}, - Config{ - DefaultBackendTimeoutSeconds: backendTimeout, - DefaultProxyBufferSize: 2, - DefaultProxyBufferBlocks: 3, - }, + }) +} + +func TestUpdaterIsNotUpdatedWhenExcludingClasslessIngresses(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress has no class annotation; feed-ingress is excluding classless ingresses", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + nil, + Config{ + DefaultAllow: ingressDefaultAllow, + DefaultBackendTimeoutSeconds: backendTimeout, + Name: defaultIngressClass, + IncludeClasslessIngresses: false, }, - { - "ingress without host definition", - createIngressesFixture("", ingressSvcName, ingressSvcPort, - map[string]string{}, ""), - createServiceFixture(ingressSvcName, "lalala land", serviceIP), - nil, - defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressClassSetToTestInIngressAndConfig(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress requesting class==test; feed has class test", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: "test", + }, ingressPath), + createDefaultServices(), + createDefaultNamespaces(), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: backendTimeout, + IngressClass: "test", + }}, + Config{ + DefaultAllow: ingressDefaultAllow, + DefaultBackendTimeoutSeconds: backendTimeout, + Name: "test", }, - { - "ingress without path definition", - createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{}, ""), - createServiceFixture(ingressSvcName, "lalala land", serviceIP), - nil, - defaultConfig(), + }) +} + +func TestUpdaterIsUpdatedForIngressInNamespaceMatchingSelector(t *testing.T) { + runAndAssertUpdates(t, expectGetIngresses, testSpec{ + "ingress is in a namespace that matches the namespace selector", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + createDefaultServices(), + createNamespaceFixture(ingressNamespace, map[string]string{"team": "theteam"}), + []IngressEntry{{ + Namespace: ingressNamespace, + Name: ingressName, + Host: ingressHost, + Path: ingressPath, + ServiceAddress: serviceIP, + ServicePort: ingressSvcPort, + LbScheme: "internal", + Allow: []string{}, + StripPaths: false, + BackendTimeoutSeconds: backendTimeout, + IngressClass: defaultIngressClass, + }}, + Config{ + DefaultAllow: ingressDefaultAllow, + DefaultBackendTimeoutSeconds: backendTimeout, + Name: defaultIngressClass, + NamespaceSelector: &k8s.NamespaceSelector{LabelName: "team", LabelValue: "theteam"}, }, - { - "ingress without rules definition", - createIngressWithoutRules(), - createServiceFixture(ingressSvcName, "lalala land", serviceIP), - nil, - defaultConfig(), + }) +} + +func TestUpdaterIsNotUpdatedForIngressInNamespaceNotMatchingSelector(t *testing.T) { + runAndAssertUpdates(t, expectGetIngresses, testSpec{ + "ingress is in a namespace that doesn't match the namespace selector", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: "", + stripPathAnnotation: "false", + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath), + nil, + createNamespaceFixture(ingressNamespace, map[string]string{"team": "otherteam"}), + nil, + Config{ + DefaultAllow: ingressDefaultAllow, + DefaultBackendTimeoutSeconds: backendTimeout, + Name: defaultIngressClass, + NamespaceSelector: &k8s.NamespaceSelector{LabelName: "team", LabelValue: "theteam"}, }, - } + }) +} - for _, test := range tests { - fmt.Printf("test: %s\n", test.description) - // add ingress pointers to entries - test.entries = addIngresses(test.ingresses, test.entries) +func TestUpdaterIsUpdatedForIngressWithoutHostDefinition(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress without host definition", + createIngressesFixture(ingressNamespace, "", ingressSvcName, ingressSvcPort, map[string]string{ + ingressClassAnnotation: defaultIngressClass, + }, ""), + createServiceFixture(ingressSvcName, "lalala land", serviceIP), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} - // setup clients - client := new(fake.FakeClient) - updater := new(fakeUpdater) +func TestUpdaterIsUpdatedForIngressWithoutPathDefinition(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress without path definition", + createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressClassAnnotation: defaultIngressClass, + }, ""), + createServiceFixture(ingressSvcName, "lalala land", serviceIP), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} - config := test.config +func TestUpdaterIsUpdatedForIngressWithoutRulesDefinition(t *testing.T) { + runAndAssertUpdates(t, expectGetAllIngresses, testSpec{ + "ingress without rules definition", + createIngressWithoutRules(), + createServiceFixture(ingressSvcName, "lalala land", serviceIP), + createDefaultNamespaces(), + nil, + defaultConfig(), + }) +} - config.KubernetesClient = client - config.Updaters = []Updater{updater} +type clientExpectation func(client *fake.FakeClient, ingresses []*v1beta1.Ingress) - controller := New(config) +var expectGetAllIngresses = func(client *fake.FakeClient, ingresses []*v1beta1.Ingress) { + client.On("GetAllIngresses").Return(ingresses, nil) +} - updater.On("Start").Return(nil) - updater.On("Stop").Return(nil) - // once for ingress update, once for service update - updater.On("Update", test.entries).Return(nil).Times(2) +var expectGetIngresses = func(client *fake.FakeClient, ingresses []*v1beta1.Ingress) { + client.On("GetIngresses", &k8s.NamespaceSelector{LabelName: "team", LabelValue: "theteam"}).Return(ingresses, nil) +} - client.On("GetIngresses").Return(test.ingresses, nil) - client.On("GetServices").Return(test.services, nil) +func runAndAssertUpdates(t *testing.T, clientExpectation clientExpectation, test testSpec) { + //given + assert := assert.New(t) - ingressWatcher, ingressCh := createFakeWatcher() - serviceWatcher, serviceCh := createFakeWatcher() - client.On("WatchIngresses").Return(ingressWatcher) - client.On("WatchServices").Return(serviceWatcher) + fmt.Printf("test: %s\n", test.description) + // add ingress pointers to entries + test.entries = addIngresses(test.ingresses, test.entries) - //when - assert.NoError(controller.Start()) - ingressCh <- struct{}{} - serviceCh <- struct{}{} - time.Sleep(smallWaitTime) + // setup clients + client := new(fake.FakeClient) + updater := new(fakeUpdater) - //then - assert.NoError(controller.Stop()) - time.Sleep(smallWaitTime) - updater.AssertExpectations(t) - } + config := test.config + + config.KubernetesClient = client + config.Updaters = []Updater{updater} + + controller := New(config) + + updater.On("Start").Return(nil) + updater.On("Stop").Return(nil) + // once for each update: ingress, service, namespace + updater.On("Update", test.entries).Return(nil).Times(3) + + clientExpectation(client, test.ingresses) + client.On("GetServices").Return(test.services, nil) + + ingressWatcher, ingressCh := createFakeWatcher() + serviceWatcher, serviceCh := createFakeWatcher() + namespaceWatcher, namespaceCh := createFakeWatcher() + client.On("WatchIngresses").Return(ingressWatcher) + client.On("WatchServices").Return(serviceWatcher) + client.On("WatchNamespaces").Return(namespaceWatcher) + + //when + assert.NoError(controller.Start()) + ingressCh <- struct{}{} + serviceCh <- struct{}{} + namespaceCh <- struct{}{} + time.Sleep(smallWaitTime) + + //then + assert.NoError(controller.Stop()) + time.Sleep(smallWaitTime) + updater.AssertExpectations(t) + client.AssertExpectations(t) } func addIngresses(ingresses []*v1beta1.Ingress, entries IngressEntries) IngressEntries { @@ -796,6 +1164,7 @@ func createLbEntriesFixture() IngressEntries { ServicePort: ingressSvcPort, Allow: strings.Split(ingressAllow, ","), LbScheme: lbScheme, + IngressClass: defaultIngressClass, BackendTimeoutSeconds: backendTimeout, }} } @@ -816,21 +1185,21 @@ const ( ) func createDefaultIngresses() []*v1beta1.Ingress { - return createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: ingressAllow, - backendTimeoutSeconds: "10", - frontendElbSchemeAnnotation: "internal", - }, ingressPath) + return createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: ingressAllow, + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath) } func createIngressesFromNonELBAnnotation() []*v1beta1.Ingress { - return createIngressesFixture(ingressHost, ingressSvcName, ingressSvcPort, - map[string]string{ - ingressAllowAnnotation: ingressAllow, - backendTimeoutSeconds: "10", - frontendSchemeAnnotation: "internal", - }, ingressPath) + return createIngressesFixture(ingressNamespace, ingressHost, ingressSvcName, ingressSvcPort, map[string]string{ + ingressAllowAnnotation: ingressAllow, + backendTimeoutSeconds: "10", + frontendSchemeAnnotation: "internal", + ingressClassAnnotation: defaultIngressClass, + }, ingressPath) } func createIngressWithoutRules() []*v1beta1.Ingress { @@ -839,6 +1208,9 @@ func createIngressWithoutRules() []*v1beta1.Ingress { ObjectMeta: v1.ObjectMeta{ Name: ingressName, Namespace: ingressNamespace, + Annotations: map[string]string{ + ingressClassAnnotation: defaultIngressClass, + }, }, Spec: v1beta1.IngressSpec{}, }, @@ -846,7 +1218,7 @@ func createIngressWithoutRules() []*v1beta1.Ingress { } -func createIngressesFixture(host string, serviceName string, servicePort int, ingressAnnotations map[string]string, path string) []*v1beta1.Ingress { +func createIngressesFixture(namespace string, host string, serviceName string, servicePort int, ingressAnnotations map[string]string, path string) []*v1beta1.Ingress { paths := []v1beta1.HTTPIngressPath{{ Path: path, @@ -866,8 +1238,6 @@ func createIngressesFixture(host string, serviceName string, servicePort int, in annotations[stripPathAnnotation] = annotationVal case exactPathAnnotation: annotations[exactPathAnnotation] = annotationVal - case frontendElbSchemeAnnotation: - annotations[frontendElbSchemeAnnotation] = annotationVal case frontendSchemeAnnotation: annotations[frontendSchemeAnnotation] = annotationVal case backendTimeoutSeconds: @@ -878,6 +1248,8 @@ func createIngressesFixture(host string, serviceName string, servicePort int, in annotations[proxyBufferSizeAnnotation] = annotationVal case proxyBufferBlocksAnnotation: annotations[proxyBufferBlocksAnnotation] = annotationVal + case ingressClassAnnotation: + annotations[ingressClassAnnotation] = annotationVal } } @@ -897,6 +1269,7 @@ func createIngressesFixture(host string, serviceName string, servicePort int, in ingressRules = append(ingressRules, ingressRule) } + ingressDefinition[0].ObjectMeta.Namespace = namespace ingressDefinition[0].ObjectMeta.Annotations = annotations ingressDefinition[0].Spec.Rules = ingressRules @@ -920,3 +1293,24 @@ func createServiceFixture(name string, namespace string, clusterIP string) []*v1 }, } } + +func createDefaultNamespaces() []*v1.Namespace { + return createNamespaceFixture(ingressNamespace, map[string]string{}) +} + +func createNamespaceFixture(name string, labels map[string]string) []*v1.Namespace { + return []*v1.Namespace{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + Labels: labels, + }, + Spec: v1.NamespaceSpec{}, + Status: v1.NamespaceStatus{}, + }, + } +} diff --git a/controller/ingress_entry.go b/controller/ingress_entry.go index 42281f07..729452ac 100644 --- a/controller/ingress_entry.go +++ b/controller/ingress_entry.go @@ -13,6 +13,8 @@ type IngressEntries []IngressEntry // IngressEntry describes the ingress for a single host, path, and service. type IngressEntry struct { + // The name of the feed-ingress instance that will manage the ingress resource. + IngressClass string // Namespace of the ingress. Namespace string // Name of the ingress. diff --git a/docker/ingress/Dockerfile b/docker/ingress/Dockerfile index 10c9c641..882f62bb 100644 --- a/docker/ingress/Dockerfile +++ b/docker/ingress/Dockerfile @@ -47,4 +47,4 @@ COPY nginx.tmpl /nginx/ RUN chown feed:feed /nginx/nginx.tmpl USER feed -ENTRYPOINT ["/feed-ingress", "-nginx-workdir", "/nginx"] +ENTRYPOINT ["/feed-ingress", "--nginx-workdir", "/nginx"] diff --git a/elb/elb.go b/elb/elb.go index 467d7269..a5beddfa 100644 --- a/elb/elb.go +++ b/elb/elb.go @@ -18,16 +18,23 @@ import ( "github.com/sky-uk/feed/util" ) -// ElbTag is the tag key used for identifying ELBs to attach to. +// ElbTag is the tag key used for identifying ELBs to attach to for a cluster. const ElbTag = "sky.uk/KubernetesClusterFrontend" +// IngressClassTag is the tag key used for identifying ELBs to attach to for a given ingress controller. +const IngressClassTag = "sky.uk/KubernetesClusterIngressClass" + // New creates a new ELB frontend -func New(region string, labelValue string, expectedNumber int, drainDelay time.Duration) (controller.Updater, error) { - if labelValue == "" { - return nil, fmt.Errorf("unable to create ELB updater: missing label value for the tag %v", ElbTag) +func New(region string, frontendTagValue string, ingressClassTagValue string, expectedNumber int, drainDelay time.Duration) (controller.Updater, error) { + if frontendTagValue == "" { + return nil, fmt.Errorf("unable to create ELB updater: missing value for the tag %v", ElbTag) + } + if ingressClassTagValue == "" { + return nil, fmt.Errorf("unable to create ELB updater: missing value for the tag %v", IngressClassTag) } + initMetrics() - log.Infof("ELB Front end region: %s cluster: %s expected frontends: %d", region, labelValue, expectedNumber) + log.Infof("ELB Front end region: %s, cluster: %s, expected frontends: %d, ingress controller: %s", region, frontendTagValue, expectedNumber, ingressClassTagValue) session, err := session.NewSession(&aws.Config{Region: ®ion}) if err != nil { @@ -35,13 +42,14 @@ func New(region string, labelValue string, expectedNumber int, drainDelay time.D } return &elb{ - metadata: ec2metadata.New(session), - awsElb: aws_elb.New(session), - labelValue: labelValue, - region: region, - expectedNumber: expectedNumber, - initialised: initialised{}, - drainDelay: drainDelay, + metadata: ec2metadata.New(session), + awsElb: aws_elb.New(session), + frontendTagValue: frontendTagValue, + ingressClassTagValue: ingressClassTagValue, + region: region, + expectedNumber: expectedNumber, + initialised: initialised{}, + drainDelay: drainDelay, }, nil } @@ -54,17 +62,18 @@ type LoadBalancerDetails struct { } type elb struct { - awsElb ELB - metadata EC2Metadata - labelValue string - region string - expectedNumber int - instanceID string - elbs map[string]LoadBalancerDetails - registeredFrontends util.SafeInt - initialised initialised - drainDelay time.Duration - readyForHealthCheck util.SafeBool + awsElb ELB + metadata EC2Metadata + frontendTagValue string + ingressClassTagValue string + region string + expectedNumber int + instanceID string + elbs map[string]LoadBalancerDetails + registeredFrontends util.SafeInt + initialised initialised + drainDelay time.Duration + readyForHealthCheck util.SafeBool } type initialised struct { @@ -100,7 +109,7 @@ func (e *elb) attachToFrontEnds() error { instance := id.InstanceID log.Infof("Attaching to ELBs from instance %s", instance) - clusterFrontEnds, err := FindFrontEndElbs(e.awsElb, e.labelValue) + clusterFrontEnds, err := FindFrontEndElbsWithIngressClassName(e.awsElb, e.frontendTagValue, e.ingressClassTagValue) if err != nil { return err @@ -141,8 +150,14 @@ func (e *elb) attachToFrontEnds() error { return nil } -// FindFrontEndElbs finds all elbs tagged with 'sky.uk/KubernetesClusterFrontend=' -func FindFrontEndElbs(awsElb ELB, labelValue string) (map[string]LoadBalancerDetails, error) { +// FindFrontEndElbs supports finding ELBs without ingress class for backwards compatibility +// with feed-dns, which does not support multiple ingress controllers +func FindFrontEndElbs(awsElb ELB, frontendTagValue string) (map[string]LoadBalancerDetails, error) { + return FindFrontEndElbsWithIngressClassName(awsElb, frontendTagValue, "") +} + +// FindFrontEndElbsWithIngressClassName finds all ELBs tagged with frontendTagValue and ingressClassValue +func FindFrontEndElbsWithIngressClassName(awsElb ELB, frontendTagValue string, ingressClassValue string) (map[string]LoadBalancerDetails, error) { maxTagQuery := 20 // Find the load balancers that are tagged with this cluster name request := &aws_elb.DescribeLoadBalancersInput{} @@ -176,7 +191,14 @@ func FindFrontEndElbs(awsElb ELB, labelValue string) (map[string]LoadBalancerDet } } - log.Debugf("Found %d loadbalancers. Checking for %s tag set to %s", len(lbNames), ElbTag, labelValue) + log.Debugf("Found %d loadbalancers.", len(lbNames)) + + requiredTags := map[string]string{ElbTag: frontendTagValue} + + if ingressClassValue != "" { + requiredTags[IngressClassTag] = ingressClassValue + } + clusterFrontEnds := make(map[string]LoadBalancerDetails) partitions := util.Partition(len(lbNames), maxTagQuery) for _, partition := range partitions { @@ -190,19 +212,31 @@ func FindFrontEndElbs(awsElb ELB, labelValue string) (map[string]LoadBalancerDet } // todo cb error out if we already have an internal or public facing elb - for _, description := range output.TagDescriptions { - for _, tag := range description.Tags { - if *tag.Key == ElbTag && *tag.Value == labelValue { - log.Infof("Found frontend elb %s", *description.LoadBalancerName) - lb := allLbs[*description.LoadBalancerName] - clusterFrontEnds[lb.Scheme] = lb - } + for _, elbDescription := range output.TagDescriptions { + if tagsDoMatch(elbDescription.Tags, requiredTags) { + log.Infof("Found frontend elb %s", *elbDescription.LoadBalancerName) + lb := allLbs[*elbDescription.LoadBalancerName] + clusterFrontEnds[lb.Scheme] = lb } } } return clusterFrontEnds, nil } +func tagsDoMatch(elbTags []*aws_elb.Tag, tagsToMatch map[string]string) bool { + matches := 0 + for name, value := range tagsToMatch { + log.Debugf("Checking for %s tag set to %s", name, value) + for _, elb := range elbTags { + if name == *elb.Key && value == *elb.Value { + matches++ + } + } + } + + return matches == len(tagsToMatch) +} + // Stop removes this instance from all the front end ELBs func (e *elb) Stop() error { var failed = false diff --git a/elb/elb_test.go b/elb/elb_test.go index fabbbc8a..3faab7bf 100644 --- a/elb/elb_test.go +++ b/elb/elb_test.go @@ -22,14 +22,21 @@ func init() { const ( clusterName = "cluster_name" + ingressName = "ingress_name" region = "eu-west-1" frontendTag = "sky.uk/KubernetesClusterFrontend" + ingressNameTag = "sky.uk/KubernetesClusterIngressClass" canonicalHostedZoneNameID = "test-id" elbDNSName = "elb-dnsname" elbInternalScheme = "internal" elbInternetFacingScheme = "internet-facing" ) +var defaultTags = []*aws_elb.Tag{ + {Key: aws.String(frontendTag), Value: aws.String(clusterName)}, + {Key: aws.String(ingressNameTag), Value: aws.String(ingressName)}, +} + type fakeElb struct { mock.Mock } @@ -129,7 +136,7 @@ func mockInstanceMetadata(mockMd *fakeMetadata, instanceID string) { } func setup() (controller.Updater, *fakeElb, *fakeMetadata) { - e, _ := New(region, clusterName, 1, 0) + e, _ := New(region, clusterName, ingressName, 1, 0) mockElb := &fakeElb{} mockMetadata := &fakeMetadata{} e.(*elb).awsElb = mockElb @@ -137,9 +144,17 @@ func setup() (controller.Updater, *fakeElb, *fakeMetadata) { return e, mockElb, mockMetadata } -func TestCanNotCreateUpdaterWithoutLabelValue(t *testing.T) { +func TestCanNotCreateUpdaterWithoutFrontEndTagValue(t *testing.T) { + //when + _, err := New(region, "", ingressName, 1, 0) + + //then + assert.Error(t, err) +} + +func TestCanNotCreateUpdaterWithoutIngressNameTagValue(t *testing.T) { //when - _, err := New(region, "", 1, 0) + _, err := New(region, clusterName, "", 1, 0) //then assert.Error(t, err) @@ -158,8 +173,11 @@ func TestAttachWithSingleMatchingLoadBalancers(t *testing.T) { lb{"other", elbInternalScheme}) mockClusterTags(mockElb, - lbTags{name: clusterFrontEnd, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, - lbTags{name: clusterFrontEndDifferentCluster, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String("different cluster")}}}, + lbTags{name: clusterFrontEnd, tags: defaultTags}, + lbTags{name: clusterFrontEndDifferentCluster, tags: []*aws_elb.Tag{ + {Key: aws.String(frontendTag), Value: aws.String("different cluster")}, + {Key: aws.String(ingressName), Value: aws.String("different cluster")}, + }}, lbTags{name: "other elb", tags: []*aws_elb.Tag{{Key: aws.String("Bannana"), Value: aws.String("Tasty")}}}, ) mockRegisterInstances(mockElb, clusterFrontEnd, instanceID) @@ -188,8 +206,11 @@ func TestReportsErrorIfExpectedNotMatched(t *testing.T) { lb{name: clusterFrontEndDifferentCluster, scheme: elbInternalScheme}, lb{name: "other", scheme: elbInternalScheme}) mockClusterTags(mockElb, - lbTags{name: clusterFrontEnd, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, - lbTags{name: clusterFrontEndDifferentCluster, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String("different cluster")}}}, + lbTags{name: clusterFrontEnd, tags: defaultTags}, + lbTags{name: clusterFrontEndDifferentCluster, tags: []*aws_elb.Tag{ + {Key: aws.String(frontendTag), Value: aws.String("different cluster")}, + {Key: aws.String(ingressNameTag), Value: aws.String("different cluster")}, + }}, lbTags{name: "other elb", tags: []*aws_elb.Tag{{Key: aws.String("Bannana"), Value: aws.String("Tasty")}}}, ) mockRegisterInstances(mockElb, clusterFrontEnd, instanceID) @@ -208,7 +229,28 @@ func TestNameAndDNSNameAndHostedZoneIDLoadBalancerDetailsAreExtracted(t *testing clusterFrontEnd := "cluster-frontend" mockLoadBalancers(mockElb, lb{name: clusterFrontEnd, scheme: elbInternalScheme}) mockClusterTags(mockElb, - lbTags{name: clusterFrontEnd, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, + lbTags{name: clusterFrontEnd, tags: defaultTags}, + ) + + //when + frontends, _ := FindFrontEndElbsWithIngressClassName(mockElb, clusterName, ingressName) + + //then + assert.Equal(t, "cluster-frontend", frontends[elbInternalScheme].Name) + assert.Equal(t, elbDNSName, frontends[elbInternalScheme].DNSName) + assert.Equal(t, canonicalHostedZoneNameID, frontends[elbInternalScheme].HostedZoneID) + assert.Equal(t, elbInternalScheme, frontends[elbInternalScheme].Scheme) +} + +func TestFindElbWithoutIngressName(t *testing.T) { + //given + mockElb := &fakeElb{} + clusterFrontEnd := "cluster-frontend" + mockLoadBalancers(mockElb, lb{name: clusterFrontEnd, scheme: elbInternalScheme}) + mockClusterTags(mockElb, + lbTags{name: clusterFrontEnd, tags: []*aws_elb.Tag{ + {Key: aws.String(frontendTag), Value: aws.String(clusterName)}, + }}, ) //when @@ -233,8 +275,8 @@ func TestAttachWithInternalAndInternetFacing(t *testing.T) { lb{name: privateFrontend, scheme: elbInternalScheme}, lb{name: publicFrontend, scheme: elbInternetFacingScheme}) mockClusterTags(mockElb, - lbTags{name: privateFrontend, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, - lbTags{name: publicFrontend, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, + lbTags{name: privateFrontend, tags: defaultTags}, + lbTags{name: publicFrontend, tags: defaultTags}, ) mockRegisterInstances(mockElb, privateFrontend, instanceID) mockRegisterInstances(mockElb, publicFrontend, instanceID) @@ -301,6 +343,46 @@ func TestNoMatchingElbs(t *testing.T) { assert.Error(t, err, "expected ELBs: 1 actual: 0") } +func TestAttachingWithoutIngressNameTagElbs(t *testing.T) { + // given + e, mockElb, mockMetadata := setup() + instanceID := "cow" + loadBalancerName := "i am not the loadbalancer you are looking for" + mockInstanceMetadata(mockMetadata, instanceID) + mockLoadBalancers(mockElb, lb{name: loadBalancerName, scheme: elbInternalScheme}) + // No cluster tags + mockClusterTags(mockElb, lbTags{name: loadBalancerName, tags: []*aws_elb.Tag{ + {Key: aws.String(frontendTag), Value: aws.String(clusterName)}, + }}) + + // when + e.Start() + err := e.Update(controller.IngressEntries{}) + + // then + assert.Error(t, err, "expected ELBs: 1 actual: 0") +} + +func TestAttachingWithoutFrontendTagElbs(t *testing.T) { + // given + e, mockElb, mockMetadata := setup() + instanceID := "cow" + loadBalancerName := "i am not the loadbalancer you are looking for" + mockInstanceMetadata(mockMetadata, instanceID) + mockLoadBalancers(mockElb, lb{name: loadBalancerName, scheme: elbInternalScheme}) + // No cluster tags + mockClusterTags(mockElb, lbTags{name: loadBalancerName, tags: []*aws_elb.Tag{ + {Key: aws.String(ingressNameTag), Value: aws.String(ingressName)}, + }}) + + // when + e.Start() + err := e.Update(controller.IngressEntries{}) + + // then + assert.Error(t, err, "expected ELBs: 1 actual: 0") +} + func TestGetLoadBalancerPages(t *testing.T) { // given e, mockElb, mockMetadata := setup() @@ -315,7 +397,7 @@ func TestGetLoadBalancerPages(t *testing.T) { }}, }, nil) mockInstanceMetadata(mockMetadata, instanceID) - mockClusterTags(mockElb, lbTags{name: loadBalancerName, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}) + mockClusterTags(mockElb, lbTags{name: loadBalancerName, tags: defaultTags}) mockRegisterInstances(mockElb, loadBalancerName, instanceID) // when @@ -338,8 +420,8 @@ func TestTagCallsPage(t *testing.T) { lb{name: loadBalancerName1, scheme: elbInternalScheme}, lb{name: loadBalancerName2, scheme: elbInternetFacingScheme}) mockClusterTags(mockElb, - lbTags{name: loadBalancerName1, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, - lbTags{name: loadBalancerName2, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}) + lbTags{name: loadBalancerName1, tags: defaultTags}, + lbTags{name: loadBalancerName2, tags: defaultTags}) mockRegisterInstances(mockElb, loadBalancerName1, instanceID) mockRegisterInstances(mockElb, loadBalancerName2, instanceID) @@ -366,8 +448,8 @@ func TestDeregistersWithAttachedELBs(t *testing.T) { lb{name: clusterFrontEnd2, scheme: elbInternetFacingScheme}, lb{name: "other", scheme: elbInternalScheme}) mockClusterTags(mockElb, - lbTags{name: clusterFrontEnd, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, - lbTags{name: clusterFrontEnd2, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, + lbTags{name: clusterFrontEnd, tags: defaultTags}, + lbTags{name: clusterFrontEnd2, tags: defaultTags}, lbTags{name: "other elb", tags: []*aws_elb.Tag{{Key: aws.String("Bannana"), Value: aws.String("Tasty")}}}, ) mockRegisterInstances(mockElb, clusterFrontEnd, instanceID) @@ -407,7 +489,7 @@ func TestRegisterInstanceError(t *testing.T) { clusterFrontEnd := "cluster-frontend" mockLoadBalancers(mockElb, lb{name: clusterFrontEnd, scheme: elbInternalScheme}) mockClusterTags(mockElb, - lbTags{name: clusterFrontEnd, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, + lbTags{name: clusterFrontEnd, tags: defaultTags}, ) mockElb.On("RegisterInstancesWithLoadBalancer", mock.Anything).Return(&aws_elb.RegisterInstancesWithLoadBalancerOutput{}, errors.New("no register for you")) @@ -427,7 +509,7 @@ func TestDeRegisterInstanceError(t *testing.T) { mockLoadBalancers(mockElb, lb{name: clusterFrontEnd, scheme: elbInternalScheme}) mockClusterTags(mockElb, - lbTags{name: clusterFrontEnd, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}, + lbTags{name: clusterFrontEnd, tags: defaultTags}, ) mockRegisterInstances(mockElb, clusterFrontEnd, instanceID) mockElb.On("DeregisterInstancesFromLoadBalancer", mock.Anything).Return(&aws_elb.DeregisterInstancesFromLoadBalancerOutput{}, errors.New("no deregister for you")) @@ -452,7 +534,7 @@ func TestRetriesUpdateIfFirstAttemptFails(t *testing.T) { mockClusterTags(mockElb, lbTags{ name: clusterFrontEnd, - tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}) + tags: defaultTags}) mockElb.On("RegisterInstancesWithLoadBalancer", mock.Anything).Return( &aws_elb.RegisterInstancesWithLoadBalancerOutput{}, errors.New("no register for you")) @@ -490,7 +572,7 @@ func TestHealthReportsUnhealthyAfterUnsuccessfulFirstUpdate(t *testing.T) { mockLoadBalancers(mockElb, lb{name: clusterFrontEnd, scheme: elbInternalScheme}) mockClusterTags(mockElb, - lbTags{name: clusterFrontEnd, tags: []*aws_elb.Tag{{Key: aws.String(frontendTag), Value: aws.String(clusterName)}}}) + lbTags{name: clusterFrontEnd, tags: defaultTags}) mockRegisterInstances(mockElb, clusterFrontEnd, instanceID) // when diff --git a/elb/status/status.go b/elb/status/status.go index 9f111a38..df158a97 100644 --- a/elb/status/status.go +++ b/elb/status/status.go @@ -18,9 +18,10 @@ import ( // Config for creating a new ELB status updater. type Config struct { - Region string - LabelValue string - KubernetesClient k8s.Client + Region string + FrontendTagValue string + IngressNameTagValue string + KubernetesClient k8s.Client } // New creates a new ELB frontend status updater. @@ -31,23 +32,25 @@ func New(conf Config) (controller.Updater, error) { } return &status{ - awsElb: aws_elb.New(session), - labelValue: conf.LabelValue, - loadBalancers: make(map[string]v1.LoadBalancerStatus), - kubernetesClient: conf.KubernetesClient, + awsElb: aws_elb.New(session), + frontendTagValue: conf.FrontendTagValue, + ingressNameTagValue: conf.IngressNameTagValue, + loadBalancers: make(map[string]v1.LoadBalancerStatus), + kubernetesClient: conf.KubernetesClient, }, nil } type status struct { - awsElb elb.ELB - labelValue string - loadBalancers map[string]v1.LoadBalancerStatus - kubernetesClient k8s.Client + awsElb elb.ELB + frontendTagValue string + ingressNameTagValue string + loadBalancers map[string]v1.LoadBalancerStatus + kubernetesClient k8s.Client } // Start discovers the elbs and generates loadBalancer statuses. func (s *status) Start() error { - clusterFrontEnds, err := elb.FindFrontEndElbs(s.awsElb, s.labelValue) + clusterFrontEnds, err := elb.FindFrontEndElbsWithIngressClassName(s.awsElb, s.frontendTagValue, s.ingressNameTagValue) if err != nil { return err } diff --git a/examples/feed-dns-deployment-aws.yml b/examples/feed-dns-deployment-aws.yml index a526bdb0..1f519e57 100644 --- a/examples/feed-dns-deployment-aws.yml +++ b/examples/feed-dns-deployment-aws.yml @@ -24,7 +24,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - - image: skycirrus/feed-dns:v1.3.0 + - image: skycirrus/feed-dns:v2.0.0 name: feed-dns resources: diff --git a/examples/feed-dns-deployment-static-hostnames.yml b/examples/feed-dns-deployment-static-hostnames.yml index f7ac1abd..33dd5b5a 100644 --- a/examples/feed-dns-deployment-static-hostnames.yml +++ b/examples/feed-dns-deployment-static-hostnames.yml @@ -23,7 +23,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - - image: skycirrus/feed-dns:v1.3.0 + - image: skycirrus/feed-dns:v2.0.0 name: feed-dns resources: diff --git a/examples/feed-ingress-deployment-gorb.yml b/examples/feed-ingress-deployment-gorb.yml index 4b7838d9..f51a988a 100644 --- a/examples/feed-ingress-deployment-gorb.yml +++ b/examples/feed-ingress-deployment-gorb.yml @@ -30,7 +30,7 @@ spec: restartPolicy: Always containers: - - image: skycirrus/feed-ingress:v1.2.0 + - image: skycirrus/feed-ingress:v2.0.0 name: feed-ingress securityContext: capabilities: @@ -65,96 +65,95 @@ spec: protocol: TCP args: + - gorb + # Ingress nginx port that ELBs will direct traffic towards. - - -ingress-port=80 + - --ingress-port=80 # Health port on nginx, used by ELBs to determine health. - - -ingress-health-port=8081 + - --ingress-health-port=8081 # Default security whitelist for ingress. Can be overridden with the sky.uk/allow annotation. - - -ingress-allow=10.0.0.0/8 + - --ingress-allow=10.0.0.0/8 # Health port for the controller to respond on. - - -health-port=12082 + - --health-port=12082 # Log level of nginx. Recommended to leave at error, or set to crit if too much spam. - - -nginx-loglevel=error + - --nginx-loglevel=error # How often to reload nginx if needed. Setting too low can cause 504s from the ELB in the case of heavy # ingress updates. - - -nginx-update-period=5m + - --nginx-update-period=5m # Use pushgateway for prometheus metrics. Optional - metrics available at /metrics on health port. - - -pushgateway=mypushgateway.com - - -pushgateway-interval=20 - - -pushgateway-label=k8s_cluster=dev - - -pushgateway-label=environment=dev - - -pushgateway-label=version=v1.0.2 - - # Specify gorb as the registration component - - -registration-frontend-type=gorb + - --pushgateway=mypushgateway.com + - --pushgateway-interval=20 + - --pushgateway-label=k8s_cluster=dev + - --pushgateway-label=environment=dev + - --pushgateway-label=version=v1.0.2 # VIP of the IPVS managed via gor - - -gorb-vip-loadbalancer=10.10.0.10 + - --gorb-vip-loadbalancer=10.10.0.10 # Define gorb base url - - -gorb-endpoint=http://10.154.0.11:4672 + - --gorb-endpoint=http://10.154.0.11:4672 # Comma separated list of virtual service definition to create via gorb - - -gorb-services-definition=http-proxy:80 + - --gorb-services-definition=http-proxy:80 # Enable loopback management to make Direct Return effective - - -gorb-management-loopback=true + - --gorb-management-loopback=true # Path to the proc file system containing the main interface definition - - -gorb-interface-proc-fs-path=/host-ipv4-proc/ + - --gorb-interface-proc-fs-path=/host-ipv4-proc/ # packet-forwarding method to set via gorb - - -gorb-backend-method=dr + - --gorb-backend-method=dr # gorb backend weight - - -gorb-backend-weight=1000 + - --gorb-backend-weight=1000 # gorb backend healthcheck interval - - -gorb-backend-healthcheck-interval=1s + - --gorb-backend-healthcheck-interval=1s # gorb backend healthcheck type - - -gorb-backend-healthcheck-type=http + - --gorb-backend-healthcheck-type=http # nginx host ip address to use when the ingress is not in AWS - - -gorb-ingress-instance-ip=$(INSTANCEIP) + - --gorb-ingress-instance-ip=$(INSTANCEIP) # LB drain time - time to wait while the load balancer drains requests from feed when stopping. Should be # at least as long as the ELBs drain timeout. - - -drain-delay=30s + - --drain-delay=30s # Each worker uses a full cpu, so scale up vertically on a box by increasing this value. - - -nginx-workers=1 + - --nginx-workers=1 # Connections*workers needs to be less than available ephemeral ports. Linux default is 60999-32768=28231. - - -nginx-worker-connections=20000 + - --nginx-worker-connections=20000 # Needs to be greater than any frontend idle timeout. - - -nginx-keepalive-seconds=304 + - --nginx-keepalive-seconds=304 # CIDRs of the ELBs to trust X-Forwarded-For, for determining client IP allow/deny. - - -nginx-trusted-frontends=10.0.0.0/8 + - --nginx-trusted-frontends=10.0.0.0/8 # Max number of idle connections to a backend. - - -nginx-backend-keepalive-count=1024 + - --nginx-backend-keepalive-count=1024 # Default max time for a request to a backend. Can be overridden by an annotation on the ingress resource. - - -nginx-default-backend-timeout-seconds=5 + - --nginx-default-backend-timeout-seconds=5 # Needs to be greater than 64 to support very large domain names. - - -nginx-server-names-hash-bucket-size=128 + - --nginx-server-names-hash-bucket-size=128 # Access logs turned on - add or remove the "-access-log" flag to turn them on/off. - - -access-log - - -access-log-dir=/var/log/nginx + - --access-log + - --access-log-dir=/var/log/nginx # Add custom headers to the access logs. - - -nginx-log-headers=X-Amzn-Trace-Id + - --nginx-log-headers=X-Amzn-Trace-Id # Controller health determines readiness. This has no effect on ingress traffic from ELBs. readinessProbe: diff --git a/examples/feed-ingress-deployment-merlin.yml b/examples/feed-ingress-deployment-merlin.yml index 13f6bc1d..b8c88835 100644 --- a/examples/feed-ingress-deployment-merlin.yml +++ b/examples/feed-ingress-deployment-merlin.yml @@ -30,7 +30,7 @@ spec: restartPolicy: Always containers: - - image: skycirrus/feed-ingress:v1.7.0 + - image: skycirrus/feed-ingress:v2.0.0 name: feed-ingress securityContext: capabilities: @@ -65,88 +65,87 @@ spec: protocol: TCP args: + - merlin + # Ingress nginx port that frontend will direct traffic towards. - - -ingress-port=80 - - -ingress-https-port=443 + - --ingress-port=80 + - --ingress-https-port=443 # Health port on nginx, used by frontend to determine health. - - -ingress-health-port=8081 + - --ingress-health-port=8081 # Default security whitelist for ingress. Can be overridden with the sky.uk/allow annotation. - - -ingress-allow=10.0.0.0/8 + - --ingress-allow=10.0.0.0/8 # Health port for the controller to respond on. - - -health-port=12082 + - --health-port=12082 # Log level of nginx. Recommended to leave at error, or set to crit if too much spam. - - -nginx-loglevel=error + - --nginx-loglevel=error # How often to reload nginx if needed. Setting too low can cause 504s from the frontend in the case of heavy # ingress updates. - - -nginx-update-period=5m + - --nginx-update-period=5m # Use pushgateway for prometheus metrics. Optional - metrics available at /metrics on health port. - - -pushgateway=mypushgateway.com - - -pushgateway-interval=20 - - -pushgateway-label=k8s_cluster=dev - - -pushgateway-label=environment=dev - - -pushgateway-label=version=v1.0.2 - - # Specify merlin as the frontend component - - -registration-frontend-type=merlin + - --pushgateway=mypushgateway.com + - --pushgateway-interval=20 + - --pushgateway-label=k8s_cluster=dev + - --pushgateway-label=environment=dev + - --pushgateway-label=version=v1.0.2 # gRPC endpoint - - -merlin-endpoint=dns:///merlin-servers + - --merlin-endpoint=dns:///merlin-servers # Virtual Service IDs to attach to - - -merlin-service-id=my-virtual-service - - -merlin-https-service-id=my-https-virtual-service + - --merlin-service-id=my-virtual-service + - --merlin-https-service-id=my-https-virtual-service # Real server IP to associate with virtual service - the IP of the ingress node. - - -merlin-instance-ip=$(INSTANCEIP) + - --merlin-instance-ip=$(INSTANCEIP) # Forward method that IPVS should use - pick route, masq, or tunnel. - - -merlin-forward-method=route + - --merlin-forward-method=route # Drain delay to bleed off connections when detaching. - - -merlin-drain-delay=60s + - --merlin-drain-delay=60s # Virtual IP to bind to the local interface for tunneling and direct routing. Usually the virtual service IP. - - -merlin-vip=10.10.10.1 + - --merlin-vip=10.10.10.1 # Interface to bind virtual IP to. - - -merlin-vip-interface=lo + - --merlin-vip-interface=lo # Internet facing Virtual IP. - - -merlin-internet-facing-vip=1.0.0.0 + - --merlin-internet-facing-vip=1.0.0.0 # Each worker uses a full cpu, so scale up vertically on a box by increasing this value. - - -nginx-workers=1 + - --nginx-workers=1 # Connections*workers needs to be less than available ephemeral ports. Linux default is 60999-32768=28231. - - -nginx-worker-connections=20000 + - --nginx-worker-connections=20000 # Needs to be greater than any frontend idle timeout. - - -nginx-keepalive-seconds=304 + - --nginx-keepalive-seconds=304 # CIDRs of the frontend to trust X-Forwarded-For, for determining client IP allow/deny. - - -nginx-trusted-frontends=10.0.0.0/8 + - --nginx-trusted-frontends=10.0.0.0/8 # Max number of idle connections to a backend. - - -nginx-backend-keepalive-count=1024 + - --nginx-backend-keepalive-count=1024 # Default max time for a request to a backend. Can be overridden by an annotation on the ingress resource. - - -nginx-default-backend-timeout-seconds=5 + - --nginx-default-backend-timeout-seconds=5 # Needs to be greater than 64 to support very large domain names. - - -nginx-server-names-hash-bucket-size=128 + - --nginx-server-names-hash-bucket-size=128 # Access logs turned on - add or remove the "-access-log" flag to turn them on/off. - - -access-log - - -access-log-dir=/var/log/nginx + - --access-log + - --access-log-dir=/var/log/nginx # Add custom headers to the access logs. - - -nginx-log-headers=X-Amzn-Trace-Id + - --nginx-log-headers=X-Amzn-Trace-Id # Controller health determines readiness. This has no effect on ingress traffic from frontend. readinessProbe: diff --git a/examples/feed-ingress-deployment-ssl.yml b/examples/feed-ingress-deployment-ssl.yml index 1b1b4872..58fea691 100644 --- a/examples/feed-ingress-deployment-ssl.yml +++ b/examples/feed-ingress-deployment-ssl.yml @@ -30,7 +30,7 @@ spec: restartPolicy: Always containers: - - image: skycirrus/feed-ingress:v1.3.0 + - image: skycirrus/feed-ingress:v2.0.0 name: feed-ingress resources: @@ -55,73 +55,84 @@ spec: protocol: TCP args: + - elb + # Ingress nginx port that ELBs will direct traffic towards. - - -ingress-port=8080 + - --ingress-port=8080 # Ingress nginx port for ssl traffic - - -ingress-https-port=8443 + - --ingress-https-port=8443 # Health port on nginx, used by ELBs to determine health. - - -ingress-health-port=8081 + - --ingress-health-port=8081 # Default security whitelist for ingress. Can be overridden with the sky.uk/allow annotation. - - -ingress-allow=10.0.0.0/8 + - --ingress-allow=10.0.0.0/8 # Set default ssl path + name file without extension - expected default-ssl.crt and default-ssl.key into /etc/ssl/default-ssl/ - - -ssl-path=/etc/ssl/default-ssl/default-ssl + - --ssl-path=/etc/ssl/default-ssl/default-ssl # Health port for the controller to respond on. - - -health-port=12082 + - --health-port=12082 # Log level of nginx. Recommended to leave at error, or set to crit if too much spam. - - -nginx-loglevel=error + - --nginx-loglevel=error # How often to reload nginx if needed. Setting too low can cause 504s from the ELB in the case of heavy # ingress updates. - - -nginx-update-period=5m + - --nginx-update-period=5m # Use pushgateway for prometheus metrics. Optional - metrics available at /metrics on health port. - - -pushgateway=mypushgateway.com - - -pushgateway-interval=20 - - -pushgateway-label=k8s_cluster=dev - - -pushgateway-label=environment=dev - - -pushgateway-label=version=v1.0.2 + - --pushgateway=mypushgateway.com + - --pushgateway-interval=20 + - --pushgateway-label=k8s_cluster=dev + - --pushgateway-label=environment=dev + - --pushgateway-label=version=v1.0.2 + + # Set status to unhealthy if fewer than this number of matching ELBs are found + - --elb-expected-number=2 # Attach to the ELBs with label sky.uk/KubernetesClusterFrontend set to this value. - - -elb-expected-number=2 - - -elb-label-value=dev + - --elb-frontend-tag-value=dev + + # Attach to the ELBs with label sky.uk/KubernetesClusterIngressClass set to this value + # and adopt ingress resources with a matching kubernetes.io/ingress.class value + - --ingress-class=main + + # Only consider ingresses in namespaces with this label. Optional. + - --ingress-controller-namespace-selector=app=myapp # ELB drain time - time to wait while ELB drains requests from feed when stopping. Should be # at least as long as the ELBs drain timeout. - - -drain-delay=30s + - --drain-delay=30s # Each worker uses a full cpu, so scale up vertically on a box by increasing this value. - - -nginx-workers=1 + - --nginx-workers=1 # Connections*workers needs to be less than available ephemeral ports. Linux default is 60999-32768=28231. - - -nginx-worker-connections=20000 + - --nginx-worker-connections=20000 # Needs to be greater than any frontend idle timeout. - - -nginx-keepalive-seconds=304 + - --nginx-keepalive-seconds=304 # CIDRs of the ELBs to trust X-Forwarded-For, for determining client IP allow/deny. - - -nginx-trusted-frontends=10.0.0.0/8 + - --nginx-trusted-frontends=10.0.0.0/8 # Max number of idle connections to a backend. - - -nginx-backend-keepalive-count=1024 + - --nginx-backend-keepalive-count=1024 # Default max time for a request to a backend. Can be overridden by an annotation on the ingress resource. - - -nginx-default-backend-timeout-seconds=5 + - --nginx-default-backend-timeout-seconds=5 # Needs to be greater than 64 to support very large domain names. - - -nginx-server-names-hash-bucket-size=128 + - --nginx-server-names-hash-bucket-size=128 # Access logs turned on - add or remove the "-access-log" flag to turn them on/off. - - -access-log - - -access-log-dir=/var/log/nginx + - --access-log + - --access-log-dir=/var/log/nginx # Add custom headers to the access logs. - - -nginx-log-headers=X-Amzn-Trace-Id + - --nginx-log-headers=X-Amzn-Trace-Id # Controller health determines readiness. This has no effect on ingress traffic from ELBs. readinessProbe: diff --git a/examples/feed-ingress-deployment.yml b/examples/feed-ingress-deployment.yml index d0fcbbca..29752ceb 100644 --- a/examples/feed-ingress-deployment.yml +++ b/examples/feed-ingress-deployment.yml @@ -30,7 +30,7 @@ spec: restartPolicy: Always containers: - - image: skycirrus/feed-ingress:v1.3.0 + - image: skycirrus/feed-ingress:v2.0.0 name: feed-ingress resources: @@ -55,67 +55,78 @@ spec: protocol: TCP args: + - elb + # Ingress nginx port that ELBs will direct traffic towards. - - -ingress-port=8080 + - --ingress-port=8080 # Health port on nginx, used by ELBs to determine health. - - -ingress-health-port=8081 + - --ingress-health-port=8081 # Default security whitelist for ingress. Can be overridden with the sky.uk/allow annotation. - - -ingress-allow=10.0.0.0/8 + - --ingress-allow=10.0.0.0/8 # Health port for the controller to respond on. - - -health-port=12082 + - --health-port=12082 # Log level of nginx. Recommended to leave at error, or set to crit if too much spam. - - -nginx-loglevel=error + - --nginx-loglevel=error # How often to reload nginx if needed. Setting too low can cause 504s from the ELB in the case of heavy # ingress updates. - - -nginx-update-period=5m + - --nginx-update-period=5m # Use pushgateway for prometheus metrics. Optional - metrics available at /metrics on health port. - - -pushgateway=mypushgateway.com - - -pushgateway-interval=20 - - -pushgateway-label=k8s_cluster=dev - - -pushgateway-label=environment=dev - - -pushgateway-label=version=v1.0.2 + - --pushgateway=mypushgateway.com + - --pushgateway-interval=20 + - --pushgateway-label=k8s_cluster=dev + - --pushgateway-label=environment=dev + - --pushgateway-label=version=v1.0.2 + + # Set status to unhealthy if fewer than this number of matching ELBs are found + - --elb-expected-number=2 # Attach to the ELBs with label sky.uk/KubernetesClusterFrontend set to this value. - - -elb-expected-number=2 - - -elb-label-value=dev + - --elb-frontend-tag-value=dev + + # Attach to the ELBs with label sky.uk/KubernetesClusterIngressClass set to this value + # and adopt ingress resources with a matching kubernetes.io/ingress.class value + - --ingress-class=main + + # Only consider ingresses in namespaces with this label. Optional. + - --ingress-controller-namespace-selector=app=myapp # ELB drain time - time to wait while ELB drains requests from feed when stopping. Should be # at least as long as the ELBs drain timeout. - - -drain-delay=30s + - --drain-delay=30s # Each worker uses a full cpu, so scale up vertically on a box by increasing this value. - - -nginx-workers=1 + - --nginx-workers=1 # Connections*workers needs to be less than available ephemeral ports. Linux default is 60999-32768=28231. - - -nginx-worker-connections=20000 + - --nginx-worker-connections=20000 # Needs to be greater than any frontend idle timeout. - - -nginx-keepalive-seconds=304 + - --nginx-keepalive-seconds=304 # CIDRs of the ELBs to trust X-Forwarded-For, for determining client IP allow/deny. - - -nginx-trusted-frontends=10.0.0.0/8 + - --nginx-trusted-frontends=10.0.0.0/8 # Max number of idle connections to a backend. - - -nginx-backend-keepalive-count=1024 + - --nginx-backend-keepalive-count=1024 # Default max time for a request to a backend. Can be overridden by an annotation on the ingress resource. - - -nginx-default-backend-timeout-seconds=5 + - --nginx-default-backend-timeout-seconds=5 # Needs to be greater than 64 to support very large domain names. - - -nginx-server-names-hash-bucket-size=128 + - --nginx-server-names-hash-bucket-size=128 # Access logs turned on - add or remove the "-access-log" flag to turn them on/off. - - -access-log - - -access-log-dir=/var/log/nginx + - --access-log + - --access-log-dir=/var/log/nginx # Add custom headers to the access logs. - - -nginx-log-headers=X-Amzn-Trace-Id + - --nginx-log-headers=X-Amzn-Trace-Id # Controller health determines readiness. This has no effect on ingress traffic from ELBs. readinessProbe: diff --git a/examples/ingress.yml b/examples/ingress.yml index 8e7f79f7..fc589860 100644 --- a/examples/ingress.yml +++ b/examples/ingress.yml @@ -8,6 +8,9 @@ metadata: # Set to internal or internet-facing, so feed-dns will point to the correct endpoint. sky.uk/frontend-scheme: internal + # Specify the feed-ingress controller that should adopt this ingress + kubernetes.io/ingress.class: main + # nginx allow clause for this ingress. sky.uk/allow: 10.10.82.0/24 diff --git a/cmd/feed-dns/main.go b/feed-dns/main.go similarity index 100% rename from cmd/feed-dns/main.go rename to feed-dns/main.go diff --git a/feed-ingress/cmd/alb.go b/feed-ingress/cmd/alb.go new file mode 100644 index 00000000..f34d5a94 --- /dev/null +++ b/feed-ingress/cmd/alb.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "github.com/sky-uk/feed/alb" + "github.com/sky-uk/feed/controller" + "github.com/sky-uk/feed/k8s" + + "github.com/spf13/cobra" +) + +var albCmd = &cobra.Command{ + Use: "alb", + Short: "Attach to AWS Application Load Balancers", + Long: `Unfortunately, ALBs have a bug that prevents non-disruptive deployments of feed. +Specifically, they don't respect the deregistration delay. As a result, we don't recommend using ALBs at this time.`, + Run: func(cmd *cobra.Command, args []string) { + runCmd(appendAlbIngressUpdaters) + }, +} + +func init() { + rootCmd.AddCommand(albCmd) + + albCmd.Flags().StringVar(®ion, "region", defaultRegion, + "AWS region for frontend attachment.") + albCmd.Flags().StringSliceVar(&targetGroupNames, "alb-target-group-names", []string{}, + "Names of ALB target groups to attach to, separated by commas.") + albCmd.Flags().DurationVar(&targetGroupDeregistrationDelay, "alb-target-group-deregistration-delay", + defaultTargetGroupDeregistrationDelay, + "Delay to wait for feed-ingress to deregister from the ALB target group on shutdown. Should match"+ + " the target group setting in AWS.") +} + +func appendAlbIngressUpdaters(kubernetesClient k8s.Client, updaters []controller.Updater) ([]controller.Updater, error) { + albUpdater, err := alb.New(region, targetGroupNames, targetGroupDeregistrationDelay) + if err != nil { + return nil, err + } + return append(updaters, albUpdater), nil +} diff --git a/feed-ingress/cmd/common.go b/feed-ingress/cmd/common.go new file mode 100644 index 00000000..588211df --- /dev/null +++ b/feed-ingress/cmd/common.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "strings" + + "github.com/sky-uk/feed/nginx" + + log "github.com/sirupsen/logrus" + "github.com/sky-uk/feed/controller" + "github.com/sky-uk/feed/k8s" + "github.com/sky-uk/feed/util/metrics" + + cmdutil "github.com/sky-uk/feed/util/cmd" +) + +type appendIngressUpdaters = func(kubernetesClient k8s.Client, updaters []controller.Updater) ([]controller.Updater, error) + +func runCmd(appender appendIngressUpdaters) { + if ingressClassName == defaultIngressClassName { + log.Fatalf("The argument --%s is required", ingressClassFlag) + } + controllerConfig.Name = ingressClassName + + cmdutil.ConfigureLogging(debug) + cmdutil.ConfigureMetrics("feed-ingress", pushgatewayLabels, pushgatewayURL, pushgatewayIntervalSeconds) + + client, err := k8s.New(kubeconfig, resyncPeriod) + if err != nil { + log.Fatal("Unable to create k8s client: ", err) + } + controllerConfig.KubernetesClient = client + + controllerConfig.Updaters, err = createIngressUpdaters(client, appender) + if err != nil { + log.Fatal("Unable to create ingress updaters: ", err) + } + + controllerConfig.NamespaceSelector, err = parseNamespaceSelector(namespaceSelector) + if err != nil { + log.Fatalf("invalid format for --%s (%s)", ingressControllerNamespaceSelectorFlag, namespaceSelector) + } + + feedController := controller.New(controllerConfig) + + cmdutil.AddHealthMetrics(feedController, metrics.PrometheusIngressSubsystem) + cmdutil.AddHealthPort(feedController, healthPort) + cmdutil.AddSignalHandler(feedController) + + if err = feedController.Start(); err != nil { + log.Fatal("Error while starting controller: ", err) + } + log.Info("Controller started") + + select {} +} + +func createIngressUpdaters(kubernetesClient k8s.Client, appender appendIngressUpdaters) ([]controller.Updater, error) { + nginxConfig.Ports = []nginx.Port{{Name: "http", Port: ingressPort}} + if ingressHTTPSPort != unset { + nginxConfig.Ports = append(nginxConfig.Ports, nginx.Port{Name: "https", Port: ingressHTTPSPort}) + } + + nginxConfig.HealthPort = ingressHealthPort + nginxConfig.SSLPath = nginxSSLPath + nginxConfig.TrustedFrontends = nginxTrustedFrontends + nginxConfig.LogHeaders = nginxLogHeaders + nginxConfig.VhostStatsSharedMemory = nginxVhostStatsSharedMemory + nginxConfig.OpenTracingPlugin = nginxOpenTracingPluginPath + nginxConfig.OpenTracingConfig = nginxOpenTracingConfigPath + nginxUpdater := nginx.New(nginxConfig) + + updaters := []controller.Updater{nginxUpdater} + updaters, err := appender(kubernetesClient, updaters) + if err != nil { + return nil, err + } + return updaters, nil +} + +func parseNamespaceSelector(nameValueStr string) (*k8s.NamespaceSelector, error) { + if len(nameValueStr) == 0 { + return nil, nil + } + + nameValue := strings.SplitN(nameValueStr, "=", 2) + if len(nameValue) != 2 { + log.Errorf("expecting name=value but was (%s)", nameValueStr) + } + return &k8s.NamespaceSelector{LabelName: nameValue[0], LabelValue: nameValue[1]}, nil +} diff --git a/feed-ingress/cmd/elb.go b/feed-ingress/cmd/elb.go new file mode 100644 index 00000000..73cd7d8d --- /dev/null +++ b/feed-ingress/cmd/elb.go @@ -0,0 +1,71 @@ +package cmd + +import ( + "time" + + "github.com/sky-uk/feed/controller" + "github.com/sky-uk/feed/k8s" + + "github.com/sky-uk/feed/elb" + elbstatus "github.com/sky-uk/feed/elb/status" + "github.com/spf13/cobra" +) + +var ( + region string + elbFrontendTagValue string + elbExpectedNumber int + drainDelay time.Duration + targetGroupNames []string + targetGroupDeregistrationDelay time.Duration +) + +const ( + defaultElbFrontendTagValue = "" + defaultDrainDelay = time.Second * 60 + defaultTargetGroupDeregistrationDelay = time.Second * 300 + defaultRegion = "eu-west-1" + defaultElbExpectedNumber = 0 +) + +var elbCmd = &cobra.Command{ + Use: "elb", + Short: "Attach to AWS Elastic Load Balancers", + Run: func(cmd *cobra.Command, args []string) { + runCmd(appendElbIngressUpdaters) + }, +} + +func init() { + rootCmd.AddCommand(elbCmd) + + elbCmd.Flags().StringVar(®ion, "region", defaultRegion, + "AWS region for frontend attachment.") + elbCmd.Flags().StringVar(&elbFrontendTagValue, "elb-frontend-tag-value", defaultElbFrontendTagValue, + "Attach to ELBs tagged with "+elb.ElbTag+"=value. Leave empty to not attach.") + elbCmd.Flags().IntVar(&elbExpectedNumber, "elb-expected-number", defaultElbExpectedNumber, + "Expected number of ELBs to attach to. If 0 the controller will not check,"+ + " otherwise it fails to start if it can't attach to this number.") + elbCmd.Flags().DurationVar(&drainDelay, "drain-delay", defaultDrainDelay, "Delay to wait"+ + " for feed-ingress to drain from the registration component on shutdown. Should match the ELB's drain time.") +} + +func appendElbIngressUpdaters(kubernetesClient k8s.Client, updaters []controller.Updater) ([]controller.Updater, error) { + elbUpdater, err := elb.New(region, elbFrontendTagValue, ingressClassName, elbExpectedNumber, drainDelay) + if err != nil { + return nil, err + } + updaters = append(updaters, elbUpdater) + + statusConfig := elbstatus.Config{ + Region: region, + FrontendTagValue: elbFrontendTagValue, + IngressNameTagValue: ingressClassName, + KubernetesClient: kubernetesClient, + } + elbStatusUpdater, err := elbstatus.New(statusConfig) + if err != nil { + return nil, err + } + return append(updaters, elbStatusUpdater), nil +} diff --git a/feed-ingress/cmd/gorb.go b/feed-ingress/cmd/gorb.go new file mode 100644 index 00000000..08d5ea04 --- /dev/null +++ b/feed-ingress/cmd/gorb.go @@ -0,0 +1,120 @@ +package cmd + +import ( + "fmt" + "strconv" + "strings" + + "github.com/sky-uk/feed/k8s" + + "github.com/sky-uk/feed/controller" + "github.com/sky-uk/feed/gorb" + "github.com/spf13/cobra" +) + +var ( + gorbIngressInstanceIP string + gorbEndpoint string + gorbServicesDefinition string + gorbBackendMethod string + gorbBackendWeight int + gorbVipLoadbalancer string + gorbManageLoopback bool + gorbBackendHealthcheckInterval string + gorbBackendHealthcheckType string + gorbInterfaceProcFsPath string +) + +const ( + defaultGorbIngressInstanceIP = "127.0.0.1" + defaultGorbEndpoint = "http://127.0.0.1:80" + defaultGorbBackendMethod = "dr" + defaultGorbBackendWeight = 1000 + defaultGorbServicesDefinition = "http-proxy:80,https-proxy:443" + defaultGorbVipLoadbalancer = "127.0.0.1" + defaultGorbManageLoopback = true + defaultGorbInterfaceProcFsPath = "/host-ipv4-proc/" + defaultGorbBackendHealthcheckInterval = "1s" + defaultGorbBackendHealthcheckType = "http" +) + +var gorbCmd = &cobra.Command{ + Use: "gorb", + Short: "Attach to Gorb Load Balancers (deprecated; use Merlin instead)", + Long: `Configure IPVS via Gorb. https://github.com/sky-uk/gorb`, + Run: func(cmd *cobra.Command, args []string) { + runCmd(appendGorbIngressUpdaters) + }, +} + +func init() { + rootCmd.AddCommand(gorbCmd) + + gorbCmd.Flags().StringVar(&gorbEndpoint, "gorb-endpoint", defaultGorbEndpoint, "Define the endpoint to talk to gorb for registration.") + gorbCmd.Flags().StringVar(&gorbIngressInstanceIP, "gorb-ingress-instance-ip", defaultGorbIngressInstanceIP, + "Define the ingress instance ip, the ip of the node where feed-ingress is running.") + gorbCmd.Flags().StringVar(&gorbServicesDefinition, "gorb-services-definition", defaultGorbServicesDefinition, + "Comma separated list of Service Definition (e.g. 'http-proxy:80,https-proxy:443') to register via Gorb") + gorbCmd.Flags().StringVar(&gorbBackendMethod, "gorb-backend-method", defaultGorbBackendMethod, + "Define the backend method (e.g. nat, dr, tunnel) to register via Gorb ") + gorbCmd.Flags().IntVar(&gorbBackendWeight, "gorb-backend-weight", defaultGorbBackendWeight, + "Define the backend weight to register via Gorb") + gorbCmd.Flags().StringVar(&gorbVipLoadbalancer, "gorb-vip-loadbalancer", defaultGorbVipLoadbalancer, + "Define the vip loadbalancer to set the loopback. Only necessary when Direct Return is enabled.") + gorbCmd.Flags().BoolVar(&gorbManageLoopback, "gorb-management-loopback", defaultGorbManageLoopback, + "Enable loopback creation. Only necessary when Direct Return is enabled") + gorbCmd.Flags().StringVar(&gorbInterfaceProcFsPath, "gorb-interface-proc-fs-path", defaultGorbInterfaceProcFsPath, + "Path to the interface proc file system. Only necessary when Direct Return is enabled") + gorbCmd.Flags().StringVar(&gorbBackendHealthcheckInterval, "gorb-backend-healthcheck-interval", defaultGorbBackendHealthcheckInterval, + "Define the gorb healthcheck interval for the backend") + gorbCmd.Flags().StringVar(&gorbBackendHealthcheckType, "gorb-backend-healthcheck-type", defaultGorbBackendHealthcheckType, + "Define the gorb healthcheck type for the backend. Must be either 'tcp', 'http' or 'none'") +} + +func appendGorbIngressUpdaters(kubernetesClient k8s.Client, updaters []controller.Updater) ([]controller.Updater, error) { + virtualServices, err := toVirtualServices(gorbServicesDefinition) + if err != nil { + return nil, fmt.Errorf("invalid gorb services definition. Must be a comma separated list - e.g. 'http-proxy:80,https-proxy:443', but was %s", gorbServicesDefinition) + } + + if gorbBackendHealthcheckType != "tcp" && gorbBackendHealthcheckType != "http" && gorbBackendHealthcheckType != "none" { + return nil, fmt.Errorf("invalid gorb backend healthcheck type. Must be either 'tcp', 'http' or 'none', but was %s", gorbBackendHealthcheckType) + } + + config := gorb.Config{ + ServerBaseURL: gorbEndpoint, + InstanceIP: gorbIngressInstanceIP, + DrainDelay: drainDelay, + ServicesDefinition: virtualServices, + BackendMethod: gorbBackendMethod, + BackendWeight: gorbBackendWeight, + VipLoadbalancer: gorbVipLoadbalancer, + ManageLoopback: gorbManageLoopback, + BackendHealthcheckInterval: gorbBackendHealthcheckInterval, + BackendHealthcheckType: gorbBackendHealthcheckType, + InterfaceProcFsPath: gorbInterfaceProcFsPath, + } + + gorbUpdater, err := gorb.New(&config) + if err != nil { + return nil, err + } + return append(updaters, gorbUpdater), nil +} + +func toVirtualServices(servicesCsv string) ([]gorb.VirtualService, error) { + virtualServices := make([]gorb.VirtualService, 0) + servicesDefinitionArr := strings.Split(servicesCsv, ",") + for _, service := range servicesDefinitionArr { + servicesArr := strings.Split(service, ":") + if len(servicesArr) != 2 { + return nil, fmt.Errorf("unable to convert %s to servicename:port combination", servicesArr) + } + port, err := strconv.Atoi(servicesArr[1]) + if err != nil { + return nil, fmt.Errorf("unable to convert port %s to int", servicesArr[1]) + } + virtualServices = append(virtualServices, gorb.VirtualService{Name: servicesArr[0], Port: port}) + } + return virtualServices, nil +} diff --git a/feed-ingress/cmd/merlin.go b/feed-ingress/cmd/merlin.go new file mode 100644 index 00000000..490e17b1 --- /dev/null +++ b/feed-ingress/cmd/merlin.go @@ -0,0 +1,125 @@ +package cmd + +import ( + "time" + + "github.com/sky-uk/feed/merlin" + + "github.com/sky-uk/feed/controller" + "github.com/sky-uk/feed/k8s" + + "github.com/sky-uk/feed/merlin/status" + "github.com/spf13/cobra" +) + +var ( + merlinEndpoint string + merlinRequestTimeout time.Duration + merlinServiceID string + merlinHTTPSServiceID string + merlinInstanceIP string + merlinForwardMethod string + merlinDrainDelay time.Duration + merlinHealthUpThreshold uint + merlinHealthDownThreshold uint + merlinHealthPeriod time.Duration + merlinHealthTimeout time.Duration + merlinVIP string + merlinVIPInterface string + merlinInternalHostname string + merlinInternetFacingHostname string +) + +const ( + defaultMerlinForwardMethod = "route" + defaultMerlinHealthUpThreshold = 3 + defaultMerlinHealthDownThreshold = 2 + defaultMerlinHealthPeriod = 10 * time.Second + defaultMerlinHealthTimeout = time.Second + defaultMerlinVIPInterface = "lo" +) + +var merlinCmd = &cobra.Command{ + Use: "merlin", + Short: "Attach to Merlin Load Balancers", + Long: `Merlin is a distributed load balancer based on IPVS, with a gRPC based API. +Feed Ingress supports attaching to merlin as a frontend. https://github.com/sky-uk/merlin`, + Run: func(cmd *cobra.Command, args []string) { + runCmd(appendMerlinIngressUpdaters) + }, +} + +func init() { + rootCmd.AddCommand(merlinCmd) + + merlinCmd.Flags().StringVar(&merlinEndpoint, "merlin-endpoint", "", + "Merlin gRPC endpoint to connect to. Expected format is scheme://authority/endpoint_name (see "+ + "https://github.com/grpc/grpc/blob/master/doc/naming.md). Will load balance between all available servers.") + merlinCmd.Flags().DurationVar(&merlinRequestTimeout, "merlin-request-timeout", time.Second*10, + "Timeout for any requests to merlin.") + merlinCmd.Flags().StringVar(&merlinServiceID, "merlin-service-id", "", "Merlin http virtual service ID to attach to.") + merlinCmd.Flags().StringVar(&merlinHTTPSServiceID, "merlin-https-service-id", "", "Merlin https virtual service ID to attach to.") + merlinCmd.Flags().StringVar(&merlinInstanceIP, "merlin-instance-ip", "", "Ingress IP to register with merlin") + merlinCmd.Flags().StringVar(&merlinForwardMethod, "merlin-forward-method", defaultMerlinForwardMethod, "IPVS forwarding method,"+ + " must be one of route, tunnel, or masq.") + merlinCmd.Flags().DurationVar(&merlinDrainDelay, "merlin-drain-delay", defaultDrainDelay, "Delay to wait after for connections"+ + " to bleed off when deregistering from merlin. Real server weight is set to 0 during this delay.") + merlinCmd.Flags().UintVar(&merlinHealthUpThreshold, "merlin-health-up-threshold", defaultMerlinHealthUpThreshold, + "Number of checks before merlin will consider this instance healthy.") + merlinCmd.Flags().UintVar(&merlinHealthDownThreshold, "merlin-health-down-threshold", defaultMerlinHealthDownThreshold, + "Number of checks before merlin will consider this instance unhealthy.") + merlinCmd.Flags().DurationVar(&merlinHealthPeriod, "merlin-health-period", defaultMerlinHealthPeriod, + "The time between health checks.") + merlinCmd.Flags().DurationVar(&merlinHealthTimeout, "merlin-health-timeout", defaultMerlinHealthTimeout, + "The timeout for health checks.") + merlinCmd.Flags().StringVar(&merlinVIP, "merlin-vip", "", "VIP to assign to loopback to support direct route and tunnel.") + merlinCmd.Flags().StringVar(&merlinVIPInterface, "merlin-vip-interface", defaultMerlinVIPInterface, + "VIP interface to assign the VIP.") + merlinCmd.Flags().StringVar(&merlinInternalHostname, "merlin-internal-hostname", "", + "Hostname of the internal facing load-balancer.") + merlinCmd.Flags().StringVar(&merlinInternetFacingHostname, "merlin-internet-facing-hostname", "", + "Hostname of the internet facing load-balancer") +} + +func appendMerlinIngressUpdaters(kubernetesClient k8s.Client, updaters []controller.Updater) ([]controller.Updater, error) { + config := merlin.Config{ + Endpoint: merlinEndpoint, + Timeout: merlinRequestTimeout, + ServiceID: merlinServiceID, + HTTPSServiceID: merlinHTTPSServiceID, + InstanceIP: merlinInstanceIP, + InstancePort: uint16(ingressPort), + InstanceHTTPSPort: uint16(ingressHTTPSPort), + ForwardMethod: merlinForwardMethod, + DrainDelay: merlinDrainDelay, + HealthPort: uint16(ingressHealthPort), + // This value is hardcoded into the nginx template. + HealthPath: "health", + HealthUpThreshold: uint32(merlinHealthUpThreshold), + HealthDownThreshold: uint32(merlinHealthDownThreshold), + HealthPeriod: merlinHealthPeriod, + HealthTimeout: merlinHealthTimeout, + VIP: merlinVIP, + VIPInterface: merlinVIPInterface, + } + merlinUpdater, err := merlin.New(config) + if err != nil { + return nil, err + } + updaters = append(updaters, merlinUpdater) + + if merlinInternalHostname != "" || merlinInternetFacingHostname != "" { + statusConfig := status.Config{ + InternalHostname: merlinInternalHostname, + InternetFacingHostname: merlinInternetFacingHostname, + KubernetesClient: kubernetesClient, + } + merlinStatusUpdater, err := status.New(statusConfig) + if err != nil { + return nil, err + } + updaters = append(updaters, merlinStatusUpdater) + } + + return updaters, nil +} diff --git a/feed-ingress/cmd/root.go b/feed-ingress/cmd/root.go new file mode 100644 index 00000000..61158c05 --- /dev/null +++ b/feed-ingress/cmd/root.go @@ -0,0 +1,242 @@ +package cmd + +import ( + "fmt" + "os" + "time" + + "github.com/sky-uk/feed/controller" + "github.com/sky-uk/feed/nginx" + "github.com/sky-uk/feed/util/cmd" + "github.com/spf13/cobra" +) + +var ( + // version of binary, injected by "go tool link -X" + version string + // buildTime of binary, injected by "go tool link -X" + buildTime string + + rootCmd = &cobra.Command{ + Use: "feed-ingress", + Version: printVersion(), + Short: "Feed Ingress is a Kubernetes Ingress Controller", + Long: `Feed Ingress attaches to AWS ELBs and routes traffic from them to the Kubernetes +Services declared by Ingress resources. It manages an NGINX instance whose +configuration it updates to keep track with changes in the cluster.`, + } +) + +var ( + debug bool + kubeconfig string + resyncPeriod time.Duration + ingressPort int + ingressHTTPSPort int + ingressHealthPort int + controllerConfig controller.Config + healthPort int + + nginxConfig nginx.Conf + nginxLogHeaders []string + nginxTrustedFrontends []string + nginxSSLPath string + nginxVhostStatsSharedMemory int + nginxOpenTracingPluginPath string + nginxOpenTracingConfigPath string + + ingressClassName string + includeUnnamedIngresses bool + namespaceSelector string + + pushgatewayURL string + pushgatewayIntervalSeconds int + pushgatewayLabels cmd.KeyValues +) + +const ( + unset = -1 + + defaultResyncPeriod = time.Minute * 15 + defaultIngressPort = 8080 + defaultIngressHTTPSPort = unset + defaultIngressHealthPort = 8081 + defaultIngressAllow = "0.0.0.0/0" + defaultIngressStripPath = true + defaultIngressExactPath = false + defaultHealthPort = 12082 + + defaultNginxBinary = "/usr/sbin/nginx" + defaultNginxWorkingDir = "/nginx" + defaultNginxWorkers = 1 + defaultNginxWorkerConnections = 1024 + defaultNginxWorkerShutdownTimeoutSeconds = 0 + defaultNginxKeepAliveSeconds = 60 + defaultNginxBackendKeepalives = 512 + defaultNginxBackendTimeoutSeconds = 60 + defaultNginxBackendConnectTimeoutSeconds = 1 + defaultNginxBackendMaxConnections = 0 + defaultNginxProxyBufferSize = 16 + defaultNginxProxyBufferBlocks = 4 + defaultNginxLogLevel = "warn" + defaultNginxServerNamesHashBucketSize = unset + defaultNginxServerNamesHashMaxSize = unset + defaultNginxProxyProtocol = false + defaultNginxUpdatePeriod = time.Second * 30 + defaultNginxSSLPath = "/etc/ssl/default-ssl/default-ssl" + defaultNginxVhostStatsSharedMemory = 1 + defaultNginxOpenTracingPluginPath = "" + defaultNginxOpenTracingConfigPath = "" + defaultAccessLogDir = "/var/log/nginx" + defaultClientHeaderBufferSize = 16 + defaultClientBodyBufferSize = 16 + defaultLargeClientHeaderBufferBlocks = 4 + + defaultIngressClassName = "" + defaultIncludeUnnamedIngresses = false + defaultIngressControllerNamespaceSelector = "" + + defaultPushgatewayIntervalSeconds = 60 +) + +const ( + ingressClassFlag = "ingress-class" + includeClasslessIngressesFlag = "include-classless-ingresses" + ingressControllerNamespaceSelectorFlag = "ingress-controller-namespace-selector" + + ingressClassAnnotation = "kubernetes.io/ingress.class" +) + +func init() { + configureGeneralFlags() + configureNginxFlags() + configurePrometheusFlags() +} + +func configureGeneralFlags() { + rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, + "Enable debug logging.") + rootCmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", "", + "Path to kubeconfig for connecting to the apiserver. Leave blank to connect inside a cluster.") + rootCmd.PersistentFlags().DurationVar(&resyncPeriod, "resync-period", defaultResyncPeriod, + "Resync with the apiserver periodically to handle missed updates.") + rootCmd.PersistentFlags().IntVar(&ingressPort, "ingress-port", defaultIngressPort, + "Port to serve ingress traffic to backend services.") + rootCmd.PersistentFlags().IntVar(&ingressHTTPSPort, "ingress-https-port", defaultIngressHTTPSPort, + "Port to serve ingress https traffic to backend services.") + rootCmd.PersistentFlags().IntVar(&ingressHealthPort, "ingress-health-port", defaultIngressHealthPort, + "Port for ingress /health and /status pages. Should be used by frontends to determine if ingress is available.") + rootCmd.PersistentFlags().StringVar(&controllerConfig.DefaultAllow, "ingress-allow", defaultIngressAllow, + "Source IP or CIDR to allow ingress access by default. This is overridden by the sky.uk/allow "+ + "annotation on ingress resources. Leave empty to deny all access by default.") + rootCmd.PersistentFlags().BoolVar(&controllerConfig.DefaultStripPath, "ingress-strip-path", defaultIngressStripPath, + "Whether to strip the ingress path from the URL before passing to backend services. For example, "+ + "if enabled 'myhost/myapp/health' would be passed as '/health' to the backend service. If disabled, "+ + "it would be passed as '/myapp/health'. Enabling this requires nginx to process the URL, which has some "+ + "limitations. URL encoded characters will not work correctly in some cases, and backend services will "+ + "need to take care to properly construct URLs, such as by using the 'X-Original-URI' header."+ + "Can be overridden with the sky.uk/strip-path annotation per ingress") + rootCmd.PersistentFlags().BoolVar(&controllerConfig.DefaultExactPath, "ingress-exact-path", defaultIngressExactPath, + "Whether to consider the ingress path to be an exact match rather than as a prefix. For example, "+ + "if enabled 'myhost/myapp/health' would match 'myhost/myapp/health' but not 'myhost/myapp/health/x'."+ + " If disabled, it would match both (and redirect requests from 'myhost/myapp/health' to "+ + " '/myhost/myapp/health/'. Can be overridden with the sky.uk/exact-path annotation per ingress") + rootCmd.PersistentFlags().IntVar(&healthPort, "health-port", defaultHealthPort, + "Port for checking the health of the ingress controller on /health. Also provides /debug/pprof.") + rootCmd.PersistentFlags().StringVar(&ingressClassName, ingressClassFlag, defaultIngressClassName, + fmt.Sprintf("The name of this instance. It will consider only ingress resources with matching %s annotation values.", ingressClassAnnotation)) + rootCmd.PersistentFlags().BoolVar(&includeUnnamedIngresses, includeClasslessIngressesFlag, defaultIncludeUnnamedIngresses, + fmt.Sprintf("In addition to ingress resources with matching %s annotations, also consider those with no such annotation.", ingressClassAnnotation)) + rootCmd.PersistentFlags().StringVar(&namespaceSelector, ingressControllerNamespaceSelectorFlag, defaultIngressControllerNamespaceSelector, + "Only consider ingresses within namespaces having labels matching this selector (e.g. app=loadtest).") + + _ = rootCmd.PersistentFlags().MarkDeprecated(includeClasslessIngressesFlag, + fmt.Sprintf("please annotate ingress resources explicitly with %s", ingressClassAnnotation)) +} + +func configureNginxFlags() { + rootCmd.PersistentFlags().StringVar(&nginxConfig.BinaryLocation, "nginx-binary", defaultNginxBinary, + "Location of nginx binary.") + rootCmd.PersistentFlags().StringVar(&nginxConfig.WorkingDir, "nginx-workdir", defaultNginxWorkingDir, + "Directory to store nginx files. Also the location of the nginx.tmpl file.") + rootCmd.PersistentFlags().IntVar(&nginxConfig.WorkerProcesses, "nginx-workers", defaultNginxWorkers, + "Number of nginx worker processes.") + rootCmd.PersistentFlags().IntVar(&nginxConfig.WorkerConnections, "nginx-worker-connections", defaultNginxWorkerConnections, + "Max number of connections per nginx worker. Includes both client and proxy connections.") + rootCmd.PersistentFlags().IntVar(&nginxConfig.WorkerShutdownTimeoutSeconds, "nginx-worker-shutdown-timeout-seconds", defaultNginxWorkerShutdownTimeoutSeconds, + "Timeout for a graceful shutdown of worker processes.") + rootCmd.PersistentFlags().IntVar(&nginxConfig.KeepaliveSeconds, "nginx-keepalive-seconds", defaultNginxKeepAliveSeconds, + "Keep alive time for persistent client connections to nginx. Should generally be set larger than frontend "+ + "keep alive times to prevent stale connections.") + rootCmd.PersistentFlags().IntVar(&nginxConfig.BackendKeepalives, "nginx-backend-keepalive-count", defaultNginxBackendKeepalives, + "Maximum number of keepalive connections per backend service. Keepalive connections count against"+ + " nginx-worker-connections limit, and will be restricted by that global limit as well.") + rootCmd.PersistentFlags().IntVar(&controllerConfig.DefaultBackendTimeoutSeconds, "nginx-default-backend-timeout-seconds", + defaultNginxBackendTimeoutSeconds, + "Timeout for requests to backends. Can be overridden per ingress with the sky.uk/backend-timeout-seconds annotation.") + rootCmd.PersistentFlags().IntVar(&nginxConfig.BackendConnectTimeoutSeconds, "nginx-backend-connect-timeout-seconds", + defaultNginxBackendConnectTimeoutSeconds, + "Connect timeout to backend services.") + rootCmd.PersistentFlags().IntVar(&controllerConfig.DefaultBackendMaxConnections, "nginx-default-backend-max-connections", + defaultNginxBackendMaxConnections, + "Maximum number of connections to a single backend. Can be overridden per ingress with the sky.uk/backend-max-connections annotation.") + rootCmd.PersistentFlags().IntVar(&controllerConfig.DefaultProxyBufferSize, "nginx-default-proxy-buffer-size", + defaultNginxProxyBufferSize, + "Proxy buffer size for response. Can be overridden per ingress with the sky.uk/proxy-buffer-size-in-kb annotation.") + rootCmd.PersistentFlags().IntVar(&controllerConfig.DefaultProxyBufferBlocks, "nginx-default-proxy-buffer-blocks", + defaultNginxProxyBufferBlocks, + "Proxy buffer blocks for response. Can be overridden per ingress with the sky.uk/proxy-buffer-blocks annotation.") + rootCmd.PersistentFlags().StringVar(&nginxConfig.LogLevel, "nginx-loglevel", defaultNginxLogLevel, + "Log level for nginx. See http://nginx.org/en/docs/ngx_core_module.html#error_log for levels.") + rootCmd.PersistentFlags().IntVar(&nginxConfig.ServerNamesHashBucketSize, "nginx-server-names-hash-bucket-size", defaultNginxServerNamesHashBucketSize, + "Sets the bucket size for the server names hash tables. Setting this to 0 or less will exclude this "+ + "config from the nginx conf file. The details of setting up hash tables are provided "+ + "in a separate document. http://nginx.org/en/docs/hash.html") + rootCmd.PersistentFlags().IntVar(&nginxConfig.ServerNamesHashMaxSize, "nginx-server-names-hash-max-size", defaultNginxServerNamesHashMaxSize, + "Sets the maximum size of the server names hash tables. Setting this to 0 or less will exclude this "+ + "config from the nginx conf file. The details of setting up hash tables are provided "+ + "in a separate document. http://nginx.org/en/docs/hash.html") + rootCmd.PersistentFlags().BoolVar(&nginxConfig.ProxyProtocol, "nginx-proxy-protocol", defaultNginxProxyProtocol, + "Enable PROXY protocol for nginx listeners.") + rootCmd.PersistentFlags().DurationVar(&nginxConfig.UpdatePeriod, "nginx-update-period", defaultNginxUpdatePeriod, + "How often nginx reloads can occur. Too frequent will result in many nginx worker processes alive at the same time.") + rootCmd.PersistentFlags().StringVar(&nginxConfig.AccessLogDir, "access-log-dir", defaultAccessLogDir, "Access logs direcoty.") + rootCmd.PersistentFlags().BoolVar(&nginxConfig.AccessLog, "access-log", false, "Enable access logs directive.") + rootCmd.PersistentFlags().StringSliceVar(&nginxLogHeaders, "nginx-log-headers", []string{}, "Comma separated list of headers to be logged in access logs") + rootCmd.PersistentFlags().StringSliceVar(&nginxTrustedFrontends, "nginx-trusted-frontends", []string{}, + "Comma separated list of CIDRs to trust when determining the client's real IP from "+ + "frontends. The client IP is used for allowing or denying ingress access. "+ + "This will typically be the ELB subnet.") + rootCmd.PersistentFlags().StringVar(&nginxSSLPath, "ssl-path", defaultNginxSSLPath, + "Set default ssl path + name file without extension. Feed expects two files: one ending in .crt (the CA) and the other in .key (the private key).") + rootCmd.PersistentFlags().IntVar(&nginxVhostStatsSharedMemory, "nginx-vhost-stats-shared-memory", defaultNginxVhostStatsSharedMemory, + "Memory (in MiB) which should be allocated for use by the vhost statistics module") + rootCmd.PersistentFlags().StringVar(&nginxOpenTracingPluginPath, "nginx-opentracing-plugin-path", defaultNginxOpenTracingPluginPath, + "Path to OpenTracing plugin on disk (eg. /usr/local/lib/libjaegertracing_plugin.so)") + rootCmd.PersistentFlags().StringVar(&nginxOpenTracingConfigPath, "nginx-opentracing-config-path", defaultNginxOpenTracingConfigPath, + "Path to OpenTracing config on disk (eg. /etc/jaeger-nginx-config.json)") + rootCmd.PersistentFlags().IntVar(&nginxConfig.ClientHeaderBufferSize, "nginx-client-header-buffer-size-in-kb", defaultClientHeaderBufferSize, "Sets buffer size for reading client request header") + rootCmd.PersistentFlags().IntVar(&nginxConfig.ClientBodyBufferSize, "nginx-client-body-buffer-size-in-kb", defaultClientBodyBufferSize, "Sets buffer size for reading client request body") + rootCmd.PersistentFlags().IntVar(&nginxConfig.LargeClientHeaderBufferBlocks, "nginx-large-client-header-buffer-blocks", defaultLargeClientHeaderBufferBlocks, "Sets the maximum number of buffers used for reading large client request header") +} + +func configurePrometheusFlags() { + rootCmd.PersistentFlags().StringVar(&pushgatewayURL, "pushgateway", "", + "Prometheus pushgateway URL for pushing metrics. Leave blank to not push metrics.") + rootCmd.PersistentFlags().IntVar(&pushgatewayIntervalSeconds, "pushgateway-interval", defaultPushgatewayIntervalSeconds, + "Interval in seconds for pushing metrics.") + rootCmd.PersistentFlags().Var(&pushgatewayLabels, "pushgateway-label", + "A label=value pair to attach to metrics pushed to prometheus. Specify multiple times for multiple labels.") +} + +func printVersion() string { + return fmt.Sprintf("%s (%s)", version, buildTime) +} + +// Execute is the entry point for Cobra commands +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/feed-ingress/main.go b/feed-ingress/main.go new file mode 100644 index 00000000..a1ce73fd --- /dev/null +++ b/feed-ingress/main.go @@ -0,0 +1,11 @@ +package main + +import ( + _ "net/http/pprof" + + "github.com/sky-uk/feed/feed-ingress/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/k8s/client.go b/k8s/client.go index 3ab20156..773002e9 100644 --- a/k8s/client.go +++ b/k8s/client.go @@ -29,8 +29,11 @@ const bufferedWatcherDuration = time.Millisecond * 50 // including reconnects, to notify that there may be new ingresses that need to be retrieved. // It's intended that client code will call the getters to retrieve the current state when notified. type Client interface { - // GetIngresses returns all the ingresses in the cluster. - GetIngresses() ([]*v1beta1.Ingress, error) + // GetAllIngresses returns all the ingresses in the cluster. + GetAllIngresses() ([]*v1beta1.Ingress, error) + + // GetIngresses returns ingresses in namespaces with matching labels + GetIngresses(*NamespaceSelector) ([]*v1beta1.Ingress, error) // GetServices returns all the services in the cluster. GetServices() ([]*v1.Service, error) @@ -41,20 +44,32 @@ type Client interface { // WatchServices watches for updates to services and notifies the Watcher. WatchServices() Watcher + // WatchNamespaces watches for updates to namespaces and notifies the Watcher. + WatchNamespaces() Watcher + // UpdateIngressStatus updates the ingress status with the loadbalancer hostname or ip address. UpdateIngressStatus(*v1beta1.Ingress) error } type client struct { sync.Mutex - clientset *kubernetes.Clientset - resyncPeriod time.Duration - ingressStore cache.Store - ingressController *cache.Controller - ingressWatcher *handlerWatcher - serviceStore cache.Store - serviceController *cache.Controller - serviceWatcher *handlerWatcher + clientset *kubernetes.Clientset + resyncPeriod time.Duration + ingressStore cache.Store + ingressController *cache.Controller + ingressWatcher *handlerWatcher + serviceStore cache.Store + serviceController *cache.Controller + serviceWatcher *handlerWatcher + namespaceStore cache.Store + namespaceController *cache.Controller + namespaceWatcher *handlerWatcher +} + +// NamespaceSelector defines the label name and value for filtering namespaces +type NamespaceSelector struct { + LabelName string + LabelValue string } // New creates a client for the kubernetes apiserver. @@ -72,19 +87,67 @@ func New(kubeconfig string, resyncPeriod time.Duration) (Client, error) { return &client{clientset: clientset, resyncPeriod: resyncPeriod}, nil } -func (c *client) GetIngresses() ([]*v1beta1.Ingress, error) { - c.createIngressSource() +func (c *client) GetAllIngresses() ([]*v1beta1.Ingress, error) { + return c.GetIngresses(nil) +} - if !c.ingressController.HasSynced() { - return nil, errors.New("Ingresses haven't synced yet") +func (c *client) GetIngresses(selector *NamespaceSelector) ([]*v1beta1.Ingress, error) { + if !c.namespaceController.HasSynced() { + return nil, errors.New("namespaces haven't synced yet") } - ingresses := []*v1beta1.Ingress{} + allIngresses := []*v1beta1.Ingress{} for _, obj := range c.ingressStore.List() { - ingresses = append(ingresses, obj.(*v1beta1.Ingress)) + allIngresses = append(allIngresses, obj.(*v1beta1.Ingress)) + } + + if selector == nil { + return allIngresses, nil } - return ingresses, nil + supportedNamespaces := supportedNamespaces(selector, toNamespaces(c.namespaceStore.List())) + + filteredIngresses := []*v1beta1.Ingress{} + for _, ingress := range allIngresses { + if ingressInNamespace(ingress, supportedNamespaces) { + filteredIngresses = append(filteredIngresses, ingress) + } + } + return filteredIngresses, nil +} + +func toNamespaces(interfaces []interface{}) []*v1.Namespace { + namespaces := make([]*v1.Namespace, len(interfaces)) + for i, obj := range interfaces { + namespaces[i] = obj.(*v1.Namespace) + } + return namespaces +} + +func supportedNamespaces(selector *NamespaceSelector, namespaces []*v1.Namespace) []*v1.Namespace { + if selector == nil { + return namespaces + } + + filteredNamespaces := []*v1.Namespace{} + for _, namespace := range namespaces { + if val, ok := namespace.Labels[selector.LabelName]; ok && val == selector.LabelValue { + filteredNamespaces = append(filteredNamespaces, namespace) + } + } + log.Debugf("Found %d of %d namespaces that match the selector %s=%s", + len(filteredNamespaces), len(namespaces), selector.LabelName, selector.LabelValue) + + return filteredNamespaces +} + +func ingressInNamespace(ingress *v1beta1.Ingress, namespaces []*v1.Namespace) bool { + for _, namespace := range namespaces { + if namespace.Name == ingress.Namespace { + return true + } + } + return false } func (c *client) WatchIngresses() Watcher { @@ -113,7 +176,7 @@ func (c *client) GetServices() ([]*v1.Service, error) { c.createServiceSource() if !c.serviceController.HasSynced() { - return nil, errors.New("Services haven't synced yet") + return nil, errors.New("services haven't synced yet") } services := []*v1.Service{} @@ -145,6 +208,28 @@ func (c *client) createServiceSource() { go controller.Run(make(chan struct{})) } +func (c *client) WatchNamespaces() Watcher { + c.createNamespaceSource() + return c.namespaceWatcher +} + +func (c *client) createNamespaceSource() { + c.Lock() + defer c.Unlock() + if c.namespaceStore != nil { + return + } + + namespaceLW := cache.NewListWatchFromClient( + c.clientset.CoreV1().RESTClient(), "namespaces", "", fields.Everything()) + c.namespaceWatcher = &handlerWatcher{bufferedWatcher: newBufferedWatcher(bufferedWatcherDuration)} + store, controller := cache.NewInformer(namespaceLW, &v1.Namespace{}, c.resyncPeriod, c.namespaceWatcher) + + c.namespaceStore = store + c.namespaceController = controller + go controller.Run(make(chan struct{})) +} + func (c *client) UpdateIngressStatus(ingress *v1beta1.Ingress) error { ingressClient := c.clientset.ExtensionsV1beta1().Ingresses(ingress.Namespace) diff --git a/merlin/mocks/merlin_client.go b/merlin/mocks/merlin_client.go index e8896ba8..ec933b9c 100644 --- a/merlin/mocks/merlin_client.go +++ b/merlin/mocks/merlin_client.go @@ -1,12 +1,15 @@ // Package mocks generated by mockery v1.0.0 package mocks -import context "context" -import empty "github.com/golang/protobuf/ptypes/empty" -import grpc "google.golang.org/grpc" -import mock "github.com/stretchr/testify/mock" -import types "github.com/sky-uk/merlin/types" -import wrappers "github.com/golang/protobuf/ptypes/wrappers" +import ( + "context" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/golang/protobuf/ptypes/wrappers" + "github.com/sky-uk/merlin/types" + "github.com/stretchr/testify/mock" + "google.golang.org/grpc" +) // MerlinClient is an autogenerated mock type for the MerlinClient type type MerlinClient struct { diff --git a/util/cmd/keyvalue.go b/util/cmd/keyvalue.go index 0b6c1a62..244a6112 100644 --- a/util/cmd/keyvalue.go +++ b/util/cmd/keyvalue.go @@ -30,3 +30,8 @@ func (kv *KeyValues) Set(value string) error { return nil } + +// Type returns the identifier for this type +func (kv *KeyValues) Type() string { + return "keyvalues" +} diff --git a/util/test/mocks.go b/util/test/mocks.go index 65349862..a77251e0 100644 --- a/util/test/mocks.go +++ b/util/test/mocks.go @@ -12,12 +12,18 @@ type FakeClient struct { mock.Mock } -// GetIngresses mocks out calls to GetIngresses -func (c *FakeClient) GetIngresses() ([]*v1beta1.Ingress, error) { +// GetAllIngresses mocks out calls to GetAllIngresses +func (c *FakeClient) GetAllIngresses() ([]*v1beta1.Ingress, error) { r := c.Called() return r.Get(0).([]*v1beta1.Ingress), r.Error(1) } +// GetIngresses mocks out calls to GetIngresses +func (c *FakeClient) GetIngresses(selector *k8s.NamespaceSelector) ([]*v1beta1.Ingress, error) { + r := c.Called(selector) + return r.Get(0).([]*v1beta1.Ingress), r.Error(1) +} + // WatchIngresses mocks out calls to WatchIngresses func (c *FakeClient) WatchIngresses() k8s.Watcher { r := c.Called() @@ -36,6 +42,12 @@ func (c *FakeClient) WatchServices() k8s.Watcher { return r.Get(0).(k8s.Watcher) } +// WatchNamespaces mocks out calls to WatchNamespaces +func (c *FakeClient) WatchNamespaces() k8s.Watcher { + r := c.Called() + return r.Get(0).(k8s.Watcher) +} + // UpdateIngressStatus mocks out calls to UpdateIngressStatus func (c *FakeClient) UpdateIngressStatus(*v1beta1.Ingress) error { r := c.Called()