diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 63d41b588b..69972eb513 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -100,13 +100,14 @@ jobs: REGISTRY: ingress-controller run: | echo "building images..." - make clean-image build image + make clean-image build image image-chroot make -C test/e2e-image image echo "creating images cache..." docker save \ nginx-ingress-controller:e2e \ ingress-controller/controller:1.0.0-dev \ + ingress-controller/controller-chroot:1.0.0-dev \ | pigz > docker.tar.gz - name: cache @@ -250,6 +251,65 @@ jobs: kind get kubeconfig > $HOME/.kube/kind-config-kind make kind-e2e-test + kubernetes-chroot: + name: Kubernetes chroot + runs-on: ubuntu-latest + needs: + - changes + - build + if: | + (needs.changes.outputs.go == 'true') + + strategy: + matrix: + k8s: [v1.21.10, v1.22.7, v1.23.4] + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: cache + uses: actions/download-artifact@v2 + with: + name: docker.tar.gz + + - name: Create Kubernetes ${{ matrix.k8s }} cluster + id: kind + uses: engineerd/setup-kind@v0.5.0 + with: + version: v0.12.0 + config: test/e2e/kind.yaml + image: kindest/node:${{ matrix.k8s }} + + - uses: geekyeggo/delete-artifact@v1 + with: + name: docker.tar.gz + failOnError: false + + - name: Prepare cluster for testing + id: local-path + run: | + kubectl version + echo + echo "installing helm 3..." + curl -sSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash + + - name: Load images from cache + run: | + echo "loading docker images..." + pigz -dc docker.tar.gz | docker load + + - name: Run e2e tests + env: + KIND_CLUSTER_NAME: kind + SKIP_CLUSTER_CREATION: true + SKIP_IMAGE_CREATION: true + IS_CHROOT: true + run: | + kind get kubeconfig > $HOME/.kube/kind-config-kind + make kind-e2e-test + test-image-build: permissions: contents: read # for dorny/paths-filter to fetch a list of changed files diff --git a/Makefile b/Makefile index c4e109d86d..58a3e95c59 100644 --- a/Makefile +++ b/Makefile @@ -75,11 +75,30 @@ image: clean-image ## Build image for a particular arch. --build-arg BUILD_ID="$(BUILD_ID)" \ -t $(REGISTRY)/controller:$(TAG) rootfs +.PHONY: image-chroot +image-chroot: clean-chroot-image ## Build image for a particular arch. + echo "Building docker image ($(ARCH))..." + @docker build \ + --no-cache \ + --build-arg BASE_IMAGE="$(BASE_IMAGE)" \ + --build-arg VERSION="$(TAG)" \ + --build-arg TARGETARCH="$(ARCH)" \ + --build-arg COMMIT_SHA="$(COMMIT_SHA)" \ + --build-arg BUILD_ID="$(BUILD_ID)" \ + -t $(REGISTRY)/controller-chroot:$(TAG) rootfs -f rootfs/Dockerfile.chroot + .PHONY: clean-image clean-image: ## Removes local image echo "removing old image $(REGISTRY)/controller:$(TAG)" @docker rmi -f $(REGISTRY)/controller:$(TAG) || true + +.PHONY: clean-chroot-image +clean-chroot-image: ## Removes local image + echo "removing old image $(REGISTRY)/controller-chroot:$(TAG)" + @docker rmi -f $(REGISTRY)/controller-chroot:$(TAG) || true + + .PHONY: build build: ## Build ingress controller, debug tool and pre-stop hook. @build/run-in-docker.sh \ @@ -221,3 +240,14 @@ release: ensure-buildx clean --build-arg COMMIT_SHA="$(COMMIT_SHA)" \ --build-arg BUILD_ID="$(BUILD_ID)" \ -t $(REGISTRY)/controller:$(TAG) rootfs + + @docker buildx build \ + --no-cache \ + --push \ + --progress plain \ + --platform $(subst $(SPACE),$(COMMA),$(PLATFORMS)) \ + --build-arg BASE_IMAGE="$(BASE_IMAGE)" \ + --build-arg VERSION="$(TAG)" \ + --build-arg COMMIT_SHA="$(COMMIT_SHA)" \ + --build-arg BUILD_ID="$(BUILD_ID)" \ + -t $(REGISTRY)/controller-chroot:$(TAG) rootfs -f rootfs/Dockerfile.chroot diff --git a/build/build.sh b/build/build.sh index 9edae604d1..a865fe927d 100755 --- a/build/build.sh +++ b/build/build.sh @@ -69,3 +69,4 @@ go build \ -X ${PKG}/version.COMMIT=${COMMIT_SHA} \ -X ${PKG}/version.REPO=${REPO_INFO}" \ -o "${TARGETS_DIR}/wait-shutdown" "${PKG}/cmd/waitshutdown" + diff --git a/build/test.sh b/build/test.sh index bb7ccdb7e4..674b9484a0 100755 --- a/build/test.sh +++ b/build/test.sh @@ -23,6 +23,7 @@ set -o errexit set -o nounset set -o pipefail +mkdir -p /tmp/nginx if [ -z "${PKG}" ]; then echo "PKG must be set" exit 1 diff --git a/charts/ingress-nginx/README.md b/charts/ingress-nginx/README.md index c52b0ae51d..c6887521c4 100644 --- a/charts/ingress-nginx/README.md +++ b/charts/ingress-nginx/README.md @@ -306,6 +306,7 @@ Kubernetes: `>=1.19.0-0` | controller.hostPort.ports.https | int | `443` | 'hostPort' https port | | controller.hostname | object | `{}` | Optionally customize the pod hostname. | | controller.image.allowPrivilegeEscalation | bool | `true` | | +| controller.image.chroot | bool | `false` | | | controller.image.digest | string | `"sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2"` | | | controller.image.image | string | `"ingress-nginx/controller"` | | | controller.image.pullPolicy | string | `"IfNotPresent"` | | diff --git a/charts/ingress-nginx/templates/_helpers.tpl b/charts/ingress-nginx/templates/_helpers.tpl index a72af5d9de..e69de0c41f 100644 --- a/charts/ingress-nginx/templates/_helpers.tpl +++ b/charts/ingress-nginx/templates/_helpers.tpl @@ -43,11 +43,40 @@ capabilities: - ALL add: - NET_BIND_SERVICE + {{- if .Values.controller.image.chroot }} + - SYS_CHROOT + {{- end }} runAsUser: {{ .Values.controller.image.runAsUser }} allowPrivilegeEscalation: {{ .Values.controller.image.allowPrivilegeEscalation }} {{- end }} {{- end -}} +{{/* +Get specific image +*/}} +{{- define "ingress-nginx.image" -}} +{{- if .chroot -}} +{{- printf "%s-chroot" .image -}} +{{- else -}} +{{- printf "%s" .image -}} +{{- end }} +{{- end -}} + +{{/* +Get specific image digest +*/}} +{{- define "ingress-nginx.imageDigest" -}} +{{- if .chroot -}} +{{- if .digestChroot -}} +{{- printf "@%s" .digestChroot -}} +{{- end }} +{{- else -}} +{{ if .digest -}} +{{- printf "@%s" .digest -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Create a default fully qualified controller name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/charts/ingress-nginx/templates/controller-daemonset.yaml b/charts/ingress-nginx/templates/controller-daemonset.yaml index 09852d0416..4d7361597b 100644 --- a/charts/ingress-nginx/templates/controller-daemonset.yaml +++ b/charts/ingress-nginx/templates/controller-daemonset.yaml @@ -74,7 +74,7 @@ spec: containers: - name: {{ .Values.controller.containerName }} {{- with .Values.controller.image }} - image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{- end -}}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }}" {{- end }} imagePullPolicy: {{ .Values.controller.image.pullPolicy }} {{- if .Values.controller.lifecycle }} diff --git a/charts/ingress-nginx/templates/controller-deployment.yaml b/charts/ingress-nginx/templates/controller-deployment.yaml index effd5b06c3..e72e7dbad9 100644 --- a/charts/ingress-nginx/templates/controller-deployment.yaml +++ b/charts/ingress-nginx/templates/controller-deployment.yaml @@ -78,7 +78,7 @@ spec: containers: - name: {{ .Values.controller.containerName }} {{- with .Values.controller.image }} - image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{- end -}}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }}" {{- end }} imagePullPolicy: {{ .Values.controller.image.pullPolicy }} {{- if .Values.controller.lifecycle }} diff --git a/charts/ingress-nginx/values.yaml b/charts/ingress-nginx/values.yaml index eb61ce78b1..500b2bb539 100644 --- a/charts/ingress-nginx/values.yaml +++ b/charts/ingress-nginx/values.yaml @@ -16,6 +16,8 @@ commonLabels: {} controller: name: controller image: + ## Keep false as default for now! + chroot: false registry: k8s.gcr.io image: ingress-nginx/controller ## for backwards compatibility consider setting the full image url via the repository value below @@ -23,6 +25,7 @@ controller: ## repository: tag: "v1.1.3" digest: sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2 + # digestChroot: "" # TODO: Fill when we have it pullPolicy: IfNotPresent # www-data -> uid 101 runAsUser: 101 diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go index f620690b55..cfeadc7db6 100644 --- a/cmd/nginx/flags.go +++ b/cmd/nginx/flags.go @@ -192,6 +192,8 @@ Takes the form ":port". If not provided, no admission controller is starte statusPort = flags.Int("status-port", 10246, `Port to use for the lua HTTP endpoint configuration.`) streamPort = flags.Int("stream-port", 10247, "Port to use for the lua TCP/UDP endpoint configuration.") + internalLoggerAddress = flags.String("internal-logger-address", "127.0.0.1:11514", "Address to be used when binding internal syslogger") + profilerPort = flags.Int("profiler-port", 10245, "Port to use for expose the ingress controller Go profiler when it is enabled.") statusUpdateInterval = flags.Int("status-update-interval", status.UpdateInterval, "Time interval in seconds in which the status should check if an update is required. Default is 60 seconds") @@ -344,6 +346,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g ValidationWebhook: *validationWebhook, ValidationWebhookCertPath: *validationWebhookCert, ValidationWebhookKeyPath: *validationWebhookKey, + InternalLoggerAddress: *internalLoggerAddress, } if *apiserverHost != "" { diff --git a/cmd/nginx/logger.go b/cmd/nginx/logger.go new file mode 100644 index 0000000000..13ec095fa7 --- /dev/null +++ b/cmd/nginx/logger.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + + "k8s.io/klog/v2" + + "gopkg.in/mcuadros/go-syslog.v2" +) + +func logger(address string) { + channel := make(syslog.LogPartsChannel) + handler := syslog.NewChannelHandler(channel) + + server := syslog.NewServer() + + server.SetFormat(syslog.RFC3164) + server.SetHandler(handler) + if err := server.ListenUDP(address); err != nil { + klog.Fatalf("failed bind internal syslog: %w", err) + } + + if err := server.Boot(); err != nil { + klog.Fatalf("failed to boot internal syslog: %w", err) + } + klog.Infof("Is Chrooted, starting logger") + + for logParts := range channel { + fmt.Printf("%s\n", logParts["content"]) + } + + server.Wait() + klog.Infof("Stopping logger") + +} diff --git a/cmd/nginx/main.go b/cmd/nginx/main.go index 7293e6b100..b8378d290f 100644 --- a/cmd/nginx/main.go +++ b/cmd/nginx/main.go @@ -152,6 +152,13 @@ func main() { registerHealthz(nginx.HealthPath, ngx, mux) registerMetrics(reg, mux) + _, errExists := os.Stat("/chroot") + if errExists == nil { + conf.IsChroot = true + go logger(conf.InternalLoggerAddress) + + } + go startHTTPServer(conf.HealthCheckHost, conf.ListenPorts.Health, mux) go ngx.Start() diff --git a/go.mod b/go.mod index daa079ac5e..de5f5938ef 100644 --- a/go.mod +++ b/go.mod @@ -130,6 +130,7 @@ require ( google.golang.org/protobuf v1.27.1 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/mcuadros/go-syslog.v2 v2.3.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect diff --git a/go.sum b/go.sum index 74067dada7..586115d44e 100644 --- a/go.sum +++ b/go.sum @@ -1214,6 +1214,8 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/internal/ingress/controller/checker_test.go b/internal/ingress/controller/checker_test.go index 3f02f40165..fd712b6a82 100644 --- a/internal/ingress/controller/checker_test.go +++ b/internal/ingress/controller/checker_test.go @@ -76,7 +76,7 @@ func TestNginxCheck(t *testing.T) { }) // create pid file - os.MkdirAll("/tmp", file.ReadWriteByUser) + os.MkdirAll("/tmp/nginx", file.ReadWriteByUser) pidFile, err := os.Create(nginx.PID) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index f37516e78c..4afb3e9f55 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -958,12 +958,11 @@ type TemplateConfig struct { EnableMetrics bool MaxmindEditionFiles *[]string MonitorMaxBatchSize int - - PID string - StatusPath string - StatusPort int - StreamPort int - StreamSnippets []string + PID string + StatusPath string + StatusPort int + StreamPort int + StreamSnippets []string } // ListenPorts describe the ports required to run the diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index 651e983a84..a7efdab918 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -120,6 +120,9 @@ type Configuration struct { PostShutdownGracePeriod int ShutdownGracePeriod int + + InternalLoggerAddress string + IsChroot bool } // GetPublishService returns the Service used to set the load-balancer status of Ingresses. diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index e4037dfb05..4a6d3d2f44 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -575,6 +575,15 @@ func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressC cfg.DefaultSSLCertificate = n.getDefaultSSLCertificate() + if n.cfg.IsChroot { + if cfg.AccessLogPath == "/var/log/nginx/access.log" { + cfg.AccessLogPath = fmt.Sprintf("syslog:server=%s", n.cfg.InternalLoggerAddress) + } + if cfg.ErrorLogPath == "/var/log/nginx/error.log" { + cfg.ErrorLogPath = fmt.Sprintf("syslog:server=%s", n.cfg.InternalLoggerAddress) + } + } + tc := ngx_config.TemplateConfig{ ProxySetHeaders: setHeaders, AddHeaders: addHeaders, @@ -614,7 +623,8 @@ func (n NGINXController) testTemplate(cfg []byte) error { if len(cfg) == 0 { return fmt.Errorf("invalid NGINX configuration (empty)") } - tmpfile, err := os.CreateTemp("", tempNginxPattern) + tmpDir := os.TempDir() + "/nginx" + tmpfile, err := os.CreateTemp(tmpDir, tempNginxPattern) if err != nil { return err } diff --git a/internal/ingress/controller/process/nginx.go b/internal/ingress/controller/process/nginx.go index 702f60da8b..70227ac2e4 100644 --- a/internal/ingress/controller/process/nginx.go +++ b/internal/ingress/controller/process/nginx.go @@ -20,7 +20,7 @@ import ( "os/exec" "syscall" - "k8s.io/klog/v2" + klog "k8s.io/klog/v2" ) // IsRespawnIfRequired checks if error type is exec.ExitError or not diff --git a/internal/ingress/controller/util.go b/internal/ingress/controller/util.go index 91e0c3acf8..a8232ed0e3 100644 --- a/internal/ingress/controller/util.go +++ b/internal/ingress/controller/util.go @@ -29,7 +29,7 @@ import ( networking "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress" - "k8s.io/klog/v2" + klog "k8s.io/klog/v2" ) // newUpstream creates an upstream without servers. @@ -98,7 +98,7 @@ func rlimitMaxNumFiles() int { } const ( - defBinary = "/usr/local/nginx/sbin/nginx" + defBinary = "/usr/bin/nginx" cfgPath = "/etc/nginx/nginx.conf" ) diff --git a/internal/ingress/metric/collectors/socket.go b/internal/ingress/metric/collectors/socket.go index 0cbeeb4a6d..ea47755839 100644 --- a/internal/ingress/metric/collectors/socket.go +++ b/internal/ingress/metric/collectors/socket.go @@ -111,7 +111,7 @@ var defObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001} // NewSocketCollector creates a new SocketCollector instance using // the ingress watch namespace and class used by the controller func NewSocketCollector(pod, namespace, class string, metricsPerHost bool, buckets HistogramBuckets) (*SocketCollector, error) { - socket := "/tmp/prometheus-nginx.socket" + socket := "/tmp/nginx/prometheus-nginx.socket" // unix sockets must be unlink()ed before being used _ = syscall.Unlink(socket) diff --git a/internal/nginx/main.go b/internal/nginx/main.go index 485b7d229a..88d2ee8770 100644 --- a/internal/nginx/main.go +++ b/internal/nginx/main.go @@ -28,7 +28,7 @@ import ( "time" ps "github.com/mitchellh/go-ps" - "k8s.io/klog/v2" + klog "k8s.io/klog/v2" ) // TODO: Check https://github.com/kubernetes/kubernetes/blob/master/pkg/master/ports/ports.go for ports already being used @@ -40,7 +40,7 @@ var ProfilerPort = 10245 var TemplatePath = "/etc/nginx/template/nginx.tmpl" // PID defines the location of the pid file used by NGINX -var PID = "/tmp/nginx.pid" +var PID = "/tmp/nginx/nginx.pid" // StatusPort port used by NGINX for the status server var StatusPort = 10246 diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 5a8af2a6d2..1eab94c580 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -54,6 +54,7 @@ RUN bash -xeu -c ' \ /etc/ingress-controller/auth \ /var/log \ /var/log/nginx \ + /tmp/nginx \ ); \ for dir in "${writeDirs[@]}"; do \ mkdir -p ${dir}; \ @@ -67,7 +68,8 @@ RUN apk add --no-cache libcap \ && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \ && setcap cap_net_bind_service=+ep /usr/bin/dumb-init \ && setcap -v cap_net_bind_service=+ep /usr/bin/dumb-init \ - && apk del libcap + && apk del libcap \ + && ln -sf /usr/local/nginx/sbin/nginx /usr/bin/nginx USER www-data diff --git a/rootfs/Dockerfile.chroot b/rootfs/Dockerfile.chroot new file mode 100644 index 0000000000..53f8e3622a --- /dev/null +++ b/rootfs/Dockerfile.chroot @@ -0,0 +1,112 @@ +# Copyright 2022 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} as chroot + +# This intermediary image will be used only to copy all the required files to the chroot +# TODO: Simplify in a future to a single Dockerfile +COPY chroot.sh /chroot.sh +RUN apk update \ + && apk upgrade \ + && /chroot.sh + +FROM alpine:3.14.2 + +ARG TARGETARCH +ARG VERSION +ARG COMMIT_SHA +ARG BUILD_ID=UNSET + +LABEL org.opencontainers.image.title="NGINX Ingress Controller for Kubernetes" +LABEL org.opencontainers.image.documentation="https://kubernetes.github.io/ingress-nginx/" +LABEL org.opencontainers.image.source="https://github.com/kubernetes/ingress-nginx" +LABEL org.opencontainers.image.vendor="The Kubernetes Authors" +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.version="${VERSION}" +LABEL org.opencontainers.image.revision="${COMMIT_SHA}" + +LABEL build_id="${BUILD_ID}" + +# This will be injected in the chroot. Don't change :) +ENV LUA_PATH="/usr/local/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/lib/lua/?.lua;;" +ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;" +ENV PATH=$PATH:/usr/local/luajit/bin:/usr/local/nginx/sbin:/usr/local/nginx/bin + +RUN apk update \ + && apk upgrade \ + && apk add -U --no-cache \ + bash \ + curl \ + openssl \ + ca-certificates \ + dumb-init \ + tzdata \ + diffutils \ + util-linux \ + && ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \ + && adduser -S -D -H -u 101 -h /usr/local/nginx \ + -s /sbin/nologin -G www-data -g www-data www-data + +COPY --from=chroot /chroot /chroot + +COPY --chown=www-data:www-data etc /chroot/etc + +COPY --chown=www-data:www-data bin/${TARGETARCH}/dbg / +COPY --chown=www-data:www-data bin/${TARGETARCH}/nginx-ingress-controller / +COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown / +COPY --chown=www-data:www-data nginx-chroot-wrapper.sh /usr/bin/nginx + +WORKDIR /chroot/etc/nginx + +# Fix permission during the build to avoid issues at runtime +# with volumes (custom templates) +RUN bash -xeu -c ' \ + writeDirs=( \ + /var/log \ + ); \ + for dir in "${writeDirs[@]}"; do \ + mkdir -p ${dir}; \ + chown -R www-data.www-data ${dir}; \ + done' + +RUN apk add --no-cache libcap \ + && setcap cap_sys_chroot,cap_net_bind_service=+ep /nginx-ingress-controller \ + && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /nginx-ingress-controller \ + && setcap cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/unshare \ + && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/unshare \ + && setcap cap_net_bind_service=+ep /chroot/usr/local/nginx/sbin/nginx \ + && setcap -v cap_net_bind_service=+ep /chroot/usr/local/nginx/sbin/nginx \ + && setcap cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/dumb-init \ + && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/dumb-init \ + && apk del libcap + +RUN ln -sf /chroot/etc/nginx /etc/nginx \ + && ln -sf /chroot/tmp/nginx /tmp/nginx \ + && ln -sf /chroot/etc/ingress-controller /etc/ingress-controller \ + && ln -sf /chroot/var/log/nginx /var/log/nginx \ + && touch /chroot/var/log/nginx/access.log \ + && chown www-data:www-data /chroot/var/log/nginx/access.log \ + && echo "" > /chroot/etc/resolv.conf \ + && chown -R www-data.www-data /chroot/var/log/nginx /chroot/etc/resolv.conf + +USER www-data + +EXPOSE 80 443 + +ENTRYPOINT ["/usr/bin/dumb-init", "--"] + +CMD ["/nginx-ingress-controller"] + diff --git a/rootfs/chroot.sh b/rootfs/chroot.sh new file mode 100755 index 0000000000..c2ae3d2fa6 --- /dev/null +++ b/rootfs/chroot.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright 2022 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +writeDirs=( \ + /chroot/etc/nginx \ + /chroot/usr/local/ \ + /chroot/etc/ingress-controller \ + /chroot/etc/ingress-controller/ssl \ + /chroot/etc/ingress-controller/auth \ + /chroot/opt/modsecurity/var/log \ + /chroot/opt/modsecurity/var/upload \ + /chroot/opt/modsecurity/var/audit \ + /chroot/var/log/audit \ + /chroot/var/lib/nginx \ + /chroot/var/log/nginx \ + /chroot/var/lib/nginx/body \ + /chroot/var/lib/nginx/fastcgi \ + /chroot/var/lib/nginx/proxy \ + /chroot/var/lib/nginx/scgi \ + /chroot/var/lib/nginx/uwsgi \ + /chroot/tmp/nginx +); + +for dir in "${writeDirs[@]}"; do + mkdir -p ${dir}; + chown -R www-data.www-data ${dir}; +done + +mkdir -p /chroot/lib /chroot/proc /chroot/usr /chroot/bin /chroot/dev /chroot/run +cp /etc/passwd /etc/group /chroot/etc/ +cp -a /usr/* /chroot/usr/ +mv /var/log/nginx /chroot/var/log/ +cp -a /etc/nginx/* /chroot/etc/nginx/ +cp /lib/ld-musl-* /lib/libcrypto* /lib/libssl* /lib/libz* /chroot/lib/ +mknod -m 0666 /chroot/dev/null c 1 3 +mknod -m 0666 /chroot/dev/random c 1 8 +mknod -m 0666 /chroot/dev/urandom c 1 9 +mknod -m 0666 /chroot/dev/full c 1 7 +mknod -m 0666 /chroot/dev/ptmx c 5 2 +mknod -m 0666 /chroot/dev/zero c 1 5 +mknod -m 0666 /chroot/dev/tty c 5 0 diff --git a/rootfs/etc/nginx/lua/monitor.lua b/rootfs/etc/nginx/lua/monitor.lua index be7a173ee9..8b0e185960 100644 --- a/rootfs/etc/nginx/lua/monitor.lua +++ b/rootfs/etc/nginx/lua/monitor.lua @@ -26,7 +26,7 @@ local _M = {} local function send(payload) local s = assert(socket()) - assert(s:connect("unix:/tmp/prometheus-nginx.socket")) + assert(s:connect("unix:/tmp/nginx/prometheus-nginx.socket")) assert(s:send(payload)) assert(s:close()) end diff --git a/rootfs/etc/nginx/lua/test/monitor_test.lua b/rootfs/etc/nginx/lua/test/monitor_test.lua index 2762a980d1..e75018adf1 100644 --- a/rootfs/etc/nginx/lua/test/monitor_test.lua +++ b/rootfs/etc/nginx/lua/test/monitor_test.lua @@ -148,7 +148,7 @@ describe("Monitor", function() }, }) - assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/prometheus-nginx.socket") + assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/nginx/prometheus-nginx.socket") assert.stub(tcp_mock.send).was_called_with(tcp_mock, expected_payload) assert.stub(tcp_mock.close).was_called_with(tcp_mock) end) diff --git a/rootfs/etc/nginx/nginx.conf b/rootfs/etc/nginx/nginx.conf index 6f8e86b90e..7f60b846d5 100644 --- a/rootfs/etc/nginx/nginx.conf +++ b/rootfs/etc/nginx/nginx.conf @@ -1,6 +1,8 @@ # A very simple nginx configuration file that forces nginx to start. -pid /tmp/nginx.pid; +pid /tmp/nginx/nginx.pid; + +error_log stderr; events {} http {} -daemon off; \ No newline at end of file +daemon off; diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 2ee76831cc..d6e7090747 100755 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -285,10 +285,10 @@ http { keepalive_timeout {{ $cfg.KeepAlive }}s; keepalive_requests {{ $cfg.KeepAliveRequests }}; - client_body_temp_path /tmp/client-body; - fastcgi_temp_path /tmp/fastcgi-temp; - proxy_temp_path /tmp/proxy-temp; - ajp_temp_path /tmp/ajp-temp; + client_body_temp_path /tmp/nginx/client-body; + fastcgi_temp_path /tmp/nginx/fastcgi-temp; + proxy_temp_path /tmp/nginx/proxy-temp; + ajp_temp_path /tmp/nginx/ajp-temp; client_header_buffer_size {{ $cfg.ClientHeaderBufferSize }}; client_header_timeout {{ $cfg.ClientHeaderTimeout }}s; @@ -536,7 +536,7 @@ http { {{ end }} # Cache for internal auth checks - proxy_cache_path /tmp/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off; + proxy_cache_path /tmp/nginx/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off; # Global filters {{ range $ip := $cfg.BlockCIDRs }}deny {{ trimSpace $ip }}; @@ -755,8 +755,8 @@ stream { access_log {{ or $cfg.StreamAccessLogPath $cfg.AccessLogPath }} log_stream {{ $cfg.AccessLogParams }}; {{ end }} + error_log {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }}; - {{ if $cfg.EnableRealIp }} {{ range $trusted_ip := $cfg.ProxyRealIPCIDR }} set_real_ip_from {{ $trusted_ip }}; diff --git a/rootfs/nginx-chroot-wrapper.sh b/rootfs/nginx-chroot-wrapper.sh new file mode 100755 index 0000000000..f7318142f4 --- /dev/null +++ b/rootfs/nginx-chroot-wrapper.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Copyright 2022 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cat /etc/resolv.conf > /chroot/etc/resolv.conf +unshare -S 101 -R /chroot nginx "$@" diff --git a/test/data/cleanConf.expected.conf b/test/data/cleanConf.expected.conf index 5edad651b8..1666c19f63 100644 --- a/test/data/cleanConf.expected.conf +++ b/test/data/cleanConf.expected.conf @@ -1,7 +1,7 @@ # Configuration checksum: # setup custom paths that do not require root access -pid /tmp/nginx.pid; +pid /tmp/nginx/nginx.pid; daemon off; diff --git a/test/data/cleanConf.src.conf b/test/data/cleanConf.src.conf index 81944e1cec..0e572faa50 100644 --- a/test/data/cleanConf.src.conf +++ b/test/data/cleanConf.src.conf @@ -1,7 +1,7 @@ # Configuration checksum: # setup custom paths that do not require root access -pid /tmp/nginx.pid; +pid /tmp/nginx/nginx.pid; diff --git a/test/e2e-image/namespace-overlays/admission/values.yaml b/test/e2e-image/namespace-overlays/admission/values.yaml index b88e8a02e6..1f0367671e 100644 --- a/test/e2e-image/namespace-overlays/admission/values.yaml +++ b/test/e2e-image/namespace-overlays/admission/values.yaml @@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress controller: image: repository: ingress-controller/controller + chroot: true tag: 1.0.0-dev digest: containerPort: diff --git a/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml b/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml index 57bd1d1058..7507d60dbb 100644 --- a/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml +++ b/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml @@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress controller: image: repository: ingress-controller/controller + chroot: true tag: 1.0.0-dev digest: extraArgs: diff --git a/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml b/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml index 4fef671a76..e0b8003d2c 100644 --- a/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml +++ b/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml @@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress controller: image: repository: ingress-controller/controller + chroot: true tag: 1.0.0-dev digest: containerPort: diff --git a/test/e2e-image/namespace-overlays/namespace-selector/values.yaml b/test/e2e-image/namespace-overlays/namespace-selector/values.yaml index e4c0e7a87f..06252da6f9 100644 --- a/test/e2e-image/namespace-overlays/namespace-selector/values.yaml +++ b/test/e2e-image/namespace-overlays/namespace-selector/values.yaml @@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress controller: image: repository: ingress-controller/controller + chroot: true tag: 1.0.0-dev digest: containerPort: diff --git a/test/e2e/framework/logs.go b/test/e2e/framework/logs.go index a4b645695d..db3eb4a6ca 100644 --- a/test/e2e/framework/logs.go +++ b/test/e2e/framework/logs.go @@ -18,12 +18,15 @@ package framework import ( "context" + "time" "k8s.io/client-go/kubernetes" ) // Logs returns the log entries of a given Pod. func Logs(client kubernetes.Interface, namespace, podName string) (string, error) { + // Logs from jails take a bigger time to get shipped due to the need of tailing them + Sleep(3 * time.Second) logs, err := client.CoreV1().RESTClient().Get(). Resource("pods"). Namespace(namespace). diff --git a/test/e2e/run.sh b/test/e2e/run.sh index bfe13b428f..1aa41e1c6a 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -58,7 +58,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}" if [ "${SKIP_CLUSTER_CREATION:-false}" = "false" ]; then echo "[dev-env] creating Kubernetes cluster with kind" - export K8S_VERSION=${K8S_VERSION:-v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6} + export K8S_VERSION=${K8S_VERSION:-v1.21.10@sha256:84709f09756ba4f863769bdcabe5edafc2ada72d3c8c44d6515fc581b66b029c} kind create cluster \ --verbosity=${KIND_LOG_LEVEL} \ @@ -77,7 +77,7 @@ if [ "${SKIP_IMAGE_CREATION:-false}" = "false" ]; then fi echo "[dev-env] building image" - make -C ${DIR}/../../ clean-image build image + make -C ${DIR}/../../ clean-image build image image-chroot make -C ${DIR}/../e2e-image image fi @@ -87,6 +87,11 @@ KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | grep worker | awk echo "[dev-env] copying docker images to cluster..." kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} nginx-ingress-controller:e2e + +if [ "${IS_CHROOT:-false}" = "true" ]; then + docker tag ${REGISTRY}/controller-chroot:${TAG} ${REGISTRY}/controller:${TAG} +fi + kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} ${REGISTRY}/controller:${TAG} echo "[dev-env] running e2e tests..." diff --git a/test/e2e/settings/access_log.go b/test/e2e/settings/access_log.go index 0e4c1d8279..0e50205e5e 100644 --- a/test/e2e/settings/access_log.go +++ b/test/e2e/settings/access_log.go @@ -31,17 +31,19 @@ var _ = framework.DescribeSetting("access-log", func() { ginkgo.It("use the default configuration", func() { f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") && - strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream") + return (strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") && + strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream")) || + (strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo") && + strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 log_stream")) }) }) ginkgo.It("use the specified configuration", func() { - f.UpdateNginxConfigMapData("access-log-path", "/tmp/access.log") + f.UpdateNginxConfigMapData("access-log-path", "/tmp/nginx/access.log") f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, "access_log /tmp/access.log upstreaminfo") && - strings.Contains(cfg, "access_log /tmp/access.log log_stream") + return strings.Contains(cfg, "access_log /tmp/nginx/access.log upstreaminfo") && + strings.Contains(cfg, "access_log /tmp/nginx/access.log log_stream") }) }) }) @@ -49,11 +51,12 @@ var _ = framework.DescribeSetting("access-log", func() { ginkgo.Context("http-access-log-path", func() { ginkgo.It("use the specified configuration", func() { - f.UpdateNginxConfigMapData("http-access-log-path", "/tmp/http-access.log") + f.UpdateNginxConfigMapData("http-access-log-path", "/tmp/nginx/http-access.log") f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, "access_log /tmp/http-access.log upstreaminfo") && - strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream") + return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") && + (strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream") || + strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 log_stream")) }) }) }) @@ -61,11 +64,12 @@ var _ = framework.DescribeSetting("access-log", func() { ginkgo.Context("stream-access-log-path", func() { ginkgo.It("use the specified configuration", func() { - f.UpdateNginxConfigMapData("stream-access-log-path", "/tmp/stream-access.log") + f.UpdateNginxConfigMapData("stream-access-log-path", "/tmp/nginx/stream-access.log") f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, "access_log /tmp/stream-access.log log_stream") && - strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") + return strings.Contains(cfg, "access_log /tmp/nginx/stream-access.log log_stream") && + (strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") || + strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo")) }) }) }) @@ -74,13 +78,13 @@ var _ = framework.DescribeSetting("access-log", func() { ginkgo.It("use the specified configuration", func() { f.SetNginxConfigMapData(map[string]string{ - "http-access-log-path": "/tmp/http-access.log", - "stream-access-log-path": "/tmp/stream-access.log", + "http-access-log-path": "/tmp/nginx/http-access.log", + "stream-access-log-path": "/tmp/nginx/stream-access.log", }) f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, "access_log /tmp/http-access.log upstreaminfo") && - strings.Contains(cfg, "access_log /tmp/stream-access.log log_stream") + return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") && + strings.Contains(cfg, "access_log /tmp/nginx/stream-access.log log_stream") }) }) }) diff --git a/test/e2e/settings/pod_security_policy_volumes.go b/test/e2e/settings/pod_security_policy_volumes.go index da88ed5be7..384f8c46c1 100644 --- a/test/e2e/settings/pod_security_policy_volumes.go +++ b/test/e2e/settings/pod_security_policy_volumes.go @@ -82,10 +82,10 @@ var _ = framework.IngressNginxDescribe("[Security] Pod Security Policies with vo deployment.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ { - Name: "ssl", MountPath: "/etc/ingress-controller", + Name: "ssl", MountPath: "/etc/my-amazing-ssl", }, { - Name: "tmp", MountPath: "/tmp", + Name: "tmp", MountPath: "/my-other-tmp", }, } diff --git a/test/e2e/wait-for-nginx.sh b/test/e2e/wait-for-nginx.sh index 9a37d1ffcf..ed2e044264 100755 --- a/test/e2e/wait-for-nginx.sh +++ b/test/e2e/wait-for-nginx.sh @@ -59,6 +59,7 @@ fullnameOverride: nginx-ingress controller: image: repository: ingress-controller/controller + chroot: true tag: 1.0.0-dev digest: scope: