Convert iptables-wrapper to a static go program

Using a static binary removes the need to include sh in the kube-proxy
image. It also drops the dependency on grep and wc.

The binary is about 1.7MB in linux amd64. This will make the resulting
image bigger, although removing dash, grep and wc will compensate for
that.

The Go code is almost a drop-in replacement for the existing sh script.
The only differences are:
* The Go program uses a new process to execute the proxied iptables
  command instead of replacing the current process with exec. It
  redirects stderr and stdout so this should be equivalent.
* iptables binary directory is detected on runtime. The cost should me
  minimum, it just looks for a file in two folders.
* The system iptables selector (alternatives vs update-alternatives) is
  detected in runtime as opposed to during wrapper installation. The
  cost should be minimum, it just inspects the sbin directory for one
  binary or the other.

Co-authored-by: Dan Winship <[email protected]>

.gitignore

@@ -0,0 +1,2 @@
+bin
+.vscode

Makefile

@@ -1,18 +1,36 @@
-all:
+BIN_DIR ?= bin
+GO ?= go
+
+all: fmt vet check
+
+$(BIN_DIR):
+	mkdir -p $(BIN_DIR)
+
+build: $(BIN_DIR)
+	CGO_ENABLED=0 $(GO) build -ldflags='-s -w -extldflags="-static" -buildid=""' -trimpath -o $(BIN_DIR)/iptables-wrapper github.com/kubernetes-sigs/iptables-wrappers
+
+vet: ## Run go vet against code.
+	$(GO) vet ./...
+
+fmt: ## Check formatting
+	if [ "$$(gofmt -e -l . | tee /dev/tty | wc -l)" -gt 0 ]; then \
+		echo "Go files need formatting"; \
+    	exit 1; \
+	fi
 
 check: check-debian check-debian-nosanity check-debian-backports check-fedora check-alpine
 
-check-debian:
+check-debian: build
 	./test/run-test.sh --build-fail debian
 
-check-debian-nosanity:
+check-debian-nosanity: build
 	./test/run-test.sh --build-arg="INSTALL_ARGS=--no-sanity-check" --nft-fail debian-nosanity
 
-check-debian-backports:
+check-debian-backports: build
 	./test/run-test.sh --build-arg="REPO=buster-backports" debian-backports
 
-check-fedora:
+check-fedora: build
 	./test/run-test.sh fedora
 
-check-alpine:
+check-alpine: build
 	./test/run-test.sh alpine

README.md

@@ -8,6 +8,12 @@ modes of iptables 1.8 ("legacy" and "nft") at runtime, so that
 hostNetwork containers that examine or modify iptables rules will work
 correctly regardless of which mode the underlying system is using.
 
+This wrapper is only compatible with Kubernetes 1.17 and newer versions.
+If you need to support older releases, then use the original shell-script
+version of iptables-wrappers, from the `[v2]` tag in this repository.
+
+[v2]: https://github.com/kubernetes-sigs/iptables-wrappers/tree/v2
+
 ## Background
 
 As of iptables 1.8, the iptables command line clients come in two
@@ -58,7 +64,7 @@ Kubernetes.
 ## iptables-wrapper
 
 The `iptables-wrapper-installer.sh` script in this repo will install
-an `iptables-wrapper` script alongside `iptables-legacy` and
+an `iptables-wrapper` binary alongside `iptables-legacy` and
 `iptables-nft` in `/usr/sbin` (or `/sbin`), and adjust the symlinks on
 `iptables`, `iptables-save`, etc, to point to the wrapper.
 
@@ -80,8 +86,8 @@ When building a container image that needs to run iptables in the host
 network namespace, install iptables 1.8.4 or later in the container
 using whatever tools you normally would. Then copy the
 [`iptables-wrapper-installer.sh`](./iptables-wrapper-installer.sh)
-script into your container, and run it to have it set up run-time
-autodetection.
+script alongside the compiled `iptables-wrapper` binary into your
+container, and run it to have it set up run-time autodetection.
 
 Some distro-specific examples:
 
@@ -93,6 +99,7 @@ Some distro-specific examples:
 
       RUN apk add --no-cache iptables
       COPY iptables-wrapper-installer.sh /
+      COPY bin/iptables-wrapper /
       RUN /iptables-wrapper-installer.sh
 
 - Debian GNU/Linux
@@ -107,6 +114,7 @@ Some distro-specific examples:
           apt-get -t buster-backports -y --no-install-recommends install iptables
 
       COPY iptables-wrapper-installer.sh /
+      COPY bin/iptables-wrapper /
       RUN /iptables-wrapper-installer.sh
 
 - Fedora
@@ -118,6 +126,7 @@ Some distro-specific examples:
       RUN dnf install -y iptables iptables-legacy iptables-nft
 
       COPY iptables-wrapper-installer.sh /
+      COPY bin/iptables-wrapper /
       RUN /iptables-wrapper-installer.sh
 
 - RHEL / CentOS / UBI

go.mod

@@ -0,0 +1,3 @@
+module github.com/kubernetes-sigs/iptables-wrappers
+
+go 1.19

internal/commands/run.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+)
+
+// RunAndReadError runs a exec.Cmd and tries to extract the error message from stderr
+// if present and it includes it in the returned error. This overrides the Stderr in cmd if
+// present.
+func RunAndReadError(cmd *exec.Cmd) error {
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+
+	if err := cmd.Run(); err != nil {
+		if stderr.Len() > 0 {
+			err = fmt.Errorf("%s: %v", stderr.String(), err)
+		}
+
+		return err
+	}
+
+	return nil
+}

internal/files/exist.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package files
+
+import "os"
+
+// ExecutableExists checks if a file exists and it's executable by someone.
+func ExecutableExists(path string) bool {
+	stat, err := os.Stat(path)
+	return err == nil && stat.Mode()&0o111 != 0
+}

internal/iptables/alternatives.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iptables
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/kubernetes-sigs/iptables-wrappers/internal/commands"
+	"github.com/kubernetes-sigs/iptables-wrappers/internal/files"
+)
+
+// AlternativeSelector allows to configure a system to use iptables in
+// nft or legacy mode.
+type AlternativeSelector interface {
+	// UseMode configures the system to use the selected iptables mode.
+	UseMode(ctx context.Context, mode Mode) error
+}
+
+// BuildAlternativeSelector builds the proper iptablesAlternativeSelector depending
+// on the machine's setup. It will use either `alternatives` or `update-alternatives` if present
+// in the sbin folder. If none is present, it will manage iptables binaries by manually
+// creating symlinks.
+func BuildAlternativeSelector(sbinPath string) AlternativeSelector {
+	if files.ExecutableExists(filepath.Join(sbinPath, "alternatives")) {
+		return alternativesSelector{sbinPath: sbinPath}
+	} else if files.ExecutableExists(filepath.Join(sbinPath, "update-alternatives")) {
+		return updateAlternativesSelector{sbinPath: sbinPath}
+	} else {
+		// if we don't find any tool to managed the alternatives, handle it manually with symlinks
+		return symlinkSelector{sbinPath: sbinPath}
+	}
+}
+
+// updateAlternativesSelector manages an iptables setup by using the `update-alternatives` binary.
+// This is most common for debian based OSs.
+type updateAlternativesSelector struct {
+	sbinPath string
+}
+
+func (u updateAlternativesSelector) UseMode(ctx context.Context, mode Mode) error {
+	modeStr := string(mode)
+
+	if err := commands.RunAndReadError(exec.CommandContext(ctx, "update-alternatives", "--set", "iptables", filepath.Join(u.sbinPath, "iptables-"+modeStr))); err != nil {
+		return fmt.Errorf("update-alternatives iptables to mode %s: %v", modeStr, err)
+	}
+
+	if err := commands.RunAndReadError(exec.CommandContext(ctx, "update-alternatives", "--set", "ip6tables", filepath.Join(u.sbinPath, "ip6tables-"+modeStr))); err != nil {
+		return fmt.Errorf("update-alternatives ip6tables to mode %s: %v", modeStr, err)
+	}
+
+	return nil
+}
+
+// alternativesSelector manages an iptables setup by using the `alternatives` binary.
+// This is most common for fedora based OSs.
+type alternativesSelector struct {
+	sbinPath string
+}
+
+func (a alternativesSelector) UseMode(ctx context.Context, mode Mode) error {
+	if err := commands.RunAndReadError(exec.CommandContext(ctx, "alternatives", "--set", "iptables", filepath.Join(a.sbinPath, "iptables-"+string(mode)))); err != nil {
+		return fmt.Errorf("alternatives to update iptables to mode %s: %v", string(mode), err)
+	}
+	return nil
+}
+
+// symlinkSelector  manages an iptables setup by manually creating symlinks
+// that point to the proper "mode" binaries.
+// It configures: `iptables`, `iptables-save`, `iptables-restore`,
+// `ip6tables`, `ip6tables-save` and `ip6tables-restore`.
+type symlinkSelector struct {
+	sbinPath string
+}
+
+func (s symlinkSelector) UseMode(ctx context.Context, mode Mode) error {
+	modeStr := string(mode)
+	xtablesForModePath := XtablesPath(s.sbinPath, mode)
+	cmds := []string{"iptables", "iptables-save", "iptables-restore", "ip6tables", "ip6tables-save", "ip6tables-restore"}
+
+	for _, cmd := range cmds {
+		cmdPath := filepath.Join(s.sbinPath, cmd)
+		// If deleting fails, ignore it and try to create symlink regardless
+		_ = os.RemoveAll(cmdPath)
+
+		if err := os.Symlink(xtablesForModePath, cmdPath); err != nil {
+			return fmt.Errorf("creating %s symlink for mode %s: %v", cmd, modeStr, err)
+		}
+	}
+
+	return nil
+}

internal/iptables/detect.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iptables
+
+import (
+	"bytes"
+	"context"
+	"errors"
+
+	"github.com/kubernetes-sigs/iptables-wrappers/internal/files"
+)
+
+// DetectBinaryDir tries to detect the `iptables` location in
+// either /usr/sbin or /sbin. If it's not there, it returns an error.
+func DetectBinaryDir() (string, error) {
+	if files.ExecutableExists("/usr/sbin/iptables") {
+		return "/usr/sbin", nil
+	} else if files.ExecutableExists("/sbin/iptables") {
+		return "/sbin", nil
+	} else {
+		return "", errors.New("iptables is not present in either /usr/sbin or /sbin")
+	}
+}
+
+// Mode represents the two different modes iptables can be
+// configured to: nft or legacy. In string form it can be used to
+// to complete all `iptables-*` commands.
+type Mode string
+
+const (
+	legacy Mode = "legacy"
+	nft    Mode = "nft"
+)
+
+// DetectMode inspects the current iptables entries and tries to
+// guess which iptables mode is being used: legacy or nft
+func DetectMode(ctx context.Context, iptables Installation) Mode {
+	// This method ignores all errors, this is on purpose. We execute all commands
+	// and try to detect patterns in a best effort basis. If somthing fails,
+	// continue with the next step. Worse case scenario if everything fails,
+	// default to nft.
+
+	// In kubernetes 1.17 and later, kubelet will have created at least
+	// one chain in the "mangle" table (either "KUBE-IPTABLES-HINT" or
+	// "KUBE-KUBELET-CANARY"), so check that first, against
+	// iptables-nft, because we can check that more efficiently and
+	// it's more common these days.
+	rulesOutput := &bytes.Buffer{}
+	_ = iptables.NFTSave(ctx, rulesOutput, "-t", "mangle")
+	if hasKubeletChains(rulesOutput.Bytes()) {
+		return nft
+	}
+	rulesOutput.Reset()
+	_ = iptables.NFTSaveIP6(ctx, rulesOutput, "-t", "mangle")
+	if hasKubeletChains(rulesOutput.Bytes()) {
+		return nft
+	}
+	rulesOutput.Reset()
+
+	// Check for kubernetes 1.17-or-later with iptables-legacy. We
+	// can't pass "-t mangle" to iptables-legacy-save because it would
+	// cause the kernel to create that table if it didn't already
+	// exist, which we don't want. So we have to grab all the rules.
+	_ = iptables.LegacySave(ctx, rulesOutput)
+	if hasKubeletChains(rulesOutput.Bytes()) {
+		return legacy
+	}
+	rulesOutput.Reset()
+	_ = iptables.LegacySaveIP6(ctx, rulesOutput)
+	if hasKubeletChains(rulesOutput.Bytes()) {
+		return legacy
+	}
+
+	// If we can't detect any of the 2 patterns, default to nft.
+	return nft
+}

internal/iptables/rules.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iptables
+
+import "regexp"
+
+var (
+	kubeletChainsRegex = regexp.MustCompile(`(?m)^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)`)
+	ruleEntryRegex     = regexp.MustCompile(`(?m)^-`)
+)
+
+// hasKubeletChains checks if the output of an iptables*-save command
+// contains any of the rules set by kubelet.
+func hasKubeletChains(output []byte) bool {
+	return kubeletChainsRegex.Match(output)
+}
+
+// ruleEntriesNum counts how many rules there are in an iptables*-save command
+// output.
+func ruleEntriesNum(iptablesOutput []byte) int {
+	return len(ruleEntryRegex.FindAllIndex(iptablesOutput, -1))
+}