[release-1.22] Backport fixes/bumps from master

go.mod

@@ -99,28 +99,28 @@ require (
 	github.com/gorilla/websocket v1.4.2
 	github.com/k3s-io/helm-controller v0.12.3
 	github.com/k3s-io/kine v0.9.3
-	github.com/klauspost/compress v1.15.1
+	github.com/klauspost/compress v1.15.9
 	github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
 	github.com/lib/pq v1.10.2
 	github.com/mattn/go-sqlite3 v1.14.8
-	github.com/minio/minio-go/v7 v7.0.7
+	github.com/minio/minio-go/v7 v7.0.33
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/onsi/ginkgo/v2 v2.1.1
 	github.com/onsi/gomega v1.17.0
 	github.com/opencontainers/runc v1.1.0
 	github.com/opencontainers/selinux v1.10.0
 	github.com/otiai10/copy v1.7.0
 	github.com/pkg/errors v0.9.1
-	github.com/rancher/dynamiclistener v0.3.3
+	github.com/rancher/dynamiclistener v0.3.4-0.20220721210816-8ebd77f8a45a
 	github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08
 	github.com/rancher/remotedialer v0.2.6-0.20220624190122-ea57207bf2b8
 	github.com/rancher/wharfie v0.5.1
 	github.com/rancher/wrangler v0.8.10
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/rootless-containers/rootlesskit v0.14.5
-	github.com/sirupsen/logrus v1.8.1
+	github.com/rootless-containers/rootlesskit v1.0.1
+	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
 	github.com/urfave/cli v1.22.9
 	github.com/vishvananda/netlink v1.2.1-beta.2
@@ -131,10 +131,10 @@ require (
 	go.etcd.io/etcd/etcdutl/v3 v3.5.4
 	go.etcd.io/etcd/server/v3 v3.5.4
 	go.uber.org/zap v1.19.0
-	golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
-	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
+	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f
 	google.golang.org/grpc v1.45.0
 	gopkg.in/yaml.v2 v2.4.0
 	inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252

install.sh

@@ -217,11 +217,7 @@ setup_env() {
     if [ -n "${INSTALL_K3S_TYPE}" ]; then
         SYSTEMD_TYPE=${INSTALL_K3S_TYPE}
     else
-        if [ "${CMD_K3S}" = server ]; then
-            SYSTEMD_TYPE=notify
-        else
-            SYSTEMD_TYPE=exec
-        fi
+        SYSTEMD_TYPE=notify
     fi
 
     # --- use binary install directory if defined or create default ---

pkg/agent/containerd/config_linux.go

@@ -52,7 +52,10 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 	if disableCgroup {
 		logrus.Warn("cgroup v2 controllers are not delegated for rootless. Disabling cgroup.")
 	} else {
-		cfg.AgentConfig.Systemd = controllers["cpuset"] && os.Getenv("NOTIFY_SOCKET") != ""
+		// note: this mutatation of the passed agent.Config is later used to set the
+		// kubelet's cgroup-driver flag. This may merit moving to somewhere else in order
+		// to avoid mutating the configuration while setting up containerd.
+		cfg.AgentConfig.Systemd = !isRunningInUserNS && controllers["cpuset"] && os.Getenv("INVOCATION_ID") != ""
 	}
 
 	var containerdTemplate string

pkg/cli/server/server.go

@@ -468,7 +468,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		systemd.SdNotify(true, "READY=1\n")
 	}()
 
-	url := fmt.Sprintf("https://%s:%d", serverConfig.ControlConfig.BindAddressOrLoopback(false), serverConfig.ControlConfig.SupervisorPort)
+	url := fmt.Sprintf("https://%s:%d", serverConfig.ControlConfig.BindAddressOrLoopback(false, true), serverConfig.ControlConfig.SupervisorPort)
 	token, err := clientaccess.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA)
 	if err != nil {
 		return err

pkg/daemons/config/types.go

@@ -215,32 +215,36 @@ type Control struct {
 	Runtime     *ControlRuntime `json:"-"`
 }
 
-// BindAddressOrLoopback returns an IPv4 or IPv6 address suitable for embedding in server
-// URLs. If a bind address was configured, that is returned. If the chooseHostInterface
-// parameter is true, and a suitable default interface can be found, that interface's
-// address is returned.  If neither of the previous were used, the loopback address is
-// returned. IPv6 addresses are enclosed in square brackets, as per RFC2732.
-func (c *Control) BindAddressOrLoopback(chooseHostInterface bool) string {
+// BindAddressOrLoopback returns an IPv4 or IPv6 address suitable for embedding in
+// server URLs. If a bind address was configured, that is returned. If the
+// chooseHostInterface parameter is true, and a suitable default interface can be
+// found, that interface's address is returned.  If neither of the previous were used,
+// the loopback address is returned. If the urlSafe parameter is true, IPv6 addresses
+// are enclosed in square brackets, as per RFC2732.
+func (c *Control) BindAddressOrLoopback(chooseHostInterface, urlSafe bool) string {
 	ip := c.BindAddress
 	if ip == "" && chooseHostInterface {
 		if hostIP, _ := utilnet.ChooseHostInterface(); len(hostIP) > 0 {
 			ip = hostIP.String()
 		}
 	}
-	if utilsnet.IsIPv6String(ip) {
+	if urlSafe && utilsnet.IsIPv6String(ip) {
 		return fmt.Sprintf("[%s]", ip)
 	} else if ip != "" {
 		return ip
 	}
-	return c.Loopback()
+	return c.Loopback(urlSafe)
 }
 
 // Loopback returns an IPv4 or IPv6 loopback address, depending on whether the cluster
-// service CIDRs indicate an IPv4/Dual-Stack or IPv6 only cluster.  IPv6 addresses are
-// enclosed in square brackets, as per RFC2732.
-func (c *Control) Loopback() string {
+// service CIDRs indicate an IPv4/Dual-Stack or IPv6 only cluster. If the urlSafe
+// parameter is true, IPv6 addresses are enclosed in square brackets, as per RFC2732.
+func (c *Control) Loopback(urlSafe bool) string {
 	if IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(c.ServiceIPRanges); IPv6OnlyService {
-		return "[::1]"
+		if urlSafe {
+			return "[::1]"
+		}
+		return "::1"
 	}
 	return "127.0.0.1"
 }

pkg/daemons/control/deps/deps.go

@@ -314,7 +314,7 @@ func genClientCerts(config *config.Control) error {
 
 	var certGen bool
 
-	apiEndpoint := fmt.Sprintf("https://%s:%d", config.Loopback(), config.APIServerPort)
+	apiEndpoint := fmt.Sprintf("https://%s:%d", config.Loopback(true), config.APIServerPort)
 
 	certGen, err = factory("system:admin", []string{user.SystemPrivilegedGroup}, runtime.ClientAdminCert, runtime.ClientAdminKey)
 	if err != nil {
@@ -734,7 +734,7 @@ func genEgressSelectorConfig(controlConfig *config.Control) error {
 			ProxyProtocol: apiserver.ProtocolHTTPConnect,
 			Transport: &apiserver.Transport{
 				TCP: &apiserver.TCPTransport{
-					URL: fmt.Sprintf("https://%s:%d", controlConfig.BindAddressOrLoopback(false), controlConfig.SupervisorPort),
+					URL: fmt.Sprintf("https://%s:%d", controlConfig.BindAddressOrLoopback(false, true), controlConfig.SupervisorPort),
 					TLSConfig: &apiserver.TLSConfig{
 						CABundle:   controlConfig.Runtime.ServerCA,
 						ClientKey:  controlConfig.Runtime.ClientKubeAPIKey,

pkg/daemons/control/server.go

@@ -31,14 +31,6 @@ import (
 	_ "k8s.io/component-base/metrics/prometheus/restclient"
 )
 
-func getLocalhostIP(serviceCIDR []*net.IPNet) net.IP {
-	IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(serviceCIDR)
-	if IPv6OnlyService {
-		return net.ParseIP("::1")
-	}
-	return net.ParseIP("127.0.0.1")
-}
-
 func Server(ctx context.Context, cfg *config.Control) error {
 	rand.Seed(time.Now().UTC().UnixNano())
 
@@ -107,10 +99,11 @@ func controllerManager(ctx context.Context, cfg *config.Control) error {
 		"authentication-kubeconfig":        runtime.KubeConfigController,
 		"service-account-private-key-file": runtime.ServiceKey,
 		"allocate-node-cidrs":              "true",
+		"service-cluster-ip-range":         util.JoinIPNets(cfg.ServiceIPRanges),
 		"cluster-cidr":                     util.JoinIPNets(cfg.ClusterIPRanges),
 		"root-ca-file":                     runtime.ServerCA,
 		"profiling":                        "false",
-		"bind-address":                     getLocalhostIP(cfg.ServiceIPRanges).String(),
+		"bind-address":                     cfg.Loopback(false),
 		"secure-port":                      "10257",
 		"use-service-account-credentials":  "true",
 		"cluster-signing-kube-apiserver-client-cert-file": runtime.ClientCA,
@@ -142,7 +135,7 @@ func scheduler(ctx context.Context, cfg *config.Control) error {
 		"kubeconfig":                runtime.KubeConfigScheduler,
 		"authorization-kubeconfig":  runtime.KubeConfigScheduler,
 		"authentication-kubeconfig": runtime.KubeConfigScheduler,
-		"bind-address":              getLocalhostIP(cfg.ServiceIPRanges).String(),
+		"bind-address":              cfg.Loopback(false),
 		"secure-port":               "10259",
 		"profiling":                 "false",
 	}
@@ -179,7 +172,7 @@ func apiServer(ctx context.Context, cfg *config.Control) error {
 	argsMap["insecure-port"] = "0"
 	argsMap["secure-port"] = strconv.Itoa(cfg.APIServerPort)
 	if cfg.APIServerBindAddress == "" {
-		argsMap["bind-address"] = getLocalhostIP(cfg.ServiceIPRanges).String()
+		argsMap["bind-address"] = cfg.Loopback(false)
 	} else {
 		argsMap["bind-address"] = cfg.APIServerBindAddress
 	}
@@ -316,7 +309,7 @@ func cloudControllerManager(ctx context.Context, cfg *config.Control) error {
 		"authorization-kubeconfig":     runtime.KubeConfigCloudController,
 		"authentication-kubeconfig":    runtime.KubeConfigCloudController,
 		"node-status-update-frequency": "1m0s",
-		"bind-address":                 getLocalhostIP(cfg.ServiceIPRanges).String(),
+		"bind-address":                 cfg.Loopback(false),
 		"port":                         "0",
 	}
 	if cfg.NoLeaderElect {

pkg/daemons/control/tunnel.go

@@ -210,7 +210,7 @@ func (t *TunnelServer) dialBackend(ctx context.Context, addr string) (net.Conn,
 	if err != nil {
 		return nil, err
 	}
-	loopback := t.config.Loopback()
+	loopback := t.config.Loopback(true)
 
 	var nodeName string
 	var toKubelet, useTunnel bool

pkg/etcd/etcd.go

@@ -53,7 +53,7 @@ import (
 )
 
 const (
-	testTimeout          = time.Second * 10
+	testTimeout          = time.Second * 30
 	manageTickerTime     = time.Second * 15
 	learnerMaxStallTime  = time.Minute * 5
 	memberRemovalTimeout = time.Minute * 1
@@ -653,7 +653,7 @@ func getEndpoints(control *config.Control) []string {
 	if len(runtime.EtcdConfig.Endpoints) > 0 {
 		return runtime.EtcdConfig.Endpoints
 	}
-	return []string{fmt.Sprintf("https://%s:2379", control.Loopback())}
+	return []string{fmt.Sprintf("https://%s:2379", control.Loopback(true))}
 }
 
 // toTLSConfig converts the ControlRuntime configuration to TLS configuration suitable
@@ -769,7 +769,7 @@ func (e *ETCD) peerURL() string {
 // During cluster reset/restore, we only listen on loopback to avoid having peers
 // connect mid-process.
 func (e *ETCD) listenPeerURLs(reset bool) string {
-	peerURLs := fmt.Sprintf("https://%s:2380", e.config.Loopback())
+	peerURLs := fmt.Sprintf("https://%s:2380", e.config.Loopback(true))
 	if !reset {
 		peerURLs += "," + e.peerURL()
 	}
@@ -785,7 +785,7 @@ func (e *ETCD) clientURL() string {
 // During cluster reset/restore, we only listen on loopback to avoid having the apiserver
 // connect mid-process.
 func (e *ETCD) listenClientURLs(reset bool) string {
-	clientURLs := fmt.Sprintf("https://%s:2379", e.config.Loopback())
+	clientURLs := fmt.Sprintf("https://%s:2379", e.config.Loopback(true))
 	if !reset {
 		clientURLs += "," + e.clientURL()
 	}
@@ -794,7 +794,7 @@ func (e *ETCD) listenClientURLs(reset bool) string {
 
 // listenMetricsURLs returns a list of URLs to bind to for metrics connections.
 func (e *ETCD) listenMetricsURLs(reset bool) string {
-	metricsURLs := fmt.Sprintf("http://%s:2381", e.config.Loopback())
+	metricsURLs := fmt.Sprintf("http://%s:2381", e.config.Loopback(true))
 	if !reset && e.config.EtcdExposeMetrics {
 		metricsURLs += "," + fmt.Sprintf("http://%s", net.JoinHostPort(e.address, "2381"))
 	}
@@ -933,7 +933,7 @@ func (e *ETCD) manageLearners(ctx context.Context) {
 	defer t.Stop()
 
 	for range t.C {
-		ctx, cancel := context.WithTimeout(ctx, testTimeout)
+		ctx, cancel := context.WithTimeout(ctx, manageTickerTime)
 		defer cancel()
 
 		// Check to see if the local node is the leader. Only the leader should do learner management.

pkg/server/server.go

@@ -313,14 +313,11 @@ func HomeKubeConfig(write, rootless bool) (string, error) {
 }
 
 func printTokens(config *config.Control) error {
-	var (
-		nodeFile string
-	)
-	if len(config.Runtime.ServerToken) > 0 {
-		p := filepath.Join(config.DataDir, "token")
-		if err := writeToken(config.Runtime.ServerToken, p, config.Runtime.ServerCA); err == nil {
-			logrus.Infof("Node token is available at %s", p)
-			nodeFile = p
+	var serverTokenFile string
+	if config.Runtime.ServerToken != "" {
+		serverTokenFile = filepath.Join(config.DataDir, "token")
+		if err := writeToken(config.Runtime.ServerToken, serverTokenFile, config.Runtime.ServerCA); err != nil {
+			return err
 		}
 
 		// backwards compatibility
@@ -329,25 +326,54 @@ func printTokens(config *config.Control) error {
 			if err := os.RemoveAll(np); err != nil {
 				return err
 			}
-			if err := os.Symlink(p, np); err != nil {
+			if err := os.Symlink(serverTokenFile, np); err != nil {
 				return err
 			}
 		}
+
+		logrus.Infof("Server node token is available at %s", serverTokenFile)
+		printToken(config.SupervisorPort, config.BindAddressOrLoopback(true, true), "To join server node to cluster:", "server", "SERVER_NODE_TOKEN")
+	}
+
+	var agentTokenFile string
+	if config.Runtime.AgentToken != "" {
+		if config.AgentToken != "" {
+			agentTokenFile = filepath.Join(config.DataDir, "agent-token")
+			if isSymlink(agentTokenFile) {
+				if err := os.RemoveAll(agentTokenFile); err != nil {
+					return err
+				}
+			}
+			if err := writeToken(config.Runtime.AgentToken, agentTokenFile, config.Runtime.ServerCA); err != nil {
+				return err
+			}
+		} else if serverTokenFile != "" {
+			agentTokenFile = filepath.Join(config.DataDir, "agent-token")
+			if !isSymlink(agentTokenFile) {
+				if err := os.RemoveAll(agentTokenFile); err != nil {
+					return err
+				}
+				if err := os.Symlink(serverTokenFile, agentTokenFile); err != nil {
+					return err
+				}
+			}
+		}
 	}
 
-	if len(nodeFile) > 0 {
-		printToken(config.SupervisorPort, config.BindAddressOrLoopback(true), "To join node to cluster:", "agent")
+	if agentTokenFile != "" {
+		logrus.Infof("Agent node token is available at %s", agentTokenFile)
+		printToken(config.SupervisorPort, config.BindAddressOrLoopback(true, true), "To join agent node to cluster:", "agent", "AGENT_NODE_TOKEN")
 	}
 
 	return nil
 }
 
 func writeKubeConfig(certs string, config *Config) error {
-	ip := config.ControlConfig.BindAddressOrLoopback(false)
+	ip := config.ControlConfig.BindAddressOrLoopback(false, true)
 	port := config.ControlConfig.HTTPSPort
 	// on servers without a local apiserver, tunnel access via the loadbalancer
 	if config.ControlConfig.DisableAPIServer {
-		ip = config.ControlConfig.Loopback()
+		ip = config.ControlConfig.Loopback(true)
 		port = config.ControlConfig.APIServerPort
 	}
 	url := fmt.Sprintf("https://%s:%d", ip, port)
@@ -423,8 +449,8 @@ func setupDataDirAndChdir(config *config.Control) error {
 	return nil
 }
 
-func printToken(httpsPort int, advertiseIP, prefix, cmd string) {
-	logrus.Infof("%s %s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, version.Program, cmd, advertiseIP, httpsPort)
+func printToken(httpsPort int, advertiseIP, prefix, cmd, varName string) {
+	logrus.Infof("%s %s %s -s https://%s:%d -t ${%s}", prefix, version.Program, cmd, advertiseIP, httpsPort, varName)
 }
 
 func writeToken(token, file, certs string) error {