feat: transparent buildx support for remote buildkit (#6732)

cmd/skaffold/app/cmd/config/set_test.go

@@ -415,7 +415,7 @@ func TestGetConfigStructWithIndex(t *testing.T) {
 			description: "survey flag set",
 			cfg:         &config.ContextConfig{},
 			survey:      true,
-			expectedIdx: []int{7},
+			expectedIdx: []int{9},
 		},
 		{
 			description: "no survey flag set",

docs-v2/content/en/schemas/v4beta12.json

@@ -1766,6 +1766,18 @@
             "[\"golang:1.10.1-alpine3.7\", \"alpine:3.7\"]"
           ]
         },
+        "cacheTo": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "description": "the Docker images used as cache destination. If omitted, cacheFrom is used with max mode to export all layers.",
+          "x-intellij-html-description": "the Docker images used as cache destination. If omitted, cacheFrom is used with max mode to export all layers.",
+          "default": "[]",
+          "examples": [
+            "[\"type=registry,ref=gcr.io/k8s-skaffold/example:cache,mode=max\"]"
+          ]
+        },
         "cliFlags": {
           "items": {
             "type": "string"
@@ -1830,6 +1842,7 @@
         "network",
         "addHost",
         "cacheFrom",
+        "cacheTo",
         "cliFlags",
         "pullParent",
         "noCache",

integration/build_test.go

@@ -53,6 +53,12 @@ func TestBuild(t *testing.T) {
 			description: "docker build",
 			dir:         "testdata/build",
 		},
+		{
+			setup:       setupBuildX,
+			description: "docker buildx",
+			args:        []string{"--config", "config", "--verbosity=trace"},
+			dir:         "testdata/buildx",
+		},
 		{
 			description: "git tagger",
 			dir:         "testdata/tagPolicy",
@@ -131,6 +137,7 @@ func TestBuildWithWithPlatform(t *testing.T) {
 		dir               string
 		args              []string
 		image             string
+		setup             func(t *testing.T, workdir string)
 		expectedPlatforms []v1.Platform
 	}{
 		{
@@ -145,11 +152,28 @@ func TestBuildWithWithPlatform(t *testing.T) {
 			args:              []string{"--platform", "linux/arm64"},
 			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}},
 		},
+		{
+			setup:             setupBuildX,
+			description:       "docker buildx linux/amd64",
+			dir:               "testdata/buildx",
+			args:              []string{"--platform", "linux/amd64", "--cache-artifacts=false", "--config", "config", "--verbosity=trace"},
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "amd64"}},
+		},
+		{
+			setup:             setupBuildX,
+			description:       "docker buildx linux/arm64",
+			dir:               "testdata/buildx",
+			args:              []string{"--platform", "linux/arm64", "--cache-artifacts=false", "--config", "config", "--verbosity=trace"},
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}},
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
 			MarkIntegrationTest(t, CanRunWithoutGcp)
+			if test.setup != nil {
+				test.setup(t, test.dir)
+			}
 			tmpfile := testutil.TempFile(t, "", []byte{})
 			args := append(test.args, "--file-output", tmpfile)
 			skaffold.Build(args...).InDir(test.dir).RunOrFail(t)
@@ -314,6 +338,35 @@ func setupGitRepo(t *testing.T, dir string) {
 	}
 }
 
+// setupBuildX sets up a docker buildx builder using buildkit
+func setupBuildX(t *testing.T, dir string) {
+	t.Cleanup(func() {
+		dockerArgs := [][]string{
+			{"buildx", "uninstall"},
+			{"buildx", "rm", "buildkit"},
+		}
+		for _, args := range dockerArgs {
+			cmd := exec.Command("docker", args...)
+			if buf, err := util.RunCmdOut(context.Background(), cmd); err != nil {
+				t.Log(string(buf))
+				t.Fatal(err)
+			}
+		}
+	})
+
+	dockerArgs := [][]string{
+		{"buildx", "install"},
+		{"buildx", "create", "--driver", "docker-container", "--name", "buildkit"},
+	}
+	for _, args := range dockerArgs {
+		cmd := exec.Command("docker", args...)
+		if buf, err := util.RunCmdOut(context.Background(), cmd); err != nil {
+			t.Log(string(buf))
+			t.Fatal(err)
+		}
+	}
+}
+
 // nowInChicago returns the dateTime string as generated by the dateTime tagger
 func nowInChicago() string {
 	loc, _ := tz.LoadLocation("America/Chicago")

integration/testdata/buildx/Dockerfile

@@ -0,0 +1,12 @@
+# syntax=docker/dockerfile:1
+
+FROM golang:1.23-alpine AS builder
+
+COPY main.go .
+RUN go build -o /app main.go
+
+FROM alpine:3
+
+COPY --from=builder /app .
+
+CMD ["/app"]

integration/testdata/buildx/config

@@ -0,0 +1,9 @@
+global:
+  cache-tag: cache
+  buildx-builder: buildkit
+  survey:
+    last-prompted: "2025-01-19T21:32:47Z"
+  collect-metrics: true
+  update:
+    last-prompted: "2025-01-19T23:10:47Z"
+kubeContexts: []

integration/testdata/buildx/main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"fmt"
+	"time"
+)
+
+func main() {
+	for counter := 0; ; counter++ {
+		fmt.Println("Hello world!", counter)
+
+		time.Sleep(time.Second * 1)
+	}
+}

integration/testdata/buildx/skaffold.yaml

@@ -0,0 +1,12 @@
+apiVersion: skaffold/v4beta12
+kind: Config
+build:
+  artifacts:
+  - image: my-app
+    docker:
+      dockerfile: Dockerfile
+  local:
+    useBuildkit: true
+    useDockerCLI: true
+    tryImportMissing: true
+    push: false

pkg/skaffold/build/cache/cache.go

@@ -58,6 +58,7 @@ type cache struct {
 	isLocalImage       func(imageName string) (bool, error)
 	importMissingImage func(imageName string) (bool, error)
 	lister             DependencyLister
+	buildx             bool
 }
 
 // DependencyLister fetches a list of dependencies for an artifact
@@ -69,6 +70,7 @@ type Config interface {
 	GetPipelines() []latest.Pipeline
 	DefaultPipeline() latest.Pipeline
 	GetCluster() config.Cluster
+	DetectBuildX() bool
 	CacheArtifacts() bool
 	CacheFile() string
 	Mode() config.RunMode
@@ -118,6 +120,9 @@ func NewCache(ctx context.Context, cfg Config, isLocalImage func(imageName strin
 		return pipeline.Build.LocalBuild.TryImportMissing, nil
 	}
 
+	// for backward compatibility, extended build capabilities with BuildKit are disabled by default
+	buildx := cfg.DetectBuildX() && docker.IsBuildXDetected()
+
 	return &cache{
 		artifactCache:      artifactCache,
 		hashByName:         hashByName,
@@ -129,6 +134,7 @@ func NewCache(ctx context.Context, cfg Config, isLocalImage func(imageName strin
 		isLocalImage:       isLocalImage,
 		importMissingImage: importMissingImage,
 		lister:             dependencies,
+		buildx:             buildx,
 	}, nil
 }
 

pkg/skaffold/build/cache/lookup.go

@@ -87,6 +87,7 @@ func (c *cache) lookup(ctx context.Context, out io.Writer, a *latest.Artifact, t
 	}
 
 	if isLocal, err := c.isLocalImage(a.ImageName); err != nil {
+		log.Entry(ctx).Debugf("isLocalImage failed %v", err)
 		return failed{err}
 	} else if isLocal {
 		return c.lookupLocal(ctx, hash, tag, entry)
@@ -122,9 +123,12 @@ func (c *cache) lookupLocal(ctx context.Context, hash, tag string, entry ImageDe
 func (c *cache) lookupRemote(ctx context.Context, hash, tag string, platforms []specs.Platform, entry ImageDetails) cacheDetails {
 	if remoteDigest, err := docker.RemoteDigest(tag, c.cfg, nil); err == nil {
 		// Image exists remotely with the same tag and digest
+		log.Entry(ctx).Debugf("RemoteDigest: %s entry.Digest %s", remoteDigest, entry.Digest)
 		if remoteDigest == entry.Digest {
 			return found{hash: hash}
 		}
+	} else {
+		log.Entry(ctx).Debugf("RemoteDigest error %v", err)
 	}
 
 	// Image exists remotely with a different tag
@@ -152,23 +156,32 @@ func (c *cache) tryImport(ctx context.Context, a *latest.Artifact, tag string, h
 		return ImageDetails{}, fmt.Errorf("import of missing images disabled")
 	}
 
-	if !c.client.ImageExists(ctx, tag) {
-		log.Entry(ctx).Debugf("Importing artifact %s from docker registry", tag)
-		err := c.client.Pull(ctx, io.Discard, tag, pl)
+	// under buildx, docker daemon is not really needed and could be disabled
+	load := true
+	if c.buildx {
+		_, err := c.client.ServerVersion(ctx)
+		load = err == nil
+		if !load {
+			log.Entry(ctx).Debugf("Docker client error, disabling image load as using buildx: %v", err)
+		}
+	}
+	if load {
+		if !c.client.ImageExists(ctx, tag) {
+			log.Entry(ctx).Debugf("Importing artifact %s from docker registry", tag)
+			err := c.client.Pull(ctx, io.Discard, tag, pl)
+			if err != nil {
+				return entry, err
+			}
+		} else {
+			log.Entry(ctx).Debugf("Importing artifact %s from local docker", tag)
+		}
+		imageID, err := c.client.ImageID(ctx, tag)
 		if err != nil {
 			return entry, err
 		}
-	} else {
-		log.Entry(ctx).Debugf("Importing artifact %s from local docker", tag)
-	}
-
-	imageID, err := c.client.ImageID(ctx, tag)
-	if err != nil {
-		return entry, err
-	}
-
-	if imageID != "" {
-		entry.ID = imageID
+		if imageID != "" {
+			entry.ID = imageID
+		}
 	}
 
 	if digest, err := docker.RemoteDigest(tag, c.cfg, nil); err == nil {