Share snapshots with host containerd

.github/README.md

@@ -0,0 +1,65 @@
+# BuildKit
+
+BuildKit is an awesome project to build oci images.
+
+This project adds features to help buildkit use host containerd as its worker, 
+then it is able to share images and snapshots with host containerd, and export new images to host containerd.
+
+With these features, users can deploy the buildkit container to a single-node minikube cluster,
+and use containerd as its worker. See [kubectl-dev](https://github.com/warm-metal/kubectl-dev) for more details.
+
+The privileged image is available on [DockerHub](https://hub.docker.com/r/warmmetal/buildkit). Fell free to test.
+
+## Installation
+
+Run the commands below to install buildkit in a k8s cluster.
+```shell script
+kubectl create ns buildkit-system
+kubectl -n buildkit-system create cm buildkitd.toml --from-file=install/buildkitd.toml
+kubectl apply -f install/buildkit.yaml
+```
+
+Buildkit exposes its service on tcp port 2375. Users can use it via `buildctl` or other compatible clients.
+
+## Features
+
+- [Sharing volumes between buildkit and host containerd](#sharing-volumes-between-buildkit-and-host-containerd)
+- ~~[HTTP_PROXY support in image building](#http_proxy-support-in-image-building)~~
+- [Multi-Arch image building](#multi-arch-image-building)
+
+#### Sharing volumes between buildkit and host containerd
+
+Both buildkit and containerd need to access snapshots created by each other.
+Therefore, we must mount the containerd root to the buildkit pod and also bind the buildkit root to the host.
+
+And, the official buildkit saves contents of context, secrets and static qemu emulator in temporary folders that are
+only visible in the pod whiling mounting them. These folders should also be available to the host containerd.
+So, we introduced a new configuration `local-mount-source-root` to indicate a directory instead of `/tmp` as the parent folder of these contents.
+This directory should be mounted to the same host path.
+
+#### Multi-Arch image building
+
+The official buildkit image has qemu-static binaries included to support multi-arch image building.
+If users has interpreters installed on the worker node for different architectures via [`binfmt_misc`](https://www.kernel.org/doc/html/latest/admin-guide/binfmt-misc.html),
+the particular interpreter will be used to execute `RUN` directives on the corresponding architecture.
+Otherwise, the built-in binaries will be used instead.
+
+Unfortunately, the built-in binaries won't work well on all cases. It may throws strange errors sometimes.
+For example, running `apk add bash` may throws the following error.
+```shell script
+ > [2/2] RUN apk add --no-cache bash:
+#5 0.347 fetch https://dl-cdn.alpinelinux.org/alpine/v3.13/main/armv7/APKINDEX.tar.gz
+#5 1.211 fetch https://dl-cdn.alpinelinux.org/alpine/v3.13/community/armv7/APKINDEX.tar.gz
+#5 2.136 (1/4) Installing ncurses-terminfo-base (6.2_p20210109-r0)
+#5 2.220 (2/4) Installing ncurses-libs (6.2_p20210109-r0)
+#5 2.308 (3/4) Installing readline (8.1.0-r0)
+#5 2.380 (4/4) Installing bash (5.1.0-r0)
+#5 2.494 Executing bash-5.1.0-r0.post-install
+#5 2.500 ERROR: bash-5.1.0-r0.post-install: script exited with error 1
+#5 2.502 Executing busybox-1.32.1-r6.trigger
+#5 2.550 1 error; 5 MiB in 18 packages
+```
+To fix this kind of failure, users need to install interpreters(like Docker did) via `binfmt_misc`,
+or update the built-in qemu-static binaries.
+Both https://github.com/tonistiigi/binfmt and https://github.com/multiarch/qemu-user-static can help.
+I would recommend the former repo.

.github/workflows/build-image.yml

@@ -0,0 +1,47 @@
+name: build-image
+
+on:
+  issue_comment:
+    types: ['created']
+
+jobs:
+  build-image:
+    # github.event.issue.pull_request && 
+    if: ${{ github.event.comment.body == '/build' && startsWith(github.event.issue.title, 'Release ') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build image
+        id: image
+        run: |
+          make images
+          title="${{ github.event.issue.title }}"
+          tag=${title#"Release "}
+          echo "::set-output name=tag::${tag}"
+          docker tag moby/buildkit:local warmmetal/buildkit:${tag}
+          docker tag moby/buildkit:local warmmetal/buildkit:latest
+          docker push warmmetal/buildkit:${tag}
+          docker push warmmetal/buildkit:latest
+      - name: Create Release
+        uses: nickatnight/releases-action@v3
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag: ${{ steps.image.outputs.tag }}
+          name: ${{ steps.image.outputs.tag }}
+          message: |
+            warmmetal/buildkit:${{ steps.image.outputs.tag }}
+      - name: Comment PR
+        uses: thollander/actions-comment-pull-request@v1
+        with:
+          message: |
+            Both image and release updated.
+          pr_number: ${{ github.event.issue.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/check-upstream-release.yml

@@ -0,0 +1,42 @@
+name: check-upstream-release
+
+on:
+  schedule:
+    - cron: '0 0 * * 2-6'
+  workflow_dispatch:
+
+jobs:
+  check-upstream-release:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get upstream latest release
+      id: upstream
+      uses: pozetroninc/[email protected]
+      with:
+          owner: moby
+          repo: buildkit
+          excludes: prerelease, draft
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Check local release
+      run: |
+        set -x
+        git remote add upstream https://github.com/moby/buildkit.git
+        git fetch origin
+        git fetch upstream
+        branch_name=${{ steps.upstream.outputs.release }}-active
+        if git branch -a | grep "${branch_name}"; then
+          echo "An active branch is already present for ${{ steps.upstream.outputs.release }}"
+          exit 0
+        fi
+        git checkout -b ${branch_name} origin/upstream-release
+        git rebase ${{ steps.upstream.outputs.release }}
+        git push origin ${branch_name}
+        set +x
+    - name: Create PR
+      uses: peter-evans/create-pull-request@v4
+      with:
+        branch: ${{ steps.upstream.outputs.release }}-active
+        base: upstream-release
+        title: "Release ${{ steps.upstream.outputs.release }}"
+        assignees: kitt1987

.gitignore

@@ -3,3 +3,5 @@ coverage
 release-out
 .certs
 .tmp
+.vscode
+oryxBuildBinary

cmd/buildkitd/config/config.go

@@ -11,6 +11,9 @@ type Config struct {
 	// Root is the path to a directory where buildkit will store persistent data
 	Root string `toml:"root"`
 
+	// Where to create local mounts.
+	LocalMountSourceRoot string `toml:"local-mount-source-root"`
+
 	// Entitlements e.g. security.insecure, network.host
 	Entitlements []string `toml:"insecure-entitlements"`
 	// GRPC configuration settings

cmd/buildkitd/main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/moby/buildkit/frontend/gateway"
 	"github.com/moby/buildkit/frontend/gateway/forwarder"
 	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver/bboltcachestorage"
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/moby/buildkit/util/appcontext"
@@ -235,6 +236,8 @@ func main() {
 		opts := []grpc.ServerOption{grpc.UnaryInterceptor(unary), grpc.StreamInterceptor(stream)}
 		server := grpc.NewServer(opts...)
 
+		snapshot.InitLocalMountSourceRoot(cfg.LocalMountSourceRoot)
+
 		// relative path does not work with nightlyone/lockfile
 		root, err := filepath.Abs(cfg.Root)
 		if err != nil {
@@ -398,6 +401,10 @@ func setDefaultConfig(cfg *config.Config) {
 		cfg.Root = appdefaults.Root
 	}
 
+	if cfg.LocalMountSourceRoot == "" {
+		cfg.LocalMountSourceRoot = appdefaults.LocalMountSourceRoot
+	}
+
 	if len(cfg.GRPC.Address) == 0 {
 		cfg.GRPC.Address = []string{appdefaults.Address}
 	}

docs/buildkitd.toml.md

@@ -27,6 +27,10 @@ root = "/var/lib/buildkit"
 # insecure-entitlements allows insecure entitlements, disabled by default.
 insecure-entitlements = [ "network.host", "security.insecure" ]
 
+# local-mount-source-root specifies a directory where source directories of local mounts would be creeated.
+# If the same directory is shared with the host, the host container runtime can mount those local mounts to containers.
+local-mount-source-root = "/var/lib/buildkit/local-mnt-src"
+
 [grpc]
   address = [ "tcp://0.0.0.0:1234" ]
   # debugAddress is address for attaching go profiles and debuggers.

snapshot/localmounter_hostshared.go

@@ -0,0 +1,19 @@
+package snapshot
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+var localMountSourceRoot = ""
+
+func InitLocalMountSourceRoot(snapshotRoot string) {
+	localMountSourceRoot = snapshotRoot
+	if err := os.MkdirAll(localMountSourceRoot, 0640); err != nil {
+		panic(err)
+	}
+}
+
+func MakeLocalMountSourceDir(pattern string) (name string, err error) {
+	return ioutil.TempDir(localMountSourceRoot, pattern)
+}

snapshot/localmounter_unix.go

@@ -4,7 +4,6 @@
 package snapshot
 
 import (
-	"io/ioutil"
 	"os"
 	"syscall"
 
@@ -38,7 +37,7 @@ func (lm *localMounter) Mount() (string, error) {
 		}
 	}
 
-	dir, err := ioutil.TempDir("", "buildkit-mount")
+	dir, err := MakeLocalMountSourceDir("buildkit-mount")
 	if err != nil {
 		return "", errors.Wrap(err, "failed to create temp dir")
 	}

solver/llbsolver/mounts/mount.go

@@ -3,7 +3,6 @@ package mounts
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
@@ -282,7 +281,7 @@ type secretMountInstance struct {
 }
 
 func (sm *secretMountInstance) Mount() ([]mount.Mount, func() error, error) {
-	dir, err := ioutil.TempDir("", "buildkit-secrets")
+	dir, err := snapshot.MakeLocalMountSourceDir("buildkit-secrets")
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to create temp dir")
 	}
@@ -320,7 +319,7 @@ func (sm *secretMountInstance) Mount() ([]mount.Mount, func() error, error) {
 
 	randID := identity.NewID()
 	fp := filepath.Join(dir, randID)
-	if err := ioutil.WriteFile(fp, sm.sm.data, 0600); err != nil {
+	if err := os.WriteFile(fp, sm.sm.data, 0600); err != nil {
 		cleanup()
 		return nil, nil, err
 	}

solver/llbsolver/ops/exec_binfmt.go

@@ -2,7 +2,6 @@ package ops
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -47,7 +46,7 @@ type staticEmulatorMount struct {
 }
 
 func (m *staticEmulatorMount) Mount() ([]mount.Mount, func() error, error) {
-	tmpdir, err := ioutil.TempDir("", "buildkit-qemu-emulator")
+	tmpdir, err := snapshot.MakeLocalMountSourceDir("buildkit-qemu-emulator")
 	if err != nil {
 		return nil, nil, err
 	}

util/appdefaults/appdefaults_unix.go

@@ -10,9 +10,10 @@ import (
 )
 
 const (
-	Address   = "unix:///run/buildkit/buildkitd.sock"
-	Root      = "/var/lib/buildkit"
-	ConfigDir = "/etc/buildkit"
+	Address              = "unix:///run/buildkit/buildkitd.sock"
+	Root                 = "/var/lib/buildkit"
+	LocalMountSourceRoot = "/var/lib/buildkit/local-mount-source"
+	ConfigDir            = "/etc/buildkit"
 )
 
 // UserAddress typically returns /run/user/$UID/buildkit/buildkitd.sock

util/archutil/check_unix.go

@@ -7,12 +7,12 @@ import (
 	"bytes"
 	"compress/gzip"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"syscall"
 
+	"github.com/moby/buildkit/snapshot"
 	"github.com/pkg/errors"
 )
 
@@ -23,7 +23,7 @@ func withChroot(cmd *exec.Cmd, dir string) {
 }
 
 func check(arch, bin string) (string, error) {
-	tmpdir, err := ioutil.TempDir("", "qemu-check")
+	tmpdir, err := snapshot.MakeLocalMountSourceDir("qemu-check")
 	if err != nil {
 		return "", err
 	}