test: add nydus e2e testing for API v2

.github/workflows/nydus-e2e.yml → .github/workflows/e2e-v1-nydus.yml

@@ -1,4 +1,4 @@
-name: E2E Test With Nydus
+name: E2E Test With Nydus(API v1 - Golang Client)
 
 on:
   push:
@@ -76,10 +76,6 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
 
-      - name: Build images of the testing tools 
-        run: |
-          make docker-build-testing-tools
-
       - name: Setup Kind
         uses: helm/[email protected]
         with:

.github/workflows/e2e-v2-nydus.yml

@@ -0,0 +1,176 @@
+name: E2E Test With Nydus(API v2 - Rust Client)
+
+on:
+  push:
+    branches: [main, release-*]
+    paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
+  pull_request:
+    branches: [main, release-*]
+    paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
+  schedule:
+    - cron: '0 4 * * *'
+
+env:
+  GO_VERSION: '1.21'
+  KIND_VERSION: v0.12.0
+  CONTAINERD_VERSION: v1.5.2
+  NERDCTL_VER: 0.22.2
+  KIND_CONFIG_PATH: test/testdata/kind/config-v2.yaml
+  DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly
+  NYDUS_SNAPSHOTTER_CHARTS_PATH: deploy/helm-charts/charts/nydus-snapshotter
+
+jobs:
+  e2e_tests_nydus:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@v3
+        id: buildx
+        with:
+          install: true
+
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Pull Rust Client Image
+        run: |
+          cd client-rs
+          CLIENT_TAG=$(git describe --tags $(git rev-parse HEAD))
+          docker pull dragonflyoss/client:$CLIENT_TAG
+          docker tag dragonflyoss/client:$CLIENT_TAG dragonflyoss/client:latest
+          docker pull dragonflyoss/dfinit:$CLIENT_TAG
+          docker tag dragonflyoss/dfinit:$CLIENT_TAG dragonflyoss/dfinit:latest
+
+      - name: Build Scheduler Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: build/images/scheduler/Dockerfile
+          push: false
+          load: true
+          tags: dragonflyoss/scheduler:latest
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Build Manager Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: build/images/manager/Dockerfile
+          push: false
+          load: true
+          tags: dragonflyoss/manager:latest
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Setup Kind
+        uses: helm/[email protected]
+        with:
+          version: ${{ env.KIND_VERSION }}
+          config: ${{ env.KIND_CONFIG_PATH }}
+          cluster_name: kind
+
+      - name: Kind load images
+        run: |
+          kind load docker-image dragonflyoss/manager:latest
+          kind load docker-image dragonflyoss/scheduler:latest
+          kind load docker-image dragonflyoss/client:latest
+          kind load docker-image dragonflyoss/dfinit:latest
+
+      - name: Setup dragonfly
+        run: |
+          helm install --wait --timeout 10m --dependency-update \
+            --create-namespace --namespace dragonfly-system \
+            -f test/testdata/charts/config-v2.yaml \
+            dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }}
+
+      - name: Setup nydus
+        run: |
+          helm install --wait --timeout 10m --dependency-update \
+            --create-namespace --namespace nydus-system \
+            -f test/testdata/charts/config-v2-nydus.yaml \
+            nydus-snapshotter ${{ env.NYDUS_SNAPSHOTTER_CHARTS_PATH }}
+
+          export ns=nydus-system
+          p=`kubectl -n $ns get pods --no-headers -o custom-columns=NAME:metadata.name`
+          echo "snapshotter pod name ${p}"
+          # helm wait is wait for daemonset, in this e2e test we ensure the pod is ready
+          kubectl -n $ns wait po $p --for=condition=ready --timeout=2m
+
+      - name: Run E2E test
+        run: |
+          docker exec kind-control-plane curl -L -o nerdctl-${NERDCTL_VER}-linux-amd64.tar.gz https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VER}/nerdctl-${NERDCTL_VER}-linux-amd64.tar.gz
+          docker exec kind-control-plane tar xzf nerdctl-${NERDCTL_VER}-linux-amd64.tar.gz
+          docker exec kind-control-plane install -D -m 755 nerdctl /usr/local/bin/nerdctl
+
+          # this is only a simple test that run `date` in container
+          docker exec kind-control-plane /usr/local/bin/nerdctl run --snapshotter nydus --network=none ghcr.io/dragonflyoss/image-service/nginx:nydus-latest date
+
+          kubectl apply -f test/testdata/k8s/nydus.yaml
+          kubectl wait po nydus-pod --for=condition=ready --timeout=1m
+          kubectl delete -f test/testdata/k8s/nydus.yaml
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Dump logs
+        if: always()
+        continue-on-error: true
+        run: |
+          # Dump nydus logs.
+          nydus_log_dir="/tmp/nydus"
+          mkdir -p $nydus_log_dir
+          export nydus_ns=nydus-system
+          for p in `kubectl -n $nydus_ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do kubectl -n $nydus_ns get pod $p -o yaml >> $nydus_log_dir/nydus-pods.log; done
+          for p in `kubectl -n $nydus_ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do kubectl -n $nydus_ns describe pod $p  >> $nydus_log_dir/nydus-pods.log; done
+
+          docker exec kind-control-plane cat /etc/containerd/config.toml >> $nydus_log_dir/containerd-config.toml
+          docker exec kind-control-plane containerd config dump >> $nydus_log_dir/containerd-config-dump.toml
+
+          docker exec kind-control-plane journalctl -u containerd >> $nydus_log_dir/containerd.log
+          docker exec kind-control-plane journalctl -u kubelet >> $nydus_log_dir/kubelet.log
+
+          # Dump dragonfly client logs.
+          dragonfly_log_dir="tmp/dragonfly"
+          mkdir -p $dragonfly_log_dir
+          export dragonfly_ns=dragonfly-system
+
+          pod_names=$(kubectl get pods -l component=client -o custom-columns=NAME:metadata.name --no-headers -n $dragonfly_ns)
+          for pod_name in $pod_names; do
+              mkdir -p "/tmp/dragonfly/${pod_name}"
+              kubectl cp "${pod_name}:/var/log/dragonfly" "/tmp/dragonfly/${pod_name}" -n $dragonfly_ns
+          done
+
+      - name: Upload Logs
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: nydus-e2e-tests-logs
+          path: |
+            /tmp/nydus
+            /tmp/dragonfly

test/e2e/v2/dfget_test.go

@@ -40,7 +40,7 @@ var _ = Describe("Download Using Dfget", func() {
 
 			fileMetadata := util.FileMetadata{
 				ID:     "1fc5ed9922a3d741063c169ec49c2071a391db5fda8de30eb6a97f60b5038c16",
-				Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641",
+				Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f",
 			}
 
 			sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID)
@@ -230,7 +230,7 @@ var _ = Describe("Download Using Dfget", func() {
 
 			fileMetadata := util.FileMetadata{
 				ID:     "1ae51fe69c381a4604517f1d00d4315afef070bab3bcb475f11770fc5b194821",
-				Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641",
+				Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f",
 			}
 
 			sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID)
@@ -420,7 +420,7 @@ var _ = Describe("Download Using Dfget", func() {
 
 			fileMetadata := util.FileMetadata{
 				ID:     "9746acdb4bd8bf2deeb5dd8a3275e51a7fdd4adf8b0dc1d9d26a4565d3ed6592",
-				Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641",
+				Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f",
 			}
 
 			sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID)
@@ -458,7 +458,7 @@ var _ = Describe("Download Using Dfget", func() {
 
 			fileMetadata := util.FileMetadata{
 				ID:     "9746acdb4bd8bf2deeb5dd8a3275e51a7fdd4adf8b0dc1d9d26a4565d3ed6592",
-				Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641",
+				Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f",
 			}
 
 			sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID)

test/e2e/v2/proxy_test.go

@@ -40,7 +40,7 @@ var _ = Describe("Download Using Proxy", func() {
 
 			fileMetadata := util.FileMetadata{
 				ID:     "0ef882badaeb3195aed759a203cfb61951d158bf614f90ab0a20504fb7f97992",
-				Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641",
+				Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f",
 			}
 
 			sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID)
@@ -498,7 +498,7 @@ var _ = Describe("Download Using Prefetch Proxy", func() {
 
 			fileMetadata := util.FileMetadata{
 				ID:     "0fa5f40dfccfcd4c20b2d2dba0bab7205bfced189cc7a013919a6d155d841dab",
-				Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641",
+				Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f",
 			}
 
 			sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID)

test/testdata/charts/config-nydus.yaml

@@ -18,7 +18,6 @@ dragonfly:
       "X-Dragonfly-Registry": "https://ghcr.io"
     ping_url: "http://127.0.0.1:40901/server/ping"
 
-
 containerRuntime:
   initContainerImage: ghcr.io/liubin/toml-cli:v0.0.7
   containerd:

test/testdata/charts/config-v2-nydus.yaml

@@ -0,0 +1,25 @@
+name: nydus-snapshotter
+pullPolicy: Always
+hostNetwork: true
+resources:
+  requests:
+    cpu: "0"
+    memory: "0"
+  limits:
+    cpu: "1"
+    memory: "1Gi"
+
+dragonfly:
+  enable: true
+  mirrorConfig:
+  - host: "http://127.0.0.1:4001"
+    auth_through: false
+    headers:
+      "X-Dragonfly-Registry": "https://ghcr.io"
+    ping_url: "http://127.0.0.1:4003/healthy"
+
+containerRuntime:
+  initContainerImage: ghcr.io/liubin/toml-cli:v0.0.7
+  containerd:
+    enable: true
+    configFile: "/etc/containerd/config.toml"

test/testdata/containerd/config-v2.toml

@@ -0,0 +1,30 @@
+# explicitly use v2 config format
+version = 2
+
+[debug]
+  level = "debug"
+
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  # save disk space when using a single snapshotter
+  discard_unpacked_layers = false
+  # explicitly use default snapshotter so we can sed it in entrypoint
+  snapshotter = "overlayfs"
+  # explicit default here, as we're configuring it below
+  default_runtime_name = "runc"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  # set default runtime handler to v2, which has a per-pod shim
+  runtime_type = "io.containerd.runc.v2"
+
+# Setup a runtime with the magic name ("test-handler") used for Kubernetes
+# runtime class tests ...
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler]
+  runtime_type = "io.containerd.runc.v2"
+
+[plugins."io.containerd.grpc.v1.cri"]
+  # use fixed sandbox image
+  sandbox_image = "registry.k8s.io/pause:3.5"
+  # allow hugepages controller to be missing
+  # see https://github.com/containerd/cri/pull/1501
+  tolerate_missing_hugepages_controller = true
+  # restrict_oom_score_adj needs to be true when running inside UserNS (rootless)
+  restrict_oom_score_adj = false

test/testdata/kind/config-v2.yaml

@@ -9,7 +9,12 @@ nodes:
       - containerPort: 4001
         hostPort: 4001
         protocol: TCP
+      - containerPort: 4003
+        hostPort: 4003
+        protocol: TCP
     extraMounts:
+      - hostPath: ./test/testdata/containerd/config-v2.toml
+        containerPath: /etc/containerd/config.toml
       - hostPath: /tmp/artifact
         containerPath: /tmp/artifact
       - hostPath: /dev/fuse