setup_cache_for_template: Improve caching mechanism for images and `n…

…erdctl` archives - Change cache key to `url-sha256:$sha256` for caching images without a digest - Include image basename in the cache key - Use `limactl validate --fill` to retrieve nerdctl archive info and set cache if needed - `test.yml`: Change cache configuration to run after `make install` Signed-off-by: Norio Nomura <[email protected]>
lima-vm · Aug 3, 2024 · a3d0d7e · a3d0d7e
1 parent 402ba1c
commit a3d0d7e
Show file tree

Hide file tree

Showing 2 changed files with 107 additions and 34 deletions.
diff --git a/.github/actions/setup_cache_for_template/action.yml b/.github/actions/setup_cache_for_template/action.yml
@@ -7,6 +7,10 @@ inputs:
   template:
     description: template yaml file
     required: true
+  detect-containerd:
+    description: detect containerd usage from template by using limactl validate
+    required: false
+    default: 'true'
 runs:
   using: "composite"
   steps:
@@ -27,10 +31,15 @@ runs:
       set -eux
       arch="${{ inputs.arch }}"
       template="${{ inputs.template }}"
+      detect_containerd="${{ inputs.detect-containerd }}"
+      if [[ $detect_containerd == "true" ]] && ! command -v limactl &>/dev/null; then
+        echo "containerd detection is disabled because limactl is not found" >&2
+        detect_containerd="false"
+      fi
       case "$template" in
         https://*)
           tmp_yaml=$(mktemp -d)/template.yaml
-          curl -sSLf "$template" > $tmp_yaml || exit 1
+          curl -sSLf "$template" > "$tmp_yaml" || exit 1
           template=$tmp_yaml
           ;;
         *)
@@ -47,27 +56,82 @@ runs:
         arm64) arch=aarch64 ;;
       esac
 
-      # extract digest and location from template using arch
-      digest="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].digest // \"\"" "$template")"
-      location="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].location // \"\"" "$template")"
-      test -n "$location" || exit 1
-
-      # path to cache
-      if command -v sha256sum > /dev/null; then
-        sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)"
-      elif command -v shasum > /dev/null; then
-        sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)"
+      # extract digest, location and size by parsing template using arch
+      readonly yq_filter="
+        [
+          .images | map(select(.arch == \"${arch}\")) | [.[0,1].location, .[0,1].digest],
+          .containerd|[.system or .user],
+          .containerd.archives | map(select(.arch == \"${arch}\")) | [.[0].location, .[0].digest]
+        ]|flatten|.[]
+      "
+      if [[ $detect_containerd == "true" ]]; then
+        parsed=$(LIMA_HOME=$(mktemp -d) limactl validate "$template" --fill 2>/dev/null | yq eval "${yq_filter}")
       else
-        echo "sha256sum or shasum not found" >&2
+        parsed=$(yq eval "${yq_filter}" "$template")
+      fi
+      readarray -t arr <<<"$parsed"
+      readonly locations=("${arr[@]:0:2}") digests=("${arr[@]:2:2}")
+      readonly containerd="${arr[4]}" containerd_location="${arr[5]}" containerd_digest="${arr[6]}"
+      for ((i = 0; i < ${#locations[@]}; i++)); do
+        [[ ${locations[i]} != "null" ]] || continue
+        http_code=$(curl -sIL -w "%{http_code}" "${locations[i]}" -o /dev/null)
+        if [[ ${http_code} -eq 200 ]]; then
+          location=${locations[i]}
+          digest=${digests[i]}
+          break
+        fi
+      done
+      if [[ -z ${location} ]]; then
+        echo "Failed to get the image location for ${template}" >&2
         exit 1
       fi
-      echo "path=.download/by-url-sha256/$sha256" >> "$GITHUB_OUTPUT"
+
+      function location_to_sha256() {
+        local location=$1
+        if command -v sha256sum > /dev/null; then
+          sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)"
+        elif command -v shasum > /dev/null; then
+          sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)"
+        else
+          echo "sha256sum or shasum not found" >&2
+          exit 1
+        fi
+        echo "$sha256"
+      }
+
+      function location_to_cache_path() {
+        sha256=$(location_to_sha256 "$location") && echo ".download/by-url-sha256/$sha256"
+      }
+
+      # path to cache
+      image_cache_path=$(location_to_cache_path "$location")
+      echo "path=$image_cache_path" >> "$GITHUB_OUTPUT"
 
       # key for cache
-      key="${digest:+image-$digest}"
-      # fallback to os and hash of template file if digest not found
-      key="${key:-${{ runner.os }}-${{ hashFiles(inputs.template) }}}"
+      image_basename=$(basename "$location")
+      if [[ "$digest" != "null" ]]; then
+        key="image:$image_basename-$digest"
+      else
+        # use sha256 of location as key if digest is not available
+        key="image:$image_basename-url-sha256:$(sha256 "$location")"
+      fi
       echo "key=$key" >> "$GITHUB_OUTPUT"
+
+      # containerd path and key for cache
+      if [[ $containerd == "true" && "$containerd_location" != "null" ]]; then
+        containerd_basename=$(basename "$containerd_location")
+        if [[ ${containerd_digest} != "null" ]]; then
+          containerd_key="containerd:$containerd_basename-$containerd_digest"
+        else
+          containerd_key="containerd:$containerd_basename-url-sha256:$(sha256 "$containerd_location")"
+        fi
+        echo "containerd-key=$containerd_key" >> "$GITHUB_OUTPUT"
+        containerd_cache_path=$(location_to_cache_path "$containerd_location")
+        echo "containerd-path=$containerd_cache_path" >> "$GITHUB_OUTPUT"
+      else
+        echo "containerd-key=" >> "$GITHUB_OUTPUT"
+        echo "containerd-path=" >> "$GITHUB_OUTPUT"
+      fi
     shell: bash
 
   - name: "Cache ${{ steps.cache-params-from-template.outputs.path }}"
@@ -78,11 +142,19 @@ runs:
       key: ${{ steps.cache-params-from-template.outputs.key }}
       enableCrossOsArchive: true
 
+  - name: "Cache ${{ steps.cache-params-from-template.outputs.containerd-key }}"
+    if: ${{ steps.cache-params-from-template.outputs.containerd-key != '' }}
+    uses: actions/cache@v4
+    with:
+      path: ${{ steps.cache-params-from-template.outputs.containerd-path }}
+      key: ${{ steps.cache-params-from-template.outputs.containerd-key }}
+      enableCrossOsArchive: true
+
   - name: "Create symbolic link named ${{ steps.detect-platform.outputs.download-dir }} pointing to .download"
     run: |
       set -eux
       [ -d .download ] || mkdir -p .download
       path_to_cache=${{ steps.detect-platform.outputs.download-dir }}
-      mkdir -p $(dirname $path_to_cache)
-      ln -sfn $PWD/.download $path_to_cache
+      mkdir -p "$(dirname "$path_to_cache")"
+      ln -sfn "$PWD/.download" "$path_to_cache"
     shell: bash
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -157,16 +157,16 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version: 1.22.x
-    - name: Cache image used by default.yaml
-      uses: ./.github/actions/setup_cache_for_template
-      with:
-        template: templates/default.yaml
     - name: Unit tests
       run: go test -v ./...
     - name: Make
       run: make
     - name: Install
       run: make install
+    - name: Cache image used by default.yaml
+      uses: ./.github/actions/setup_cache_for_template
+      with:
+        template: templates/default.yaml
     - name: Validate templates
       run: find -L templates -name '*.yaml' | xargs limactl validate
     - name: Install test dependencies
@@ -225,6 +225,10 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version: 1.22.x
+    - name: Make
+      run: make
+    - name: Install
+      run: sudo make install
     - name: normalize template path
       id: normalize_template_path
       # `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
@@ -233,10 +237,6 @@ jobs:
       uses: ./.github/actions/setup_cache_for_template
       with:
         template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
-    - name: Make
-      run: make
-    - name: Install
-      run: sudo make install
     - name: Install test dependencies
       run: |
         sudo apt-get update
@@ -325,14 +325,14 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version: 1.22.x
-    - name: Cache image used by vmnet.yaml
-      uses: ./.github/actions/setup_cache_for_template
-      with:
-        template: templates/vmnet.yaml
     - name: Make
       run: make
     - name: Install
       run: make install
+    - name: Cache image used by vmnet.yaml
+      uses: ./.github/actions/setup_cache_for_template
+      with:
+        template: templates/vmnet.yaml
     - name: Install test dependencies
       run: brew install qemu bash coreutils iperf3
     - name: Install socket_vmnet
@@ -379,6 +379,7 @@ jobs:
       uses: ./.github/actions/setup_cache_for_template
       with:
         template: https://raw.githubusercontent.com/lima-vm/lima/${{ matrix.oldver }}/examples/ubuntu-lts.yaml
+        detect-containerd: "false"
     - name: Install test dependencies
       run: brew install qemu bash coreutils
     - name: Test
@@ -408,6 +409,10 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version: 1.22.x
+    - name: Make
+      run: make
+    - name: Install
+      run: make install
     - name: normalize template path
       id: normalize_template_path
       # `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
@@ -416,10 +421,6 @@ jobs:
       uses: ./.github/actions/setup_cache_for_template
       with:
         template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
-    - name: Make
-      run: make
-    - name: Install
-      run: make install
     - name: Install test dependencies
       run: brew install bash coreutils jq
     - name: Uninstall qemu