setup_cache_for_template: Improve caching mechanism for images and nerdctl archives

norio-nomura · norio-nomura · commit 99c1247434ec · 2024-08-19T08:32:48.000+09:00
- Change cache key to `url-sha256:$sha256` for caching images without a digest
- Include image basename in the cache key
- Use `limactl validate --fill` to retrieve nerdctl archive info and set cache if needed
- `test.yml`: Change cache configuration to run after `make install`

Signed-off-by: Norio Nomura &lt;norio.nomura@gmail.com&gt;

test.yml: Remove `normalize_template_path` since `hashFile` is no longer used.

Signed-off-by: Norio Nomura &lt;norio.nomura@gmail.com&gt;
diff --git a/.github/actions/setup_cache_for_template/action.yml b/.github/actions/setup_cache_for_template/action.yml
@@ -7,6 +7,10 @@ inputs:
   template:
     description: template yaml file
     required: true
+  detect-containerd:
+    description: detect containerd usage from template by using limactl validate
+    required: false
+    default: 'true'
 runs:
   using: "composite"
   steps:
@@ -27,10 +31,15 @@ runs:
       set -eux
       arch="${{ inputs.arch }}"
       template="${{ inputs.template }}"
+      detect_containerd="${{ inputs.detect-containerd }}"
+      if [[ $detect_containerd == "true" ]] && ! command -v limactl &>/dev/null; then
+        echo "containerd detection is disabled because limactl is not found" >&2
+        detect_containerd="false"
+      fi
       case "$template" in
         https://*)
           tmp_yaml=$(mktemp -d)/template.yaml
-          curl -sSLf "$template" > $tmp_yaml || exit 1
+          curl -sSLf "$template" > "$tmp_yaml" || exit 1
           template=$tmp_yaml
           ;;
         *)
@@ -47,27 +56,85 @@ runs:
         arm64) arch=aarch64 ;;
       esac
 
-      # extract digest and location from template using arch
-      digest="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].digest // \"\"" "$template")"
-      location="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].location // \"\"" "$template")"
-      test -n "$location" || exit 1
-
-      # path to cache
-      if command -v sha256sum > /dev/null; then
-        sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)"
-      elif command -v shasum > /dev/null; then
-        sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)"
+      # extract digest, location and size by parsing template using arch
+      readonly yq_filter="
+        [
+          .images | map(select(.arch == \"${arch}\")) | [.[0,1].location, .[0,1].digest],
+          .containerd|[.system or .user],
+          .containerd.archives | map(select(.arch == \"${arch}\")) | [.[0].location, .[0].digest]
+        ]|flatten|.[]
+      "
+      if [[ $detect_containerd == "true" ]]; then
+        parsed=$(LIMA_HOME=$(mktemp -d) limactl validate "$template" --fill 2>/dev/null | yq eval "${yq_filter}")
       else
-        echo "sha256sum or shasum not found" >&2
+        parsed=$(yq eval "${yq_filter}" "$template")
+      fi
+      # macOS earlier than 15.0 uses bash 3.2.57, which does not support readarray -t
+      # readarray -t arr <<<"$parsed"
+      while IFS= read -r line; do arr+=("$line"); done <<<"${parsed}"
+      readonly locations=("${arr[@]:0:2}") digests=("${arr[@]:2:2}")
+      readonly containerd="${arr[4]}" containerd_location="${arr[5]}" containerd_digest="${arr[6]}"
+      for ((i = 0; i < ${#locations[@]}; i++)); do
+        [[ ${locations[i]} != "null" ]] || continue
+        http_code=$(curl -sIL -w "%{http_code}" "${locations[i]}" -o /dev/null)
+        if [[ ${http_code} -eq 200 ]]; then
+          location=${locations[i]}
+          digest=${digests[i]}
+          break
+        fi
+      done
+      if [[ -z ${location} ]]; then
+        echo "Failed to get the image location for ${template}" >&2
         exit 1
       fi
-      echo "path=.download/by-url-sha256/$sha256" >> "$GITHUB_OUTPUT"
+
+      function location_to_sha256() {
+        local location=$1
+        if command -v sha256sum > /dev/null; then
+          sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)"
+        elif command -v shasum > /dev/null; then
+          sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)"
+        else
+          echo "sha256sum or shasum not found" >&2
+          exit 1
+        fi
+        echo "$sha256"
+      }
+
+      function location_to_cache_path() {
+        local location=$1
+        sha256=$(location_to_sha256 "$location") && echo ".download/by-url-sha256/$sha256"
+      }
+
+      # path to cache
+      image_cache_path=$(location_to_cache_path "$location")
+      echo "path=$image_cache_path" >> "$GITHUB_OUTPUT"
 
       # key for cache
-      key="${digest:+image-$digest}"
-      # fallback to os and hash of template file if digest not found
-      key="${key:-${{ runner.os }}-${{ hashFiles(inputs.template) }}}"
+      image_basename=$(basename "$location")
+      if [[ "$digest" != "null" ]]; then
+        key="image:$image_basename-$digest"
+      else
+        # use sha256 of location as key if digest is not available
+        key="image:$image_basename-url-sha256:$(location_to_sha256 "$location")"
+      fi
       echo "key=$key" >> "$GITHUB_OUTPUT"
+
+      # containerd path and key for cache
+      if [[ $containerd == "true" && "$containerd_location" != "null" ]]; then
+        containerd_basename=$(basename "$containerd_location")
+        if [[ ${containerd_digest} != "null" ]]; then
+          containerd_key="containerd:$containerd_basename-$containerd_digest"
+        else
+          containerd_key="containerd:$containerd_basename-url-sha256:$(sha256 "$containerd_location")"
+        fi
+        echo "containerd-key=$containerd_key" >> "$GITHUB_OUTPUT"
+        containerd_cache_path=$(location_to_cache_path "$containerd_location")
+        echo "containerd-path=$containerd_cache_path" >> "$GITHUB_OUTPUT"
+      else
+        echo "containerd-key=" >> "$GITHUB_OUTPUT"
+        echo "containerd-path=" >> "$GITHUB_OUTPUT"
+      fi
     shell: bash
 
   - name: "Cache ${{ steps.cache-params-from-template.outputs.path }}"
@@ -78,11 +145,19 @@ runs:
       key: ${{ steps.cache-params-from-template.outputs.key }}
       enableCrossOsArchive: true
 
+  - name: "Cache ${{ steps.cache-params-from-template.outputs.containerd-key }}"
+    if: ${{ steps.cache-params-from-template.outputs.containerd-key != '' }}
+    uses: actions/cache@v4
+    with:
+      path: ${{ steps.cache-params-from-template.outputs.containerd-path }}
+      key: ${{ steps.cache-params-from-template.outputs.containerd-key }}
+      enableCrossOsArchive: true
+
   - name: "Create symbolic link named ${{ steps.detect-platform.outputs.download-dir }} pointing to .download"
     run: |
       set -eux
       [ -d .download ] || mkdir -p .download
       path_to_cache=${{ steps.detect-platform.outputs.download-dir }}
-      mkdir -p $(dirname $path_to_cache)
-      ln -sfn $PWD/.download $path_to_cache
+      mkdir -p "$(dirname "$path_to_cache")"
+      ln -sfn "$PWD/.download" "$path_to_cache"
     shell: bash
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -157,16 +157,16 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version: 1.23.x
-    - name: Cache image used by default.yaml
-      uses: ./.github/actions/setup_cache_for_template
-      with:
-        template: templates/default.yaml
     - name: Unit tests
       run: go test -v ./...
     - name: Make
       run: make
     - name: Install
       run: make install
+    - name: Cache image used by default.yaml
+      uses: ./.github/actions/setup_cache_for_template
+      with:
+        template: templates/default.yaml
     - name: Validate templates
       run: find -L templates -name '*.yaml' | xargs limactl validate
     - name: Install test dependencies
@@ -228,18 +228,14 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version: 1.23.x
-    - name: normalize template path
-      id: normalize_template_path
-      # `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
-      run: echo "NORMALIZED=$(realpath templates/${{ matrix.template }})" >> "$GITHUB_OUTPUT"
-    - name: Cache image used by ${{ steps.normalize_template_path.outputs.NORMALIZED }}
-      uses: ./.github/actions/setup_cache_for_template
-      with:
-        template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
     - name: Make
       run: make
     - name: Install
       run: sudo make install
+    - name: Cache image used by templates/${{ matrix.template }}
+      uses: ./.github/actions/setup_cache_for_template
+      with:
+        template: templates/${{ matrix.template }}
     - name: Install test dependencies
       run: |
         sudo apt-get update
@@ -328,14 +324,14 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version: 1.23.x
-    - name: Cache image used by vmnet.yaml
-      uses: ./.github/actions/setup_cache_for_template
-      with:
-        template: templates/vmnet.yaml
     - name: Make
       run: make
     - name: Install
       run: make install
+    - name: Cache image used by vmnet.yaml
+      uses: ./.github/actions/setup_cache_for_template
+      with:
+        template: templates/vmnet.yaml
     - name: Install test dependencies
       run: brew install qemu bash coreutils iperf3
     - name: Install socket_vmnet
@@ -385,6 +381,7 @@ jobs:
       uses: ./.github/actions/setup_cache_for_template
       with:
         template: https://raw.githubusercontent.com/lima-vm/lima/${{ matrix.oldver }}/examples/ubuntu-lts.yaml
+        detect-containerd: "false"
     - name: Install test dependencies
       run: brew install qemu bash coreutils
     - name: Test
@@ -414,18 +411,14 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version: 1.23.x
-    - name: normalize template path
-      id: normalize_template_path
-      # `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
-      run: echo "NORMALIZED=$(realpath templates/${{ matrix.template }})" >> "$GITHUB_OUTPUT"
-    - name: Cache image used by ${{ steps.normalize_template_path.outputs.NORMALIZED }}
-      uses: ./.github/actions/setup_cache_for_template
-      with:
-        template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
     - name: Make
       run: make
     - name: Install
       run: make install
+    - name: Cache image used by templates/${{ matrix.template }}
+      uses: ./.github/actions/setup_cache_for_template
+      with:
+        template: templates/${{ matrix.template }}
     - name: Install test dependencies
       run: brew install bash coreutils jq
     - name: Uninstall qemu
