Ensure using the correct variant of yq (#275)

.gitignore

@@ -1,3 +1,4 @@
+/bin
 # exclude CRDs kustomize build output directory
 /config/helm/input
 /out

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Build: ensure using the correct variant of yq
+
 ## [2.25.0] - 2024-11-15
 
 ### Changed

Makefile.custom.mk

@@ -1,6 +1,21 @@
 # Generate kustomize patches and all helm charts
+
+OS ?= $(shell go env GOOS 2>/dev/null || echo linux)
+ARCH ?= $(shell go env GOARCH 2>/dev/null || echo amd64)
+YQ_BIN_VERSION := 4.44.5
+YQ_BIN := ./bin/yq
+
+$(YQ_BIN): ## Download yq locally if necessary.
+	mkdir -p $(dir $@)
+	curl -sfL https://github.com/mikefarah/yq/releases/download/v$(YQ_BIN_VERSION)/yq_$(OS)_$(ARCH) -o $@
+	chmod +x $@
+
+.PHONY: clean
+clean:
+	rm -f $(YQ_BIN)
+
 .PHONY: generate
-generate:
+generate: $(YQ_BIN)
 	./hack/generate-kustomize-patches.sh
 	$(MAKE) delete-generated-helm-charts
 	kustomize build config/helm -o helm/cluster-api-provider-aws/templates

hack/generate-crd-version-patches.sh

@@ -4,12 +4,12 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+YQ="${YQ:-$(realpath "$(dirname "$0")/../bin/yq")}"
+
 ROOT_DIR="./$(dirname "$0")/.."
 ROOT_DIR="$(realpath "$ROOT_DIR")"
 KUSTOMIZE_CRD_DIR="$ROOT_DIR/helm/cluster-api-provider-aws/files"
 
-# YQ="./$(dirname "$0")/tools/bin/yq"
-
 for CRD_DIR in "$KUSTOMIZE_CRD_DIR/infrastructure" "$KUSTOMIZE_CRD_DIR/bootstrap" "$KUSTOMIZE_CRD_DIR/controlplane"; do
     CRD_BASE_DIR="${CRD_DIR}/bases"
     CRD_VERSION_PATCHES_DIR="${CRD_DIR}/patches/versions"
@@ -28,34 +28,34 @@ patches:
 EOF
     else
         # clean up resource list
-        yq e -i '.resources = null' "$KUSTOMIZATION_FILE"
+        "${YQ}" e -i '.resources = null' "$KUSTOMIZATION_FILE"
 
         # clean up API version patches
-        for ((i=$(yq eval '.patches | length' "$KUSTOMIZATION_FILE")-1; i>=0; i--)); do
-            patch_path=$(j="$i" yq e '.patches[env(j)].path' "$KUSTOMIZATION_FILE")
+        for ((i=$("${YQ}" eval '.patches | length' "$KUSTOMIZATION_FILE")-1; i>=0; i--)); do
+            patch_path=$(j="$i" "${YQ}" e '.patches[env(j)].path' "$KUSTOMIZATION_FILE")
 
             if [[ "$patch_path" = patches/versions* ]]; then
-                j="$i" yq e -i 'del .patches[env(j)]' "$KUSTOMIZATION_FILE"
+                j="$i" "${YQ}" e -i 'del .patches[env(j)]' "$KUSTOMIZATION_FILE"
             fi
         done
 
-        patch_len=$(yq eval '.patches | length' "$KUSTOMIZATION_FILE")
+        patch_len=$("${YQ}" eval '.patches | length' "$KUSTOMIZATION_FILE")
         if [ "$patch_len" -eq "0" ]; then
-            yq e -i '.patches = null' "$KUSTOMIZATION_FILE"
+            "${YQ}" e -i '.patches = null' "$KUSTOMIZATION_FILE"
         fi
     fi
 
     for crd in "${CRD_BASE_DIR}"/*.yaml
     do
-        crd_name="$(yq e '.metadata.name' "$crd")"
+        crd_name="$("${YQ}" e '.metadata.name' "$crd")"
         echo "$crd_name"
         crd_filename="$(basename "$crd")"
 
         # Add CRD base to kustomization.yaml
-        yq eval -i '.resources += ["bases/'"$crd_filename"'"]' "$KUSTOMIZATION_FILE"
+        "${YQ}" eval -i '.resources += ["bases/'"$crd_filename"'"]' "$KUSTOMIZATION_FILE"
 
         version_index=0
-        for version in $(yq e '.spec.versions[].name' "$crd")
+        for version in $("${YQ}" e '.spec.versions[].name' "$crd")
         do
             version_patches_dir="$CRD_VERSION_PATCHES_DIR/$version"
             mkdir -p "$version_patches_dir"
@@ -68,8 +68,8 @@ EOF
   value:
 " > "$patch_file"
 
-            version_obj="$(yq e ".spec.versions[$version_index].schema" "$crd")" \
-                yq e -i '.[0].value = env(version_obj)' "$patch_file"
+            version_obj="$("${YQ}" e ".spec.versions[$version_index].schema" "$crd")" \
+                "${YQ}" e -i '.[0].value = env(version_obj)' "$patch_file"
 
             # Add CRD version patches to kustomization.yaml
             version_patch_entry="path: patches/versions/$version/$crd_filename
@@ -79,10 +79,10 @@ target:
     kind: CustomResourceDefinition
     name: $crd_name
 " \
-            yq eval -i '.patches += [env(version_patch_entry)]' "$KUSTOMIZATION_FILE"
+            "${YQ}" eval -i '.patches += [env(version_patch_entry)]' "$KUSTOMIZATION_FILE"
 
             # Delete version data from the CRD base
-            yq e -i ".spec.versions[$version_index].schema = {}" "$crd"
+            "${YQ}" e -i ".spec.versions[$version_index].schema = {}" "$crd"
             version_index=$((version_index+1))
         done
     done

hack/generate-kustomize-patches.sh

@@ -11,6 +11,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+YQ="${YQ:-$(realpath "$(dirname "$0")/../bin/yq")}"
+
 # Directories
 ROOT_DIR="./$(dirname "$0")/.."
 ROOT_DIR="$(realpath "$ROOT_DIR")"
@@ -22,29 +24,29 @@ KUSTOMIZE_INPUT_DIR="$ROOT_DIR/config/helm/input"
 helm_values="$HELM_DIR/values.yaml"
 org="giantswarm"
 repo="cluster-api-provider-aws"
-version="$(yq e -e '.tag' "$helm_values")" || { >&2 echo "Could not find image tag value"; exit 1; }
+version="$("${YQ}" e -e '.tag' "$helm_values")" || { >&2 echo "Could not find image tag value"; exit 1; }
 release_asset_filename="infrastructure-components.yaml"
 url="https://github.com/$org/$repo/releases/download/$version/${release_asset_filename}"
 mkdir -p "$KUSTOMIZE_INPUT_DIR"
 curl -fsSL "$url" -o "$KUSTOMIZE_INPUT_DIR/${release_asset_filename}" || { >&2 echo "Failed to get release manifest from $url"; exit 1; }
 
 # Update kustomize patches for webhooks. We do this for every CRD
 
-watch_filter="$(yq e -e '.watchfilter' "$helm_values")" || { echo "Could not find watch filter value"; exit 1; }
+watch_filter="$("${YQ}" e -e '.watchfilter' "$helm_values")" || { echo "Could not find watch filter value"; exit 1; }
 
 # For every CRD, add webhook label selector
 for webhook_kind_prefix in Mutating Validating; do
     output_path="${KUSTOMIZE_DIR}/webhook-$(echo "${webhook_kind_prefix}" | tr '[:upper:]' '[:lower:]')-watchfilter.yaml"
     echo "# Generated by 'generate-kustomize-patches.sh'. Do not edit." > "${output_path}"
 
-    for webhook_cr_name in $(yq e -N "select(.kind==\"${webhook_kind_prefix}WebhookConfiguration\") | .metadata.name" "$KUSTOMIZE_INPUT_DIR/$release_asset_filename"); do
+    for webhook_cr_name in $("${YQ}" e -N "select(.kind==\"${webhook_kind_prefix}WebhookConfiguration\") | .metadata.name" "$KUSTOMIZE_INPUT_DIR/$release_asset_filename"); do
         webhook="$(
             webhook_cr_name="$webhook_cr_name" \
-            yq e "select((.kind==\"${webhook_kind_prefix}WebhookConfiguration\") and .metadata.name==env(webhook_cr_name))" \
+            "${YQ}" e "select((.kind==\"${webhook_kind_prefix}WebhookConfiguration\") and .metadata.name==env(webhook_cr_name))" \
                 "$KUSTOMIZE_INPUT_DIR/$release_asset_filename"
         )"
 
-        webhook_api_version="$(echo "$webhook" | yq e ".apiVersion" -)"
+        webhook_api_version="$(echo "$webhook" | "${YQ}" e ".apiVersion" -)"
 
         echo "Generating watch-filter patches for ${webhook_kind_prefix}WebhookConfiguration $webhook_cr_name"
 
@@ -56,19 +58,19 @@ webhooks: null
 "
 
         # Get all CRDs for this provider
-        for webhook_name in $(webhook_cr_name="$webhook_cr_name" yq e "select((.kind==\"${webhook_kind_prefix}WebhookConfiguration\") and .metadata.name==env(webhook_cr_name)) | .webhooks[].name" "$KUSTOMIZE_INPUT_DIR/$release_asset_filename"); do
+        for webhook_name in $(webhook_cr_name="$webhook_cr_name" "${YQ}" e "select((.kind==\"${webhook_kind_prefix}WebhookConfiguration\") and .metadata.name==env(webhook_cr_name)) | .webhooks[].name" "$KUSTOMIZE_INPUT_DIR/$release_asset_filename"); do
             object_selector_patch="$(
                 webhook_name="$webhook_name" \
                 watch_filter="$watch_filter" \
-                yq e --null-input \
+                "${YQ}" e --null-input \
                     '.name = env(webhook_name) |
                     .objectSelector.matchLabels["cluster.x-k8s.io/watch-filter"] = env(watch_filter)'
                 )"
 
             webhook_patch="$(
                 echo "$webhook_patch" | \
                     object_selector_patch="$object_selector_patch" \
-                    yq e '.webhooks += [env(object_selector_patch)]' -
+                    "${YQ}" e '.webhooks += [env(object_selector_patch)]' -
             )"
         done
 

hack/move-generated-crds.sh

@@ -4,6 +4,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+YQ="${YQ:-$(realpath "$(dirname "$0")/../bin/yq")}"
+
 # dirs
 HELM_DIR="helm/cluster-api-provider-aws"
 HELM_TEMPLATES_DIR="$HELM_DIR/templates"
@@ -51,7 +53,7 @@ move-infrastructure-manifests() {
         mv "$crd_file" "$new_crd_file"
 
         # Consistently remove `creationTimestamp`
-        yq e -i 'del .metadata.creationTimestamp' "$new_crd_file"
+        "${YQ}" e -i 'del .metadata.creationTimestamp' "$new_crd_file"
     done
     cd ../../../../..
 }