feat: set variables on find images (#2282)

## Description

Currently when you run `zarf prepare find-images` with a package that
has variables in it, the variables don't get set, even to their
defaults. This can break the helm chart, or make us unable to find
images. This PR introduces a `--deploy-set` flag. Additionally we take
in the default values to the chart.

## Type of change

- [X] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [ ] Test, docs, adr added or updated as needed
- [x] [Contributor Guide
Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow)
followed

---------

Co-authored-by: Wayne Starr <[email protected]>
Co-authored-by: razzle <[email protected]>

Author: 3 people

.github/actions/k3d/action.yaml

@@ -7,5 +7,5 @@ runs:
     - run: "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash"
       shell: bash
 
-    - run: k3d cluster delete && k3d cluster create
+    - run: k3d cluster delete && k3d cluster create --k3s-arg="--disable=traefik@server:0"
       shell: bash

docs/2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md

@@ -16,11 +16,14 @@ zarf dev find-images [ PACKAGE ] [flags]
 ## Options
 
 ```
-  -h, --help                     help for find-images
-      --kube-version string      Override the default helm template KubeVersion when performing a package chart template
-  -p, --repo-chart-path string   If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"
-      --set stringToString       Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]. (default [])
-      --why string               Find the location of the image given as an argument and print it to the console.
+      --create-set stringToString   Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]. (default [])
+      --deploy-set stringToString   Specify deployment variables to set on the command line (KEY=value) (default [])
+  -f, --flavor string               The flavor of components to include in the resulting package (i.e. have a matching or empty "only.flavor" key)
+  -h, --help                        help for find-images
+      --kube-version string         Override the default helm template KubeVersion when performing a package chart template
+      --registry-url string         Override the ###ZARF_REGISTRY### value (default "127.0.0.1:31999")
+  -p, --repo-chart-path string      If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"
+      --why string                  Prints the source manifest for the specified image
 ```
 
 ## Options inherited from parent commands

src/cmd/dev.go

@@ -21,6 +21,7 @@ import (
 	"github.com/defenseunicorns/zarf/src/pkg/transform"
 	"github.com/defenseunicorns/zarf/src/pkg/utils"
 	"github.com/defenseunicorns/zarf/src/pkg/utils/helpers"
+	"github.com/defenseunicorns/zarf/src/types"
 	"github.com/mholt/archiver/v3"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -190,8 +191,11 @@ var devFindImagesCmd = &cobra.Command{
 
 		// Ensure uppercase keys from viper
 		v := common.GetViper()
+
 		pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap(
 			v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper)
+		pkgConfig.PkgOpts.SetVariables = helpers.TransformAndMergeMap(
+			v.GetStringMapString(common.VPkgDeploySet), pkgConfig.PkgOpts.SetVariables, strings.ToUpper)
 
 		// Configure the packager
 		pkgClient := packager.NewOrDie(&pkgConfig)
@@ -265,14 +269,23 @@ func init() {
 	devFindImagesCmd.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdDevFlagRepoChartPath)
 	// use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables
 	devFindImagesCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdDevFlagSet)
+
+	devFindImagesCmd.Flags().MarkDeprecated("set", "this field is replaced by create-set")
+	devFindImagesCmd.Flags().MarkHidden("set")
+	devFindImagesCmd.Flags().StringVarP(&pkgConfig.CreateOpts.Flavor, "flavor", "f", v.GetString(common.VPkgCreateFlavor), lang.CmdPackageCreateFlagFlavor)
+	devFindImagesCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "create-set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdDevFlagSet)
+	devFindImagesCmd.Flags().StringToStringVar(&pkgConfig.PkgOpts.SetVariables, "deploy-set", v.GetStringMapString(common.VPkgDeploySet), lang.CmdPackageDeployFlagSet)
 	// allow for the override of the default helm KubeVersion
 	devFindImagesCmd.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdDevFlagKubeVersion)
 	// check which manifests are using this particular image
 	devFindImagesCmd.Flags().StringVar(&pkgConfig.FindImagesOpts.Why, "why", "", lang.CmdDevFlagFindImagesWhy)
 
+	defaultRegistry := fmt.Sprintf("%s:%d", helpers.IPV4Localhost, types.ZarfInClusterContainerRegistryNodePort)
+	devFindImagesCmd.Flags().StringVar(&pkgConfig.FindImagesOpts.RegistryURL, "registry-url", defaultRegistry, lang.CmdDevFlagFindImagesRegistry)
+
 	devLintCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet)
 	devLintCmd.Flags().StringVarP(&pkgConfig.CreateOpts.Flavor, "flavor", "f", v.GetString(common.VPkgCreateFlavor), lang.CmdPackageCreateFlagFlavor)
-	devTransformGitLinksCmd.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdDevFlagGitAccount)
+	devTransformGitLinksCmd.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", types.ZarfGitPushUser, lang.CmdDevFlagGitAccount)
 }
 
 func bindDevDeployFlags(v *viper.Viper) {

src/cmd/initialize.go

@@ -23,6 +23,7 @@ import (
 	"github.com/defenseunicorns/zarf/src/pkg/utils"
 	"github.com/defenseunicorns/zarf/src/pkg/utils/helpers"
 	"github.com/defenseunicorns/zarf/src/pkg/zoci"
+	"github.com/defenseunicorns/zarf/src/types"
 
 	"github.com/spf13/cobra"
 )
@@ -181,8 +182,8 @@ func init() {
 
 	// Init package variable defaults that are non-zero values
 	// NOTE: these are not in common.setDefaults so that zarf tools update-creds does not erroneously update values back to the default
-	v.SetDefault(common.VInitGitPushUser, config.ZarfGitPushUser)
-	v.SetDefault(common.VInitRegistryPushUser, config.ZarfRegistryPushUser)
+	v.SetDefault(common.VInitGitPushUser, types.ZarfGitPushUser)
+	v.SetDefault(common.VInitRegistryPushUser, types.ZarfRegistryPushUser)
 
 	// Init package set variable flags
 	initCmd.Flags().StringToStringVar(&pkgConfig.PkgOpts.SetVariables, "set", v.GetStringMapString(common.VPkgDeploySet), lang.CmdInitFlagSet)

src/cmd/package.go

@@ -15,6 +15,7 @@ import (
 	"github.com/defenseunicorns/zarf/src/pkg/message"
 	"github.com/defenseunicorns/zarf/src/pkg/packager/sources"
 	"github.com/defenseunicorns/zarf/src/pkg/utils"
+	"github.com/defenseunicorns/zarf/src/types"
 
 	"oras.land/oras-go/v2/registry"
 
@@ -395,8 +396,8 @@ func bindMirrorFlags(v *viper.Viper) {
 
 	// Init package variable defaults that are non-zero values
 	// NOTE: these are not in common.setDefaults so that zarf tools update-creds does not erroneously update values back to the default
-	v.SetDefault(common.VInitGitPushUser, config.ZarfGitPushUser)
-	v.SetDefault(common.VInitRegistryPushUser, config.ZarfRegistryPushUser)
+	v.SetDefault(common.VInitGitPushUser, types.ZarfGitPushUser)
+	v.SetDefault(common.VInitRegistryPushUser, types.ZarfRegistryPushUser)
 
 	// Always require confirm flag (no viper)
 	mirrorFlags.BoolVar(&config.CommonOptions.Confirm, "confirm", false, lang.CmdPackageDeployFlagConfirm)

src/config/config.go

@@ -26,9 +26,7 @@ const (
 	GithubProject = "defenseunicorns/zarf"
 
 	// ZarfMaxChartNameLength limits helm chart name size to account for K8s/helm limits and zarf prefix
-	ZarfMaxChartNameLength   = 40
-	ZarfGeneratedPasswordLen = 24
-	ZarfGeneratedSecretLen   = 48
+	ZarfMaxChartNameLength = 40
 
 	ZarfAgentHost = "agent-hook.zarf.svc"
 
@@ -53,16 +51,6 @@ const (
 	ZarfImagePullSecretName = "private-registry"
 	ZarfGitServerSecretName = "private-git-server"
 
-	ZarfRegistryPushUser                   = "zarf-push"
-	ZarfRegistryPullUser                   = "zarf-pull"
-	ZarfInClusterContainerRegistryNodePort = 31999
-
-	ZarfGitPushUser = "zarf-git-user"
-	ZarfGitReadUser = "zarf-git-read-user"
-
-	ZarfInClusterGitServiceURL      = "http://zarf-gitea-http.zarf.svc.cluster.local:3000"
-	ZarfInClusterArtifactServiceURL = ZarfInClusterGitServiceURL + "/api/packages/" + ZarfGitPushUser
-
 	ZarfLoggingUser = "zarf-admin"
 )
 

src/config/lang/english.go

@@ -379,12 +379,13 @@ $ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 -a sk
 		"NOTE: This file must not already exist. If no filename is provided, the config will be written to the current working directory as zarf-config.toml."
 	CmdDevGenerateConfigErr = "Unable to write the config file %s, make sure the file doesn't already exist"
 
-	CmdDevFlagExtractPath   = `The path inside of an archive to use to calculate the sha256sum (i.e. for use with "files.extractPath")`
-	CmdDevFlagSet           = "Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]."
-	CmdDevFlagRepoChartPath = `If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"`
-	CmdDevFlagGitAccount    = "User or organization name for the git account that the repos are created under."
-	CmdDevFlagKubeVersion   = "Override the default helm template KubeVersion when performing a package chart template"
-	CmdDevFlagFindImagesWhy = "Find the location of the image given as an argument and print it to the console."
+	CmdDevFlagExtractPath        = `The path inside of an archive to use to calculate the sha256sum (i.e. for use with "files.extractPath")`
+	CmdDevFlagSet                = "Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]."
+	CmdDevFlagRepoChartPath      = `If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"`
+	CmdDevFlagGitAccount         = "User or organization name for the git account that the repos are created under."
+	CmdDevFlagKubeVersion        = "Override the default helm template KubeVersion when performing a package chart template"
+	CmdDevFlagFindImagesRegistry = "Override the ###ZARF_REGISTRY### value"
+	CmdDevFlagFindImagesWhy      = "Prints the source manifest for the specified image"
 
 	CmdDevLintShort = "Lints the given package for valid schema and recommended practices"
 	CmdDevLintLong  = "Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files"

src/extensions/bigbang/bigbang.go

@@ -94,6 +94,7 @@ func Run(YOLO bool, tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (typ
 		},
 		path.Join(tmpPaths.Temp, bb),
 		path.Join(tmpPaths.Temp, bb, "values"),
+		helm.WithPackageConfig(&types.PackagerConfig{}),
 	)
 
 	// Download the chart from Git and save it to a temporary directory.

src/internal/agent/hooks/argocd-repository.go

@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/defenseunicorns/zarf/src/config"
 	"github.com/defenseunicorns/zarf/src/config/lang"
 	"github.com/defenseunicorns/zarf/src/internal/agent/operations"
 	"github.com/defenseunicorns/zarf/src/internal/agent/state"
@@ -102,7 +101,7 @@ func mutateRepository(r *v1.AdmissionRequest) (result *operations.Result, err er
 func populateArgoRepositoryPatchOperations(repoURL string, zarfGitPullPassword string) []operations.PatchOperation {
 	var patches []operations.PatchOperation
 	patches = append(patches, operations.ReplacePatchOperation("/data/url", base64.StdEncoding.EncodeToString([]byte(repoURL))))
-	patches = append(patches, operations.ReplacePatchOperation("/data/username", base64.StdEncoding.EncodeToString([]byte(config.ZarfGitReadUser))))
+	patches = append(patches, operations.ReplacePatchOperation("/data/username", base64.StdEncoding.EncodeToString([]byte(types.ZarfGitReadUser))))
 	patches = append(patches, operations.ReplacePatchOperation("/data/password", base64.StdEncoding.EncodeToString([]byte(zarfGitPullPassword))))
 
 	return patches

src/internal/packager/helm/chart.go

@@ -139,12 +139,12 @@ func (h *Helm) InstallOrUpgradeChart() (types.ConnectStrings, string, error) {
 }
 
 // TemplateChart generates a helm template from a given chart.
-func (h *Helm) TemplateChart() (string, chartutil.Values, error) {
+func (h *Helm) TemplateChart() (manifest string, chartValues chartutil.Values, err error) {
 	message.Debugf("helm.TemplateChart()")
 	spinner := message.NewProgressSpinner("Templating helm chart %s", h.chart.Name)
 	defer spinner.Stop()
 
-	err := h.createActionConfig(h.chart.Namespace, spinner)
+	err = h.createActionConfig(h.chart.Namespace, spinner)
 
 	// Setup K8s connection.
 	if err != nil {
@@ -183,13 +183,18 @@ func (h *Helm) TemplateChart() (string, chartutil.Values, error) {
 		return "", nil, fmt.Errorf("unable to load chart data: %w", err)
 	}
 
+	client.PostRenderer, err = h.newRenderer()
+	if err != nil {
+		return "", nil, fmt.Errorf("unable to create helm renderer: %w", err)
+	}
+
 	// Perform the loadedChart installation.
 	templatedChart, err := client.Run(loadedChart, chartValues)
 	if err != nil {
 		return "", nil, fmt.Errorf("error generating helm chart template: %w", err)
 	}
 
-	manifest := templatedChart.Manifest
+	manifest = templatedChart.Manifest
 
 	for _, hook := range templatedChart.Hooks {
 		manifest += fmt.Sprintf("\n---\n%s", hook.Manifest)
@@ -243,7 +248,7 @@ func (h *Helm) UpdateReleaseValues(updatedValues map[string]interface{}) error {
 		// Namespace must be specified.
 		client.Namespace = h.chart.Namespace
 
-		// Post-processing our manifests for reasons....
+		// Post-processing our manifests to apply vars and run zarf helm logic in cluster
 		client.PostRenderer = postRender
 
 		// Set reuse values to only override the values we are explicitly given
@@ -285,7 +290,7 @@ func (h *Helm) installChart(postRender *renderer) (*release.Release, error) {
 	// Namespace must be specified.
 	client.Namespace = h.chart.Namespace
 
-	// Post-processing our manifests for reasons....
+	// Post-processing our manifests to apply vars and run zarf helm logic in cluster
 	client.PostRenderer = postRender
 
 	loadedChart, chartValues, err := h.loadChartData()
@@ -298,10 +303,6 @@ func (h *Helm) installChart(postRender *renderer) (*release.Release, error) {
 }
 
 func (h *Helm) upgradeChart(lastRelease *release.Release, postRender *renderer) (*release.Release, error) {
-	// Print the postRender object piece by piece to not print the htpasswd
-	message.Debugf("helm.upgradeChart(%#v, %#v, %#v, %#v, %s)", postRender.actionConfig, postRender.connectStrings,
-		postRender.namespaces, postRender.Helm, fmt.Sprintf("values:template.Values{ registry: \"%s\" }", postRender.values.GetRegistry()))
-
 	// Migrate any deprecated APIs (if applicable)
 	err := h.migrateDeprecatedAPIs(lastRelease)
 	if err != nil {
@@ -322,7 +323,7 @@ func (h *Helm) upgradeChart(lastRelease *release.Release, postRender *renderer)
 	// Namespace must be specified.
 	client.Namespace = h.chart.Namespace
 
-	// Post-processing our manifests for reasons....
+	// Post-processing our manifests to apply vars and run zarf helm logic in cluster
 	client.PostRenderer = postRender
 
 	loadedChart, chartValues, err := h.loadChartData()

src/internal/packager/helm/common.go

@@ -148,7 +148,19 @@ func WithKubeVersion(kubeVersion string) Modifier {
 	}
 }
 
+// WithPackageConfig sets the packager config for the chart
+func WithPackageConfig(cfg *types.PackagerConfig) Modifier {
+	return func(h *Helm) {
+		h.cfg = cfg
+	}
+}
+
 // StandardName generates a predictable full path for a helm chart for Zarf.
 func StandardName(destination string, chart types.ZarfChart) string {
 	return filepath.Join(destination, chart.Name+"-"+chart.Version)
 }
+
+// StandardValuesName generates a predictable full path for the values file for a helm chart for zarf
+func StandardValuesName(destination string, chart types.ZarfChart, idx int) string {
+	return fmt.Sprintf("%s-%d", StandardName(destination, chart), idx)
+}