Merge pull request #664 from fluxcd/embed-manifests

Use embedded Flux manifests for air-gapped bootstrap

Author: stefanprodan

.github/workflows/golangci-lint.yaml

@@ -16,6 +16,8 @@ jobs:
         with:
           go-version-file: 'go.mod'
           cache: true
+      - name: Download Flux manifests
+        run: make manifests
       - name: Run tidy
         run: make tidy
       - name: Check if working tree is dirty

.github/workflows/release.yaml

@@ -30,6 +30,8 @@ jobs:
         with:
           gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.PASSPHRASE }}
+      - name: Download Flux manifests
+        run: make manifests
       - name: Run GoReleaser
         id: run-goreleaser
         uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0

.github/workflows/tests.yaml

@@ -90,11 +90,8 @@ jobs:
         with:
           terraform_version: ${{ matrix.terraform }}
           terraform_wrapper: false
-      - run: go mod download
-      - env:
-          TF_ACC: "1"
-        run: go test -v -cover ./internal/provider/
-        timeout-minutes: 10
+      - name: Run tests
+        run: make testacc
   e2e-flux-bootstrap-with-github:
     runs-on: ubuntu-latest
     needs: build
@@ -126,10 +123,12 @@ jobs:
         with:
           terraform_version: "${{env.TERRAFORM_VERSION}}"
           terraform_wrapper: false
-      - name: Apply Terraform
+      - name: Build provider
         run: |
           make build
           make terraformrc
+      - name: Apply Terraform
+        run: |
           export TF_CLI_CONFIG_FILE="${PWD}/.terraformrc"
           cd examples/github-via-ssh
           terraform init
@@ -216,10 +215,12 @@ jobs:
         with:
           terraform_version: "${{env.TERRAFORM_VERSION}}"
           terraform_wrapper: false
-      - name: Apply Terraform
+      - name: Build provider
         run: |
           make build
           make terraformrc
+      - name: Apply Terraform
+        run: |
           export TF_CLI_CONFIG_FILE="${PWD}/.terraformrc"
           cd examples/helm-install
           terraform init
@@ -279,10 +280,12 @@ jobs:
         with:
           terraform_version: "${{env.TERRAFORM_VERSION}}"
           terraform_wrapper: false
-      - name: Apply Terraform
+      - name: Build provider
         run: |
           make build
           make terraformrc
+      - name: Apply Terraform
+        run: |
           export TF_CLI_CONFIG_FILE="${PWD}/.terraformrc"
           cd examples/github-via-pat
           terraform init

.gitignore

@@ -36,3 +36,8 @@ cover.out
 
 # ignore .terraformrc for dev overrides
 .terraformrc
+
+# ignore embedded manifests
+manifests/
+manifests.tar.gz
+.manifests.done

Makefile

@@ -4,35 +4,50 @@ SHELL := /bin/bash
 TEST_COUNT?=1
 ACCTEST_PARALLELISM?=5
 ACCTEST_TIMEOUT?=10m
+EMBEDDED_MANIFESTS_TARGET=.manifests.done
+FLUX_VERSION:=$(shell grep 'DefaultFluxVersion' internal/utils/flux.go | awk '{ print $$5 }' | tr -d '"')
+
+rwildcard=$(foreach d,$(wildcard $(addsuffix *,$(1))),$(call rwildcard,$(d)/,$(2)) $(filter $(subst *,%,$(2)),$(d)))
 
 all: test testacc build
 
+$(EMBEDDED_MANIFESTS_TARGET): $(call rwildcard,manifests/,*.yaml)
+	echo "Downloading manifests for Flux $(FLUX_VERSION)"
+	rm -rf manifests && mkdir -p manifests
+	curl -sLO https://github.com/fluxcd/flux2/releases/download/$(FLUX_VERSION)/manifests.tar.gz
+	tar xzf manifests.tar.gz -C manifests
+	rm -rf manifests.tar.gz
+	touch $@
+
+.PHONY: manifests
+manifests: $(EMBEDDED_MANIFESTS_TARGET)
+
 tidy:
 	rm -f go.sum; go mod tidy -compat=1.22
 
 fmt:
 	go fmt ./...
 
-vet:
+vet: $(EMBEDDED_MANIFESTS_TARGET)
 	go vet ./...
 
-test: tidy fmt vet
+test: $(EMBEDDED_MANIFESTS_TARGET) tidy fmt vet
 	go test ./... -coverprofile cover.out
 
-testacc: tidy fmt vet
-	TF_ACC=1 go test ./... -v -count $(TEST_COUNT) -parallel $(ACCTEST_PARALLELISM) -timeout $(ACCTEST_TIMEOUT)
+testacc: $(EMBEDDED_MANIFESTS_TARGET) tidy fmt vet
+	TF_ACC=1 go test ./... -v -count $(TEST_COUNT) -parallel $(ACCTEST_PARALLELISM) -timeout $(ACCTEST_TIMEOUT) -coverprofile cover.out
 
 # Run acceptance tests on macOS with the gitea-flux instance
 # Requires the following entry in /etc/hosts:
 # 127.0.0.1 gitea-flux
-testmacos: tidy fmt vet
+testmacos: $(EMBEDDED_MANIFESTS_TARGET) tidy fmt vet
 	TF_ACC=1 GITEA_HOSTNAME=gitea-flux go test ./... -v -parallel 1 -run TestAccBootstrapGit_Drift
 
-build:
+build: $(EMBEDDED_MANIFESTS_TARGET)
 	CGO_ENABLED=0 go build -o ./bin/terraform-provider-flux main.go
 
 .PHONY: docs
-docs: tools
+docs: $(EMBEDDED_MANIFESTS_TARGET) tools
 	tfplugindocs generate --ignore-deprecated true
 
 tools:

docs/resources/bootstrap_git.md

@@ -32,12 +32,13 @@ The following examples are available to help you use the provider:
 - `components_extra` (Set of String) List of extra components to include in the install manifests.
 - `delete_git_manifests` (Boolean) Delete manifests from git repository. Defaults to `true`.
 - `disable_secret_creation` (Boolean) Use the existing secret for flux controller and don't create one from bootstrap
+- `embedded_manifests` (Boolean) When enabled, the Flux manifests will be extracted from the provider binary instead of being downloaded from github.com. Defaults to `false`.
 - `image_pull_secret` (String) Kubernetes secret name used for pulling the toolkit images from a private registry.
 - `interval` (String) Interval at which to reconcile from bootstrap repository. Defaults to `1m0s`.
 - `keep_namespace` (Boolean) Keep the namespace after uninstalling Flux components. Defaults to `false`.
 - `kustomization_override` (String) Kustomization to override configuration set by default.
 - `log_level` (String) Log level for toolkit components. Defaults to `info`.
-- `manifests_path` (String) The install manifests are built from a GitHub release or kustomize overlay if using a local path. Defaults to `https://github.com/fluxcd/flux2/releases`.
+- `manifests_path` (String, Deprecated) The install manifests are built from a GitHub release or kustomize overlay if using a local path. Defaults to `https://github.com/fluxcd/flux2/releases`.
 - `namespace` (String) The namespace scope for install manifests. Defaults to `flux-system`. It will be created if it does not exist.
 - `network_policy` (Boolean) Deny ingress access to the toolkit controllers from other namespaces using network policies. Defaults to `true`.
 - `path` (String) Path relative to the repository root, when specified the cluster sync will be scoped to this path (immutable).
@@ -46,7 +47,7 @@ The following examples are available to help you use the provider:
 - `secret_name` (String) Name of the secret the sync credentials can be found in or stored to. Defaults to `flux-system`.
 - `timeouts` (Attributes) (see [below for nested schema](#nestedatt--timeouts))
 - `toleration_keys` (Set of String) List of toleration keys used to schedule the components pods onto nodes with matching taints.
-- `version` (String) Flux version. Defaults to `v2.2.3`.
+- `version` (String) Flux version. Defaults to `v2.2.3`. Has no effect when `embedded_manifests` is enabled.
 - `watch_all_namespaces` (Boolean) If true watch for custom resources in all namespaces. Defaults to `true`.
 
 ### Read-Only

go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.27.0
 	github.com/hashicorp/terraform-plugin-testing v1.3.0
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/otiai10/copy v1.14.0
 	github.com/stretchr/testify v1.9.0
 	k8s.io/api v0.29.3
 	k8s.io/apiextensions-apiserver v0.29.3

go.sum

@@ -365,6 +365,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8=
 github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
+github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
+github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
+github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
+github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
 github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=

internal/provider/provider.go

@@ -40,6 +40,8 @@ const (
 	defaultAuthor = "Flux"
 )
 
+var EmbeddedManifests string
+
 type Ssh struct {
 	Username   types.String `tfsdk:"username"`
 	Password   types.String `tfsdk:"password"`

internal/provider/resource_bootstrap_git.go

@@ -98,6 +98,7 @@ type bootstrapGitResourceData struct {
 	ComponentsExtra       types.Set            `tfsdk:"components_extra"`
 	DeleteGitManifests    types.Bool           `tfsdk:"delete_git_manifests"`
 	DisableSecretCreation types.Bool           `tfsdk:"disable_secret_creation"`
+	EmbeddedManifests     types.Bool           `tfsdk:"embedded_manifests"`
 	ID                    types.String         `tfsdk:"id"`
 	ImagePullSecret       types.String         `tfsdk:"image_pull_secret"`
 	Interval              customtypes.Duration `tfsdk:"interval"`
@@ -199,6 +200,10 @@ func (r *bootstrapGitResource) Schema(ctx context.Context, req resource.SchemaRe
 				Description: "Use the existing secret for flux controller and don't create one from bootstrap",
 				Optional:    true,
 			},
+			"embedded_manifests": schema.BoolAttribute{
+				Description: "When enabled, the Flux manifests will be extracted from the provider binary instead of being downloaded from github.com. Defaults to `false`.",
+				Optional:    true,
+			},
 			"id": schema.StringAttribute{
 				Computed: true,
 				PlanModifiers: []planmodifier.String{
@@ -241,8 +246,9 @@ func (r *bootstrapGitResource) Schema(ctx context.Context, req resource.SchemaRe
 				},
 			},
 			"manifests_path": schema.StringAttribute{
-				Description: fmt.Sprintf("The install manifests are built from a GitHub release or kustomize overlay if using a local path. Defaults to `%s`.", defaultOpts.BaseURL),
-				Optional:    true,
+				Description:        fmt.Sprintf("The install manifests are built from a GitHub release or kustomize overlay if using a local path. Defaults to `%s`.", defaultOpts.BaseURL),
+				Optional:           true,
+				DeprecationMessage: "This attribute is deprecated. Use the `embedded_manifests` attribute when running bootstrap on air-gapped environments.",
 			},
 			"namespace": schema.StringAttribute{
 				Description: fmt.Sprintf("The namespace scope for install manifests. Defaults to `%s`. It will be created if it does not exist.", defaultOpts.Namespace),
@@ -309,7 +315,7 @@ func (r *bootstrapGitResource) Schema(ctx context.Context, req resource.SchemaRe
 				},
 			},
 			"version": schema.StringAttribute{
-				Description: fmt.Sprintf("Flux version. Defaults to `%s`.", utils.DefaultFluxVersion),
+				Description: fmt.Sprintf("Flux version. Defaults to `%s`. Has no effect when `embedded_manifests` is enabled.", utils.DefaultFluxVersion),
 				Optional:    true,
 				Computed:    true,
 				Default:     stringdefault.StaticString(utils.DefaultFluxVersion),
@@ -446,6 +452,9 @@ func (r *bootstrapGitResource) Create(ctx context.Context, req resource.CreateRe
 	}
 
 	manifestsBase := ""
+	if data.EmbeddedManifests.ValueBool() {
+		manifestsBase = EmbeddedManifests
+	}
 	err = bootstrap.Run(ctx, bootstrapProvider, manifestsBase, installOpts, secretOpts, syncOpts, 2*time.Second, timeout)
 	if err != nil {
 		resp.Diagnostics.AddError("Bootstrap run error", err.Error())
@@ -679,6 +688,9 @@ func (r bootstrapGitResource) Update(ctx context.Context, req resource.UpdateReq
 		}
 
 		manifestsBase := ""
+		if data.EmbeddedManifests.ValueBool() {
+			manifestsBase = EmbeddedManifests
+		}
 		err = bootstrap.Run(ctx, bootstrapProvider, manifestsBase, installOpts, secretOpts, syncOpts, 2*time.Second, timeout)
 		if err != nil {
 			resp.Diagnostics.AddError("Bootstrap run error", err.Error())
@@ -1116,18 +1128,27 @@ func getSyncOptions(data bootstrapGitResourceData, url *url.URL, branch string)
 func getExpectedRepositoryFiles(data bootstrapGitResourceData, url *url.URL, branch string) (map[string]string, error) {
 	repositoryFiles := map[string]string{}
 	installOpts := getInstallOptions(data)
-	installManifests, err := install.Generate(installOpts, "")
+	manifestsBase := ""
+	if data.EmbeddedManifests.ValueBool() {
+		manifestsBase = EmbeddedManifests
+	}
+
+	installManifests, err := install.Generate(installOpts, manifestsBase)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate install manifests: %w", err)
 	}
+
 	repositoryFiles[installManifests.Path] = installManifests.Content
+
 	syncOpts := getSyncOptions(data, url, branch)
 	syncManifests, err := sync.Generate(syncOpts)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate sync manifests: %w", err)
 	}
+
 	repositoryFiles[syncManifests.Path] = syncManifests.Content
 	repositoryFiles[filepath.Join(data.Path.ValueString(), data.Namespace.ValueString(), konfig.DefaultKustomizationFileName())] = getKustomizationFile(data)
+
 	return repositoryFiles, nil
 }
 

internal/provider/resource_bootstrap_git_test.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -45,6 +46,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-testing/helper/acctest"
 	"github.com/hashicorp/terraform-plugin-testing/helper/resource"
 	"github.com/hashicorp/terraform-plugin-testing/terraform"
+	cp "github.com/otiai10/copy"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -182,6 +184,34 @@ func TestAccBootstrapGit_SSH(t *testing.T) {
 	})
 }
 
+func TestAccBootstrapGit_AirGapped(t *testing.T) {
+	env := setupEnvironment(t)
+	resource.ParallelTest(t, resource.TestCase{
+		ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
+		Steps: []resource.TestStep{
+			{
+				PreConfig: func() {
+					tmpBase := t.TempDir()
+					tmpBase, err := filepath.EvalSymlinks(tmpBase)
+					require.NoError(t, err)
+
+					_, b, _, _ := runtime.Caller(0)
+					manifestsBase := filepath.Join(filepath.Dir(b), "../..", "manifests")
+					err = cp.Copy(manifestsBase, tmpBase)
+					require.NoError(t, err)
+					EmbeddedManifests = tmpBase
+				},
+				Config: bootstrapAirGapped(env),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttrSet("flux_bootstrap_git.this", "repository_files.flux-system/kustomization.yaml"),
+					resource.TestCheckResourceAttrSet("flux_bootstrap_git.this", "repository_files.flux-system/gotk-components.yaml"),
+					resource.TestCheckResourceAttrSet("flux_bootstrap_git.this", "repository_files.flux-system/gotk-sync.yaml"),
+				),
+			},
+		},
+	})
+}
+
 func TestAccBootstrapGit_Drift(t *testing.T) {
 	env := setupEnvironment(t)
 	resource.ParallelTest(t, resource.TestCase{
@@ -526,6 +556,30 @@ EOF
 	`, env.kubeCfgPath, env.sshClone, env.privateKey)
 }
 
+func bootstrapAirGapped(env environment) string {
+	return fmt.Sprintf(`
+    provider "flux" {
+	  kubernetes = {
+        config_path = "%s"
+	  }
+	  git = {
+        url = "%s"
+        ssh = {
+          username = "git"
+          private_key = <<EOF
+%s
+EOF
+        }
+	  }
+    }
+
+    resource "flux_bootstrap_git" "this" {
+		embedded_manifests = true
+		version = "v0.0.0"
+	}
+	`, env.kubeCfgPath, env.sshClone, env.privateKey)
+}
+
 func bootstrapGitVersion(env environment, version string) string {
 	return fmt.Sprintf(`
     provider "flux" {