diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml index 4fe64ebb..0a74c2aa 100644 --- a/.github/workflows/master.yaml +++ b/.github/workflows/master.yaml @@ -44,15 +44,6 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.19.x - - name: Install Ginkgo testing framework - run: | - # Do the install from outside the code tree to avoid messing with go.sum - cd /tmp; go install github.com/onsi/ginkgo/ginkgo@v1.16.4 - - name: Setup gcloud CLI for GKE testing cluster - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.GKE_SA_KEY }} - export_default_credentials: true - name: Configure AWS credentials to use in AWS Stack tests uses: aws-actions/configure-aws-credentials@v1 with: @@ -72,15 +63,14 @@ jobs: - name: Tests run: | # Create GKE test cluster to install CRDs and use with the test operator. - scripts/ci-cluster-create.sh + scripts/ci-infra-create.sh # Source the env variables created in the script above cat ~/.envfile . ~/.envfile # Run tests - make codegen install-crds make test - name: Cleanup if: ${{ always() }} run: | - scripts/ci-cluster-destroy.sh + scripts/ci-infra-destroy.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e6cfe749..6e6d1db5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,15 +23,6 @@ jobs: uses: actions/setup-go@v2 with: go-version: "1.19.x" - - name: Install Ginkgo testing framework - run: | - # Do the install from outside the code tree to avoid messing with go.sum - cd /tmp; go install github.com/onsi/ginkgo/ginkgo@v1.16.4 - - name: Setup gcloud CLI for GKE testing cluster - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.GKE_SA_KEY }} - export_default_credentials: true - name: Configure AWS credentials to use in AWS Stack tests uses: aws-actions/configure-aws-credentials@v1 with: @@ -44,25 +35,24 @@ jobs: role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - name: Install Pulumi CLI uses: pulumi/setup-pulumi@v2 - - name: Set env variables + - name: Set env variables and path run: | echo '$HOME/.pulumi/bin' >> $GITHUB_PATH echo "STACK=ci-cluster-$(head /dev/urandom | LC_CTYPE=C tr -dc '[:lower:]' | head -c5)" >> $GITHUB_ENV - name: Tests run: | # Create GKE test cluster to install CRDs and use with the test operator. - scripts/ci-cluster-create.sh + scripts/ci-infra-create.sh # Source the env variables created in the script above cat ~/.envfile . ~/.envfile # Run tests - make codegen install-crds make test - name: Cleanup if: ${{ always() }} run: | - scripts/ci-cluster-destroy.sh + scripts/ci-infra-destroy.sh release: needs: [operator-int-tests] runs-on: ubuntu-latest diff --git a/.github/workflows/run-acceptance-tests.yaml b/.github/workflows/run-acceptance-tests.yaml index dca4d78c..caa1a9a4 100644 --- a/.github/workflows/run-acceptance-tests.yaml +++ b/.github/workflows/run-acceptance-tests.yaml @@ -56,15 +56,6 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.19.x - - name: Install Ginkgo testing framework - run: | - # Do the install from outside the code tree to avoid messing with go.sum - cd /tmp; go install github.com/onsi/ginkgo/ginkgo@v1.16.4 - - name: Setup gcloud CLI for GKE testing cluster - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.GKE_SA_KEY }} - export_default_credentials: true - name: Configure AWS credentials to use in AWS Stack tests uses: aws-actions/configure-aws-credentials@v1 with: @@ -84,14 +75,14 @@ jobs: - name: Tests run: | # Create GKE test cluster to install CRDs and use with the test operator. - scripts/ci-cluster-create.sh + scripts/ci-infra-create.sh # Source the env variables created in the script above cat ~/.envfile . ~/.envfile + # Run tests - make codegen install-crds make test - name: Cleanup if: ${{ always() }} run: | - scripts/ci-cluster-destroy.sh + scripts/ci-infra-destroy.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 16626355..05a1f1c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ CHANGELOG - Exit processing early when a stack is ready to be garbage collected [#322](https://github.com/pulumi/pulumi-kubernetes-operator/pull/322) +- Fix a goroutine leak [#319](https://github.com/pulumi/pulumi-kubernetes-operator/pull/319) ## 1.8.0 (2022-09-01) - Use go 1.18 for builds diff --git a/Makefile b/Makefile index 62dc0ed5..172e4507 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,10 @@ RELEASE ?= $(shell git describe --abbrev=0 --tags) default: build +.PHONY: download-test-deps +download-test-deps: + go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + install-crds: kubectl apply -f deploy/crds/pulumi.com_stacks.yaml @@ -40,8 +44,8 @@ build-static: push-image: docker push $(IMAGE_NAME):$(VERSION) -test: install-crds - ginkgo -v ./test/... +test: codegen download-test-deps + KUBEBUILDER_ASSETS="$(shell setup-envtest --use-env use -p path)" go test -v ./test/... deploy: kubectl apply -f deploy/yaml/service_account.yaml diff --git a/pkg/controller/stack/stack_controller.go b/pkg/controller/stack/stack_controller.go index ee6b687c..e367bd17 100644 --- a/pkg/controller/stack/stack_controller.go +++ b/pkg/controller/stack/stack_controller.go @@ -9,7 +9,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -620,38 +619,45 @@ func (sess *reconcileStackSession) runCmd(title string, cmd *exec.Cmd, workspace } // Capture stdout and stderr. - stdoutR, stdoutW := io.Pipe() - stderrR, stderrW := io.Pipe() - cmd.Stdout = stdoutW - cmd.Stderr = stderrW + stdoutR, err := cmd.StdoutPipe() + if err != nil { + return "", "", err + } + stderrR, err := cmd.StderrPipe() + if err != nil { + return "", "", err + } // Start the command asynchronously. - err := cmd.Start() + err = cmd.Start() if err != nil { return "", "", err } - // Kick off some goroutines to stream the output asynchronously. Since Pulumi can take - // a while to run, this helps to debug issues that might be ongoing before a command completes. var stdout bytes.Buffer var stderr bytes.Buffer + + // We want to echo both stderr and stdout as they are written; so at least one of them must be + // in another goroutine. + stderrClosed := make(chan struct{}) + errs := bufio.NewScanner(stderrR) go func() { - outs := bufio.NewScanner(stdoutR) - for outs.Scan() { - text := outs.Text() - sess.logger.Debug(title, "Dir", cmd.Dir, "Path", cmd.Path, "Args", cmd.Args, "Stdout", text) - stdout.WriteString(text + "\n") - } - }() - go func() { - errs := bufio.NewScanner(stderrR) for errs.Scan() { text := errs.Text() sess.logger.Debug(title, "Dir", cmd.Dir, "Path", cmd.Path, "Args", cmd.Args, "Text", text) stderr.WriteString(text + "\n") } + close(stderrClosed) }() + outs := bufio.NewScanner(stdoutR) + for outs.Scan() { + text := outs.Text() + sess.logger.Debug(title, "Dir", cmd.Dir, "Path", cmd.Path, "Args", cmd.Args, "Stdout", text) + stdout.WriteString(text + "\n") + } + <-stderrClosed + // Now wait for the command to finish. No matter what, return everything written to stdout and // stderr, in addition to the resulting error, if any. err = cmd.Wait() @@ -1139,9 +1145,15 @@ func (sess *reconcileStackSession) updateResource(o client.Object) error { }) } -func (sess *reconcileStackSession) updateResourceStatus(o client.Object) error { +func (sess *reconcileStackSession) updateResourceStatus(o *pulumiv1.Stack) error { + name := types.NamespacedName{Name: o.Name, Namespace: o.Namespace} return retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return sess.kubeClient.Status().Update(context.TODO(), o) + var s pulumiv1.Stack + if err := sess.kubeClient.Get(context.TODO(), name, &s); err != nil { + return err + } + s.Status = o.Status + return sess.kubeClient.Status().Update(context.TODO(), &s) }) } diff --git a/scripts/ci-cluster-create.sh b/scripts/ci-infra-create.sh similarity index 58% rename from scripts/ci-cluster-create.sh rename to scripts/ci-infra-create.sh index 4eac43a6..cad77a8c 100755 --- a/scripts/ci-cluster-create.sh +++ b/scripts/ci-infra-create.sh @@ -1,18 +1,6 @@ #!/usr/bin/env bash set -o nounset -o errexit -o pipefail -echo Creating ephemeral Kubernetes cluster for CI testing... - -pushd test/ci-cluster -yarn install --json --verbose >out -tail -10 out -pulumi stack init "${STACK}" -pulumi up --skip-preview --yes - -mkdir -p "$HOME/.kube/" -pulumi stack output kubeconfig --show-secrets >~/.kube/config -popd - echo Creating S3 backend and KMS key pushd test/s3backend diff --git a/scripts/ci-cluster-destroy.sh b/scripts/ci-infra-destroy.sh similarity index 67% rename from scripts/ci-cluster-destroy.sh rename to scripts/ci-infra-destroy.sh index 9cd46c95..9e98f086 100755 --- a/scripts/ci-cluster-destroy.sh +++ b/scripts/ci-infra-destroy.sh @@ -1,14 +1,6 @@ #!/usr/bin/env bash set -o nounset -o errexit -o pipefail -echo Deleting ephemeral Kubernetes cluster... - -pushd test/ci-cluster -pulumi stack select "${STACK}" && \ - pulumi destroy --skip-preview --yes && \ - pulumi stack rm --yes -popd - echo Deleting S3 backend and KMS Key... pushd test/s3backend @@ -21,4 +13,3 @@ echo Destroying stack pulumi destroy --skip-preview --yes && \ pulumi stack rm --yes popd - diff --git a/test/ci-cluster/.gitignore b/test/ci-cluster/.gitignore deleted file mode 100644 index f13a50af..00000000 --- a/test/ci-cluster/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/bin/ -/node_modules/ -/.vscode diff --git a/test/ci-cluster/Pulumi.yaml b/test/ci-cluster/Pulumi.yaml deleted file mode 100644 index c3d25308..00000000 --- a/test/ci-cluster/Pulumi.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name: k8s-ci-cluster -description: GKE cluster for pulumi-kubernetes CI -runtime: nodejs diff --git a/test/ci-cluster/config.ts b/test/ci-cluster/config.ts deleted file mode 100644 index 4bd7c3d9..00000000 --- a/test/ci-cluster/config.ts +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2016-2021, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import { Config } from "@pulumi/pulumi"; -import * as random from "@pulumi/random"; - -const config = new Config(); - -export const gcpProject = "pulumi-k8s-operator"; -export const gcpZone = "a"; -export const gcpLocation = "us-west1-a"; - -// nodeCount is the number of cluster nodes to provision. Defaults to 3 if unspecified. -export const nodeCount = config.getNumber("nodeCount") || 2; - -// nodeMachineType is the machine type to use for cluster nodes. Defaults to n1-standard-2 if unspecified. -// See https://cloud.google.com/compute/docs/machine-types for more details on available machine types. -export const nodeMachineType = config.get("nodeMachineType") || "n1-standard-2"; diff --git a/test/ci-cluster/gke.ts b/test/ci-cluster/gke.ts deleted file mode 100644 index db29c16d..00000000 --- a/test/ci-cluster/gke.ts +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2016-2021, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import * as gcp from "@pulumi/gcp"; -import * as k8s from "@pulumi/kubernetes"; -import * as pulumi from "@pulumi/pulumi"; -import * as config from "./config"; - -export class GkeCluster extends pulumi.ComponentResource { - public cluster: gcp.container.Cluster; - public kubeconfig: pulumi.Output; - public provider: k8s.Provider; - - constructor(name: string, - opts: pulumi.ComponentResourceOptions = {}) { - super("pulumi-kubernetes:ci:GkeCluster", name, {}, opts); - - // Find the latest 1.20.x engine version. - const engineVersion = gcp.container.getEngineVersions({ - location: config.gcpLocation, - project: config.gcpProject, - }).then(v => v.validMasterVersions.filter(v => v.startsWith("1.20"))[0]); - - // Create the GKE cluster. - const k8sCluster = new gcp.container.Cluster("ephemeral-ci-cluster", { - initialNodeCount: config.nodeCount, - nodeVersion: engineVersion, - minMasterVersion: engineVersion, - nodeConfig: { - machineType: config.nodeMachineType, - oauthScopes: [ - "https://www.googleapis.com/auth/compute", - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring" - ], - }, - project: config.gcpProject, - location: config.gcpLocation, - }, {parent: this}); - this.cluster = k8sCluster; - - // Manufacture a GKE-style Kubeconfig. Note that this is slightly "different" because of the way GKE requires - // gcloud to be in the picture for cluster authentication (rather than using the client cert/key directly). - this.kubeconfig = pulumi.all([k8sCluster.name, k8sCluster.endpoint, k8sCluster.masterAuth]).apply( - ([name, endpoint, auth]) => { - const context = `${config.gcpProject}_${config.gcpZone}_${name}`; - return `apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: ${auth.clusterCaCertificate} - server: https://${endpoint} - name: ${context} -contexts: -- context: - cluster: ${context} - user: ${context} - name: ${context} -current-context: ${context} -kind: Config -preferences: {} -users: -- name: ${context} - user: - auth-provider: - config: - cmd-args: config config-helper --format=json - cmd-path: gcloud - expiry-key: '{.credential.token_expiry}' - token-key: '{.credential.access_token}' - name: gcp -`; - }); - - // Export a Kubernetes provider instance that uses our cluster from above. - this.provider = new k8s.Provider("gke", {kubeconfig: this.kubeconfig}, {parent: this}); - } -} - diff --git a/test/ci-cluster/index.ts b/test/ci-cluster/index.ts deleted file mode 100644 index 859ce113..00000000 --- a/test/ci-cluster/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2016-2019, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import * as gke from "./gke"; - -// Create Kubernetes clusters. -const gkeCluster = new gke.GkeCluster("ci-cluster"); -export const k8sProvider = gkeCluster.provider; -export const kubeconfig = gkeCluster.kubeconfig; diff --git a/test/ci-cluster/package.json b/test/ci-cluster/package.json deleted file mode 100644 index f4692834..00000000 --- a/test/ci-cluster/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "kubernetes-ci-cluster", - "devDependencies": { - "@types/node": "latest" - }, - "dependencies": { - "@pulumi/gcp": "^5.1.0", - "@pulumi/kubernetes": "^3.0.0", - "@pulumi/pulumi": "^3.1.0", - "@pulumi/random": "^4.0.0" - } -} diff --git a/test/ci-cluster/tsconfig.json b/test/ci-cluster/tsconfig.json deleted file mode 100644 index e9f2c3a1..00000000 --- a/test/ci-cluster/tsconfig.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "compilerOptions": { - "outDir": "bin", - "target": "es6", - "lib": [ - "es6" - ], - "module": "commonjs", - "moduleResolution": "node", - "declaration": true, - "sourceMap": true, - "stripInternal": true, - "experimentalDecorators": true, - "pretty": true, - "noFallthroughCasesInSwitch": true, - "noImplicitAny": true, - "noImplicitReturns": true, - "forceConsistentCasingInFileNames": true, - "strictNullChecks": true - }, - "files": [ - "config.ts", - "index.ts", - "gke.ts" - ] -} diff --git a/test/stack_controller_test.go b/test/stack_controller_test.go index 4044d452..5298bdbc 100644 --- a/test/stack_controller_test.go +++ b/test/stack_controller_test.go @@ -44,8 +44,9 @@ var ( const namespace = "default" const pulumiAPISecretName = "pulumi-api-secret" const pulumiAWSSecretName = "pulumi-aws-secrets" -const timeout = time.Minute * 10 -const interval = time.Second * 1 +const k8sOpTimeout = 10 * time.Second +const stackExecTimeout = 3 * time.Minute +const interval = time.Second * 5 var _ = Describe("Stack Controller", func() { whoami, err := exec.Command("pulumi", "whoami").Output() @@ -102,7 +103,7 @@ var _ = Describe("Stack Controller", func() { return false } return !fetched.CreationTimestamp.IsZero() && fetched.Data != nil - }, timeout, interval).Should(BeTrue()) + }, k8sOpTimeout, interval).Should(BeTrue()) // Create the Pulumi AWS k8s secret pulumiAWSSecret = generateSecret(pulumiAWSSecretName, namespace, @@ -121,7 +122,7 @@ var _ = Describe("Stack Controller", func() { return false } return !fetched.CreationTimestamp.IsZero() && fetched.Data != nil - }, timeout, interval).Should(BeTrue()) + }, k8sOpTimeout, interval).Should(BeTrue()) // Create the passphrase secret passphraseSecret = generateSecret("passphrase-secret", namespace, @@ -138,19 +139,16 @@ var _ = Describe("Stack Controller", func() { return false } return !fetched.CreationTimestamp.IsZero() && fetched.Data != nil - }, timeout, interval).Should(BeTrue()) + }, k8sOpTimeout, interval).Should(BeTrue()) }) AfterEach(func() { By("Deleting left over stacks") - deletionPolicy := metav1.DeletePropagationForeground Expect(k8sClient.DeleteAllOf( ctx, &pulumiv1alpha1.Stack{}, client.InNamespace(namespace), - &client.DeleteAllOfOptions{ - DeleteOptions: client.DeleteOptions{PropagationPolicy: &deletionPolicy}, - }, + &client.DeleteAllOfOptions{}, )).Should(Succeed()) Eventually(func() bool { @@ -159,7 +157,7 @@ var _ = Describe("Stack Controller", func() { return false } return len(stacksList.Items) == 0 - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) // stacks will be finalized, so allow time for that to happen if pulumiAPISecret != nil { By("Deleting the Stack Pulumi API Secret") @@ -167,7 +165,7 @@ var _ = Describe("Stack Controller", func() { Eventually(func() bool { err := k8sClient.Get(ctx, types.NamespacedName{Name: pulumiAPISecret.Name, Namespace: namespace}, pulumiAPISecret) return k8serrors.IsNotFound(err) - }, timeout, interval).Should(BeTrue()) + }, k8sOpTimeout, interval).Should(BeTrue()) } if pulumiAWSSecret != nil { By("Deleting the Stack AWS Credentials Secret") @@ -175,7 +173,7 @@ var _ = Describe("Stack Controller", func() { Eventually(func() bool { err := k8sClient.Get(ctx, types.NamespacedName{Name: pulumiAWSSecret.Name, Namespace: namespace}, pulumiAWSSecret) return k8serrors.IsNotFound(err) - }, timeout, interval).Should(BeTrue()) + }, k8sOpTimeout, interval).Should(BeTrue()) } if passphraseSecret != nil { By("Deleting the Passphrase Secret") @@ -183,7 +181,7 @@ var _ = Describe("Stack Controller", func() { Eventually(func() bool { err := k8sClient.Get(ctx, types.NamespacedName{Name: passphraseSecret.Name, Namespace: namespace}, passphraseSecret) return k8serrors.IsNotFound(err) - }, timeout, interval).Should(BeTrue()) + }, k8sOpTimeout, interval).Should(BeTrue()) } }) @@ -237,7 +235,7 @@ var _ = Describe("Stack Controller", func() { } return stackUpdatedToCommit(&fetched.Status, stack.Spec.Commit) - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) // Validate outputs. Expect(fetched.Status.Outputs).Should(BeEquivalentTo(shared.StackOutputs{ "region": v1.JSON{Raw: []byte(`"us-west-2"`)}, @@ -254,7 +252,7 @@ var _ = Describe("Stack Controller", func() { Eventually(func() bool { err := k8sClient.Get(ctx, types.NamespacedName{Name: stack.Name, Namespace: namespace}, toDelete) return k8serrors.IsNotFound(err) - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) // allow time for finalizer to run }) It("Should deploy an AWS S3 Stack successfully, then deploy a commit update successfully", func() { @@ -292,7 +290,7 @@ var _ = Describe("Stack Controller", func() { return false } return stackUpdatedToCommit(&original.Status, stack.Spec.Commit) - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) // Update the stack config (this time to cause a failure) original.Spec.Config["aws:region"] = "us-nonexistent-1" @@ -311,7 +309,7 @@ var _ = Describe("Stack Controller", func() { configChanged.Status.LastUpdate.State == shared.FailedStackStateMessage } return false - }) + }, stackExecTimeout, interval).Should(BeTrue()) // Update the stack commit to a different commit. Need retries because of // competing retries within the operator due to failure. @@ -324,7 +322,7 @@ var _ = Describe("Stack Controller", func() { return false } return true - }, timeout, interval).Should(BeTrue(), "%#v", configChanged) + }, stackExecTimeout, interval).Should(BeTrue(), "%#v", configChanged) // Check that the stack update was attempted but failed fetched := &pulumiv1.Stack{} @@ -339,7 +337,7 @@ var _ = Describe("Stack Controller", func() { fetched.Status.LastUpdate.State == shared.FailedStackStateMessage } return false - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) Eventually(func() bool { if err := k8sClient.Get(ctx, types.NamespacedName{Name: stack.Name, Namespace: namespace}, fetched); err != nil { @@ -351,7 +349,7 @@ var _ = Describe("Stack Controller", func() { return false } return true - }, timeout, interval).Should(BeTrue()) + }, k8sOpTimeout, interval).Should(BeTrue()) // Check that the stack update attempted and succeeded after the region fix Eventually(func() bool { @@ -360,7 +358,7 @@ var _ = Describe("Stack Controller", func() { return false } return stackUpdatedToCommit(&fetched.Status, commit) - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) // Delete the Stack toDelete := &pulumiv1.Stack{} @@ -370,7 +368,7 @@ var _ = Describe("Stack Controller", func() { Eventually(func() bool { err := k8sClient.Get(ctx, types.NamespacedName{Name: stack.Name, Namespace: namespace}, toDelete) return k8serrors.IsNotFound(err) - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) }) It("Should deploy an AWS S3 Stack successfully with credentials passed through EnvRefs", func() { @@ -381,12 +379,7 @@ var _ = Describe("Stack Controller", func() { // Write a secret to a temp directory. This is a stand-in for other mechanisms to reify the secrets // on the file system. This is not a recommended way to store/pass secrets. - Eventually(func() bool { - if err = os.WriteFile(filepath.Join(secretsDir, pulumiAPISecretName), []byte(pulumiAccessToken), 0600); err != nil { - return false - } - return true - }, timeout, interval).Should(BeTrue()) + Expect(os.WriteFile(filepath.Join(secretsDir, pulumiAPISecretName), []byte(pulumiAccessToken), 0600)).To(Succeed()) // Define the stack spec spec := shared.StackSpec{ @@ -421,7 +414,7 @@ var _ = Describe("Stack Controller", func() { return false } return stackUpdatedToCommit(&fetched.Status, stack.Spec.Commit) - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) }) It("Should deploy an AWS S3 Stack successfully using S3 backend", func() { @@ -500,7 +493,7 @@ var _ = Describe("Stack Controller", func() { return false } return stackUpdatedToCommit(&initial.Status, stack.Spec.Commit) - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) // Check that secrets are not leaked Expect(initial.Status.Outputs).Should(HaveKeyWithValue( @@ -514,7 +507,7 @@ var _ = Describe("Stack Controller", func() { Eventually(func() bool { err := k8sClient.Get(ctx, types.NamespacedName{Name: stack.Name, Namespace: namespace}, toDelete) return k8serrors.IsNotFound(err) - }, timeout, interval).Should(BeTrue()) + }, stackExecTimeout, interval).Should(BeTrue()) }) }) diff --git a/test/suite_test.go b/test/suite_test.go index 3ab73341..3cd14bb5 100644 --- a/test/suite_test.go +++ b/test/suite_test.go @@ -3,15 +3,15 @@ package tests import ( + "context" "os" + "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" - "github.com/onsi/gomega/gexec" // Used to auth against GKE clusters that use gcloud creds. _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -41,6 +41,7 @@ var cfg *rest.Config var k8sClient client.Client var k8sManager ctrl.Manager var testEnv *envtest.Environment +var shutdownController func() func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) @@ -63,13 +64,12 @@ func TestAPIs(t *testing.T) { } var secretsDir string -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) By("bootstrapping test environment") - t := true testEnv = &envtest.Environment{ - UseExistingCluster: &t, + CRDDirectoryPaths: []string{filepath.Join("..", "deploy", "crds")}, } cfg, err := testEnv.Start() @@ -93,10 +93,12 @@ var _ = BeforeSuite(func(done Done) { err = controller.Add(k8sManager) Expect(err).ToNot(HaveOccurred()) + ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) go func() { - err = k8sManager.Start(ctrl.SetupSignalHandler()) + err = k8sManager.Start(ctx) Expect(err).ToNot(HaveOccurred()) }() + shutdownController = cancel k8sClient = k8sManager.GetClient() Expect(k8sClient).ToNot(BeNil()) @@ -106,13 +108,13 @@ var _ = BeforeSuite(func(done Done) { if err != nil { Fail("Failed to create secret temp directory") } - - close(done) }, 60) var _ = AfterSuite(func() { By("tearing down the test environment") - gexec.KillAndWait(5 * time.Second) + if shutdownController != nil { + shutdownController() + } err := testEnv.Stop() Expect(err).ToNot(HaveOccurred())