Run (or run in trap) deferred commands explicitly

The test runner `bats` installs its own trap(s), and in doing so,
appears to interfere with the trap `defer.bash` sets. So, instead of
setting the trap within the sourced file, require the
deferred-command-runner to be invoked explicitly. `run.bash` can set a
trap to do so; the tests must call `run_deferred` (formerly `on_exit`)
in their teardown function.

test/e2e/12_sync.bats

@@ -17,10 +17,15 @@ function setup() {
   export GIT_SSH_COMMAND="$git_ssh_cmd"
   # shellcheck disable=SC2154
   git_port_forward_pid="${git_srv_result[1]}"
+  # Teardown the created port-forward to gitsrv and restore Git settings.
+  defer kill "$git_port_forward_pid"
+
   install_flux_with_fluxctl
+
   # Clone the repo and
   clone_dir="$(mktemp -d)"
   git clone -b master ssh://git@localhost/git-server/repos/cluster.git "$clone_dir"
+  defer rm -rf "$clone_dir"
   # shellcheck disable=SC2164
   cd "$clone_dir"
 }
@@ -81,9 +86,7 @@ function setup() {
 }
 
 function teardown() {
-  rm -rf "$clone_dir"
-  # Teardown the created port-forward to gitsrv and restore Git settings.
-  kill "$git_port_forward_pid"
+  run_deferred
   # Uninstall Flux and the global resources it installs.
   uninstall_flux_with_fluxctl
   # Removing the namespace also takes care of removing gitsrv.

test/e2e/13_sync_gc.bats

@@ -16,6 +16,7 @@ function setup() {
   export GIT_SSH_COMMAND="$git_ssh_cmd"
   # shellcheck disable=SC2154
   git_port_forward_pid="${git_srv_result[1]}"
+  defer kill "$git_port_forward_pid"
   install_flux_with_fluxctl "13_sync_gc"
 }
 
@@ -52,7 +53,7 @@ function setup() {
 }
 
 function teardown() {
-  kill "$git_port_forward_pid"
+  run_deferred
   # Removing the namespace also takes care of removing Flux and gitsrv.
   kubectl delete namespace "$FLUX_NAMESPACE"
   # Only remove the demo workloads after Flux, so that they cannot be recreated.

test/e2e/20_commit_signing.bats

@@ -19,10 +19,13 @@ function setup() {
   export GIT_SSH_COMMAND="$git_ssh_cmd"
   # shellcheck disable=SC2154
   git_port_forward_pid="${git_srv_result[1]}"
+  # Teardown the created port-forward to gitsrv.
+  defer kill "$git_port_forward_pid"
 
   # Create a temporary GNUPGHOME
   tmp_gnupghome=$(mktemp -d)
   export GNUPGHOME="$tmp_gnupghome"
+  defer rm -rf "$tmp_gnupghome"
 
   # Install Flux, with a new GPG key and signing enabled
   gpg_key=$(create_gpg_key)
@@ -66,11 +69,9 @@ function setup() {
 }
 
 function teardown() {
-  # Teardown the created port-forward to gitsrv.
-  kill "$git_port_forward_pid"
+  run_deferred
   # Kill the agent and remove temporary GNUPGHOME
   gpgconf --kill gpg-agent
-  rm -rf "$tmp_gnupghome"
   # Uninstall Flux and the global resources it installs.
   uninstall_flux_gpg
   # Removing the namespace also takes care of removing Flux and gitsrv.

test/e2e/lib/defer.bash

@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 
+# This lets you call `defer` to record an action to do later;
+# `run_deferred` should be called in an EXIT trap, either explicitly:
+#
+#    trap run_deferred EXIT
+#
+# or when using with tests, by calling it in the teardown function
+# (which bats will arrange to run).
+
 declare -a on_exit_items
 
-function on_exit() {
+function run_deferred() {
   if [ "${#on_exit_items[@]}" -gt 0 ]; then
     echo -e '\nRunning deferred items, please do not interrupt until they are done:'
   fi
@@ -12,8 +20,6 @@ function on_exit() {
   done
 }
 
-trap on_exit EXIT
-
 function defer() {
   on_exit_items=("$*" "${on_exit_items[@]}")
 }

test/e2e/lib/install.bash

@@ -53,9 +53,10 @@ fluxctl_install_cmd="fluxctl install --git-url=ssh://git@gitsrv/git-server/repos
 function install_flux_with_fluxctl() {
   local kustomtmp
   kustomtmp="$(mktemp -d)"
+  defer "if [ -d \"${kustomtmp}\" ]; then rm -r \"${kustomtmp}\"; fi"
   mkdir "${kustomtmp}/base"
   # This generates the base manifests, which we'll then patch with a kustomization
-  echo ">>> writing base configuration to ${kustomtmp}" >&3
+  echo ">>> writing base configuration to ${kustomtmp}/base" >&3
   $fluxctl_install_cmd --namespace "${FLUX_NAMESPACE}" -o "${kustomtmp}/base/"
   # Everything goes into one directory, but not everything is
   # necessarily used by the kustomization

test/e2e/run.bash

@@ -17,6 +17,7 @@ USING_KIND=false
 
 # shellcheck disable=SC1090
 source "${E2E_DIR}/lib/defer.bash"
+trap run_deferred EXIT
 
 # Check if there is a kubernetes cluster running, otherwise use Kind
 if ! kubectl version > /dev/null 2>&1; then