internal/e2e: add a "cue mod upload" end-to-end test with gcloud

Now that we have the first experimental version of "cue mod upload",
and we support authenticating with cloud provider registries
via docker credential helpers such as gcloud,
we can write an end-to-end test with Google Cloud Artifact Registry.

This test does the same as our existing GitHub App test at a high level,
publishing one module version and then using it as a dependency.
However, instead of creating a GitHub repo and pushing a git tag,
we use "cue mod upload" with gcloud as a credential helper,
making sure that gcloud shares the same configuration as the host.

The test script is named after "gcloud" since all the auth bits
are particular to Google Cloud right now.
We can later add another script, or make this script more generic,
if we want to test against other common OCI/Docker registry providers.

Note that the test is skipped for now for the sake of our CI,
since we don't have a service account set up with a key on it yet.
The next commit will do that bit of setup in our CI script.

Signed-off-by: Daniel Martí <[email protected]>
Change-Id: I2e45865bf86788c42ba3ef2d7422fd21b9af221a
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1171911
Reviewed-by: Roger Peppe <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>

Author: mvdan

internal/e2e/script_test.go

@@ -15,12 +15,14 @@
 package e2e_test
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -78,6 +80,11 @@ var (
 	githubOrg = envOr("GITHUB_ORG", "cue-labs-modules-testing")
 	// githubKeep leaves the newly created repo around when set to true.
 	githubKeep = envOr("GITHUB_KEEP", "false")
+
+	// gcloudRegistry is an existing Google Cloud Artifact Registry repository
+	// to upload module versions to via "cue mod upload",
+	// and authenticated via gcloud's configuration in the host environment.
+	gcloudRegistry = envOr("GCLOUD_REGISTRY", "europe-west1-docker.pkg.dev/project-unity-377819/modules-e2e-registry")
 )
 
 func TestScript(t *testing.T) {
@@ -104,9 +111,7 @@ func TestScript(t *testing.T) {
 				// Not a global, since
 				githubToken := envMust(t, "GITHUB_TOKEN")
 
-				// TODO: name the repo after ts.Name once the API lands
-				// TODO: add a short random suffix to prevent time collisions
-				repoName := time.Now().UTC().Format("2006-01-02.15-04-05")
+				repoName := testModuleName(ts)
 				client := github.NewClient(nil).WithAuthToken(githubToken)
 				ctx := context.TODO()
 
@@ -165,6 +170,31 @@ func TestScript(t *testing.T) {
 				}
 				ts.Fatalf("timed out waiting for module")
 			},
+			// gcloud-auth-docker configures gcloud so that it uses the host's existing configuration,
+			// and sets CUE_REGISTRY and CUE_REGISTRY_HOST according to gcloudRegistry.
+			"gcloud-auth-docker": func(ts *testscript.TestScript, neg bool, args []string) {
+				if neg || len(args) > 0 {
+					ts.Fatalf("usage: gcloud-auth-docker")
+				}
+				// The test script needs to be able to run gcloud as a docker credential helper.
+				// gcloud will be accessible via $PATH without issue, but it needs to use its host config,
+				// so we pass it along as $CLOUDSDK_CONFIG to not share the host's entire $HOME.
+				//
+				// We assume that the host already has gcloud authorized to upload OCI artifacts,
+				// via either a user account (gcloud auth login) or a service account key (gcloud auth activate-service-account).
+				gcloudConfigPath, err := exec.Command("gcloud", "info", "--format=value(config.paths.global_config_dir)").Output()
+				ts.Check(err)
+				ts.Setenv("CLOUDSDK_CONFIG", string(bytes.TrimSpace(gcloudConfigPath)))
+
+				// The module path can be anything we want in this case,
+				// but we might as well make it unique and realistic.
+				ts.Setenv("MODULE", "domain.test/"+testModuleName(ts))
+
+				ts.Setenv("CUE_REGISTRY", gcloudRegistry)
+				// TODO: reuse internal/mod/modresolve.parseRegistry, returning a Location with Host.
+				gcloudRegistryHost, _, _ := strings.Cut(gcloudRegistry, "/")
+				ts.Setenv("CUE_REGISTRY_HOST", gcloudRegistryHost)
+			},
 		},
 	}
 	testscript.Run(t, p)
@@ -190,3 +220,13 @@ func tsExpand(ts *testscript.TestScript, s string) string {
 		return ts.Getenv(key)
 	})
 }
+
+// testModuleName creates a unique string without any slashes
+// which can be used as the base name for a module path to publish,
+// so that test runs don't conflict with one another
+// and can be easily attributed to a point in time.
+func testModuleName(ts *testscript.TestScript) string {
+	// TODO: name the repo after ts.Name once the API lands
+	// TODO: add a short random suffix to prevent time collisions
+	return time.Now().UTC().Format("2006-01-02.15-04-05")
+}

internal/e2e/testdata/script/gcloud_upload.txtar

@@ -0,0 +1,50 @@
+# Publish a version for a simple CUE module via "cue mod upload"
+# to an off-the-shelf OCI registry which requires authentication.
+# Then fetch that module as a dependency via cmd/cue.
+
+skip 'TODO(mvdan): set up a service account key on GitHub Actions'
+
+gcloud-auth-docker # sets: MODULE, CUE_REGISTRY, CUE_REGISTRY_HOST, CLOUDSDK_CONFIG
+env DOCKER_CONFIG=$WORK/docker-config
+env-fill docker-config/config.json
+
+env VERSION=v0.0.1
+env MODVER=${MODULE}@v0
+
+cd publish
+
+# TODO: replace by "cue mod init" once it supports versioned module names.
+# cue mod init ${MODVER}
+env-fill cue.mod/module.cue
+
+exec cue mod upload ${VERSION}
+
+cd ../depend
+
+env-fill cue.mod/module.cue out_foo.cue
+exec cue export
+cmp stdout export.golden
+
+-- docker-config/config.json --
+{"credHelpers": {"${CUE_REGISTRY_HOST}": "gcloud"}}
+-- publish/cue.mod/module.cue --
+module: "${MODVER}"
+-- publish/foo.cue --
+package publish
+
+foo: "foo value"
+
+-- depend/cue.mod/module.cue --
+module: "depend.localhost"
+
+deps: "${MODVER}": v: "${VERSION}"
+-- depend/out_foo.cue --
+package depend
+
+import mt "${MODVER}:publish"
+
+out: mt.foo
+-- depend/export.golden --
+{
+    "out": "foo value"
+}

internal/e2e/testdata/script/github_app_public.txtar

@@ -9,7 +9,7 @@ env MODVER=${MODULE}@v0
 
 cd publish
 
-# TODO: replace by "cue mod init" once it's available.
+# TODO: replace by "cue mod init" once it supports versioned module names.
 # cue mod init ${MODVER}
 env-fill cue.mod/module.cue