test: add support for sourcing secrets from secret manager (#919)

For now testing and hooking up the idtoken integration tests to
run on continuous.

integration-tests/idtoken/idtoken_test.go

@@ -20,13 +20,17 @@ import (
 )
 
 const (
-	envCredentialFile = "API_GO_CLIENT_SA"
-	envTokenAudience  = "API_GO_CLIENT_TOKEN_AUDIENCE"
+	envCredentialFile = "GCLOUD_TESTS_GOLANG_KEY"
+
+	aud = "http://example.com"
 )
 
 func TestNewTokenSource(t *testing.T) {
-	aud := os.Getenv(envTokenAudience)
-	ts, err := idtoken.NewTokenSource(context.Background(), aud, option.WithCredentialsFile(os.Getenv(envCredentialFile)))
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+	os.Setenv(envCredentialFile, "/Users/codyoss/creds/codyoss-workspace-2b1fb7cd40c0.json")
+	ts, err := idtoken.NewTokenSource(context.Background(), "http://example.com", option.WithCredentialsFile(os.Getenv(envCredentialFile)))
 	if err != nil {
 		t.Fatalf("unable to create TokenSource: %v", err)
 	}
@@ -49,7 +53,10 @@ func TestNewTokenSource(t *testing.T) {
 }
 
 func TestNewClient_WithCredentialFile(t *testing.T) {
-	aud := os.Getenv(envTokenAudience)
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+	os.Setenv(envCredentialFile, "/Users/codyoss/creds/codyoss-workspace-2b1fb7cd40c0.json")
 	client, err := idtoken.NewClient(context.Background(), aud, option.WithCredentialsFile(os.Getenv(envCredentialFile)))
 	if err != nil {
 		t.Fatalf("unable to create Client: %v", err)
@@ -68,7 +75,9 @@ func TestNewClient_WithCredentialFile(t *testing.T) {
 }
 
 func TestNewClient_WithCredentialJSON(t *testing.T) {
-	aud := os.Getenv(envTokenAudience)
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
 	ctx := context.Background()
 	creds, err := google.FindDefaultCredentials(ctx)
 	if err != nil {

internal/kokoro/populate-secrets.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2021 Google LLC.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -eo pipefail
+
+function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; }
+function msg() { println "$*" >&2; }
+function println() { printf '%s\n' "$(now) $*"; }
+
+# Populates requested secrets set in SECRET_MANAGER_KEYS from service account:
+# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com
+SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
+msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
+mkdir -p ${SECRET_LOCATION}
+for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g"); do
+    msg "Retrieving secret ${key}"
+    docker run --entrypoint=gcloud \
+        --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \
+        gcr.io/google.com/cloudsdktool/cloud-sdk \
+        secrets versions access latest \
+        --credential-file-override=${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json \
+        --project cloud-devrel-kokoro-resources \
+        --secret ${key} > \
+        "${SECRET_LOCATION}/${key}"
+    if [[ $? == 0 ]]; then
+        msg "Secret written to ${SECRET_LOCATION}/${key}"
+    else
+        msg "Error retrieving secret ${key}"
+    fi
+done

internal/kokoro/test.sh

@@ -4,11 +4,11 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-# TODO(deklerk) Add integration tests when it's secure to do so. b/64723143
-
 # Fail on any error
 set -eo pipefail
 
+export GCLOUD_TESTS_GOLANG_KEY="${KOKORO_GFILE_DIR}/secret_manager/go-cloud-integration-service-account"
+
 # Display commands being run
 set -x
 
@@ -35,7 +35,13 @@ try3 go mod download
 ./internal/kokoro/vet.sh
 
 # Testing the generator itself depends on a generation step
-cd google-api-go-generator; go generate; cd ..
+cd google-api-go-generator
+go generate
+cd ..
 
 # Run tests and tee output to log file, to be pushed to GCS as artifact.
-go test -race -v -short ./... 2>&1 | tee $KOKORO_ARTIFACTS_DIR/$KOKORO_GERRIT_CHANGE_NUMBER.txt
+if [[ $KOKORO_JOB_NAME == *"continuous"* ]]; then
+    go test -race -v ./... 2>&1 | tee $KOKORO_ARTIFACTS_DIR/$KOKORO_GERRIT_CHANGE_NUMBER.txt
+else
+    go test -race -v -short ./... 2>&1 | tee $KOKORO_ARTIFACTS_DIR/$KOKORO_GERRIT_CHANGE_NUMBER.txt
+fi

internal/kokoro/trampoline.sh

@@ -13,4 +13,5 @@ function cleanup() {
     echo "cleanup";
 }
 trap cleanup EXIT
+$(dirname $0)/populate-secrets.sh # Secret Manager secrets.
 python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py"