roachtest: run workload from the tenant node

The secure URL refers to paths on disk on the clusters in the
node. Since we only create the tenant-scoped certs on the tenant node,
we need to run workload from that node.

BACKPORT NOTES: Also adds the `multi-tenant` owner to the roachtest
registry and TEAMS.yml, which were not yet added to the release branch.

Fixes cockroachdb#82266
Depends on cockroachdb#83703

Release note: None

TEAMS.yaml

@@ -73,6 +73,8 @@ cockroachdb/cluster-ui:
   triage_column_id: 6598672
 cockroachdb/obs-inf-prs:
   triage_column_id: 14196277
+cockroachdb/multi-tenant:
+  triage_column_id: 16575985
 cockroachdb/unowned:
   aliases:
     cockroachdb/rfc-prs: other

pkg/cmd/roachtest/registry/owners.go

@@ -28,4 +28,5 @@ const (
 	OwnerStorage       Owner = `storage`
 	OwnerTestEng       Owner = `test-eng`
 	OwnerDevInf        Owner = `dev-inf`
+	OwnerMultiTenant   Owner = `multi-tenant`
 )

pkg/cmd/roachtest/tests/acceptance.go

@@ -40,11 +40,6 @@ func registerAcceptance(r registry.Registry) {
 				encryptionSupport: registry.EncryptionAlwaysDisabled,
 			},
 			{name: "gossip/locality-address", fn: runCheckLocalityIPAddress},
-			{
-				name:       "multitenant",
-				minVersion: "v20.2.0", // multitenancy is introduced in this cycle
-				fn:         runAcceptanceMultitenant,
-			},
 			{name: "reset-quorum", fn: runResetQuorum, numNodes: 8},
 			{
 				name: "many-splits", fn: runManySplits,
@@ -65,6 +60,13 @@ func registerAcceptance(r registry.Registry) {
 				timeout:    30 * time.Minute,
 			},
 		},
+		registry.OwnerMultiTenant: {
+			{
+				name: "multitenant",
+				skip: "https://github.com/cockroachdb/cockroach/issues/81506",
+				fn:   runAcceptanceMultitenant,
+			},
+		},
 		registry.OwnerServer: {
 			{name: "build-info", fn: RunBuildInfo},
 			{name: "build-analyze", fn: RunBuildAnalyze},

pkg/cmd/roachtest/tests/multitenant_upgrade.go

@@ -29,7 +29,7 @@ func registerMultiTenantUpgrade(r registry.Registry) {
 	r.Add(registry.TestSpec{
 		Name:              "multitenant-upgrade",
 		Cluster:           r.MakeClusterSpec(2),
-		Owner:             registry.OwnerKV,
+		Owner:             registry.OwnerMultiTenant,
 		NonReleaseBlocker: false,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runMultiTenantUpgrade(ctx, t, c, *t.BuildVersion())

pkg/cmd/roachtest/tests/smoketest_secure.go

@@ -12,13 +12,15 @@ package tests
 
 import (
 	"context"
+	gosql "database/sql"
 	"fmt"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
+	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/stretchr/testify/require"
 )
 
@@ -40,4 +42,50 @@ func registerSecure(r registry.Registry) {
 			},
 		})
 	}
+	r.Add(registry.TestSpec{
+		Name:    "smoketest/secure/multitenant",
+		Owner:   registry.OwnerMultiTenant,
+		Cluster: r.MakeClusterSpec(2),
+		Run:     multitenantSmokeTest,
+	})
+}
+
+// multitenantSmokeTest verifies that a secure sql pod can connect to kv server
+// and that tenant is is properly transmitted via cert.
+func multitenantSmokeTest(ctx context.Context, t test.Test, c cluster.Cluster) {
+	c.Put(ctx, t.Cockroach(), "./cockroach")
+	settings := install.MakeClusterSettings(install.SecureOption(true))
+	c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, c.Node(1))
+
+	// make sure connections to kvserver work
+	db := c.Conn(ctx, t.L(), 1)
+	defer db.Close()
+	_, err := db.QueryContext(ctx, `SELECT 1`)
+	require.NoError(t, err)
+
+	tenID := 11
+	ten := createTenantNode(ctx, t, c, c.Node(1), tenID, 2, 8011, 9011)
+	runner := sqlutils.MakeSQLRunner(c.Conn(ctx, t.L(), 1))
+	runner.Exec(t, `SELECT crdb_internal.create_tenant($1)`, tenID)
+	ten.start(ctx, t, c, "./cockroach")
+
+	// this doesn't work yet, roachprod knows nothing about tenants
+	// db = c.Conn(ctx, t.L(), 2)
+	// defer db.Close()
+
+	tdb, err := gosql.Open("postgres", ten.pgURL)
+	require.NoError(t, err)
+	_, err = tdb.QueryContext(ctx, `SELECT 1`)
+	require.NoError(t, err)
+
+	// init kv and check new database was done right
+	cmd := fmt.Sprintf("./cockroach workload init kv '%s'", ten.secureURL())
+	err = c.RunE(ctx, c.Node(2), cmd)
+	require.NoError(t, err)
+
+	sqlutils.MakeSQLRunner(db).CheckQueryResultsRetry(t, fmt.Sprintf(`
+    SELECT count(*) > 0
+    FROM crdb_internal.ranges
+    WHERE start_pretty LIKE '/Tenant/%d/%%';
+`, tenID), [][]string{{"true"}})
 }