cloudtest,mzcompose: generalize external service configuration

This commit avoids duplicating the configuration of CockroachDB and
MinIO across our various test harnesses. The mzcompose `Cockroach`,
`MinioSetup`, and `Materialized` services learn convenience options that
configure them all appropriately for one another. Additionally, the
CockroachDB initialization is shared between cloudtest and mzcompose.
This yields a natural spot to install the workaround for the CockroachDB
bug discovered in #16726.

Fix #16726.

Author: benesch

ci/nightly/pipeline.yml

@@ -378,7 +378,7 @@ steps:
     plugins:
       - ./ci/plugins/mzcompose:
           composition: platform-checks
-          args: [--scenario=RestartPostgresBackend, --execution-mode=oneatatime]
+          args: [--scenario=RestartCockroach, --execution-mode=oneatatime]
 
   - id: checks-oneatatime-restart-redpanda-debezium
     label: "Checks oneatatime + restart Redpanda & Debezium"
@@ -448,7 +448,7 @@ steps:
     plugins:
       - ./ci/plugins/mzcompose:
           composition: platform-checks
-          args: [--scenario=RestartPostgresBackend, --execution-mode=parallel]
+          args: [--scenario=RestartCockroach, --execution-mode=parallel]
 
   - id: checks-parallel-restart-redpanda
     label: "Checks parallel + restart Redpanda & Debezium"

ci/test/pipeline.template.yml

@@ -508,8 +508,8 @@ steps:
           composition: platform-checks
           args: [--scenario=KillClusterdStorage]
 
-  - id: checks-restart-postgres-backend
-    label: "Checks + restart Postgres backend"
+  - id: checks-restart-cockroach
+    label: "Checks + restart Cockroach"
     depends_on: build-x86_64
     inputs: [misc/python/materialize/checks]
     timeout_in_minutes: 30
@@ -518,7 +518,7 @@ steps:
     plugins:
       - ./ci/plugins/mzcompose:
           composition: platform-checks
-          args: [--scenario=RestartPostgresBackend]
+          args: [--scenario=RestartCockroach]
 
   - id: checks-restart-source-postgres
     label: "Checks + restart source Postgres"

misc/cockroach/setup_materialize.sql

@@ -0,0 +1,20 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- Sets up a CockroachDB cluster for use by Materialize.
+
+-- See: https://github.com/cockroachdb/cockroach/issues/93892
+-- See: https://github.com/MaterializeInc/materialize/issues/16726
+-- TODO: remove this workaround before upgrading to CockroachDB 22.2 in
+-- production.
+SET CLUSTER SETTING sql.stats.forecasts.enabled = false;
+
+CREATE SCHEMA consensus;
+CREATE SCHEMA adapter;
+CREATE SCHEMA storage;

misc/dbt-materialize/mzcompose.py

@@ -11,17 +11,15 @@
 from typing import Dict, List, Optional
 
 from materialize.mzcompose import Composition, WorkflowArgumentParser
-from materialize.mzcompose.services import Materialized, Redpanda, Service, TestCerts
+from materialize.mzcompose.services import Materialized, Redpanda, Service
 
 SERVICES = [
-    TestCerts(),
     Materialized(),
     Redpanda(),
     Service(
         "dbt-test",
         {
             "mzbuild": "dbt-materialize",
-            "depends_on": ["test-certs"],
             "environment": [
                 "TMPDIR=/share/tmp",
             ],
@@ -63,7 +61,6 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
             materialized = Materialized(
                 options=test_case.materialized_options,
                 image=test_case.materialized_image,
-                depends_on=["test-certs"],
                 volumes_extra=["secrets:/secrets"],
             )
 

misc/python/materialize/checks/debezium.py

@@ -19,7 +19,7 @@ def initialize(self) -> Testdrive:
         return Testdrive(
             dedent(
                 """
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 CREATE TABLE debezium_table (f1 TEXT, f2 INTEGER, f3 INTEGER, f4 TEXT, PRIMARY KEY (f1, f2));
                 ALTER TABLE debezium_table REPLICA IDENTITY FULL;
                 INSERT INTO debezium_table SELECT 'A', generate_series, 1, REPEAT('X', 16) FROM generate_series(1,1000);
@@ -29,7 +29,7 @@ def initialize(self) -> Testdrive:
                     "name": "psql-connector",
                   "config": {
                     "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
-                    "database.hostname": "postgres-source",
+                    "database.hostname": "postgres",
                     "database.port": "5432",
                     "database.user": "postgres",
                     "database.password": "postgres",
@@ -59,7 +59,7 @@ def initialize(self) -> Testdrive:
                   FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_conn
                   ENVELOPE DEBEZIUM;
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO debezium_table SELECT 'B', generate_series, 1, REPEAT('X', 16) FROM generate_series(1,1000);
 
                 > CREATE MATERIALIZED VIEW debezium_view1 AS SELECT f1, f3, SUM(LENGTH(f4)) FROM debezium_source1 GROUP BY f1, f3;
@@ -76,7 +76,7 @@ def manipulate(self) -> List[Testdrive]:
             Testdrive(dedent(s))
             for s in [
                 """
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 BEGIN;
                 INSERT INTO debezium_table SELECT 'C', generate_series, 1, REPEAT('X', 16) FROM generate_series(1,1000);
                 UPDATE debezium_table SET f3 = f3 + 1;
@@ -87,7 +87,7 @@ def manipulate(self) -> List[Testdrive]:
                   FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_conn
                   ENVELOPE DEBEZIUM;
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 BEGIN;
                 INSERT INTO debezium_table SELECT 'D', generate_series, 1, REPEAT('X', 16) FROM generate_series(1,1000);
                 UPDATE debezium_table SET f3 = f3 + 1;
@@ -96,7 +96,7 @@ def manipulate(self) -> List[Testdrive]:
                 > CREATE MATERIALIZED VIEW debezium_view2 AS SELECT f1, f3, SUM(LENGTH(f4)) FROM debezium_source2 GROUP BY f1, f3;
                 """,
                 """
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 BEGIN;
                 INSERT INTO debezium_table SELECT 'E', generate_series, 1, REPEAT('X', 16) FROM generate_series(1,1000);
                 UPDATE debezium_table SET f3 = f3 + 1;
@@ -107,7 +107,7 @@ def manipulate(self) -> List[Testdrive]:
                   FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_conn
                   ENVELOPE DEBEZIUM;
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 BEGIN;
                 INSERT INTO debezium_table SELECT 'F', generate_series, 1, REPEAT('X', 16) FROM generate_series(1,1000);
                 UPDATE debezium_table SET f3 = f3 + 1;

misc/python/materialize/checks/mzcompose_actions.py

@@ -27,12 +27,6 @@ class MzcomposeAction(Action):
 
 
 class StartMz(MzcomposeAction):
-    DEFAULT_MZ_OPTIONS = [
-        "--persist-consensus-url=postgresql://postgres:postgres@postgres-backend:5432?options=--search_path=consensus",
-        "--storage-stash-url=postgresql://postgres:postgres@postgres-backend:5432?options=--search_path=storage",
-        "--adapter-stash-url=postgresql://postgres:postgres@postgres-backend:5432?options=--search_path=adapter",
-    ]
-
     def __init__(
         self, tag: Optional[str] = None, environment_extra: List[str] = []
     ) -> None:
@@ -46,7 +40,7 @@ def execute(self, e: Executor) -> None:
         print(f"Starting Mz using image {image}")
         mz = Materialized(
             image=image,
-            options=StartMz.DEFAULT_MZ_OPTIONS,
+            external_cockroach=True,
             environment_extra=self.environment_extra,
         )
 
@@ -134,22 +128,21 @@ def execute(self, e: Executor) -> None:
             c.start_and_wait_for_tcp(services=[service])
 
 
-class RestartPostgresBackend(MzcomposeAction):
+class RestartCockroach(MzcomposeAction):
     def execute(self, e: Executor) -> None:
         c = e.mzcompose_composition()
 
-        c.kill("postgres-backend")
-        c.up("postgres-backend")
-        c.wait_for_postgres(service="postgres-backend")
+        c.kill("cockroach")
+        c.up("cockroach")
 
 
 class RestartSourcePostgres(MzcomposeAction):
     def execute(self, e: Executor) -> None:
         c = e.mzcompose_composition()
 
-        c.kill("postgres-source")
-        c.up("postgres-source")
-        c.wait_for_postgres(service="postgres-source")
+        c.kill("postgres")
+        c.up("postgres")
+        c.wait_for_postgres(service="postgres")
 
 
 class KillClusterdStorage(MzcomposeAction):

misc/python/materialize/checks/pg_cdc.py

@@ -22,12 +22,12 @@ def initialize(self) -> Testdrive:
                 > CREATE SECRET pgpass1 AS 'postgres';
 
                 > CREATE CONNECTION pg1 FOR POSTGRES
-                  HOST 'postgres-source',
+                  HOST 'postgres',
                   DATABASE postgres,
                   USER postgres1,
                   PASSWORD SECRET pgpass1
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 CREATE USER postgres1 WITH SUPERUSER PASSWORD 'postgres';
                 ALTER USER postgres1 WITH replication;
                 DROP PUBLICATION IF EXISTS postgres_source;
@@ -54,25 +54,25 @@ def manipulate(self) -> List[Testdrive]:
                   (PUBLICATION 'postgres_source')
                   FOR TABLES (postgres_source_table AS postgres_source_tableA);
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_source_table SELECT 'B', 1, REPEAT('X', 1024) FROM generate_series(1,100);
                 UPDATE postgres_source_table SET f2 = f2 + 1;
 
                 > CREATE SECRET pgpass2 AS 'postgres';
 
                 > CREATE CONNECTION pg2 FOR POSTGRES
-                  HOST 'postgres-source',
+                  HOST 'postgres',
                   DATABASE postgres,
                   USER postgres1,
                   PASSWORD SECRET pgpass1
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_source_table SELECT 'C', 1, REPEAT('X', 1024) FROM generate_series(1,100);
                 UPDATE postgres_source_table SET f2 = f2 + 1;
                 """,
                 """
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_source_table SELECT 'D', 1, REPEAT('X', 1024) FROM generate_series(1,100);
                 UPDATE postgres_source_table SET f2 = f2 + 1;
 
@@ -81,18 +81,18 @@ def manipulate(self) -> List[Testdrive]:
                   (PUBLICATION 'postgres_source')
                   FOR TABLES (postgres_source_table AS postgres_source_tableB);
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_source_table SELECT 'E', 1, REPEAT('X', 1024) FROM generate_series(1,100);
                 UPDATE postgres_source_table SET f2 = f2 + 1;
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_source_table SELECT 'F', 1, REPEAT('X', 1024) FROM generate_series(1,100);
                 UPDATE postgres_source_table SET f2 = f2 + 1;
 
                 > CREATE SECRET pgpass3 AS 'postgres';
 
                 > CREATE CONNECTION pg3 FOR POSTGRES
-                  HOST 'postgres-source',
+                  HOST 'postgres',
                   DATABASE postgres,
                   USER postgres1,
                   PASSWORD SECRET pgpass3
@@ -102,12 +102,12 @@ def manipulate(self) -> List[Testdrive]:
                   (PUBLICATION 'postgres_source')
                   FOR TABLES (postgres_source_table AS postgres_source_tableC);
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_source_table SELECT 'G', 1, REPEAT('X', 1024) FROM generate_series(1,100);
                 UPDATE postgres_source_table SET f2 = f2 + 1;
 
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_source_table SELECT 'H', 1, REPEAT('X', 1024) FROM generate_series(1,100);
                 UPDATE postgres_source_table SET f2 = f2 + 1;
                 """,
@@ -160,12 +160,12 @@ def initialize(self) -> Testdrive:
                 > CREATE SECRET postgres_mz_now_pass AS 'postgres';
 
                 > CREATE CONNECTION postgres_mz_now_conn FOR POSTGRES
-                  HOST 'postgres-source',
+                  HOST 'postgres',
                   DATABASE postgres,
                   USER postgres2,
                   PASSWORD SECRET postgres_mz_now_pass
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 CREATE USER postgres2 WITH SUPERUSER PASSWORD 'postgres';
                 ALTER USER postgres2 WITH replication;
                 DROP PUBLICATION IF EXISTS postgres_mz_now_publication;
@@ -201,7 +201,7 @@ def manipulate(self) -> List[Testdrive]:
             Testdrive(dedent(s))
             for s in [
                 """
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'A2');
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'B2');
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'C2');
@@ -211,7 +211,7 @@ def manipulate(self) -> List[Testdrive]:
                 UPDATE postgres_mz_now_table SET f1 = NOW() WHERE f2 = 'C1';
                 """,
                 """
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'A3');
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'B3');
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'C3');
@@ -230,7 +230,7 @@ def validate(self) -> Testdrive:
                 > SELECT COUNT(*) FROM postgres_mz_now_table;
                 13
 
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'A4');
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'B4');
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'C4');
@@ -250,7 +250,7 @@ def validate(self) -> Testdrive:
                 0
 
                 # Rollback the last INSERTs so that validate() can be called multiple times
-                $ postgres-execute connection=postgres://postgres:postgres@postgres-source
+                $ postgres-execute connection=postgres://postgres:postgres@postgres
                 INSERT INTO postgres_mz_now_table VALUES (NOW(), 'B3');
                 DELETE FROM postgres_mz_now_table WHERE f2 LIKE '%4%';
                 """

misc/python/materialize/checks/scenarios.py

@@ -29,7 +29,7 @@
 )
 from materialize.checks.mzcompose_actions import KillMz
 from materialize.checks.mzcompose_actions import (
-    RestartPostgresBackend as RestartPostgresBackendAction,
+    RestartCockroach as RestartCockroachAction,
 )
 from materialize.checks.mzcompose_actions import (
     RestartRedpandaDebezium as RestartRedpandaDebeziumAction,
@@ -155,16 +155,16 @@ def actions(self) -> List[Action]:
         ]
 
 
-class RestartPostgresBackend(Scenario):
+class RestartCockroach(Scenario):
     def actions(self) -> List[Action]:
         return [
             StartMz(),
             Initialize(self.checks),
-            RestartPostgresBackendAction(),
+            RestartCockroachAction(),
             Manipulate(self.checks, phase=1),
-            RestartPostgresBackendAction(),
+            RestartCockroachAction(),
             Manipulate(self.checks, phase=2),
-            RestartPostgresBackendAction(),
+            RestartCockroachAction(),
             Validate(self.checks),
         ]