feat(test): add etcd simulator (#5084)

* add etcd to simulation Signed-off-by: Runji Wang <[email protected]> * update madsim and note the etcd timeout issue Signed-off-by: Runji Wang <[email protected]> * enable etcd service timeout Signed-off-by: Runji Wang <[email protected]> * update madsim Signed-off-by: Runji Wang <[email protected]> * ci: disable recovery test since it's too slow Signed-off-by: Runji Wang <[email protected]> * add --etcd-timeout-rate argument Signed-off-by: Runji Wang <[email protected]> * meta: fix lock order Signed-off-by: Runji Wang <[email protected]> * improve error info readability Signed-off-by: Runji Wang <[email protected]> Signed-off-by: Runji Wang <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
risingwavelabs · Sep 6, 2022 · ede7432 · ede7432
1 parent d574cf5
commit ede7432
Show file tree

Hide file tree

Showing 5 changed files with 59 additions and 13 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh
@@ -30,12 +30,12 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, batch"
 seq 10 | parallel MADSIM_TEST_SEED={} $RUNNER -j 16 './e2e_test/batch/\*\*/\*.slt'
 
 # bugs here! Tracking issue https://github.com/risingwavelabs/risingwave/issues/4527
-echo "--- deterministic simulation e2e, ci-3cn-1fe, recovery, streaming"
-seq 1 | parallel MADSIM_TEST_SEED={} $RUNNER --kill-compute './e2e_test/streaming/\*\*/\*.slt' || true
+# echo "--- deterministic simulation e2e, ci-3cn-1fe, recovery, streaming"
+# seq 1 | parallel MADSIM_TEST_SEED={} $RUNNER --kill-compute './e2e_test/streaming/\*\*/\*.slt' || true
 
 # bugs here! Tracking issue https://github.com/risingwavelabs/risingwave/issues/4527
-echo "--- deterministic simulation e2e, ci-3cn-1fe, recovery, batch"
-seq 1 | parallel MADSIM_TEST_SEED={} $RUNNER --kill-compute './e2e_test/batch/\*\*/\*.slt' || true
+# echo "--- deterministic simulation e2e, ci-3cn-1fe, recovery, batch"
+# seq 1 | parallel MADSIM_TEST_SEED={} $RUNNER --kill-compute './e2e_test/batch/\*\*/\*.slt' || true
 
 echo "--- deterministic simulation e2e, ci-3cn-1fe, fuzzing"
 seq 1 | parallel MADSIM_TEST_SEED={} $RUNNER --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata
diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml
@@ -15,7 +15,7 @@ clap = { version = "3", features = ["derive", "env"] }
 crc32fast = "1"
 derivative = "2"
 either = "1"
-etcd-client = "0.10"
+etcd-client = { version = "0.2", package = "madsim-etcd-client" }
 fail = "0.5"
 function_name = "0.3.0"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }

diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml
@@ -7,9 +7,10 @@ edition = "2021"
 [dependencies]
 async-trait = "0.1"
 clap = "3"
+etcd-client = { version = "0.2.6", package = "madsim-etcd-client" }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 glob = "0.3"
-madsim = "0.2.4"
+madsim = "0.2.6"
 rand = "0.8"
 risingwave_compactor = { path = "../../storage/compactor" }
 risingwave_compute = { path = "../../compute" }

diff --git a/src/tests/simulation/src/main.rs b/src/tests/simulation/src/main.rs
@@ -66,6 +66,10 @@ pub struct Args {
     #[clap(short, long)]
     jobs: Option<usize>,
 
+    /// The probability of etcd request timeout.
+    #[clap(long, default_value = "0.0")]
+    etcd_timeout_rate: f32,
+
     /// Randomly kill the meta node after each query.
     ///
     /// Currently only available when `-j` is not set.
@@ -109,6 +113,23 @@ async fn main() {
     println!("seed = {}", handle.seed());
     println!("{:?}", args);
 
+    // etcd node
+    handle
+        .create_node()
+        .name("etcd")
+        .ip("192.168.10.1".parse().unwrap())
+        .init(|| async {
+            let addr = "0.0.0.0:2388".parse().unwrap();
+            etcd_client::SimServer::builder()
+                .timeout_rate(args.etcd_timeout_rate)
+                .serve(addr)
+                .await
+                .unwrap();
+        })
+        .build();
+    // wait for the service to be ready
+    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+
     // meta node
     handle
         .create_node()
@@ -121,12 +142,16 @@ async fn main() {
                 // "src/config/risingwave.toml",
                 "--listen-addr",
                 "0.0.0.0:5690",
+                "--backend",
+                "etcd",
+                "--etcd-endpoints",
+                "192.168.10.1:2388",
             ]);
             risingwave_meta::start(opts).await
         })
         .build();
     // wait for the service to be ready
-    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+    tokio::time::sleep(std::time::Duration::from_secs(30)).await;
 
     // frontend node
     let mut frontend_ip = vec![];
@@ -309,7 +334,11 @@ async fn run_slt_task(glob: &str, host: &str) {
         let file = file.unwrap();
         let path = file.as_path();
         println!("{}", path.display());
-        tester.run_file_async(path).await.unwrap();
+        tester
+            .run_file_async(path)
+            .await
+            .map_err(|e| panic!("{e}"))
+            .unwrap()
     }
 }
 
@@ -324,6 +353,7 @@ async fn run_parallel_slt_task(
     tester
         .run_parallel_async(glob, hosts.to_vec(), Risingwave::connect, jobs)
         .await
+        .map_err(|e| panic!("{e}"))
 }
 
 struct Risingwave {