DNM: proof of concept test

This test fails with the "overlapping range" error if the
WriteSyncNoop call is removed.

Release note: None

pkg/storage/merge_queue.go

@@ -38,12 +38,12 @@ const (
 	// purgatory make merge attempts. Since merges are relatively untested, the
 	// reasons that a range may fail to merge are unknown, so the merge queue has
 	// a large purgatory interval.
-	mergeQueuePurgatoryCheckInterval = 1 * time.Minute
+	mergeQueuePurgatoryCheckInterval = 10 * time.Millisecond
 
 	// The current implementation of merges requires rewriting the right-hand data
 	// onto the left-hand range, even when the ranges are collocated. This is
 	// expensive, so limit to one merge at a time.
-	mergeQueueConcurrency = 1
+	mergeQueueConcurrency = 8
 )
 
 // MergeQueueInterval is a setting that controls how often the merge queue waits

pkg/storage/replica_command.go

@@ -556,7 +556,9 @@ func waitForApplication(
 				return err
 			})
 		}
-		return g.Wait()
+		err := g.Wait()
+		time.Sleep(5 * time.Millisecond)
+		return err
 	})
 }
 

pkg/storage/store.go

@@ -2227,6 +2227,19 @@ func (s *Store) MergeRange(
 		return err
 	}
 
+	// if crash := rand.Intn(3) == 0; crash && s.StoreID() == 1 {
+	// 	go func() {
+	// 		time.Sleep(time.Duration(rand.Intn(int(5 * time.Millisecond.Nanoseconds()))))
+	// 		p, err := os.FindProcess(os.Getpid())
+	// 		if err != nil {
+	// 			log.Fatal(ctx, err)
+	// 		}
+	// 		if err := p.Kill(); err != nil {
+	// 			log.Fatal(ctx, err)
+	// 		}
+	// 	}()
+	// }
+
 	leftRepl.raftMu.AssertHeld()
 	rightRepl.raftMu.AssertHeld()
 

pkg/storage/stores_server.go

@@ -17,6 +17,8 @@ package storage
 import (
 	"bytes"
 	"context"
+	"math/rand"
+	"os"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
@@ -117,6 +119,20 @@ func (is Server) WaitForApplication(
 				if err := engine.WriteSyncNoop(ctx, s.engine); err != nil {
 					return err
 				}
+
+				if crash := rand.Intn(3) == 0; crash && s.StoreID() == 1 {
+					go func() {
+						time.Sleep(time.Duration(rand.Intn(int(5 * time.Millisecond.Nanoseconds()))))
+						p, err := os.FindProcess(os.Getpid())
+						if err != nil {
+							log.Fatal(ctx, err)
+						}
+						if err := p.Kill(); err != nil {
+							log.Fatal(ctx, err)
+						}
+					}()
+				}
+
 				return nil
 			}
 		}

run.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export COCKROACH_ENGINE_MAX_SYNC_DURATION=60s
+
+killall -9 cockroach || true
+make build
+rm -rf cockroach-data* || true
+for i in 0 1 2 3; do
+  ./cockroach start --max-offset 10ms --insecure --host 127.0.0.1 --port $((26257+i)) --http-port $((8080+i)) --background --store "cockroach-data${i}" --join 127.0.0.1:26257
+  if [ $i -eq 0 ]; then ./cockroach init --insecure; fi
+done
+echo "
+SET CLUSTER SETTING kv.range_merge.queue_interval = '1ns';
+SET CLUSTER SETTING kv.range_merge.queue_enabled = false;
+CREATE TABLE IF NOT EXISTS data (id INT PRIMARY KEY);
+ALTER TABLE data SPLIT AT SELECT i FROM generate_series(1, 1000) AS g(i);
+SET CLUSTER SETTING kv.range_merge.queue_enabled = true;
+" | ./cockroach sql --insecure
+
+./cockroach quit --insecure || true
+sleep 1
+
+
+retval=137
+while [ $retval -eq 137 ];
+do
+  set +e
+  ./cockroach start --max-offset 10ms --insecure --logtostderr --host 127.0.0.1 --store cockroach-data0
+  retval=$?
+  set -e
+  echo "exit code: $retval"
+done