storage: add env var for aggressive consistency checks

We saw a consistency failure in cockroachdb#29252 that would've been much more
useful had it occurred close to the time around which the inconsistency
must have been introduced. Instead of leaving it to chance, add a switch
that runs aggressive checks in (roach) tests that want them such as the
clearrange test.

Release note: None

Author: tbg

pkg/storage/consistency_queue.go

@@ -22,6 +22,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/gossip"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/settings"
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/grpcutil"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -33,6 +34,8 @@ var consistencyCheckInterval = settings.RegisterNonNegativeDurationSetting(
 	24*time.Hour,
 )
 
+var testingAggressiveConsistencyChecks = envutil.EnvOrDefaultBool("COCKROACH_CONSISTENCY_AGGRESSIVE", false)
+
 type consistencyQueue struct {
 	*baseQueue
 	interval       func() time.Duration

pkg/storage/merge_queue.go

@@ -273,6 +273,11 @@ func (mq *mergeQueue) process(
 		// as purgatory-worthy.
 		return rangeMergePurgatoryError{err}
 	}
+	if testingAggressiveConsistencyChecks {
+		if err := mq.store.consistencyQueue.process(ctx, lhsRepl, sysCfg); err != nil {
+			log.Warning(ctx, err)
+		}
+	}
 	return nil
 }
 

pkg/storage/replicate_queue.go

@@ -265,6 +265,11 @@ func (rq *replicateQueue) process(
 			// Enqueue this replica again to see if there are more changes to be made.
 			rq.MaybeAdd(repl, rq.store.Clock().Now())
 		}
+		if testingAggressiveConsistencyChecks {
+			if err := rq.store.consistencyQueue.process(ctx, repl, sysCfg); err != nil {
+				log.Warning(ctx, err)
+			}
+		}
 		return nil
 	}
 	return errors.Errorf("failed to replicate after %d retries", retryOpts.MaxRetries)