From 7d7a24e37b84814820c01233c7689904fc6397aa Mon Sep 17 00:00:00 2001 From: Henrik Johansson Date: Fri, 19 Jul 2019 09:27:09 +0200 Subject: [PATCH] store: oracle mutations more resilient Gemini used to treat mutation errors for both systems as equal. Now gemini treats failures in the Oracle as unrecoverable and it will simply not attempt the same mutation in the test system. The idea is that since the Oracle is the blueprint, there is no need to to attempt a test mutation if the blueprint didn't change. --- store/store.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/store/store.go b/store/store.go index 0aa598fb..913bb470 100644 --- a/store/store.go +++ b/store/store.go @@ -69,23 +69,28 @@ func New(schema *gemini.Schema, testCluster *gocql.ClusterConfig, oracleCluster schema: schema, system: "oracle", ops: ops, - maxRetriesMutate: cfg.MaxRetriesMutate, + maxRetriesMutate: cfg.MaxRetriesMutate + 10, maxRetriesMutateSleep: cfg.MaxRetriesMutateSleep, logger: logger, }, + logger: logger.Named("delegating_store"), } } type delegatingStore struct { oracleStore storeLoader testStore storeLoader + logger *zap.Logger } -func (ds delegatingStore) Mutate(ctx context.Context, builder qb.Builder, values ...interface{}) (err error) { +func (ds delegatingStore) Mutate(ctx context.Context, builder qb.Builder, values ...interface{}) error { ts := time.Now() - err = multierr.Append(err, mutate(ctx, ds.testStore, ts, builder, values...)) - err = multierr.Append(err, mutate(ctx, ds.oracleStore, ts, builder, values...)) - return + if err := mutate(ctx, ds.oracleStore, ts, builder, values...); err != nil { + // Oracle failed, transition cannot take place + ds.logger.Info("oracle failed mutation, transition to next state impossible so continuing with next mutation", zap.Error(err)) + return nil + } + return mutate(ctx, ds.testStore, ts, builder, values...) } func mutate(ctx context.Context, s storeLoader, ts time.Time, builder qb.Builder, values ...interface{}) error {