From 929402cb9141b0a4ff7d3a938486dc0913274694 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 11 Nov 2021 21:02:04 +0000 Subject: [PATCH 1/8] qual subset handling --- cache/index_item.go | 28 +++++++++++++++-- cache/pending_index_item.go | 2 +- cache/query_cache.go | 61 +++++++++++++++++++++--------------- cache/query_cache_pending.go | 4 +-- grpc/proto/plugin.pb.go | 5 +-- grpc/proto/quals.go | 27 ++++++++++++++++ 6 files changed, 94 insertions(+), 33 deletions(-) diff --git a/cache/index_item.go b/cache/index_item.go index e05492c0..15b5cdc8 100644 --- a/cache/index_item.go +++ b/cache/index_item.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" ) // IndexBucket contains index items for all cache results for a given table and qual set @@ -22,9 +23,9 @@ func (b *IndexBucket) Append(item *IndexItem) *IndexBucket { } // Get finds an index item which satisfies all columns -func (b *IndexBucket) Get(columns []string, limit int64) *IndexItem { +func (b *IndexBucket) Get(qualMap map[string]*proto.Quals, columns []string, limit int64) *IndexItem { for _, item := range b.Items { - if item.SatisfiesColumns(columns) && item.SatisfiesLimit(limit) { + if item.SatisfiesQuals(qualMap) && item.SatisfiesColumns(columns) && item.SatisfiesLimit(limit) { return item } } @@ -37,13 +38,15 @@ type IndexItem struct { Columns []string Key string Limit int64 + Quals map[string]*proto.Quals } -func NewIndexItem(columns []string, key string, limit int64) *IndexItem { +func NewIndexItem(columns []string, key string, limit int64, quals map[string]*proto.Quals) *IndexItem { return &IndexItem{ Columns: columns, Key: key, Limit: limit, + Quals: quals, } } @@ -78,3 +81,22 @@ func (i IndexItem) SatisfiesLimit(limit int64) bool { return res } + +// SatisfiesQuals +//check quals must be MORE restrictive the our quals +// i.e. our quals must be a subset of check quals +// eg +//our quals [], check quals [id="1"] -> SATISFIED +//our quals [id="1"], check quals [id="1"] -> SATISFIED +//our quals [id="1"], check quals [id="1", foo=2] -> SATISFIED +//our quals [id="1", foo=2], check quals [id="1"] -> NOT SATISFIED +func (i IndexItem) SatisfiesQuals(checkQualMap map[string]*proto.Quals) bool { + for col, indexQuals := range i.Quals { + // if we have quals the passed in map does not, we DO NOT satisfy + checkQuals, ok := checkQualMap[col] + if !ok || !indexQuals.IsASubsetOf(checkQuals) { + return false + } + } + return true +} diff --git a/cache/pending_index_item.go b/cache/pending_index_item.go index 09baee55..5c237982 100644 --- a/cache/pending_index_item.go +++ b/cache/pending_index_item.go @@ -77,7 +77,7 @@ func (p *pendingIndexItem) Wait() { func NewPendingIndexItem(columns []string, key string, limit int64) *pendingIndexItem { res := &pendingIndexItem{ - item: NewIndexItem(columns, key, limit), + item: NewIndexItem(columns, key, limit, nil), lock: new(sync.WaitGroup), } // increment wait group - indicate this item is pending diff --git a/cache/query_cache.go b/cache/query_cache.go index 68e1f489..981bda81 100644 --- a/cache/query_cache.go +++ b/cache/query_cache.go @@ -77,8 +77,8 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns } } sort.Strings(columns) - log.Printf("[TRACE] QueryCache Set - connectionName: %s, table: %s, columns: %s\n", c.connectionName, table, columns) - defer log.Printf("[TRACE] QueryCache Set() DONE") + log.Printf("[INFO] QueryCache Set - connectionName: %s, table: %s, columns: %s\n", c.connectionName, table, columns) + defer log.Printf("[INFO] QueryCache Set() DONE") // write to the result cache // set the insertion time @@ -88,19 +88,19 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns // now update the index // get the index bucket for this table and quals - indexBucketKey := c.buildIndexKey(c.connectionName, table, qualMap) + indexBucketKey := c.buildIndexKey(c.connectionName, table) indexBucket, ok := c.getIndexBucket(indexBucketKey) - log.Printf("[TRACE] index key %s, result key %s", indexBucketKey, resultKey) + log.Printf("[INFO] index key %s, result key %s", indexBucketKey, resultKey) if ok { - indexBucket.Append(&IndexItem{columns, resultKey, limit}) + indexBucket.Append(&IndexItem{Columns: columns, Key: resultKey, Limit: limit, Quals: qualMap}) } else { // create new index bucket - indexBucket = newIndexBucket().Append(NewIndexItem(columns, resultKey, limit)) + indexBucket = newIndexBucket().Append(NewIndexItem(columns, resultKey, limit, qualMap)) } if res := c.cache.SetWithTTL(indexBucketKey, indexBucket, 1, ttl); !res { - log.Printf("[TRACE] Set failed") + log.Printf("[INFO] Set failed") return res } @@ -112,7 +112,7 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns // CancelPendingItem cancels a pending item - called when an execute call fails for any reason func (c *QueryCache) CancelPendingItem(table string, qualMap map[string]*proto.Quals, columns []string, limit int64) { - log.Printf("[Trace] QueryCache CancelPendingItem %s", table) + log.Printf("[INFO] QueryCache CancelPendingItem %s", table) // clear the corresponding pending item c.pendingItemComplete(table, qualMap, columns, limit) } @@ -120,23 +120,35 @@ func (c *QueryCache) CancelPendingItem(table string, qualMap map[string]*proto.Q func (c *QueryCache) Get(ctx context.Context, table string, qualMap map[string]*proto.Quals, columns []string, limit, ttlSeconds int64) *QueryCacheResult { // get the index bucket for this table and quals // - this contains cache keys for all cache entries for specified table and quals - indexBucketKey := c.buildIndexKey(c.connectionName, table, qualMap) - - log.Printf("[TRACE] QueryCache Get - indexBucketKey %s", indexBucketKey) - + indexBucketKey := c.buildIndexKey(c.connectionName, table) + + log.Printf("[INFO] QueryCache Get - indexBucketKey %s", indexBucketKey) + + // build a map containing only the quals which we sue for building a cache key + shouldIncludeQual := c.getShouldIncludeQualInKey(table) + cacheQualMap := make(map[string]*proto.Quals) + for col, quals := range qualMap { + log.Printf("[INFO] col %s", col) + if shouldIncludeQual(col) { + log.Printf("[INFO] sgould include %v", shouldIncludeQual(col)) + cacheQualMap[col] = quals + } + } + log.Printf("[INFO] getCachedResult") // do we have a cached result? - res := c.getCachedResult(indexBucketKey, columns, limit, ttlSeconds) + res := c.getCachedResult(indexBucketKey, table, cacheQualMap, columns, limit, ttlSeconds) if res != nil { log.Printf("[INFO] CACHE HIT") // cache hit! return res } + log.Printf("[INFO] getPendingResultItem") // there was no cached result - is there data fetch in progress? - if pendingItem := c.getPendingResultItem(indexBucketKey, table, qualMap, columns, limit); pendingItem != nil { + if pendingItem := c.getPendingResultItem(indexBucketKey, table, cacheQualMap, columns, limit); pendingItem != nil { log.Printf("[INFO] found pending item - waiting for it") // so there is a pending result, wait for it - return c.waitForPendingItem(ctx, pendingItem, indexBucketKey, table, qualMap, columns, limit, ttlSeconds) + return c.waitForPendingItem(ctx, pendingItem, indexBucketKey, table, cacheQualMap, columns, limit, ttlSeconds) } log.Printf("[INFO] CACHE MISS") @@ -148,8 +160,8 @@ func (c *QueryCache) Clear() { c.cache.Clear() } -func (c *QueryCache) getCachedResult(indexBucketKey string, columns []string, limit int64, ttlSeconds int64) *QueryCacheResult { - log.Printf("[TRACE] QueryCache getCachedResult - index bucket key: %s\n", indexBucketKey) +func (c *QueryCache) getCachedResult(indexBucketKey, table string, qualMap map[string]*proto.Quals, columns []string, limit int64, ttlSeconds int64) *QueryCacheResult { + log.Printf("[INFO] QueryCache getCachedResult - index bucket key: %s\n", indexBucketKey) indexBucket, ok := c.getIndexBucket(indexBucketKey) if !ok { c.Stats.Misses++ @@ -157,8 +169,8 @@ func (c *QueryCache) getCachedResult(indexBucketKey string, columns []string, li return nil } - // now check whether we have a cache entry that covers the required columns - check the index - indexItem := indexBucket.Get(columns, limit) + // now check whether we have a cache entry that covers the required quals and columns - check the index + indexItem := indexBucket.Get(qualMap, columns, limit) if indexItem == nil { limitString := "NONE" if limit != -1 { @@ -204,11 +216,10 @@ func (c *QueryCache) getResult(resultKey string) (*QueryCacheResult, bool) { return result.(*QueryCacheResult), true } -func (c *QueryCache) buildIndexKey(connectionName, table string, qualMap map[string]*proto.Quals) string { - str := c.sanitiseKey(fmt.Sprintf("index__%s%s%s", +func (c *QueryCache) buildIndexKey(connectionName, table string) string { + str := c.sanitiseKey(fmt.Sprintf("index__%s%s", connectionName, - table, - c.formatQualMapForKey(table, qualMap))) + table)) return str } @@ -235,7 +246,7 @@ func (c *QueryCache) formatQualMapForKey(table string, qualMap map[string]*proto idx++ } sort.Strings(keys) - log.Printf("[TRACE] formatQualMapForKey sorted keys %v\n", keys) + log.Printf("[INFO] formatQualMapForKey sorted keys %v\n", keys) // now construct cache key from ordered quals @@ -277,7 +288,7 @@ func (c *QueryCache) getShouldIncludeQualInKey(table string) func(string) bool { return func(column string) bool { res := helpers.StringSliceContains(cols, column) - log.Printf("[TRACE] shouldIncludeQual, column %s, include = %v", column, res) + log.Printf("[INFO] shouldIncludeQual, column %s, include = %v", column, res) return res } diff --git a/cache/query_cache_pending.go b/cache/query_cache_pending.go index 1d506b38..9e79785e 100644 --- a/cache/query_cache_pending.go +++ b/cache/query_cache_pending.go @@ -70,7 +70,7 @@ func (c *QueryCache) waitForPendingItem(ctx context.Context, pendingItem *pendin log.Printf("[TRACE] waitForPendingItem transfer complete - trying cache again, indexBucketKey: %s", indexBucketKey) // now try to read from the cache again - res = c.getCachedResult(indexBucketKey, columns, limit, ttlSeconds) + res = c.getCachedResult(indexBucketKey, table, qualMap, columns, limit, ttlSeconds) // if the data is still not in the cache, create a pending item if res == nil { log.Printf("[TRACE] waitForPendingItem item still not in the cache - add pending item, indexBucketKey: %s", indexBucketKey) @@ -110,7 +110,7 @@ func (c *QueryCache) addPendingResult(indexBucketKey, table string, qualMap map[ // unlock pending result items from the map func (c *QueryCache) pendingItemComplete(table string, qualMap map[string]*proto.Quals, columns []string, limit int64) { - indexBucketKey := c.buildIndexKey(c.connectionName, table, qualMap) + indexBucketKey := c.buildIndexKey(c.connectionName, table) log.Printf("[TRACE] pendingItemComplete indexBucketKey %s, columns %v, limit %d", indexBucketKey, columns, limit) defer log.Printf("[TRACE] pendingItemComplete done") diff --git a/grpc/proto/plugin.pb.go b/grpc/proto/plugin.pb.go index dabab025..b2cbcb64 100644 --- a/grpc/proto/plugin.pb.go +++ b/grpc/proto/plugin.pb.go @@ -8,6 +8,9 @@ package proto import ( context "context" + reflect "reflect" + sync "sync" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" @@ -15,8 +18,6 @@ import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" - reflect "reflect" - sync "sync" ) const ( diff --git a/grpc/proto/quals.go b/grpc/proto/quals.go index 6468815a..03b88001 100644 --- a/grpc/proto/quals.go +++ b/grpc/proto/quals.go @@ -1,6 +1,8 @@ package proto import ( + "fmt" + "github.com/golang/protobuf/ptypes/timestamp" typehelpers "github.com/turbot/go-kit/types" ) @@ -9,6 +11,31 @@ func (x *Quals) Append(q *Qual) { x.Quals = append(x.Quals, q) } +func (x *Quals) IsASubsetOf(other *Quals) bool { + // all quals in x must exist in other + for _, q := range x.Quals { + if !other.Contains(q) { + return false + } + } + return true +} + +func (x *Quals) Contains(otherQual *Qual) bool { + for _, q := range x.Quals { + if q.Equals(otherQual) { + return true + } + } + return false +} + +func (x *Qual) Equals(other *Qual) bool { + return fmt.Sprintf("%v", x.Value.Value) == fmt.Sprintf("%v", other.Value.Value) && + x.FieldName == other.FieldName && + x.Operator == other.Operator +} + // NewQualValue creates a QualValue object from a raw value func NewQualValue(value interface{}) *QualValue { // TODO handle lists separately From 01ce0e1559c8079088c8bb01fb432cf7e5bc4312 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 12 Nov 2021 16:07:12 +0000 Subject: [PATCH 2/8] qual operator subsets in progress --- cache/index_item.go | 9 +- cache/pending_index_item.go | 2 + cache/query_cache.go | 48 +++---- cache/query_cache_pending.go | 2 +- grpc/proto/quals.go | 48 ++++++- grpc/proto/quals_test.go | 249 +++++++++++++++++++++++++++++++++++ 6 files changed, 324 insertions(+), 34 deletions(-) create mode 100644 grpc/proto/quals_test.go diff --git a/cache/index_item.go b/cache/index_item.go index 15b5cdc8..29e47a4d 100644 --- a/cache/index_item.go +++ b/cache/index_item.go @@ -91,10 +91,17 @@ func (i IndexItem) SatisfiesLimit(limit int64) bool { //our quals [id="1"], check quals [id="1", foo=2] -> SATISFIED //our quals [id="1", foo=2], check quals [id="1"] -> NOT SATISFIED func (i IndexItem) SatisfiesQuals(checkQualMap map[string]*proto.Quals) bool { + log.Printf("[INFO] SatisfiesQuals") for col, indexQuals := range i.Quals { + log.Printf("[INFO] col %s", col) // if we have quals the passed in map does not, we DO NOT satisfy checkQuals, ok := checkQualMap[col] - if !ok || !indexQuals.IsASubsetOf(checkQuals) { + var isSubset bool + if ok { + isSubset = indexQuals.IsASubsetOf(checkQuals) + } + log.Printf("[INFO] get check qual %v, isSubset %v", ok, isSubset) + if !ok || !isSubset { return false } } diff --git a/cache/pending_index_item.go b/cache/pending_index_item.go index 5c237982..0d808ad9 100644 --- a/cache/pending_index_item.go +++ b/cache/pending_index_item.go @@ -20,7 +20,9 @@ func newPendingIndexBucket() *pendingIndexBucket { // GetItemWhichSatisfiesColumnsAndLimit finds an index item which satisfies all columns // used to find an IndexItem to satisfy a cache Get request func (b *pendingIndexBucket) GetItemWhichSatisfiesColumnsAndLimit(columns []string, limit int64) *pendingIndexItem { + log.Printf("[TRACE] found pending index item to satisfy columns %v and limit %d", columns, limit) for _, item := range b.Items { + if item.SatisfiesColumns(columns) && item.SatisfiesLimit(limit) { log.Printf("[TRACE] found pending index item to satisfy columns %s, limit %d", strings.Join(columns, ","), limit) return item diff --git a/cache/query_cache.go b/cache/query_cache.go index 981bda81..05bbf4a4 100644 --- a/cache/query_cache.go +++ b/cache/query_cache.go @@ -67,7 +67,7 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns // (we need to do this even if the cache set fails) c.pendingItemComplete(table, qualMap, columns, limit) }() - + cacheQualMap := c.buildCacheQualMap(table, qualMap) // if any data was returned, extract the columns from the first row if len(result.Rows) > 0 { for col := range result.Rows[0].Columns { @@ -83,7 +83,7 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns // write to the result cache // set the insertion time result.InsertionTime = time.Now() - resultKey := c.buildResultKey(table, qualMap, columns) + resultKey := c.buildResultKey(table, cacheQualMap, columns) c.cache.SetWithTTL(resultKey, result, 1, ttl) // now update the index @@ -94,10 +94,10 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns log.Printf("[INFO] index key %s, result key %s", indexBucketKey, resultKey) if ok { - indexBucket.Append(&IndexItem{Columns: columns, Key: resultKey, Limit: limit, Quals: qualMap}) + indexBucket.Append(&IndexItem{Columns: columns, Key: resultKey, Limit: limit, Quals: cacheQualMap}) } else { // create new index bucket - indexBucket = newIndexBucket().Append(NewIndexItem(columns, resultKey, limit, qualMap)) + indexBucket = newIndexBucket().Append(NewIndexItem(columns, resultKey, limit, cacheQualMap)) } if res := c.cache.SetWithTTL(indexBucketKey, indexBucket, 1, ttl); !res { log.Printf("[INFO] Set failed") @@ -124,16 +124,8 @@ func (c *QueryCache) Get(ctx context.Context, table string, qualMap map[string]* log.Printf("[INFO] QueryCache Get - indexBucketKey %s", indexBucketKey) - // build a map containing only the quals which we sue for building a cache key - shouldIncludeQual := c.getShouldIncludeQualInKey(table) - cacheQualMap := make(map[string]*proto.Quals) - for col, quals := range qualMap { - log.Printf("[INFO] col %s", col) - if shouldIncludeQual(col) { - log.Printf("[INFO] sgould include %v", shouldIncludeQual(col)) - cacheQualMap[col] = quals - } - } + // build a map containing only the quals which we use for building a cache key (i.e. key column quals) + cacheQualMap := c.buildCacheQualMap(table, qualMap) log.Printf("[INFO] getCachedResult") // do we have a cached result? res := c.getCachedResult(indexBucketKey, table, cacheQualMap, columns, limit, ttlSeconds) @@ -156,6 +148,18 @@ func (c *QueryCache) Get(ctx context.Context, table string, qualMap map[string]* return nil } +func (c *QueryCache) buildCacheQualMap(table string, qualMap map[string]*proto.Quals) map[string]*proto.Quals { + shouldIncludeQual := c.getShouldIncludeQualInKey(table) + cacheQualMap := make(map[string]*proto.Quals) + for col, quals := range qualMap { + log.Printf("[INFO] col %s", col) + if shouldIncludeQual(col) { + cacheQualMap[col] = quals + } + } + return cacheQualMap +} + func (c *QueryCache) Clear() { c.cache.Clear() } @@ -249,29 +253,17 @@ func (c *QueryCache) formatQualMapForKey(table string, qualMap map[string]*proto log.Printf("[INFO] formatQualMapForKey sorted keys %v\n", keys) // now construct cache key from ordered quals - - // get a predicate function which tells us whether to include a qual - shouldIncludeQualInKey := c.getShouldIncludeQualInKey(table) - for i, key := range keys { - strs[i] = c.formatQualsForKey(qualMap[key], shouldIncludeQualInKey) - } - return strings.Join(strs, "-") -} - -func (c *QueryCache) formatQualsForKey(quals *proto.Quals, shouldIncludeQualInKey func(string) bool) string { - var strs []string - for _, q := range quals.Quals { - if shouldIncludeQualInKey(q.FieldName) { + for _, q := range qualMap[key].Quals { strs = append(strs, fmt.Sprintf("%s-%s-%v", q.FieldName, q.GetStringValue(), grpc.GetQualValue(q.Value))) } + strs[i] = strings.Join(strs, "-") } return strings.Join(strs, "-") } // only include key column quals and optional quals func (c *QueryCache) getShouldIncludeQualInKey(table string) func(string) bool { - // build a list of all key columns tableSchema, ok := c.PluginSchema[table] if !ok { diff --git a/cache/query_cache_pending.go b/cache/query_cache_pending.go index 9e79785e..dba054c7 100644 --- a/cache/query_cache_pending.go +++ b/cache/query_cache_pending.go @@ -45,7 +45,7 @@ func (c *QueryCache) waitForPendingItem(ctx context.Context, pendingItem *pendin log.Printf("[TRACE] waitForPendingItem indexBucketKey: %s", indexBucketKey) - transferCompleteChan := make(chan (bool), 1) + transferCompleteChan := make(chan bool, 1) go func() { pendingItem.Wait() close(transferCompleteChan) diff --git a/grpc/proto/quals.go b/grpc/proto/quals.go index 03b88001..30241c76 100644 --- a/grpc/proto/quals.go +++ b/grpc/proto/quals.go @@ -1,7 +1,7 @@ package proto import ( - "fmt" + "log" "github.com/golang/protobuf/ptypes/timestamp" typehelpers "github.com/turbot/go-kit/types" @@ -12,8 +12,10 @@ func (x *Quals) Append(q *Qual) { } func (x *Quals) IsASubsetOf(other *Quals) bool { + log.Printf("[INFO] IsASubsetOf") // all quals in x must exist in other for _, q := range x.Quals { + if !other.Contains(q) { return false } @@ -23,17 +25,55 @@ func (x *Quals) IsASubsetOf(other *Quals) bool { func (x *Quals) Contains(otherQual *Qual) bool { for _, q := range x.Quals { + log.Printf("[INFO] Contains me %+v, other %+v", q, otherQual) if q.Equals(otherQual) { return true } + log.Printf("[INFO] !=") } return false } func (x *Qual) Equals(other *Qual) bool { - return fmt.Sprintf("%v", x.Value.Value) == fmt.Sprintf("%v", other.Value.Value) && - x.FieldName == other.FieldName && - x.Operator == other.Operator + log.Printf("[INFO] me %s, other %s", x.String(), other.String()) + return x.String() == other.String() +} + +func (x *Qual) IsASubsetOf(other *Qual) bool { + operator, ok := x.Operator.(*Qual_StringValue) + if !ok { + return false + } + otherOperator, ok := x.Operator.(*Qual_StringValue) + if !ok { + return false + } + + // if operators are different then we are not a subset + if operator != otherOperator { + return false + } + // if operators are both equals then the quals must be equal + if operator.StringValue == "=" { + return x.Equals(other) + } + + switch x.Value.Value.(type) { + //case *QualValue_StringValue: + case *QualValue_Int64Value: + case *QualValue_DoubleValue: + // whiuch operatros can ewe handle for this subset check + operators := []string{""} + if + //case *QualValue_BoolValue: + //case *QualValue_InetValue : + //case *QualValue_JsonbValue: + case *QualValue_TimestampValue: + case *QualValue_ListValue: + + } + log.Printf("[INFO] me %s, other %s", x.String(), other.String()) + return x.String() == other.String() } // NewQualValue creates a QualValue object from a raw value diff --git a/grpc/proto/quals_test.go b/grpc/proto/quals_test.go new file mode 100644 index 00000000..4282cf0c --- /dev/null +++ b/grpc/proto/quals_test.go @@ -0,0 +1,249 @@ +package proto + +import ( + "fmt" + "testing" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +// is q1 a subset of q2 +type isSubsetTest struct { + q1 *Qual + q2 *Qual + expected bool +} + +var now = time.Now() +var testCasesIsSubset = map[string]isSubsetTest{ + "= same string": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, + true, + }, + "= same int64": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, + true, + }, + "= same double": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, + true, + }, + "= same inet": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, + true, + }, + "= same bool": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, + true, + }, + "= same jsonb": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, + true, + }, + "= same timestamp": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now)}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now)}, + true, + }, + "= same list": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, + true, + }, + "= subset list": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a")}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, + true, + }, + "!= same string": { + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, + true, + }, + "!= same int64": { + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, + true, + }, + "!= same double": { + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, + true, + }, + "!= same inet": { + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, + true, + }, + "!= same bool": { + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, + true, + }, + "!= same jsonb": { + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, + true, + }, + "!= same timestamp": { + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: getTimestampValue(now)}, + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: getTimestampValue(now)}, + true, + }, + "!= same list": {&Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: toStringList("a", "b")}, + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: toStringList("a", "b")}, + true, + }, + "int64 < smaller number": { + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, + true, + }, + "int64 <= smaller number": { + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, + true, + }, + "int64 > bigger number": { + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + true, + }, + "int64 >= bigger number": { + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + true, + }, + "double < smaller number": { + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 10}}}, + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, + true, + }, + "double <= smaller number": { + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 10}}}, + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, + true, + }, + "double > bigger number": { + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 10}}}, + true, + }, + "double >= bigger number": { + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 10}}}, + true, + }, "timestamp < earlier": { + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: getTimestampValue(now)}, + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, + true, + }, + "timestamp <= earlier": { + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: getTimestampValue(now)}, + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, + true, + }, + "timestamp > later": { + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: getTimestampValue(now)}, + true, + }, + "timestamp >= later": { + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: getTimestampValue(now)}, + true, + }, + "= same string different field": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f2", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, + false, + }, + "= same string different operator": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, + &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f2", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, + false, + }, + "Different string": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "b"}}}, + false, + }, + "Different int64": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 101}}}, + false, + }, + "Different double": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 101}}}, + false, + }, + "Different inet": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.2")}}, + false, + }, + "Different bool": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: false}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, + false, + }, + "Different jsonb": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "101"}}}, + false, + }, + "Different timestamp": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now)}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, + false, + }, + "Different list": {&Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b", "c")}, + false, + }, +} + +func getTimestampValue(t time.Time) *QualValue { + return &QualValue{Value: &QualValue_TimestampValue{TimestampValue: ×tamppb.Timestamp{Seconds: t.Unix()}}} + +} + +func TestIsSubset(t *testing.T) { + for name, test := range testCasesIsSubset { + isSubset := test.q1.IsASubsetOf(test.q2) + if isSubset != test.expected { + t.Errorf("Test: '%s' FAILED : \nexpected:\n %v \ngot:\n %v\n", name, test.expected, isSubset) + } + } +} + +func ToInet(ipString string) *QualValue_InetValue { + var netmaskBits int32 = 0xffff + protocolVersion := "IPv4" + return &QualValue_InetValue{ + InetValue: &Inet{ + Mask: netmaskBits, + Addr: ipString, + Cidr: fmt.Sprintf("%s/%d", ipString, netmaskBits), + ProtocolVersion: protocolVersion, + }, + } +} + +func toStringList(items ...string) *QualValue { + list := make([]*QualValue, len(items)) + for i, item := range items { + list[i] = &QualValue{Value: &QualValue_StringValue{StringValue: item}} + + } + + return &QualValue{Value: &QualValue_ListValue{ListValue: &QualValueList{Values: list}}} +} From 73a83cbf217cf62b99db15592152e10f11cf9145 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 19 Nov 2021 10:01:47 +0000 Subject: [PATCH 3/8] update subset logic. respect cancellation when waiting for pending result --- cache/index_item.go | 28 ++-- cache/query_cache.go | 15 +- grpc/proto/quals.go | 333 +++++++++++++++++++++++++++++++++++---- grpc/proto/quals_test.go | 76 ++++----- plugin/plugin.go | 2 +- plugin/query_data.go | 1 - plugin/query_status.go | 1 - 7 files changed, 369 insertions(+), 87 deletions(-) diff --git a/cache/index_item.go b/cache/index_item.go index 29e47a4d..cb2bdfc4 100644 --- a/cache/index_item.go +++ b/cache/index_item.go @@ -83,24 +83,30 @@ func (i IndexItem) SatisfiesLimit(limit int64) bool { } // SatisfiesQuals -//check quals must be MORE restrictive the our quals -// i.e. our quals must be a subset of check quals +// does this index item satisfy the check quals +// all data returned by check quals is returned by index quals +// i.e. check quals must be a 'subset' of index quals // eg -//our quals [], check quals [id="1"] -> SATISFIED -//our quals [id="1"], check quals [id="1"] -> SATISFIED -//our quals [id="1"], check quals [id="1", foo=2] -> SATISFIED -//our quals [id="1", foo=2], check quals [id="1"] -> NOT SATISFIED +// our quals [], check quals [id="1"] -> SATISFIED +// our quals [id="1"], check quals [id="1"] -> SATISFIED +// our quals [id="1"], check quals [id="1", foo=2] -> SATISFIED +// our quals [id="1", foo=2], check quals [id="1"] -> NOT SATISFIED func (i IndexItem) SatisfiesQuals(checkQualMap map[string]*proto.Quals) bool { - log.Printf("[INFO] SatisfiesQuals") + log.Printf("[TRACE] SatisfiesQuals") for col, indexQuals := range i.Quals { - log.Printf("[INFO] col %s", col) - // if we have quals the passed in map does not, we DO NOT satisfy + log.Printf("[TRACE] col %s", col) + // if we have quals the check quals do not, we DO NOT satisfy checkQuals, ok := checkQualMap[col] var isSubset bool if ok { - isSubset = indexQuals.IsASubsetOf(checkQuals) + log.Printf("[TRACE] SatisfiesQuals index item has quals for %s which check quals also have - check if our quals for this colummn are a subset of the check quals", col) + log.Printf("[TRACE] indexQuals %+v, checkQuals %+v", indexQuals, checkQuals) + // isSubset means all data returned by check quals is returned by index quals + isSubset = checkQuals.IsASubsetOf(indexQuals) + } else { + log.Printf("[TRACE] SatisfiesQuals index item has qual for %s which check quals do not - NOT SATISFIED") } - log.Printf("[INFO] get check qual %v, isSubset %v", ok, isSubset) + log.Printf("[TRACE] get check qual %v, isSubset %v", ok, isSubset) if !ok || !isSubset { return false } diff --git a/cache/query_cache.go b/cache/query_cache.go index 05bbf4a4..ffd4671a 100644 --- a/cache/query_cache.go +++ b/cache/query_cache.go @@ -68,6 +68,7 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns c.pendingItemComplete(table, qualMap, columns, limit) }() cacheQualMap := c.buildCacheQualMap(table, qualMap) + // if any data was returned, extract the columns from the first row if len(result.Rows) > 0 { for col := range result.Rows[0].Columns { @@ -90,17 +91,15 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns // get the index bucket for this table and quals indexBucketKey := c.buildIndexKey(c.connectionName, table) indexBucket, ok := c.getIndexBucket(indexBucketKey) - - log.Printf("[INFO] index key %s, result key %s", indexBucketKey, resultKey) - - if ok { - indexBucket.Append(&IndexItem{Columns: columns, Key: resultKey, Limit: limit, Quals: cacheQualMap}) - } else { + indexItem := NewIndexItem(columns, resultKey, limit, cacheQualMap) + if !ok { // create new index bucket - indexBucket = newIndexBucket().Append(NewIndexItem(columns, resultKey, limit, cacheQualMap)) + indexBucket = newIndexBucket() } + indexBucket.Append(indexItem) + if res := c.cache.SetWithTTL(indexBucketKey, indexBucket, 1, ttl); !res { - log.Printf("[INFO] Set failed") + log.Printf("[WARN] cache Set failed") return res } diff --git a/grpc/proto/quals.go b/grpc/proto/quals.go index 30241c76..0449fd3f 100644 --- a/grpc/proto/quals.go +++ b/grpc/proto/quals.go @@ -2,78 +2,357 @@ package proto import ( "log" + "time" "github.com/golang/protobuf/ptypes/timestamp" typehelpers "github.com/turbot/go-kit/types" + "google.golang.org/protobuf/types/known/timestamppb" ) func (x *Quals) Append(q *Qual) { x.Quals = append(x.Quals, q) } +// IsASubsetOf determines whether we are a subset of other Quals func (x *Quals) IsASubsetOf(other *Quals) bool { - log.Printf("[INFO] IsASubsetOf") - // all quals in x must exist in other - for _, q := range x.Quals { - - if !other.Contains(q) { + log.Printf("[TRACE] IsASubsetOf me %+v other %+v", x, other) + // all quals in x must be a subset of all the quals in other + for _, qual := range x.Quals { + if !other.QualIsASubset(qual) { return false } } return true } -func (x *Quals) Contains(otherQual *Qual) bool { +// QualIsASubset determines whether otherQual is a subset of all our quals +func (x *Quals) QualIsASubset(otherQual *Qual) bool { + log.Printf("[TRACE] QualIsASubset my quals %+v, other %+v", x, otherQual) for _, q := range x.Quals { - log.Printf("[INFO] Contains me %+v, other %+v", q, otherQual) - if q.Equals(otherQual) { - return true + if !otherQual.IsASubsetOf(q) { + log.Printf("[TRACE] otherQual %+v is NOT a subset of %+v", otherQual, q) + return false } - log.Printf("[INFO] !=") + log.Printf("[TRACE] otherQual %+v IS a subset of %+v", otherQual, q) } - return false + + log.Printf("[WARN] QualIsASubset returning true") + return true } func (x *Qual) Equals(other *Qual) bool { - log.Printf("[INFO] me %s, other %s", x.String(), other.String()) + log.Printf("[TRACE] me %s, other %s", x.String(), other.String()) return x.String() == other.String() } func (x *Qual) IsASubsetOf(other *Qual) bool { operator, ok := x.Operator.(*Qual_StringValue) if !ok { + log.Printf("[TRACE] IsASubsetOf my operator is not a string - returning false") return false } otherOperator, ok := x.Operator.(*Qual_StringValue) if !ok { + log.Printf("[TRACE] IsASubsetOf other operator is not a string - returning false") return false } - // if operators are different then we are not a subset - if operator != otherOperator { - return false - } - // if operators are both equals then the quals must be equal + // if operators are both equals then the quals must qqbe equal if operator.StringValue == "=" { + log.Printf("[TRACE] IsASubsetOf operator is equals - returning x.Equals(other)") return x.Equals(other) } - switch x.Value.Value.(type) { - //case *QualValue_StringValue: + switch value := x.Value.Value.(type) { case *QualValue_Int64Value: + if otherValue, ok := other.Value.Value.(*QualValue_Int64Value); !ok { + return false + } else { + return intOperatorIsASubset(operator.StringValue, value.Int64Value, otherOperator.StringValue, otherValue.Int64Value) + } + case *QualValue_DoubleValue: - // whiuch operatros can ewe handle for this subset check - operators := []string{""} - if - //case *QualValue_BoolValue: - //case *QualValue_InetValue : - //case *QualValue_JsonbValue: + if otherVal, ok := other.Value.Value.(*QualValue_DoubleValue); !ok { + return false + } else { + return doubleOperatorIsASubset(operator.StringValue, value.DoubleValue, otherOperator.StringValue, otherVal.DoubleValue) + } + case *QualValue_TimestampValue: + if otherVal, ok := other.Value.Value.(*QualValue_TimestampValue); !ok { + return false + } else { + return timeOperatorIsASubset(operator.StringValue, value.TimestampValue, otherOperator.StringValue, otherVal.TimestampValue) + } case *QualValue_ListValue: + log.Printf("[TRACE] IsASubsetOf list not implemented yet") + } + + log.Printf("[TRACE] IsASubsetOf no supported types = returning false") + return false +} +// is operator and value a subset of otherOperator and otherValue +func doubleOperatorIsASubset(operator string, value float64, otherOperator string, otherValue float64) bool { + switch operator { + case "=": + switch otherOperator { + case "=": + return value == otherValue + case "<": + // value = 9.9, otherValue < 10 - subset + // value = 10, otherValue < 10 - NOT subset + return value < otherValue + case "<=": + // value = 9.9, otherValue <= 10 - subset + // value = 10, otherValue <= 10 - subset + // value = 10.1, otherValue <= 10 - NOT subset + return value <= otherValue + case ">": + // value = 10, otherValue > 10 - NOT subset + // value = 10.1, otherValue > 10 - subset + return value > otherValue + case ">=": + // value = 9.9, otherValue >= 10 - NOT subset + // value = 10, otherValue >= 10 - subset + // value = 10.1, otherValue >= 10 - subset + return value >= otherValue + default: + return false + } + case "!=": + switch otherOperator { + case "!=": + return value == otherValue + default: + return false + } + + case "<": + switch otherOperator { + case "<": + // value < 9.9, otherValue < 10 - subset + // value < 10, otherValue < 10 - subset + // value < 10.1, otherValue < 10 - NOT subset + return value <= otherValue + case "<=": + // value < 9.9, otherValue <= 10 - subset + // value < 10, otherValue <= 10 - subset + // value < 10.1, otherValue <= 10 - NOT subset + return value <= otherValue + default: + return false + } + case "<=": + switch otherOperator { + case "<": + // value <= 9.9, otherValue < 10 - subset + // value <= 10, otherValue < 10 - NOT subset + return value < otherValue + case "<=": + // value <= 9.9, otherValue <= 10 - subset + // value <= 10, otherValue <= 10 - subset + // value <= 10.1, otherValue <= 10 - NOT subset + return value <= otherValue + default: + return false + } + case ">": + switch otherOperator { + case ">": + // value > 9.9, otherValue > 10 - NOT subset + // value > 10, otherValue > 10 - subset + // value > 10.1, otherValue > 10 - subset + return value >= otherValue + case ">=": + // value > 9.9, otherValue >= 10 - NOT subset + // value > 10, otherValue >= 10 - subset + return value >= otherValue + default: + return false + } + case ">=": + switch otherOperator { + case ">": + // value >= 10, otherValue > 10 - NOT subset + // value >= 10.1, otherValue > 10 - subset + // value >= 10.2, otherValue > 10 - subset + return value > otherValue + case ">=": + // value >= 9.9, otherValue >= 10 - NOT subset + // value >= 10, otherValue >= 10 - subset + // value >= 10.1, otherValue >= 10 - subset + return value >= otherValue + default: + return false + } } - log.Printf("[INFO] me %s, other %s", x.String(), other.String()) - return x.String() == other.String() + + return false +} + +func intOperatorIsASubset(operator string, value int64, otherOperator string, otherValue int64) bool { + switch operator { + case "=": + switch otherOperator { + case "=": + return value == otherValue + case "<": + // value = 9, otherValue < 10 - subset + // value = 10, otherValue < 10 - NOT subset + return value < otherValue + case "<=": + // value = 9, otherValue <= 10 - subset + // value = 10, otherValue <= 10 - subset + // value = 11, otherValue <= 10 - NOT subset + return value <= otherValue + case ">": + // value = 10, otherValue > 10 - NOT subset + // value = 11, otherValue > 10 - subset + return value > otherValue + case ">=": + // value = 9, otherValue >= 10 - NOT subset + // value = 10, otherValue >= 10 - subset + // value = 11, otherValue >= 10 - subset + return value >= otherValue + default: + return false + } + case "!=": + switch otherOperator { + case "!=": + return value == otherValue + default: + return false + } + case "<": + switch otherOperator { + case "<": + // value < 9, otherValue < 10 - subset + // value < 10, otherValue < 10 - subset + // value < 11, otherValue < 10 - NOT subset + return value <= otherValue + case "<=": + // value < 10, otherValue <= 10 - subset + // value < 11, otherValue <= 10 - subset + // value < 12, otherValue <= 10 - NOT subset + return value+1 <= otherValue + default: + return false + } + case "<=": + switch otherOperator { + case "<": + // value <= 8, otherValue < 10 - subset + // value <= 9, otherValue < 10 - subset + // value <= 10, otherValue < 10 - NOT subset + return value < otherValue + case "<=": + // value <= 9, otherValue <= 10 - subset + // value <= 10, otherValue <= 10 - subset + // value <= 11, otherValue <= 10 - NOT subset + return value <= otherValue + default: + return false + } + case ">": + switch otherOperator { + case ">": + // value > 9, otherValue > 10 - NOT subset + // value > 10, otherValue > 10 - subset + // value > 11, otherValue > 10 - subset + return value >= otherValue + case ">=": + // value > 8, otherValue >= 10 - NOT subset + // value > 9, otherValue >= 10 - subset + // value > 10, otherValue >= 10 - subset + return value+1 >= otherValue + default: + return false + } + case ">=": + switch otherOperator { + case ">": + // value >= 10, otherValue > 10 - NOT subset + // value >= 11, otherValue > 10 - subset + // value >= 12, otherValue > 10 - subset + return value > otherValue + case ">=": + // value >= 9, otherValue >= 10 - NOT subset + // value >= 10, otherValue >= 10 - subset + // value >= 11, otherValue >= 10 - subset + return value >= otherValue + default: + return false + } + } + + return false + +} + +func timeOperatorIsASubset(operator string, value *timestamppb.Timestamp, otherOperator string, otherValue *timestamppb.Timestamp) bool { + timeVal := time.Unix(value.Seconds, int64(value.Nanos)) + otherTimeVal := time.Unix(value.Seconds, int64(value.Nanos)) + switch operator { + case "=": + switch otherOperator { + case "=": + return timeVal == otherTimeVal + case "<": + return timeVal.Before(otherTimeVal) + case "<=": + return timeVal.Before(otherTimeVal) || timeVal == otherTimeVal + case ">": + return timeVal.After(otherTimeVal) + case ">=": + return timeVal.After(otherTimeVal) || timeVal == otherTimeVal + default: + return false + } + case "!=": + switch otherOperator { + case "!=": + return timeVal == otherTimeVal + default: + return false + } + case "<": + switch otherOperator { + case "<", "<=": + return timeVal.Before(otherTimeVal) || timeVal == otherTimeVal + default: + return false + } + case "<=": + switch otherOperator { + case "<": + return timeVal.Before(otherTimeVal) + case "<=": + return timeVal.Before(otherTimeVal) || timeVal == otherTimeVal + default: + return false + } + case ">": + switch otherOperator { + case ">", ">=": + return timeVal.After(otherTimeVal) || timeVal == otherTimeVal + default: + return false + } + case ">=": + switch otherOperator { + case ">": + return timeVal.After(otherTimeVal) + case ">=": + return timeVal.After(otherTimeVal) || timeVal == otherTimeVal + default: + return false + } + } + + return false + } // NewQualValue creates a QualValue object from a raw value diff --git a/grpc/proto/quals_test.go b/grpc/proto/quals_test.go index 4282cf0c..63a5a7fd 100644 --- a/grpc/proto/quals_test.go +++ b/grpc/proto/quals_test.go @@ -17,126 +17,126 @@ type isSubsetTest struct { var now = time.Now() var testCasesIsSubset = map[string]isSubsetTest{ - "= same string": { + "both = same string": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, true, }, - "= same int64": { + "both = same int64": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, true, }, - "= same double": { + "both = same double": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, true, }, - "= same inet": { + "both = same inet": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, true, }, - "= same bool": { + "both = same bool": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, true, }, - "= same jsonb": { + "both = same jsonb": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, true, }, - "= same timestamp": { + "both = same timestamp": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now)}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now)}, true, }, - "= same list": { + "both = same list": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, true, }, - "= subset list": { + "both = subset list": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a")}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, true, }, - "!= same string": { + "both != same string": { &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, true, }, - "!= same int64": { + "both != same int64": { &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, true, }, - "!= same double": { + "both != same double": { &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, true, }, - "!= same inet": { + "both != same inet": { &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, true, }, - "!= same bool": { + "both != same bool": { &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, true, }, - "!= same jsonb": { + "both != same jsonb": { &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, true, }, - "!= same timestamp": { + "both != same timestamp": { &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: getTimestampValue(now)}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: getTimestampValue(now)}, true, }, - "!= same list": {&Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: toStringList("a", "b")}, + "both != same list": {&Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: toStringList("a", "b")}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: toStringList("a", "b")}, true, }, - "int64 < smaller number": { + "both int64 < smaller number": { &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, true, }, - "int64 <= smaller number": { + "both int64 <= smaller number": { &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, true, }, - "int64 > bigger number": { + "both int64 > bigger number": { &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, true, }, - "int64 >= bigger number": { + "both int64 >= bigger number": { &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, true, }, - "double < smaller number": { + "both double < smaller number": { &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 10}}}, &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, true, }, - "double <= smaller number": { + "both double <= smaller number": { &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 10}}}, &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, true, }, - "double > bigger number": { + "both double > bigger number": { &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 10}}}, true, }, - "double >= bigger number": { + "both double >= bigger number": { &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 10}}}, true, @@ -145,67 +145,67 @@ var testCasesIsSubset = map[string]isSubsetTest{ &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, true, }, - "timestamp <= earlier": { + "both timestamp <= earlier": { &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: getTimestampValue(now)}, &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, true, }, - "timestamp > later": { + "both timestamp > later": { &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: getTimestampValue(now)}, true, }, - "timestamp >= later": { + "both timestamp >= later": { &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: getTimestampValue(now)}, true, }, - "= same string different field": { + "both = same string different field": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f2", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, false, }, - "= same string different operator": { + "both = same string different operator": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f2", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, false, }, - "Different string": { + "different string": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "a"}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_StringValue{StringValue: "b"}}}, false, }, - "Different int64": { + "different int64": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 101}}}, false, }, - "Different double": { + "different double": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 100}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_DoubleValue{DoubleValue: 101}}}, false, }, - "Different inet": { + "different inet": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.1")}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: ToInet("192.168.0.2")}}, false, }, - "Different bool": { + "different bool": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: false}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, false, }, - "Different jsonb": { + "different jsonb": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "101"}}}, false, }, - "Different timestamp": { + "different timestamp": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now)}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, false, }, - "Different list": {&Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, + "different list": {&Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b", "c")}, false, }, diff --git a/plugin/plugin.go b/plugin/plugin.go index f05bf49e..d1dd5549 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -273,7 +273,7 @@ func (p *Plugin) Execute(req *proto.ExecuteRequest, stream proto.WrapperPlugin_E // the cache will have added a pending item for this transfer // and it is our responsibility to either call 'set' or 'cancel' for this pending item defer func() { - if err != nil { + if err != nil || ctx.Err() != nil { log.Printf("[WARN] Execute call failed - cancelling pending item in cache") p.queryCache.CancelPendingItem(table.Name, queryContext.UnsafeQuals, queryContext.Columns, limit) } diff --git a/plugin/query_data.go b/plugin/query_data.go index 5b963bba..bb8f55dd 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -395,7 +395,6 @@ func (d *QueryData) streamLeafListItem(ctx context.Context, item interface{}) { } // have we streamed enough already? if d.QueryStatus.RowsRemaining(ctx) == 0 { - log.Printf("[TRACE] d.QueryStatus.RowsRemaining is 0 - streamLeafListItem NOT streaming item") return } // increment the stream count diff --git a/plugin/query_status.go b/plugin/query_status.go index df9b8d4b..0a1afb9c 100644 --- a/plugin/query_status.go +++ b/plugin/query_status.go @@ -28,7 +28,6 @@ func newQueryStatus(limit *int64) *QueryStatus { // - if the context has been cancelled, it will return zero func (s *QueryStatus) RowsRemaining(ctx context.Context) int { if IsCancelled(ctx) { - log.Printf("[TRACE] RowsRemaining returning 0: context is cancelled") return 0 } rowsRemaining := s.rowsRequired - s.rowsStreamed From 33049ef4e6d42f8feb90a46670c4b36776172750 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 19 Nov 2021 12:09:44 +0000 Subject: [PATCH 4/8] subset unit tests --- grpc/proto/quals.go | 152 +++++++++++++++++++++++++++++++++------ grpc/proto/quals_test.go | 112 +++++++++++++++++++++++------ 2 files changed, 223 insertions(+), 41 deletions(-) diff --git a/grpc/proto/quals.go b/grpc/proto/quals.go index 0449fd3f..799ff6a7 100644 --- a/grpc/proto/quals.go +++ b/grpc/proto/quals.go @@ -51,47 +51,159 @@ func (x *Qual) IsASubsetOf(other *Qual) bool { log.Printf("[TRACE] IsASubsetOf my operator is not a string - returning false") return false } - otherOperator, ok := x.Operator.(*Qual_StringValue) + otherOperator, ok := other.Operator.(*Qual_StringValue) if !ok { log.Printf("[TRACE] IsASubsetOf other operator is not a string - returning false") return false } - - // if operators are both equals then the quals must qqbe equal - if operator.StringValue == "=" { - log.Printf("[TRACE] IsASubsetOf operator is equals - returning x.Equals(other)") - return x.Equals(other) + if x.FieldName != other.FieldName { + log.Printf("[TRACE] IsASubsetOf field names different - returning false") + return false } switch value := x.Value.Value.(type) { + case *QualValue_StringValue: + otherValue, ok := other.Value.Value.(*QualValue_StringValue) + if !ok { + return false + } + return stringOperatorIsASubset(operator.StringValue, value.StringValue, otherOperator.StringValue, otherValue.StringValue) case *QualValue_Int64Value: - if otherValue, ok := other.Value.Value.(*QualValue_Int64Value); !ok { + otherValue, ok := other.Value.Value.(*QualValue_Int64Value) + if !ok { return false - } else { - return intOperatorIsASubset(operator.StringValue, value.Int64Value, otherOperator.StringValue, otherValue.Int64Value) } - + return intOperatorIsASubset(operator.StringValue, value.Int64Value, otherOperator.StringValue, otherValue.Int64Value) case *QualValue_DoubleValue: - if otherVal, ok := other.Value.Value.(*QualValue_DoubleValue); !ok { + otherVal, ok := other.Value.Value.(*QualValue_DoubleValue) + if !ok { return false - } else { - return doubleOperatorIsASubset(operator.StringValue, value.DoubleValue, otherOperator.StringValue, otherVal.DoubleValue) } - + return doubleOperatorIsASubset(operator.StringValue, value.DoubleValue, otherOperator.StringValue, otherVal.DoubleValue) case *QualValue_TimestampValue: - if otherVal, ok := other.Value.Value.(*QualValue_TimestampValue); !ok { + otherVal, ok := other.Value.Value.(*QualValue_TimestampValue) + if !ok { return false - } else { - return timeOperatorIsASubset(operator.StringValue, value.TimestampValue, otherOperator.StringValue, otherVal.TimestampValue) } + return timeOperatorIsASubset(operator.StringValue, value.TimestampValue, otherOperator.StringValue, otherVal.TimestampValue) + case *QualValue_BoolValue: + otherVal, ok := other.Value.Value.(*QualValue_BoolValue) + if !ok { + return false + } + return boolOperatorIsASubset(operator.StringValue, value.BoolValue, otherOperator.StringValue, otherVal.BoolValue) + + case *QualValue_InetValue: + otherVal, ok := other.Value.Value.(*QualValue_InetValue) + if !ok { + return false + } + return inetOperatorIsASubset(operator.StringValue, value.InetValue.Addr, otherOperator.StringValue, otherVal.InetValue.Addr) + case *QualValue_ListValue: - log.Printf("[TRACE] IsASubsetOf list not implemented yet") + otherVal, ok := other.Value.Value.(*QualValue_ListValue) + if !ok { + return false + } + return listOperatorIsASubset(operator.StringValue, value.ListValue.Values, otherOperator.StringValue, otherVal.ListValue.Values) } log.Printf("[TRACE] IsASubsetOf no supported types = returning false") return false } +func stringOperatorIsASubset(operator string, value string, otherOperator string, otherValue string) bool { + switch operator { + case "=": + switch otherOperator { + case "=": + return value == otherValue + default: + return false + } + case "!=": + switch otherOperator { + case "!=": + return value == otherValue + default: + return false + } + } + + return false +} + +func listOperatorIsASubset(operator string, value []*QualValue, otherOperator string, otherValue []*QualValue) bool { + // only support equals + switch operator { + case "=": + switch otherOperator { + case "=": + // all elements in value must be contained in otherValue + for _, e := range value { + if !qualValueListContains(otherValue, e) { + return false + } + } + return true + default: + return false + } + } + + return false +} + +func qualValueListContains(list []*QualValue, otherElement *QualValue) bool { + for _, e := range list { + if e.String() == otherElement.String() { + return true + } + } + return false +} + +func inetOperatorIsASubset(operator string, value string, otherOperator string, otherValue string) bool { + switch operator { + case "=": + switch otherOperator { + case "=": + return value == otherValue + default: + return false + } + case "!=": + switch otherOperator { + case "!=": + return value == otherValue + default: + return false + } + } + + return false +} +func boolOperatorIsASubset(operator string, value bool, otherOperator string, otherValue bool) bool { + switch operator { + case "=": + switch otherOperator { + case "=": + return value == otherValue + default: + return false + } + case "!=": + switch otherOperator { + case "!=": + return value == otherValue + default: + return false + } + } + + return false +} + // is operator and value a subset of otherOperator and otherValue func doubleOperatorIsASubset(operator string, value float64, otherOperator string, otherValue float64) bool { switch operator { @@ -236,7 +348,7 @@ func intOperatorIsASubset(operator string, value int64, otherOperator string, ot // value < 10, otherValue <= 10 - subset // value < 11, otherValue <= 10 - subset // value < 12, otherValue <= 10 - NOT subset - return value+1 <= otherValue + return value-1 <= otherValue default: return false } @@ -293,7 +405,7 @@ func intOperatorIsASubset(operator string, value int64, otherOperator string, ot func timeOperatorIsASubset(operator string, value *timestamppb.Timestamp, otherOperator string, otherValue *timestamppb.Timestamp) bool { timeVal := time.Unix(value.Seconds, int64(value.Nanos)) - otherTimeVal := time.Unix(value.Seconds, int64(value.Nanos)) + otherTimeVal := time.Unix(otherValue.Seconds, int64(otherValue.Nanos)) switch operator { case "=": switch otherOperator { diff --git a/grpc/proto/quals_test.go b/grpc/proto/quals_test.go index 63a5a7fd..ecd07ee2 100644 --- a/grpc/proto/quals_test.go +++ b/grpc/proto/quals_test.go @@ -42,11 +42,6 @@ var testCasesIsSubset = map[string]isSubsetTest{ &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, true, }, - "both = same jsonb": { - &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, - &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, - true, - }, "both = same timestamp": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now)}, &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now)}, @@ -87,20 +82,95 @@ var testCasesIsSubset = map[string]isSubsetTest{ &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, true, }, - "both != same jsonb": { - &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, - &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, - true, - }, "both != same timestamp": { &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: getTimestampValue(now)}, &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: getTimestampValue(now)}, true, }, - "both != same list": {&Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: toStringList("a", "b")}, - &Qual{Operator: &Qual_StringValue{"!="}, FieldName: "f1", Value: toStringList("a", "b")}, + // int64 = + "int64 =, < NOT subset": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + "int64 =, < subset": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 9}}}, + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + true, + }, + "int64 =, <= NOT subset": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 11}}}, + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + "int64 =, <= subset": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + true, + }, + "int64 =, > NOT subset": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + "int64 =, > subset": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 11}}}, + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + true, + }, + "int64 =, >= NOT subset": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 9}}}, + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + "int64 =, >= subset": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + true, + }, + // int64 < + "int64 <, <= NOT subset": { + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 12}}}, + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + "int64 <, <= subset": { + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 11}}}, + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, true, }, + "int64 <, > NOT subset": { + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + "int64 <, >= NOT subset": { + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + // int64 < + "int64 <=, < NOT subset": { + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + "int64 <=, < subset": { + &Qual{Operator: &Qual_StringValue{"<="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 9}}}, + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + true, + }, + "int64 <=, > NOT subset": { + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{">"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + "int64 <=, >= NOT subset": { + &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + &Qual{Operator: &Qual_StringValue{">="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, + false, + }, + "both int64 < smaller number": { &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 10}}}, &Qual{Operator: &Qual_StringValue{"<"}, FieldName: "f1", Value: &QualValue{Value: &QualValue_Int64Value{Int64Value: 100}}}, @@ -195,25 +265,25 @@ var testCasesIsSubset = map[string]isSubsetTest{ &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_BoolValue{BoolValue: true}}}, false, }, - "different jsonb": { - &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "10"}}}, - &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: &QualValue{Value: &QualValue_JsonbValue{JsonbValue: "101"}}}, - false, - }, "different timestamp": { &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now)}, - &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Second))}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: getTimestampValue(now.Add(1 * time.Minute))}, false, }, - "different list": {&Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, - &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b", "c")}, + "overlapping different list": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("b", "c", "d")}, + false, + }, + "different list": { + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("a", "b")}, + &Qual{Operator: &Qual_StringValue{"="}, FieldName: "f1", Value: toStringList("c", "d", "e")}, false, }, } func getTimestampValue(t time.Time) *QualValue { return &QualValue{Value: &QualValue_TimestampValue{TimestampValue: ×tamppb.Timestamp{Seconds: t.Unix()}}} - } func TestIsSubset(t *testing.T) { From 6f7d24f2a39e0bb9dbba2d2ca0dba1f23facf42c Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 19 Nov 2021 14:17:27 +0000 Subject: [PATCH 5/8] logging --- grpc/proto/quals.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grpc/proto/quals.go b/grpc/proto/quals.go index 799ff6a7..d35f77f7 100644 --- a/grpc/proto/quals.go +++ b/grpc/proto/quals.go @@ -36,7 +36,7 @@ func (x *Quals) QualIsASubset(otherQual *Qual) bool { log.Printf("[TRACE] otherQual %+v IS a subset of %+v", otherQual, q) } - log.Printf("[WARN] QualIsASubset returning true") + log.Printf("[TRACE] QualIsASubset returning true") return true } From 9de533cdfe306b188b0398adb7d5704f15cbad87 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 22 Nov 2021 10:08:56 +0000 Subject: [PATCH 6/8] logging --- cache/query_cache.go | 20 +++++++++----------- plugin/plugin.go | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/cache/query_cache.go b/cache/query_cache.go index ffd4671a..b55c34c5 100644 --- a/cache/query_cache.go +++ b/cache/query_cache.go @@ -78,8 +78,8 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns } } sort.Strings(columns) - log.Printf("[INFO] QueryCache Set - connectionName: %s, table: %s, columns: %s\n", c.connectionName, table, columns) - defer log.Printf("[INFO] QueryCache Set() DONE") + log.Printf("[TRACE] QueryCache Set - connectionName: %s, table: %s, columns: %s\n", c.connectionName, table, columns) + defer log.Printf("[TRACE] QueryCache Set() DONE") // write to the result cache // set the insertion time @@ -111,7 +111,7 @@ func (c *QueryCache) Set(table string, qualMap map[string]*proto.Quals, columns // CancelPendingItem cancels a pending item - called when an execute call fails for any reason func (c *QueryCache) CancelPendingItem(table string, qualMap map[string]*proto.Quals, columns []string, limit int64) { - log.Printf("[INFO] QueryCache CancelPendingItem %s", table) + log.Printf("[TRACE] QueryCache CancelPendingItem table: %s", table) // clear the corresponding pending item c.pendingItemComplete(table, qualMap, columns, limit) } @@ -121,11 +121,11 @@ func (c *QueryCache) Get(ctx context.Context, table string, qualMap map[string]* // - this contains cache keys for all cache entries for specified table and quals indexBucketKey := c.buildIndexKey(c.connectionName, table) - log.Printf("[INFO] QueryCache Get - indexBucketKey %s", indexBucketKey) + log.Printf("[TRACE] QueryCache Get - indexBucketKey %s", indexBucketKey) // build a map containing only the quals which we use for building a cache key (i.e. key column quals) cacheQualMap := c.buildCacheQualMap(table, qualMap) - log.Printf("[INFO] getCachedResult") + // do we have a cached result? res := c.getCachedResult(indexBucketKey, table, cacheQualMap, columns, limit, ttlSeconds) if res != nil { @@ -134,10 +134,9 @@ func (c *QueryCache) Get(ctx context.Context, table string, qualMap map[string]* return res } - log.Printf("[INFO] getPendingResultItem") // there was no cached result - is there data fetch in progress? if pendingItem := c.getPendingResultItem(indexBucketKey, table, cacheQualMap, columns, limit); pendingItem != nil { - log.Printf("[INFO] found pending item - waiting for it") + log.Printf("[TRACE] found pending item - waiting for it") // so there is a pending result, wait for it return c.waitForPendingItem(ctx, pendingItem, indexBucketKey, table, cacheQualMap, columns, limit, ttlSeconds) } @@ -151,7 +150,6 @@ func (c *QueryCache) buildCacheQualMap(table string, qualMap map[string]*proto.Q shouldIncludeQual := c.getShouldIncludeQualInKey(table) cacheQualMap := make(map[string]*proto.Quals) for col, quals := range qualMap { - log.Printf("[INFO] col %s", col) if shouldIncludeQual(col) { cacheQualMap[col] = quals } @@ -164,7 +162,7 @@ func (c *QueryCache) Clear() { } func (c *QueryCache) getCachedResult(indexBucketKey, table string, qualMap map[string]*proto.Quals, columns []string, limit int64, ttlSeconds int64) *QueryCacheResult { - log.Printf("[INFO] QueryCache getCachedResult - index bucket key: %s\n", indexBucketKey) + log.Printf("[TRACE] QueryCache getCachedResult - index bucket key: %s\n", indexBucketKey) indexBucket, ok := c.getIndexBucket(indexBucketKey) if !ok { c.Stats.Misses++ @@ -249,7 +247,7 @@ func (c *QueryCache) formatQualMapForKey(table string, qualMap map[string]*proto idx++ } sort.Strings(keys) - log.Printf("[INFO] formatQualMapForKey sorted keys %v\n", keys) + log.Printf("[TRACE] formatQualMapForKey sorted keys %v\n", keys) // now construct cache key from ordered quals for i, key := range keys { @@ -279,7 +277,7 @@ func (c *QueryCache) getShouldIncludeQualInKey(table string) func(string) bool { return func(column string) bool { res := helpers.StringSliceContains(cols, column) - log.Printf("[INFO] shouldIncludeQual, column %s, include = %v", column, res) + log.Printf("[TRACE] shouldIncludeQual, column %s, include = %v", column, res) return res } diff --git a/plugin/plugin.go b/plugin/plugin.go index d1dd5549..fdc8f2c7 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -259,7 +259,7 @@ func (p *Plugin) Execute(req *proto.ExecuteRequest, stream proto.WrapperPlugin_E } // can we satisfy this request from the cache? if req.CacheEnabled { - log.Printf("[INFO] Cache ENABLED callId: %s", req.CallId) + log.Printf("[TRACE] Cache ENABLED callId: %s", req.CallId) cachedResult := p.queryCache.Get(ctx, table.Name, queryContext.UnsafeQuals, queryContext.Columns, limit, req.CacheTtl) if cachedResult != nil { log.Printf("[TRACE] stream cached result callId: %s", req.CallId) @@ -279,7 +279,7 @@ func (p *Plugin) Execute(req *proto.ExecuteRequest, stream proto.WrapperPlugin_E } }() } else { - log.Printf("[INFO] Cache DISABLED callId: %s", req.CallId) + log.Printf("[TRACE] Cache DISABLED callId: %s", req.CallId) } log.Printf("[TRACE] fetch items callId: %s", req.CallId) From 0ed723662fe232457779984aa90da7fa178998b2 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 22 Nov 2021 15:19:50 +0000 Subject: [PATCH 7/8] fix time quals subset --- grpc/proto/quals.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/grpc/proto/quals.go b/grpc/proto/quals.go index d35f77f7..5b8bf802 100644 --- a/grpc/proto/quals.go +++ b/grpc/proto/quals.go @@ -410,29 +410,29 @@ func timeOperatorIsASubset(operator string, value *timestamppb.Timestamp, otherO case "=": switch otherOperator { case "=": - return timeVal == otherTimeVal + return timeVal.Equal(otherTimeVal) case "<": return timeVal.Before(otherTimeVal) case "<=": - return timeVal.Before(otherTimeVal) || timeVal == otherTimeVal + return timeVal.Before(otherTimeVal) || timeVal.Equal(otherTimeVal) case ">": return timeVal.After(otherTimeVal) case ">=": - return timeVal.After(otherTimeVal) || timeVal == otherTimeVal + return timeVal.After(otherTimeVal) || timeVal.Equal(otherTimeVal) default: return false } case "!=": switch otherOperator { case "!=": - return timeVal == otherTimeVal + return timeVal.Equal(otherTimeVal) default: return false } case "<": switch otherOperator { case "<", "<=": - return timeVal.Before(otherTimeVal) || timeVal == otherTimeVal + return timeVal.Before(otherTimeVal) || timeVal.Equal(otherTimeVal) default: return false } @@ -441,14 +441,14 @@ func timeOperatorIsASubset(operator string, value *timestamppb.Timestamp, otherO case "<": return timeVal.Before(otherTimeVal) case "<=": - return timeVal.Before(otherTimeVal) || timeVal == otherTimeVal + return timeVal.Before(otherTimeVal) || timeVal.Equal(otherTimeVal) default: return false } case ">": switch otherOperator { case ">", ">=": - return timeVal.After(otherTimeVal) || timeVal == otherTimeVal + return timeVal.After(otherTimeVal) || timeVal.Equal(otherTimeVal) default: return false } @@ -457,7 +457,7 @@ func timeOperatorIsASubset(operator string, value *timestamppb.Timestamp, otherO case ">": return timeVal.After(otherTimeVal) case ">=": - return timeVal.After(otherTimeVal) || timeVal == otherTimeVal + return timeVal.After(otherTimeVal) || timeVal.Equal(otherTimeVal) default: return false } From 0220a34d5a161f6f753f0ceac5bbe7bf103a5965 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 22 Nov 2021 15:32:45 +0000 Subject: [PATCH 8/8] logging --- cache/query_cache.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cache/query_cache.go b/cache/query_cache.go index b55c34c5..c2b7b665 100644 --- a/cache/query_cache.go +++ b/cache/query_cache.go @@ -121,7 +121,7 @@ func (c *QueryCache) Get(ctx context.Context, table string, qualMap map[string]* // - this contains cache keys for all cache entries for specified table and quals indexBucketKey := c.buildIndexKey(c.connectionName, table) - log.Printf("[TRACE] QueryCache Get - indexBucketKey %s", indexBucketKey) + log.Printf("[TRACE] QueryCache Get - indexBucketKey %s, quals", indexBucketKey) // build a map containing only the quals which we use for building a cache key (i.e. key column quals) cacheQualMap := c.buildCacheQualMap(table, qualMap) @@ -150,8 +150,12 @@ func (c *QueryCache) buildCacheQualMap(table string, qualMap map[string]*proto.Q shouldIncludeQual := c.getShouldIncludeQualInKey(table) cacheQualMap := make(map[string]*proto.Quals) for col, quals := range qualMap { + log.Printf("[TRACE] buildCacheQualMap col %s, quals %+v", col, quals) if shouldIncludeQual(col) { + log.Printf("[TRACE] INCLUDING COLUMN") cacheQualMap[col] = quals + } else { + log.Printf("[TRACE] EXCLUDING COLUMN") } } return cacheQualMap