Skip to content

Commit 830aa21

Browse files
kaidaguerrekaidaguerre
authored
Add 'RequiresExactCacheMatch' property to KeyColumn, to support key columns which require exact matching to be considered a cache hit. Closes #298 
1 parent 17db334 commit 830aa21

18 files changed

+1027
-515
lines changed

‎.github/workflows/acceptance-test.yml

+1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ jobs:
2626
echo "PATH=$PATH:$HOME/build:/home/runner" >> $GITHUB_ENV
2727
cd /home/runner/work/steampipe-plugin-sdk/steampipe-plugin-sdk/steampipe
2828
go get github.com/turbot/steampipe-plugin-sdk/v2@${{ github.event.pull_request.head.sha }}
29+
go get
2930
go build -o /home/runner/steampipe
3031
3132
- name: Install Chaos plugin from registry

‎cache/consts.go

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package cache
2+
3+
const (
4+
5+
// Key column CacheMatch values
6+
CacheMatchSubset = "subset"
7+
CacheMatchExact = "exact"
8+
)

‎cache/index_bucket.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ func (b *IndexBucket) Append(item *IndexItem) *IndexBucket {
2121
}
2222

2323
// Get finds an index item which satisfies all columns
24-
func (b *IndexBucket) Get(qualMap map[string]*proto.Quals, columns []string, limit, ttlSeconds int64) *IndexItem {
24+
func (b *IndexBucket) Get(qualMap map[string]*proto.Quals, columns []string, limit, ttlSeconds int64, keyColumns map[string]*proto.KeyColumn) *IndexItem {
2525
for _, item := range b.Items {
2626
log.Printf("[TRACE] IndexBucket.Get key %s limit %d", item.Key, item.Limit)
27-
if item.SatisfiesQuals(qualMap) && item.SatisfiesColumns(columns) && item.SatisfiesLimit(limit) && item.SatisfiesTtl(ttlSeconds) {
27+
if item.SatisfiesQuals(qualMap, keyColumns) && item.SatisfiesColumns(columns) && item.SatisfiesLimit(limit) && item.SatisfiesTtl(ttlSeconds) {
2828
return item
2929
}
3030
}

‎cache/index_item.go

+17-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,11 @@ func (i IndexItem) SatisfiesLimit(limit int64) bool {
6969
// our quals [id="1"], check quals [id="1"] -> SATISFIED
7070
// our quals [id="1"], check quals [id="1", foo=2] -> SATISFIED
7171
// our quals [id="1", foo=2], check quals [id="1"] -> NOT SATISFIED
72-
func (i IndexItem) SatisfiesQuals(checkQualMap map[string]*proto.Quals) bool {
72+
//
73+
// NOTE: some columns cannot use this subset logic. Generally this applies to columns which represent a filter which
74+
// is executed server side to filter the data returned.
75+
// In this case, we only identify a cache hit if the cached data has the _same_ value for the given colummn
76+
func (i IndexItem) SatisfiesQuals(checkQualMap map[string]*proto.Quals, keyColumns map[string]*proto.KeyColumn) bool {
7377
log.Printf("[TRACE] SatisfiesQuals")
7478
for col, indexQuals := range i.Quals {
7579
log.Printf("[TRACE] col %s", col)
@@ -89,6 +93,18 @@ func (i IndexItem) SatisfiesQuals(checkQualMap map[string]*proto.Quals) bool {
8993
return false
9094
}
9195
}
96+
97+
// now for each of the check quals, see whether it requires an exact match in the cached data.
98+
// i.e. the same qual must exist in the cached data
99+
for col, checkQuals := range checkQualMap {
100+
if keyColumn, ok := keyColumns[col]; ok && keyColumn.CacheMatch == CacheMatchExact {
101+
quals, ok := i.Quals[col]
102+
if !ok || !quals.Equals(checkQuals) {
103+
return false
104+
}
105+
}
106+
}
107+
92108
return true
93109
}
94110

‎cache/query_cache.go

+16-23
Original file line numberDiff line numberDiff line change
@@ -167,11 +167,13 @@ func (c *QueryCache) Get(ctx context.Context, table string, qualMap map[string]*
167167
}
168168

169169
func (c *QueryCache) buildCacheQualMap(table string, qualMap map[string]*proto.Quals) map[string]*proto.Quals {
170-
shouldIncludeQual := c.getShouldIncludeQualInKey(table)
170+
keyColumns := c.getKeyColumnsForTable(table)
171+
171172
cacheQualMap := make(map[string]*proto.Quals)
172173
for col, quals := range qualMap {
173174
log.Printf("[TRACE] buildCacheQualMap col %s, quals %+v", col, quals)
174-
if shouldIncludeQual(col) {
175+
// if this column is a key column, include in key
176+
if _, ok := keyColumns[col]; ok {
175177
log.Printf("[TRACE] INCLUDING COLUMN")
176178
cacheQualMap[col] = quals
177179
} else {
@@ -185,7 +187,9 @@ func (c *QueryCache) Clear() {
185187
c.cache.Clear()
186188
}
187189

188-
func (c *QueryCache) getCachedResult(indexBucketKey, table string, qualMap map[string]*proto.Quals, columns []string, limit int64, ttlSeconds int64) *QueryCacheResult {
190+
func (c *QueryCache) getCachedResult(indexBucketKey, table string, qualMap map[string]*proto.Quals, columns []string, limit, ttlSeconds int64) *QueryCacheResult {
191+
keyColumns := c.getKeyColumnsForTable(table)
192+
189193
log.Printf("[TRACE] QueryCache getCachedResult - index bucket key: %s ttlSeconds %d\n", indexBucketKey, ttlSeconds)
190194
indexBucket, ok := c.getIndexBucket(indexBucketKey)
191195
if !ok {
@@ -195,7 +199,7 @@ func (c *QueryCache) getCachedResult(indexBucketKey, table string, qualMap map[s
195199
}
196200

197201
// now check whether we have a cache entry that covers the required quals and columns - check the index
198-
indexItem := indexBucket.Get(qualMap, columns, limit, ttlSeconds)
202+
indexItem := indexBucket.Get(qualMap, columns, limit, ttlSeconds, keyColumns)
199203
if indexItem == nil {
200204
limitString := "NONE"
201205
if limit != -1 {
@@ -280,28 +284,17 @@ func (c *QueryCache) formatQualMapForKey(table string, qualMap map[string]*proto
280284
return strings.Join(strs, "-")
281285
}
282286

283-
// only include key column quals and optional quals
284-
func (c *QueryCache) getShouldIncludeQualInKey(table string) func(string) bool {
287+
// return a map of key column for the given table
288+
func (c *QueryCache) getKeyColumnsForTable(table string) map[string]*proto.KeyColumn {
289+
res := make(map[string]*proto.KeyColumn)
285290
// build a list of all key columns
286291
tableSchema, ok := c.PluginSchema[table]
287-
if !ok {
288-
// any errors, just default to including the column
289-
return func(string) bool { return true }
290-
}
291-
var cols []string
292-
for _, k := range tableSchema.ListCallKeyColumnList {
293-
cols = append(cols, k.Name)
294-
}
295-
for _, k := range tableSchema.GetCallKeyColumnList {
296-
cols = append(cols, k.Name)
297-
}
298-
299-
return func(column string) bool {
300-
res := helpers.StringSliceContains(cols, column)
301-
log.Printf("[TRACE] shouldIncludeQual, column %s, include = %v", column, res)
302-
return res
292+
if ok {
293+
for _, k := range append(tableSchema.ListCallKeyColumnList, tableSchema.GetCallKeyColumnList...) {
294+
res[k.Name] = k
295+
}
303296
}
304-
297+
return res
305298
}
306299

307300
func (c *QueryCache) sanitiseKey(str string) string {

‎go.mod

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ require (
1919
github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
2020
github.com/turbot/go-kit v0.3.0
2121
github.com/zclconf/go-cty v1.8.2
22-
golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c // indirect
22+
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
23+
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect
2324
google.golang.org/grpc v1.41.0
2425
google.golang.org/protobuf v1.27.1
2526
)

‎go.sum

+9-4
Original file line numberDiff line numberDiff line change
@@ -165,8 +165,9 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
165165
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
166166
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
167167
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
168-
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
169168
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
169+
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
170+
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
170171
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
171172
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
172173
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -181,12 +182,16 @@ golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7w
181182
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
182183
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
183184
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
184-
golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c h1:QOfDMdrf/UwlVR0UBq2Mpr58UzNtvgJRXA4BgPfFACs=
185-
golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
185+
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
186+
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
187+
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng=
188+
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
189+
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
186190
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
187191
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
188-
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
189192
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
193+
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
194+
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
190195
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
191196
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
192197
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

0 commit comments

Comments
 (0)