Skip to content

Commit 22c7018

Browse files
authored
Key columns should allow specifying supported operators. Closes #121
- deprecated KeyColumnSet - can now specify key columns using an array of KeyColumn objects, which have an 'operators' and 'require' property
1 parent 72db53f commit 22c7018

24 files changed

+831
-631
lines changed

go.mod

+5-12
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,16 @@ require (
77
github.com/gertd/go-pluralize v0.1.7
88
github.com/ghodss/yaml v1.0.0
99
github.com/golang/protobuf v1.4.3
10-
github.com/hashicorp/go-hclog v0.14.1
11-
github.com/hashicorp/go-plugin v1.3.0
10+
github.com/hashicorp/go-hclog v0.15.0
11+
github.com/hashicorp/go-plugin v1.4.1
1212
github.com/hashicorp/go-version v1.2.1
13-
github.com/hashicorp/hcl/v2 v2.8.2
13+
github.com/hashicorp/hcl/v2 v2.9.1
1414
github.com/iancoleman/strcase v0.1.2
15-
github.com/mattn/go-runewidth v0.0.9 // indirect
16-
github.com/mitchellh/go-testing-interface v1.0.0 // indirect
1715
github.com/olekukonko/tablewriter v0.0.4
1816
github.com/sethvargo/go-retry v0.1.0
1917
github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
20-
github.com/turbot/go-kit v0.1.1
21-
github.com/zclconf/go-cty v1.7.1
22-
golang.org/x/net v0.0.0-20201029221708-28c70e62bb1d // indirect
23-
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect
24-
golang.org/x/text v0.3.4 // indirect
25-
google.golang.org/genproto v0.0.0-20201029200359-8ce4113da6f7 // indirect
18+
github.com/turbot/go-kit v0.2.2-0.20210628165333-268ba0a30be3
19+
github.com/zclconf/go-cty v1.8.2
2620
google.golang.org/grpc v1.33.1
2721
google.golang.org/protobuf v1.25.0
28-
gopkg.in/yaml.v2 v2.2.8 // indirect
2922
)

go.sum

+25-56
Large diffs are not rendered by default.

grpc/proto/key_columns_set.go

-22
This file was deleted.

grpc/proto/plugin.pb.go

+258-140
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

grpc/proto/plugin.proto

+19-7
Original file line numberDiff line numberDiff line change
@@ -99,17 +99,29 @@ message TableSchema
9999
{
100100
repeated ColumnDefinition columns = 1;
101101
string description = 2;
102-
KeyColumnsSet getCallKeyColumns = 3;
103-
KeyColumnsSet listCallKeyColumns = 4;
104-
KeyColumnsSet listCallOptionalKeyColumns = 5;
102+
KeyColumnsSet getCallKeyColumns = 3 [deprecated=true];
103+
KeyColumnsSet listCallKeyColumns = 4 [deprecated=true];
104+
KeyColumnsSet listCallOptionalKeyColumns = 5 [deprecated=true];
105+
106+
repeated KeyColumn getCallKeyColumnList = 6;
107+
repeated KeyColumn listCallKeyColumnList = 7;
105108
}
106109

107-
// a set of Key Columns, all of which are reuired
110+
// a set of Key Columns, required for get/list calls
111+
// deprecated - kept for compatibility
108112
message KeyColumnsSet
109113
{
110-
string single = 1;
111-
repeated string all = 2;
112-
repeated string any = 3;
114+
option deprecated = true;
115+
string single = 1;
116+
repeated string all = 2;
117+
repeated string any = 3;
118+
}
119+
120+
message KeyColumn
121+
{
122+
string name = 1;
123+
repeated string operators = 2;
124+
string require=3;
113125
}
114126

115127
message Schema {

plugin/key_column.go

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package plugin
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"github.com/gertd/go-pluralize"
8+
"github.com/turbot/go-kit/helpers"
9+
"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
10+
)
11+
12+
const (
13+
Required = "required"
14+
Optional = "optional"
15+
AnyOf = "any_of"
16+
)
17+
18+
// KeyColumn is a struct representing the definition of a KeyColumn used to filter and Get/List call
19+
type KeyColumn struct {
20+
Name string
21+
Operators []string
22+
Require string
23+
}
24+
25+
func (k KeyColumn) String() string {
26+
return fmt.Sprintf("column:'%s' %s: %s", k.Name, pluralize.NewClient().Pluralize("operator", len(k.Operators), false), strings.Join(k.Operators, ","))
27+
}
28+
29+
// ToProtobuf converts the KeyColumn to a protobuf object
30+
func (k *KeyColumn) ToProtobuf() *proto.KeyColumn {
31+
return &proto.KeyColumn{
32+
Name: k.Name,
33+
Operators: k.Operators,
34+
Require: k.Require,
35+
}
36+
}
37+
38+
// SingleEqualsQual returns whether this key column has a single = operator
39+
func (k *KeyColumn) SingleEqualsQual() bool {
40+
return len(k.Operators) == 1 && k.Operators[0] == "="
41+
}
42+
43+
func (k *KeyColumn) Validate() []string {
44+
// ensure operators are valid
45+
46+
// map "!=" operator to "<>"
47+
validOperators := []string{"=", "<>", "<", "<=", ">", ">="}
48+
validRequire := []string{Required, Optional, AnyOf}
49+
var res []string
50+
51+
for _, op := range k.Operators {
52+
// convert "!=" to "<>"
53+
if op == "!=" {
54+
op = "<>"
55+
}
56+
if !helpers.StringSliceContains(validOperators, op) {
57+
res = append(res, fmt.Sprintf("operator %s is not valid, it must be one of: %s", op, strings.Join(validOperators, ",")))
58+
}
59+
}
60+
// default Require to Required
61+
if k.Require == "" {
62+
k.Require = Required
63+
}
64+
if !helpers.StringSliceContains(validRequire, k.Require) {
65+
res = append(res, fmt.Sprintf("Require value '%s' is not valid, it must be one of: %s", k.Require, strings.Join(validRequire, ",")))
66+
}
67+
return res
68+
}

plugin/key_column_qual.go

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package plugin
2+
3+
import (
4+
"github.com/turbot/steampipe-plugin-sdk/plugin/quals"
5+
6+
"github.com/turbot/go-kit/helpers"
7+
)
8+
9+
// KeyColumnQuals is a struct representing all quals for a specific column
10+
type KeyColumnQuals struct {
11+
Name string
12+
Quals quals.QualSlice
13+
}
14+
15+
func (k KeyColumnQuals) SatisfiesKeyColumn(keyColumn *KeyColumn) bool {
16+
if keyColumn.Name != k.Name {
17+
return false
18+
}
19+
for _, q := range k.Quals {
20+
if helpers.StringSliceContains(keyColumn.Operators, q.Operator) {
21+
return true
22+
}
23+
}
24+
return false
25+
}
26+
27+
func (k KeyColumnQuals) SingleEqualsQual() bool {
28+
return k.Quals.SingleEqualsQual()
29+
}

plugin/key_column_qual_map.go

+143
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package plugin
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"strings"
7+
8+
"github.com/turbot/go-kit/helpers"
9+
"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
10+
"github.com/turbot/steampipe-plugin-sdk/plugin/quals"
11+
)
12+
13+
// KeyColumnQualMap is a map of KeyColumnQuals keyed by column name
14+
type KeyColumnQualMap map[string]*KeyColumnQuals
15+
16+
// ToEqualsQualValueMap converts a KeyColumnQualMap to a column-qual value map, including only the
17+
func (m KeyColumnQualMap) ToEqualsQualValueMap() map[string]*proto.QualValue {
18+
res := make(map[string]*proto.QualValue, len(m))
19+
for k, v := range m {
20+
if v.SingleEqualsQual() {
21+
res[k] = v.Quals[0].Value
22+
}
23+
}
24+
return res
25+
}
26+
27+
func (m KeyColumnQualMap) String() string {
28+
strs := make([]string, len(m))
29+
for _, k := range m {
30+
var values = make([]interface{}, len(k.Quals))
31+
for i, v := range k.Quals {
32+
values[i] = v.Value
33+
}
34+
strs = append(strs, fmt.Sprintf("%s - %v", k.Name, values))
35+
}
36+
return strings.Join(strs, "\n")
37+
}
38+
39+
func (m KeyColumnQualMap) SatisfiesKeyColumns(columns KeyColumnSlice) (bool, KeyColumnSlice) {
40+
log.Printf("[WARN] SatisfiesKeyColumns %v", columns)
41+
42+
if columns == nil {
43+
return true, nil
44+
}
45+
var unsatisfiedKeyColumns KeyColumnSlice
46+
satisfiedCount := map[string]int{
47+
Required: 0,
48+
AnyOf: 0,
49+
Optional: 0,
50+
}
51+
unsatisfiedCount := map[string]int{
52+
Required: 0,
53+
AnyOf: 0,
54+
Optional: 0,
55+
}
56+
57+
for _, keyColumn := range columns {
58+
// look for this key column in our map
59+
k := m[keyColumn.Name]
60+
satisfied := k != nil && k.SatisfiesKeyColumn(keyColumn)
61+
if satisfied {
62+
satisfiedCount[keyColumn.Require]++
63+
64+
log.Printf("[TRACE] key column satisfied %v", keyColumn)
65+
66+
} else {
67+
unsatisfiedCount[keyColumn.Require]++
68+
unsatisfiedKeyColumns = append(unsatisfiedKeyColumns, keyColumn)
69+
log.Printf("[TRACE] key column NOT satisfied %v", keyColumn)
70+
// if this was NOT an optional key column, we are not satisfied
71+
}
72+
}
73+
74+
// we are satisfied if:
75+
// all Required key columns are satisfied
76+
// either there is at least 1 satisfied AnyOf key columns, or there are no AnyOf columns
77+
res := unsatisfiedCount[Required] == 0 && (satisfiedCount[AnyOf] > 0 || unsatisfiedCount[AnyOf] == 0)
78+
79+
log.Printf("[WARN] SatisfiesKeyColumns result: %v, satisfiedCount %v, unsatisfiedCount %v, unsatisfiedKeyColumns %v", res, satisfiedCount, unsatisfiedCount, unsatisfiedKeyColumns)
80+
return res, unsatisfiedKeyColumns
81+
}
82+
83+
// ToQualMap converts the map into a simpler map of column to []Quals
84+
// this is needed to avoid the transform package needing to reference plugin
85+
func (m KeyColumnQualMap) ToQualMap() map[string]quals.QualSlice {
86+
var res = make(map[string]quals.QualSlice)
87+
for k, v := range m {
88+
res[k] = v.Quals
89+
}
90+
return res
91+
}
92+
93+
// NewKeyColumnQualValueMap creates a KeyColumnQualMap from a qual map and a KeyColumnSlice
94+
func NewKeyColumnQualValueMap(qualMap map[string]*proto.Quals, keyColumns KeyColumnSlice) KeyColumnQualMap {
95+
res := KeyColumnQualMap{}
96+
97+
for _, col := range keyColumns {
98+
matchingQuals := getMatchingQuals(col, qualMap)
99+
for _, q := range matchingQuals {
100+
// convert proto.Qual into a qual.Qual (which is easier to use)
101+
qual := quals.NewQual(q)
102+
103+
// if there is already an entry for this column, add a value to the array
104+
if mapEntry, mapEntryExists := res[col.Name]; mapEntryExists {
105+
mapEntry.Quals = append(mapEntry.Quals, qual)
106+
res[col.Name] = mapEntry
107+
} else {
108+
// crate a new map entry for this column
109+
res[col.Name] = &KeyColumnQuals{
110+
Name: col.Name,
111+
Quals: quals.QualSlice{qual},
112+
}
113+
}
114+
}
115+
}
116+
return res
117+
}
118+
119+
// look in a column-qual map for quals with column and operator matching the key column
120+
func getMatchingQuals(keyColumn *KeyColumn, qualMap map[string]*proto.Quals) []*proto.Qual {
121+
log.Printf("[TRACE] getMatchingQuals keyColumn %s qualMap %s", keyColumn, qualMap)
122+
123+
quals, ok := qualMap[keyColumn.Name]
124+
if !ok {
125+
log.Printf("[TRACE] getMatchingQuals returning false - qualMap does not contain any quals for colums %s", keyColumn.Name)
126+
return nil
127+
}
128+
129+
var res []*proto.Qual
130+
for _, q := range quals.Quals {
131+
operator := q.GetStringValue()
132+
if helpers.StringSliceContains(keyColumn.Operators, operator) {
133+
res = append(res, q)
134+
}
135+
}
136+
if len(res) > 0 {
137+
log.Printf("[TRACE] getMatchingQuals found %d quals matching key column %s", len(res), keyColumn)
138+
} else {
139+
log.Printf("[TRACE] getMatchingQuals returning false - qualMap does not contain any matching quals for quals for key column %s", keyColumn)
140+
}
141+
142+
return res
143+
}

plugin/key_column_set.go

-42
This file was deleted.

0 commit comments

Comments
 (0)