Skip to content

Commit

Permalink
support keyword and key/value search pagination (#70)
Browse files Browse the repository at this point in the history 
* support keyword and key/value search pagination
1. query?keyword=test&first=1&offset=1
2. query?objtype=pod&first=1&offset=4
"first" default value is 1000 and cannot exceed maximum value 10000

* change grpc client maximum receive size from default 4M to 20M

* revert back entity service unit tests
  • Loading branch information
@formuzi
formuzi authored Jan 29, 2019
1 parent c02ec8c commit d21430f
Show file tree
Hide file tree
Showing 9 changed files with 166 additions and 99 deletions.
14 changes: 8 additions & 6 deletions service/apis/entity_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,14 @@ func (s EntityService) DeleteEntity(uuid string) error {

// DeleteEntityByResourceID remove object by given resourceid
func (s EntityService) DeleteEntityByResourceID(meta string, rid string) error {
qm := map[string][]string{util.ResourceID: {rid}, util.ObjType: {meta}}
qm := map[string][]string{util.ResourceID: {rid}, util.ObjType: {meta}, util.Print: {util.ResourceID}}
queryService := NewQueryService(s.dbclient)
node, err := queryService.GetQueryResult(qm)
if err != nil {
log.Error(err)
return err
}
if len(node[util.Objects].([]interface{})) > 0 {
if node[util.Count].(float64) > 0 {
// got existing object id
for _, obj := range node[util.Objects].([]interface{}) {
err = s.dbclient.DeleteEntity(obj.(map[string]interface{})[util.UID].(string))
Expand Down Expand Up @@ -147,14 +147,16 @@ func (s EntityService) CreateEntity(meta string, data map[string]interface{}) (m
if mutex.TryLock(data[util.ResourceID]) {
defer mutex.Unlock(data[util.ResourceID])
// check if entity already exist
qm := map[string][]string{util.ResourceID: {data[util.ResourceID].(string)}, util.ObjType: {meta}}
qm := map[string][]string{util.ResourceID: {data[util.ResourceID].(string)},
util.ObjType: {meta},
util.Print: {util.ResourceVersion}}
queryService := NewQueryService(s.dbclient)
node, err := queryService.GetQueryResult(qm)
if err != nil {
log.Error(err)
return nil, err
}
if len(node[util.Objects].([]interface{})) > 0 {
if node[util.Count].(float64) > 0 {
// got existing object id
uid := node[util.Objects].([]interface{})[0].(map[string]interface{})[util.UID].(string)
data[util.UID] = uid
Expand Down Expand Up @@ -292,15 +294,15 @@ func buildDataMap(k8sObj interface{}, relData interface{}, relType string, clust
// get uid from relationship object, if object not present, create it
func (s EntityService) getUIDFromRelData(data map[string]interface{}, objType string) (*string, error) {
// query by ResourceID to get uid
qm := map[string][]string{util.ResourceID: {data[util.ResourceID].(string)}, util.ObjType: {objType}}
qm := map[string][]string{util.ResourceID: {data[util.ResourceID].(string)}, util.ObjType: {objType}, util.Print: {util.ResourceID}}
queryService := NewQueryService(s.dbclient)
node, err := queryService.GetQueryResult(qm)
if err != nil {
log.Error(err)
return nil, err
}
var uid string
if len(node[util.Objects].([]interface{})) > 0 {
if node[util.Count].(float64) > 0 {
// got existing object id
uid = node[util.Objects].([]interface{})[0].(map[string]interface{})[util.UID].(string)
} else {
Expand Down
30 changes: 16 additions & 14 deletions service/apis/entity_service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,22 @@ func TestCreateEntity(t *testing.T) {
s := NewEntityService(dc)
// create node
node := map[string]interface{}{
"objtype": "K8sNode",
"objtype": "k8snode",
"name": "node02",
"labels": "testingnode02",
}
// create index for query
dc.CreateSchema(db.Schema{Predicate: "name", Type: "string", Index: true, Tokenizer: []string{"term"}})
dc.CreateSchema(db.Schema{Predicate: "resourceid", Type: "string", Index: true, Tokenizer: []string{"term"}})
dc.CreateSchema(db.Schema{Predicate: "objtype", Type: "string", Index: true, Tokenizer: []string{"term"}})
nids, _ := s.CreateEntity("K8sNode", node)
nids, _ := s.CreateEntity("k8snode", node)
var nid string
for _, v := range nids {
nid = v
break
}
defer s.DeleteEntity(nid)
n, _ := s.GetEntity("K8sNode", nid)
n, _ := s.GetEntity("k8snode", nid)
o := n["objects"].([]interface{})[0].(map[string]interface{})
if val, ok := o["labels"]; ok {
assert.Equal(t, val, "testingnode02", "node label not equals to testnode02")
Expand All @@ -46,13 +46,13 @@ func TestDeleteEntityByRid(t *testing.T) {
s := NewEntityService(dc)
// create node
node := map[string]interface{}{
"objtype": "K8sNode",
"objtype": "k8snode",
"name": "node02",
"labels": "testingnode02",
"resourceid": "noderid",
}
s.CreateEntity("K8sNode", node)
err := s.DeleteEntityByResourceID("K8sNode", "noderid")
s.CreateEntity("k8snode", node)
err := s.DeleteEntityByResourceID("k8snode", "noderid")
assert.Nil(t, err)
dc.Close()
}
Expand Down Expand Up @@ -444,21 +444,23 @@ func TestMultiCreateEntity(t *testing.T) {
dc := db.NewDGClient("127.0.0.1:9080")
q := NewQueryService(dc)
defer dc.Close()
dc.CreateSchema(db.Schema{Predicate: "name", Type: "string", Index: true, Tokenizer: []string{"term"}})
dc.CreateSchema(db.Schema{Predicate: "resourceid", Type: "string", Index: true, Tokenizer: []string{"term"}})
dc.CreateSchema(db.Schema{Predicate: "objtype", Type: "string", Index: true, Tokenizer: []string{"term"}})
s := NewEntityService(dc)

var wg sync.WaitGroup
rest := make(chan map[string]string)
wg.Add(100)
for i := 0; i < 50; i++ {
go func(version string) {
defer wg.Done()
node := map[string]interface{}{
"objtype": "K8sNode",
"objtype": "k8snode",
"name": "multinode",
"label": version,
"resourceid": "cluster:ns:multinode",
}
nids, _ := s.CreateEntity("K8sNode", node)
nids, _ := s.CreateEntity("k8snode", node)
rest <- nids
}(strconv.Itoa(i))
}
Expand All @@ -467,13 +469,13 @@ func TestMultiCreateEntity(t *testing.T) {
go func(version string) {
defer wg.Done()
node := map[string]interface{}{
"objtype": "K8sNode",
"objtype": "k8snode",
"name": "multinode2",
"label": version,
"resourceid": "cluster:ns:multinode2",
"resourceversion": version,
}
nids, _ := s.CreateEntity("K8sNode", node)
nids, _ := s.CreateEntity("k8snode", node)
rest <- nids
}(strconv.Itoa(i))
}
Expand All @@ -484,10 +486,10 @@ func TestMultiCreateEntity(t *testing.T) {
}
}()
wg.Wait()
qm := map[string][]string{"resourceid": {"cluster:ns:multinode"}, "objtype": {"K8sNode"}}
qm := map[string][]string{"resourceid": {"cluster:ns:multinode"}, "objtype": {"k8snode"}}
n, _ := q.GetQueryResult(qm)
o := n["objects"].([]interface{})
assert.Equal(t, 1, len(o), "only one object expect to be created with same resourceid")
s.DeleteEntityByResourceID("K8sNode", "cluster:ns:multinode")
s.DeleteEntityByResourceID("K8sNode", "cluster:ns:multinode2")
s.DeleteEntityByResourceID("k8snode", "cluster:ns:multinode")
s.DeleteEntityByResourceID("k8snode", "cluster:ns:multinode2")
}
31 changes: 17 additions & 14 deletions service/apis/qsl_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"fmt"
log "github.com/Sirupsen/logrus"
"github.com/intuit/katlas/service/db"
"github.com/intuit/katlas/service/util"
"strconv"
)

Expand All @@ -23,7 +24,6 @@ var filterRegex = `\@([a-zA-Z0-9\(\)]*)([\!\<\>\=]*)(\"?[a-zA-Z0-9\-\.\|\&\:_]*\
// QSLService service for QSL
type QSLService struct {
DBclient db.IDGClient
metaSvc *MetaService
}

// MaximumLimit define pagination limit
Expand Down Expand Up @@ -51,7 +51,8 @@ func IsStar(s string) bool {

// GetMetadata - get a list of the fields supoorted for this object type
func (qa *QSLService) GetMetadata(objtype string) ([]MetadataField, error) {
metafieldslist, err := qa.metaSvc.GetMetadataFields(objtype)
m := NewMetaService(qa.DBclient)
metafieldslist, err := m.GetMetadataFields(objtype)
if err != nil {
log.Error(err)
return []MetadataField{}, errors.New("Failed to connect to dgraph to get metadata")
Expand All @@ -64,8 +65,8 @@ func (qa *QSLService) GetMetadata(objtype string) ([]MetadataField, error) {
}

// NewQSLService creates an instance of a QSLService
func NewQSLService(host db.IDGClient, m *MetaService) *QSLService {
return &QSLService{host, m}
func NewQSLService(host db.IDGClient) *QSLService {
return &QSLService{host}
}

// CreateFiltersQuery translates the filters part of the qsl string to dgraph
Expand Down Expand Up @@ -95,16 +96,19 @@ func CreateFiltersQuery(filterlist string) (string, string, string, error) {
for _, item := range splitlist {
splitval := strings.Split(item, "=")

if splitval[0] == "first" || splitval[0] == "offset" {
switch splitval[0] {
case util.First, util.Offset:
paginate += "," + splitval[0] + ": " + splitval[1]
val, err := strconv.Atoi(splitval[1])
if err != nil || val > MaximumLimit {
return "", "", "", fmt.Errorf("pagination format error or exceeding maxiumum limit %d", MaximumLimit)
}
} else {
default:
return "", "", "", errors.New("Invalid pagination filters in " + filterlist)
}

val, err := strconv.Atoi(splitval[1])
if err != nil {
return "", "", "", errors.New("Pagination format error " + filterlist)
}
if splitval[0] == util.First && val > MaximumLimit {
return "", "", "", fmt.Errorf("pagination exceeding maxiumum limit %d", MaximumLimit)
}
}
// get rid of the first comma
paginate = paginate[0:]
Expand Down Expand Up @@ -472,14 +476,13 @@ func (qa *QSLService) getRelationName(objType string, parent string) (string, er

if !found {
// if not, see if we can find the relation from the parent to this object
metafieldslist2, err := qa.metaSvc.GetMetadataFields(parent)
m := NewMetaService(qa.DBclient)
metafieldslist2, err := m.GetMetadataFields(parent)
if err != nil {
log.Error(err)
return "", errors.New("Failed to connect to dgraph to get metadata")
}
log.Debugf("couldn't find relation for %s->%s,", parent, objType)
log.Debugf("metadata fields for %s: %#v", parent, metafieldslist)

for _, item := range metafieldslist2 {
if item.FieldType == "relationship" {
log.Debugf("2 found relationship for %s-%s->%s", parent, item.FieldName, item.RefDataType)
Expand Down
2 changes: 1 addition & 1 deletion service/apis/qsl_service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ func TestCreateDgraphQuery(t *testing.T) {
dc := db.NewDGClient(dgraphHost)
defer dc.Close()
metaSvc := NewMetaService(dc)
qslSvc := NewQSLService(dc, metaSvc)
qslSvc := NewQSLService(dc)

// Initialize metadata
meta, err := ioutil.ReadFile("../data/meta.json")
Expand Down
Loading

0 comments on commit d21430f

Please sign in to comment.