Initial (temporary) implementation of search doc.

Document describing how to convert a kustomization file into a
searchable document on appengine (will be changed to elasticsearch)
soon.

internal/search/doc/doc.go

@@ -0,0 +1,169 @@
+package doc
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"sigs.k8s.io/yaml"
+
+	"google.golang.org/appengine/search"
+)
+
+const (
+	identifierStr   = "identifier"
+	documentStr     = "document"
+	repoURLStr      = "repo_url"
+	filePathStr     = "file_path"
+	creationTimeStr = "creation_time"
+)
+
+// Represents an unbreakable character stream.
+type Atom = search.Atom
+
+// Implements search.FieldLoadSaver in order to index this representation of a kustomization.yaml
+// file.
+type KustomizationDocument struct {
+	identifiers   []Atom
+	FilePath      Atom
+	RepositoryURL Atom
+	DocumentData  string
+	CreationTime  time.Time
+}
+
+// Partially implements search.FieldLoadSaver.
+func (k *KustomizationDocument) Load(fields []search.Field, metadata *search.DocumentMetadata) error {
+	k.identifiers = make([]search.Atom, 0)
+	wrongTypeError := func(name string, expected interface{}, actual interface{}) error {
+		return fmt.Errorf("%s expects type %T, found %#v", name, expected, actual)
+	}
+
+	for _, f := range fields {
+		switch f.Name {
+		case identifierStr:
+			identifier, ok := f.Value.(search.Atom)
+			if !ok {
+				return wrongTypeError(f.Name, identifier, f.Value)
+			}
+			k.identifiers = append(k.identifiers, identifier)
+
+		case documentStr:
+			document, ok := f.Value.(string)
+			if !ok {
+				return wrongTypeError(f.Name, document, f.Value)
+			}
+			k.DocumentData = document
+
+		case filePathStr:
+			fp, ok := f.Value.(search.Atom)
+			if !ok {
+				return wrongTypeError(f.Name, fp, f.Value)
+			}
+			k.FilePath = fp
+
+		case repoURLStr:
+			url, ok := f.Value.(search.Atom)
+			if !ok {
+				return wrongTypeError(f.Name, url, f.Value)
+			}
+			k.RepositoryURL = url
+
+		case creationTimeStr:
+			time, ok := f.Value.(time.Time)
+			if !ok {
+				return wrongTypeError(f.Name, time, f.Value)
+			}
+			k.CreationTime = time
+		default:
+			return fmt.Errorf("KustomizationDocument field %s not recognized", f.Name)
+		}
+	}
+
+	return nil
+}
+
+// Partially implements search.FieldLoadSaver.
+func (k *KustomizationDocument) Save() ([]search.Field, *search.DocumentMetadata, error) {
+	err := k.ParseYAML()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	extraFields := []search.Field{
+		{Name: documentStr, Value: k.DocumentData},
+		{Name: filePathStr, Value: k.FilePath},
+		{Name: repoURLStr, Value: k.RepositoryURL},
+		{Name: creationTimeStr, Value: k.CreationTime},
+	}
+
+	fields := make([]search.Field, 0, len(k.identifiers)+len(extraFields))
+	for _, identifier := range k.identifiers {
+		fields = append(fields, search.Field{Name: identifierStr, Value: identifier})
+	}
+	fields = append(fields, extraFields...)
+
+	return fields, nil, nil
+}
+
+func (k *KustomizationDocument) ParseYAML() error {
+	k.identifiers = make([]Atom, 0)
+
+	var kustomization map[string]interface{}
+	err := yaml.Unmarshal([]byte(k.DocumentData), &kustomization)
+	if err != nil {
+		return fmt.Errorf("unable to parse kustomization file: %s", err)
+	}
+
+	type Map struct {
+		data   map[string]interface{}
+		prefix Atom
+	}
+
+	toVisit := []Map{
+		{
+			data:   kustomization,
+			prefix: "",
+		},
+	}
+
+	atomJoin := func(vals ...interface{}) Atom {
+		strs := make([]string, 0, len(vals))
+		for _, val := range vals {
+			strs = append(strs, fmt.Sprint(val))
+		}
+		return Atom(strings.Trim(strings.Join(strs, " "), " "))
+	}
+
+	set := make(map[Atom]struct{})
+
+	for i := 0; i < len(toVisit); i++ {
+		visiting := toVisit[i]
+		for k, v := range visiting.data {
+			set[atomJoin(visiting.prefix, k)] = struct{}{}
+			switch value := v.(type) {
+			case map[string]interface{}:
+				toVisit = append(toVisit, Map{
+					data:   value,
+					prefix: atomJoin(visiting.prefix, fmt.Sprint(k)),
+				})
+			case []interface{}:
+				for _, val := range value {
+					submap, ok := val.(map[string]interface{})
+					if !ok {
+						continue
+					}
+					toVisit = append(toVisit, Map{
+						data:   submap,
+						prefix: atomJoin(visiting.prefix, fmt.Sprint(k)),
+					})
+				}
+			}
+		}
+	}
+
+	for key := range set {
+		k.identifiers = append(k.identifiers, key)
+	}
+
+	return nil
+}

internal/search/doc/doc_test.go

@@ -0,0 +1,153 @@
+package doc
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/appengine/search"
+)
+
+func TestLoadFailures(t *testing.T) {
+	type sentinelType struct{}
+	sentinel := sentinelType{}
+
+	testCases := [][]search.Field{
+		{{Name: identifierStr, Value: sentinel}},
+		{{Name: documentStr, Value: sentinel}},
+		{{Name: repoURLStr, Value: sentinel}},
+		{{Name: filePathStr, Value: sentinel}},
+		{{Name: creationTimeStr, Value: sentinel}},
+	}
+
+	for _, test := range testCases {
+		var k KustomizationDocument
+		err := k.Load(test, nil)
+		if err == nil {
+			t.Errorf("Type missmatch %#v should not be loadable", test)
+		}
+	}
+}
+
+func TestFieldLoadSaver(t *testing.T) {
+
+	commonTestCases := []KustomizationDocument{
+		{
+			identifiers:   []Atom{"namePrefix", "metadata.name", "kind"},
+			FilePath:      "some/path/kustomization.yaml",
+			RepositoryURL: "https://example.com/kustomize",
+			CreationTime:  time.Now(),
+			DocumentData: `
+namePrefix: dev-
+metadata:
+  name: app
+kind: Deployment
+`,
+		},
+	}
+
+	for _, test := range commonTestCases {
+		fields, metadata, err := test.Save()
+		if err != nil {
+			t.Errorf("Error calling Save(): %s\n", err)
+		}
+		doc := KustomizationDocument{}
+		err = doc.Load(fields, metadata)
+		if err != nil {
+			t.Errorf("Doc failed to load: %s\n", err)
+		}
+		if !reflect.DeepEqual(test, doc) {
+			t.Errorf("Expected loaded document (%+v) to be equal to (%+v)\n", doc, test)
+		}
+	}
+}
+
+func TestParseYAML(t *testing.T) {
+	testCases := []struct {
+		identifiers []Atom
+		yaml        string
+	}{
+		{
+			identifiers: []Atom{
+				"namePrefix",
+				"metadata",
+				"metadata name",
+				"kind",
+			},
+			yaml: `
+namePrefix: dev-
+metadata:
+  name: app
+kind: Deployment
+`,
+		},
+		{
+			identifiers: []Atom{
+				"namePrefix",
+				"metadata",
+				"metadata name",
+				"metadata spec",
+				"metadata spec replicas",
+				"kind",
+				"replicas",
+				"replicas name",
+				"replicas count",
+				"resource",
+			},
+			yaml: `
+namePrefix: dev-
+# map of map
+metadata:
+  name: n1
+  spec:
+    replicas: 3
+kind: Deployment
+
+#list of map
+replicas:
+- name: n1
+  count: 3
+- name: n2
+  count: 3
+
+# list
+resource:
+- file1.yaml
+- file2.yaml
+`,
+		},
+	}
+
+	atomStrs := func(atoms []Atom) []string {
+		strs := make([]string, 0, len(atoms))
+		for _, val := range atoms {
+			strs = append(strs, fmt.Sprintf("%v", val))
+		}
+		return strs
+	}
+
+	for _, test := range testCases {
+		doc := KustomizationDocument{
+			DocumentData: test.yaml,
+			FilePath:     "example/path/kustomization.yaml",
+		}
+
+		err := doc.ParseYAML()
+		if err != nil {
+			t.Errorf("Document error error: %s", err)
+		}
+
+		docIDs := atomStrs(doc.identifiers)
+		expectedIDs := atomStrs(test.identifiers)
+		sort.Strings(docIDs)
+		sort.Strings(expectedIDs)
+
+		if !reflect.DeepEqual(docIDs, expectedIDs) {
+			t.Errorf("Expected loaded document (%v) to be equal to (%v)\n",
+				strings.Join(docIDs, ","), strings.Join(expectedIDs, ","))
+		}
+	}
+}

internal/search/go.mod

@@ -0,0 +1,9 @@
+module sigs.k8s.io/kustomize/internal/search
+
+go 1.12
+
+require (
+	google.golang.org/appengine v1.6.1
+	gopkg.in/yaml.v2 v2.2.2 // indirect
+	sigs.k8s.io/yaml v1.1.0
+)

internal/search/go.sum

@@ -0,0 +1,24 @@
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

travis/pre-commit.sh

@@ -30,6 +30,7 @@ function testGoLangCILint {
 
 function testGoTest {
   go test -v ./...
+  (cd ./internal/search; go test -v ./...)
 }
 
 # These tests require the helm program, and at the moment