add experimental support to expand helm templates in qbec

.travis.yml

@@ -7,7 +7,7 @@ before_install:
   - go get github.com/mattn/goveralls
 
 install:
-  make install get
+  make install-ci install get
 
 script:
   - make

Makefile

@@ -30,6 +30,12 @@ lint:
 	go list ./... | grep -v vendor | xargs go vet
 	go list ./... | grep -v vendor | xargs golint
 
+.PHONY: install-ci
+install-ci:
+	curl -sSL -o helm.tar.gz https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz
+	tar -xvzf helm.tar.gz
+	mv linux-amd64/helm $(GOPATH)/bin/
+
 .PHONY: install
 install:
 	go get github.com/golang/dep/cmd/dep

internal/vm/helm.go

@@ -0,0 +1,100 @@
+package vm
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+)
+
+// helmOptions are options that can be passed to the helm template command as well
+// as a `thisFile` option that the caller needs to set from `std.thisFile` to make
+// relative references to charts work correctly.
+type helmOptions struct {
+	Execute      []string `json:"execute"`      // --execute option
+	KubeVersion  string   `json:"kubeVersion"`  // --kube-version option
+	Name         string   `json:"name"`         // --name option
+	NameTemplate string   `json:"nameTemplate"` // --name-template option
+	Namespace    string   `json:"namespace"`    // --namespace option
+	ThisFile     string   `json:"thisFile"`     // use supplied file as current file to resolve relative refs, should be set to std.thisFile
+	Verbose      bool     `json:"verbose"`      // print helm template command before executing it
+	//IsUpgrade    bool     `json:"isUpgrade"` // --is-upgrade option, defer adding this until implications are known,
+}
+
+// toArgs converts options to a slice of command-line args.
+func (h helmOptions) toArgs() []string {
+	var ret []string
+	if len(h.Execute) > 0 {
+		for _, e := range h.Execute {
+			ret = append(ret, "--execute", e)
+		}
+	}
+	if h.KubeVersion != "" {
+		ret = append(ret, "--kube-version", h.KubeVersion)
+	}
+	if h.Name != "" {
+		ret = append(ret, "--name", h.Name)
+	}
+	if h.NameTemplate != "" {
+		ret = append(ret, "--name-template", h.NameTemplate)
+	}
+	if h.Namespace != "" {
+		ret = append(ret, "--namespace", h.Namespace)
+	}
+	//if h.IsUpgrade {
+	//	ret = append(ret, "--is-upgrade")
+	//}
+	return ret
+}
+
+// expandHelmTemplate produces an array of objects parsed from the output of running `helm template` with
+// the supplied values and helm options.
+func expandHelmTemplate(chart string, values map[string]interface{}, options helmOptions) (out []interface{}, finalErr error) {
+	// run command from the directory containing current file or the OS temp dir if `thisFile` not specified. That is,
+	// explicitly fail to resolve relative refs unless the calling file is specified; don't let them work by happenstance.
+	workDir := os.TempDir()
+	if options.ThisFile != "" {
+		dir := filepath.Dir(options.ThisFile)
+		if !filepath.IsAbs(dir) {
+			wd, err := os.Getwd()
+			if err != nil {
+				return nil, errors.Wrap(err, "get working directory")
+			}
+			dir = filepath.Join(wd, dir)
+		}
+		workDir = dir
+	}
+
+	valueBytes, err := yaml.Marshal(values)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshal values to YAML")
+	}
+
+	args := append([]string{"template", chart}, options.toArgs()...)
+	args = append(args, "--values", "-")
+
+	var stdout bytes.Buffer
+	cmd := exec.Command("helm", args...)
+	cmd.Stdin = bytes.NewBuffer(valueBytes)
+	cmd.Stdout = &stdout
+	cmd.Stderr = os.Stderr
+	cmd.Dir = workDir
+
+	if options.Verbose {
+		fmt.Fprintf(os.Stderr, "[helm template] cd %s && helm %s\n", workDir, strings.Join(args, " "))
+	}
+
+	if err := cmd.Run(); err != nil {
+		if options.ThisFile == "" {
+			fmt.Fprintln(os.Stderr, "[WARN] helm template command failed, you may need to set the 'thisFile' option to make relative chart paths work")
+		}
+		return nil, errors.Wrap(err, "run helm template command")
+	}
+
+	return parseYAMLDocuments(bytes.NewReader(stdout.Bytes()))
+}

internal/vm/helm_test.go

@@ -0,0 +1,111 @@
+package vm
+
+import (
+	"encoding/json"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHelmOptions(t *testing.T) {
+	a := assert.New(t)
+	var h helmOptions
+	a.Nil(h.toArgs())
+	h = helmOptions{
+		Execute:     []string{"a.yaml", "b.yaml"},
+		KubeVersion: "1.10",
+		Name:        "foo",
+		Namespace:   "foobar",
+		ThisFile:    "/path/to/my.jsonnet",
+		Verbose:     true,
+	}
+	a.EqualValues([]string{
+		"--execute", "a.yaml",
+		"--execute", "b.yaml",
+		"--kube-version", "1.10",
+		"--name", "foo",
+		"--namespace", "foobar",
+	}, h.toArgs())
+}
+
+type cmOrSecret struct {
+	APIVersion string `json:"apiVersion"`
+	Kind       string `json:"kind"`
+	Metadata   struct {
+		Namespace string `json:"namespace"`
+		Name      string `json:"name"`
+	}
+	Data map[string]string `json:"data"`
+}
+
+func TestHelmSimpleExpand(t *testing.T) {
+	a := assert.New(t)
+	jvm := New(Config{})
+	file := "./consumer.jsonnet"
+	inputCode := `
+local expandHelmTemplate = std.native('expandHelmTemplate');
+
+expandHelmTemplate(
+    './testdata/charts/foobar',
+    {
+        foo: 'barbar',
+    },
+    {
+        namespace: 'my-ns',
+        name: 'my-name',
+        thisFile: std.thisFile,
+		verbose: true,
+    }
+)
+`
+	code, err := jvm.EvaluateSnippet(file, inputCode)
+	require.Nil(t, err)
+
+	var output []cmOrSecret
+	err = json.Unmarshal([]byte(code), &output)
+
+	require.Equal(t, 2, len(output))
+
+	sort.Slice(output, func(i, j int) bool {
+		return output[i].Kind < output[j].Kind
+	})
+
+	ob := output[0]
+	a.Equal("ConfigMap", ob.Kind)
+	a.Equal("my-ns", ob.Metadata.Namespace)
+	a.Equal("my-name", ob.Metadata.Name)
+	a.Equal("barbar", ob.Data["foo"])
+	a.Equal("baz", ob.Data["bar"])
+
+	ob = output[1]
+	a.Equal("Secret", ob.Kind)
+	a.Equal("my-ns", ob.Metadata.Namespace)
+	a.Equal("my-name", ob.Metadata.Name)
+	a.Equal("Y2hhbmdlbWUK", ob.Data["secret"])
+}
+
+func TestHelmBadRelative(t *testing.T) {
+	a := assert.New(t)
+	jvm := New(Config{})
+	file := "./consumer.jsonnet"
+	inputCode := `
+local expandHelmTemplate = std.native('expandHelmTemplate');
+
+expandHelmTemplate(
+    './testdata/charts/foobar',
+    {
+        foo: 'barbar',
+    },
+    {
+        namespace: 'my-ns',
+        name: 'my-name',
+		verbose: true,
+    }
+)
+`
+	_, err := jvm.EvaluateSnippet(file, inputCode)
+	require.NotNil(t, err)
+	a.Contains(err.Error(), "exit status 1")
+}

internal/vm/nativefuncs.go

@@ -21,12 +21,11 @@ package vm
 import (
 	"bytes"
 	"encoding/json"
-	"io"
 	"regexp"
 
 	"github.com/google/go-jsonnet"
 	"github.com/google/go-jsonnet/ast"
-	"k8s.io/apimachinery/pkg/util/yaml"
+	"github.com/pkg/errors"
 )
 
 // registerNativeFuncs adds kubecfg's native jsonnet functions to provided VM
@@ -51,20 +50,8 @@ func registerNativeFuncs(vm *jsonnet.VM) {
 		Name:   "parseYaml",
 		Params: []ast.Identifier{"yaml"},
 		Func: func(args []interface{}) (res interface{}, err error) {
-			ret := []interface{}{}
 			data := []byte(args[0].(string))
-			d := yaml.NewYAMLToJSONDecoder(bytes.NewReader(data))
-			for {
-				var doc interface{}
-				if err := d.Decode(&doc); err != nil {
-					if err == io.EOF {
-						break
-					}
-					return nil, err
-				}
-				ret = append(ret, doc)
-			}
-			return ret, nil
+			return parseYAMLDocuments(bytes.NewReader(data))
 		},
 	})
 
@@ -99,4 +86,24 @@ func registerNativeFuncs(vm *jsonnet.VM) {
 			return r.ReplaceAllString(src, repl), nil
 		},
 	})
+
+	vm.NativeFunction(&jsonnet.NativeFunction{
+		Name:   "expandHelmTemplate",
+		Params: []ast.Identifier{"chart", "values", "options"},
+		Func: func(args []interface{}) (res interface{}, err error) {
+			chart := args[0].(string)
+			values := args[1].(map[string]interface{})
+			options := args[2].(map[string]interface{})
+			var h helmOptions
+			b, err := json.Marshal(options)
+			if err != nil {
+				return nil, errors.Wrap(err, "marshal options to JSON")
+			}
+			if err := json.Unmarshal(b, &h); err != nil {
+				return nil, errors.Wrap(err, "unmarshal options from JSON")
+			}
+			return expandHelmTemplate(chart, values, h)
+		},
+	})
+
 }

internal/vm/testdata/charts/foobar/Chart.yaml

@@ -0,0 +1,3 @@
+apiVersion: v1
+name: foobar
+version: 1.0

internal/vm/testdata/charts/foobar/templates/config-map.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ .Release.Name }}
+data:
+  foo: {{.Values.foo}}
+  bar: {{default "baz" .Values.bar}}

internal/vm/testdata/charts/foobar/templates/secret.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ .Release.Name }}
+data:
+  secret: {{.Values.secret}}

internal/vm/testdata/charts/foobar/values.yaml

@@ -0,0 +1,3 @@
+foo: bar
+secret: Y2hhbmdlbWUK
+

internal/vm/yaml.go

@@ -0,0 +1,25 @@
+package vm
+
+import (
+	"io"
+
+	"k8s.io/apimachinery/pkg/util/yaml"
+)
+
+func parseYAMLDocuments(reader io.Reader) ([]interface{}, error) {
+	ret := []interface{}{}
+	d := yaml.NewYAMLToJSONDecoder(reader)
+	for {
+		var doc interface{}
+		if err := d.Decode(&doc); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		if doc != nil {
+			ret = append(ret, doc)
+		}
+	}
+	return ret, nil
+}

site/content/comparison-with-other-tools/_index.md

@@ -95,7 +95,10 @@ We are not averse to adding this feature but would like to do it consciously aft
 
 ### Support for community components, prototypes, helm charts etc.
 
-Only ksonnet has these feature. In this author's opinion these features have made the ksonnet surface area large and
+Only ksonnet has these features. In this author's opinion these features have made the ksonnet surface area large and
 difficult to reason about so we have left them out. 
 
 It is probably useful to provide support for helm charts in qbec.
+
+**Update**: As of v0.6.1, qbec has experimental support to expand helm chart templates. This is implemented as a
+native function that can be called from jsonnet code.