add experimental support to expand helm templates in qbec

add a new native function called `expandHelmTemplate` that accepts a chart name,
values as a JSON object and additional helm options. Since native functions do not
have caller context, the caller needs to pass a `thisFile` value in the options
object for relative references to charts succeed. This should be set to `std.thisFile`

This allows a jsonnet caller to produce a list of objects by running `helm template`
and return them, possibly modifying them as needed.

.travis.yml

@@ -7,7 +7,7 @@ before_install:
   - go get github.com/mattn/goveralls
 
 install:
-  make install get
+  make install-ci install get
 
 script:
   - make

Makefile

@@ -30,6 +30,12 @@ lint:
 	go list ./... | grep -v vendor | xargs go vet
 	go list ./... | grep -v vendor | xargs golint
 
+.PHONY: install-ci
+install-ci:
+	curl -sSL -o helm.tar.gz https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz
+	tar -xvzf helm.tar.gz
+	mv linux-amd64/helm $(GOPATH)/bin/
+
 .PHONY: install
 install:
 	go get github.com/golang/dep/cmd/dep

internal/vm/helm.go

@@ -0,0 +1,100 @@
+package vm
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+)
+
+// helmOptions are options that can be passed to the helm template command as well
+// as a `thisFile` option that the caller needs to set from `std.thisFile` to make
+// relative references to charts work correctly.
+type helmOptions struct {
+	Execute      []string `json:"execute"`      // --execute option
+	KubeVersion  string   `json:"kubeVersion"`  // --kube-version option
+	Name         string   `json:"name"`         // --name option
+	NameTemplate string   `json:"nameTemplate"` // --name-template option
+	Namespace    string   `json:"namespace"`    // --namespace option
+	ThisFile     string   `json:"thisFile"`     // use supplied file as current file to resolve relative refs, should be set to std.thisFile
+	Verbose      bool     `json:"verbose"`      // print helm template command before executing it
+	//IsUpgrade    bool     `json:"isUpgrade"` // --is-upgrade option, defer adding this until implications are known,
+}
+
+// toArgs converts options to a slice of command-line args.
+func (h helmOptions) toArgs() []string {
+	var ret []string
+	if len(h.Execute) > 0 {
+		for _, e := range h.Execute {
+			ret = append(ret, "--execute", e)
+		}
+	}
+	if h.KubeVersion != "" {
+		ret = append(ret, "--kube-version", h.KubeVersion)
+	}
+	if h.Name != "" {
+		ret = append(ret, "--name", h.Name)
+	}
+	if h.NameTemplate != "" {
+		ret = append(ret, "--name-template", h.NameTemplate)
+	}
+	if h.Namespace != "" {
+		ret = append(ret, "--namespace", h.Namespace)
+	}
+	//if h.IsUpgrade {
+	//	ret = append(ret, "--is-upgrade")
+	//}
+	return ret
+}
+
+// expandHelmTemplate produces an array of objects parsed from the output of running `helm template` with
+// the supplied values and helm options.
+func expandHelmTemplate(chart string, values map[string]interface{}, options helmOptions) (out []interface{}, finalErr error) {
+	// run command from the directory containing current file or the OS temp dir if `thisFile` not specified. That is,
+	// explicitly fail to resolve relative refs unless the calling file is specified; don't let them work by happenstance.
+	workDir := os.TempDir()
+	if options.ThisFile != "" {
+		dir := filepath.Dir(options.ThisFile)
+		if !filepath.IsAbs(dir) {
+			wd, err := os.Getwd()
+			if err != nil {
+				return nil, errors.Wrap(err, "get working directory")
+			}
+			dir = filepath.Join(wd, dir)
+		}
+		workDir = dir
+	}
+
+	valueBytes, err := yaml.Marshal(values)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshal values to YAML")
+	}
+
+	args := append([]string{"template", chart}, options.toArgs()...)
+	args = append(args, "--values", "-")
+
+	var stdout bytes.Buffer
+	cmd := exec.Command("helm", args...)
+	cmd.Stdin = bytes.NewBuffer(valueBytes)
+	cmd.Stdout = &stdout
+	cmd.Stderr = os.Stderr
+	cmd.Dir = workDir
+
+	if options.Verbose {
+		fmt.Fprintf(os.Stderr, "[helm template] cd %s && helm %s\n", workDir, strings.Join(args, " "))
+	}
+
+	if err := cmd.Run(); err != nil {
+		if options.ThisFile == "" {
+			fmt.Fprintln(os.Stderr, "[WARN] helm template command failed, you may need to set the 'thisFile' option to make relative chart paths work")
+		}
+		return nil, errors.Wrap(err, "run helm template command")
+	}
+
+	return parseYAMLDocuments(bytes.NewReader(stdout.Bytes()))
+}

internal/vm/helm_test.go

@@ -0,0 +1,111 @@
+package vm
+
+import (
+	"encoding/json"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHelmOptions(t *testing.T) {
+	a := assert.New(t)
+	var h helmOptions
+	a.Nil(h.toArgs())
+	h = helmOptions{
+		Execute:     []string{"a.yaml", "b.yaml"},
+		KubeVersion: "1.10",
+		Name:        "foo",
+		Namespace:   "foobar",
+		ThisFile:    "/path/to/my.jsonnet",
+		Verbose:     true,
+	}
+	a.EqualValues([]string{
+		"--execute", "a.yaml",
+		"--execute", "b.yaml",
+		"--kube-version", "1.10",
+		"--name", "foo",
+		"--namespace", "foobar",
+	}, h.toArgs())
+}
+
+type cmOrSecret struct {
+	APIVersion string `json:"apiVersion"`
+	Kind       string `json:"kind"`
+	Metadata   struct {
+		Namespace string `json:"namespace"`
+		Name      string `json:"name"`
+	}
+	Data map[string]string `json:"data"`
+}
+
+func TestHelmSimpleExpand(t *testing.T) {
+	a := assert.New(t)
+	jvm := New(Config{})
+	file := "./consumer.jsonnet"
+	inputCode := `
+local expandHelmTemplate = std.native('expandHelmTemplate');
+
+expandHelmTemplate(
+    './testdata/charts/foobar',
+    {
+        foo: 'barbar',
+    },
+    {
+        namespace: 'my-ns',
+        name: 'my-name',
+        thisFile: std.thisFile,
+		verbose: true,
+    }
+)
+`
+	code, err := jvm.EvaluateSnippet(file, inputCode)
+	require.Nil(t, err)
+
+	var output []cmOrSecret
+	err = json.Unmarshal([]byte(code), &output)
+
+	require.Equal(t, 2, len(output))
+
+	sort.Slice(output, func(i, j int) bool {
+		return output[i].Kind < output[j].Kind
+	})
+
+	ob := output[0]
+	a.Equal("ConfigMap", ob.Kind)
+	a.Equal("my-ns", ob.Metadata.Namespace)
+	a.Equal("my-name", ob.Metadata.Name)
+	a.Equal("barbar", ob.Data["foo"])
+	a.Equal("baz", ob.Data["bar"])
+
+	ob = output[1]
+	a.Equal("Secret", ob.Kind)
+	a.Equal("my-ns", ob.Metadata.Namespace)
+	a.Equal("my-name", ob.Metadata.Name)
+	a.Equal("Y2hhbmdlbWUK", ob.Data["secret"])
+}
+
+func TestHelmBadRelative(t *testing.T) {
+	a := assert.New(t)
+	jvm := New(Config{})
+	file := "./consumer.jsonnet"
+	inputCode := `
+local expandHelmTemplate = std.native('expandHelmTemplate');
+
+expandHelmTemplate(
+    './testdata/charts/foobar',
+    {
+        foo: 'barbar',
+    },
+    {
+        namespace: 'my-ns',
+        name: 'my-name',
+		verbose: true,
+    }
+)
+`
+	_, err := jvm.EvaluateSnippet(file, inputCode)
+	require.NotNil(t, err)
+	a.Contains(err.Error(), "exit status 1")
+}

internal/vm/nativefuncs.go

@@ -21,12 +21,11 @@ package vm
 import (
 	"bytes"
 	"encoding/json"
-	"io"
 	"regexp"
 
 	"github.com/google/go-jsonnet"
 	"github.com/google/go-jsonnet/ast"
-	"k8s.io/apimachinery/pkg/util/yaml"
+	"github.com/pkg/errors"
 )
 
 // registerNativeFuncs adds kubecfg's native jsonnet functions to provided VM
@@ -51,20 +50,8 @@ func registerNativeFuncs(vm *jsonnet.VM) {
 		Name:   "parseYaml",
 		Params: []ast.Identifier{"yaml"},
 		Func: func(args []interface{}) (res interface{}, err error) {
-			ret := []interface{}{}
 			data := []byte(args[0].(string))
-			d := yaml.NewYAMLToJSONDecoder(bytes.NewReader(data))
-			for {
-				var doc interface{}
-				if err := d.Decode(&doc); err != nil {
-					if err == io.EOF {
-						break
-					}
-					return nil, err
-				}
-				ret = append(ret, doc)
-			}
-			return ret, nil
+			return parseYAMLDocuments(bytes.NewReader(data))
 		},
 	})
 
@@ -99,4 +86,24 @@ func registerNativeFuncs(vm *jsonnet.VM) {
 			return r.ReplaceAllString(src, repl), nil
 		},
 	})
+
+	vm.NativeFunction(&jsonnet.NativeFunction{
+		Name:   "expandHelmTemplate",
+		Params: []ast.Identifier{"chart", "values", "options"},
+		Func: func(args []interface{}) (res interface{}, err error) {
+			chart := args[0].(string)
+			values := args[1].(map[string]interface{})
+			options := args[2].(map[string]interface{})
+			var h helmOptions
+			b, err := json.Marshal(options)
+			if err != nil {
+				return nil, errors.Wrap(err, "marshal options to JSON")
+			}
+			if err := json.Unmarshal(b, &h); err != nil {
+				return nil, errors.Wrap(err, "unmarshal options from JSON")
+			}
+			return expandHelmTemplate(chart, values, h)
+		},
+	})
+
 }

internal/vm/testdata/charts/foobar/Chart.yaml

@@ -0,0 +1,3 @@
+apiVersion: v1
+name: foobar
+version: 1.0

internal/vm/testdata/charts/foobar/templates/config-map.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ .Release.Name }}
+data:
+  foo: {{.Values.foo}}
+  bar: {{default "baz" .Values.bar}}

internal/vm/testdata/charts/foobar/templates/secret.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ .Release.Name }}
+data:
+  secret: {{.Values.secret}}

internal/vm/testdata/charts/foobar/values.yaml

@@ -0,0 +1,3 @@
+foo: bar
+secret: Y2hhbmdlbWUK
+

internal/vm/yaml.go

@@ -0,0 +1,25 @@
+package vm
+
+import (
+	"io"
+
+	"k8s.io/apimachinery/pkg/util/yaml"
+)
+
+func parseYAMLDocuments(reader io.Reader) ([]interface{}, error) {
+	ret := []interface{}{}
+	d := yaml.NewYAMLToJSONDecoder(reader)
+	for {
+		var doc interface{}
+		if err := d.Decode(&doc); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		if doc != nil {
+			ret = append(ret, doc)
+		}
+	}
+	return ret, nil
+}