add structured support for external and top-level variables in qbec, f…

…ixes #4 (#10)

Refactor the options structs and interfaces such that config is now
created in the commands package. This allows for a single implmentation
for real code and tests such that the newly introduced interactions
between VM config and app can be handled in one place and tested.

Add a `vars` section in qbec.yaml that allows definition of external
and top-level variables. External variables can have defaults that
are used when not specified on the command line. Top level variables
must be associated with components such that these are only set
for the components that need them.

Add `-V` and `-A` shorthands (similar to `jsonnet eval`) to set
string vars for external and top-level variables from the command line.
Code values are also supported but do not have short flags for
assigment.

Introduce a `--strict-vars` flag that restricts commands from
only setting up variables that are defined in qbec and requires
them to set _all_ such variables at the time of command invocation.

cmd/jsonnet-qbec/main.go

@@ -61,7 +61,7 @@ func main() {
 			}
 		},
 	}
-	configInit = vm.ConfigFromCommandParams(root, "")
+	configInit = vm.ConfigFromCommandParams(root, "", true)
 	if err := root.Execute(); err != nil {
 		log.Fatalln(err)
 	}

examples/test-app/components/service1.jsonnet

@@ -1,5 +1,6 @@
 local objects = import 'objects.libsonnet';
+local fooValue = std.extVar('externalFoo');
 
 {
-    configMap: objects.configmap('foo-system','svc1-cm', { foo : 'bar' }),
+    configMap: objects.configmap('foo-system','svc1-cm', { foo : fooValue }),
 }

examples/test-app/components/service2.jsonnet

@@ -1,7 +1,9 @@
 local objects = import 'objects.libsonnet';
 
-{
-    configMap: objects.configmap('bar-system','svc2-cm', { foo : 'bar' }),
-    secret: objects.secret('bar-system','svc2-secret', { foo : std.base64('bar') }),
-}
+function (tlaFoo = 'bar') (
+    {
+        configMap: objects.configmap('bar-system','svc2-cm', { foo : tlaFoo }),
+        secret: objects.secret('bar-system','svc2-secret', { foo : std.base64('bar') }),
+    }
+)
 

examples/test-app/qbec.yaml

@@ -10,6 +10,13 @@ spec:
   - lib
   excludes:
   - service2
+  vars:
+    topLevel:
+      - name: tlaFoo
+        components: [ 'service2' ]
+    external:
+      - name: externalFoo
+        default: 'bar'
   environments:
     dev:
       server: https://dev-server

internal/commands/apply.go

@@ -49,20 +49,11 @@ func (a *applyStats) update(name string, s *remote.SyncResult) {
 	}
 }
 
-// applyClient is the remote interface needed for apply operations.
-type applyClient interface {
-	listClient
-	DisplayName(o model.K8sMeta) string
-	Sync(obj model.K8sLocalObject, opts remote.SyncOptions) (*remote.SyncResult, error)
-	Delete(obj model.K8sMeta, dryRun bool) (*remote.SyncResult, error)
-}
-
 type applyCommandConfig struct {
-	StdOptions
-	syncOptions    remote.SyncOptions
-	gc             bool
-	filterFunc     func() (filterParams, error)
-	clientProvider func(env string) (applyClient, error)
+	*Config
+	syncOptions remote.SyncOptions
+	gc          bool
+	filterFunc  func() (filterParams, error)
 }
 
 func doApply(args []string, config applyCommandConfig) error {
@@ -77,20 +68,20 @@ func doApply(args []string, config applyCommandConfig) error {
 	if err != nil {
 		return err
 	}
-	objects, err := filteredObjects(config, env, fp)
+	objects, err := filteredObjects(config.Config, env, fp)
 	if err != nil {
 		return err
 	}
 
-	client, err := config.clientProvider(env)
+	client, err := config.Client(env)
 	if err != nil {
 		return err
 	}
 
 	// prepare for GC with object list of deletions
 	var lister lister = &stubLister{}
 	if config.gc {
-		all, err := allObjects(config, env)
+		all, err := allObjects(config.Config, env)
 		if err != nil {
 			return err
 		}
@@ -113,7 +104,7 @@ func doApply(args []string, config applyCommandConfig) error {
 	}
 
 	// continue with apply
-	objects = objsort.Sort(objects, config.SortConfig(client.IsNamespaced))
+	objects = objsort.Sort(objects, sortConfig(client.IsNamespaced))
 
 	opts := config.syncOptions
 	dryRun := ""
@@ -156,7 +147,7 @@ func doApply(args []string, config applyCommandConfig) error {
 		}
 	}
 
-	deletions = objsort.SortMeta(deletions, config.SortConfig(client.IsNamespaced))
+	deletions = objsort.SortMeta(deletions, sortConfig(client.IsNamespaced))
 	for i := len(deletions) - 1; i >= 0; i-- {
 		ob := deletions[i]
 		name := client.DisplayName(ob)
@@ -177,17 +168,14 @@ func doApply(args []string, config applyCommandConfig) error {
 
 }
 
-func newApplyCommand(op OptionsProvider) *cobra.Command {
+func newApplyCommand(cp ConfigProvider) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "apply [-n] <environment>",
 		Short:   "apply one or more components to a Kubernetes cluster",
 		Example: applyExamples(),
 	}
 
 	config := applyCommandConfig{
-		clientProvider: func(env string) (applyClient, error) {
-			return op().Client(env)
-		},
 		filterFunc: addFilterParams(cmd, true),
 	}
 
@@ -197,7 +185,7 @@ func newApplyCommand(op OptionsProvider) *cobra.Command {
 	cmd.Flags().BoolVar(&config.gc, "gc", true, "garbage collect extra objects on the server")
 
 	cmd.RunE = func(c *cobra.Command, args []string) error {
-		config.StdOptions = op()
+		config.Config = cp()
 		return wrapError(doApply(args, config))
 	}
 	return cmd

internal/commands/apply_test.go

@@ -31,7 +31,7 @@ func TestApplyBasic(t *testing.T) {
 	defer s.reset()
 	first := true
 	var captured remote.SyncOptions
-	s.opts.client.syncFunc = func(obj model.K8sLocalObject, opts remote.SyncOptions) (*remote.SyncResult, error) {
+	s.client.syncFunc = func(obj model.K8sLocalObject, opts remote.SyncOptions) (*remote.SyncResult, error) {
 		if first {
 			first = false
 			captured = opts
@@ -62,7 +62,7 @@ func TestApplyFlags(t *testing.T) {
 	defer s.reset()
 	first := true
 	var captured remote.SyncOptions
-	s.opts.client.syncFunc = func(obj model.K8sLocalObject, opts remote.SyncOptions) (*remote.SyncResult, error) {
+	s.client.syncFunc = func(obj model.K8sLocalObject, opts remote.SyncOptions) (*remote.SyncResult, error) {
 		if first {
 			first = false
 			captured = opts

internal/commands/common.go

@@ -27,10 +27,8 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/splunk/qbec/internal/model"
-	"github.com/splunk/qbec/internal/objsort"
 	"github.com/splunk/qbec/internal/remote"
 	"github.com/splunk/qbec/internal/sio"
-	"github.com/splunk/qbec/internal/vm"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
@@ -82,19 +80,6 @@ func wrapError(err error) error {
 	return newRuntimeError(err)
 }
 
-// StdOptions provides standardized access to information available to every command.
-type StdOptions interface {
-	App() *model.App                                       // the app loaded for the command
-	VM() *vm.VM                                            // the base VM constructed out of command line args and potentially app information
-	Colorize() bool                                        // returns if colorized output is needed
-	Verbosity() int                                        // returns the verbosity level
-	SortConfig(provider objsort.Namespaced) objsort.Config // returns the apply sort config potentially using hints from the app
-	Stdout() io.Writer                                     // output to write to
-	DefaultNamespace(env string) string                    // the default namespace for the supplied environment
-	Confirm(context string) error                          // confirmation function for dangerous operations
-	EvalConcurrency() int                                  // the concurrency using which to evaluate components
-}
-
 // Client encapsulates all remote operations needed for the superset of all commands.
 type Client interface {
 	DisplayName(o model.K8sMeta) string
@@ -106,24 +91,18 @@ type Client interface {
 	Delete(obj model.K8sMeta, dryRun bool) (*remote.SyncResult, error)
 }
 
-// StdOptionsWithClient provides a remote client in addition to standard options.
-type StdOptionsWithClient interface {
-	StdOptions                         // base options
-	Client(env string) (Client, error) // a client valid for the supplied environment
-}
-
-// OptionsProvider provides standard configuration available to all commands
-type OptionsProvider func() StdOptionsWithClient
+// ConfigProvider provides standard configuration available to all commands
+type ConfigProvider func() *Config
 
 // Setup sets up all subcommands for the supplied root command.
-func Setup(root *cobra.Command, op OptionsProvider) {
-	root.AddCommand(newApplyCommand(op))
-	root.AddCommand(newValidateCommand(op))
-	root.AddCommand(newShowCommand(op))
-	root.AddCommand(newDiffCommand(op))
-	root.AddCommand(newDeleteCommand(op))
-	root.AddCommand(newComponentCommand(op))
-	root.AddCommand(newParamCommand(op))
+func Setup(root *cobra.Command, cp ConfigProvider) {
+	root.AddCommand(newApplyCommand(cp))
+	root.AddCommand(newValidateCommand(cp))
+	root.AddCommand(newShowCommand(cp))
+	root.AddCommand(newDiffCommand(cp))
+	root.AddCommand(newDeleteCommand(cp))
+	root.AddCommand(newComponentCommand(cp))
+	root.AddCommand(newParamCommand(cp))
 	root.AddCommand(newInitCommand())
 }
 

internal/commands/component.go

@@ -28,12 +28,12 @@ import (
 	"github.com/splunk/qbec/internal/model"
 )
 
-func newComponentCommand(op OptionsProvider) *cobra.Command {
+func newComponentCommand(cp ConfigProvider) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "component <subcommand>",
 		Short: "component lists and diffs",
 	}
-	cmd.AddCommand(newComponentListCommand(op), newComponentDiffCommand(op))
+	cmd.AddCommand(newComponentListCommand(cp), newComponentDiffCommand(cp))
 	return cmd
 }
 
@@ -64,7 +64,7 @@ func listComponents(components []model.Component, formatSpecified bool, format s
 }
 
 type componentListCommandConfig struct {
-	StdOptions
+	*Config
 	format  string
 	objects bool
 }
@@ -75,7 +75,7 @@ func doComponentList(args []string, config componentListCommandConfig) error {
 	}
 	env := args[0]
 	if config.objects {
-		objects, err := filteredObjects(config, env, filterParams{})
+		objects, err := filteredObjects(config.Config, env, filterParams{})
 		if err != nil {
 			return err
 		}
@@ -88,7 +88,7 @@ func doComponentList(args []string, config componentListCommandConfig) error {
 	return listComponents(components, config.format != "", config.format, config.Stdout())
 }
 
-func newComponentListCommand(op OptionsProvider) *cobra.Command {
+func newComponentListCommand(cp ConfigProvider) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "list [-objects] <environment>",
 		Short:   "list all components for an environment, optionally listing all objects as well",
@@ -100,14 +100,14 @@ func newComponentListCommand(op OptionsProvider) *cobra.Command {
 	cmd.Flags().StringVarP(&config.format, "format", "o", "", "use json|yaml to display machine readable input")
 
 	cmd.RunE = func(c *cobra.Command, args []string) error {
-		config.StdOptions = op()
+		config.Config = cp()
 		return wrapError(doComponentList(args, config))
 	}
 	return cmd
 }
 
 type componentDiffCommandConfig struct {
-	StdOptions
+	*Config
 	objects bool
 }
 
@@ -142,7 +142,7 @@ func doComponentDiff(args []string, config componentDiffCommandConfig) error {
 	}
 
 	getObjects := func(env string) (str string, name string, err error) {
-		objs, err := filteredObjects(config, env, filterParams{})
+		objs, err := filteredObjects(config.Config, env, filterParams{})
 		if err != nil {
 			return
 		}
@@ -189,7 +189,7 @@ func doComponentDiff(args []string, config componentDiffCommandConfig) error {
 	return nil
 }
 
-func newComponentDiffCommand(op OptionsProvider) *cobra.Command {
+func newComponentDiffCommand(cp ConfigProvider) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "diff [-objects] <environment>|_ [<environment>|_]",
 		Short:   "diff component lists across two environments or between the baseline (use _ for baseline) and an environment",
@@ -200,7 +200,7 @@ func newComponentDiffCommand(op OptionsProvider) *cobra.Command {
 	cmd.Flags().BoolVarP(&config.objects, "objects", "O", false, "set to true to also list objects in each component")
 
 	cmd.RunE = func(c *cobra.Command, args []string) error {
-		config.StdOptions = op()
+		config.Config = cp()
 		return wrapError(doComponentDiff(args, config))
 	}
 	return cmd