databricks bundle init template v2: optional stubs, DLT support (#700)

## Changes

This follows up on #686. This PR
makes our stubs optional + it adds DLT stubs:

```
$ databricks bundle init
Template to use [default-python]: default-python
Unique name for this project [my_project]: my_project
Include a stub (sample) notebook in 'my_project/src' [yes]: yes
Include a stub (sample) DLT pipeline in 'my_project/src' [yes]: yes
Include a stub (sample) Python package 'my_project/src' [yes]: yes
✨ Successfully initialized template
```

## Tests
Manual testing, matrix tests.

---------

Co-authored-by: Andrew Nester <[email protected]>
Co-authored-by: PaulCornellDB <[email protected]>
Co-authored-by: Pieter Noordhuis <[email protected]>

bundle/bundle.go

@@ -37,6 +37,10 @@ type Bundle struct {
 	// Stores an initialized copy of this bundle's Terraform wrapper.
 	Terraform *tfexec.Terraform
 
+	// Indicates that the Terraform definition based on this bundle is empty,
+	// i.e. that it would deploy no resources.
+	TerraformHasNoResources bool
+
 	// Stores the locker responsible for acquiring/releasing a deployment lock.
 	Locker *locker.Locker
 

bundle/config/mutator/populate_current_user.go

@@ -21,6 +21,10 @@ func (m *populateCurrentUser) Name() string {
 }
 
 func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) error {
+	if b.Config.Workspace.CurrentUser != nil {
+		return nil
+	}
+
 	w := b.WorkspaceClient()
 	me, err := w.CurrentUser.Me(ctx)
 	if err != nil {

bundle/deploy/terraform/apply.go

@@ -16,6 +16,10 @@ func (w *apply) Name() string {
 }
 
 func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) error {
+	if b.TerraformHasNoResources {
+		cmdio.LogString(ctx, "Note: there are no resources to deploy for this bundle")
+		return nil
+	}
 	tf := b.Terraform
 	if tf == nil {
 		return fmt.Errorf("terraform not initialized")

bundle/deploy/terraform/convert.go

@@ -49,12 +49,14 @@ func convPermission(ac resources.Permission) schema.ResourcePermissionsAccessCon
 //
 // NOTE: THIS IS CURRENTLY A HACK. WE NEED A BETTER WAY TO
 // CONVERT TO/FROM TERRAFORM COMPATIBLE FORMAT.
-func BundleToTerraform(config *config.Root) *schema.Root {
+func BundleToTerraform(config *config.Root) (*schema.Root, bool) {
 	tfroot := schema.NewRoot()
 	tfroot.Provider = schema.NewProviders()
 	tfroot.Resource = schema.NewResources()
+	noResources := true
 
 	for k, src := range config.Resources.Jobs {
+		noResources = false
 		var dst schema.ResourceJob
 		conv(src, &dst)
 
@@ -100,6 +102,7 @@ func BundleToTerraform(config *config.Root) *schema.Root {
 	}
 
 	for k, src := range config.Resources.Pipelines {
+		noResources = false
 		var dst schema.ResourcePipeline
 		conv(src, &dst)
 
@@ -127,6 +130,7 @@ func BundleToTerraform(config *config.Root) *schema.Root {
 	}
 
 	for k, src := range config.Resources.Models {
+		noResources = false
 		var dst schema.ResourceMlflowModel
 		conv(src, &dst)
 		tfroot.Resource.MlflowModel[k] = &dst
@@ -139,6 +143,7 @@ func BundleToTerraform(config *config.Root) *schema.Root {
 	}
 
 	for k, src := range config.Resources.Experiments {
+		noResources = false
 		var dst schema.ResourceMlflowExperiment
 		conv(src, &dst)
 		tfroot.Resource.MlflowExperiment[k] = &dst
@@ -150,7 +155,7 @@ func BundleToTerraform(config *config.Root) *schema.Root {
 		}
 	}
 
-	return tfroot
+	return tfroot, noResources
 }
 
 func TerraformToBundle(state *tfjson.State, config *config.Root) error {

bundle/deploy/terraform/convert_test.go

@@ -40,7 +40,7 @@ func TestConvertJob(t *testing.T) {
 		},
 	}
 
-	out := BundleToTerraform(&config)
+	out, _ := BundleToTerraform(&config)
 	assert.Equal(t, "my job", out.Resource.Job["my_job"].Name)
 	assert.Len(t, out.Resource.Job["my_job"].JobCluster, 1)
 	assert.Equal(t, "https://github.com/foo/bar", out.Resource.Job["my_job"].GitSource.Url)
@@ -65,7 +65,7 @@ func TestConvertJobPermissions(t *testing.T) {
 		},
 	}
 
-	out := BundleToTerraform(&config)
+	out, _ := BundleToTerraform(&config)
 	assert.NotEmpty(t, out.Resource.Permissions["job_my_job"].JobId)
 	assert.Len(t, out.Resource.Permissions["job_my_job"].AccessControl, 1)
 
@@ -101,7 +101,7 @@ func TestConvertJobTaskLibraries(t *testing.T) {
 		},
 	}
 
-	out := BundleToTerraform(&config)
+	out, _ := BundleToTerraform(&config)
 	assert.Equal(t, "my job", out.Resource.Job["my_job"].Name)
 	require.Len(t, out.Resource.Job["my_job"].Task, 1)
 	require.Len(t, out.Resource.Job["my_job"].Task[0].Library, 1)
@@ -135,7 +135,7 @@ func TestConvertPipeline(t *testing.T) {
 		},
 	}
 
-	out := BundleToTerraform(&config)
+	out, _ := BundleToTerraform(&config)
 	assert.Equal(t, "my pipeline", out.Resource.Pipeline["my_pipeline"].Name)
 	assert.Len(t, out.Resource.Pipeline["my_pipeline"].Library, 2)
 	assert.Nil(t, out.Data)
@@ -159,7 +159,7 @@ func TestConvertPipelinePermissions(t *testing.T) {
 		},
 	}
 
-	out := BundleToTerraform(&config)
+	out, _ := BundleToTerraform(&config)
 	assert.NotEmpty(t, out.Resource.Permissions["pipeline_my_pipeline"].PipelineId)
 	assert.Len(t, out.Resource.Permissions["pipeline_my_pipeline"].AccessControl, 1)
 
@@ -194,7 +194,7 @@ func TestConvertModel(t *testing.T) {
 		},
 	}
 
-	out := BundleToTerraform(&config)
+	out, _ := BundleToTerraform(&config)
 	assert.Equal(t, "name", out.Resource.MlflowModel["my_model"].Name)
 	assert.Equal(t, "description", out.Resource.MlflowModel["my_model"].Description)
 	assert.Len(t, out.Resource.MlflowModel["my_model"].Tags, 2)
@@ -223,7 +223,7 @@ func TestConvertModelPermissions(t *testing.T) {
 		},
 	}
 
-	out := BundleToTerraform(&config)
+	out, _ := BundleToTerraform(&config)
 	assert.NotEmpty(t, out.Resource.Permissions["mlflow_model_my_model"].RegisteredModelId)
 	assert.Len(t, out.Resource.Permissions["mlflow_model_my_model"].AccessControl, 1)
 
@@ -247,7 +247,7 @@ func TestConvertExperiment(t *testing.T) {
 		},
 	}
 
-	out := BundleToTerraform(&config)
+	out, _ := BundleToTerraform(&config)
 	assert.Equal(t, "name", out.Resource.MlflowExperiment["my_experiment"].Name)
 	assert.Nil(t, out.Data)
 }
@@ -270,7 +270,7 @@ func TestConvertExperimentPermissions(t *testing.T) {
 		},
 	}
 
-	out := BundleToTerraform(&config)
+	out, _ := BundleToTerraform(&config)
 	assert.NotEmpty(t, out.Resource.Permissions["mlflow_experiment_my_experiment"].ExperimentId)
 	assert.Len(t, out.Resource.Permissions["mlflow_experiment_my_experiment"].AccessControl, 1)
 

bundle/deploy/terraform/write.go

@@ -21,7 +21,8 @@ func (w *write) Apply(ctx context.Context, b *bundle.Bundle) error {
 		return err
 	}
 
-	root := BundleToTerraform(&b.Config)
+	root, noResources := BundleToTerraform(&b.Config)
+	b.TerraformHasNoResources = noResources
 	f, err := os.Create(filepath.Join(dir, "bundle.tf.json"))
 	if err != nil {
 		return err

libs/template/helpers.go

@@ -26,9 +26,10 @@ type pair struct {
 	v any
 }
 
+var cachedUser *iam.User
+var cachedIsServicePrincipal *bool
+
 func loadHelpers(ctx context.Context) template.FuncMap {
-	var user *iam.User
-	var is_service_principal *bool
 	w := root.WorkspaceClient(ctx)
 	return template.FuncMap{
 		"fail": func(format string, args ...any) (any, error) {
@@ -80,32 +81,32 @@ func loadHelpers(ctx context.Context) template.FuncMap {
 			return w.Config.Host, nil
 		},
 		"user_name": func() (string, error) {
-			if user == nil {
+			if cachedUser == nil {
 				var err error
-				user, err = w.CurrentUser.Me(ctx)
+				cachedUser, err = w.CurrentUser.Me(ctx)
 				if err != nil {
 					return "", err
 				}
 			}
-			result := user.UserName
+			result := cachedUser.UserName
 			if result == "" {
-				result = user.Id
+				result = cachedUser.Id
 			}
 			return result, nil
 		},
 		"is_service_principal": func() (bool, error) {
-			if is_service_principal != nil {
-				return *is_service_principal, nil
+			if cachedIsServicePrincipal != nil {
+				return *cachedIsServicePrincipal, nil
 			}
-			if user == nil {
+			if cachedUser == nil {
 				var err error
-				user, err = w.CurrentUser.Me(ctx)
+				cachedUser, err = w.CurrentUser.Me(ctx)
 				if err != nil {
 					return false, err
 				}
 			}
-			result := auth.IsServicePrincipal(user.Id)
-			is_service_principal = &result
+			result := auth.IsServicePrincipal(cachedUser.Id)
+			cachedIsServicePrincipal = &result
 			return result, nil
 		},
 	}

libs/template/renderer.go

@@ -9,6 +9,7 @@ import (
 	"path"
 	"path/filepath"
 	"slices"
+	"sort"
 	"strings"
 	"text/template"
 
@@ -214,17 +215,22 @@ func (r *renderer) walk() error {
 		// Add skip function, which accumulates skip patterns relative to current
 		// directory
 		r.baseTemplate.Funcs(template.FuncMap{
-			"skip": func(relPattern string) string {
+			"skip": func(relPattern string) (string, error) {
 				// patterns are specified relative to current directory of the file
 				// the {{skip}} function is called from.
-				pattern := path.Join(currentDirectory, relPattern)
+				patternRaw := path.Join(currentDirectory, relPattern)
+				pattern, err := r.executeTemplate(patternRaw)
+				if err != nil {
+					return "", err
+				}
+
 				if !slices.Contains(r.skipPatterns, pattern) {
 					logger.Infof(r.ctx, "adding skip pattern: %s", pattern)
 					r.skipPatterns = append(r.skipPatterns, pattern)
 				}
 				// return empty string will print nothing at function call site
 				// when executing the template
-				return ""
+				return "", nil
 			},
 		})
 
@@ -239,6 +245,10 @@ func (r *renderer) walk() error {
 		if err != nil {
 			return err
 		}
+		// Sort by name to ensure deterministic ordering
+		sort.Slice(entries, func(i, j int) bool {
+			return entries[i].Name() < entries[j].Name()
+		})
 		for _, entry := range entries {
 			if entry.IsDir() {
 				// Add to slice, for BFS traversal

libs/template/renderer_test.go

@@ -12,7 +12,14 @@ import (
 	"testing"
 	"text/template"
 
+	"github.com/databricks/cli/bundle"
+	bundleConfig "github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/phases"
 	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/databricks-sdk-go"
+	workspaceConfig "github.com/databricks/databricks-sdk-go/config"
+	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -29,6 +36,95 @@ func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
 	assert.Equal(t, perm, info.Mode().Perm())
 }
 
+func assertBuiltinTemplateValid(t *testing.T, settings map[string]any, target string, isServicePrincipal bool, build bool, tempDir string) {
+	ctx := context.Background()
+
+	templatePath, err := prepareBuiltinTemplates("default-python", tempDir)
+	require.NoError(t, err)
+
+	w := &databricks.WorkspaceClient{
+		Config: &workspaceConfig.Config{Host: "https://myhost.com"},
+	}
+
+	// Prepare helpers
+	cachedUser = &iam.User{UserName: "[email protected]"}
+	cachedIsServicePrincipal = &isServicePrincipal
+	ctx = root.SetWorkspaceClient(ctx, w)
+	helpers := loadHelpers(ctx)
+
+	renderer, err := newRenderer(ctx, settings, helpers, templatePath, "./testdata/template-in-path/library", tempDir)
+	require.NoError(t, err)
+
+	// Evaluate template
+	err = renderer.walk()
+	require.NoError(t, err)
+	err = renderer.persistToDisk()
+	require.NoError(t, err)
+	b, err := bundle.Load(ctx, filepath.Join(tempDir, "template", "my_project"))
+	require.NoError(t, err)
+
+	// Apply initialize / validation mutators
+	b.Config.Workspace.CurrentUser = &bundleConfig.User{User: cachedUser}
+	b.WorkspaceClient()
+	b.Config.Bundle.Terraform = &bundleConfig.Terraform{
+		ExecPath: "sh",
+	}
+	err = bundle.Apply(ctx, b, bundle.Seq(
+		bundle.Seq(mutator.DefaultMutators()...),
+		mutator.SelectTarget(target),
+		phases.Initialize(),
+	))
+	require.NoError(t, err)
+
+	// Apply build mutator
+	if build {
+		err = bundle.Apply(ctx, b, phases.Build())
+		require.NoError(t, err)
+	}
+}
+
+func TestBuiltinTemplateValid(t *testing.T) {
+	// Test option combinations
+	options := []string{"yes", "no"}
+	isServicePrincipal := false
+	build := false
+	for _, includeNotebook := range options {
+		for _, includeDlt := range options {
+			for _, includePython := range options {
+				for _, isServicePrincipal := range []bool{true, false} {
+					config := map[string]any{
+						"project_name":     "my_project",
+						"include_notebook": includeNotebook,
+						"include_dlt":      includeDlt,
+						"include_python":   includePython,
+					}
+					tempDir := t.TempDir()
+					assertBuiltinTemplateValid(t, config, "dev", isServicePrincipal, build, tempDir)
+				}
+			}
+		}
+	}
+
+	// Test prod mode + build
+	config := map[string]any{
+		"project_name":     "my_project",
+		"include_notebook": "yes",
+		"include_dlt":      "yes",
+		"include_python":   "yes",
+	}
+	isServicePrincipal = false
+	build = true
+
+	// On Windows, we can't always remove the resulting temp dir since background
+	// processes might have it open, so we use 'defer' for a best-effort cleanup
+	tempDir, err := os.MkdirTemp("", "templates")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	assertBuiltinTemplateValid(t, config, "prod", isServicePrincipal, build, tempDir)
+	defer os.RemoveAll(tempDir)
+}
+
 func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) {
 	tmpDir := t.TempDir()