Merge remote-tracking branch 'databricks/main' into template

.github/workflows/push.yml

@@ -26,10 +26,9 @@ jobs:
         run: git fetch --prune --unshallow
 
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 1.21.0
-          cache: true
 
       - name: Set go env
         run: |
@@ -54,9 +53,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.21
+          go-version: 1.21.0
 
           # No need to download cached dependencies when running gofmt.
           cache: false

.github/workflows/release-snapshot.yml

@@ -19,27 +19,10 @@ jobs:
         run: git fetch --prune --unshallow
 
       - name: Setup Go
-        id: go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 1.21.0
 
-      - name: Locate cache paths
-        id: cache
-        run: |
-          echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
-          echo "GOCACHE=$(go env GOCACHE)" >> $GITHUB_OUTPUT
-
-      # Note: use custom caching because below performs a cross platform build
-      # through goreleaser and don't want to share a cache with the test builds.
-      - name: Setup caching
-        uses: actions/cache@v3
-        with:
-          path: |
-            ${{ steps.cache.outputs.GOMODCACHE }}
-            ${{ steps.cache.outputs.GOCACHE }}
-          key: release-${{ runner.os }}-go-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', '.goreleaser.yaml') }}
-
       - name: Hide snapshot tag to outsmart GoReleaser
         run: git tag -d snapshot || true
 

.github/workflows/release.yml

@@ -18,27 +18,10 @@ jobs:
         run: git fetch --prune --unshallow
 
       - name: Setup Go
-        id: go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 1.21.0
 
-      - name: Locate cache paths
-        id: cache
-        run: |
-          echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
-          echo "GOCACHE=$(go env GOCACHE)" >> $GITHUB_OUTPUT
-
-      # Note: use custom caching because below performs a cross platform build
-      # through goreleaser and don't want to share a cache with the test builds.
-      - name: Setup caching
-        uses: actions/cache@v3
-        with:
-          path: |
-            ${{ steps.cache.outputs.GOMODCACHE }}
-            ${{ steps.cache.outputs.GOCACHE }}
-          key: release-${{ runner.os }}-go-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', '.goreleaser.yaml') }}
-
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Version changelog
 
+## 0.203.3
+
+Bundles:
+ * Support cluster overrides with cluster_key and compute_key ([#696](https://github.com/databricks/cli/pull/696)).
+ * Allow referencing local Python wheels without artifacts section defined ([#703](https://github.com/databricks/cli/pull/703)).
+ * Fixed --environment flag ([#705](https://github.com/databricks/cli/pull/705)).
+ * Correctly identify local paths in libraries section ([#702](https://github.com/databricks/cli/pull/702)).
+ * Fixed path joining in FindFilesWithSuffixInPath ([#704](https://github.com/databricks/cli/pull/704)).
+ *  Added transformation mutator for Python wheel task for them to work on DBR <13.1 ([#635](https://github.com/databricks/cli/pull/635)).
+
+Internal:
+ * Add a foundation for built-in templates ([#685](https://github.com/databricks/cli/pull/685)).
+ * Test transform when no Python wheel tasks defined ([#714](https://github.com/databricks/cli/pull/714)).
+ * Pin Terraform binary version to 1.5.5 ([#715](https://github.com/databricks/cli/pull/715)).
+ * Cleanup after "Add a foundation for built-in templates" ([#707](https://github.com/databricks/cli/pull/707)).
+ * Filter down to Python wheel tasks only for trampoline ([#712](https://github.com/databricks/cli/pull/712)).
+ * Update Terraform provider schema structs from 1.23.0 ([#713](https://github.com/databricks/cli/pull/713)).
+
 ## 0.203.2
 
 CLI:

bundle/artifacts/artifacts.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/artifacts/whl"
@@ -107,7 +108,7 @@ func uploadArtifact(ctx context.Context, a *config.Artifact, b *bundle.Bundle) e
 	for i := range a.Files {
 		f := &a.Files[i]
 		if f.NeedsUpload() {
-			filename := path.Base(f.Source)
+			filename := filepath.Base(f.Source)
 			cmdio.LogString(ctx, fmt.Sprintf("artifacts.Upload(%s): Uploading...", filename))
 			remotePath, err := uploadArtifactFile(ctx, f.Source, b)
 			if err != nil {
@@ -136,7 +137,7 @@ func uploadArtifactFile(ctx context.Context, file string, b *bundle.Bundle) (str
 	}
 
 	fileHash := sha256.Sum256(raw)
-	remotePath := path.Join(uploadPath, fmt.Sprintf("%x", fileHash), path.Base(file))
+	remotePath := path.Join(uploadPath, fmt.Sprintf("%x", fileHash), filepath.Base(file))
 	// Make sure target directory exists.
 	err = b.WorkspaceClient().Workspace.MkdirsByPath(ctx, path.Dir(remotePath))
 	if err != nil {

bundle/artifacts/artifacts_test.go

@@ -0,0 +1,89 @@
+package artifacts
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"regexp"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+	"github.com/stretchr/testify/require"
+)
+
+func touchEmptyFile(t *testing.T, path string) {
+	err := os.MkdirAll(filepath.Dir(path), 0700)
+	require.NoError(t, err)
+	f, err := os.Create(path)
+	require.NoError(t, err)
+	f.Close()
+}
+
+type MockWorkspaceService struct {
+}
+
+// Delete implements workspace.WorkspaceService.
+func (MockWorkspaceService) Delete(ctx context.Context, request workspace.Delete) error {
+	panic("unimplemented")
+}
+
+// Export implements workspace.WorkspaceService.
+func (MockWorkspaceService) Export(ctx context.Context, request workspace.ExportRequest) (*workspace.ExportResponse, error) {
+	panic("unimplemented")
+}
+
+// GetStatus implements workspace.WorkspaceService.
+func (MockWorkspaceService) GetStatus(ctx context.Context, request workspace.GetStatusRequest) (*workspace.ObjectInfo, error) {
+	panic("unimplemented")
+}
+
+// Import implements workspace.WorkspaceService.
+func (MockWorkspaceService) Import(ctx context.Context, request workspace.Import) error {
+	return nil
+}
+
+// List implements workspace.WorkspaceService.
+func (MockWorkspaceService) List(ctx context.Context, request workspace.ListWorkspaceRequest) (*workspace.ListResponse, error) {
+	panic("unimplemented")
+}
+
+// Mkdirs implements workspace.WorkspaceService.
+func (MockWorkspaceService) Mkdirs(ctx context.Context, request workspace.Mkdirs) error {
+	return nil
+}
+
+func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) {
+	dir := t.TempDir()
+	whlPath := filepath.Join(dir, "dist", "test.whl")
+	touchEmptyFile(t, whlPath)
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Path: dir,
+			Bundle: config.Bundle{
+				Target: "whatever",
+			},
+			Workspace: config.Workspace{
+				ArtifactsPath: "/Users/[email protected]/whatever",
+			},
+		},
+	}
+
+	b.WorkspaceClient().Workspace.WithImpl(MockWorkspaceService{})
+	artifact := &config.Artifact{
+		Files: []config.ArtifactFile{
+			{
+				Source: whlPath,
+				Libraries: []*compute.Library{
+					{Whl: "dist\\test.whl"},
+				},
+			},
+		},
+	}
+
+	err := uploadArtifact(context.Background(), artifact, b)
+	require.NoError(t, err)
+	require.Regexp(t, regexp.MustCompile("/Users/[email protected]/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].RemotePath)
+}

bundle/config/mutator/process_target_mode.go

@@ -160,7 +160,10 @@ func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) error {
 		}
 		return transformDevelopmentMode(b)
 	case config.Production:
-		isPrincipal := auth.IsServicePrincipal(ctx, b.WorkspaceClient(), b.Config.Workspace.CurrentUser.Id)
+		isPrincipal, err := auth.IsServicePrincipal(ctx, b.WorkspaceClient(), b.Config.Workspace.CurrentUser.Id)
+		if err != nil {
+			return err
+		}
 		return validateProductionMode(ctx, b, isPrincipal)
 	case "":
 		// No action

bundle/deploy/terraform/init.go

@@ -55,10 +55,10 @@ func (m *initialize) findExecPath(ctx context.Context, b *bundle.Bundle, tf *con
 	}
 
 	// Download Terraform to private bin directory.
-	installer := &releases.LatestVersion{
-		Product:     product.Terraform,
-		Constraints: version.MustConstraints(version.NewConstraint("<=1.5.5")),
-		InstallDir:  binDir,
+	installer := &releases.ExactVersion{
+		Product:    product.Terraform,
+		Version:    version.Must(version.NewVersion("1.5.5")),
+		InstallDir: binDir,
 	}
 	execPath, err = installer.Install(ctx)
 	if err != nil {

bundle/internal/tf/codegen/generator/generator.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"text/template"
 
+	schemapkg "github.com/databricks/cli/bundle/internal/tf/codegen/schema"
 	tfjson "github.com/hashicorp/terraform-json"
 )
 
@@ -32,6 +33,23 @@ func (c *collection) Generate(path string) error {
 	return tmpl.Execute(f, c)
 }
 
+type root struct {
+	OutputFile      string
+	ProviderVersion string
+}
+
+func (r *root) Generate(path string) error {
+	tmpl := template.Must(template.ParseFiles(fmt.Sprintf("./templates/%s.tmpl", r.OutputFile)))
+	f, err := os.Create(filepath.Join(path, r.OutputFile))
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	return tmpl.Execute(f, r)
+}
+
 func Run(ctx context.Context, schema *tfjson.ProviderSchema, path string) error {
 	// Generate types for resources.
 	var resources []*namedBlock
@@ -105,5 +123,17 @@ func Run(ctx context.Context, schema *tfjson.ProviderSchema, path string) error
 		}
 	}
 
+	// Generate root.go
+	{
+		r := &root{
+			OutputFile:      "root.go",
+			ProviderVersion: schemapkg.ProviderVersion,
+		}
+		err := r.Generate(path)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

bundle/internal/tf/codegen/schema/generate.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/hc-install/product"
 	"github.com/hashicorp/hc-install/releases"
 	"github.com/hashicorp/terraform-exec/tfexec"
@@ -19,7 +20,7 @@ func (s *Schema) writeTerraformBlock(_ context.Context) error {
 			"required_providers": map[string]interface{}{
 				"databricks": map[string]interface{}{
 					"source":  "databricks/databricks",
-					"version": ">= 1.0.0",
+					"version": ProviderVersion,
 				},
 			},
 		},
@@ -40,9 +41,10 @@ func (s *Schema) installTerraform(ctx context.Context) (path string, err error)
 		return
 	}
 
-	installer := &releases.LatestVersion{
-		InstallDir: installDir,
+	installer := &releases.ExactVersion{
 		Product:    product.Terraform,
+		Version:    version.Must(version.NewVersion("1.5.5")),
+		InstallDir: installDir,
 	}
 
 	installer.SetLogger(log.Default())

bundle/internal/tf/codegen/schema/version.go

@@ -0,0 +1,3 @@
+package schema
+
+const ProviderVersion = "1.23.0"

bundle/internal/tf/codegen/templates/root.go.tmpl

@@ -0,0 +1,32 @@
+package schema
+
+type Providers struct {
+	Databricks *Config `json:"databricks,omitempty"`
+}
+
+func NewProviders() *Providers {
+	return &Providers{
+		Databricks: &Config{},
+	}
+}
+
+type Root struct {
+	Terraform map[string]any `json:"terraform"`
+
+	Provider *Providers   `json:"provider,omitempty"`
+	Data     *DataSources `json:"data,omitempty"`
+	Resource *Resources   `json:"resource,omitempty"`
+}
+
+func NewRoot() *Root {
+	return &Root{
+		Terraform: map[string]interface{}{
+			"required_providers": map[string]interface{}{
+				"databricks": map[string]interface{}{
+					"source":  "databricks/databricks",
+					"version": "1.23.0",
+				},
+			},
+		},
+	}
+}

bundle/internal/tf/schema/data_source_cluster.go

@@ -90,6 +90,7 @@ type DataSourceClusterClusterInfoGcpAttributes struct {
 	Availability            string `json:"availability,omitempty"`
 	BootDiskSize            int    `json:"boot_disk_size,omitempty"`
 	GoogleServiceAccount    string `json:"google_service_account,omitempty"`
+	LocalSsdCount           int    `json:"local_ssd_count,omitempty"`
 	UsePreemptibleExecutors bool   `json:"use_preemptible_executors,omitempty"`
 	ZoneId                  string `json:"zone_id,omitempty"`
 }

bundle/internal/tf/schema/data_source_instance_pool.go

@@ -26,6 +26,7 @@ type DataSourceInstancePoolPoolInfoDiskSpec struct {
 
 type DataSourceInstancePoolPoolInfoGcpAttributes struct {
 	GcpAvailability string `json:"gcp_availability,omitempty"`
+	LocalSsdCount   int    `json:"local_ssd_count,omitempty"`
 }
 
 type DataSourceInstancePoolPoolInfoInstancePoolFleetAttributesFleetOnDemandOption struct {