Feature/azuredatabricks (#47)

* added initial files

* added example

* added Readme.md

* updated

* added diagnostic settings for ADB

* updated diagnostic setting

* fixed

* added diagnostic setting

* updated example

* updated docs

* updated diagnostic setting

* updated example

* fixed

* fixed

* fixed

* updated adb

* updated

* fixed

* removed log_analytics_destination_type

* Update azurerm/modules/azurerm-adb/example/constraints.tf

Co-authored-by: Rishi <[email protected]>

---------

Co-authored-by: Rishi <[email protected]>

azurerm/modules/azurerm-adb/constraints.tf

@@ -0,0 +1,12 @@
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~> 3.0"
+    }
+    databricks = {
+      source = "databricks/databricks"
+    }
+  }
+}

azurerm/modules/azurerm-adb/example/constraints.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~> 3.0"
+    }
+#TODO: note this is just added right now without any use, can be used In future for databricks provider
+    databricks = {
+      source = "databricks/databricks"
+    }
+  }
+}

azurerm/modules/azurerm-adb/example/main.tf

@@ -0,0 +1,42 @@
+data "azurerm_client_config" "current" {}
+
+module "default_label" {
+  source     = "git::https://github.com/cloudposse/terraform-null-label.git?ref=0.25.0"
+  namespace  = "${var.name_company}-${var.name_project}"
+  stage      = var.stage
+  name       = "${lookup(var.location_name_map, var.resource_group_location, "uksouth")}-${var.name_component}"
+  attributes = var.attributes
+  delimiter  = "-"
+  tags       = var.tags
+}
+
+##################################################
+# ResourceGroups
+##################################################
+
+resource "azurerm_resource_group" "default" {
+  name     = module.default_label.id
+  location = var.resource_group_location
+  tags     = var.tags
+}
+
+resource "azurerm_log_analytics_workspace" "la" {
+  name                = module.default_label.id
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  sku                 = var.la_sku
+  retention_in_days   = var.la_retention
+  tags                = module.default_label.tags
+}
+
+module "adb" {
+  source                  = "../../azurerm-adb"
+  resource_namer          = module.default_label.id
+  resource_group_name     = azurerm_resource_group.default.name
+  resource_group_location = azurerm_resource_group.default.location
+  databricks_sku          = var.databricks_sku
+  resource_tags           = module.default_label.tags
+  enable_databricksws_diagnostic       = var.enable_databricksws_diagnostic
+  #databricksws_diagnostic_setting_name = var.databricksws_diagnostic_setting_name
+  data_platform_log_analytics_workspace_id = azurerm_log_analytics_workspace.la.id
+}

azurerm/modules/azurerm-adb/example/output.tf

@@ -0,0 +1,7 @@
+output "adb_databricks_id" {
+  value = module.adb.adb_databricks_id
+}
+
+output "databricks_hosturl" {
+  value = module.adb.databricks_hosturl
+}

azurerm/modules/azurerm-adb/example/provider.tf

@@ -0,0 +1,3 @@
+provider "azurerm" {
+  features {}
+}

azurerm/modules/azurerm-adb/example/var.tf

@@ -0,0 +1,96 @@
+############################################
+# NAMING
+############################################
+
+variable "name_company" {
+  description = "Company Name - should/will be used in conventional resource naming"
+  type        = string
+}
+
+variable "name_project" {
+  description = "Project Name - should/will be used in conventional resource naming"
+  type        = string
+}
+
+variable "name_component" {
+  description = "Component Name - should/will be used in conventional resource naming. Typically this will be a logical name for this part of the system i.e. `API` || `middleware` or more generic like `Billing`"
+  type        = string
+}
+
+variable "name_environment" {
+  type = string
+}
+
+variable "stage" {
+  type    = string
+  default = "dev"
+}
+
+variable "attributes" {
+  description = "Additional attributes for tagging"
+  default     = []
+}
+
+variable "tags" {
+  description = "Tags to be assigned to all resources, NB if global tagging is enabled these will get overwritten periodically"
+  type        = map(string)
+  default     = {}
+}
+
+
+variable "resource_group_location" {
+  type    = string
+  default = "uksouth"
+}
+
+
+# Each region must have corresponding a shortend name for resource naming purposes 
+variable "location_name_map" {
+  type = map(string)
+
+  default = {
+    northeurope   = "eun"
+    westeurope    = "euw"
+    uksouth       = "uks"
+    ukwest        = "ukw"
+    eastus        = "use"
+    eastus2       = "use2"
+    westus        = "usw"
+    eastasia      = "ase"
+    southeastasia = "asse"
+  }
+}
+variable "databricks_sku" {
+  type        = string
+  default    = "premium"
+  description = "The SKU to use for the databricks instance"
+
+  validation {
+    condition     = can(regex("standard|premium|trial", var.databricks_sku))
+    error_message = "Err: Valid options are 'standard', 'premium' or 'trial'."
+  }
+}
+
+variable "enable_databricksws_diagnostic" {
+  type        = bool
+  description = "Whether to enable diagnostic settings for the Azure Databricks workspace"
+  default     = true
+}
+
+variable "databricksws_diagnostic_setting_name" {
+  type        = string
+  default = "Databricks to Log Analytics"
+  description = "The Databricks workspace diagnostic setting name."
+}
+
+variable "la_sku" {
+  type        = string
+  default     = "PerGB2018"
+  description = "Specifies the SKU of the Log Analytics Workspace."
+}
+
+variable "la_retention" {
+  type        = number
+  default     = 30
+  description = "The workspace data retention in days. Possible values are either 7 (Free Tier only) or range between 30 and 730."
+}

azurerm/modules/azurerm-adb/main.tf

@@ -0,0 +1,60 @@
+data "azurerm_client_config" "current" {
+}
+
+
+resource "azurerm_databricks_workspace" "example" {
+  name                = var.resource_namer
+  location            = var.resource_group_location
+  resource_group_name = var.resource_group_name
+  sku                 = var.databricks_sku
+
+
+  tags = var.resource_tags
+  lifecycle {
+    ignore_changes = [
+      tags,
+    ]
+  }
+}
+
+
+# Enable diagnostic settings for ADB
+data "azurerm_monitor_diagnostic_categories" "adb_log_analytics_categories" {
+  resource_id = azurerm_databricks_workspace.example.id
+}
+
+resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" {
+  count = var.enable_databricksws_diagnostic ? 1 : 0
+  #for_each = var.enable_databricksws_diagnostic ? { "enabled" = true } : {}
+  name                       = var.databricksws_diagnostic_setting_name
+  target_resource_id         = azurerm_databricks_workspace.example.id
+  log_analytics_workspace_id = var.data_platform_log_analytics_workspace_id
+
+  dynamic "log" {
+    for_each = data.azurerm_monitor_diagnostic_categories.adb_log_analytics_categories.logs
+
+    content {
+      category = log.value
+      enabled  = true
+
+      retention_policy {
+        enabled = false
+        days    = 0
+      }
+    }
+  }
+
+  dynamic "metric" {
+    for_each = data.azurerm_monitor_diagnostic_categories.adb_log_analytics_categories.metrics
+
+    content {
+      category = metric.value
+      enabled  = true
+
+      retention_policy {
+        enabled = false
+        days    = 0
+      }
+    }
+  }
+}

azurerm/modules/azurerm-adb/output.tf

@@ -0,0 +1,8 @@
+output "adb_databricks_id" {
+  value = azurerm_databricks_workspace.example.id
+}
+
+output "databricks_hosturl" {
+  description = "Azure Databricks HostUrl"
+  value = "https://${azurerm_databricks_workspace.example.workspace_url}/"
+}

azurerm/modules/azurerm-adb/var.tf

@@ -0,0 +1,63 @@
+############################################
+# NAMING
+############################################
+
+variable "resource_namer" {
+  type        = string
+  description = "User defined naming convention applied to all resources created as part of this module"
+}
+
+variable "resource_tags" {
+  description = "Map of tags to be applied to all resources created as part of this module"
+  type        = map(string)
+  default     = {}
+}
+
+############################################
+# RESOURCE INFORMATION
+############################################
+
+variable "resource_group_location" {
+  type        = string
+  default     = "uksouth"
+  description = "Location of Resource group"
+}
+
+variable "resource_group_name" {
+  type        = string
+  description = "Name of resource group"
+}
+
+variable "databricks_sku" {
+  type        = string
+  default    = "premium"
+  description = "The SKU to use for the databricks instance"
+
+  validation {
+    condition     = can(regex("standard|premium|trial", var.databricks_sku))
+    error_message = "Err: Valid options are 'standard', 'premium' or 'trial'."
+  }
+}
+
+
+############################################
+# Resource Diagnostic Setting 
+############################################
+
+variable "enable_databricksws_diagnostic" {
+  type        = bool
+  description = "Whether to enable diagnostic settings for the Azure Databricks workspace"
+  default     = false
+}
+
+variable "databricksws_diagnostic_setting_name" {
+  type        = string
+  default = "Databricks to Log Analytics"
+  description = "The Databricks workspace diagnostic setting name."
+}
+
+variable "data_platform_log_analytics_workspace_id" {
+  type        = string
+  default = null
+  description = "The Log Analytics Workspace used for the whole Data Platform."
+}