diff --git a/azurerm/modules/azurerm-adb/README.md b/azurerm/modules/azurerm-adb/README.md new file mode 100644 index 00000000..64976d66 Binary files /dev/null and b/azurerm/modules/azurerm-adb/README.md differ diff --git a/azurerm/modules/azurerm-adb/constraints.tf b/azurerm/modules/azurerm-adb/constraints.tf new file mode 100644 index 00000000..9baec76e --- /dev/null +++ b/azurerm/modules/azurerm-adb/constraints.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + databricks = { + source = "databricks/databricks" + } + } +} diff --git a/azurerm/modules/azurerm-adb/example/constraints.tf b/azurerm/modules/azurerm-adb/example/constraints.tf new file mode 100644 index 00000000..7ed79660 --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/constraints.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 0.13" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } +#TODO: note this is just added right now without any use, can be used In future for databricks provider + databricks = { + source = "databricks/databricks" + } + } +} diff --git a/azurerm/modules/azurerm-adb/example/main.tf b/azurerm/modules/azurerm-adb/example/main.tf new file mode 100644 index 00000000..9282bb77 --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/main.tf @@ -0,0 +1,42 @@ +data "azurerm_client_config" "current" {} + +module "default_label" { + source = "git::https://github.com/cloudposse/terraform-null-label.git?ref=0.25.0" + namespace = "${var.name_company}-${var.name_project}" + stage = var.stage + name = "${lookup(var.location_name_map, var.resource_group_location, "uksouth")}-${var.name_component}" + attributes = var.attributes + delimiter = "-" + tags = var.tags +} + +################################################## +# ResourceGroups +################################################## + +resource "azurerm_resource_group" "default" { + name = module.default_label.id + location = var.resource_group_location + tags = var.tags +} + +resource "azurerm_log_analytics_workspace" "la" { + name = module.default_label.id + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + sku = var.la_sku + retention_in_days = var.la_retention + tags = module.default_label.tags +} + +module "adb" { + source = "../../azurerm-adb" + resource_namer = module.default_label.id + resource_group_name = azurerm_resource_group.default.name + resource_group_location = azurerm_resource_group.default.location + databricks_sku = var.databricks_sku + resource_tags = module.default_label.tags + enable_databricksws_diagnostic = var.enable_databricksws_diagnostic + #databricksws_diagnostic_setting_name = var.databricksws_diagnostic_setting_name + data_platform_log_analytics_workspace_id = azurerm_log_analytics_workspace.la.id +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/example/output.tf b/azurerm/modules/azurerm-adb/example/output.tf new file mode 100644 index 00000000..d337ac39 --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/output.tf @@ -0,0 +1,7 @@ +output "adb_databricks_id" { + value = module.adb.adb_databricks_id +} + +output "databricks_hosturl" { + value = module.adb.databricks_hosturl +} diff --git a/azurerm/modules/azurerm-adb/example/provider.tf b/azurerm/modules/azurerm-adb/example/provider.tf new file mode 100644 index 00000000..615cb43d --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/provider.tf @@ -0,0 +1,3 @@ +provider "azurerm" { + features {} +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/example/var.tf b/azurerm/modules/azurerm-adb/example/var.tf new file mode 100644 index 00000000..5a67940b --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/var.tf @@ -0,0 +1,96 @@ +############################################ +# NAMING +############################################ + +variable "name_company" { + description = "Company Name - should/will be used in conventional resource naming" + type = string +} + +variable "name_project" { + description = "Project Name - should/will be used in conventional resource naming" + type = string +} + +variable "name_component" { + description = "Component Name - should/will be used in conventional resource naming. Typically this will be a logical name for this part of the system i.e. `API` || `middleware` or more generic like `Billing`" + type = string +} + +variable "name_environment" { + type = string +} + +variable "stage" { + type = string + default = "dev" +} + +variable "attributes" { + description = "Additional attributes for tagging" + default = [] +} + +variable "tags" { + description = "Tags to be assigned to all resources, NB if global tagging is enabled these will get overwritten periodically" + type = map(string) + default = {} +} + + +variable "resource_group_location" { + type = string + default = "uksouth" +} + + +# Each region must have corresponding a shortend name for resource naming purposes +variable "location_name_map" { + type = map(string) + + default = { + northeurope = "eun" + westeurope = "euw" + uksouth = "uks" + ukwest = "ukw" + eastus = "use" + eastus2 = "use2" + westus = "usw" + eastasia = "ase" + southeastasia = "asse" + } +} +variable "databricks_sku" { + type = string + default = "premium" + description = "The SKU to use for the databricks instance" + + validation { + condition = can(regex("standard|premium|trial", var.databricks_sku)) + error_message = "Err: Valid options are 'standard', 'premium' or 'trial'." + } +} + +variable "enable_databricksws_diagnostic" { + type = bool + description = "Whether to enable diagnostic settings for the Azure Databricks workspace" + default = true +} + +variable "databricksws_diagnostic_setting_name" { + type = string + default = "Databricks to Log Analytics" + description = "The Databricks workspace diagnostic setting name." +} + +variable "la_sku" { + type = string + default = "PerGB2018" + description = "Specifies the SKU of the Log Analytics Workspace." +} + +variable "la_retention" { + type = number + default = 30 + description = "The workspace data retention in days. Possible values are either 7 (Free Tier only) or range between 30 and 730." +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf new file mode 100644 index 00000000..7ad5306a --- /dev/null +++ b/azurerm/modules/azurerm-adb/main.tf @@ -0,0 +1,60 @@ +data "azurerm_client_config" "current" { +} + + +resource "azurerm_databricks_workspace" "example" { + name = var.resource_namer + location = var.resource_group_location + resource_group_name = var.resource_group_name + sku = var.databricks_sku + + + tags = var.resource_tags + lifecycle { + ignore_changes = [ + tags, + ] + } +} + + +# Enable diagnostic settings for ADB +data "azurerm_monitor_diagnostic_categories" "adb_log_analytics_categories" { + resource_id = azurerm_databricks_workspace.example.id +} + +resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { + count = var.enable_databricksws_diagnostic ? 1 : 0 + #for_each = var.enable_databricksws_diagnostic ? { "enabled" = true } : {} + name = var.databricksws_diagnostic_setting_name + target_resource_id = azurerm_databricks_workspace.example.id + log_analytics_workspace_id = var.data_platform_log_analytics_workspace_id + + dynamic "log" { + for_each = data.azurerm_monitor_diagnostic_categories.adb_log_analytics_categories.logs + + content { + category = log.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } + + dynamic "metric" { + for_each = data.azurerm_monitor_diagnostic_categories.adb_log_analytics_categories.metrics + + content { + category = metric.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/output.tf b/azurerm/modules/azurerm-adb/output.tf new file mode 100644 index 00000000..2390541f --- /dev/null +++ b/azurerm/modules/azurerm-adb/output.tf @@ -0,0 +1,8 @@ +output "adb_databricks_id" { + value = azurerm_databricks_workspace.example.id +} + +output "databricks_hosturl" { + description = "Azure Databricks HostUrl" + value = "https://${azurerm_databricks_workspace.example.workspace_url}/" +} diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf new file mode 100644 index 00000000..832fbbb6 --- /dev/null +++ b/azurerm/modules/azurerm-adb/var.tf @@ -0,0 +1,63 @@ +############################################ +# NAMING +############################################ + +variable "resource_namer" { + type = string + description = "User defined naming convention applied to all resources created as part of this module" +} + +variable "resource_tags" { + description = "Map of tags to be applied to all resources created as part of this module" + type = map(string) + default = {} +} + +############################################ +# RESOURCE INFORMATION +############################################ + +variable "resource_group_location" { + type = string + default = "uksouth" + description = "Location of Resource group" +} + +variable "resource_group_name" { + type = string + description = "Name of resource group" +} + +variable "databricks_sku" { + type = string + default = "premium" + description = "The SKU to use for the databricks instance" + + validation { + condition = can(regex("standard|premium|trial", var.databricks_sku)) + error_message = "Err: Valid options are 'standard', 'premium' or 'trial'." + } +} + + +############################################ +# Resource Diagnostic Setting +############################################ + +variable "enable_databricksws_diagnostic" { + type = bool + description = "Whether to enable diagnostic settings for the Azure Databricks workspace" + default = false +} + +variable "databricksws_diagnostic_setting_name" { + type = string + default = "Databricks to Log Analytics" + description = "The Databricks workspace diagnostic setting name." +} + +variable "data_platform_log_analytics_workspace_id" { + type = string + default = null + description = "The Log Analytics Workspace used for the whole Data Platform." +} \ No newline at end of file