From d18627cc7ee1fec742bca17c659f1d955d67be3b Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 7 Jun 2023 15:35:07 +0100 Subject: [PATCH 001/137] initial commit of databricks changes --- azurerm/modules/azurerm-adb/data.tf | 25 +++++++++++ azurerm/modules/azurerm-adb/main.tf | 25 +++++++++-- azurerm/modules/azurerm-adb/network.tf | 57 ++++++++++++++++++++++++++ azurerm/modules/azurerm-adb/output.tf | 40 ++++++++++++++++++ azurerm/modules/azurerm-adb/var.tf | 17 ++++++++ 5 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 azurerm/modules/azurerm-adb/data.tf create mode 100644 azurerm/modules/azurerm-adb/network.tf diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf new file mode 100644 index 00000000..16dadd57 --- /dev/null +++ b/azurerm/modules/azurerm-adb/data.tf @@ -0,0 +1,25 @@ +data "azurerm_virtual_network" "vnet" { + count = var.enable_private_network ? 1 : 0 + name = var.vnet_name + resource_group_name = var.vnet_name_resource_group +} + +data "azurerm_subnet" "public_subnet" { + count = var.enable_private_network ? 1 : 0 + name = var.subnet_name + virtual_network_name = var.vnet_name + resource_group_name = var.vnet_resource_group +} + +data "azurerm_subnet" "private_subnet" { + count = var.enable_private_network ? 1 : 0 + name = var.subnet_name + virtual_network_name = var.vnet_name + resource_group_name = var.vnet_resource_group +} + +data "azurerm_databricks_workspace_private_endpoint_connection" "example" { + count = var.enable_private_network ? 1 : 0 + workspace_id = azurerm_databricks_workspace.example.id + private_endpoint_id = azurerm_private_endpoint.databricks.id +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index eeec33ba..df8eeff0 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -1,9 +1,26 @@ resource "azurerm_databricks_workspace" "example" { - name = var.resource_namer - location = var.resource_group_location - resource_group_name = var.resource_group_name - sku = var.databricks_sku + name = var.resource_namer + location = var.resource_group_location + resource_group_name = var.resource_group_name + sku = var.databricks_sku + public_network_access_enabled = var.enable_private_network ? false : true + network_security_group_rules_required = var.network_security_group_rules_required + + dynamic "custom_parameters" { + for_each = var.enable_private_network == false ? toset([]) : toset([1]) + content { + no_public_ip = true + public_subnet_name = data.azurerm_subnet.public.name + private_subnet_name = data.azurerm_subnet.private.name + virtual_network_id = data.azurerm_virtual_network.example.id + vnet_address_prefix = var.vnet_address_prefix ? var.vnet_address_prefix : null + public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id + private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id + nat_gateway_name = var.nat_gateway_name ? var.nat_gateway_name : null + public_ip_name = var.public_ip_name ? var.public_ip_name : null + } + } tags = var.resource_tags diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf new file mode 100644 index 00000000..15a65faa --- /dev/null +++ b/azurerm/modules/azurerm-adb/network.tf @@ -0,0 +1,57 @@ +############################################ +# NSG +############################################ + +resource "azurerm_network_security_group" "nsg" { + count = var.enable_private_network ? 1 : 0 + name = "${var.prefix}-nsg-databricks" + location = var.resource_group_location + resource_group_name = var.resource_group_name +} + +resource "azurerm_subnet_network_security_group_association" "private" { + count = var.enable_private_network ? 1 : 0 + subnet_id = azurerm_subnet.private.id + network_security_group_id = azurerm_network_security_group.nsg.id +} + +resource "azurerm_subnet_network_security_group_association" "public" { + count = var.enable_private_network ? 1 : 0 + subnet_id = azurerm_subnet.public.id + network_security_group_id = azurerm_network_security_group.nsg.id +} + +############################################ +# PRIVATE ENDPOINT +############################################ + +resource "azurerm_private_endpoint" "databricks" { + count = var.enable_private_network ? 1 : 0 + name = "${var.prefix}-pe-databricks" + location = var.resource_group_location + resource_group_name = var.resource_group_name + subnet_id = azurerm_subnet.endpoint.id + + private_service_connection { + name = "${var.prefix}-psc" + is_manual_connection = false + private_connection_resource_id = azurerm_databricks_workspace.example.id + subresource_names = ["databricks_ui_api"] + } +} + +resource "azurerm_private_dns_zone" "dns" { + count = var.enable_private_network ? 1 : 0 + depends_on = [azurerm_private_endpoint.databricks] + name = "privatelink.azuredatabricks.net" + resource_group_name = var.resource_group_name +} + +resource "azurerm_private_dns_cname_record" "cname" { + count = var.enable_private_network ? 1 : 0 + name = azurerm_databricks_workspace.example.workspace_url + zone_name = azurerm_private_dns_zone.example.name + resource_group_name = var.resource_group_name + ttl = 300 + record = "eastus1-c2.azuredatabricks.net" +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/output.tf b/azurerm/modules/azurerm-adb/output.tf index b172ee23..eb971ab2 100644 --- a/azurerm/modules/azurerm-adb/output.tf +++ b/azurerm/modules/azurerm-adb/output.tf @@ -6,3 +6,43 @@ output "databricks_hosturl" { description = "Azure Databricks HostUrl" value = "https://${azurerm_databricks_workspace.example.workspace_url}/" } + + +############################################ +# PRIVATE ENDPOINT +############################################ + +output "databricks_workspace_private_endpoint_connection_workspace_id" { + count = var.enable_private_network ? 1 : 0 + value = data.azurerm_databricks_workspace_private_endpoint_connection.example.workspace_id +} + +output "databricks_workspace_private_endpoint_connection_private_endpoint_id" { + count = var.enable_private_network ? 1 : 0 + value = data.azurerm_databricks_workspace_private_endpoint_connection.example.private_endpoint_id +} + +output "databricks_workspace_private_endpoint_connection_name" { + count = var.enable_private_network ? 1 : 0 + value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.name +} + +output "databricks_workspace_private_endpoint_connection_workspace_private_endpoint_id" { + count = var.enable_private_network ? 1 : 0 + value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.workspace_private_endpoint_id +} + +output "databricks_workspace_private_endpoint_connection_status" { + count = var.enable_private_network ? 1 : 0 + value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.status +} + +output "databricks_workspace_private_endpoint_connection_description" { + count = var.enable_private_network ? 1 : 0 + value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.description +} + +output "databricks_workspace_private_endpoint_connection_action_required" { + count = var.enable_private_network ? 1 : 0 + value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.action_required +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index caa4e94b..9d7f4ef7 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -72,6 +72,23 @@ variable "enable_enableDbfsFileBrowser" { default = false } +variable "public_network_access_enabled" { + type = bool + default = true + description = "Enables or Disabled Public Access to Databricks Workspace." +} + +variable "network_security_group_rules_required" { + type = string + default = "NoAzureDatabricksRules" + description = " Does the data plane (clusters) to control plane communication happen over private link endpoint only or publicly? Possible values AllRules, NoAzureDatabricksRules or NoAzureServiceRules." +} + +variable "enable_private_network" { + type = bool + default = false + description = "Enable Secure Data Platform." +} ############################################ # Resource Databricks user From 274c80b9f8a5307f71066e1179700781c55fe883 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 7 Jun 2023 15:48:12 +0100 Subject: [PATCH 002/137] add nat --- azurerm/modules/azurerm-adb/local.tf | 4 +++ azurerm/modules/azurerm-adb/network.tf | 42 ++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 azurerm/modules/azurerm-adb/local.tf diff --git a/azurerm/modules/azurerm-adb/local.tf b/azurerm/modules/azurerm-adb/local.tf new file mode 100644 index 00000000..b7de44d3 --- /dev/null +++ b/azurerm/modules/azurerm-adb/local.tf @@ -0,0 +1,4 @@ +locals { + public_ip_name = "${var.resource_namer}-pip" + nat_gatewat_name = "${var.resource_namer}-nat-gw" +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 15a65faa..aced7d1e 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -41,8 +41,8 @@ resource "azurerm_private_endpoint" "databricks" { } resource "azurerm_private_dns_zone" "dns" { - count = var.enable_private_network ? 1 : 0 - depends_on = [azurerm_private_endpoint.databricks] + count = var.enable_private_network ? 1 : 0 + depends_on = [azurerm_private_endpoint.databricks] name = "privatelink.azuredatabricks.net" resource_group_name = var.resource_group_name } @@ -54,4 +54,42 @@ resource "azurerm_private_dns_cname_record" "cname" { resource_group_name = var.resource_group_name ttl = 300 record = "eastus1-c2.azuredatabricks.net" +} + + +############################################ +# NAT GATEWAY +############################################ + + +resource "azurerm_public_ip" "pip" { + count = var.enable_private_network ? 1 : 0 + name = local.public_ip_name + location = var.resource_group_location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" + zones = ["1", "2", "3"] +} + +resource "azurerm_nat_gateway" "nat" { + count = var.enable_private_network ? 1 : 0 + name = local.nat_gatewat_name + location = var.resource_group_location + resource_group_name = var.resource_group_name + sku_name = "Standard" + idle_timeout_in_minutes = 10 + zones = ["1", "2", "3"] +} + +resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { + count = var.enable_private_network ? 1 : 0 + nat_gateway_id = azurerm_nat_gateway.nat.id + public_ip_address_id = azurerm_public_ip.pip.id +} + +resource "azurerm_subnet_nat_gateway_association" "subnet_nat" { + count = var.enable_private_network ? 1 : 0 + subnet_id = data.azurerm_subnet.subnet.id + nat_gateway_id = azurerm_nat_gateway.nat.id } \ No newline at end of file From df7dce89d28a97801f37febb54b8c72e957206dc Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 7 Jun 2023 15:51:18 +0100 Subject: [PATCH 003/137] use nat and ip reosurces --- azurerm/modules/azurerm-adb/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index df8eeff0..14b6b135 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -17,8 +17,8 @@ resource "azurerm_databricks_workspace" "example" { vnet_address_prefix = var.vnet_address_prefix ? var.vnet_address_prefix : null public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id - nat_gateway_name = var.nat_gateway_name ? var.nat_gateway_name : null - public_ip_name = var.public_ip_name ? var.public_ip_name : null + nat_gateway_name = azurerm_nat_gateway.nat.name + public_ip_name = azurerm_public_ip.pip.name } } From 54982428aaab825960e20c1a714ca20fb6a1e565 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 7 Jun 2023 15:55:39 +0100 Subject: [PATCH 004/137] paramatise nat timeout --- azurerm/modules/azurerm-adb/network.tf | 2 +- azurerm/modules/azurerm-adb/var.tf | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index aced7d1e..624f9a2c 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -78,7 +78,7 @@ resource "azurerm_nat_gateway" "nat" { location = var.resource_group_location resource_group_name = var.resource_group_name sku_name = "Standard" - idle_timeout_in_minutes = 10 + idle_timeout_in_minutes = var.nat_idle_timeout zones = ["1", "2", "3"] } diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 9d7f4ef7..6fe83390 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -126,4 +126,10 @@ variable "enable_sql_access" { type = bool description = "Whether to enable sql access for the databricks group" default = true +} + +variable "nat_idle_timeout" { + type = number + default = 10 + description = "Idle timeout period in minutes." } \ No newline at end of file From d947215d31d4c2423f81f51572b8cf74cfa204ca Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 7 Jun 2023 16:03:37 +0100 Subject: [PATCH 005/137] add vars --- azurerm/modules/azurerm-adb/data.tf | 4 ++-- azurerm/modules/azurerm-adb/var.tf | 34 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index 16dadd57..5e198c0c 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -6,14 +6,14 @@ data "azurerm_virtual_network" "vnet" { data "azurerm_subnet" "public_subnet" { count = var.enable_private_network ? 1 : 0 - name = var.subnet_name + name = var.public_subnet_name virtual_network_name = var.vnet_name resource_group_name = var.vnet_resource_group } data "azurerm_subnet" "private_subnet" { count = var.enable_private_network ? 1 : 0 - name = var.subnet_name + name = var.private_subnet_name virtual_network_name = var.vnet_name resource_group_name = var.vnet_resource_group } diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 6fe83390..26595573 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -132,4 +132,38 @@ variable "nat_idle_timeout" { type = number default = 10 description = "Idle timeout period in minutes." +} + +############################################ +# Network Details +############################################ + +variable "vnet_name" { + type = string + default = "" + description = "Name of the VNET inwhich the Databricks Workspace will be provisioned." +} + +variable "vnet_name_resource_group" { + type = string + default = "" + description = "The Resource Group which the VNET is provisioned." +} + +variable "public_subnet_name" { + type = string + default = "" + description = "Name of the Public Databricks Subnet." +} + +variable "private_subnet_name" { + type = string + default = "" + description = "Name of the Private Databricks Subnet." +} + +variable "vnet_address_prefix" { + type = string + default = "10.139" + description = "Address Prefix of the VNET." } \ No newline at end of file From 560045c34ae29a5dce598ff5bd1e9aade1cedea3 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 10:33:21 +0100 Subject: [PATCH 006/137] add nat to both subnets --- azurerm/modules/azurerm-adb/network.tf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 624f9a2c..30a416ca 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -88,8 +88,14 @@ resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { public_ip_address_id = azurerm_public_ip.pip.id } -resource "azurerm_subnet_nat_gateway_association" "subnet_nat" { +resource "azurerm_subnet_nat_gateway_association" "public_subnet_nat" { count = var.enable_private_network ? 1 : 0 - subnet_id = data.azurerm_subnet.subnet.id + subnet_id = data.azurerm_subnet.public_subnet.id + nat_gateway_id = azurerm_nat_gateway.nat.id +} + +resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { + count = var.enable_private_network ? 1 : 0 + subnet_id = data.azurerm_subnet.private_subnet.id nat_gateway_id = azurerm_nat_gateway.nat.id } \ No newline at end of file From 46be20c99e5def85602868ed8151c101f8600c24 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 10:50:35 +0100 Subject: [PATCH 007/137] add pe subnet --- azurerm/modules/azurerm-adb/data.tf | 7 +++++ azurerm/modules/azurerm-adb/network.tf | 18 ++++++------- azurerm/modules/azurerm-adb/var.tf | 36 +++++++++++++++++--------- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index 5e198c0c..388d8b2d 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -18,6 +18,13 @@ data "azurerm_subnet" "private_subnet" { resource_group_name = var.vnet_resource_group } +data "azurerm_subnet" "pe_subnet" { + count = var.enable_private_network ? 1 : 0 + name = var.pe_subnet_name + virtual_network_name = var.vnet_name + resource_group_name = var.vnet_resource_group +} + data "azurerm_databricks_workspace_private_endpoint_connection" "example" { count = var.enable_private_network ? 1 : 0 workspace_id = azurerm_databricks_workspace.example.id diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 30a416ca..3aeb6d3f 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -4,20 +4,20 @@ resource "azurerm_network_security_group" "nsg" { count = var.enable_private_network ? 1 : 0 - name = "${var.prefix}-nsg-databricks" + name = "${var.resource_namer}-nsg-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name } resource "azurerm_subnet_network_security_group_association" "private" { count = var.enable_private_network ? 1 : 0 - subnet_id = azurerm_subnet.private.id + subnet_id = data.azurerm_subnet.private_subnet.id network_security_group_id = azurerm_network_security_group.nsg.id } resource "azurerm_subnet_network_security_group_association" "public" { count = var.enable_private_network ? 1 : 0 - subnet_id = azurerm_subnet.public.id + subnet_id = data.azurerm_subnet.public_subnet.id network_security_group_id = azurerm_network_security_group.nsg.id } @@ -27,13 +27,13 @@ resource "azurerm_subnet_network_security_group_association" "public" { resource "azurerm_private_endpoint" "databricks" { count = var.enable_private_network ? 1 : 0 - name = "${var.prefix}-pe-databricks" + name = "${var.resource_namer}-pe-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name - subnet_id = azurerm_subnet.endpoint.id + subnet_id = data.azurerm_subnet.pe_subnet.id private_service_connection { - name = "${var.prefix}-psc" + name = "${var.resource_namer}-psc" is_manual_connection = false private_connection_resource_id = azurerm_databricks_workspace.example.id subresource_names = ["databricks_ui_api"] @@ -50,10 +50,10 @@ resource "azurerm_private_dns_zone" "dns" { resource "azurerm_private_dns_cname_record" "cname" { count = var.enable_private_network ? 1 : 0 name = azurerm_databricks_workspace.example.workspace_url - zone_name = azurerm_private_dns_zone.example.name + zone_name = azurerm_private_dns_zone.dns.name resource_group_name = var.resource_group_name - ttl = 300 - record = "eastus1-c2.azuredatabricks.net" + ttl = var.dns_record_ttl + record = "${var.resource_namer}.azuredatabricks.net" } diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 26595573..122ae1bf 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -129,8 +129,8 @@ variable "enable_sql_access" { } variable "nat_idle_timeout" { - type = number - default = 10 + type = number + default = 10 description = "Idle timeout period in minutes." } @@ -139,31 +139,43 @@ variable "nat_idle_timeout" { ############################################ variable "vnet_name" { - type = string - default = "" + type = string + default = "" description = "Name of the VNET inwhich the Databricks Workspace will be provisioned." } variable "vnet_name_resource_group" { - type = string - default = "" + type = string + default = "" description = "The Resource Group which the VNET is provisioned." } variable "public_subnet_name" { - type = string - default = "" + type = string + default = "" description = "Name of the Public Databricks Subnet." } variable "private_subnet_name" { - type = string - default = "" + type = string + default = "" description = "Name of the Private Databricks Subnet." } +variable "pe_subnet_name" { + type = string + default = "" + description = "Name of the Subnet used to provision Private Endpoints into." +} + variable "vnet_address_prefix" { - type = string - default = "10.139" + type = string + default = "10.139" description = "Address Prefix of the VNET." +} + +variable "dns_record_ttl" { + type = number + default = 300 + description = "TTL for DNS Record." } \ No newline at end of file From 4aaeb242b4c168ff2c045aadc844be04954916ab Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 11:27:16 +0100 Subject: [PATCH 008/137] add ability to create subnets --- azurerm/modules/azurerm-adb/data.tf | 4 +- azurerm/modules/azurerm-adb/network.tf | 56 ++++++++++++++++++++++++-- azurerm/modules/azurerm-adb/var.tf | 19 +++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index 388d8b2d..1471df07 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -5,14 +5,14 @@ data "azurerm_virtual_network" "vnet" { } data "azurerm_subnet" "public_subnet" { - count = var.enable_private_network ? 1 : 0 + count = var.enable_private_network == true && var.create_subnets == false ? 1 : 0 name = var.public_subnet_name virtual_network_name = var.vnet_name resource_group_name = var.vnet_resource_group } data "azurerm_subnet" "private_subnet" { - count = var.enable_private_network ? 1 : 0 + count = var.enable_private_network == true && var.create_subnets == false ? 1 : 0 name = var.private_subnet_name virtual_network_name = var.vnet_name resource_group_name = var.vnet_resource_group diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 3aeb6d3f..9e5f1a15 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -1,3 +1,51 @@ +############################################ +# SUBNETS +############################################ + +resource "azurerm_subnet" "public_subnt" { + count = var.enable_private_network == true && var.create_subnets == true ? 1 : 0 + + name = var.public_subnet_name + resource_group_name = var.resource_group_name + virtual_network_name = var.vnet_name + address_prefixes = var.public_subnet_prefix + + delegation { + name = "${var.public_subnet_name}-databricks-del" + + service_delegation { + actions = [ + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action", + "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action", + ] + name = "Microsoft.Databricks/workspaces" + } + } +} + +resource "azurerm_subnet" "private_subnet" { + count = var.enable_private_network == true && var.create_subnets == true ? 1 : 0 + + name = var.private_subnet_name + resource_group_name = var.resource_group_name + virtual_network_name = var.vnet_name + address_prefixes = var.private_subnet_prefix + + delegation { + name = "${var.private_subnet_name}-databricks-del" + + service_delegation { + actions = [ + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action", + "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action", + ] + name = "Microsoft.Databricks/workspaces" + } + } +} + ############################################ # NSG ############################################ @@ -11,13 +59,13 @@ resource "azurerm_network_security_group" "nsg" { resource "azurerm_subnet_network_security_group_association" "private" { count = var.enable_private_network ? 1 : 0 - subnet_id = data.azurerm_subnet.private_subnet.id + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet.id : data.azurerm_subnet.private_subnet.id network_security_group_id = azurerm_network_security_group.nsg.id } resource "azurerm_subnet_network_security_group_association" "public" { count = var.enable_private_network ? 1 : 0 - subnet_id = data.azurerm_subnet.public_subnet.id + subnet_id = var.create_subnets ? azurerm_subnet.public_subnet.id : data.azurerm_subnet.public_subnet.id network_security_group_id = azurerm_network_security_group.nsg.id } @@ -90,12 +138,12 @@ resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { resource "azurerm_subnet_nat_gateway_association" "public_subnet_nat" { count = var.enable_private_network ? 1 : 0 - subnet_id = data.azurerm_subnet.public_subnet.id + subnet_id = var.create_subnets ? azurerm_subnet.public_subnet.id : data.azurerm_subnet.public_subnet.id nat_gateway_id = azurerm_nat_gateway.nat.id } resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { count = var.enable_private_network ? 1 : 0 - subnet_id = data.azurerm_subnet.private_subnet.id + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet.id : data.azurerm_subnet.private_subnet.id nat_gateway_id = azurerm_nat_gateway.nat.id } \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 122ae1bf..5aa80847 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -138,6 +138,12 @@ variable "nat_idle_timeout" { # Network Details ############################################ +variable "create_subnets" { + type = bool + default = false + description = "Set to true if you need the module to create the subnets for you." +} + variable "vnet_name" { type = string default = "" @@ -162,6 +168,19 @@ variable "private_subnet_name" { description = "Name of the Private Databricks Subnet." } +variable "public_subnet_prefix" { + type = string + default = "" + description = "IP Address Space fo the Public Databricks Subnet." +} + +variable "private_subnet_prefix" { + type = string + default = "" + description = "IP Address Space fo the Private Databricks Subnet." + +} + variable "pe_subnet_name" { type = string default = "" From 4560dc7524b34972811f630809185eb18263c40f Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 11:42:38 +0100 Subject: [PATCH 009/137] temp comment out count on outputs --- azurerm/modules/azurerm-adb/output.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azurerm/modules/azurerm-adb/output.tf b/azurerm/modules/azurerm-adb/output.tf index eb971ab2..fe8f5f18 100644 --- a/azurerm/modules/azurerm-adb/output.tf +++ b/azurerm/modules/azurerm-adb/output.tf @@ -13,36 +13,36 @@ output "databricks_hosturl" { ############################################ output "databricks_workspace_private_endpoint_connection_workspace_id" { - count = var.enable_private_network ? 1 : 0 + # count = var.enable_private_network ? 1 : 0 value = data.azurerm_databricks_workspace_private_endpoint_connection.example.workspace_id } output "databricks_workspace_private_endpoint_connection_private_endpoint_id" { - count = var.enable_private_network ? 1 : 0 + # count = var.enable_private_network ? 1 : 0 value = data.azurerm_databricks_workspace_private_endpoint_connection.example.private_endpoint_id } output "databricks_workspace_private_endpoint_connection_name" { - count = var.enable_private_network ? 1 : 0 + # count = var.enable_private_network ? 1 : 0 value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.name } output "databricks_workspace_private_endpoint_connection_workspace_private_endpoint_id" { - count = var.enable_private_network ? 1 : 0 + # count = var.enable_private_network ? 1 : 0 value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.workspace_private_endpoint_id } output "databricks_workspace_private_endpoint_connection_status" { - count = var.enable_private_network ? 1 : 0 + # count = var.enable_private_network ? 1 : 0 value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.status } output "databricks_workspace_private_endpoint_connection_description" { - count = var.enable_private_network ? 1 : 0 + # count = var.enable_private_network ? 1 : 0 value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.description } output "databricks_workspace_private_endpoint_connection_action_required" { - count = var.enable_private_network ? 1 : 0 + # count = var.enable_private_network ? 1 : 0 value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.action_required } \ No newline at end of file From 0ff35469c978508adf84cbbbcbf823f9773dbd49 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 11:47:47 +0100 Subject: [PATCH 010/137] update prefix vars to lists --- azurerm/modules/azurerm-adb/var.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 5aa80847..8afae363 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -169,14 +169,14 @@ variable "private_subnet_name" { } variable "public_subnet_prefix" { - type = string - default = "" + type = list(string) + default = [] description = "IP Address Space fo the Public Databricks Subnet." } variable "private_subnet_prefix" { - type = string - default = "" + type = list(string) + default = [] description = "IP Address Space fo the Private Databricks Subnet." } From 7ecb0940a9a490d8bb360617f793b372300f2bd2 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 11:54:11 +0100 Subject: [PATCH 011/137] update --- azurerm/modules/azurerm-adb/data.tf | 2 +- azurerm/modules/azurerm-adb/network.tf | 4 ++-- azurerm/modules/azurerm-adb/var.tf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index 1471df07..f934c874 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -1,7 +1,7 @@ data "azurerm_virtual_network" "vnet" { count = var.enable_private_network ? 1 : 0 name = var.vnet_name - resource_group_name = var.vnet_name_resource_group + resource_group_name = var.vnet_resource_group } data "azurerm_subnet" "public_subnet" { diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 9e5f1a15..97eddc9e 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -6,7 +6,7 @@ resource "azurerm_subnet" "public_subnt" { count = var.enable_private_network == true && var.create_subnets == true ? 1 : 0 name = var.public_subnet_name - resource_group_name = var.resource_group_name + resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name address_prefixes = var.public_subnet_prefix @@ -28,7 +28,7 @@ resource "azurerm_subnet" "private_subnet" { count = var.enable_private_network == true && var.create_subnets == true ? 1 : 0 name = var.private_subnet_name - resource_group_name = var.resource_group_name + resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name address_prefixes = var.private_subnet_prefix diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 8afae363..7bf85fba 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -150,7 +150,7 @@ variable "vnet_name" { description = "Name of the VNET inwhich the Databricks Workspace will be provisioned." } -variable "vnet_name_resource_group" { +variable "vnet_resource_group" { type = string default = "" description = "The Resource Group which the VNET is provisioned." From 94e721afc1b8fd6b923e05ab3b3b9f3f7858f1b6 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 13:36:53 +0100 Subject: [PATCH 012/137] add index --- azurerm/modules/azurerm-adb/network.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 97eddc9e..f4e19b92 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -59,13 +59,13 @@ resource "azurerm_network_security_group" "nsg" { resource "azurerm_subnet_network_security_group_association" "private" { count = var.enable_private_network ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.private_subnet.id : data.azurerm_subnet.private_subnet.id + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id network_security_group_id = azurerm_network_security_group.nsg.id } resource "azurerm_subnet_network_security_group_association" "public" { count = var.enable_private_network ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.public_subnet.id : data.azurerm_subnet.public_subnet.id + subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id network_security_group_id = azurerm_network_security_group.nsg.id } @@ -138,12 +138,12 @@ resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { resource "azurerm_subnet_nat_gateway_association" "public_subnet_nat" { count = var.enable_private_network ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.public_subnet.id : data.azurerm_subnet.public_subnet.id + subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id nat_gateway_id = azurerm_nat_gateway.nat.id } resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { count = var.enable_private_network ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.private_subnet.id : data.azurerm_subnet.private_subnet.id + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id nat_gateway_id = azurerm_nat_gateway.nat.id } \ No newline at end of file From d2f11f5c2b5eae83980b462c1e03258aada6a0e5 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 13:44:55 +0100 Subject: [PATCH 013/137] add index to nsg --- azurerm/modules/azurerm-adb/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index f4e19b92..07b0a54b 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -60,13 +60,13 @@ resource "azurerm_network_security_group" "nsg" { resource "azurerm_subnet_network_security_group_association" "private" { count = var.enable_private_network ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id - network_security_group_id = azurerm_network_security_group.nsg.id + network_security_group_id = azurerm_network_security_group.nsg[0].id } resource "azurerm_subnet_network_security_group_association" "public" { count = var.enable_private_network ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id - network_security_group_id = azurerm_network_security_group.nsg.id + network_security_group_id = azurerm_network_security_group.nsg[0].id } ############################################ From 8ecb3bf76177591640c9d749227c1f8cbb931a0f Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 13:46:12 +0100 Subject: [PATCH 014/137] add index to pip and nat --- azurerm/modules/azurerm-adb/network.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 07b0a54b..513c1780 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -132,18 +132,18 @@ resource "azurerm_nat_gateway" "nat" { resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { count = var.enable_private_network ? 1 : 0 - nat_gateway_id = azurerm_nat_gateway.nat.id - public_ip_address_id = azurerm_public_ip.pip.id + nat_gateway_id = azurerm_nat_gateway.nat[0].id + public_ip_address_id = azurerm_public_ip.pip[0].id } resource "azurerm_subnet_nat_gateway_association" "public_subnet_nat" { count = var.enable_private_network ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id - nat_gateway_id = azurerm_nat_gateway.nat.id + nat_gateway_id = azurerm_nat_gateway.nat[0].id } resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { count = var.enable_private_network ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id - nat_gateway_id = azurerm_nat_gateway.nat.id + nat_gateway_id = azurerm_nat_gateway.nat[0].id } \ No newline at end of file From d6d20e81e478c7efe143322ef66e12d02020f1ed Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 13:51:54 +0100 Subject: [PATCH 015/137] fix typo --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 513c1780..97396946 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -2,7 +2,7 @@ # SUBNETS ############################################ -resource "azurerm_subnet" "public_subnt" { +resource "azurerm_subnet" "public_subnet" { count = var.enable_private_network == true && var.create_subnets == true ? 1 : 0 name = var.public_subnet_name From a4bac126a9e25f645cdd7d782deb33290606121c Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 13:56:06 +0100 Subject: [PATCH 016/137] add count to pe subnet --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 97396946..35789cea 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -78,7 +78,7 @@ resource "azurerm_private_endpoint" "databricks" { name = "${var.resource_namer}-pe-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name - subnet_id = data.azurerm_subnet.pe_subnet.id + subnet_id = data.azurerm_subnet.pe_subnet[0].id private_service_connection { name = "${var.resource_namer}-psc" From 49d387a7db6b56e25e790b39c12e1e74fc7acbed Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 14:06:05 +0100 Subject: [PATCH 017/137] add index --- azurerm/modules/azurerm-adb/data.tf | 2 +- azurerm/modules/azurerm-adb/network.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index f934c874..3a7d1345 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -28,5 +28,5 @@ data "azurerm_subnet" "pe_subnet" { data "azurerm_databricks_workspace_private_endpoint_connection" "example" { count = var.enable_private_network ? 1 : 0 workspace_id = azurerm_databricks_workspace.example.id - private_endpoint_id = azurerm_private_endpoint.databricks.id + private_endpoint_id = azurerm_private_endpoint[0].databricks.id } \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 35789cea..160d0095 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -98,7 +98,7 @@ resource "azurerm_private_dns_zone" "dns" { resource "azurerm_private_dns_cname_record" "cname" { count = var.enable_private_network ? 1 : 0 name = azurerm_databricks_workspace.example.workspace_url - zone_name = azurerm_private_dns_zone.dns.name + zone_name = azurerm_private_dns_zone.dns[0].name resource_group_name = var.resource_group_name ttl = var.dns_record_ttl record = "${var.resource_namer}.azuredatabricks.net" From e78f01af4290dc0bc128210c5f39663573890973 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 14:10:17 +0100 Subject: [PATCH 018/137] fix typo --- azurerm/modules/azurerm-adb/data.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index 3a7d1345..367a5e0b 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -28,5 +28,5 @@ data "azurerm_subnet" "pe_subnet" { data "azurerm_databricks_workspace_private_endpoint_connection" "example" { count = var.enable_private_network ? 1 : 0 workspace_id = azurerm_databricks_workspace.example.id - private_endpoint_id = azurerm_private_endpoint[0].databricks.id + private_endpoint_id = azurerm_private_endpoint.databricks[0].id } \ No newline at end of file From 0be934be25d59cd5b7446e81f520227b4e4e787d Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 14:15:18 +0100 Subject: [PATCH 019/137] add index to outputs --- azurerm/modules/azurerm-adb/output.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azurerm/modules/azurerm-adb/output.tf b/azurerm/modules/azurerm-adb/output.tf index fe8f5f18..2154367b 100644 --- a/azurerm/modules/azurerm-adb/output.tf +++ b/azurerm/modules/azurerm-adb/output.tf @@ -14,35 +14,35 @@ output "databricks_hosturl" { output "databricks_workspace_private_endpoint_connection_workspace_id" { # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example.workspace_id + value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].workspace_id } output "databricks_workspace_private_endpoint_connection_private_endpoint_id" { # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example.private_endpoint_id + value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].private_endpoint_id } output "databricks_workspace_private_endpoint_connection_name" { # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.name + value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.name } output "databricks_workspace_private_endpoint_connection_workspace_private_endpoint_id" { # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.workspace_private_endpoint_id + value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.workspace_private_endpoint_id } output "databricks_workspace_private_endpoint_connection_status" { # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.status + value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.status } output "databricks_workspace_private_endpoint_connection_description" { # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.description + value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.description } output "databricks_workspace_private_endpoint_connection_action_required" { # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example.connections.0.action_required + value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.action_required } \ No newline at end of file From a749d62f98cecb13c4261ae83d2f49876f19a13d Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 14:36:15 +0100 Subject: [PATCH 020/137] update --- azurerm/modules/azurerm-adb/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 14b6b135..b7237052 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -11,14 +11,14 @@ resource "azurerm_databricks_workspace" "example" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) content { no_public_ip = true - public_subnet_name = data.azurerm_subnet.public.name - private_subnet_name = data.azurerm_subnet.private.name - virtual_network_id = data.azurerm_virtual_network.example.id + public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id + private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id + virtual_network_id = data.azurerm_virtual_network.vnet[0].id vnet_address_prefix = var.vnet_address_prefix ? var.vnet_address_prefix : null - public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id - private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id - nat_gateway_name = azurerm_nat_gateway.nat.name - public_ip_name = azurerm_public_ip.pip.name + public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id + private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id + nat_gateway_name = azurerm_nat_gateway.nat[0].name + public_ip_name = azurerm_public_ip.pip[0].name } } From a1d19a54694a52dd9022a68e6254e9712440a5ae Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 14:47:30 +0100 Subject: [PATCH 021/137] update --- azurerm/modules/azurerm-adb/main.tf | 2 +- azurerm/modules/azurerm-adb/var.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index b7237052..853d9a14 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -14,7 +14,7 @@ resource "azurerm_databricks_workspace" "example" { public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id virtual_network_id = data.azurerm_virtual_network.vnet[0].id - vnet_address_prefix = var.vnet_address_prefix ? var.vnet_address_prefix : null + vnet_address_prefix = var.vnet_address_prefix == "" ? null : var.vnet_address_prefix public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id nat_gateway_name = azurerm_nat_gateway.nat[0].name diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 7bf85fba..23d8ef04 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -189,7 +189,7 @@ variable "pe_subnet_name" { variable "vnet_address_prefix" { type = string - default = "10.139" + default = "" description = "Address Prefix of the VNET." } From 6d599b54f990d09abdabeb4f53af0ee8cdd08f47 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 15:05:02 +0100 Subject: [PATCH 022/137] make nat gateway single region --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 160d0095..83986afc 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -127,7 +127,7 @@ resource "azurerm_nat_gateway" "nat" { resource_group_name = var.resource_group_name sku_name = "Standard" idle_timeout_in_minutes = var.nat_idle_timeout - zones = ["1", "2", "3"] + zones = ["1"] } resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { From 9255de76384dc70200f937b6e5dd89dc5426e24a Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 15:11:21 +0100 Subject: [PATCH 023/137] make pip single zone --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 83986afc..506f863b 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -117,7 +117,7 @@ resource "azurerm_public_ip" "pip" { resource_group_name = var.resource_group_name allocation_method = "Static" sku = "Standard" - zones = ["1", "2", "3"] + zones = ["1"] } resource "azurerm_nat_gateway" "nat" { From e6916c51c98cc65bfe2f34d5ebb34e16241292d9 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 16:29:19 +0100 Subject: [PATCH 024/137] add depends on --- azurerm/modules/azurerm-adb/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 853d9a14..c47f7c14 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -29,6 +29,7 @@ resource "azurerm_databricks_workspace" "example" { tags, ] } + depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet] } From 0f2c8db8b3faa305e3a2dc62c3551b56ffc41be0 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 8 Jun 2023 16:40:21 +0100 Subject: [PATCH 025/137] add service endpoints --- azurerm/modules/azurerm-adb/network.tf | 2 ++ azurerm/modules/azurerm-adb/var.tf | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 506f863b..2650ea57 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -9,6 +9,7 @@ resource "azurerm_subnet" "public_subnet" { resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name address_prefixes = var.public_subnet_prefix + service_endpoints = var.service_endpoints delegation { name = "${var.public_subnet_name}-databricks-del" @@ -31,6 +32,7 @@ resource "azurerm_subnet" "private_subnet" { resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name address_prefixes = var.private_subnet_prefix + service_endpoints = var.service_endpoints delegation { name = "${var.private_subnet_name}-databricks-del" diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 23d8ef04..bdc4a574 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -197,4 +197,10 @@ variable "dns_record_ttl" { type = number default = 300 description = "TTL for DNS Record." +} + +variable "service_endpoints" { + type = list(string) + default = ["Microsoft.AzureActiveDirectory", "Microsoft.KeyVault", "Microsoft.ServiceBus", "Microsoft.Sql", "Microsoft.Storage"] + description = "List of Service Endpoints Enabled on the Subnet." } \ No newline at end of file From 26856553721ed2e18c4a6c56c147918d3c777441 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 09:56:26 +0100 Subject: [PATCH 026/137] add permissions for network and dns --- azurerm/modules/azurerm-adb/data.tf | 6 ++++++ azurerm/modules/azurerm-adb/main.tf | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index 367a5e0b..5d5c0f88 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -1,3 +1,9 @@ +data "azurerm_client_config" "current" {} + +data "azurerm_resource_group" "vnet_rg" { + name = var.vnet_resource_group +} + data "azurerm_virtual_network" "vnet" { count = var.enable_private_network ? 1 : 0 name = var.vnet_name diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index c47f7c14..e0673499 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -101,4 +101,16 @@ resource "databricks_group_member" "project_users" { for_each = var.add_rbac_users ? databricks_user.rbac_users : {} group_id = databricks_group.project_users[0].id member_id = each.value.id -} \ No newline at end of file +} + +resource "azurerm_role_assignment" "network" { + scope = data.azurerm_resource_group.vnet_rg.id + role_definition_name = "Network Contributor" + principal_id = data.azurerm_client_config.current.client_id +} + +resource "azurerm_role_assignment" "dns" { + scope = azurerm_private_dns_zone.dns.id + role_definition_name = "Private DNS Zone Contributor" + principal_id = data.azurerm_client_config.current.client_id +} From 71f6ce92d1862343442923665faad5383d6d3380 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 10:10:49 +0100 Subject: [PATCH 027/137] add index --- azurerm/modules/azurerm-adb/data.tf | 3 ++- azurerm/modules/azurerm-adb/main.tf | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index 5d5c0f88..d2adc506 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -1,7 +1,8 @@ data "azurerm_client_config" "current" {} data "azurerm_resource_group" "vnet_rg" { - name = var.vnet_resource_group + count = var.enable_private_network ? 1 : 0 + name = var.vnet_resource_group } data "azurerm_virtual_network" "vnet" { diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index e0673499..04227966 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -104,13 +104,13 @@ resource "databricks_group_member" "project_users" { } resource "azurerm_role_assignment" "network" { - scope = data.azurerm_resource_group.vnet_rg.id + scope = data.azurerm_resource_group.vnet_rg[0].id role_definition_name = "Network Contributor" principal_id = data.azurerm_client_config.current.client_id } resource "azurerm_role_assignment" "dns" { - scope = azurerm_private_dns_zone.dns.id + scope = azurerm_private_dns_zone.dns[0].id role_definition_name = "Private DNS Zone Contributor" principal_id = data.azurerm_client_config.current.client_id } From 12f6df905f2edc30d6993ec2a415ac85f9545e44 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 10:16:15 +0100 Subject: [PATCH 028/137] add permissions to depends on --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 04227966..d98b077d 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -29,7 +29,7 @@ resource "azurerm_databricks_workspace" "example" { tags, ] } - depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet] + depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet, azurerm_role_assignment.dns, azurerm_role_assignment.network] } From 2150428dfcf00097f713dc5585eea456104749eb Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 10:54:10 +0100 Subject: [PATCH 029/137] add databricks user --- azurerm/modules/azurerm-adb/data.tf | 5 +++++ azurerm/modules/azurerm-adb/main.tf | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index d2adc506..e0ac0860 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -1,5 +1,10 @@ data "azurerm_client_config" "current" {} +data "databricks_current_user" "db" { + count = var.enable_private_network ? 1 : 0 + depends_on = [azurerm_databricks_workspace.example] +} + data "azurerm_resource_group" "vnet_rg" { count = var.enable_private_network ? 1 : 0 name = var.vnet_resource_group diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index d98b077d..8d67684d 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -114,3 +114,15 @@ resource "azurerm_role_assignment" "dns" { role_definition_name = "Private DNS Zone Contributor" principal_id = data.azurerm_client_config.current.client_id } + +resource "azurerm_role_assignment" "network" { + scope = data.databricks_current_user.db[0].external_id + role_definition_name = "Network Contributor" + principal_id = data.azurerm_client_config.current.client_id +} + +resource "azurerm_role_assignment" "dns" { + scope = data.databricks_current_user.db[0].external_id + role_definition_name = "Private DNS Zone Contributor" + principal_id = data.azurerm_client_config.current.client_id +} \ No newline at end of file From 4c713ca30b15ac77a4bde32ff0a2fcfa52f653ef Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 10:54:28 +0100 Subject: [PATCH 030/137] update --- azurerm/modules/azurerm-adb/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 8d67684d..618b71f3 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -115,13 +115,13 @@ resource "azurerm_role_assignment" "dns" { principal_id = data.azurerm_client_config.current.client_id } -resource "azurerm_role_assignment" "network" { +resource "azurerm_role_assignment" "network_db" { scope = data.databricks_current_user.db[0].external_id role_definition_name = "Network Contributor" principal_id = data.azurerm_client_config.current.client_id } -resource "azurerm_role_assignment" "dns" { +resource "azurerm_role_assignment" "dns_db" { scope = data.databricks_current_user.db[0].external_id role_definition_name = "Private DNS Zone Contributor" principal_id = data.azurerm_client_config.current.client_id From bd2f7e431a4ba6417c0ef50154b0d8b418553160 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 11:00:15 +0100 Subject: [PATCH 031/137] add peering --- azurerm/modules/azurerm-adb/main.tf | 52 +++++++++++++++++------------ 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 618b71f3..256daf25 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -103,26 +103,36 @@ resource "databricks_group_member" "project_users" { member_id = each.value.id } -resource "azurerm_role_assignment" "network" { - scope = data.azurerm_resource_group.vnet_rg[0].id - role_definition_name = "Network Contributor" - principal_id = data.azurerm_client_config.current.client_id +resource "azurerm_databricks_virtual_network_peering" "peering" { + name = "databricks-vnet-peer" + resource_group_name = var.resource_group_name + workspace_id = azurerm_databricks_workspace.example.id + + remote_address_space_prefixes = data.azurerm_virtual_network.vnet.address_space + remote_virtual_network_id = data.azurerm_virtual_network.vnet.id + allow_virtual_network_access = true } -resource "azurerm_role_assignment" "dns" { - scope = azurerm_private_dns_zone.dns[0].id - role_definition_name = "Private DNS Zone Contributor" - principal_id = data.azurerm_client_config.current.client_id -} - -resource "azurerm_role_assignment" "network_db" { - scope = data.databricks_current_user.db[0].external_id - role_definition_name = "Network Contributor" - principal_id = data.azurerm_client_config.current.client_id -} - -resource "azurerm_role_assignment" "dns_db" { - scope = data.databricks_current_user.db[0].external_id - role_definition_name = "Private DNS Zone Contributor" - principal_id = data.azurerm_client_config.current.client_id -} \ No newline at end of file +# resource "azurerm_role_assignment" "network" { +# scope = data.azurerm_resource_group.vnet_rg[0].id +# role_definition_name = "Network Contributor" +# principal_id = data.azurerm_client_config.current.client_id +# } + +# resource "azurerm_role_assignment" "dns" { +# scope = azurerm_private_dns_zone.dns[0].id +# role_definition_name = "Private DNS Zone Contributor" +# principal_id = data.azurerm_client_config.current.client_id +# } + +# resource "azurerm_role_assignment" "network_db" { +# scope = data.databricks_current_user.db[0].external_id +# role_definition_name = "Network Contributor" +# principal_id = data.azurerm_client_config.current.client_id +# } + +# resource "azurerm_role_assignment" "dns_db" { +# scope = data.databricks_current_user.db[0].external_id +# role_definition_name = "Private DNS Zone Contributor" +# principal_id = data.azurerm_client_config.current.client_id +# } \ No newline at end of file From 228ddd0820a7efc5cba00126c37d66ef7d805e04 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 11:06:43 +0100 Subject: [PATCH 032/137] remove reference to data lookup --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 256daf25..ca55a970 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -29,7 +29,7 @@ resource "azurerm_databricks_workspace" "example" { tags, ] } - depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet, azurerm_role_assignment.dns, azurerm_role_assignment.network] + depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet] } From 40a7c1b7ba99077dc5981b88a3541771e0aa4c86 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 11:10:24 +0100 Subject: [PATCH 033/137] add index to vnet --- azurerm/modules/azurerm-adb/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index ca55a970..6764d721 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -108,8 +108,8 @@ resource "azurerm_databricks_virtual_network_peering" "peering" { resource_group_name = var.resource_group_name workspace_id = azurerm_databricks_workspace.example.id - remote_address_space_prefixes = data.azurerm_virtual_network.vnet.address_space - remote_virtual_network_id = data.azurerm_virtual_network.vnet.id + remote_address_space_prefixes = data.azurerm_virtual_network.vnet[0].address_space + remote_virtual_network_id = data.azurerm_virtual_network.vnet[0].id allow_virtual_network_access = true } From c4efbb105275e98990b161205639d7103eee4878 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 11:22:13 +0100 Subject: [PATCH 034/137] add permissions to db --- azurerm/modules/azurerm-adb/main.tf | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 6764d721..f8f774cf 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -113,17 +113,17 @@ resource "azurerm_databricks_virtual_network_peering" "peering" { allow_virtual_network_access = true } -# resource "azurerm_role_assignment" "network" { -# scope = data.azurerm_resource_group.vnet_rg[0].id -# role_definition_name = "Network Contributor" -# principal_id = data.azurerm_client_config.current.client_id -# } +resource "azurerm_role_assignment" "network" { + scope = data.azurerm_resource_group.vnet_rg[0].id + role_definition_name = "Network Contributor" + principal_id = "9a74af6f-d153-4348-988a-e2672920bee9" +} -# resource "azurerm_role_assignment" "dns" { -# scope = azurerm_private_dns_zone.dns[0].id -# role_definition_name = "Private DNS Zone Contributor" -# principal_id = data.azurerm_client_config.current.client_id -# } +resource "azurerm_role_assignment" "dns" { + scope = azurerm_private_dns_zone.dns[0].id + role_definition_name = "Private DNS Zone Contributor" + principal_id = "9a74af6f-d153-4348-988a-e2672920bee9" +} # resource "azurerm_role_assignment" "network_db" { # scope = data.databricks_current_user.db[0].external_id From 1315241b5e34c6e092be6b1a599a9028edbc9d3a Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 13:10:36 +0100 Subject: [PATCH 035/137] comment out perms --- azurerm/modules/azurerm-adb/main.tf | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index f8f774cf..37ef613f 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -113,17 +113,17 @@ resource "azurerm_databricks_virtual_network_peering" "peering" { allow_virtual_network_access = true } -resource "azurerm_role_assignment" "network" { - scope = data.azurerm_resource_group.vnet_rg[0].id - role_definition_name = "Network Contributor" - principal_id = "9a74af6f-d153-4348-988a-e2672920bee9" -} +# resource "azurerm_role_assignment" "network" { +# scope = data.azurerm_resource_group.vnet_rg[0].id +# role_definition_name = "Network Contributor" +# principal_id = "9a74af6f-d153-4348-988a-e2672920bee9" +# } -resource "azurerm_role_assignment" "dns" { - scope = azurerm_private_dns_zone.dns[0].id - role_definition_name = "Private DNS Zone Contributor" - principal_id = "9a74af6f-d153-4348-988a-e2672920bee9" -} +# resource "azurerm_role_assignment" "dns" { +# scope = azurerm_private_dns_zone.dns[0].id +# role_definition_name = "Private DNS Zone Contributor" +# principal_id = "9a74af6f-d153-4348-988a-e2672920bee9" +# } # resource "azurerm_role_assignment" "network_db" { # scope = data.databricks_current_user.db[0].external_id From 236d9627a382d3360bd28548fdb9609c6bd52306 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 13:33:30 +0100 Subject: [PATCH 036/137] add rg --- azurerm/modules/azurerm-adb/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 37ef613f..7a7461dd 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -6,6 +6,7 @@ resource "azurerm_databricks_workspace" "example" { sku = var.databricks_sku public_network_access_enabled = var.enable_private_network ? false : true network_security_group_rules_required = var.network_security_group_rules_required + managed_resource_group_name = "databricks-rg-${var.resource_group_name}" dynamic "custom_parameters" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) From ca5809da268e6e092929bb4aa2c2ce7820be5982 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 13:54:27 +0100 Subject: [PATCH 037/137] add depends on --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 7a7461dd..e6d28f0d 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -30,7 +30,7 @@ resource "azurerm_databricks_workspace" "example" { tags, ] } - depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet] + depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet, azurerm_private_dns_zone.dns] } From f9d167ec10671b3d5eb1f35e5c99f52cba8c0fcc Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 14:11:34 +0100 Subject: [PATCH 038/137] use name instead of id --- azurerm/modules/azurerm-adb/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index e6d28f0d..657eb19a 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -12,9 +12,9 @@ resource "azurerm_databricks_workspace" "example" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) content { no_public_ip = true - public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id - private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id - virtual_network_id = data.azurerm_virtual_network.vnet[0].id + public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name + private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name + virtual_network_id = data.azurerm_virtual_network.vnet[0].name vnet_address_prefix = var.vnet_address_prefix == "" ? null : var.vnet_address_prefix public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id From 0fa42b7e48d6dfb3f602b02af11054309db1d3d8 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 14:21:47 +0100 Subject: [PATCH 039/137] remove depends on --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 657eb19a..c1ab7ae7 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -30,7 +30,7 @@ resource "azurerm_databricks_workspace" "example" { tags, ] } - depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet, azurerm_private_dns_zone.dns] + depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet] } From 6bfe2add987834100da57cb9e3f41e47e7d84ff3 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 14:27:55 +0100 Subject: [PATCH 040/137] update id --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index c1ab7ae7..01b80a9c 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -14,7 +14,7 @@ resource "azurerm_databricks_workspace" "example" { no_public_ip = true public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name - virtual_network_id = data.azurerm_virtual_network.vnet[0].name + virtual_network_id = data.azurerm_virtual_network.vnet[0].id vnet_address_prefix = var.vnet_address_prefix == "" ? null : var.vnet_address_prefix public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id From 6e503979f79482dfb2ce4f1daed6e7724db226c6 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 14:37:49 +0100 Subject: [PATCH 041/137] remove db peer --- azurerm/modules/azurerm-adb/main.tf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 01b80a9c..e875b9bb 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -104,16 +104,6 @@ resource "databricks_group_member" "project_users" { member_id = each.value.id } -resource "azurerm_databricks_virtual_network_peering" "peering" { - name = "databricks-vnet-peer" - resource_group_name = var.resource_group_name - workspace_id = azurerm_databricks_workspace.example.id - - remote_address_space_prefixes = data.azurerm_virtual_network.vnet[0].address_space - remote_virtual_network_id = data.azurerm_virtual_network.vnet[0].id - allow_virtual_network_access = true -} - # resource "azurerm_role_assignment" "network" { # scope = data.azurerm_resource_group.vnet_rg[0].id # role_definition_name = "Network Contributor" From 82aa16382f2b565bbd41caf37033f37bf0ef21d2 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 14:47:13 +0100 Subject: [PATCH 042/137] update var for public access --- azurerm/modules/azurerm-adb/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index e875b9bb..a7707ac3 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -4,7 +4,8 @@ resource "azurerm_databricks_workspace" "example" { location = var.resource_group_location resource_group_name = var.resource_group_name sku = var.databricks_sku - public_network_access_enabled = var.enable_private_network ? false : true + # public_network_access_enabled = var.enable_private_network ? false : true + public_network_access_enabled = var.public_network_access_enabled network_security_group_rules_required = var.network_security_group_rules_required managed_resource_group_name = "databricks-rg-${var.resource_group_name}" From fd5b5629c3c4fec38597babf7942e321f4edcb90 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 16:01:04 +0100 Subject: [PATCH 043/137] add route table --- azurerm/modules/azurerm-adb/main.tf | 8 +++--- azurerm/modules/azurerm-adb/network.tf | 36 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index a7707ac3..b8522fa9 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -1,9 +1,9 @@ resource "azurerm_databricks_workspace" "example" { - name = var.resource_namer - location = var.resource_group_location - resource_group_name = var.resource_group_name - sku = var.databricks_sku + name = var.resource_namer + location = var.resource_group_location + resource_group_name = var.resource_group_name + sku = var.databricks_sku # public_network_access_enabled = var.enable_private_network ? false : true public_network_access_enabled = var.public_network_access_enabled network_security_group_rules_required = var.network_security_group_rules_required diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 2650ea57..0443f693 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -148,4 +148,40 @@ resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { count = var.enable_private_network ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id nat_gateway_id = azurerm_nat_gateway.nat[0].id +} + +############################################ +# ROUTE TABLES +############################################ + +resource "azurerm_route_table" "adb-route-table" { + count = var.enable_private_network ? 1 : 0 + name = var.resource_namer + location = var.resource_group_location + resource_group_name = var.resource_group_name + + route { + name = "to-nat" + address_prefix = "0.0.0.0/0" + next_hop_type = "VirtualAppliance" + next_hop_in_ip_address = azurerm_public_ip.pip[0].id + } + + route { + name = "to-scc-relay" + address_prefix = azurerm_public_ip.pip[0].id + next_hop_type = "Internet" + } +} + +resource "azurerm_subnet_route_table_association" "adb-pubic-rt-assocation" { + count = var.enable_private_network ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id + route_table_id = azurerm_route_table.adb-route-table[0].id +} + +resource "azurerm_subnet_route_table_association" "adb-private-rt-assocation" { + count = var.enable_private_network ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id + route_table_id = azurerm_route_table.adb-route-table[0].id } \ No newline at end of file From f0a741d676bfac79355382377d1501ec220f8c13 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 16:08:58 +0100 Subject: [PATCH 044/137] fix typo --- azurerm/modules/azurerm-adb/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 0443f693..e046b353 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -164,12 +164,12 @@ resource "azurerm_route_table" "adb-route-table" { name = "to-nat" address_prefix = "0.0.0.0/0" next_hop_type = "VirtualAppliance" - next_hop_in_ip_address = azurerm_public_ip.pip[0].id + next_hop_in_ip_address = azurerm_public_ip.pip[0].ip_address } route { name = "to-scc-relay" - address_prefix = azurerm_public_ip.pip[0].id + address_prefix = azurerm_public_ip.pip[0].ip_address next_hop_type = "Internet" } } From 614562abdfc978c94a4a1b41483a3d23a4a45335 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 16:15:58 +0100 Subject: [PATCH 045/137] add prefix/ --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index e046b353..70f8b8dd 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -169,7 +169,7 @@ resource "azurerm_route_table" "adb-route-table" { route { name = "to-scc-relay" - address_prefix = azurerm_public_ip.pip[0].ip_address + address_prefix = "${azurerm_public_ip.pip[0].ip_address}/31" next_hop_type = "Internet" } } From 42242b18e325c741a9ef1a4bd92d56f19fdbf1cc Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 16:36:43 +0100 Subject: [PATCH 046/137] remove rt --- azurerm/modules/azurerm-adb/network.tf | 36 -------------------------- 1 file changed, 36 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 70f8b8dd..d6ee6043 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -149,39 +149,3 @@ resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id nat_gateway_id = azurerm_nat_gateway.nat[0].id } - -############################################ -# ROUTE TABLES -############################################ - -resource "azurerm_route_table" "adb-route-table" { - count = var.enable_private_network ? 1 : 0 - name = var.resource_namer - location = var.resource_group_location - resource_group_name = var.resource_group_name - - route { - name = "to-nat" - address_prefix = "0.0.0.0/0" - next_hop_type = "VirtualAppliance" - next_hop_in_ip_address = azurerm_public_ip.pip[0].ip_address - } - - route { - name = "to-scc-relay" - address_prefix = "${azurerm_public_ip.pip[0].ip_address}/31" - next_hop_type = "Internet" - } -} - -resource "azurerm_subnet_route_table_association" "adb-pubic-rt-assocation" { - count = var.enable_private_network ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id - route_table_id = azurerm_route_table.adb-route-table[0].id -} - -resource "azurerm_subnet_route_table_association" "adb-private-rt-assocation" { - count = var.enable_private_network ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id - route_table_id = azurerm_route_table.adb-route-table[0].id -} \ No newline at end of file From 19bd52ec2c89e04c1ee7dd7a4caa1592f6cc4c33 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 17:38:51 +0100 Subject: [PATCH 047/137] add config for lb --- azurerm/modules/azurerm-adb/load-balancer.tf | 39 ++++++++++++++++++++ azurerm/modules/azurerm-adb/main.tf | 4 +- azurerm/modules/azurerm-adb/nat.tf | 31 ++++++++++++++++ azurerm/modules/azurerm-adb/network.tf | 36 +----------------- azurerm/modules/azurerm-adb/var.tf | 12 ++++++ 5 files changed, 85 insertions(+), 37 deletions(-) create mode 100644 azurerm/modules/azurerm-adb/load-balancer.tf create mode 100644 azurerm/modules/azurerm-adb/nat.tf diff --git a/azurerm/modules/azurerm-adb/load-balancer.tf b/azurerm/modules/azurerm-adb/load-balancer.tf new file mode 100644 index 00000000..ae8208f3 --- /dev/null +++ b/azurerm/modules/azurerm-adb/load-balancer.tf @@ -0,0 +1,39 @@ +resource "azurerm_lb" "lb" { + count = var.enable_private_network && var.create_lb ? 1 : 0 + + name = var.resource_namer + location = var.resource_group_location + resource_group_name = var.resource_group_name + + sku = "Standard" + + frontend_ip_configuration { + name = "Databricks-PIP" + public_ip_address_id = azurerm_public_ip.pip.id + } +} + +resource "azurerm_lb_outbound_rule" "lb_rule" { + count = var.enable_private_network && var.create_lb ? 1 : 0 + + name = "Databricks-LB-Outbound-Rules" + resource_group_name = var.resource_group_name + + loadbalancer_id = azurerm_lb.lb.id + protocol = "All" + enable_tcp_reset = true + allocated_outbound_ports = 1024 + idle_timeout_in_minutes = 4 + + backend_address_pool_id = azurerm_lb_backend_address_pool.lb_be_pool.id + + frontend_ip_configuration { + name = azurerm_lb.lb.frontend_ip_configuration.0.name + } +} + +resource "azurerm_lb_backend_address_pool" "lb_be_pool" { + count = var.enable_private_network && var.create_lb ? 1 : 0 + loadbalancer_id = azurerm_lb.lb.id + name = "Databricks-BE" +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index b8522fa9..b196d9fb 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -4,10 +4,10 @@ resource "azurerm_databricks_workspace" "example" { location = var.resource_group_location resource_group_name = var.resource_group_name sku = var.databricks_sku - # public_network_access_enabled = var.enable_private_network ? false : true public_network_access_enabled = var.public_network_access_enabled network_security_group_rules_required = var.network_security_group_rules_required managed_resource_group_name = "databricks-rg-${var.resource_group_name}" + load_balancer_backend_address_pool_id = var.create_lb ? azurerm_lb_backend_address_pool.lb_be_pool.id : null dynamic "custom_parameters" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) @@ -19,7 +19,7 @@ resource "azurerm_databricks_workspace" "example" { vnet_address_prefix = var.vnet_address_prefix == "" ? null : var.vnet_address_prefix public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id - nat_gateway_name = azurerm_nat_gateway.nat[0].name + nat_gateway_name = var.create_nat ? azurerm_nat_gateway.nat[0].name : null public_ip_name = azurerm_public_ip.pip[0].name } } diff --git a/azurerm/modules/azurerm-adb/nat.tf b/azurerm/modules/azurerm-adb/nat.tf new file mode 100644 index 00000000..eb904f44 --- /dev/null +++ b/azurerm/modules/azurerm-adb/nat.tf @@ -0,0 +1,31 @@ +############################################ +# NAT GATEWAY +############################################ + +resource "azurerm_nat_gateway" "nat" { + count = var.enable_private_network && var.create_nat ? 1 : 0 + name = local.nat_gatewat_name + location = var.resource_group_location + resource_group_name = var.resource_group_name + sku_name = "Standard" + idle_timeout_in_minutes = var.nat_idle_timeout + zones = ["1"] +} + +resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { + count = var.enable_private_network && var.create_nat ? 1 : 0 + nat_gateway_id = azurerm_nat_gateway.nat[0].id + public_ip_address_id = azurerm_public_ip.pip[0].id +} + +resource "azurerm_subnet_nat_gateway_association" "public_subnet_nat" { + count = var.enable_private_network && var.create_nat ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id + nat_gateway_id = azurerm_nat_gateway.nat[0].id +} + +resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { + count = var.enable_private_network && var.create_nat ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id + nat_gateway_id = azurerm_nat_gateway.nat[0].id +} diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index d6ee6043..63a57544 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -106,12 +106,6 @@ resource "azurerm_private_dns_cname_record" "cname" { record = "${var.resource_namer}.azuredatabricks.net" } - -############################################ -# NAT GATEWAY -############################################ - - resource "azurerm_public_ip" "pip" { count = var.enable_private_network ? 1 : 0 name = local.public_ip_name @@ -120,32 +114,4 @@ resource "azurerm_public_ip" "pip" { allocation_method = "Static" sku = "Standard" zones = ["1"] -} - -resource "azurerm_nat_gateway" "nat" { - count = var.enable_private_network ? 1 : 0 - name = local.nat_gatewat_name - location = var.resource_group_location - resource_group_name = var.resource_group_name - sku_name = "Standard" - idle_timeout_in_minutes = var.nat_idle_timeout - zones = ["1"] -} - -resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { - count = var.enable_private_network ? 1 : 0 - nat_gateway_id = azurerm_nat_gateway.nat[0].id - public_ip_address_id = azurerm_public_ip.pip[0].id -} - -resource "azurerm_subnet_nat_gateway_association" "public_subnet_nat" { - count = var.enable_private_network ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id - nat_gateway_id = azurerm_nat_gateway.nat[0].id -} - -resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { - count = var.enable_private_network ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id - nat_gateway_id = azurerm_nat_gateway.nat[0].id -} +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index bdc4a574..01532543 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -203,4 +203,16 @@ variable "service_endpoints" { type = list(string) default = ["Microsoft.AzureActiveDirectory", "Microsoft.KeyVault", "Microsoft.ServiceBus", "Microsoft.Sql", "Microsoft.Storage"] description = "List of Service Endpoints Enabled on the Subnet." +} + +variable "create_nat" { + type = bool + default = false + description = "Deploy Databricks with a NAT Gateway." +} + +variable "create_lb" { + type = bool + default = false + description = "Deploy Databricks with a Load Balancer." } \ No newline at end of file From e8ac7c301cd05d6035dac2f5f3adfc8ddca58e45 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 17:42:02 +0100 Subject: [PATCH 048/137] update lb name to local --- azurerm/modules/azurerm-adb/load-balancer.tf | 6 +++--- azurerm/modules/azurerm-adb/local.tf | 3 ++- azurerm/modules/azurerm-adb/main.tf | 8 ++++---- azurerm/modules/azurerm-adb/nat.tf | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/azurerm/modules/azurerm-adb/load-balancer.tf b/azurerm/modules/azurerm-adb/load-balancer.tf index ae8208f3..859cdf15 100644 --- a/azurerm/modules/azurerm-adb/load-balancer.tf +++ b/azurerm/modules/azurerm-adb/load-balancer.tf @@ -1,9 +1,9 @@ resource "azurerm_lb" "lb" { count = var.enable_private_network && var.create_lb ? 1 : 0 - name = var.resource_namer - location = var.resource_group_location - resource_group_name = var.resource_group_name + name = local.lb_name + location = var.resource_group_location + resource_group_name = var.resource_group_name sku = "Standard" diff --git a/azurerm/modules/azurerm-adb/local.tf b/azurerm/modules/azurerm-adb/local.tf index b7de44d3..d11792be 100644 --- a/azurerm/modules/azurerm-adb/local.tf +++ b/azurerm/modules/azurerm-adb/local.tf @@ -1,4 +1,5 @@ locals { public_ip_name = "${var.resource_namer}-pip" - nat_gatewat_name = "${var.resource_namer}-nat-gw" + nat_gateway_name = "${var.resource_namer}-nat-gw" + lb_name = "${var.resource_namer}-lb" } \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index b196d9fb..1bf3b7ca 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -1,9 +1,9 @@ resource "azurerm_databricks_workspace" "example" { - name = var.resource_namer - location = var.resource_group_location - resource_group_name = var.resource_group_name - sku = var.databricks_sku + name = var.resource_namer + location = var.resource_group_location + resource_group_name = var.resource_group_name + sku = var.databricks_sku public_network_access_enabled = var.public_network_access_enabled network_security_group_rules_required = var.network_security_group_rules_required managed_resource_group_name = "databricks-rg-${var.resource_group_name}" diff --git a/azurerm/modules/azurerm-adb/nat.tf b/azurerm/modules/azurerm-adb/nat.tf index eb904f44..431f6b02 100644 --- a/azurerm/modules/azurerm-adb/nat.tf +++ b/azurerm/modules/azurerm-adb/nat.tf @@ -4,7 +4,7 @@ resource "azurerm_nat_gateway" "nat" { count = var.enable_private_network && var.create_nat ? 1 : 0 - name = local.nat_gatewat_name + name = local.nat_gateway_name location = var.resource_group_location resource_group_name = var.resource_group_name sku_name = "Standard" From b37b190e08d27a312437ca90d2498abe5801e147 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 18:14:21 +0100 Subject: [PATCH 049/137] add index --- azurerm/modules/azurerm-adb/load-balancer.tf | 8 ++++---- azurerm/modules/azurerm-adb/main.tf | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azurerm/modules/azurerm-adb/load-balancer.tf b/azurerm/modules/azurerm-adb/load-balancer.tf index 859cdf15..6b9f2f46 100644 --- a/azurerm/modules/azurerm-adb/load-balancer.tf +++ b/azurerm/modules/azurerm-adb/load-balancer.tf @@ -9,7 +9,7 @@ resource "azurerm_lb" "lb" { frontend_ip_configuration { name = "Databricks-PIP" - public_ip_address_id = azurerm_public_ip.pip.id + public_ip_address_id = azurerm_public_ip.pip[0].id } } @@ -19,13 +19,13 @@ resource "azurerm_lb_outbound_rule" "lb_rule" { name = "Databricks-LB-Outbound-Rules" resource_group_name = var.resource_group_name - loadbalancer_id = azurerm_lb.lb.id + loadbalancer_id = azurerm_lb.lb[0].id protocol = "All" enable_tcp_reset = true allocated_outbound_ports = 1024 idle_timeout_in_minutes = 4 - backend_address_pool_id = azurerm_lb_backend_address_pool.lb_be_pool.id + backend_address_pool_id = azurerm_lb_backend_address_pool.lb_be_pool[0].id frontend_ip_configuration { name = azurerm_lb.lb.frontend_ip_configuration.0.name @@ -34,6 +34,6 @@ resource "azurerm_lb_outbound_rule" "lb_rule" { resource "azurerm_lb_backend_address_pool" "lb_be_pool" { count = var.enable_private_network && var.create_lb ? 1 : 0 - loadbalancer_id = azurerm_lb.lb.id + loadbalancer_id = azurerm_lb.lb[0].id name = "Databricks-BE" } \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 1bf3b7ca..232eaa10 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -7,7 +7,7 @@ resource "azurerm_databricks_workspace" "example" { public_network_access_enabled = var.public_network_access_enabled network_security_group_rules_required = var.network_security_group_rules_required managed_resource_group_name = "databricks-rg-${var.resource_group_name}" - load_balancer_backend_address_pool_id = var.create_lb ? azurerm_lb_backend_address_pool.lb_be_pool.id : null + load_balancer_backend_address_pool_id = var.create_lb ? azurerm_lb_backend_address_pool.lb_be_pool[0].id : null dynamic "custom_parameters" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) From d2b74fc5a79bbdf1ed707979ec76de6bc6c41d8c Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 18:24:24 +0100 Subject: [PATCH 050/137] update --- azurerm/modules/azurerm-adb/load-balancer.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/load-balancer.tf b/azurerm/modules/azurerm-adb/load-balancer.tf index 6b9f2f46..ff7cb837 100644 --- a/azurerm/modules/azurerm-adb/load-balancer.tf +++ b/azurerm/modules/azurerm-adb/load-balancer.tf @@ -28,7 +28,7 @@ resource "azurerm_lb_outbound_rule" "lb_rule" { backend_address_pool_id = azurerm_lb_backend_address_pool.lb_be_pool[0].id frontend_ip_configuration { - name = azurerm_lb.lb.frontend_ip_configuration.0.name + name = azurerm_lb.lb[0].frontend_ip_configuration[0].name } } From 64cb044f9fc66d27a86c9842a920b1e63da13bd6 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 9 Jun 2023 18:28:48 +0100 Subject: [PATCH 051/137] remove param --- azurerm/modules/azurerm-adb/load-balancer.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/load-balancer.tf b/azurerm/modules/azurerm-adb/load-balancer.tf index ff7cb837..ecaab3cc 100644 --- a/azurerm/modules/azurerm-adb/load-balancer.tf +++ b/azurerm/modules/azurerm-adb/load-balancer.tf @@ -17,7 +17,6 @@ resource "azurerm_lb_outbound_rule" "lb_rule" { count = var.enable_private_network && var.create_lb ? 1 : 0 name = "Databricks-LB-Outbound-Rules" - resource_group_name = var.resource_group_name loadbalancer_id = azurerm_lb.lb[0].id protocol = "All" From 26a3f874ac55edbef283df328eeadc77b7597bcc Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Sun, 11 Jun 2023 18:01:30 +0100 Subject: [PATCH 052/137] temp comment out all params --- azurerm/modules/azurerm-adb/main.tf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 232eaa10..27b6fe40 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -13,14 +13,14 @@ resource "azurerm_databricks_workspace" "example" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) content { no_public_ip = true - public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name - private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name - virtual_network_id = data.azurerm_virtual_network.vnet[0].id - vnet_address_prefix = var.vnet_address_prefix == "" ? null : var.vnet_address_prefix - public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id - private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id - nat_gateway_name = var.create_nat ? azurerm_nat_gateway.nat[0].name : null - public_ip_name = azurerm_public_ip.pip[0].name + # public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name + # private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name + # virtual_network_id = data.azurerm_virtual_network.vnet[0].id + # vnet_address_prefix = var.vnet_address_prefix == "" ? null : var.vnet_address_prefix + # public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id + # private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id + # nat_gateway_name = var.create_nat ? azurerm_nat_gateway.nat[0].name : null + # public_ip_name = azurerm_public_ip.pip[0].name } } From 5505b80fbcf541d006c2e7155e328516b7e5ca1a Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Sun, 11 Jun 2023 18:18:41 +0100 Subject: [PATCH 053/137] comment out rules temp --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 27b6fe40..19a37ded 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -5,7 +5,7 @@ resource "azurerm_databricks_workspace" "example" { resource_group_name = var.resource_group_name sku = var.databricks_sku public_network_access_enabled = var.public_network_access_enabled - network_security_group_rules_required = var.network_security_group_rules_required + # network_security_group_rules_required = var.network_security_group_rules_required managed_resource_group_name = "databricks-rg-${var.resource_group_name}" load_balancer_backend_address_pool_id = var.create_lb ? azurerm_lb_backend_address_pool.lb_be_pool[0].id : null From 6243cc6e24fd291bf80e7c194cff796deaebf8d5 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Sun, 11 Jun 2023 19:43:52 +0100 Subject: [PATCH 054/137] temp commetn out pe --- azurerm/modules/azurerm-adb/network.tf | 60 +++++++++++++------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 63a57544..befcd4e2 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -75,36 +75,36 @@ resource "azurerm_subnet_network_security_group_association" "public" { # PRIVATE ENDPOINT ############################################ -resource "azurerm_private_endpoint" "databricks" { - count = var.enable_private_network ? 1 : 0 - name = "${var.resource_namer}-pe-databricks" - location = var.resource_group_location - resource_group_name = var.resource_group_name - subnet_id = data.azurerm_subnet.pe_subnet[0].id - - private_service_connection { - name = "${var.resource_namer}-psc" - is_manual_connection = false - private_connection_resource_id = azurerm_databricks_workspace.example.id - subresource_names = ["databricks_ui_api"] - } -} - -resource "azurerm_private_dns_zone" "dns" { - count = var.enable_private_network ? 1 : 0 - depends_on = [azurerm_private_endpoint.databricks] - name = "privatelink.azuredatabricks.net" - resource_group_name = var.resource_group_name -} - -resource "azurerm_private_dns_cname_record" "cname" { - count = var.enable_private_network ? 1 : 0 - name = azurerm_databricks_workspace.example.workspace_url - zone_name = azurerm_private_dns_zone.dns[0].name - resource_group_name = var.resource_group_name - ttl = var.dns_record_ttl - record = "${var.resource_namer}.azuredatabricks.net" -} +# resource "azurerm_private_endpoint" "databricks" { +# count = var.enable_private_network ? 1 : 0 +# name = "${var.resource_namer}-pe-databricks" +# location = var.resource_group_location +# resource_group_name = var.resource_group_name +# subnet_id = data.azurerm_subnet.pe_subnet[0].id + +# private_service_connection { +# name = "${var.resource_namer}-psc" +# is_manual_connection = false +# private_connection_resource_id = azurerm_databricks_workspace.example.id +# subresource_names = ["databricks_ui_api"] +# } +# } + +# resource "azurerm_private_dns_zone" "dns" { +# count = var.enable_private_network ? 1 : 0 +# depends_on = [azurerm_private_endpoint.databricks] +# name = "privatelink.azuredatabricks.net" +# resource_group_name = var.resource_group_name +# } + +# resource "azurerm_private_dns_cname_record" "cname" { +# count = var.enable_private_network ? 1 : 0 +# name = azurerm_databricks_workspace.example.workspace_url +# zone_name = azurerm_private_dns_zone.dns[0].name +# resource_group_name = var.resource_group_name +# ttl = var.dns_record_ttl +# record = "${var.resource_namer}.azuredatabricks.net" +# } resource "azurerm_public_ip" "pip" { count = var.enable_private_network ? 1 : 0 From 964e0b0291bee01319c26bdd01524037d955a417 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Sun, 11 Jun 2023 19:54:22 +0100 Subject: [PATCH 055/137] temp commetn out pe --- azurerm/modules/azurerm-adb/data.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index e0ac0860..d84655f5 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -37,8 +37,8 @@ data "azurerm_subnet" "pe_subnet" { resource_group_name = var.vnet_resource_group } -data "azurerm_databricks_workspace_private_endpoint_connection" "example" { - count = var.enable_private_network ? 1 : 0 - workspace_id = azurerm_databricks_workspace.example.id - private_endpoint_id = azurerm_private_endpoint.databricks[0].id -} \ No newline at end of file +# data "azurerm_databricks_workspace_private_endpoint_connection" "example" { +# count = var.enable_private_network ? 1 : 0 +# workspace_id = azurerm_databricks_workspace.example.id +# private_endpoint_id = azurerm_private_endpoint.databricks[0].id +# } \ No newline at end of file From acab715c38f2649f1638455e88680e2b9360dede Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Sun, 11 Jun 2023 20:01:57 +0100 Subject: [PATCH 056/137] temp commetn out pe --- azurerm/modules/azurerm-adb/output.tf | 68 +++++++++++++-------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/azurerm/modules/azurerm-adb/output.tf b/azurerm/modules/azurerm-adb/output.tf index 2154367b..11dfa0d1 100644 --- a/azurerm/modules/azurerm-adb/output.tf +++ b/azurerm/modules/azurerm-adb/output.tf @@ -12,37 +12,37 @@ output "databricks_hosturl" { # PRIVATE ENDPOINT ############################################ -output "databricks_workspace_private_endpoint_connection_workspace_id" { - # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].workspace_id -} - -output "databricks_workspace_private_endpoint_connection_private_endpoint_id" { - # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].private_endpoint_id -} - -output "databricks_workspace_private_endpoint_connection_name" { - # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.name -} - -output "databricks_workspace_private_endpoint_connection_workspace_private_endpoint_id" { - # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.workspace_private_endpoint_id -} - -output "databricks_workspace_private_endpoint_connection_status" { - # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.status -} - -output "databricks_workspace_private_endpoint_connection_description" { - # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.description -} - -output "databricks_workspace_private_endpoint_connection_action_required" { - # count = var.enable_private_network ? 1 : 0 - value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.action_required -} \ No newline at end of file +# output "databricks_workspace_private_endpoint_connection_workspace_id" { +# # count = var.enable_private_network ? 1 : 0 +# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].workspace_id +# } + +# output "databricks_workspace_private_endpoint_connection_private_endpoint_id" { +# # count = var.enable_private_network ? 1 : 0 +# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].private_endpoint_id +# } + +# output "databricks_workspace_private_endpoint_connection_name" { +# # count = var.enable_private_network ? 1 : 0 +# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.name +# } + +# output "databricks_workspace_private_endpoint_connection_workspace_private_endpoint_id" { +# # count = var.enable_private_network ? 1 : 0 +# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.workspace_private_endpoint_id +# } + +# output "databricks_workspace_private_endpoint_connection_status" { +# # count = var.enable_private_network ? 1 : 0 +# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.status +# } + +# output "databricks_workspace_private_endpoint_connection_description" { +# # count = var.enable_private_network ? 1 : 0 +# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.description +# } + +# output "databricks_workspace_private_endpoint_connection_action_required" { +# # count = var.enable_private_network ? 1 : 0 +# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.action_required +# } \ No newline at end of file From 3f1d18cca519a099b8af3f5f4bf1423ad5b8936a Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 08:37:22 +0100 Subject: [PATCH 057/137] add configurable managed vnet --- azurerm/modules/azurerm-adb/data.tf | 16 ++-- azurerm/modules/azurerm-adb/load-balancer.tf | 8 +- azurerm/modules/azurerm-adb/main.tf | 18 ++-- azurerm/modules/azurerm-adb/nat.tf | 8 +- azurerm/modules/azurerm-adb/network.tf | 87 ++++++++++++-------- azurerm/modules/azurerm-adb/var.tf | 6 ++ 6 files changed, 82 insertions(+), 61 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index d84655f5..ae11daef 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -17,28 +17,28 @@ data "azurerm_virtual_network" "vnet" { } data "azurerm_subnet" "public_subnet" { - count = var.enable_private_network == true && var.create_subnets == false ? 1 : 0 + count = var.enable_private_network == true && var.create_subnets == false && var.managed_vnet == false ? 1 : 0 name = var.public_subnet_name virtual_network_name = var.vnet_name resource_group_name = var.vnet_resource_group } data "azurerm_subnet" "private_subnet" { - count = var.enable_private_network == true && var.create_subnets == false ? 1 : 0 + count = var.enable_private_network == true && var.create_subnets == false && var.managed_vnet == false ? 1 : 0 name = var.private_subnet_name virtual_network_name = var.vnet_name resource_group_name = var.vnet_resource_group } data "azurerm_subnet" "pe_subnet" { - count = var.enable_private_network ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = var.pe_subnet_name virtual_network_name = var.vnet_name resource_group_name = var.vnet_resource_group } -# data "azurerm_databricks_workspace_private_endpoint_connection" "example" { -# count = var.enable_private_network ? 1 : 0 -# workspace_id = azurerm_databricks_workspace.example.id -# private_endpoint_id = azurerm_private_endpoint.databricks[0].id -# } \ No newline at end of file +data "azurerm_databricks_workspace_private_endpoint_connection" "example" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + workspace_id = azurerm_databricks_workspace.example.id + private_endpoint_id = azurerm_private_endpoint.databricks[0].id +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/load-balancer.tf b/azurerm/modules/azurerm-adb/load-balancer.tf index ecaab3cc..b4f573eb 100644 --- a/azurerm/modules/azurerm-adb/load-balancer.tf +++ b/azurerm/modules/azurerm-adb/load-balancer.tf @@ -1,5 +1,5 @@ resource "azurerm_lb" "lb" { - count = var.enable_private_network && var.create_lb ? 1 : 0 + count = var.enable_private_network && var.create_lb && var.managed_vnet == false ? 1 : 0 name = local.lb_name location = var.resource_group_location @@ -14,9 +14,9 @@ resource "azurerm_lb" "lb" { } resource "azurerm_lb_outbound_rule" "lb_rule" { - count = var.enable_private_network && var.create_lb ? 1 : 0 + count = var.enable_private_network && var.create_lb && var.managed_vnet == false ? 1 : 0 - name = "Databricks-LB-Outbound-Rules" + name = "Databricks-LB-Outbound-Rules" loadbalancer_id = azurerm_lb.lb[0].id protocol = "All" @@ -32,7 +32,7 @@ resource "azurerm_lb_outbound_rule" "lb_rule" { } resource "azurerm_lb_backend_address_pool" "lb_be_pool" { - count = var.enable_private_network && var.create_lb ? 1 : 0 + count = var.enable_private_network && var.create_lb && var.managed_vnet == false ? 1 : 0 loadbalancer_id = azurerm_lb.lb[0].id name = "Databricks-BE" } \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 19a37ded..64f8bf9a 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -5,7 +5,7 @@ resource "azurerm_databricks_workspace" "example" { resource_group_name = var.resource_group_name sku = var.databricks_sku public_network_access_enabled = var.public_network_access_enabled - # network_security_group_rules_required = var.network_security_group_rules_required + network_security_group_rules_required = var.managed_vnet ? null : var.network_security_group_rules_required managed_resource_group_name = "databricks-rg-${var.resource_group_name}" load_balancer_backend_address_pool_id = var.create_lb ? azurerm_lb_backend_address_pool.lb_be_pool[0].id : null @@ -13,14 +13,14 @@ resource "azurerm_databricks_workspace" "example" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) content { no_public_ip = true - # public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name - # private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name - # virtual_network_id = data.azurerm_virtual_network.vnet[0].id - # vnet_address_prefix = var.vnet_address_prefix == "" ? null : var.vnet_address_prefix - # public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id - # private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id - # nat_gateway_name = var.create_nat ? azurerm_nat_gateway.nat[0].name : null - # public_ip_name = azurerm_public_ip.pip[0].name + public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) + private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) + virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id + vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) + public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id + private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id + nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) + public_ip_name = var.managed_vnet ? null : azurerm_public_ip.pip[0].name } } diff --git a/azurerm/modules/azurerm-adb/nat.tf b/azurerm/modules/azurerm-adb/nat.tf index 431f6b02..d403056a 100644 --- a/azurerm/modules/azurerm-adb/nat.tf +++ b/azurerm/modules/azurerm-adb/nat.tf @@ -3,7 +3,7 @@ ############################################ resource "azurerm_nat_gateway" "nat" { - count = var.enable_private_network && var.create_nat ? 1 : 0 + count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 name = local.nat_gateway_name location = var.resource_group_location resource_group_name = var.resource_group_name @@ -13,19 +13,19 @@ resource "azurerm_nat_gateway" "nat" { } resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { - count = var.enable_private_network && var.create_nat ? 1 : 0 + count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 nat_gateway_id = azurerm_nat_gateway.nat[0].id public_ip_address_id = azurerm_public_ip.pip[0].id } resource "azurerm_subnet_nat_gateway_association" "public_subnet_nat" { - count = var.enable_private_network && var.create_nat ? 1 : 0 + count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id nat_gateway_id = azurerm_nat_gateway.nat[0].id } resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { - count = var.enable_private_network && var.create_nat ? 1 : 0 + count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id nat_gateway_id = azurerm_nat_gateway.nat[0].id } diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index befcd4e2..d5707770 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -3,7 +3,7 @@ ############################################ resource "azurerm_subnet" "public_subnet" { - count = var.enable_private_network == true && var.create_subnets == true ? 1 : 0 + count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 name = var.public_subnet_name resource_group_name = var.vnet_resource_group @@ -26,7 +26,7 @@ resource "azurerm_subnet" "public_subnet" { } resource "azurerm_subnet" "private_subnet" { - count = var.enable_private_network == true && var.create_subnets == true ? 1 : 0 + count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 name = var.private_subnet_name resource_group_name = var.vnet_resource_group @@ -53,20 +53,35 @@ resource "azurerm_subnet" "private_subnet" { ############################################ resource "azurerm_network_security_group" "nsg" { - count = var.enable_private_network ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "${var.resource_namer}-nsg-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name } +resource "azurerm_network_security_rule" "worker" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "DatabricksWorkerToWorker" + priority = 200 + direction = "Inbound" + access = "Allow" + protocol = "Any" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "*" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.nsg.name +} + resource "azurerm_subnet_network_security_group_association" "private" { - count = var.enable_private_network ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id network_security_group_id = azurerm_network_security_group.nsg[0].id } resource "azurerm_subnet_network_security_group_association" "public" { - count = var.enable_private_network ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id network_security_group_id = azurerm_network_security_group.nsg[0].id } @@ -75,39 +90,39 @@ resource "azurerm_subnet_network_security_group_association" "public" { # PRIVATE ENDPOINT ############################################ -# resource "azurerm_private_endpoint" "databricks" { -# count = var.enable_private_network ? 1 : 0 -# name = "${var.resource_namer}-pe-databricks" -# location = var.resource_group_location -# resource_group_name = var.resource_group_name -# subnet_id = data.azurerm_subnet.pe_subnet[0].id - -# private_service_connection { -# name = "${var.resource_namer}-psc" -# is_manual_connection = false -# private_connection_resource_id = azurerm_databricks_workspace.example.id -# subresource_names = ["databricks_ui_api"] -# } -# } - -# resource "azurerm_private_dns_zone" "dns" { -# count = var.enable_private_network ? 1 : 0 -# depends_on = [azurerm_private_endpoint.databricks] -# name = "privatelink.azuredatabricks.net" -# resource_group_name = var.resource_group_name -# } - -# resource "azurerm_private_dns_cname_record" "cname" { -# count = var.enable_private_network ? 1 : 0 -# name = azurerm_databricks_workspace.example.workspace_url -# zone_name = azurerm_private_dns_zone.dns[0].name -# resource_group_name = var.resource_group_name -# ttl = var.dns_record_ttl -# record = "${var.resource_namer}.azuredatabricks.net" -# } +resource "azurerm_private_endpoint" "databricks" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "${var.resource_namer}-pe-databricks" + location = var.resource_group_location + resource_group_name = var.resource_group_name + subnet_id = data.azurerm_subnet.pe_subnet[0].id + + private_service_connection { + name = "${var.resource_namer}-psc" + is_manual_connection = false + private_connection_resource_id = azurerm_databricks_workspace.example.id + subresource_names = ["databricks_ui_api"] + } +} + +resource "azurerm_private_dns_zone" "dns" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + depends_on = [azurerm_private_endpoint.databricks] + name = "privatelink.azuredatabricks.net" + resource_group_name = var.resource_group_name +} + +resource "azurerm_private_dns_cname_record" "cname" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = azurerm_databricks_workspace.example.workspace_url + zone_name = azurerm_private_dns_zone.dns[0].name + resource_group_name = var.resource_group_name + ttl = var.dns_record_ttl + record = "${var.resource_namer}.azuredatabricks.net" +} resource "azurerm_public_ip" "pip" { - count = var.enable_private_network ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = local.public_ip_name location = var.resource_group_location resource_group_name = var.resource_group_name diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 01532543..d561dd75 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -215,4 +215,10 @@ variable "create_lb" { type = bool default = false description = "Deploy Databricks with a Load Balancer." +} + +variable "managed_vnet" { + type = bool + default = false + description = "Used to determine if Databricks is created in a managed vnet configuration." } \ No newline at end of file From 4308143770440ced10deaa741ab0fca4f113d7f4 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 10:16:37 +0100 Subject: [PATCH 058/137] update priority --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index d5707770..7aec9051 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -62,7 +62,7 @@ resource "azurerm_network_security_group" "nsg" { resource "azurerm_network_security_rule" "worker" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "DatabricksWorkerToWorker" - priority = 200 + priority = 100 direction = "Inbound" access = "Allow" protocol = "Any" From 22eb1009cfcd0a4f3a0fd4b4f76c8505bdf1044b Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 10:55:45 +0100 Subject: [PATCH 059/137] update vnet id --- azurerm/modules/azurerm-adb/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 64f8bf9a..e102eeb6 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -15,7 +15,8 @@ resource "azurerm_databricks_workspace" "example" { no_public_ip = true public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) - virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id + # virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id + virtual_network_id = data.azurerm_virtual_network.vnet[0].id vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id From 46f204d7aea1dc675812448588a4150029a10b65 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 11:02:34 +0100 Subject: [PATCH 060/137] update index --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 7aec9051..391d0a11 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -71,7 +71,7 @@ resource "azurerm_network_security_rule" "worker" { source_address_prefix = "VirtualNetwork" destination_address_prefix = "*" resource_group_name = var.resource_group_name - network_security_group_name = azurerm_network_security_group.nsg.name + network_security_group_name = azurerm_network_security_group.nsg[0].name } resource "azurerm_subnet_network_security_group_association" "private" { From 1796fd72d58f9bd8b581d507a952027887a180c3 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 11:02:49 +0100 Subject: [PATCH 061/137] temp comment out nsg --- azurerm/modules/azurerm-adb/network.tf | 66 +++++++++++++------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 391d0a11..446eee79 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -52,39 +52,39 @@ resource "azurerm_subnet" "private_subnet" { # NSG ############################################ -resource "azurerm_network_security_group" "nsg" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = "${var.resource_namer}-nsg-databricks" - location = var.resource_group_location - resource_group_name = var.resource_group_name -} - -resource "azurerm_network_security_rule" "worker" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = "DatabricksWorkerToWorker" - priority = 100 - direction = "Inbound" - access = "Allow" - protocol = "Any" - source_port_range = "*" - destination_port_range = "*" - source_address_prefix = "VirtualNetwork" - destination_address_prefix = "*" - resource_group_name = var.resource_group_name - network_security_group_name = azurerm_network_security_group.nsg[0].name -} - -resource "azurerm_subnet_network_security_group_association" "private" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id - network_security_group_id = azurerm_network_security_group.nsg[0].id -} - -resource "azurerm_subnet_network_security_group_association" "public" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id - network_security_group_id = azurerm_network_security_group.nsg[0].id -} +# resource "azurerm_network_security_group" "nsg" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# name = "${var.resource_namer}-nsg-databricks" +# location = var.resource_group_location +# resource_group_name = var.resource_group_name +# } + +# resource "azurerm_network_security_rule" "worker" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# name = "DatabricksWorkerToWorker" +# priority = 100 +# direction = "Inbound" +# access = "Allow" +# protocol = "Any" +# source_port_range = "*" +# destination_port_range = "*" +# source_address_prefix = "VirtualNetwork" +# destination_address_prefix = "*" +# resource_group_name = var.resource_group_name +# network_security_group_name = azurerm_network_security_group.nsg[0].name +# } + +# resource "azurerm_subnet_network_security_group_association" "private" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id +# network_security_group_id = azurerm_network_security_group.nsg[0].id +# } + +# resource "azurerm_subnet_network_security_group_association" "public" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id +# network_security_group_id = azurerm_network_security_group.nsg[0].id +# } ############################################ # PRIVATE ENDPOINT From 3f07aa253cbb4d289ee666b504e56eb5cbab2ea9 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 11:08:57 +0100 Subject: [PATCH 062/137] remove reference to nsg ass --- azurerm/modules/azurerm-adb/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index e102eeb6..dd00f15d 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -18,8 +18,8 @@ resource "azurerm_databricks_workspace" "example" { # virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id virtual_network_id = data.azurerm_virtual_network.vnet[0].id vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) - public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id - private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id + # public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id + # private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) public_ip_name = var.managed_vnet ? null : azurerm_public_ip.pip[0].name } From d2f152848596d44b257f33d0528eeb4e8b77753f Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 11:23:44 +0100 Subject: [PATCH 063/137] temp fix subnet names --- azurerm/modules/azurerm-adb/main.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index dd00f15d..a4fa612d 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -13,8 +13,10 @@ resource "azurerm_databricks_workspace" "example" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) content { no_public_ip = true - public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) - private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) + # public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) + # private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) + public_subnet_name = "databricks-public-test" + private_subnet_name = "databricks-private-test" # virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id virtual_network_id = data.azurerm_virtual_network.vnet[0].id vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) From 4f6c74ebb610afa6b121bed5ecb612b8e462ba11 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 11:29:51 +0100 Subject: [PATCH 064/137] add nsg ass to custom param --- azurerm/modules/azurerm-adb/main.tf | 2 + azurerm/modules/azurerm-adb/network.tf | 66 +++++++++++++------------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index a4fa612d..30ca884d 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -22,6 +22,8 @@ resource "azurerm_databricks_workspace" "example" { vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) # public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id # private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id + public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id + private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) public_ip_name = var.managed_vnet ? null : azurerm_public_ip.pip[0].name } diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 446eee79..391d0a11 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -52,39 +52,39 @@ resource "azurerm_subnet" "private_subnet" { # NSG ############################################ -# resource "azurerm_network_security_group" "nsg" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# name = "${var.resource_namer}-nsg-databricks" -# location = var.resource_group_location -# resource_group_name = var.resource_group_name -# } - -# resource "azurerm_network_security_rule" "worker" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# name = "DatabricksWorkerToWorker" -# priority = 100 -# direction = "Inbound" -# access = "Allow" -# protocol = "Any" -# source_port_range = "*" -# destination_port_range = "*" -# source_address_prefix = "VirtualNetwork" -# destination_address_prefix = "*" -# resource_group_name = var.resource_group_name -# network_security_group_name = azurerm_network_security_group.nsg[0].name -# } - -# resource "azurerm_subnet_network_security_group_association" "private" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id -# network_security_group_id = azurerm_network_security_group.nsg[0].id -# } - -# resource "azurerm_subnet_network_security_group_association" "public" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id -# network_security_group_id = azurerm_network_security_group.nsg[0].id -# } +resource "azurerm_network_security_group" "nsg" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "${var.resource_namer}-nsg-databricks" + location = var.resource_group_location + resource_group_name = var.resource_group_name +} + +resource "azurerm_network_security_rule" "worker" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "DatabricksWorkerToWorker" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Any" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "*" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.nsg[0].name +} + +resource "azurerm_subnet_network_security_group_association" "private" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id + network_security_group_id = azurerm_network_security_group.nsg[0].id +} + +resource "azurerm_subnet_network_security_group_association" "public" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id + network_security_group_id = azurerm_network_security_group.nsg[0].id +} ############################################ # PRIVATE ENDPOINT From 4718ad787f9b1760790ee19dbb051db064caa250 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 11:38:07 +0100 Subject: [PATCH 065/137] update nsg rule --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 391d0a11..6ba4630c 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -65,7 +65,7 @@ resource "azurerm_network_security_rule" "worker" { priority = 100 direction = "Inbound" access = "Allow" - protocol = "Any" + protocol = "*" source_port_range = "*" destination_port_range = "*" source_address_prefix = "VirtualNetwork" From c5a609d50718ca6bdbb6a231aef889e39c5fb2a2 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 11:43:30 +0100 Subject: [PATCH 066/137] temp comment out count --- azurerm/modules/azurerm-adb/main.tf | 4 ++-- azurerm/modules/azurerm-adb/network.tf | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 30ca884d..ba8a3c7d 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -22,8 +22,8 @@ resource "azurerm_databricks_workspace" "example" { vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) # public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id # private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id - public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public[0].id - private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private[0].id + public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id + private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private.id nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) public_ip_name = var.managed_vnet ? null : azurerm_public_ip.pip[0].name } diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 6ba4630c..75a5f66d 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -53,14 +53,14 @@ resource "azurerm_subnet" "private_subnet" { ############################################ resource "azurerm_network_security_group" "nsg" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "${var.resource_namer}-nsg-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name } resource "azurerm_network_security_rule" "worker" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "DatabricksWorkerToWorker" priority = 100 direction = "Inbound" @@ -75,13 +75,13 @@ resource "azurerm_network_security_rule" "worker" { } resource "azurerm_subnet_network_security_group_association" "private" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id network_security_group_id = azurerm_network_security_group.nsg[0].id } resource "azurerm_subnet_network_security_group_association" "public" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id network_security_group_id = azurerm_network_security_group.nsg[0].id } From eee25b2bb237554c316ae9057640f5cb40137205 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 11:46:37 +0100 Subject: [PATCH 067/137] update index --- azurerm/modules/azurerm-adb/main.tf | 10 +++---- azurerm/modules/azurerm-adb/network.tf | 39 ++++++++++++++------------ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index ba8a3c7d..c3c3d9ec 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -12,14 +12,14 @@ resource "azurerm_databricks_workspace" "example" { dynamic "custom_parameters" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) content { - no_public_ip = true + no_public_ip = true # public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) # private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) - public_subnet_name = "databricks-public-test" - private_subnet_name = "databricks-private-test" + public_subnet_name = "databricks-public-test" + private_subnet_name = "databricks-private-test" # virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id - virtual_network_id = data.azurerm_virtual_network.vnet[0].id - vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) + virtual_network_id = data.azurerm_virtual_network.vnet[0].id + vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) # public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id # private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 75a5f66d..34f1a4ca 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -53,37 +53,40 @@ resource "azurerm_subnet" "private_subnet" { ############################################ resource "azurerm_network_security_group" "nsg" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + # count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "${var.resource_namer}-nsg-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name } resource "azurerm_network_security_rule" "worker" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = "DatabricksWorkerToWorker" - priority = 100 - direction = "Inbound" - access = "Allow" - protocol = "*" - source_port_range = "*" - destination_port_range = "*" - source_address_prefix = "VirtualNetwork" - destination_address_prefix = "*" - resource_group_name = var.resource_group_name - network_security_group_name = azurerm_network_security_group.nsg[0].name + # count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "DatabricksWorkerToWorker" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "*" + resource_group_name = var.resource_group_name + # network_security_group_name = azurerm_network_security_group.nsg[0].name + network_security_group_name = azurerm_network_security_group.nsg.name } resource "azurerm_subnet_network_security_group_association" "private" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id - network_security_group_id = azurerm_network_security_group.nsg[0].id + # count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id + # network_security_group_id = azurerm_network_security_group.nsg[0].id + network_security_group_id = azurerm_network_security_group.nsg.id } resource "azurerm_subnet_network_security_group_association" "public" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + # count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id - network_security_group_id = azurerm_network_security_group.nsg[0].id + network_security_group_id = azurerm_network_security_group.nsg.id + # network_security_group_id = azurerm_network_security_group.nsg[0].id } ############################################ From fffa8c55bf6e272f7438640d5f2bae0c2b245d98 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 12:28:32 +0100 Subject: [PATCH 068/137] update" --- azurerm/modules/azurerm-adb/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index c3c3d9ec..64de5164 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -15,8 +15,8 @@ resource "azurerm_databricks_workspace" "example" { no_public_ip = true # public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) # private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) - public_subnet_name = "databricks-public-test" - private_subnet_name = "databricks-private-test" + public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : null + private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : null # virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id virtual_network_id = data.azurerm_virtual_network.vnet[0].id vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) From 3b3ed043d515f149c297d4069ee41c02586ec378 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 12:29:30 +0100 Subject: [PATCH 069/137] temp comment out count on subnets --- azurerm/modules/azurerm-adb/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 34f1a4ca..3ebedfe7 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -3,7 +3,7 @@ ############################################ resource "azurerm_subnet" "public_subnet" { - count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 +# count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 name = var.public_subnet_name resource_group_name = var.vnet_resource_group @@ -26,7 +26,7 @@ resource "azurerm_subnet" "public_subnet" { } resource "azurerm_subnet" "private_subnet" { - count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 +# count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 name = var.private_subnet_name resource_group_name = var.vnet_resource_group From 51c973a7b06ab236f141c4b1466e792985103c3d Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 12:41:54 +0100 Subject: [PATCH 070/137] revert stuff --- azurerm/modules/azurerm-adb/main.tf | 28 ++++++++--------- azurerm/modules/azurerm-adb/network.tf | 43 ++++++++++++-------------- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 64de5164..cfa09fad 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -12,20 +12,20 @@ resource "azurerm_databricks_workspace" "example" { dynamic "custom_parameters" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) content { - no_public_ip = true - # public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) - # private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) - public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : null - private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : null - # virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id - virtual_network_id = data.azurerm_virtual_network.vnet[0].id - vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) - # public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id - # private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id - public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id - private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private.id - nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) - public_ip_name = var.managed_vnet ? null : azurerm_public_ip.pip[0].name + no_public_ip = true + public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) + private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) + # public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : null + # private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : null + virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id + # virtual_network_id = data.azurerm_virtual_network.vnet[0].id + vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) + public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id + private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id + # public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id + # private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private.id + nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) + public_ip_name = var.managed_vnet ? null : azurerm_public_ip.pip[0].name } } diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 3ebedfe7..6ba4630c 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -3,7 +3,7 @@ ############################################ resource "azurerm_subnet" "public_subnet" { -# count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 name = var.public_subnet_name resource_group_name = var.vnet_resource_group @@ -26,7 +26,7 @@ resource "azurerm_subnet" "public_subnet" { } resource "azurerm_subnet" "private_subnet" { -# count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 name = var.private_subnet_name resource_group_name = var.vnet_resource_group @@ -53,40 +53,37 @@ resource "azurerm_subnet" "private_subnet" { ############################################ resource "azurerm_network_security_group" "nsg" { - # count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "${var.resource_namer}-nsg-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name } resource "azurerm_network_security_rule" "worker" { - # count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = "DatabricksWorkerToWorker" - priority = 100 - direction = "Inbound" - access = "Allow" - protocol = "*" - source_port_range = "*" - destination_port_range = "*" - source_address_prefix = "VirtualNetwork" - destination_address_prefix = "*" - resource_group_name = var.resource_group_name - # network_security_group_name = azurerm_network_security_group.nsg[0].name - network_security_group_name = azurerm_network_security_group.nsg.name + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "DatabricksWorkerToWorker" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "*" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.nsg[0].name } resource "azurerm_subnet_network_security_group_association" "private" { - # count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id - # network_security_group_id = azurerm_network_security_group.nsg[0].id - network_security_group_id = azurerm_network_security_group.nsg.id + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id + network_security_group_id = azurerm_network_security_group.nsg[0].id } resource "azurerm_subnet_network_security_group_association" "public" { - # count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id - network_security_group_id = azurerm_network_security_group.nsg.id - # network_security_group_id = azurerm_network_security_group.nsg[0].id + network_security_group_id = azurerm_network_security_group.nsg[0].id } ############################################ From 9c21a1c9700c32517a9a47aeefcf40e8add6851f Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 12:47:58 +0100 Subject: [PATCH 071/137] comment out pe --- azurerm/modules/azurerm-adb/network.tf | 60 +++++++++++++------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 6ba4630c..7daab537 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -90,36 +90,36 @@ resource "azurerm_subnet_network_security_group_association" "public" { # PRIVATE ENDPOINT ############################################ -resource "azurerm_private_endpoint" "databricks" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = "${var.resource_namer}-pe-databricks" - location = var.resource_group_location - resource_group_name = var.resource_group_name - subnet_id = data.azurerm_subnet.pe_subnet[0].id - - private_service_connection { - name = "${var.resource_namer}-psc" - is_manual_connection = false - private_connection_resource_id = azurerm_databricks_workspace.example.id - subresource_names = ["databricks_ui_api"] - } -} - -resource "azurerm_private_dns_zone" "dns" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - depends_on = [azurerm_private_endpoint.databricks] - name = "privatelink.azuredatabricks.net" - resource_group_name = var.resource_group_name -} - -resource "azurerm_private_dns_cname_record" "cname" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = azurerm_databricks_workspace.example.workspace_url - zone_name = azurerm_private_dns_zone.dns[0].name - resource_group_name = var.resource_group_name - ttl = var.dns_record_ttl - record = "${var.resource_namer}.azuredatabricks.net" -} +# resource "azurerm_private_endpoint" "databricks" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# name = "${var.resource_namer}-pe-databricks" +# location = var.resource_group_location +# resource_group_name = var.resource_group_name +# subnet_id = data.azurerm_subnet.pe_subnet[0].id + +# private_service_connection { +# name = "${var.resource_namer}-psc" +# is_manual_connection = false +# private_connection_resource_id = azurerm_databricks_workspace.example.id +# subresource_names = ["databricks_ui_api"] +# } +# } + +# resource "azurerm_private_dns_zone" "dns" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# depends_on = [azurerm_private_endpoint.databricks] +# name = "privatelink.azuredatabricks.net" +# resource_group_name = var.resource_group_name +# } + +# resource "azurerm_private_dns_cname_record" "cname" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# name = azurerm_databricks_workspace.example.workspace_url +# zone_name = azurerm_private_dns_zone.dns[0].name +# resource_group_name = var.resource_group_name +# ttl = var.dns_record_ttl +# record = "${var.resource_namer}.azuredatabricks.net" +# } resource "azurerm_public_ip" "pip" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 From b391aeec19fc3edec1552adb21d4e4de75d89fb2 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 12:55:47 +0100 Subject: [PATCH 072/137] comment out data --- azurerm/modules/azurerm-adb/data.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index ae11daef..e4167bc4 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -30,12 +30,12 @@ data "azurerm_subnet" "private_subnet" { resource_group_name = var.vnet_resource_group } -data "azurerm_subnet" "pe_subnet" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = var.pe_subnet_name - virtual_network_name = var.vnet_name - resource_group_name = var.vnet_resource_group -} +# data "azurerm_subnet" "pe_subnet" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# name = var.pe_subnet_name +# virtual_network_name = var.vnet_name +# resource_group_name = var.vnet_resource_group +# } data "azurerm_databricks_workspace_private_endpoint_connection" "example" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 From d3a16166b42d9fb36c92abbc963879f44ab420ac Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 13:04:52 +0100 Subject: [PATCH 073/137] comment out data --- azurerm/modules/azurerm-adb/data.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index e4167bc4..546cdd99 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -37,8 +37,8 @@ data "azurerm_subnet" "private_subnet" { # resource_group_name = var.vnet_resource_group # } -data "azurerm_databricks_workspace_private_endpoint_connection" "example" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - workspace_id = azurerm_databricks_workspace.example.id - private_endpoint_id = azurerm_private_endpoint.databricks[0].id -} \ No newline at end of file +# data "azurerm_databricks_workspace_private_endpoint_connection" "example" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# workspace_id = azurerm_databricks_workspace.example.id +# private_endpoint_id = azurerm_private_endpoint.databricks[0].id +# } \ No newline at end of file From 4f6cd4d831f532fd1b0255a1267d1dc4c66ef094 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 13:46:47 +0100 Subject: [PATCH 074/137] comment out rule --- azurerm/modules/azurerm-adb/network.tf | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 7daab537..5b3de9d3 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -59,20 +59,20 @@ resource "azurerm_network_security_group" "nsg" { resource_group_name = var.resource_group_name } -resource "azurerm_network_security_rule" "worker" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = "DatabricksWorkerToWorker" - priority = 100 - direction = "Inbound" - access = "Allow" - protocol = "*" - source_port_range = "*" - destination_port_range = "*" - source_address_prefix = "VirtualNetwork" - destination_address_prefix = "*" - resource_group_name = var.resource_group_name - network_security_group_name = azurerm_network_security_group.nsg[0].name -} +# resource "azurerm_network_security_rule" "worker" { +# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 +# name = "DatabricksWorkerToWorker" +# priority = 100 +# direction = "Inbound" +# access = "Allow" +# protocol = "*" +# source_port_range = "*" +# destination_port_range = "*" +# source_address_prefix = "VirtualNetwork" +# destination_address_prefix = "*" +# resource_group_name = var.resource_group_name +# network_security_group_name = azurerm_network_security_group.nsg[0].name +# } resource "azurerm_subnet_network_security_group_association" "private" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 From c193d624f90fbd8c434fb07a7f1cb7ac31bd6612 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 14:23:38 +0100 Subject: [PATCH 075/137] comment out service endpoints --- azurerm/modules/azurerm-adb/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 5b3de9d3..672236ac 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -9,7 +9,7 @@ resource "azurerm_subnet" "public_subnet" { resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name address_prefixes = var.public_subnet_prefix - service_endpoints = var.service_endpoints +# service_endpoints = var.service_endpoints delegation { name = "${var.public_subnet_name}-databricks-del" @@ -32,7 +32,7 @@ resource "azurerm_subnet" "private_subnet" { resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name address_prefixes = var.private_subnet_prefix - service_endpoints = var.service_endpoints +# service_endpoints = var.service_endpoints delegation { name = "${var.private_subnet_name}-databricks-del" From 4cecf472b4b3a9d996d5e897b95dc1e3b994c65e Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 14:45:12 +0100 Subject: [PATCH 076/137] create pr subnet and pe --- azurerm/modules/azurerm-adb/network.tf | 64 +++++++++++++++----------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 672236ac..cabcdf9a 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -48,6 +48,17 @@ resource "azurerm_subnet" "private_subnet" { } } +resource "azurerm_subnet" "pe_subnet" { + count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 + + name = "private-endpoints" + resource_group_name = var.vnet_resource_group + virtual_network_name = var.vnet_name + address_prefixes = var.private_subnet_prefix +} + + + ############################################ # NSG ############################################ @@ -90,36 +101,37 @@ resource "azurerm_subnet_network_security_group_association" "public" { # PRIVATE ENDPOINT ############################################ -# resource "azurerm_private_endpoint" "databricks" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# name = "${var.resource_namer}-pe-databricks" -# location = var.resource_group_location -# resource_group_name = var.resource_group_name +resource "azurerm_private_endpoint" "databricks" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "${var.resource_namer}-pe-databricks" + location = var.resource_group_location + resource_group_name = var.resource_group_name # subnet_id = data.azurerm_subnet.pe_subnet[0].id + subnet_id = azurerm_subnet.pe_subnet[0].id -# private_service_connection { -# name = "${var.resource_namer}-psc" -# is_manual_connection = false -# private_connection_resource_id = azurerm_databricks_workspace.example.id -# subresource_names = ["databricks_ui_api"] -# } -# } + private_service_connection { + name = "${var.resource_namer}-psc" + is_manual_connection = false + private_connection_resource_id = azurerm_databricks_workspace.example.id + subresource_names = ["databricks_ui_api"] + } +} -# resource "azurerm_private_dns_zone" "dns" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# depends_on = [azurerm_private_endpoint.databricks] -# name = "privatelink.azuredatabricks.net" -# resource_group_name = var.resource_group_name -# } +resource "azurerm_private_dns_zone" "dns" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + depends_on = [azurerm_private_endpoint.databricks] + name = "privatelink.azuredatabricks.net" + resource_group_name = var.resource_group_name +} -# resource "azurerm_private_dns_cname_record" "cname" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# name = azurerm_databricks_workspace.example.workspace_url -# zone_name = azurerm_private_dns_zone.dns[0].name -# resource_group_name = var.resource_group_name -# ttl = var.dns_record_ttl -# record = "${var.resource_namer}.azuredatabricks.net" -# } +resource "azurerm_private_dns_cname_record" "cname" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = azurerm_databricks_workspace.example.workspace_url + zone_name = azurerm_private_dns_zone.dns[0].name + resource_group_name = var.resource_group_name + ttl = var.dns_record_ttl + record = "${var.resource_namer}.azuredatabricks.net" +} resource "azurerm_public_ip" "pip" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 From 5b8019fa1ffb827688ac26a33e5b709027a9fe1d Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 14:51:26 +0100 Subject: [PATCH 077/137] update subnet range --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index cabcdf9a..64a3177a 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -54,7 +54,7 @@ resource "azurerm_subnet" "pe_subnet" { name = "private-endpoints" resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name - address_prefixes = var.private_subnet_prefix + address_prefixes = "10.12.2.0/24" } From 1d5a9eb2404d8a3611529827bafdd53a62668404 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 14:57:55 +0100 Subject: [PATCH 078/137] update prefix to list --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 64a3177a..fb9bbdcb 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -54,7 +54,7 @@ resource "azurerm_subnet" "pe_subnet" { name = "private-endpoints" resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name - address_prefixes = "10.12.2.0/24" + address_prefixes = ["10.12.2.0/24"] } From 809f52648093963cefffc483dcb0acbd12770734 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 12 Jun 2023 16:54:50 +0100 Subject: [PATCH 079/137] configure pip --- azurerm/modules/azurerm-adb/main.tf | 2 +- azurerm/modules/azurerm-adb/network.tf | 2 +- azurerm/modules/azurerm-adb/var.tf | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index cfa09fad..821aed7e 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -25,7 +25,7 @@ resource "azurerm_databricks_workspace" "example" { # public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id # private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private.id nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) - public_ip_name = var.managed_vnet ? null : azurerm_public_ip.pip[0].name + public_ip_name = var.managed_vnet ? null : (var.create_nat ? azurerm_public_ip.pip[0].name : null) } } diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index fb9bbdcb..a6502968 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -134,7 +134,7 @@ resource "azurerm_private_dns_cname_record" "cname" { } resource "azurerm_public_ip" "pip" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network && var.create_pip && var.managed_vnet == false ? 1 : 0 name = local.public_ip_name location = var.resource_group_location resource_group_name = var.resource_group_name diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index d561dd75..fed73d5f 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -221,4 +221,10 @@ variable "managed_vnet" { type = bool default = false description = "Used to determine if Databricks is created in a managed vnet configuration." +} + +variable "create_pip" { + type = bool + default = false + description = "Create Databricks with a Public IP." } \ No newline at end of file From dd528cd083896c15b96177b6354fc934d6f23184 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 07:35:08 +0100 Subject: [PATCH 080/137] update networking --- azurerm/modules/azurerm-adb/main.tf | 26 +----------- azurerm/modules/azurerm-adb/network.tf | 57 +++++++++++++++----------- azurerm/modules/azurerm-adb/var.tf | 4 +- 3 files changed, 37 insertions(+), 50 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 821aed7e..95e321ab 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -17,7 +17,7 @@ resource "azurerm_databricks_workspace" "example" { private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) # public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : null # private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : null - virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id + virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id # virtual_network_id = data.azurerm_virtual_network.vnet[0].id vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id @@ -109,27 +109,3 @@ resource "databricks_group_member" "project_users" { group_id = databricks_group.project_users[0].id member_id = each.value.id } - -# resource "azurerm_role_assignment" "network" { -# scope = data.azurerm_resource_group.vnet_rg[0].id -# role_definition_name = "Network Contributor" -# principal_id = "9a74af6f-d153-4348-988a-e2672920bee9" -# } - -# resource "azurerm_role_assignment" "dns" { -# scope = azurerm_private_dns_zone.dns[0].id -# role_definition_name = "Private DNS Zone Contributor" -# principal_id = "9a74af6f-d153-4348-988a-e2672920bee9" -# } - -# resource "azurerm_role_assignment" "network_db" { -# scope = data.databricks_current_user.db[0].external_id -# role_definition_name = "Network Contributor" -# principal_id = data.azurerm_client_config.current.client_id -# } - -# resource "azurerm_role_assignment" "dns_db" { -# scope = data.databricks_current_user.db[0].external_id -# role_definition_name = "Private DNS Zone Contributor" -# principal_id = data.azurerm_client_config.current.client_id -# } \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index a6502968..85fb2147 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -9,18 +9,15 @@ resource "azurerm_subnet" "public_subnet" { resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name address_prefixes = var.public_subnet_prefix -# service_endpoints = var.service_endpoints delegation { - name = "${var.public_subnet_name}-databricks-del" - + name = "databricks" service_delegation { + name = "Microsoft.Databricks/workspaces" actions = [ "Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action", - "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action", - ] - name = "Microsoft.Databricks/workspaces" + "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"] } } } @@ -32,33 +29,35 @@ resource "azurerm_subnet" "private_subnet" { resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name address_prefixes = var.private_subnet_prefix -# service_endpoints = var.service_endpoints - delegation { - name = "${var.private_subnet_name}-databricks-del" + enforce_private_link_endpoint_network_policies = true + enforce_private_link_service_network_policies = true + delegation { + name = "databricks" service_delegation { + name = "Microsoft.Databricks/workspaces" actions = [ "Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action", - "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action", - ] - name = "Microsoft.Databricks/workspaces" + "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"] } } + + service_endpoints = var.service_endpoints } resource "azurerm_subnet" "pe_subnet" { count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 - name = "private-endpoints" - resource_group_name = var.vnet_resource_group - virtual_network_name = var.vnet_name - address_prefixes = ["10.12.2.0/24"] + name = "private-endpoints" + resource_group_name = var.vnet_resource_group + virtual_network_name = var.vnet_name + address_prefixes = ["10.12.2.0/24"] + enforce_private_link_endpoint_network_policies = true + # private_endpoint_network_policies_enabled = true } - - ############################################ # NSG ############################################ @@ -102,24 +101,28 @@ resource "azurerm_subnet_network_security_group_association" "public" { ############################################ resource "azurerm_private_endpoint" "databricks" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "${var.resource_namer}-pe-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name -# subnet_id = data.azurerm_subnet.pe_subnet[0].id subnet_id = azurerm_subnet.pe_subnet[0].id private_service_connection { - name = "${var.resource_namer}-psc" - is_manual_connection = false + name = "${var.resource_namer}-db-uiapi" private_connection_resource_id = azurerm_databricks_workspace.example.id + is_manual_connection = false subresource_names = ["databricks_ui_api"] } + + private_dns_zone_group { + name = "private-dns-zone-uiapi" + private_dns_zone_ids = [azurerm_private_dns_zone.dns.id] + } } resource "azurerm_private_dns_zone" "dns" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - depends_on = [azurerm_private_endpoint.databricks] name = "privatelink.azuredatabricks.net" resource_group_name = var.resource_group_name } @@ -133,6 +136,14 @@ resource "azurerm_private_dns_cname_record" "cname" { record = "${var.resource_namer}.azuredatabricks.net" } +resource "azurerm_private_dns_zone_virtual_network_link" "uiapidnszonevnetlink" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "db-dns-vnet-link" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.dns.name + virtual_network_id = data.azurerm_virtual_network.vnet.id +} + resource "azurerm_public_ip" "pip" { count = var.enable_private_network && var.create_pip && var.managed_vnet == false ? 1 : 0 name = local.public_ip_name diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index fed73d5f..dfc317cb 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -224,7 +224,7 @@ variable "managed_vnet" { } variable "create_pip" { - type = bool - default = false + type = bool + default = false description = "Create Databricks with a Public IP." } \ No newline at end of file From a303b862e5f8e0b576561bd527f966473b1a2149 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 07:47:25 +0100 Subject: [PATCH 081/137] updates --- azurerm/modules/azurerm-adb/network.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 85fb2147..0d9850bf 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -117,7 +117,7 @@ resource "azurerm_private_endpoint" "databricks" { private_dns_zone_group { name = "private-dns-zone-uiapi" - private_dns_zone_ids = [azurerm_private_dns_zone.dns.id] + private_dns_zone_ids = [azurerm_private_dns_zone.dns[0].id] } } @@ -140,8 +140,8 @@ resource "azurerm_private_dns_zone_virtual_network_link" "uiapidnszonevnetlink" count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "db-dns-vnet-link" resource_group_name = var.resource_group_name - private_dns_zone_name = azurerm_private_dns_zone.dns.name - virtual_network_id = data.azurerm_virtual_network.vnet.id + private_dns_zone_name = azurerm_private_dns_zone.dns[0].name + virtual_network_id = data.azurerm_virtual_network.vnet[0].id } resource "azurerm_public_ip" "pip" { From 462c964999d0947a3bdf36cda2690743c9ea5aaf Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 07:58:58 +0100 Subject: [PATCH 082/137] update name --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 0d9850bf..d9eb7cce 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -109,7 +109,7 @@ resource "azurerm_private_endpoint" "databricks" { subnet_id = azurerm_subnet.pe_subnet[0].id private_service_connection { - name = "${var.resource_namer}-db-uiapi" + name = "${var.resource_namer}-db-pe" private_connection_resource_id = azurerm_databricks_workspace.example.id is_manual_connection = false subresource_names = ["databricks_ui_api"] From 92b041e2ec5db490a467e4808083db2129be6803 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 08:10:29 +0100 Subject: [PATCH 083/137] variable for pe prefix --- azurerm/modules/azurerm-adb/network.tf | 3 +-- azurerm/modules/azurerm-adb/var.tf | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index d9eb7cce..d8501fb7 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -53,9 +53,8 @@ resource "azurerm_subnet" "pe_subnet" { name = "private-endpoints" resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name - address_prefixes = ["10.12.2.0/24"] + address_prefixes = var.pe_subnet_prefix enforce_private_link_endpoint_network_policies = true - # private_endpoint_network_policies_enabled = true } ############################################ diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index dfc317cb..a4932ee5 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -181,6 +181,13 @@ variable "private_subnet_prefix" { } +variable "pe_subnet_prefix" { + type = list(string) + default = [] + description = "IP Address Space fo the Private Endpoints Databricks Subnet." + +} + variable "pe_subnet_name" { type = string default = "" From 9dc576991ffedb07bb050ca33b279767c53575ab Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 10:00:17 +0100 Subject: [PATCH 084/137] add datalookup for pe subnet --- azurerm/modules/azurerm-adb/network.tf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index d8501fb7..75296380 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -57,6 +57,14 @@ resource "azurerm_subnet" "pe_subnet" { enforce_private_link_endpoint_network_policies = true } +data "azurerm_subnet" "pe_subnet" { + count = var.enable_private_network == true && var.create_subnets == false && var.managed_vnet == false ? 1 : 0 + + name = "private-endpoints" + resource_group_name = var.vnet_resource_group + virtual_network_name = var.vnet_name +} + ############################################ # NSG ############################################ @@ -105,7 +113,7 @@ resource "azurerm_private_endpoint" "databricks" { name = "${var.resource_namer}-pe-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name - subnet_id = azurerm_subnet.pe_subnet[0].id + subnet_id = var.create_subnets ? azurerm_subnet.pe_subnet[0].id : data.azurerm_subnet.pe_subnet[0].id private_service_connection { name = "${var.resource_namer}-db-pe" From 38a57d5462ebdf3321a79267299b231e6731cd69 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 10:01:02 +0100 Subject: [PATCH 085/137] update pe subnet name to var --- azurerm/modules/azurerm-adb/network.tf | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 75296380..a5339b01 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -50,7 +50,7 @@ resource "azurerm_subnet" "private_subnet" { resource "azurerm_subnet" "pe_subnet" { count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 - name = "private-endpoints" + name = var.pe_subnet_name resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name address_prefixes = var.pe_subnet_prefix @@ -60,7 +60,7 @@ resource "azurerm_subnet" "pe_subnet" { data "azurerm_subnet" "pe_subnet" { count = var.enable_private_network == true && var.create_subnets == false && var.managed_vnet == false ? 1 : 0 - name = "private-endpoints" + name = var.pe_subnet_name resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name } @@ -76,21 +76,6 @@ resource "azurerm_network_security_group" "nsg" { resource_group_name = var.resource_group_name } -# resource "azurerm_network_security_rule" "worker" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# name = "DatabricksWorkerToWorker" -# priority = 100 -# direction = "Inbound" -# access = "Allow" -# protocol = "*" -# source_port_range = "*" -# destination_port_range = "*" -# source_address_prefix = "VirtualNetwork" -# destination_address_prefix = "*" -# resource_group_name = var.resource_group_name -# network_security_group_name = azurerm_network_security_group.nsg[0].name -# } - resource "azurerm_subnet_network_security_group_association" "private" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id From 0c3ffe94fd06719c3be1d3add0833c3df6781187 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 10:22:32 +0100 Subject: [PATCH 086/137] split var for creating pe subnet --- azurerm/modules/azurerm-adb/network.tf | 10 +++++----- azurerm/modules/azurerm-adb/var.tf | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index a5339b01..310325cb 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -48,7 +48,7 @@ resource "azurerm_subnet" "private_subnet" { } resource "azurerm_subnet" "pe_subnet" { - count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network == true && var.create_pe_subnet == true && var.managed_vnet == false ? 1 : 0 name = var.pe_subnet_name resource_group_name = var.vnet_resource_group @@ -58,11 +58,11 @@ resource "azurerm_subnet" "pe_subnet" { } data "azurerm_subnet" "pe_subnet" { - count = var.enable_private_network == true && var.create_subnets == false && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network == true && var.create_pe_subnet == false && var.managed_vnet == false ? 1 : 0 - name = var.pe_subnet_name - resource_group_name = var.vnet_resource_group - virtual_network_name = var.vnet_name + name = var.pe_subnet_name + resource_group_name = var.vnet_resource_group + virtual_network_name = var.vnet_name } ############################################ diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index a4932ee5..7fa3d239 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -144,6 +144,12 @@ variable "create_subnets" { description = "Set to true if you need the module to create the subnets for you." } +variable "create_pe_subnet" { + type = bool + default = false + description = "Set to true if you need the module to create the private endpoint subnet." +} + variable "vnet_name" { type = string default = "" From a7f7d811e3abd3a3d8692b175497ec388d96d60c Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 10:27:56 +0100 Subject: [PATCH 087/137] update condition --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 310325cb..296d8142 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -98,7 +98,7 @@ resource "azurerm_private_endpoint" "databricks" { name = "${var.resource_namer}-pe-databricks" location = var.resource_group_location resource_group_name = var.resource_group_name - subnet_id = var.create_subnets ? azurerm_subnet.pe_subnet[0].id : data.azurerm_subnet.pe_subnet[0].id + subnet_id = var.create_pe_subnet ? azurerm_subnet.pe_subnet[0].id : data.azurerm_subnet.pe_subnet[0].id private_service_connection { name = "${var.resource_namer}-db-pe" From f699c6f06276a09a9634f464603c6a7c4a63964c Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 10:40:33 +0100 Subject: [PATCH 088/137] update condition --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 296d8142..f109deba 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -129,7 +129,7 @@ resource "azurerm_private_dns_cname_record" "cname" { } resource "azurerm_private_dns_zone_virtual_network_link" "uiapidnszonevnetlink" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network == true && var.create_pe_subnet == true && var.managed_vnet == false ? 1 : 0 name = "db-dns-vnet-link" resource_group_name = var.resource_group_name private_dns_zone_name = azurerm_private_dns_zone.dns[0].name From f177feb2829c6efb290f099871ac2374ccd9c57d Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 11:08:37 +0100 Subject: [PATCH 089/137] update names --- azurerm/modules/azurerm-adb/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index f109deba..89c7b96d 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -108,7 +108,7 @@ resource "azurerm_private_endpoint" "databricks" { } private_dns_zone_group { - name = "private-dns-zone-uiapi" + name = "databricks_ui_api" private_dns_zone_ids = [azurerm_private_dns_zone.dns[0].id] } } @@ -130,7 +130,7 @@ resource "azurerm_private_dns_cname_record" "cname" { resource "azurerm_private_dns_zone_virtual_network_link" "uiapidnszonevnetlink" { count = var.enable_private_network == true && var.create_pe_subnet == true && var.managed_vnet == false ? 1 : 0 - name = "db-dns-vnet-link" + name = var.resource_namer resource_group_name = var.resource_group_name private_dns_zone_name = azurerm_private_dns_zone.dns[0].name virtual_network_id = data.azurerm_virtual_network.vnet[0].id From d3754826e114ec9ed56c6bdea189a675056d2fff Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 11:09:22 +0100 Subject: [PATCH 090/137] update condition --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 89c7b96d..23190b2a 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -129,7 +129,7 @@ resource "azurerm_private_dns_cname_record" "cname" { } resource "azurerm_private_dns_zone_virtual_network_link" "uiapidnszonevnetlink" { - count = var.enable_private_network == true && var.create_pe_subnet == true && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network == true && var.managed_vnet == false ? 1 : 0 name = var.resource_namer resource_group_name = var.resource_group_name private_dns_zone_name = azurerm_private_dns_zone.dns[0].name From 3c566a446e7442d5005ce15b6233449ab0541c80 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 11:09:44 +0100 Subject: [PATCH 091/137] update name --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 23190b2a..39e03e73 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -128,7 +128,7 @@ resource "azurerm_private_dns_cname_record" "cname" { record = "${var.resource_namer}.azuredatabricks.net" } -resource "azurerm_private_dns_zone_virtual_network_link" "uiapidnszonevnetlink" { +resource "azurerm_private_dns_zone_virtual_network_link" "db_dns_vnet_link" { count = var.enable_private_network == true && var.managed_vnet == false ? 1 : 0 name = var.resource_namer resource_group_name = var.resource_group_name From 0434bdf261ce2d463a4dc6f3afb8efe014ece827 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 11:40:08 +0100 Subject: [PATCH 092/137] update readme and tidy up --- azurerm/modules/azurerm-adb/README.md | Bin 14142 -> 29500 bytes azurerm/modules/azurerm-adb/data.tf | 17 ++++------- azurerm/modules/azurerm-adb/main.tf | 17 ++++------- azurerm/modules/azurerm-adb/network.tf | 9 +----- azurerm/modules/azurerm-adb/output.tf | 40 ------------------------- 5 files changed, 13 insertions(+), 70 deletions(-) diff --git a/azurerm/modules/azurerm-adb/README.md b/azurerm/modules/azurerm-adb/README.md index d51fbf5f8c65ffd11d1c57f1f5e6fb903c5003e6..c0749eed4e69d65c92bd41513adf3acb2cf92dee 100644 GIT binary patch literal 29500 zcmeI5Yj0b}5r)s_0{suH62LJK$qDk!uv5sFq-tzMmE;&f5>R(rjV!669ml!!r?-8d z9S!$#mgFHtxn>d&63KJ+GW*VDXJ&T~|NEai&0m^jy?V{P=3(=Bv(~IP{bsxQO#dG= zUpDufE#0HUr_G<6KQ-^^+eWjgJAYUC`zqO2jn(F{zHy(w2l~9Fdz)%)LnYQ!dcC3 z?a`2yRrRi4iY(D&E6L;v(0DLGe9XUz|~ zI?&&l?pf(`eLL6ZlTbdT>WxZ4X)lcGJe1hi=a-lDd+O~l)IHTFdV8j_v>baq(|s=w zF4ksGC_Gn5C|((kXro2Raq~j;PgLrK{!Z0W+PkAS9%GPZY=?m_M%`=mebIccE9Yu) zKivB;qi#2hg1duI&#-(gs-fUeI35OM+AYN~zE)XihR}#&KFSc>R9pQ0qj@5E-O}H-zHPOncD4CwD1}@4w5L(-i%R-1 z=|IkUdZANWqGVm~e$B6M2NaNc;{_u``b;iOB25;MIzFB0Khk!Q)Nozd6&>~&)Q&|@ zjoV%t4Ovrbl%l^ui#nIJND(q`Qtz$osJ=-DefHJsL4za=3QSyvV~M;Tk_>)_YK8j9 z$mu2jt2s4}U=I=I$20#JIPSIR^Be4#$L2NtGxht185vU}>`Op;?I(5PL}MvZ-4H*H04 zOjF8!NA=ND&y2|0a=mw}$NLk>DxAb?U zIv|g1f(Ba6Y=a!5WdoJk*VW}v_O{B}C@Im?y*=GK4eyaQqt@jxr`;^$Ve_LXKo4kT zRd)EXEXsXZ_BF|>S@kt}6?aw6UcKhqFz$L;JOaFrhmz^LLfBtrtu1N9J?TX0sr1z^ zy?H%&l=sKnfY+1K7E3eMeAMvM^auSk{c_oGYwPyNxwoT|KdEPQ=UKpYN8cPr(^ez{ z%*>`CD_E*h&Pwg`9;1DaGp@}vsqJIV+ST9AeU)$vFut#zC>`G$?TzXwpOyM5>9bNl@ESSrS<`zmnwh(>IYzTn+m}Y-BjIzI zJ*iV)(pC{DcTA1iUh|7ywf^gEAi1wq-fFk0 zs?p+eDb_!qrI&-e5M3Qh_U>w4!EUEs)XGo}PjolcaQc2P#DJTk>0gpZxD=xd^Cc<8 z_|}K|#J8_hj#=8gWH7LD77^P2Uqo&B@^ZcAj@s)Dwe+s^fWF&Q^edcQ636doq`QIh z-&9I`r&;m65iNcz>(EB|rl?_Lp9XvIk+Ax>`6OI(1iDju34RrRAFf_4)|=6??g&zJ63~&t8DpbD{lGR1AWzdMjus{t$D} zh3LaVwe2`o=9X%eQhh7<)cDQt4tsqb5Kd+7mQam;?JeO<^n&k;y~(lTZd)E|V{?v{ zZA5H)O&hrr$r<)|uUP~vpmX+d_EkHbhtS>wy$rE|G=(VPOlTM)ls`6N%uH6N|5? zR88Y8PD5t#%!&J{f!$aHd}l#oHq1eXxb9ha$Fn8dvJy~xxe!>@M=(z`uyo{>$;BXn zWS5aFa_Y=QNU6t&j+~WR1X-)&H_e!5#Tq^KD9!YGUpV8(y_5{5u{fv>B%_OgXZLZ< zh8%uW%n4&BL5Iu(yd*+rb|K2ejvh#x<+6{(JXMAdEcbNZCDrqPFm1Ile*Xc3Vu zM>Z>7<*dCTw3xjY1Cx^a*-)@GVAD*-&+L$1TLdgpdgp^eK7*$1|32|ZW$pKkhOIoc zW1`U>f9bEJ_<3uKCRI*%85xd3;p$pBZI9TpQBvz|{egNdi&@HN?z7n*pn4oOiu>>) zwWy1T`_)mHj}Ba&j$AYT;6=JdkmvX7xx=^zz~uU@@pbsVEDrj?irjVJTBE1FrZL^B zNE}f&8LiQg=41LH%kUy({4W%Dp#8rkl=AUi1v8hI6(MBnDT^vYK9Z}T;n}{pW7+o0 zu>XOTL)HNoOS8s}s2fuGnmBln6R}-35gm4ikjGC>V%5dKw`RR8ro9{B%mfvxsx(ZsUC(bPTuIFY(Bb1J0o`|GT#k-xidi&Un zihdWzM$JoV#;-#}yR$gRj33c78s>7D?vP_{OFfC`liEtnb>LK^W7^op+h|+pQRC@` zRNTq8CeMnz5+@bvOH6W5&T*HXl{`{i_WH1Eni89; z9rwv{M!Dz$jS1s(~DWU9kZ0X zE3GU$C%rVEJz6kt{yq6!-n#7rrTx)$oNH@}3vAam*x6|a_8Gnsf_6U5&Zpw*`2NfE z{7-&z!lftG1Ws#sM9;|zR@B%bA0y_t$C*1fidZ)v>kXBNC!*q6CT4c_iLlGtPVVKs zaF{Bx5OWSO?x~?XylxfV1zrETySd&LxQParX91TFGeJ z>!T^h*xUO^WszylV1ok4gC=>7(O5*p=7EOo$tg9h9H*7SR!>62O3sc6r>nL#`p*M? zahKTJ!lUgOPr`;8$V}>CbU~_C&vDyQn{7M9i9xKCpW|YsH9wu3Tc_HYF1HkU7TwNN zrYro|LcBg-&3$|LvuZvWb6;DZB#*WV@3`KVZXI4qRyQ5p2q=(K*cb9V*Wm5xX3BxV zL`?G1T_iBC;TT=N?B)H&)59j$*NLys%j`L{z5ad<-_GE18W1a-@8hlQ>bLWWrJQ>E z*n_eMn^!o~$)|QNTy4*}T;()wji1-oo6Ews@}!&sVW#3)6{{_5dd*Ya%byM5o(?lx zd5WDULa>vcR(+1-$s##7q72w+4}7|3I*IC5=u}$Cd40b!oj>5XrQTVZJ+l4aWR9VV z{h>$%UehRkr!B#;=+p6y-Q}@0u6TRnxH@Z_&O7sHZ;0~o9x$FH{#R|jvDxA@l*m0x z{Jk2Esn4t5$j%o>>j&h-=eU^My{Jdz4}R zd0SKJ=%NXB3fFwaE=c0-A8VYPFPxvoycQiM+mlXCM0xL`KeX#wQ?|@?lT|;JF^e|M z-b2Yrw_vHR)+49d2@}^lA4}AaN%n7PXgl+s&Ep$wt!L-I?fb@i^1a@ijD0B!+2$!_ z;Y(VY?^eF9KjZ5}+_yvA>9fiMoi*N)CXwfGx*~SW5+3Ke-0$e5?niq0_vvO1R>L>} z%KU}ww`07~T2bPZwgoz)TC;oY^GP)vuiB>L{4~t_X8n1xh{;vjy4m(pKTC;bmT+EK zYxod;<}!K!xfHj(Ah&Yu%)nKn;dD%WaW(Syf%tqRN&ZZ>6Mo+da}xX|LZ8IV?k#oU zWB1mk;h>MOHyk6+Mq}pnQ7#tu@|c!cC*9#RiDP+fv@WmTg-?^(aPB-zl>9f*UckuH;CF*%O5cZNai5^|rQG62S4I*NjOL^Lb zJ--0o982Tby}D0n+43|mOIy?g>HHaR+;M9C*>d*O7i}adP?)RC(NV%6eshR z=#?1S&T^M{TX}FM+d&OJ=XGkxcd6$*Si5W)9;CGR6Ba=wqH26$eU25lh*jOI{CH-c zGM{BEzOmV$eL5AN+MoSl{&l*#*TyuR|G*R9C2gB)d9EV1&pl?Ow1+we=G}|)>w)+I z_VCN{P;SBDn5~zfH$2nvM7R(a+m2aU@a(reSrM;>XX~Z)h!|ez#d8wK??%n&sXv}f zOU?rg;|V#mO#O792v2E42fx=HO6D1KdbZ~Pk|CpR+(*Inzv|kC+U9gO&)!4o?Wu;0 z#zwxXt0#(Yoa?MbuB%S;qHFddmWOuw&grdO8KbFOy17;zGsa#K|Nnj z8J$6>t)d{g*afUjO=2h2ars-nYy%8gcX27)3VbxZO>>CDl5o1 zazqQ0c`#30p_X;oV?BSTv>&5zO6yF6Z(7eWW<`aVFn!(&9$uspTV2O`w7rS#jLHnwZTS4HO{Anm zFxB;GQB3RBElunf)Jra;^&QRW?M&=rI+U-|%J_c1dKNt0<5yztHNopZI$S=Xk?U~> zTn+WZUTztId7&Qnx8_RF)mT6JnD?J?)ch#JM~go;C012|v*9DBKCz zU7mnzuY7ejdL;I=-&Cv0n%;L>$J*rSyyI$O>q{`r3jn`2Jpq(P!nPYEjR@Ovm)Azl@<^JUfN2YL*_*Z%mp|6{($+mj*(^p?%WtOq z*%_j_Bt2ng$N%p1w_haFn`LUrEgneBd#q+6{Vse8fW>ROYP#;#>; zENBRO@!U)JuHqd`H)r{32@!3YiDDNgDAx01UJGf=>v1KmS(D7}wOu~TEZIgvo6D@{ zD8Ew1m)}8&ab~K;uC+Ttan^a>>5ubJsk2@povS7x;d7dgxtyoV@mb!KU8QKo^F;UR zUUNGVw^JE5W0|#ub9n}r+3js0SZ*&so<7+!7wlCLUHl+`#-^^JJ z@Uq{csd2((5nkDBKTmw)iMlUE15b>f{Y;&m!AnoG)lR(m{c(PtdC^9TSqFP6WjjKx z?|O4IY}bLmaaVZS7k9eVb-qT=$xk4*e+?n+p&n=VB^~zcY}S}-y8`Q@$ZMIT rroAzqQE`1<*T)lA#zsZXf4ijgNuZQyl99(>#N;=g9;n~?uVMZV77ga_ delta 264 zcmdnw&EWS)BkBL$)LVs1qnYnZH;U E0FX*KBLDyZ diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index 546cdd99..eef6a1ea 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -30,15 +30,10 @@ data "azurerm_subnet" "private_subnet" { resource_group_name = var.vnet_resource_group } -# data "azurerm_subnet" "pe_subnet" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# name = var.pe_subnet_name -# virtual_network_name = var.vnet_name -# resource_group_name = var.vnet_resource_group -# } +data "azurerm_subnet" "pe_subnet" { + count = var.enable_private_network == true && var.create_pe_subnet == false && var.managed_vnet == false ? 1 : 0 -# data "azurerm_databricks_workspace_private_endpoint_connection" "example" { -# count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 -# workspace_id = azurerm_databricks_workspace.example.id -# private_endpoint_id = azurerm_private_endpoint.databricks[0].id -# } \ No newline at end of file + name = var.pe_subnet_name + resource_group_name = var.vnet_resource_group + virtual_network_name = var.vnet_name +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 95e321ab..12c43d66 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -12,20 +12,15 @@ resource "azurerm_databricks_workspace" "example" { dynamic "custom_parameters" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) content { - no_public_ip = true - public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) - private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) - # public_subnet_name = var.create_subnets ? azurerm_subnet.public_subnet[0].name : null - # private_subnet_name = var.create_subnets ? azurerm_subnet.private_subnet[0].name : null - virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id - # virtual_network_id = data.azurerm_virtual_network.vnet[0].id + no_public_ip = true + public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) + private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) + virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id - # public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id - # private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private.id - nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) - public_ip_name = var.managed_vnet ? null : (var.create_nat ? azurerm_public_ip.pip[0].name : null) + nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) + public_ip_name = var.managed_vnet ? null : (var.create_nat ? azurerm_public_ip.pip[0].name : null) } } diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 39e03e73..fd65551d 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -57,13 +57,6 @@ resource "azurerm_subnet" "pe_subnet" { enforce_private_link_endpoint_network_policies = true } -data "azurerm_subnet" "pe_subnet" { - count = var.enable_private_network == true && var.create_pe_subnet == false && var.managed_vnet == false ? 1 : 0 - - name = var.pe_subnet_name - resource_group_name = var.vnet_resource_group - virtual_network_name = var.vnet_name -} ############################################ # NSG @@ -129,7 +122,7 @@ resource "azurerm_private_dns_cname_record" "cname" { } resource "azurerm_private_dns_zone_virtual_network_link" "db_dns_vnet_link" { - count = var.enable_private_network == true && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network == true && var.managed_vnet == false ? 1 : 0 name = var.resource_namer resource_group_name = var.resource_group_name private_dns_zone_name = azurerm_private_dns_zone.dns[0].name diff --git a/azurerm/modules/azurerm-adb/output.tf b/azurerm/modules/azurerm-adb/output.tf index 11dfa0d1..b172ee23 100644 --- a/azurerm/modules/azurerm-adb/output.tf +++ b/azurerm/modules/azurerm-adb/output.tf @@ -6,43 +6,3 @@ output "databricks_hosturl" { description = "Azure Databricks HostUrl" value = "https://${azurerm_databricks_workspace.example.workspace_url}/" } - - -############################################ -# PRIVATE ENDPOINT -############################################ - -# output "databricks_workspace_private_endpoint_connection_workspace_id" { -# # count = var.enable_private_network ? 1 : 0 -# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].workspace_id -# } - -# output "databricks_workspace_private_endpoint_connection_private_endpoint_id" { -# # count = var.enable_private_network ? 1 : 0 -# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].private_endpoint_id -# } - -# output "databricks_workspace_private_endpoint_connection_name" { -# # count = var.enable_private_network ? 1 : 0 -# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.name -# } - -# output "databricks_workspace_private_endpoint_connection_workspace_private_endpoint_id" { -# # count = var.enable_private_network ? 1 : 0 -# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.workspace_private_endpoint_id -# } - -# output "databricks_workspace_private_endpoint_connection_status" { -# # count = var.enable_private_network ? 1 : 0 -# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.status -# } - -# output "databricks_workspace_private_endpoint_connection_description" { -# # count = var.enable_private_network ? 1 : 0 -# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.description -# } - -# output "databricks_workspace_private_endpoint_connection_action_required" { -# # count = var.enable_private_network ? 1 : 0 -# value = data.azurerm_databricks_workspace_private_endpoint_connection.example[0].connections.0.action_required -# } \ No newline at end of file From a21f56f06a2860e06e8497c2e67cfafd447a01f4 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 17:12:05 +0100 Subject: [PATCH 093/137] add nsg rule to allow databricks into vnet --- azurerm/modules/azurerm-adb/network.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index fd65551d..3252fb74 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -69,6 +69,21 @@ resource "azurerm_network_security_group" "nsg" { resource_group_name = var.resource_group_name } +resource "azurerm_network_security_rule" "nsg_rule" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "adf-db-inbound" + priority = 200 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "DataFactory.WestEurope" + destination_address_prefix = "VirtualNetwork" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.nsg[0].name +} + resource "azurerm_subnet_network_security_group_association" "private" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id From b0f3c932feff3b7c6f5cdf7527c97dcfebb6811d Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 13 Jun 2023 17:55:58 +0100 Subject: [PATCH 094/137] add nsg rules --- azurerm/modules/azurerm-adb/network.tf | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 3252fb74..a5b65059 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -84,6 +84,37 @@ resource "azurerm_network_security_rule" "nsg_rule" { network_security_group_name = azurerm_network_security_group.nsg[0].name } +resource "azurerm_network_security_rule" "aad" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "AllowAAD" + priority = 200 + direction = "Outbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "AzureActiveDirectory" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.nsg[0].name +} + +resource "azurerm_network_security_rule" "azfrontdoor" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "AllowAzureFrontDoor" + priority = 201 + direction = "Outbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "AzureFrontDoor.Frontend" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.nsg[0].name +} + + resource "azurerm_subnet_network_security_group_association" "private" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id From d91b5835243e4bb431929e64839689a277efa53b Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 14 Jun 2023 08:33:49 +0100 Subject: [PATCH 095/137] add depends on --- azurerm/modules/azurerm-adb/network.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index a5b65059..29cc206b 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -150,6 +150,8 @@ resource "azurerm_private_endpoint" "databricks" { name = "databricks_ui_api" private_dns_zone_ids = [azurerm_private_dns_zone.dns[0].id] } + + depends_on = [ azurerm_private_dns_zone.dns ] } resource "azurerm_private_dns_zone" "dns" { From ecd8b1aafcbd9c3b36367219505090fa905fcbdb Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 14 Jun 2023 08:34:26 +0100 Subject: [PATCH 096/137] add depends on --- azurerm/modules/azurerm-adb/network.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 29cc206b..4090cc83 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -70,7 +70,7 @@ resource "azurerm_network_security_group" "nsg" { } resource "azurerm_network_security_rule" "nsg_rule" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "adf-db-inbound" priority = 200 direction = "Inbound" @@ -85,7 +85,7 @@ resource "azurerm_network_security_rule" "nsg_rule" { } resource "azurerm_network_security_rule" "aad" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "AllowAAD" priority = 200 direction = "Outbound" @@ -100,7 +100,7 @@ resource "azurerm_network_security_rule" "aad" { } resource "azurerm_network_security_rule" "azfrontdoor" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "AllowAzureFrontDoor" priority = 201 direction = "Outbound" @@ -151,7 +151,7 @@ resource "azurerm_private_endpoint" "databricks" { private_dns_zone_ids = [azurerm_private_dns_zone.dns[0].id] } - depends_on = [ azurerm_private_dns_zone.dns ] + depends_on = [azurerm_databricks_workspace.example, azurerm_private_dns_zone.dns] } resource "azurerm_private_dns_zone" "dns" { From d154b7e39dbb4acb7a070d04ff86d6de20a1653b Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 14 Jun 2023 09:45:41 +0100 Subject: [PATCH 097/137] add auth pe --- azurerm/modules/azurerm-adb/network.tf | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 4090cc83..3a985a12 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -154,6 +154,28 @@ resource "azurerm_private_endpoint" "databricks" { depends_on = [azurerm_databricks_workspace.example, azurerm_private_dns_zone.dns] } +resource "azurerm_private_endpoint" "auth" { + name = "${var.resource_namer}-pe-databricks-auth" + location = var.resource_group_location + resource_group_name = var.resource_group_name + subnet_id = var.create_pe_subnet ? azurerm_subnet.pe_subnet[0].id : data.azurerm_subnet.pe_subnet[0].id + + private_service_connection { + name = "${var.resource_namer}-db-pe-auth" + private_connection_resource_id = azurerm_databricks_workspace.example.id + is_manual_connection = false + subresource_names = ["browser_authentication"] + } + + private_dns_zone_group { + name = "databricks_auth" + private_dns_zone_ids = [azurerm_private_dns_zone.dns[0].id] + } + + depends_on = [azurerm_databricks_workspace.example, azurerm_private_dns_zone.dns] + +} + resource "azurerm_private_dns_zone" "dns" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = "privatelink.azuredatabricks.net" From df5e4860dda9cf76d273e8b93fe98813cf1370f2 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 22 Jun 2023 14:44:41 +0100 Subject: [PATCH 098/137] comment out data --- azurerm/modules/azurerm-adb/data.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index eef6a1ea..d0ec3f7c 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -1,9 +1,9 @@ data "azurerm_client_config" "current" {} -data "databricks_current_user" "db" { - count = var.enable_private_network ? 1 : 0 - depends_on = [azurerm_databricks_workspace.example] -} +# data "databricks_current_user" "db" { +# count = var.enable_private_network ? 1 : 0 +# depends_on = [azurerm_databricks_workspace.example] +# } data "azurerm_resource_group" "vnet_rg" { count = var.enable_private_network ? 1 : 0 From 25eeebfa11da50f8e08a15660b5813c57799f545 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 22 Jun 2023 16:14:07 +0100 Subject: [PATCH 099/137] update depends on --- azurerm/modules/azurerm-adb/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 12c43d66..3206094c 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -82,6 +82,7 @@ resource "databricks_workspace_conf" "this" { "enableDbfsFileBrowser" : true } + depends_on = [ azurerm_databricks_workspace.example ] } From e58efc64346276f5a861e2e827bd814155a9acf8 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 22 Jun 2023 16:14:42 +0100 Subject: [PATCH 100/137] update --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 3206094c..395a0531 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -82,7 +82,7 @@ resource "databricks_workspace_conf" "this" { "enableDbfsFileBrowser" : true } - depends_on = [ azurerm_databricks_workspace.example ] + depends_on = [azurerm_databricks_workspace.example] } From fa4863210ed8ab51318868aa188acb56c058cfb9 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 22 Jun 2023 16:38:31 +0100 Subject: [PATCH 101/137] add depends on to db resources --- azurerm/modules/azurerm-adb/main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 395a0531..be1343d8 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -91,6 +91,8 @@ resource "databricks_user" "rbac_users" { display_name = each.value.display_name user_name = each.value.user_name active = each.value.active + + depends_on = [azurerm_databricks_workspace.example] } resource "databricks_group" "project_users" { @@ -98,10 +100,16 @@ resource "databricks_group" "project_users" { display_name = var.databricks_group_display_name workspace_access = var.enable_workspace_access databricks_sql_access = var.enable_sql_access + + depends_on = [azurerm_databricks_workspace.example] + } resource "databricks_group_member" "project_users" { for_each = var.add_rbac_users ? databricks_user.rbac_users : {} group_id = databricks_group.project_users[0].id member_id = each.value.id + + depends_on = [azurerm_databricks_workspace.example] + } From 64e7de492c861f40f7a747c18550262bc42637a0 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 22 Jun 2023 18:33:34 +0100 Subject: [PATCH 102/137] make string --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index be1343d8..138bb917 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -79,7 +79,7 @@ resource "databricks_workspace_conf" "this" { count = var.enable_enableDbfsFileBrowser ? 1 : 0 custom_config = { - "enableDbfsFileBrowser" : true + "enableDbfsFileBrowser" : "true" } depends_on = [azurerm_databricks_workspace.example] From cd48c4a56301f0fea1ae54b11b5a865c696fa2df Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Thu, 22 Jun 2023 18:40:52 +0100 Subject: [PATCH 103/137] update --- azurerm/modules/azurerm-adb/main.tf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 138bb917..41797cf5 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -78,9 +78,7 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { resource "databricks_workspace_conf" "this" { count = var.enable_enableDbfsFileBrowser ? 1 : 0 custom_config = { - - "enableDbfsFileBrowser" : "true" - + "enableDbfsFileBrowser" : true } depends_on = [azurerm_databricks_workspace.example] } From 88ab416d59ad7d5599d4dae182995aa99072fc15 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 26 Jun 2023 14:23:33 +0100 Subject: [PATCH 104/137] remove whitespace --- azurerm/modules/azurerm-adb/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 41797cf5..e6746ba4 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -100,7 +100,6 @@ resource "databricks_group" "project_users" { databricks_sql_access = var.enable_sql_access depends_on = [azurerm_databricks_workspace.example] - } resource "databricks_group_member" "project_users" { @@ -109,5 +108,4 @@ resource "databricks_group_member" "project_users" { member_id = each.value.id depends_on = [azurerm_databricks_workspace.example] - } From fa9b806c706e2dfbcb7ed285555a077b14a41166 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 26 Jun 2023 17:14:55 +0100 Subject: [PATCH 105/137] update dbfs to string --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index e6746ba4..7ee84532 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -78,7 +78,7 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { resource "databricks_workspace_conf" "this" { count = var.enable_enableDbfsFileBrowser ? 1 : 0 custom_config = { - "enableDbfsFileBrowser" : true + "enableDbfsFileBrowser" : "true" } depends_on = [azurerm_databricks_workspace.example] } From 8779e46a34616cf0bde1cc81c1d20f02474ba719 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 26 Jun 2023 17:17:38 +0100 Subject: [PATCH 106/137] temp remove dbfs explore --- azurerm/modules/azurerm-adb/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 7ee84532..c71c8c72 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,13 +75,13 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -resource "databricks_workspace_conf" "this" { - count = var.enable_enableDbfsFileBrowser ? 1 : 0 - custom_config = { - "enableDbfsFileBrowser" : "true" - } - depends_on = [azurerm_databricks_workspace.example] -} +# resource "databricks_workspace_conf" "this" { +# count = var.enable_enableDbfsFileBrowser ? 1 : 0 +# custom_config = { +# "enableDbfsFileBrowser" : "true" +# } +# depends_on = [azurerm_databricks_workspace.example] +# } resource "databricks_user" "rbac_users" { From 0e07fe3233dab87a141a44641ad500624f13698c Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 26 Jun 2023 17:26:54 +0100 Subject: [PATCH 107/137] add dbfs explorer --- azurerm/modules/azurerm-adb/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index c71c8c72..7ee84532 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,13 +75,13 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -# resource "databricks_workspace_conf" "this" { -# count = var.enable_enableDbfsFileBrowser ? 1 : 0 -# custom_config = { -# "enableDbfsFileBrowser" : "true" -# } -# depends_on = [azurerm_databricks_workspace.example] -# } +resource "databricks_workspace_conf" "this" { + count = var.enable_enableDbfsFileBrowser ? 1 : 0 + custom_config = { + "enableDbfsFileBrowser" : "true" + } + depends_on = [azurerm_databricks_workspace.example] +} resource "databricks_user" "rbac_users" { From 2e8a0df0769b22b6c7b7a9ed89c4f56a4901dd00 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Mon, 26 Jun 2023 17:40:23 +0100 Subject: [PATCH 108/137] temp remove dbfs --- azurerm/modules/azurerm-adb/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 7ee84532..c71c8c72 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,13 +75,13 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -resource "databricks_workspace_conf" "this" { - count = var.enable_enableDbfsFileBrowser ? 1 : 0 - custom_config = { - "enableDbfsFileBrowser" : "true" - } - depends_on = [azurerm_databricks_workspace.example] -} +# resource "databricks_workspace_conf" "this" { +# count = var.enable_enableDbfsFileBrowser ? 1 : 0 +# custom_config = { +# "enableDbfsFileBrowser" : "true" +# } +# depends_on = [azurerm_databricks_workspace.example] +# } resource "databricks_user" "rbac_users" { From d26ba66c2ef2bb93260a9518f82a4b42b76fb3cc Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 14:47:34 +0100 Subject: [PATCH 109/137] update --- azurerm/modules/azurerm-adb/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index c71c8c72..7ee84532 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,13 +75,13 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -# resource "databricks_workspace_conf" "this" { -# count = var.enable_enableDbfsFileBrowser ? 1 : 0 -# custom_config = { -# "enableDbfsFileBrowser" : "true" -# } -# depends_on = [azurerm_databricks_workspace.example] -# } +resource "databricks_workspace_conf" "this" { + count = var.enable_enableDbfsFileBrowser ? 1 : 0 + custom_config = { + "enableDbfsFileBrowser" : "true" + } + depends_on = [azurerm_databricks_workspace.example] +} resource "databricks_user" "rbac_users" { From 81b0f5ff464c28129fe191655450d182aac3bbfd Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 17:29:21 +0100 Subject: [PATCH 110/137] update dns zone name --- azurerm/modules/azurerm-adb/network.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 3a985a12..50144fa0 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -178,7 +178,8 @@ resource "azurerm_private_endpoint" "auth" { resource "azurerm_private_dns_zone" "dns" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = "privatelink.azuredatabricks.net" + name = "${var.resource_namer}.azuredatabricks.net" + # name = "privatelink.azuredatabricks.net" TODO resource_group_name = var.resource_group_name } From 9adfd3d87d3a71b7cdf097ee8cf3cdd9f9234337 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 17:33:14 +0100 Subject: [PATCH 111/137] run fmt --- azurerm/modules/azurerm-adb/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 50144fa0..987231a9 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -177,8 +177,8 @@ resource "azurerm_private_endpoint" "auth" { } resource "azurerm_private_dns_zone" "dns" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = "${var.resource_namer}.azuredatabricks.net" + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "${var.resource_namer}.azuredatabricks.net" # name = "privatelink.azuredatabricks.net" TODO resource_group_name = var.resource_group_name } From 5a84b0786016a3fc01094f2ebacd10c80faaa53f Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 21:06:29 +0100 Subject: [PATCH 112/137] add permissions --- azurerm/modules/azurerm-adb/main.tf | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 7ee84532..fae36570 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,12 +75,20 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } +resource "azurerm_role_assignment" "tf_spn" { + scope = azurerm_databricks_workspace.example.id + role_definition_name = "Contributor" + principal_id = data.azurerm_client_config.spn_client.object_id + + depends_on = [azurerm_databricks_workspace.example] +} + resource "databricks_workspace_conf" "this" { count = var.enable_enableDbfsFileBrowser ? 1 : 0 custom_config = { "enableDbfsFileBrowser" : "true" } - depends_on = [azurerm_databricks_workspace.example] + depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] } @@ -90,7 +98,7 @@ resource "databricks_user" "rbac_users" { user_name = each.value.user_name active = each.value.active - depends_on = [azurerm_databricks_workspace.example] + depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] } resource "databricks_group" "project_users" { @@ -99,7 +107,7 @@ resource "databricks_group" "project_users" { workspace_access = var.enable_workspace_access databricks_sql_access = var.enable_sql_access - depends_on = [azurerm_databricks_workspace.example] + depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] } resource "databricks_group_member" "project_users" { @@ -107,5 +115,5 @@ resource "databricks_group_member" "project_users" { group_id = databricks_group.project_users[0].id member_id = each.value.id - depends_on = [azurerm_databricks_workspace.example] + depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] } From c8e10a585fa39596a16425bb2119cc6c7947c217 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 21:17:58 +0100 Subject: [PATCH 113/137] fix name --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index fae36570..a2bb7255 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -78,7 +78,7 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { resource "azurerm_role_assignment" "tf_spn" { scope = azurerm_databricks_workspace.example.id role_definition_name = "Contributor" - principal_id = data.azurerm_client_config.spn_client.object_id + principal_id = data.azurerm_client_config.current.object_id depends_on = [azurerm_databricks_workspace.example] } From 5f02b10835c35057eb7ef869894d04d48d5cd883 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 21:41:32 +0100 Subject: [PATCH 114/137] temp comment out --- azurerm/modules/azurerm-adb/main.tf | 68 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index a2bb7255..50e24e22 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -83,37 +83,37 @@ resource "azurerm_role_assignment" "tf_spn" { depends_on = [azurerm_databricks_workspace.example] } -resource "databricks_workspace_conf" "this" { - count = var.enable_enableDbfsFileBrowser ? 1 : 0 - custom_config = { - "enableDbfsFileBrowser" : "true" - } - depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] -} - - -resource "databricks_user" "rbac_users" { - for_each = var.add_rbac_users ? var.rbac_databricks_users : {} - display_name = each.value.display_name - user_name = each.value.user_name - active = each.value.active - - depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] -} - -resource "databricks_group" "project_users" { - count = var.add_rbac_users ? 1 : 0 - display_name = var.databricks_group_display_name - workspace_access = var.enable_workspace_access - databricks_sql_access = var.enable_sql_access - - depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] -} - -resource "databricks_group_member" "project_users" { - for_each = var.add_rbac_users ? databricks_user.rbac_users : {} - group_id = databricks_group.project_users[0].id - member_id = each.value.id - - depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] -} +# resource "databricks_workspace_conf" "this" { +# count = var.enable_enableDbfsFileBrowser ? 1 : 0 +# custom_config = { +# "enableDbfsFileBrowser" : "true" +# } +# depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] +# } + + +# resource "databricks_user" "rbac_users" { +# for_each = var.add_rbac_users ? var.rbac_databricks_users : {} +# display_name = each.value.display_name +# user_name = each.value.user_name +# active = each.value.active + +# depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] +# } + +# resource "databricks_group" "project_users" { +# count = var.add_rbac_users ? 1 : 0 +# display_name = var.databricks_group_display_name +# workspace_access = var.enable_workspace_access +# databricks_sql_access = var.enable_sql_access + +# depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] +# } + +# resource "databricks_group_member" "project_users" { +# for_each = var.add_rbac_users ? databricks_user.rbac_users : {} +# group_id = databricks_group.project_users[0].id +# member_id = each.value.id + +# depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] +# } From ac0de06478bedf25971e1e80015f43664607aa3b Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 21:51:19 +0100 Subject: [PATCH 115/137] remove perm --- azurerm/modules/azurerm-adb/main.tf | 8 -------- 1 file changed, 8 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 50e24e22..878acfcc 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,14 +75,6 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -resource "azurerm_role_assignment" "tf_spn" { - scope = azurerm_databricks_workspace.example.id - role_definition_name = "Contributor" - principal_id = data.azurerm_client_config.current.object_id - - depends_on = [azurerm_databricks_workspace.example] -} - # resource "databricks_workspace_conf" "this" { # count = var.enable_enableDbfsFileBrowser ? 1 : 0 # custom_config = { From ebbfe50b28784efb98dc1bd3e1d8abf67511e3d7 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 21:57:15 +0100 Subject: [PATCH 116/137] add db back in --- azurerm/modules/azurerm-adb/main.tf | 68 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 878acfcc..93d6987b 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,37 +75,37 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -# resource "databricks_workspace_conf" "this" { -# count = var.enable_enableDbfsFileBrowser ? 1 : 0 -# custom_config = { -# "enableDbfsFileBrowser" : "true" -# } -# depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] -# } - - -# resource "databricks_user" "rbac_users" { -# for_each = var.add_rbac_users ? var.rbac_databricks_users : {} -# display_name = each.value.display_name -# user_name = each.value.user_name -# active = each.value.active - -# depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] -# } - -# resource "databricks_group" "project_users" { -# count = var.add_rbac_users ? 1 : 0 -# display_name = var.databricks_group_display_name -# workspace_access = var.enable_workspace_access -# databricks_sql_access = var.enable_sql_access - -# depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] -# } - -# resource "databricks_group_member" "project_users" { -# for_each = var.add_rbac_users ? databricks_user.rbac_users : {} -# group_id = databricks_group.project_users[0].id -# member_id = each.value.id - -# depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] -# } +resource "databricks_workspace_conf" "this" { + count = var.enable_enableDbfsFileBrowser ? 1 : 0 + custom_config = { + "enableDbfsFileBrowser" : "true" + } + depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] +} + + +resource "databricks_user" "rbac_users" { + for_each = var.add_rbac_users ? var.rbac_databricks_users : {} + display_name = each.value.display_name + user_name = each.value.user_name + active = each.value.active + + depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] +} + +resource "databricks_group" "project_users" { + count = var.add_rbac_users ? 1 : 0 + display_name = var.databricks_group_display_name + workspace_access = var.enable_workspace_access + databricks_sql_access = var.enable_sql_access + + depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] +} + +resource "databricks_group_member" "project_users" { + for_each = var.add_rbac_users ? databricks_user.rbac_users : {} + group_id = databricks_group.project_users[0].id + member_id = each.value.id + + depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] +} From d1649acd217f2fce037f5432b768d86d99c1d73e Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 21:59:17 +0100 Subject: [PATCH 117/137] remove depends on --- azurerm/modules/azurerm-adb/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 93d6987b..7ee84532 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -80,7 +80,7 @@ resource "databricks_workspace_conf" "this" { custom_config = { "enableDbfsFileBrowser" : "true" } - depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] + depends_on = [azurerm_databricks_workspace.example] } @@ -90,7 +90,7 @@ resource "databricks_user" "rbac_users" { user_name = each.value.user_name active = each.value.active - depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] + depends_on = [azurerm_databricks_workspace.example] } resource "databricks_group" "project_users" { @@ -99,7 +99,7 @@ resource "databricks_group" "project_users" { workspace_access = var.enable_workspace_access databricks_sql_access = var.enable_sql_access - depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] + depends_on = [azurerm_databricks_workspace.example] } resource "databricks_group_member" "project_users" { @@ -107,5 +107,5 @@ resource "databricks_group_member" "project_users" { group_id = databricks_group.project_users[0].id member_id = each.value.id - depends_on = [azurerm_databricks_workspace.example, azurerm_role_assignment.tf_spn] + depends_on = [azurerm_databricks_workspace.example] } From 3f61804d92f1b0fa86eff31bdd341f7d25033337 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 22:01:31 +0100 Subject: [PATCH 118/137] comment out db --- azurerm/modules/azurerm-adb/main.tf | 68 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 7ee84532..df67f00d 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,37 +75,37 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -resource "databricks_workspace_conf" "this" { - count = var.enable_enableDbfsFileBrowser ? 1 : 0 - custom_config = { - "enableDbfsFileBrowser" : "true" - } - depends_on = [azurerm_databricks_workspace.example] -} - - -resource "databricks_user" "rbac_users" { - for_each = var.add_rbac_users ? var.rbac_databricks_users : {} - display_name = each.value.display_name - user_name = each.value.user_name - active = each.value.active - - depends_on = [azurerm_databricks_workspace.example] -} - -resource "databricks_group" "project_users" { - count = var.add_rbac_users ? 1 : 0 - display_name = var.databricks_group_display_name - workspace_access = var.enable_workspace_access - databricks_sql_access = var.enable_sql_access - - depends_on = [azurerm_databricks_workspace.example] -} - -resource "databricks_group_member" "project_users" { - for_each = var.add_rbac_users ? databricks_user.rbac_users : {} - group_id = databricks_group.project_users[0].id - member_id = each.value.id - - depends_on = [azurerm_databricks_workspace.example] -} +# resource "databricks_workspace_conf" "this" { +# count = var.enable_enableDbfsFileBrowser ? 1 : 0 +# custom_config = { +# "enableDbfsFileBrowser" : "true" +# } +# depends_on = [azurerm_databricks_workspace.example] +# } + + +# resource "databricks_user" "rbac_users" { +# for_each = var.add_rbac_users ? var.rbac_databricks_users : {} +# display_name = each.value.display_name +# user_name = each.value.user_name +# active = each.value.active + +# depends_on = [azurerm_databricks_workspace.example] +# } + +# resource "databricks_group" "project_users" { +# count = var.add_rbac_users ? 1 : 0 +# display_name = var.databricks_group_display_name +# workspace_access = var.enable_workspace_access +# databricks_sql_access = var.enable_sql_access + +# depends_on = [azurerm_databricks_workspace.example] +# } + +# resource "databricks_group_member" "project_users" { +# for_each = var.add_rbac_users ? databricks_user.rbac_users : {} +# group_id = databricks_group.project_users[0].id +# member_id = each.value.id + +# depends_on = [azurerm_databricks_workspace.example] +# } From 13b70fc6c2d9c6796230115a96f175cec40140d1 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 22:53:46 +0100 Subject: [PATCH 119/137] update pe --- azurerm/modules/azurerm-adb/main.tf | 68 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index df67f00d..9c4bd3f3 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,37 +75,37 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -# resource "databricks_workspace_conf" "this" { -# count = var.enable_enableDbfsFileBrowser ? 1 : 0 -# custom_config = { -# "enableDbfsFileBrowser" : "true" -# } -# depends_on = [azurerm_databricks_workspace.example] -# } - - -# resource "databricks_user" "rbac_users" { -# for_each = var.add_rbac_users ? var.rbac_databricks_users : {} -# display_name = each.value.display_name -# user_name = each.value.user_name -# active = each.value.active - -# depends_on = [azurerm_databricks_workspace.example] -# } - -# resource "databricks_group" "project_users" { -# count = var.add_rbac_users ? 1 : 0 -# display_name = var.databricks_group_display_name -# workspace_access = var.enable_workspace_access -# databricks_sql_access = var.enable_sql_access - -# depends_on = [azurerm_databricks_workspace.example] -# } - -# resource "databricks_group_member" "project_users" { -# for_each = var.add_rbac_users ? databricks_user.rbac_users : {} -# group_id = databricks_group.project_users[0].id -# member_id = each.value.id - -# depends_on = [azurerm_databricks_workspace.example] -# } +resource "databricks_workspace_conf" "this" { + count = var.enable_enableDbfsFileBrowser ? 1 : 0 + custom_config = { + "enableDbfsFileBrowser" : "true" + } + depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +} + + +resource "databricks_user" "rbac_users" { + for_each = var.add_rbac_users ? var.rbac_databricks_users : {} + display_name = each.value.display_name + user_name = each.value.user_name + active = each.value.active + + depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +} + +resource "databricks_group" "project_users" { + count = var.add_rbac_users ? 1 : 0 + display_name = var.databricks_group_display_name + workspace_access = var.enable_workspace_access + databricks_sql_access = var.enable_sql_access + + depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +} + +resource "databricks_group_member" "project_users" { + for_each = var.add_rbac_users ? databricks_user.rbac_users : {} + group_id = databricks_group.project_users[0].id + member_id = each.value.id + + depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +} From e002a0b0457f1f29b48689d6f4644cb836d571f6 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Tue, 27 Jun 2023 22:57:57 +0100 Subject: [PATCH 120/137] comment out db --- azurerm/modules/azurerm-adb/main.tf | 68 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 9c4bd3f3..294011fc 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,37 +75,37 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -resource "databricks_workspace_conf" "this" { - count = var.enable_enableDbfsFileBrowser ? 1 : 0 - custom_config = { - "enableDbfsFileBrowser" : "true" - } - depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -} - - -resource "databricks_user" "rbac_users" { - for_each = var.add_rbac_users ? var.rbac_databricks_users : {} - display_name = each.value.display_name - user_name = each.value.user_name - active = each.value.active - - depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -} - -resource "databricks_group" "project_users" { - count = var.add_rbac_users ? 1 : 0 - display_name = var.databricks_group_display_name - workspace_access = var.enable_workspace_access - databricks_sql_access = var.enable_sql_access - - depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -} - -resource "databricks_group_member" "project_users" { - for_each = var.add_rbac_users ? databricks_user.rbac_users : {} - group_id = databricks_group.project_users[0].id - member_id = each.value.id - - depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -} +# resource "databricks_workspace_conf" "this" { +# count = var.enable_enableDbfsFileBrowser ? 1 : 0 +# custom_config = { +# "enableDbfsFileBrowser" : "true" +# } +# depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +# } + + +# resource "databricks_user" "rbac_users" { +# for_each = var.add_rbac_users ? var.rbac_databricks_users : {} +# display_name = each.value.display_name +# user_name = each.value.user_name +# active = each.value.active + +# depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +# } + +# resource "databricks_group" "project_users" { +# count = var.add_rbac_users ? 1 : 0 +# display_name = var.databricks_group_display_name +# workspace_access = var.enable_workspace_access +# databricks_sql_access = var.enable_sql_access + +# depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +# } + +# resource "databricks_group_member" "project_users" { +# for_each = var.add_rbac_users ? databricks_user.rbac_users : {} +# group_id = databricks_group.project_users[0].id +# member_id = each.value.id + +# depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +# } From 692d0a19b0146fc0d741b94e88d9cc3581e8d2bf Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 09:18:31 +0100 Subject: [PATCH 121/137] rename dns zone --- azurerm/modules/azurerm-adb/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 987231a9..0dbb6599 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -178,8 +178,8 @@ resource "azurerm_private_endpoint" "auth" { resource "azurerm_private_dns_zone" "dns" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 - name = "${var.resource_namer}.azuredatabricks.net" - # name = "privatelink.azuredatabricks.net" TODO + # name = "${var.resource_namer}.azuredatabricks.net" + name = "privatelink.azuredatabricks.net" resource_group_name = var.resource_group_name } From b7ebbf683411d4b22ca5cb70bb5ad0dd9a0aae2a Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 09:50:22 +0100 Subject: [PATCH 122/137] add users and conf --- azurerm/modules/azurerm-adb/main.tf | 68 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 294011fc..9c4bd3f3 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,37 +75,37 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -# resource "databricks_workspace_conf" "this" { -# count = var.enable_enableDbfsFileBrowser ? 1 : 0 -# custom_config = { -# "enableDbfsFileBrowser" : "true" -# } -# depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -# } - - -# resource "databricks_user" "rbac_users" { -# for_each = var.add_rbac_users ? var.rbac_databricks_users : {} -# display_name = each.value.display_name -# user_name = each.value.user_name -# active = each.value.active - -# depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -# } - -# resource "databricks_group" "project_users" { -# count = var.add_rbac_users ? 1 : 0 -# display_name = var.databricks_group_display_name -# workspace_access = var.enable_workspace_access -# databricks_sql_access = var.enable_sql_access - -# depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -# } - -# resource "databricks_group_member" "project_users" { -# for_each = var.add_rbac_users ? databricks_user.rbac_users : {} -# group_id = databricks_group.project_users[0].id -# member_id = each.value.id - -# depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -# } +resource "databricks_workspace_conf" "this" { + count = var.enable_enableDbfsFileBrowser ? 1 : 0 + custom_config = { + "enableDbfsFileBrowser" : "true" + } + depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +} + + +resource "databricks_user" "rbac_users" { + for_each = var.add_rbac_users ? var.rbac_databricks_users : {} + display_name = each.value.display_name + user_name = each.value.user_name + active = each.value.active + + depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +} + +resource "databricks_group" "project_users" { + count = var.add_rbac_users ? 1 : 0 + display_name = var.databricks_group_display_name + workspace_access = var.enable_workspace_access + databricks_sql_access = var.enable_sql_access + + depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +} + +resource "databricks_group_member" "project_users" { + for_each = var.add_rbac_users ? databricks_user.rbac_users : {} + group_id = databricks_group.project_users[0].id + member_id = each.value.id + + depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] +} From 046a1895fec756c04e57e03684106b8c8b20f399 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 09:56:35 +0100 Subject: [PATCH 123/137] add dns update --- azurerm/modules/azurerm-adb/network.tf | 12 +++++++++--- azurerm/modules/azurerm-adb/var.tf | 6 ++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 0dbb6599..88976112 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -177,23 +177,29 @@ resource "azurerm_private_endpoint" "auth" { } resource "azurerm_private_dns_zone" "dns" { - count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false && var.create_dns_zone ? 1 : 0 # name = "${var.resource_namer}.azuredatabricks.net" name = "privatelink.azuredatabricks.net" resource_group_name = var.resource_group_name } +data "azurerm_private_dns_zone" "dns" { + count = var.create_dns_zone == false ? 1 : 0 + name = "privatelink.azuredatabricks.net" + resource_group_name = var.resource_group_name +} + resource "azurerm_private_dns_cname_record" "cname" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = azurerm_databricks_workspace.example.workspace_url - zone_name = azurerm_private_dns_zone.dns[0].name + zone_name = var.create_dns_zone ? azurerm_private_dns_zone.dns[0].name : data.azurerm_private_dns_zone.dns[0].name resource_group_name = var.resource_group_name ttl = var.dns_record_ttl record = "${var.resource_namer}.azuredatabricks.net" } resource "azurerm_private_dns_zone_virtual_network_link" "db_dns_vnet_link" { - count = var.enable_private_network == true && var.managed_vnet == false ? 1 : 0 + count = var.enable_private_network == true && var.managed_vnet == false && var.create_dns_zone == true ? 1 : 0 name = var.resource_namer resource_group_name = var.resource_group_name private_dns_zone_name = azurerm_private_dns_zone.dns[0].name diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 7fa3d239..d68f1e96 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -150,6 +150,12 @@ variable "create_pe_subnet" { description = "Set to true if you need the module to create the private endpoint subnet." } +variable "create_dns_zone" { + type = bool + default = false + description = "Create DNS Zone for Azure Databricks." +} + variable "vnet_name" { type = string default = "" From be00219afd589a0e7c360e23c8aabbd75899b2b0 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 10:02:05 +0100 Subject: [PATCH 124/137] update default value --- azurerm/modules/azurerm-adb/var.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index d68f1e96..13c80dd9 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -152,7 +152,7 @@ variable "create_pe_subnet" { variable "create_dns_zone" { type = bool - default = false + default = true description = "Create DNS Zone for Azure Databricks." } From 20f6fef1269b0c73f5c77e5fd01ccc8aaabd41e7 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 10:20:04 +0100 Subject: [PATCH 125/137] update var name --- azurerm/modules/azurerm-adb/network.tf | 8 ++++---- azurerm/modules/azurerm-adb/var.tf | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 88976112..1d5711ea 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -177,14 +177,14 @@ resource "azurerm_private_endpoint" "auth" { } resource "azurerm_private_dns_zone" "dns" { - count = var.enable_private_network && var.managed_vnet == false && var.create_dns_zone ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false && var.create_db_dns_zone ? 1 : 0 # name = "${var.resource_namer}.azuredatabricks.net" name = "privatelink.azuredatabricks.net" resource_group_name = var.resource_group_name } data "azurerm_private_dns_zone" "dns" { - count = var.create_dns_zone == false ? 1 : 0 + count = var.create_db_dns_zone == false ? 1 : 0 name = "privatelink.azuredatabricks.net" resource_group_name = var.resource_group_name } @@ -192,14 +192,14 @@ data "azurerm_private_dns_zone" "dns" { resource "azurerm_private_dns_cname_record" "cname" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = azurerm_databricks_workspace.example.workspace_url - zone_name = var.create_dns_zone ? azurerm_private_dns_zone.dns[0].name : data.azurerm_private_dns_zone.dns[0].name + zone_name = var.create_db_dns_zone ? azurerm_private_dns_zone.dns[0].name : data.azurerm_private_dns_zone.dns[0].name resource_group_name = var.resource_group_name ttl = var.dns_record_ttl record = "${var.resource_namer}.azuredatabricks.net" } resource "azurerm_private_dns_zone_virtual_network_link" "db_dns_vnet_link" { - count = var.enable_private_network == true && var.managed_vnet == false && var.create_dns_zone == true ? 1 : 0 + count = var.enable_private_network == true && var.managed_vnet == false && var.create_db_dns_zone == true ? 1 : 0 name = var.resource_namer resource_group_name = var.resource_group_name private_dns_zone_name = azurerm_private_dns_zone.dns[0].name diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 13c80dd9..87a79a6b 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -150,7 +150,7 @@ variable "create_pe_subnet" { description = "Set to true if you need the module to create the private endpoint subnet." } -variable "create_dns_zone" { +variable "create_db_dns_zone" { type = bool default = true description = "Create DNS Zone for Azure Databricks." From 06dfdc648d4aa49e4aa1da6511621ad48e1e8914 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 10:26:12 +0100 Subject: [PATCH 126/137] add condition --- azurerm/modules/azurerm-adb/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 1d5711ea..03f8886b 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -148,10 +148,10 @@ resource "azurerm_private_endpoint" "databricks" { private_dns_zone_group { name = "databricks_ui_api" - private_dns_zone_ids = [azurerm_private_dns_zone.dns[0].id] + private_dns_zone_ids = [var.create_db_dns_zone ? azurerm_private_dns_zone.dns[0].id : data.azurerm_private_dns_zone.dns[0].id] } - depends_on = [azurerm_databricks_workspace.example, azurerm_private_dns_zone.dns] + depends_on = [azurerm_databricks_workspace.example, azurerm_private_dns_zone.dns, data.azurerm_private_dns_zone.dns] } resource "azurerm_private_endpoint" "auth" { From 7191000d05b2bd3c8de0144b80578854cbd38c00 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 10:26:41 +0100 Subject: [PATCH 127/137] add condition --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 03f8886b..79523c16 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -169,7 +169,7 @@ resource "azurerm_private_endpoint" "auth" { private_dns_zone_group { name = "databricks_auth" - private_dns_zone_ids = [azurerm_private_dns_zone.dns[0].id] + private_dns_zone_ids = [var.create_db_dns_zone ? azurerm_private_dns_zone.dns[0].id : data.azurerm_private_dns_zone.dns[0].id] } depends_on = [azurerm_databricks_workspace.example, azurerm_private_dns_zone.dns] From 7e4cca9c88f0b1aad53e7e0847278a9848da2543 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 10:33:20 +0100 Subject: [PATCH 128/137] update rg name --- azurerm/modules/azurerm-adb/main.tf | 1 - azurerm/modules/azurerm-adb/network.tf | 2 +- azurerm/modules/azurerm-adb/var.tf | 6 ++++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 9c4bd3f3..f7640500 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -83,7 +83,6 @@ resource "databricks_workspace_conf" "this" { depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] } - resource "databricks_user" "rbac_users" { for_each = var.add_rbac_users ? var.rbac_databricks_users : {} display_name = each.value.display_name diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 79523c16..e6b8cefb 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -186,7 +186,7 @@ resource "azurerm_private_dns_zone" "dns" { data "azurerm_private_dns_zone" "dns" { count = var.create_db_dns_zone == false ? 1 : 0 name = "privatelink.azuredatabricks.net" - resource_group_name = var.resource_group_name + resource_group_name = var.db_dns_zone_rg } resource "azurerm_private_dns_cname_record" "cname" { diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 87a79a6b..464a4032 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -156,6 +156,12 @@ variable "create_db_dns_zone" { description = "Create DNS Zone for Azure Databricks." } +variable "db_dns_zone_rg" { + type = string + default = "value" + description = "Resource Group where DNS is created." +} + variable "vnet_name" { type = string default = "" From eb427a2d367836e72eb38d17e27b6ae3f1110e78 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 10:44:10 +0100 Subject: [PATCH 129/137] add condition to cname --- azurerm/modules/azurerm-adb/network.tf | 2 +- azurerm/modules/azurerm-adb/var.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index e6b8cefb..d03d7201 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -193,7 +193,7 @@ resource "azurerm_private_dns_cname_record" "cname" { count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 name = azurerm_databricks_workspace.example.workspace_url zone_name = var.create_db_dns_zone ? azurerm_private_dns_zone.dns[0].name : data.azurerm_private_dns_zone.dns[0].name - resource_group_name = var.resource_group_name + resource_group_name = var.create_db_dns_zone ? var.resource_group_name : var.db_dns_zone_rg ttl = var.dns_record_ttl record = "${var.resource_namer}.azuredatabricks.net" } diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 464a4032..cfb19142 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -157,8 +157,8 @@ variable "create_db_dns_zone" { } variable "db_dns_zone_rg" { - type = string - default = "value" + type = string + default = "value" description = "Resource Group where DNS is created." } From 66df3fc380b6074f3c7e0683f5366cbd2c298013 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 10:51:25 +0100 Subject: [PATCH 130/137] tidy up comments --- azurerm/modules/azurerm-adb/network.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index d03d7201..7014a762 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -178,7 +178,6 @@ resource "azurerm_private_endpoint" "auth" { resource "azurerm_private_dns_zone" "dns" { count = var.enable_private_network && var.managed_vnet == false && var.create_db_dns_zone ? 1 : 0 - # name = "${var.resource_namer}.azuredatabricks.net" name = "privatelink.azuredatabricks.net" resource_group_name = var.resource_group_name } From d60cf2429145ecc94b91f01d99eb865a7fc1a9eb Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Wed, 28 Jun 2023 10:51:41 +0100 Subject: [PATCH 131/137] tidy up comments --- azurerm/modules/azurerm-adb/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 7014a762..14a9b900 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -177,7 +177,7 @@ resource "azurerm_private_endpoint" "auth" { } resource "azurerm_private_dns_zone" "dns" { - count = var.enable_private_network && var.managed_vnet == false && var.create_db_dns_zone ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false && var.create_db_dns_zone ? 1 : 0 name = "privatelink.azuredatabricks.net" resource_group_name = var.resource_group_name } From 9463a0f1d7f30dc70feda06c1260ed42c86620ee Mon Sep 17 00:00:00 2001 From: Trishisingh Date: Mon, 10 Jul 2023 18:22:16 +0100 Subject: [PATCH 132/137] updated for new --- azurerm/modules/azurerm-adb/data.tf | 8 +++++++- azurerm/modules/azurerm-adb/network.tf | 14 ++++++++------ azurerm/modules/azurerm-adb/var.tf | 14 +++++++++++++- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf index d0ec3f7c..20fa1ae1 100644 --- a/azurerm/modules/azurerm-adb/data.tf +++ b/azurerm/modules/azurerm-adb/data.tf @@ -36,4 +36,10 @@ data "azurerm_subnet" "pe_subnet" { name = var.pe_subnet_name resource_group_name = var.vnet_resource_group virtual_network_name = var.vnet_name -} \ No newline at end of file +} + +data "azurerm_private_dns_zone" "adb_pvt_dns" { + count = var.enable_private_network ? 1 : 0 + name = var.private_dns_zone_name + resource_group_name = var.dns_resource_group_name +} diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index 14a9b900..a98208ae 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -147,11 +147,12 @@ resource "azurerm_private_endpoint" "databricks" { } private_dns_zone_group { + name = "databricks_ui_api" - private_dns_zone_ids = [var.create_db_dns_zone ? azurerm_private_dns_zone.dns[0].id : data.azurerm_private_dns_zone.dns[0].id] + private_dns_zone_ids = [data.azurerm_private_dns_zone.adb_pvt_dns[0].id] } - depends_on = [azurerm_databricks_workspace.example, azurerm_private_dns_zone.dns, data.azurerm_private_dns_zone.dns] + depends_on = [azurerm_databricks_workspace.example, data.azurerm_private_dns_zone.adb_pvt_dns] } resource "azurerm_private_endpoint" "auth" { @@ -169,13 +170,13 @@ resource "azurerm_private_endpoint" "auth" { private_dns_zone_group { name = "databricks_auth" - private_dns_zone_ids = [var.create_db_dns_zone ? azurerm_private_dns_zone.dns[0].id : data.azurerm_private_dns_zone.dns[0].id] + private_dns_zone_ids = [data.azurerm_private_dns_zone.adb_pvt_dns[0].id] } - depends_on = [azurerm_databricks_workspace.example, azurerm_private_dns_zone.dns] + depends_on = [azurerm_databricks_workspace.example] } - +/* resource "azurerm_private_dns_zone" "dns" { count = var.enable_private_network && var.managed_vnet == false && var.create_db_dns_zone ? 1 : 0 name = "privatelink.azuredatabricks.net" @@ -213,4 +214,5 @@ resource "azurerm_public_ip" "pip" { allocation_method = "Static" sku = "Standard" zones = ["1"] -} \ No newline at end of file +} +*/ diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index cfb19142..13894623 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -252,4 +252,16 @@ variable "create_pip" { type = bool default = false description = "Create Databricks with a Public IP." -} \ No newline at end of file +} + +variable "private_dns_zone_name" { + type = string + default = "privatelink.azuredatabricks.net" + description = "Specifies the Name of the Private DNS Zone Group." +} + +variable "dns_resource_group_name" { + type = string + default = "amido-stacks-euw-de-hub-network" + description = "Name of the resource group where pvt dns is present." +} From 604474b4ee051cfb6dee96387c455eba9076f114 Mon Sep 17 00:00:00 2001 From: Trishisingh Date: Mon, 10 Jul 2023 18:31:22 +0100 Subject: [PATCH 133/137] feat: added dns for adls --- azurerm/modules/azurerm-adb/load-balancer.tf | 4 +++- azurerm/modules/azurerm-adb/nat.tf | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adb/load-balancer.tf b/azurerm/modules/azurerm-adb/load-balancer.tf index b4f573eb..b6b1c581 100644 --- a/azurerm/modules/azurerm-adb/load-balancer.tf +++ b/azurerm/modules/azurerm-adb/load-balancer.tf @@ -1,3 +1,4 @@ +/* resource "azurerm_lb" "lb" { count = var.enable_private_network && var.create_lb && var.managed_vnet == false ? 1 : 0 @@ -35,4 +36,5 @@ resource "azurerm_lb_backend_address_pool" "lb_be_pool" { count = var.enable_private_network && var.create_lb && var.managed_vnet == false ? 1 : 0 loadbalancer_id = azurerm_lb.lb[0].id name = "Databricks-BE" -} \ No newline at end of file +} +*/ diff --git a/azurerm/modules/azurerm-adb/nat.tf b/azurerm/modules/azurerm-adb/nat.tf index d403056a..eebaad40 100644 --- a/azurerm/modules/azurerm-adb/nat.tf +++ b/azurerm/modules/azurerm-adb/nat.tf @@ -1,7 +1,7 @@ ############################################ # NAT GATEWAY ############################################ - +/* resource "azurerm_nat_gateway" "nat" { count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 name = local.nat_gateway_name @@ -29,3 +29,4 @@ resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id nat_gateway_id = azurerm_nat_gateway.nat[0].id } +*/ From dcefe589a58bed7a7d0d6e0f8444702b6bda503e Mon Sep 17 00:00:00 2001 From: Trishisingh Date: Mon, 10 Jul 2023 18:36:15 +0100 Subject: [PATCH 134/137] updated --- azurerm/modules/azurerm-adb/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index f7640500..9020e4a8 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -7,7 +7,7 @@ resource "azurerm_databricks_workspace" "example" { public_network_access_enabled = var.public_network_access_enabled network_security_group_rules_required = var.managed_vnet ? null : var.network_security_group_rules_required managed_resource_group_name = "databricks-rg-${var.resource_group_name}" - load_balancer_backend_address_pool_id = var.create_lb ? azurerm_lb_backend_address_pool.lb_be_pool[0].id : null + load_balancer_backend_address_pool_id = var.create_lb ? null : null dynamic "custom_parameters" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) From 2af0a3f5961b06f293265689506c3c492c6b1542 Mon Sep 17 00:00:00 2001 From: Trishisingh Date: Mon, 10 Jul 2023 18:42:46 +0100 Subject: [PATCH 135/137] updated --- azurerm/modules/azurerm-adb/load-balancer.tf | 3 +-- azurerm/modules/azurerm-adb/main.tf | 2 +- azurerm/modules/azurerm-adb/nat.tf | 3 +-- azurerm/modules/azurerm-adb/network.tf | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/azurerm/modules/azurerm-adb/load-balancer.tf b/azurerm/modules/azurerm-adb/load-balancer.tf index b6b1c581..f0a09ef0 100644 --- a/azurerm/modules/azurerm-adb/load-balancer.tf +++ b/azurerm/modules/azurerm-adb/load-balancer.tf @@ -1,4 +1,4 @@ -/* + resource "azurerm_lb" "lb" { count = var.enable_private_network && var.create_lb && var.managed_vnet == false ? 1 : 0 @@ -37,4 +37,3 @@ resource "azurerm_lb_backend_address_pool" "lb_be_pool" { loadbalancer_id = azurerm_lb.lb[0].id name = "Databricks-BE" } -*/ diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index 9020e4a8..f7640500 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -7,7 +7,7 @@ resource "azurerm_databricks_workspace" "example" { public_network_access_enabled = var.public_network_access_enabled network_security_group_rules_required = var.managed_vnet ? null : var.network_security_group_rules_required managed_resource_group_name = "databricks-rg-${var.resource_group_name}" - load_balancer_backend_address_pool_id = var.create_lb ? null : null + load_balancer_backend_address_pool_id = var.create_lb ? azurerm_lb_backend_address_pool.lb_be_pool[0].id : null dynamic "custom_parameters" { for_each = var.enable_private_network == false ? toset([]) : toset([1]) diff --git a/azurerm/modules/azurerm-adb/nat.tf b/azurerm/modules/azurerm-adb/nat.tf index eebaad40..d403056a 100644 --- a/azurerm/modules/azurerm-adb/nat.tf +++ b/azurerm/modules/azurerm-adb/nat.tf @@ -1,7 +1,7 @@ ############################################ # NAT GATEWAY ############################################ -/* + resource "azurerm_nat_gateway" "nat" { count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 name = local.nat_gateway_name @@ -29,4 +29,3 @@ resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id nat_gateway_id = azurerm_nat_gateway.nat[0].id } -*/ diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index a98208ae..d042b625 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -205,7 +205,7 @@ resource "azurerm_private_dns_zone_virtual_network_link" "db_dns_vnet_link" { private_dns_zone_name = azurerm_private_dns_zone.dns[0].name virtual_network_id = data.azurerm_virtual_network.vnet[0].id } - +*/ resource "azurerm_public_ip" "pip" { count = var.enable_private_network && var.create_pip && var.managed_vnet == false ? 1 : 0 name = local.public_ip_name @@ -215,4 +215,3 @@ resource "azurerm_public_ip" "pip" { sku = "Standard" zones = ["1"] } -*/ From bd60aac7cf24b2f4af2630c63d9111356483a6f3 Mon Sep 17 00:00:00 2001 From: Satenderrathee Date: Mon, 10 Jul 2023 20:17:16 +0100 Subject: [PATCH 136/137] removed databrick provider code --- azurerm/modules/azurerm-adb/constraints.tf | 4 -- azurerm/modules/azurerm-adb/main.tf | 33 ---------------- azurerm/modules/azurerm-adb/var.tf | 44 ---------------------- 3 files changed, 81 deletions(-) diff --git a/azurerm/modules/azurerm-adb/constraints.tf b/azurerm/modules/azurerm-adb/constraints.tf index 60408b3a..5257ed4f 100644 --- a/azurerm/modules/azurerm-adb/constraints.tf +++ b/azurerm/modules/azurerm-adb/constraints.tf @@ -5,9 +5,5 @@ terraform { source = "hashicorp/azurerm" version = "~> 3.0" } - #TODO: note this is just added right now without any use, can be used In future for databricks provider - databricks = { - source = "databricks/databricks" - } } } diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index f7640500..fed92ed8 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -75,36 +75,3 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -resource "databricks_workspace_conf" "this" { - count = var.enable_enableDbfsFileBrowser ? 1 : 0 - custom_config = { - "enableDbfsFileBrowser" : "true" - } - depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -} - -resource "databricks_user" "rbac_users" { - for_each = var.add_rbac_users ? var.rbac_databricks_users : {} - display_name = each.value.display_name - user_name = each.value.user_name - active = each.value.active - - depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -} - -resource "databricks_group" "project_users" { - count = var.add_rbac_users ? 1 : 0 - display_name = var.databricks_group_display_name - workspace_access = var.enable_workspace_access - databricks_sql_access = var.enable_sql_access - - depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -} - -resource "databricks_group_member" "project_users" { - for_each = var.add_rbac_users ? databricks_user.rbac_users : {} - group_id = databricks_group.project_users[0].id - member_id = each.value.id - - depends_on = [azurerm_databricks_workspace.example, azurerm_private_endpoint.databricks, azurerm_private_endpoint.auth] -} diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 13894623..5a488ae7 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -66,12 +66,6 @@ variable "data_platform_log_analytics_workspace_id" { # Resource Databricks workspace setting ############################################ -variable "enable_enableDbfsFileBrowser" { - type = bool - description = "Whether to enable Dbfs File browser for the Azure Databricks workspace" - default = false -} - variable "public_network_access_enabled" { type = bool default = true @@ -90,44 +84,6 @@ variable "enable_private_network" { description = "Enable Secure Data Platform." } -############################################ -# Resource Databricks user -############################################ - -variable "add_rbac_users" { - description = "If set to true, the module will create databricks users and group named 'project_users' with the specified users as members, and grant workspace and SQL access to this group. Default is false." - type = bool - default = true -} - -variable "rbac_databricks_users" { - type = map(object({ - display_name = string - user_name = string - active = bool - })) - description = "If 'add_rbac_users' set to true then specifies RBAC Databricks users" - default = null -} - -variable "databricks_group_display_name" { - type = string - description = "If 'add_rbac_users' set to true then specifies databricks group display name" - default = "project_users" -} - -variable "enable_workspace_access" { - type = bool - description = "Whether to enable workspace access for the databricks group" - default = true -} - -variable "enable_sql_access" { - type = bool - description = "Whether to enable sql access for the databricks group" - default = true -} - variable "nat_idle_timeout" { type = number default = 10 From 5965e2275a2426db3d1dea70eb1af50080d9e25f Mon Sep 17 00:00:00 2001 From: Trishisingh Date: Mon, 10 Jul 2023 21:51:27 +0100 Subject: [PATCH 137/137] updated for pe --- azurerm/modules/azurerm-adb/network.tf | 1 + azurerm/modules/azurerm-adb/var.tf | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index d042b625..f0b761cc 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -156,6 +156,7 @@ resource "azurerm_private_endpoint" "databricks" { } resource "azurerm_private_endpoint" "auth" { + count = var.enable_private_network && var.managed_vnet == false && var.browser_authentication_enabled == true ? 1 : 0 name = "${var.resource_namer}-pe-databricks-auth" location = var.resource_group_location resource_group_name = var.resource_group_name diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index 5a488ae7..bfeb6d71 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -90,6 +90,12 @@ variable "nat_idle_timeout" { description = "Idle timeout period in minutes." } +variable "browser_authentication_enabled" { + type = bool + default = false + description = "Specify wether to create to private endpoint for browser authentication, False in Dev and True in Production should be enable in on enviroment." +} + ############################################ # Network Details ############################################