From 58337d5453956d1e3dc5239f4a8746d00591c12e Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 12:06:09 +0100 Subject: [PATCH 01/14] add diag settings to adf --- azurerm/modules/azurerm-adb/network.tf | 4 +- azurerm/modules/azurerm-adf/observability.tf | 40 ++++++++++++++++++++ azurerm/modules/azurerm-adf/vars.tf | 6 +++ azurerm/modules/azurerm-hub-spoke/dns.tf | 2 +- azurerm/modules/azurerm-kv/example/main.tf | 8 ++-- 5 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 azurerm/modules/azurerm-adf/observability.tf diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf index f0b761cc..1c79edab 100644 --- a/azurerm/modules/azurerm-adb/network.tf +++ b/azurerm/modules/azurerm-adb/network.tf @@ -147,7 +147,7 @@ resource "azurerm_private_endpoint" "databricks" { } private_dns_zone_group { - + name = "databricks_ui_api" private_dns_zone_ids = [data.azurerm_private_dns_zone.adb_pvt_dns[0].id] } @@ -156,7 +156,7 @@ resource "azurerm_private_endpoint" "databricks" { } resource "azurerm_private_endpoint" "auth" { - count = var.enable_private_network && var.managed_vnet == false && var.browser_authentication_enabled == true ? 1 : 0 + count = var.enable_private_network && var.managed_vnet == false && var.browser_authentication_enabled == true ? 1 : 0 name = "${var.resource_namer}-pe-databricks-auth" location = var.resource_group_location resource_group_name = var.resource_group_name diff --git a/azurerm/modules/azurerm-adf/observability.tf b/azurerm/modules/azurerm-adf/observability.tf new file mode 100644 index 00000000..33c71f81 --- /dev/null +++ b/azurerm/modules/azurerm-adf/observability.tf @@ -0,0 +1,40 @@ +data "azurerm_monitor_diagnostic_categories" "adf_log_analytics_categories" { + count = var.la_workspace_id == "" ? 0 : 1 + resource_id = module.adf.adf_factory_id +} + +resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" { + count = var.la_workspace_id == "" ? 0 : 1 + name = "ADF to Log Analytics" + target_resource_id = azurerm_data_factory.default.id + log_analytics_workspace_id = var.la_workspace_id + log_analytics_destination_type = "Dedicated" + + dynamic "log" { + for_each = data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories.logs + + content { + category = log.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } + + dynamic "metric" { + for_each = data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories.metrics + + content { + category = metric.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adf/vars.tf b/azurerm/modules/azurerm-adf/vars.tf index 005c818a..4b38062f 100644 --- a/azurerm/modules/azurerm-adf/vars.tf +++ b/azurerm/modules/azurerm-adf/vars.tf @@ -95,6 +95,12 @@ variable "runtime_virtual_network_enabled" { description = "Is Integration Runtime compute provisioned within Managed Virtual Network? Changing this forces a new resource to be created." } +variable "la_workspace_id" { + type = string + default = "" + description = "Log Analytics Workspace ID" +} + ########################### # Global parameter for ADF SETTINGS diff --git a/azurerm/modules/azurerm-hub-spoke/dns.tf b/azurerm/modules/azurerm-hub-spoke/dns.tf index e772de89..a0d269cf 100644 --- a/azurerm/modules/azurerm-hub-spoke/dns.tf +++ b/azurerm/modules/azurerm-hub-spoke/dns.tf @@ -7,7 +7,7 @@ resource "azurerm_private_dns_zone" "example" { } resource "azurerm_private_dns_zone_virtual_network_link" "hub-privatelink-dns" { - for_each = var.link_dns_network == true ? toset(var.dns_zone_name ) : toset([]) + for_each = var.link_dns_network == true ? toset(var.dns_zone_name) : toset([]) name = each.key resource_group_name = local.hub_resource_group_name[0] registration_enabled = false # Auto registration_enabled set to false as we cannot add multiple Private DNS to 1 Vnet diff --git a/azurerm/modules/azurerm-kv/example/main.tf b/azurerm/modules/azurerm-kv/example/main.tf index 47c5ea5f..e317c3ef 100644 --- a/azurerm/modules/azurerm-kv/example/main.tf +++ b/azurerm/modules/azurerm-kv/example/main.tf @@ -21,8 +21,8 @@ resource "azurerm_resource_group" "default" { } module "kv_default" { - source = "../../azurerm-kv" - resource_namer = substr(replace(module.default_label.id, "-", ""), 0, 24) + source = "../../azurerm-kv" + resource_namer = substr(replace(module.default_label.id, "-", ""), 0, 24) resource_group_name = azurerm_resource_group.default.name resource_group_location = azurerm_resource_group.default.location create_kv_networkacl = false @@ -33,7 +33,7 @@ module "kv_default" { pe_subnet_id = data.azurerm_subnet.pe_subnet.id pe_resource_group_name = data.azurerm_subnet.pe_subnet.resource_group_name pe_resource_group_location = "UK South" - # private_dns_zone_name = data.azurerm_private_dns_zone.private_dns.name - # private_dns_zone_ids = ["${data.azurerm_private_dns_zone.private_dns.id}"] + # private_dns_zone_name = data.azurerm_private_dns_zone.private_dns.name + # private_dns_zone_ids = ["${data.azurerm_private_dns_zone.private_dns.id}"] dns_resource_group_name = "hub-rg" } From 20c8ec616bf31666ab2fab1190e7d82d79f6d656 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 13:29:17 +0100 Subject: [PATCH 02/14] update id --- azurerm/modules/azurerm-adf/observability.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-adf/observability.tf b/azurerm/modules/azurerm-adf/observability.tf index 33c71f81..77eaade9 100644 --- a/azurerm/modules/azurerm-adf/observability.tf +++ b/azurerm/modules/azurerm-adf/observability.tf @@ -1,6 +1,6 @@ data "azurerm_monitor_diagnostic_categories" "adf_log_analytics_categories" { count = var.la_workspace_id == "" ? 0 : 1 - resource_id = module.adf.adf_factory_id + resource_id = azurerm_data_factory.default.id } resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" { From 5ce56e8ab58dc4164c1a323d6eb2471a4b0bdbc2 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 13:31:29 +0100 Subject: [PATCH 03/14] add observability to kv --- azurerm/modules/azurerm-adf/observability.tf | 4 +- azurerm/modules/azurerm-kv/observability.tf | 40 ++++++++++++++++++++ azurerm/modules/azurerm-kv/vars.tf | 6 +++ 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 azurerm/modules/azurerm-kv/observability.tf diff --git a/azurerm/modules/azurerm-adf/observability.tf b/azurerm/modules/azurerm-adf/observability.tf index 77eaade9..db221806 100644 --- a/azurerm/modules/azurerm-adf/observability.tf +++ b/azurerm/modules/azurerm-adf/observability.tf @@ -1,12 +1,12 @@ data "azurerm_monitor_diagnostic_categories" "adf_log_analytics_categories" { count = var.la_workspace_id == "" ? 0 : 1 - resource_id = azurerm_data_factory.default.id + resource_id = azurerm_data_factory.example.id } resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" { count = var.la_workspace_id == "" ? 0 : 1 name = "ADF to Log Analytics" - target_resource_id = azurerm_data_factory.default.id + target_resource_id = azurerm_data_factory.example.id log_analytics_workspace_id = var.la_workspace_id log_analytics_destination_type = "Dedicated" diff --git a/azurerm/modules/azurerm-kv/observability.tf b/azurerm/modules/azurerm-kv/observability.tf new file mode 100644 index 00000000..7e7d204a --- /dev/null +++ b/azurerm/modules/azurerm-kv/observability.tf @@ -0,0 +1,40 @@ +data "azurerm_monitor_diagnostic_categories" "kv_log_analytics_categories" { + count = var.la_workspace_id == "" ? 0 : 1 + resource_id = azurerm_key_vault.example.id +} + +resource "azurerm_monitor_diagnostic_setting" "kv_log_analytics" { + count = var.la_workspace_id == "" ? 0 : 1 + name = "KV to Log Analytics" + target_resource_id = azurerm_key_vault.example.id + log_analytics_workspace_id = var.la_workspace_id + log_analytics_destination_type = "Dedicated" + + dynamic "log" { + for_each = data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories.logs + + content { + category = log.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } + + dynamic "metric" { + for_each = data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories.metrics + + content { + category = metric.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-kv/vars.tf b/azurerm/modules/azurerm-kv/vars.tf index ef7f8d27..32560087 100644 --- a/azurerm/modules/azurerm-kv/vars.tf +++ b/azurerm/modules/azurerm-kv/vars.tf @@ -194,3 +194,9 @@ variable "pe_resource_group_location" { default = "" description = "Location of the resource group to provision private endpoint in." } + +variable "la_workspace_id" { + type = string + default = "" + description = "Log Analytics Workspace ID" +} \ No newline at end of file From 819ac088b860bcfaad8f5df312d41c6ed8dae103 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 13:36:56 +0100 Subject: [PATCH 04/14] initial commit of storage to la --- azurerm/modules/azurerm-adls/observability.tf | 40 +++++++++++++++++++ azurerm/modules/azurerm-adls/vars.tf | 6 +++ 2 files changed, 46 insertions(+) create mode 100644 azurerm/modules/azurerm-adls/observability.tf diff --git a/azurerm/modules/azurerm-adls/observability.tf b/azurerm/modules/azurerm-adls/observability.tf new file mode 100644 index 00000000..98999aeb --- /dev/null +++ b/azurerm/modules/azurerm-adls/observability.tf @@ -0,0 +1,40 @@ +data "azurerm_monitor_diagnostic_categories" "adls_log_analytics_categories" { + for_each = var.storage_account_details && var.la_workspace_id != "" + resource_id = azurerm_storage_account.storage_account_default["${each.value.name}"].id +} + +resource "azurerm_monitor_diagnostic_setting" "adls_log_analytics" { + for_each = var.storage_account_details && var.la_workspace_id != "" + name = "Storage to Log Analytics" + target_resource_id = azurerm_storage_account.storage_account_default["${each.value.name}"].id + log_analytics_workspace_id = var.la_workspace_id + log_analytics_destination_type = "Dedicated" + + dynamic "log" { + for_each = data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories.logs + + content { + category = log.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } + + dynamic "metric" { + for_each = data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories.metrics + + content { + category = metric.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adls/vars.tf b/azurerm/modules/azurerm-adls/vars.tf index d816d284..4a73dbc0 100644 --- a/azurerm/modules/azurerm-adls/vars.tf +++ b/azurerm/modules/azurerm-adls/vars.tf @@ -170,3 +170,9 @@ variable "dfs_dns_resource_group_name" { default = "amido-stacks-euw-de-hub-network" description = "Name of the resource group where pvt dns is present for blob." } + +variable "la_workspace_id" { + type = string + default = "" + description = "Log Analytics Workspace ID" +} \ No newline at end of file From d54b5a136f479e6e03171aaf179b9f4a8dfee165 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 13:41:00 +0100 Subject: [PATCH 05/14] update count index --- azurerm/modules/azurerm-adf/observability.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adf/observability.tf b/azurerm/modules/azurerm-adf/observability.tf index db221806..bd9a2951 100644 --- a/azurerm/modules/azurerm-adf/observability.tf +++ b/azurerm/modules/azurerm-adf/observability.tf @@ -1,12 +1,12 @@ data "azurerm_monitor_diagnostic_categories" "adf_log_analytics_categories" { count = var.la_workspace_id == "" ? 0 : 1 - resource_id = azurerm_data_factory.example.id + resource_id = azurerm_data_factory.example[0].id } resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" { count = var.la_workspace_id == "" ? 0 : 1 name = "ADF to Log Analytics" - target_resource_id = azurerm_data_factory.example.id + target_resource_id = azurerm_data_factory.example[0].id log_analytics_workspace_id = var.la_workspace_id log_analytics_destination_type = "Dedicated" From 2398fdcddc8d02dddb159806344dc46bc07b08b2 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 13:43:39 +0100 Subject: [PATCH 06/14] update --- azurerm/modules/azurerm-adf/observability.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azurerm/modules/azurerm-adf/observability.tf b/azurerm/modules/azurerm-adf/observability.tf index bd9a2951..6e0e85cc 100644 --- a/azurerm/modules/azurerm-adf/observability.tf +++ b/azurerm/modules/azurerm-adf/observability.tf @@ -1,6 +1,8 @@ data "azurerm_monitor_diagnostic_categories" "adf_log_analytics_categories" { count = var.la_workspace_id == "" ? 0 : 1 resource_id = azurerm_data_factory.example[0].id + + depends_on = [ azurerm_data_factory.example ] } resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" { @@ -37,4 +39,5 @@ resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" { } } } + depends_on = [ data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories ] } \ No newline at end of file From f117b92531b72b3624ffad4115b7afc7a48f52e1 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 13:45:32 +0100 Subject: [PATCH 07/14] update --- azurerm/modules/azurerm-adf/observability.tf | 12 ++++++------ azurerm/modules/azurerm-adls/observability.tf | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/azurerm/modules/azurerm-adf/observability.tf b/azurerm/modules/azurerm-adf/observability.tf index 6e0e85cc..fd691291 100644 --- a/azurerm/modules/azurerm-adf/observability.tf +++ b/azurerm/modules/azurerm-adf/observability.tf @@ -1,19 +1,19 @@ data "azurerm_monitor_diagnostic_categories" "adf_log_analytics_categories" { - count = var.la_workspace_id == "" ? 0 : 1 + count = var.la_workspace_id != "" ? 1 : 0 resource_id = azurerm_data_factory.example[0].id - depends_on = [ azurerm_data_factory.example ] + depends_on = [azurerm_data_factory.example] } resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" { - count = var.la_workspace_id == "" ? 0 : 1 + count = var.la_workspace_id != "" ? 1 : 0 name = "ADF to Log Analytics" target_resource_id = azurerm_data_factory.example[0].id log_analytics_workspace_id = var.la_workspace_id log_analytics_destination_type = "Dedicated" dynamic "log" { - for_each = data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories.logs + for_each = data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories[0].logs content { category = log.value @@ -27,7 +27,7 @@ resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" { } dynamic "metric" { - for_each = data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories.metrics + for_each = data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories[0].metrics content { category = metric.value @@ -39,5 +39,5 @@ resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" { } } } - depends_on = [ data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories ] + depends_on = [data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories] } \ No newline at end of file diff --git a/azurerm/modules/azurerm-adls/observability.tf b/azurerm/modules/azurerm-adls/observability.tf index 98999aeb..a7db0bcd 100644 --- a/azurerm/modules/azurerm-adls/observability.tf +++ b/azurerm/modules/azurerm-adls/observability.tf @@ -1,10 +1,10 @@ data "azurerm_monitor_diagnostic_categories" "adls_log_analytics_categories" { - for_each = var.storage_account_details && var.la_workspace_id != "" + for_each = var.storage_account_details && var.la_workspace_id != "" resource_id = azurerm_storage_account.storage_account_default["${each.value.name}"].id } resource "azurerm_monitor_diagnostic_setting" "adls_log_analytics" { - for_each = var.storage_account_details && var.la_workspace_id != "" + for_each = var.storage_account_details && var.la_workspace_id != "" name = "Storage to Log Analytics" target_resource_id = azurerm_storage_account.storage_account_default["${each.value.name}"].id log_analytics_workspace_id = var.la_workspace_id From 344a3f130e6b03d13d78aa1622c56b2c03267c81 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 13:47:19 +0100 Subject: [PATCH 08/14] update kv --- azurerm/modules/azurerm-kv/observability.tf | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-kv/observability.tf b/azurerm/modules/azurerm-kv/observability.tf index 7e7d204a..859bf5f5 100644 --- a/azurerm/modules/azurerm-kv/observability.tf +++ b/azurerm/modules/azurerm-kv/observability.tf @@ -1,17 +1,19 @@ data "azurerm_monitor_diagnostic_categories" "kv_log_analytics_categories" { - count = var.la_workspace_id == "" ? 0 : 1 + count = var.la_workspace_id != "" ? 1 : 0 resource_id = azurerm_key_vault.example.id + + depends_on = [azurerm_key_vault] } resource "azurerm_monitor_diagnostic_setting" "kv_log_analytics" { - count = var.la_workspace_id == "" ? 0 : 1 + count = var.la_workspace_id != "" ? 1 : 0 name = "KV to Log Analytics" target_resource_id = azurerm_key_vault.example.id log_analytics_workspace_id = var.la_workspace_id log_analytics_destination_type = "Dedicated" dynamic "log" { - for_each = data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories.logs + for_each = data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories[0].logs content { category = log.value @@ -25,7 +27,7 @@ resource "azurerm_monitor_diagnostic_setting" "kv_log_analytics" { } dynamic "metric" { - for_each = data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories.metrics + for_each = data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories[0].metrics content { category = metric.value @@ -37,4 +39,6 @@ resource "azurerm_monitor_diagnostic_setting" "kv_log_analytics" { } } } + + depends_on = [data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories] } \ No newline at end of file From 3883caaf1e73a9c47b67bcaa49e6196bc8c79ea3 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 13:53:41 +0100 Subject: [PATCH 09/14] add index --- azurerm/modules/azurerm-kv/observability.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-kv/observability.tf b/azurerm/modules/azurerm-kv/observability.tf index 859bf5f5..e84ad7d2 100644 --- a/azurerm/modules/azurerm-kv/observability.tf +++ b/azurerm/modules/azurerm-kv/observability.tf @@ -1,6 +1,6 @@ data "azurerm_monitor_diagnostic_categories" "kv_log_analytics_categories" { count = var.la_workspace_id != "" ? 1 : 0 - resource_id = azurerm_key_vault.example.id + resource_id = azurerm_key_vault.example[0].id depends_on = [azurerm_key_vault] } @@ -8,7 +8,7 @@ data "azurerm_monitor_diagnostic_categories" "kv_log_analytics_categories" { resource "azurerm_monitor_diagnostic_setting" "kv_log_analytics" { count = var.la_workspace_id != "" ? 1 : 0 name = "KV to Log Analytics" - target_resource_id = azurerm_key_vault.example.id + target_resource_id = azurerm_key_vault.example[0].id log_analytics_workspace_id = var.la_workspace_id log_analytics_destination_type = "Dedicated" From 6f483fd7c5d7d238f4e5a0094fcf671ecc07556c Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 13:54:58 +0100 Subject: [PATCH 10/14] fix depends on --- azurerm/modules/azurerm-kv/observability.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/modules/azurerm-kv/observability.tf b/azurerm/modules/azurerm-kv/observability.tf index e84ad7d2..21434b3d 100644 --- a/azurerm/modules/azurerm-kv/observability.tf +++ b/azurerm/modules/azurerm-kv/observability.tf @@ -2,7 +2,7 @@ data "azurerm_monitor_diagnostic_categories" "kv_log_analytics_categories" { count = var.la_workspace_id != "" ? 1 : 0 resource_id = azurerm_key_vault.example[0].id - depends_on = [azurerm_key_vault] + depends_on = [azurerm_key_vault.example] } resource "azurerm_monitor_diagnostic_setting" "kv_log_analytics" { From 9cd8a0bcdf5299d5b245be1b95717671ade5682a Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 14:05:39 +0100 Subject: [PATCH 11/14] update adls --- azurerm/modules/azurerm-adls/observability.tf | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/azurerm/modules/azurerm-adls/observability.tf b/azurerm/modules/azurerm-adls/observability.tf index a7db0bcd..be2928cd 100644 --- a/azurerm/modules/azurerm-adls/observability.tf +++ b/azurerm/modules/azurerm-adls/observability.tf @@ -1,12 +1,21 @@ data "azurerm_monitor_diagnostic_categories" "adls_log_analytics_categories" { - for_each = var.storage_account_details && var.la_workspace_id != "" - resource_id = azurerm_storage_account.storage_account_default["${each.value.name}"].id + for_each = { + for account_name, account_details in var.storage_account_details : account_name => account_details + if var.la_workspace_id != "" + } + resource_id = azurerm_storage_account.storage_account_default["${each.key}"].id + + depends_on = [azurerm_storage_account.storage_account_default] } resource "azurerm_monitor_diagnostic_setting" "adls_log_analytics" { - for_each = var.storage_account_details && var.la_workspace_id != "" + for_each = { + for account_name, account_details in var.storage_account_details : account_name => account_details + if var.la_workspace_id != "" + } + name = "Storage to Log Analytics" - target_resource_id = azurerm_storage_account.storage_account_default["${each.value.name}"].id + target_resource_id = azurerm_storage_account.storage_account_default["${each.key}"].id log_analytics_workspace_id = var.la_workspace_id log_analytics_destination_type = "Dedicated" @@ -37,4 +46,6 @@ resource "azurerm_monitor_diagnostic_setting" "adls_log_analytics" { } } } + + depends_on = [data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories] } \ No newline at end of file From 19167b31d9751b3a8bd09d187ea3d52022335fce Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 14:07:50 +0100 Subject: [PATCH 12/14] update --- azurerm/modules/azurerm-adls/observability.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/modules/azurerm-adls/observability.tf b/azurerm/modules/azurerm-adls/observability.tf index be2928cd..d6dc8c21 100644 --- a/azurerm/modules/azurerm-adls/observability.tf +++ b/azurerm/modules/azurerm-adls/observability.tf @@ -20,7 +20,7 @@ resource "azurerm_monitor_diagnostic_setting" "adls_log_analytics" { log_analytics_destination_type = "Dedicated" dynamic "log" { - for_each = data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories.logs + for_each = data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories[each.key].logs content { category = log.value @@ -34,7 +34,7 @@ resource "azurerm_monitor_diagnostic_setting" "adls_log_analytics" { } dynamic "metric" { - for_each = data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories.metrics + for_each = data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories[each.key].metrics content { category = metric.value From c200decce48e9bc53f22b715dcca24516f9e5690 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 14:41:33 +0100 Subject: [PATCH 13/14] add observability module --- .../modules/azurerm-observability/README.md | 52 +++++++++ .../azurerm-observability/constraints.tf | 9 ++ azurerm/modules/azurerm-observability/data.tf | 6 + .../example/constraints.tf | 9 ++ .../azurerm-observability/example/main.tf | 31 ++++++ .../azurerm-observability/example/vars.tf | 73 +++++++++++++ azurerm/modules/azurerm-observability/main.tf | 44 ++++++++ .../modules/azurerm-observability/output.tf | 23 ++++ azurerm/modules/azurerm-observability/vars.tf | 103 ++++++++++++++++++ 9 files changed, 350 insertions(+) create mode 100644 azurerm/modules/azurerm-observability/README.md create mode 100644 azurerm/modules/azurerm-observability/constraints.tf create mode 100644 azurerm/modules/azurerm-observability/data.tf create mode 100644 azurerm/modules/azurerm-observability/example/constraints.tf create mode 100644 azurerm/modules/azurerm-observability/example/main.tf create mode 100644 azurerm/modules/azurerm-observability/example/vars.tf create mode 100644 azurerm/modules/azurerm-observability/main.tf create mode 100644 azurerm/modules/azurerm-observability/output.tf create mode 100644 azurerm/modules/azurerm-observability/vars.tf diff --git a/azurerm/modules/azurerm-observability/README.md b/azurerm/modules/azurerm-observability/README.md new file mode 100644 index 00000000..3d8c15bf --- /dev/null +++ b/azurerm/modules/azurerm-observability/README.md @@ -0,0 +1,52 @@ +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [azurerm](#requirement\_azurerm) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [azurerm_application_insights.default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/application_insights) | resource | +| [azurerm_log_analytics_solution.default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_solution) | resource | +| [azurerm_log_analytics_workspace.default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource | +| [azurerm_client_config.spn_client](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_resource_group.default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [app\_insights\_name](#input\_app\_insights\_name) | Name of the App Insights Instance to be created. | `string` | `""` | no | +| [attributes](#input\_attributes) | Additional attributes for tagging | `list` | `[]` | no | +| [key\_vault\_name](#input\_key\_vault\_name) | Key Vault name - if not specificied will default to computed naming convention | `string` | `""` | no | +| [la\_name](#input\_la\_name) | Name of the Log Analtics Instance to be created. | `string` | `""` | no | +| [log\_application\_type](#input\_log\_application\_type) | Log application type | `string` | `"other"` | no | +| [resource\_group\_location](#input\_resource\_group\_location) | Location of the RG | `string` | `"useast"` | no | +| [resource\_group\_name](#input\_resource\_group\_name) | Name of the Data Platform Resource Group. | `string` | `""` | no | +| [resource\_group\_tags](#input\_resource\_group\_tags) | Tags at a RG level | `map(string)` | `{}` | no | +| [retention\_in\_days](#input\_retention\_in\_days) | n/a | `number` | `30` | no | +| [stage](#input\_stage) | n/a | `string` | `"dev"` | no | +| [tags](#input\_tags) | Tags to be assigned to all resources, NB if global tagging is enabled these will get overwritten periodically | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [app\_insights\_id](#output\_app\_insights\_id) | n/a | +| [app\_insights\_key](#output\_app\_insights\_key) | n/a | +| [app\_insights\_name](#output\_app\_insights\_name) | n/a | +| [app\_insights\_resource\_group\_name](#output\_app\_insights\_resource\_group\_name) | n/a | +| [log\_analytics\_workspace\_id](#output\_log\_analytics\_workspace\_id) | n/a | \ No newline at end of file diff --git a/azurerm/modules/azurerm-observability/constraints.tf b/azurerm/modules/azurerm-observability/constraints.tf new file mode 100644 index 00000000..5257ed4f --- /dev/null +++ b/azurerm/modules/azurerm-observability/constraints.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 0.13" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + } +} diff --git a/azurerm/modules/azurerm-observability/data.tf b/azurerm/modules/azurerm-observability/data.tf new file mode 100644 index 00000000..3791d500 --- /dev/null +++ b/azurerm/modules/azurerm-observability/data.tf @@ -0,0 +1,6 @@ +data "azurerm_client_config" "spn_client" { +} + +data "azurerm_resource_group" "default" { + name = var.resource_group_name +} diff --git a/azurerm/modules/azurerm-observability/example/constraints.tf b/azurerm/modules/azurerm-observability/example/constraints.tf new file mode 100644 index 00000000..5257ed4f --- /dev/null +++ b/azurerm/modules/azurerm-observability/example/constraints.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 0.13" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + } +} diff --git a/azurerm/modules/azurerm-observability/example/main.tf b/azurerm/modules/azurerm-observability/example/main.tf new file mode 100644 index 00000000..c9aa8563 --- /dev/null +++ b/azurerm/modules/azurerm-observability/example/main.tf @@ -0,0 +1,31 @@ + +module "default_label" { + source = "git::https://github.com/cloudposse/terraform-null-label.git?ref=0.25.0" + namespace = "${var.name_company}-${var.name_project}" + stage = var.stage + name = "${lookup(var.location_name_map, var.resource_group_location, "uksouth")}-${var.name_component}" + attributes = var.attributes + delimiter = "-" + tags = var.tags +} + +################################################## +# ResourceGroups +################################################## + +resource "azurerm_resource_group" "default" { + name = module.default_label.id + location = var.resource_group_location + tags = var.tags +} + + +module "observability" { + source = "../../azurerm-observability" + resource_group_name = azurerm_resource_group.default.name + resource_group_location = azurerm_resource_group.default.location + la_name = module.default_label.id + app_insights_name = module.default_label.id + log_application_type = "other" + retention_in_days = var.retention_in_days +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-observability/example/vars.tf b/azurerm/modules/azurerm-observability/example/vars.tf new file mode 100644 index 00000000..c722e2b8 --- /dev/null +++ b/azurerm/modules/azurerm-observability/example/vars.tf @@ -0,0 +1,73 @@ +############################################ +# NAMING +############################################ + +variable "stage" { + type = string + default = "dev" +} + +variable "attributes" { + description = "Additional attributes for tagging" + default = [] +} + +variable "tags" { + description = "Tags to be assigned to all resources, NB if global tagging is enabled these will get overwritten periodically" + type = map(string) + default = {} +} + +########################### +# OBSERVABILITY +########################## + +variable "resource_group_name" { + type = string + default = "" + description = "Name of the Data Platform Resource Group." +} + +variable "resource_group_location" { + description = "Location of the RG" + type = string + default = "useast" +} + +variable "resource_group_tags" { + description = "Tags at a RG level" + type = map(string) + default = {} +} + +variable "retention_in_days" { + type = number + default = 30 +} + +variable "log_application_type" { + description = "Log application type" + type = string + default = "other" +} + +variable "key_vault_name" { + description = "Key Vault name - if not specificied will default to computed naming convention" + type = string + default = "" +} + +variable "la_name" { + type = string + default = "" + description = "Name of the Log Analtics Instance to be created." +} + +variable "app_insights_name" { + type = string + default = "" + description = "Name of the App Insights Instance to be created." +} + + + diff --git a/azurerm/modules/azurerm-observability/main.tf b/azurerm/modules/azurerm-observability/main.tf new file mode 100644 index 00000000..4037dd49 --- /dev/null +++ b/azurerm/modules/azurerm-observability/main.tf @@ -0,0 +1,44 @@ +######################################### +# OBSERVABILITY +######################################### + +resource "azurerm_log_analytics_workspace" "default" { + name = var.la_name + resource_group_name = var.resource_group_name + location = var.resource_group_location + sku = "PerGB2018" + retention_in_days = var.retention_in_days + depends_on = [data.azurerm_resource_group.default] + lifecycle { + ignore_changes = [ + tags, + ] + } +} + +resource "azurerm_log_analytics_solution" "default" { + solution_name = "ContainerInsights" + resource_group_name = var.resource_group_name + location = var.resource_group_location + workspace_resource_id = azurerm_log_analytics_workspace.default.id + workspace_name = azurerm_log_analytics_workspace.default.name + depends_on = [data.azurerm_resource_group.default, azurerm_log_analytics_workspace.default] + plan { + publisher = "Microsoft" + product = "OMSGallery/ContainerInsights" + } +} + +resource "azurerm_application_insights" "default" { + name = var.app_insights_name + resource_group_name = var.resource_group_name + location = var.resource_group_location + application_type = var.log_application_type + workspace_id = azurerm_log_analytics_workspace.default.id + depends_on = [data.azurerm_resource_group.default, azurerm_log_analytics_workspace.default] + lifecycle { + ignore_changes = [ + tags, + ] + } +} diff --git a/azurerm/modules/azurerm-observability/output.tf b/azurerm/modules/azurerm-observability/output.tf new file mode 100644 index 00000000..68339b6c --- /dev/null +++ b/azurerm/modules/azurerm-observability/output.tf @@ -0,0 +1,23 @@ +######################################### +# Application Insights +######################################### + +output "app_insights_resource_group_name" { + value = azurerm_log_analytics_workspace.default.resource_group_name +} +output "app_insights_name" { + value = azurerm_log_analytics_workspace.default.name +} + +output "app_insights_id" { + value = azurerm_log_analytics_workspace.default.id +} + +output "app_insights_key" { + value = azurerm_log_analytics_workspace.default.primary_shared_key +} + +output "log_analytics_workspace_id" { + value = azurerm_log_analytics_workspace.default.id +} + diff --git a/azurerm/modules/azurerm-observability/vars.tf b/azurerm/modules/azurerm-observability/vars.tf new file mode 100644 index 00000000..315caa2a --- /dev/null +++ b/azurerm/modules/azurerm-observability/vars.tf @@ -0,0 +1,103 @@ +############################################ +# NAMING +############################################ + +variable "name_company" { + description = "Company Name - should/will be used in conventional resource naming" + type = string +} + +variable "name_project" { + description = "Project Name - should/will be used in conventional resource naming" + type = string +} + +variable "name_component" { + description = "Component Name - should/will be used in conventional resource naming. Typically this will be a logical name for this part of the system i.e. `API` || `middleware` or more generic like `Billing`" + type = string +} + +variable "name_environment" { + type = string +} + +variable "stage" { + type = string + default = "dev" +} + +variable "attributes" { + description = "Additional attributes for tagging" + default = [] +} + +variable "tags" { + description = "Tags to be assigned to all resources, NB if global tagging is enabled these will get overwritten periodically" + type = map(string) + default = {} +} + +variable "resource_namer" { + type = string + description = "User defined naming convention applied to all resources created as part of this module" +} + +variable "resource_tags" { + description = "Map of tags to be applied to all resources created as part of this module" + type = map(string) + default = {} +} + +########################### +# OBSERVABILITY +########################## + +variable "resource_group_name" { + type = string + default = "" + description = "Name of the Data Platform Resource Group." +} + +variable "resource_group_location" { + description = "Location of the RG" + type = string + default = "useast" +} + +variable "resource_group_tags" { + description = "Tags at a RG level" + type = map(string) + default = {} +} + +variable "retention_in_days" { + type = number + default = 30 +} + +variable "log_application_type" { + description = "Log application type" + type = string + default = "other" +} + +variable "key_vault_name" { + description = "Key Vault name - if not specificied will default to computed naming convention" + type = string + default = "" +} + +variable "la_name" { + type = string + default = "" + description = "Name of the Log Analtics Instance to be created." +} + +variable "app_insights_name" { + type = string + default = "" + description = "Name of the App Insights Instance to be created." +} + + + From 52305695004f8fea0432952939d8a2de8f33b325 Mon Sep 17 00:00:00 2001 From: Rhys Bushnell Date: Fri, 21 Jul 2023 14:45:07 +0100 Subject: [PATCH 14/14] update vars --- .../azurerm-observability/example/vars.tf | 30 +++++++++++++++++++ azurerm/modules/azurerm-observability/vars.tf | 30 ------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/azurerm/modules/azurerm-observability/example/vars.tf b/azurerm/modules/azurerm-observability/example/vars.tf index c722e2b8..315caa2a 100644 --- a/azurerm/modules/azurerm-observability/example/vars.tf +++ b/azurerm/modules/azurerm-observability/example/vars.tf @@ -2,6 +2,25 @@ # NAMING ############################################ +variable "name_company" { + description = "Company Name - should/will be used in conventional resource naming" + type = string +} + +variable "name_project" { + description = "Project Name - should/will be used in conventional resource naming" + type = string +} + +variable "name_component" { + description = "Component Name - should/will be used in conventional resource naming. Typically this will be a logical name for this part of the system i.e. `API` || `middleware` or more generic like `Billing`" + type = string +} + +variable "name_environment" { + type = string +} + variable "stage" { type = string default = "dev" @@ -18,6 +37,17 @@ variable "tags" { default = {} } +variable "resource_namer" { + type = string + description = "User defined naming convention applied to all resources created as part of this module" +} + +variable "resource_tags" { + description = "Map of tags to be applied to all resources created as part of this module" + type = map(string) + default = {} +} + ########################### # OBSERVABILITY ########################## diff --git a/azurerm/modules/azurerm-observability/vars.tf b/azurerm/modules/azurerm-observability/vars.tf index 315caa2a..c722e2b8 100644 --- a/azurerm/modules/azurerm-observability/vars.tf +++ b/azurerm/modules/azurerm-observability/vars.tf @@ -2,25 +2,6 @@ # NAMING ############################################ -variable "name_company" { - description = "Company Name - should/will be used in conventional resource naming" - type = string -} - -variable "name_project" { - description = "Project Name - should/will be used in conventional resource naming" - type = string -} - -variable "name_component" { - description = "Component Name - should/will be used in conventional resource naming. Typically this will be a logical name for this part of the system i.e. `API` || `middleware` or more generic like `Billing`" - type = string -} - -variable "name_environment" { - type = string -} - variable "stage" { type = string default = "dev" @@ -37,17 +18,6 @@ variable "tags" { default = {} } -variable "resource_namer" { - type = string - description = "User defined naming convention applied to all resources created as part of this module" -} - -variable "resource_tags" { - description = "Map of tags to be applied to all resources created as part of this module" - type = map(string) - default = {} -} - ########################### # OBSERVABILITY ##########################