feat: log analytics diagnostic settings for adls, kv and adf (#77)

azurerm/modules/azurerm-adb/network.tf

@@ -147,7 +147,7 @@ resource "azurerm_private_endpoint" "databricks" {
   }
 
   private_dns_zone_group {
-    
+
     name                 = "databricks_ui_api"
     private_dns_zone_ids = [data.azurerm_private_dns_zone.adb_pvt_dns[0].id]
   }
@@ -156,7 +156,7 @@ resource "azurerm_private_endpoint" "databricks" {
 }
 
 resource "azurerm_private_endpoint" "auth" {
-  count = var.enable_private_network && var.managed_vnet == false  && var.browser_authentication_enabled == true ? 1 : 0
+  count               = var.enable_private_network && var.managed_vnet == false && var.browser_authentication_enabled == true ? 1 : 0
   name                = "${var.resource_namer}-pe-databricks-auth"
   location            = var.resource_group_location
   resource_group_name = var.resource_group_name

azurerm/modules/azurerm-adf/observability.tf

@@ -0,0 +1,43 @@
+data "azurerm_monitor_diagnostic_categories" "adf_log_analytics_categories" {
+  count       = var.la_workspace_id != "" ? 1 : 0
+  resource_id = azurerm_data_factory.example[0].id
+
+  depends_on = [azurerm_data_factory.example]
+}
+
+resource "azurerm_monitor_diagnostic_setting" "adf_log_analytics" {
+  count                          = var.la_workspace_id != "" ? 1 : 0
+  name                           = "ADF to Log Analytics"
+  target_resource_id             = azurerm_data_factory.example[0].id
+  log_analytics_workspace_id     = var.la_workspace_id
+  log_analytics_destination_type = "Dedicated"
+
+  dynamic "log" {
+    for_each = data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories[0].logs
+
+    content {
+      category = log.value
+      enabled  = true
+
+      retention_policy {
+        enabled = false
+        days    = 0
+      }
+    }
+  }
+
+  dynamic "metric" {
+    for_each = data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories[0].metrics
+
+    content {
+      category = metric.value
+      enabled  = true
+
+      retention_policy {
+        enabled = false
+        days    = 0
+      }
+    }
+  }
+  depends_on = [data.azurerm_monitor_diagnostic_categories.adf_log_analytics_categories]
+}

azurerm/modules/azurerm-adf/vars.tf

@@ -95,6 +95,12 @@ variable "runtime_virtual_network_enabled" {
   description = "Is Integration Runtime compute provisioned within Managed Virtual Network? Changing this forces a new resource to be created."
 }
 
+variable "la_workspace_id" {
+  type        = string
+  default     = ""
+  description = "Log Analytics Workspace ID"
+}
+
 
 ###########################
 # Global parameter  for ADF SETTINGS

azurerm/modules/azurerm-adls/observability.tf

@@ -0,0 +1,51 @@
+data "azurerm_monitor_diagnostic_categories" "adls_log_analytics_categories" {
+  for_each = {
+    for account_name, account_details in var.storage_account_details : account_name => account_details
+    if var.la_workspace_id != ""
+  }
+  resource_id = azurerm_storage_account.storage_account_default["${each.key}"].id
+
+  depends_on = [azurerm_storage_account.storage_account_default]
+}
+
+resource "azurerm_monitor_diagnostic_setting" "adls_log_analytics" {
+  for_each = {
+    for account_name, account_details in var.storage_account_details : account_name => account_details
+    if var.la_workspace_id != ""
+  }
+
+  name                           = "Storage to Log Analytics"
+  target_resource_id             = azurerm_storage_account.storage_account_default["${each.key}"].id
+  log_analytics_workspace_id     = var.la_workspace_id
+  log_analytics_destination_type = "Dedicated"
+
+  dynamic "log" {
+    for_each = data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories[each.key].logs
+
+    content {
+      category = log.value
+      enabled  = true
+
+      retention_policy {
+        enabled = false
+        days    = 0
+      }
+    }
+  }
+
+  dynamic "metric" {
+    for_each = data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories[each.key].metrics
+
+    content {
+      category = metric.value
+      enabled  = true
+
+      retention_policy {
+        enabled = false
+        days    = 0
+      }
+    }
+  }
+
+  depends_on = [data.azurerm_monitor_diagnostic_categories.adls_log_analytics_categories]
+}

azurerm/modules/azurerm-adls/vars.tf

@@ -170,3 +170,9 @@ variable "dfs_dns_resource_group_name" {
   default     = "amido-stacks-euw-de-hub-network"
   description = "Name of the resource group where pvt dns is present for blob."
 }
+
+variable "la_workspace_id" {
+  type        = string
+  default     = ""
+  description = "Log Analytics Workspace ID"
+}

azurerm/modules/azurerm-hub-spoke/dns.tf

@@ -7,7 +7,7 @@ resource "azurerm_private_dns_zone" "example" {
 }
 
 resource "azurerm_private_dns_zone_virtual_network_link" "hub-privatelink-dns" {
-  for_each              = var.link_dns_network == true ? toset(var.dns_zone_name ) : toset([])
+  for_each              = var.link_dns_network == true ? toset(var.dns_zone_name) : toset([])
   name                  = each.key
   resource_group_name   = local.hub_resource_group_name[0]
   registration_enabled  = false # Auto registration_enabled set to false as we cannot add multiple Private DNS to 1 Vnet

azurerm/modules/azurerm-kv/example/main.tf

@@ -21,8 +21,8 @@ resource "azurerm_resource_group" "default" {
 }
 
 module "kv_default" {
-  source                    = "../../azurerm-kv"
-    resource_namer             = substr(replace(module.default_label.id, "-", ""), 0, 24)
+  source                     = "../../azurerm-kv"
+  resource_namer             = substr(replace(module.default_label.id, "-", ""), 0, 24)
   resource_group_name        = azurerm_resource_group.default.name
   resource_group_location    = azurerm_resource_group.default.location
   create_kv_networkacl       = false
@@ -33,7 +33,7 @@ module "kv_default" {
   pe_subnet_id               = data.azurerm_subnet.pe_subnet.id
   pe_resource_group_name     = data.azurerm_subnet.pe_subnet.resource_group_name
   pe_resource_group_location = "UK South"
- # private_dns_zone_name      = data.azurerm_private_dns_zone.private_dns.name
- # private_dns_zone_ids       = ["${data.azurerm_private_dns_zone.private_dns.id}"]
+  # private_dns_zone_name      = data.azurerm_private_dns_zone.private_dns.name
+  # private_dns_zone_ids       = ["${data.azurerm_private_dns_zone.private_dns.id}"]
   dns_resource_group_name = "hub-rg"
 }

azurerm/modules/azurerm-kv/observability.tf

@@ -0,0 +1,44 @@
+data "azurerm_monitor_diagnostic_categories" "kv_log_analytics_categories" {
+  count       = var.la_workspace_id != "" ? 1 : 0
+  resource_id = azurerm_key_vault.example[0].id
+
+  depends_on = [azurerm_key_vault.example]
+}
+
+resource "azurerm_monitor_diagnostic_setting" "kv_log_analytics" {
+  count                          = var.la_workspace_id != "" ? 1 : 0
+  name                           = "KV to Log Analytics"
+  target_resource_id             = azurerm_key_vault.example[0].id
+  log_analytics_workspace_id     = var.la_workspace_id
+  log_analytics_destination_type = "Dedicated"
+
+  dynamic "log" {
+    for_each = data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories[0].logs
+
+    content {
+      category = log.value
+      enabled  = true
+
+      retention_policy {
+        enabled = false
+        days    = 0
+      }
+    }
+  }
+
+  dynamic "metric" {
+    for_each = data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories[0].metrics
+
+    content {
+      category = metric.value
+      enabled  = true
+
+      retention_policy {
+        enabled = false
+        days    = 0
+      }
+    }
+  }
+
+  depends_on = [data.azurerm_monitor_diagnostic_categories.kv_log_analytics_categories]
+}

azurerm/modules/azurerm-kv/vars.tf

@@ -194,3 +194,9 @@ variable "pe_resource_group_location" {
   default     = ""
   description = "Location of the resource group to provision private endpoint in."
 }
+
+variable "la_workspace_id" {
+  type        = string
+  default     = ""
+  description = "Log Analytics Workspace ID"
+}

azurerm/modules/azurerm-observability/README.md

@@ -0,0 +1,52 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13 |
+| <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm) | ~> 3.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | ~> 3.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [azurerm_application_insights.default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/application_insights) | resource |
+| [azurerm_log_analytics_solution.default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_solution) | resource |
+| [azurerm_log_analytics_workspace.default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource |
+| [azurerm_client_config.spn_client](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source |
+| [azurerm_resource_group.default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_app_insights_name"></a> [app\_insights\_name](#input\_app\_insights\_name) | Name of the App Insights Instance to be created. | `string` | `""` | no |
+| <a name="input_attributes"></a> [attributes](#input\_attributes) | Additional attributes for tagging | `list` | `[]` | no |
+| <a name="input_key_vault_name"></a> [key\_vault\_name](#input\_key\_vault\_name) | Key Vault name - if not specificied will default to computed naming convention | `string` | `""` | no |
+| <a name="input_la_name"></a> [la\_name](#input\_la\_name) | Name of the Log Analtics Instance to be created. | `string` | `""` | no |
+| <a name="input_log_application_type"></a> [log\_application\_type](#input\_log\_application\_type) | Log application type | `string` | `"other"` | no |
+| <a name="input_resource_group_location"></a> [resource\_group\_location](#input\_resource\_group\_location) | Location of the RG | `string` | `"useast"` | no |
+| <a name="input_resource_group_name"></a> [resource\_group\_name](#input\_resource\_group\_name) | Name of the Data Platform Resource Group. | `string` | `""` | no |
+| <a name="input_resource_group_tags"></a> [resource\_group\_tags](#input\_resource\_group\_tags) | Tags at a RG level | `map(string)` | `{}` | no |
+| <a name="input_retention_in_days"></a> [retention\_in\_days](#input\_retention\_in\_days) | n/a | `number` | `30` | no |
+| <a name="input_stage"></a> [stage](#input\_stage) | n/a | `string` | `"dev"` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | Tags to be assigned to all resources, NB if global tagging is enabled these will get overwritten periodically | `map(string)` | `{}` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_app_insights_id"></a> [app\_insights\_id](#output\_app\_insights\_id) | n/a |
+| <a name="output_app_insights_key"></a> [app\_insights\_key](#output\_app\_insights\_key) | n/a |
+| <a name="output_app_insights_name"></a> [app\_insights\_name](#output\_app\_insights\_name) | n/a |
+| <a name="output_app_insights_resource_group_name"></a> [app\_insights\_resource\_group\_name](#output\_app\_insights\_resource\_group\_name) | n/a |
+| <a name="output_log_analytics_workspace_id"></a> [log\_analytics\_workspace\_id](#output\_log\_analytics\_workspace\_id) | n/a |

azurerm/modules/azurerm-observability/constraints.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~> 3.0"
+    }
+  }
+}

azurerm/modules/azurerm-observability/data.tf

@@ -0,0 +1,6 @@
+data "azurerm_client_config" "spn_client" {
+}
+
+data "azurerm_resource_group" "default" {
+  name = var.resource_group_name
+}

azurerm/modules/azurerm-observability/example/constraints.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~> 3.0"
+    }
+  }
+}

azurerm/modules/azurerm-observability/example/main.tf

@@ -0,0 +1,31 @@
+
+module "default_label" {
+  source     = "git::https://github.com/cloudposse/terraform-null-label.git?ref=0.25.0"
+  namespace  = "${var.name_company}-${var.name_project}"
+  stage      = var.stage
+  name       = "${lookup(var.location_name_map, var.resource_group_location, "uksouth")}-${var.name_component}"
+  attributes = var.attributes
+  delimiter  = "-"
+  tags       = var.tags
+}
+
+##################################################
+# ResourceGroups
+##################################################
+
+resource "azurerm_resource_group" "default" {
+  name     = module.default_label.id
+  location = var.resource_group_location
+  tags     = var.tags
+}
+
+
+module "observability" {
+  source                     = "../../azurerm-observability"
+  resource_group_name     = azurerm_resource_group.default.name
+  resource_group_location = azurerm_resource_group.default.location
+  la_name                 = module.default_label.id
+  app_insights_name       = module.default_label.id
+  log_application_type    = "other"
+  retention_in_days       = var.retention_in_days
+}