From 9e0fd92903dd55e2c4e7bae4b4702e9e46971435 Mon Sep 17 00:00:00 2001 From: satenderrathee <88006547+satenderrathee@users.noreply.github.com> Date: Tue, 18 Apr 2023 16:06:09 +0100 Subject: [PATCH] Feature/azuredatabricks (#47) * added initial files * added example * added Readme.md * updated * added diagnostic settings for ADB * updated diagnostic setting * fixed * added diagnostic setting * updated example * updated docs * updated diagnostic setting * updated example * fixed * fixed * fixed * updated adb * updated * fixed * removed log_analytics_destination_type * Update azurerm/modules/azurerm-adb/example/constraints.tf Co-authored-by: Rishi <51043972+Trishisingh@users.noreply.github.com> --------- Co-authored-by: Rishi <51043972+Trishisingh@users.noreply.github.com> --- azurerm/modules/azurerm-adb/README.md | Bin 0 -> 10182 bytes azurerm/modules/azurerm-adb/constraints.tf | 12 +++ .../azurerm-adb/example/constraints.tf | 13 +++ azurerm/modules/azurerm-adb/example/main.tf | 42 ++++++++ azurerm/modules/azurerm-adb/example/output.tf | 7 ++ .../modules/azurerm-adb/example/provider.tf | 3 + azurerm/modules/azurerm-adb/example/var.tf | 96 ++++++++++++++++++ azurerm/modules/azurerm-adb/main.tf | 60 +++++++++++ azurerm/modules/azurerm-adb/output.tf | 8 ++ azurerm/modules/azurerm-adb/var.tf | 63 ++++++++++++ 10 files changed, 304 insertions(+) create mode 100644 azurerm/modules/azurerm-adb/README.md create mode 100644 azurerm/modules/azurerm-adb/constraints.tf create mode 100644 azurerm/modules/azurerm-adb/example/constraints.tf create mode 100644 azurerm/modules/azurerm-adb/example/main.tf create mode 100644 azurerm/modules/azurerm-adb/example/output.tf create mode 100644 azurerm/modules/azurerm-adb/example/provider.tf create mode 100644 azurerm/modules/azurerm-adb/example/var.tf create mode 100644 azurerm/modules/azurerm-adb/main.tf create mode 100644 azurerm/modules/azurerm-adb/output.tf create mode 100644 azurerm/modules/azurerm-adb/var.tf diff --git a/azurerm/modules/azurerm-adb/README.md b/azurerm/modules/azurerm-adb/README.md new file mode 100644 index 0000000000000000000000000000000000000000..64976d6647fe4cba7dbf5468189ae1d5a6443229 GIT binary patch literal 10182 zcmdU#ZEqXL5y$s+fP9Cm9H2E2Nh$o!umZ@Ik}9!eRg~izNkCE(Wvh`Wmb4-}b^6iU z{(n1~+r7ObO)CzP;J}C5o!yz8`Om}d-T(gkAp9w;>Z-!SupPb(J7F*M!(sSB-&^5H zcocf-gW~h>arh|Q(cOMHP|sJQe~4wR+$W?W!$7w2G^mM2chme|0my61PztJJvtAy$~%kC$5+3cY5?17Vl~JK`j%_cpUpaZAdqatDtul$C)m# zB{dR^q{mUzOnauZ#%s|cGj>Mz7y5^#tHRdcY+ts25}rr=?L{;m>MmKnlBA`bo=3bs zZfNj8bMg6mcq+X1^f}aBuZ7yp@KcS#p8g$amB*429)=EZR_Vg0dXi*MzuxBUhfxcl z-fY1NL7(B$5NTKdb^N>3H)xw?BivVplEeO^wKK`nWo@U%Lv}P9RPdX%q;sq#iNL&} z-e(zTyrBc0eQ`YrAYoSM#4(&Gn&Ut+`WtB$#)FaZjs0!RpvHNZ){aDbEJQNuq+SWS zXU>`Qh9`p5N?rz-iqItr<8o>@u-cS>P&%_&SfsMr9 zeaWMArQWgRtfH(_QM;ee4i3>&iYmjK`%*Ty=S*DD8qy6T0*GOWe>;va5-l}-_dKou zTc9z+l3}dmjJ41P>o&Bx)mT4hV1BX#NKWy75V+9&OcdDeB^+T9 zbT#Vh6QO^iRh|f!_rtozl55uWIn@|64>r*TUQM1FX3{zio3dME9EI1C06$=sO~v6|MU+R1>^s7$MfDw36&s?ns|w%6b$8QJ5m0?> z3)36YuwUh2Ae4?yW`r->WABT;a{KySdJ-KX&nzc4pjXlkO@K5tE$A(8+k4J93 zfhd0xXME>H)ayWZu1E7$kb%xDrhyeARf)6G`KKY3v06+fDDVT4fU@`PRHzO z1GV7Z=3AAn!11ayMK!V#H9D1s`=am&u$WgRNf&ctzImLQS*b5d&r1DhYjEIM(>a;U z4y9O-fb!R z9hO~@#UE&;!)W<$J1D+aR=hK3#_tpz+FHIVX;|6kF&=y-UHvuOi*2rf1>-hC8nReO6zb{L@uSUu_uv&AL z<#e$tO`OOY_-)d|o?5A~^S)G*C9lZ$9BFjh9}ZPtu%lbX6Os>nTdb%rNy!&O`7pas ztJ3-%v1m=aUW?9820+nEjeI33X0Z-jkLah?Aihr}AKu@Nk$CXBMwN(JkM$}wDRK}C zC(+|_JFZKssb*Z4^z00%EAi!~?&w`^ZEn|cj-=Zvv^9q8r&exSw1uMp?TiwjEtr;kbv8 zvr7)NS$wsOUPf42+k9x{`~pnh(LY}c?Y|y3N7tdUAi4dq{F2lCP4RE$g(^IYKAWGW zvy&&^be%VHhJGnMFzZ(IU8>>z!Wum30CNM)Lk_pBHK<$15qIb6H#%}%Lff(^Xx17t z?WnI>HLL45r$zThy~rxvBTCKo!q+;#VHd>;{LH*OE$x!RPTXt{%gLM7VDojhB1Dpv zI9s@i)%l0h-nI0xT;-9@C+7K1t~cg2q_=AtvqE0g&;mZ$YcFe=+jA%_>1fX7@7@V_ zRcN-yP8C|(A->BOR%$x{6wEc)jT?)IM~W0gQ1{+J%)>kZ z8`W*^7Z1z*#^Tew(uU<|?3hjixg+c4=Yd^Td;eM7P9Qv=ooCRe()oe((X3OGREx9A zisB>rES*%ku0Nz+JyN~DK7Hiv^ys5qe|t9()Ad$bqWQ#eepcp*3Ar4KH&OS?XXKJ+ zisRE(MlNads6XwD`-?}o9-c-I!qYjsVK>K#B{hs`%xVySy2%>=?C<;>kT*Z5_}IHJ z#-ata63@U&35W^#(+Nh{b2`qy?Fo{hnD}E)&+{{zMfm-&cHm6rg*3AaN}YCDTspm5 zU{}0&s_iQ`&aO*)`uN-EndhJrRp5Cwdbf_Y1HIMpbA6Y`h_wHDqL{)N)$Mm+c8ckH z9)7#_N8ax{udZ6y;+pS?e3#u-V|MRyY{a~LO?%q0T*5HjrH}sJMaHZ=m%v_6$OYKW=A`h-46fI4s2JuCEgTkztv;A zHDV2W17Zv(CcI&gpO|_*avtLx@h-iK!1HU~c;eZ)?a|ng*t;0KxkGhkXQ{lKkj{+< W1rPG2pE9I(^LX04CB1gPm-m0dq~9$7 literal 0 HcmV?d00001 diff --git a/azurerm/modules/azurerm-adb/constraints.tf b/azurerm/modules/azurerm-adb/constraints.tf new file mode 100644 index 00000000..9baec76e --- /dev/null +++ b/azurerm/modules/azurerm-adb/constraints.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.13" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + databricks = { + source = "databricks/databricks" + } + } +} diff --git a/azurerm/modules/azurerm-adb/example/constraints.tf b/azurerm/modules/azurerm-adb/example/constraints.tf new file mode 100644 index 00000000..7ed79660 --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/constraints.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 0.13" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } +#TODO: note this is just added right now without any use, can be used In future for databricks provider + databricks = { + source = "databricks/databricks" + } + } +} diff --git a/azurerm/modules/azurerm-adb/example/main.tf b/azurerm/modules/azurerm-adb/example/main.tf new file mode 100644 index 00000000..9282bb77 --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/main.tf @@ -0,0 +1,42 @@ +data "azurerm_client_config" "current" {} + +module "default_label" { + source = "git::https://github.com/cloudposse/terraform-null-label.git?ref=0.25.0" + namespace = "${var.name_company}-${var.name_project}" + stage = var.stage + name = "${lookup(var.location_name_map, var.resource_group_location, "uksouth")}-${var.name_component}" + attributes = var.attributes + delimiter = "-" + tags = var.tags +} + +################################################## +# ResourceGroups +################################################## + +resource "azurerm_resource_group" "default" { + name = module.default_label.id + location = var.resource_group_location + tags = var.tags +} + +resource "azurerm_log_analytics_workspace" "la" { + name = module.default_label.id + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + sku = var.la_sku + retention_in_days = var.la_retention + tags = module.default_label.tags +} + +module "adb" { + source = "../../azurerm-adb" + resource_namer = module.default_label.id + resource_group_name = azurerm_resource_group.default.name + resource_group_location = azurerm_resource_group.default.location + databricks_sku = var.databricks_sku + resource_tags = module.default_label.tags + enable_databricksws_diagnostic = var.enable_databricksws_diagnostic + #databricksws_diagnostic_setting_name = var.databricksws_diagnostic_setting_name + data_platform_log_analytics_workspace_id = azurerm_log_analytics_workspace.la.id +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/example/output.tf b/azurerm/modules/azurerm-adb/example/output.tf new file mode 100644 index 00000000..d337ac39 --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/output.tf @@ -0,0 +1,7 @@ +output "adb_databricks_id" { + value = module.adb.adb_databricks_id +} + +output "databricks_hosturl" { + value = module.adb.databricks_hosturl +} diff --git a/azurerm/modules/azurerm-adb/example/provider.tf b/azurerm/modules/azurerm-adb/example/provider.tf new file mode 100644 index 00000000..615cb43d --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/provider.tf @@ -0,0 +1,3 @@ +provider "azurerm" { + features {} +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/example/var.tf b/azurerm/modules/azurerm-adb/example/var.tf new file mode 100644 index 00000000..5a67940b --- /dev/null +++ b/azurerm/modules/azurerm-adb/example/var.tf @@ -0,0 +1,96 @@ +############################################ +# NAMING +############################################ + +variable "name_company" { + description = "Company Name - should/will be used in conventional resource naming" + type = string +} + +variable "name_project" { + description = "Project Name - should/will be used in conventional resource naming" + type = string +} + +variable "name_component" { + description = "Component Name - should/will be used in conventional resource naming. Typically this will be a logical name for this part of the system i.e. `API` || `middleware` or more generic like `Billing`" + type = string +} + +variable "name_environment" { + type = string +} + +variable "stage" { + type = string + default = "dev" +} + +variable "attributes" { + description = "Additional attributes for tagging" + default = [] +} + +variable "tags" { + description = "Tags to be assigned to all resources, NB if global tagging is enabled these will get overwritten periodically" + type = map(string) + default = {} +} + + +variable "resource_group_location" { + type = string + default = "uksouth" +} + + +# Each region must have corresponding a shortend name for resource naming purposes +variable "location_name_map" { + type = map(string) + + default = { + northeurope = "eun" + westeurope = "euw" + uksouth = "uks" + ukwest = "ukw" + eastus = "use" + eastus2 = "use2" + westus = "usw" + eastasia = "ase" + southeastasia = "asse" + } +} +variable "databricks_sku" { + type = string + default = "premium" + description = "The SKU to use for the databricks instance" + + validation { + condition = can(regex("standard|premium|trial", var.databricks_sku)) + error_message = "Err: Valid options are 'standard', 'premium' or 'trial'." + } +} + +variable "enable_databricksws_diagnostic" { + type = bool + description = "Whether to enable diagnostic settings for the Azure Databricks workspace" + default = true +} + +variable "databricksws_diagnostic_setting_name" { + type = string + default = "Databricks to Log Analytics" + description = "The Databricks workspace diagnostic setting name." +} + +variable "la_sku" { + type = string + default = "PerGB2018" + description = "Specifies the SKU of the Log Analytics Workspace." +} + +variable "la_retention" { + type = number + default = 30 + description = "The workspace data retention in days. Possible values are either 7 (Free Tier only) or range between 30 and 730." +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf new file mode 100644 index 00000000..7ad5306a --- /dev/null +++ b/azurerm/modules/azurerm-adb/main.tf @@ -0,0 +1,60 @@ +data "azurerm_client_config" "current" { +} + + +resource "azurerm_databricks_workspace" "example" { + name = var.resource_namer + location = var.resource_group_location + resource_group_name = var.resource_group_name + sku = var.databricks_sku + + + tags = var.resource_tags + lifecycle { + ignore_changes = [ + tags, + ] + } +} + + +# Enable diagnostic settings for ADB +data "azurerm_monitor_diagnostic_categories" "adb_log_analytics_categories" { + resource_id = azurerm_databricks_workspace.example.id +} + +resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { + count = var.enable_databricksws_diagnostic ? 1 : 0 + #for_each = var.enable_databricksws_diagnostic ? { "enabled" = true } : {} + name = var.databricksws_diagnostic_setting_name + target_resource_id = azurerm_databricks_workspace.example.id + log_analytics_workspace_id = var.data_platform_log_analytics_workspace_id + + dynamic "log" { + for_each = data.azurerm_monitor_diagnostic_categories.adb_log_analytics_categories.logs + + content { + category = log.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } + + dynamic "metric" { + for_each = data.azurerm_monitor_diagnostic_categories.adb_log_analytics_categories.metrics + + content { + category = metric.value + enabled = true + + retention_policy { + enabled = false + days = 0 + } + } + } +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/output.tf b/azurerm/modules/azurerm-adb/output.tf new file mode 100644 index 00000000..2390541f --- /dev/null +++ b/azurerm/modules/azurerm-adb/output.tf @@ -0,0 +1,8 @@ +output "adb_databricks_id" { + value = azurerm_databricks_workspace.example.id +} + +output "databricks_hosturl" { + description = "Azure Databricks HostUrl" + value = "https://${azurerm_databricks_workspace.example.workspace_url}/" +} diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf new file mode 100644 index 00000000..832fbbb6 --- /dev/null +++ b/azurerm/modules/azurerm-adb/var.tf @@ -0,0 +1,63 @@ +############################################ +# NAMING +############################################ + +variable "resource_namer" { + type = string + description = "User defined naming convention applied to all resources created as part of this module" +} + +variable "resource_tags" { + description = "Map of tags to be applied to all resources created as part of this module" + type = map(string) + default = {} +} + +############################################ +# RESOURCE INFORMATION +############################################ + +variable "resource_group_location" { + type = string + default = "uksouth" + description = "Location of Resource group" +} + +variable "resource_group_name" { + type = string + description = "Name of resource group" +} + +variable "databricks_sku" { + type = string + default = "premium" + description = "The SKU to use for the databricks instance" + + validation { + condition = can(regex("standard|premium|trial", var.databricks_sku)) + error_message = "Err: Valid options are 'standard', 'premium' or 'trial'." + } +} + + +############################################ +# Resource Diagnostic Setting +############################################ + +variable "enable_databricksws_diagnostic" { + type = bool + description = "Whether to enable diagnostic settings for the Azure Databricks workspace" + default = false +} + +variable "databricksws_diagnostic_setting_name" { + type = string + default = "Databricks to Log Analytics" + description = "The Databricks workspace diagnostic setting name." +} + +variable "data_platform_log_analytics_workspace_id" { + type = string + default = null + description = "The Log Analytics Workspace used for the whole Data Platform." +} \ No newline at end of file