From 90002591570871067a1fdbcbc90ee583d4ff4e2f Mon Sep 17 00:00:00 2001 From: Rishi <51043972+Trishisingh@users.noreply.github.com> Date: Thu, 20 Jul 2023 09:38:29 +0100 Subject: [PATCH] Feature/new secure databricks (#76) * initial commit of databricks changes * add nat * use nat and ip reosurces * paramatise nat timeout * add vars * add nat to both subnets * add pe subnet * add ability to create subnets * temp comment out count on outputs * update prefix vars to lists * update * add index * add index to nsg * add index to pip and nat * fix typo * add count to pe subnet * add index * fix typo * add index to outputs * update * update * make nat gateway single region * make pip single zone * add depends on * add service endpoints * add permissions for network and dns * add index * add permissions to depends on * add databricks user * update * add peering * remove reference to data lookup * add index to vnet * add permissions to db * comment out perms * add rg * add depends on * use name instead of id * remove depends on * update id * remove db peer * update var for public access * add route table * fix typo * add prefix/ * remove rt * add config for lb * update lb name to local * add index * update * remove param * temp comment out all params * comment out rules temp * temp commetn out pe * temp commetn out pe * temp commetn out pe * add configurable managed vnet * update priority * update vnet id * update index * temp comment out nsg * remove reference to nsg ass * temp fix subnet names * add nsg ass to custom param * update nsg rule * temp comment out count * update index * update" * temp comment out count on subnets * revert stuff * comment out pe * comment out data * comment out data * comment out rule * comment out service endpoints * create pr subnet and pe * update subnet range * update prefix to list * configure pip * update networking * updates * update name * variable for pe prefix * add datalookup for pe subnet * update pe subnet name to var * split var for creating pe subnet * update condition * update condition * update names * update condition * update name * update readme and tidy up * add nsg rule to allow databricks into vnet * add nsg rules * add depends on * add depends on * add auth pe * comment out data * update depends on * update * add depends on to db resources * make string * update * remove whitespace * update dbfs to string * temp remove dbfs explore * add dbfs explorer * temp remove dbfs * update * update dns zone name * run fmt * add permissions * fix name * temp comment out * remove perm * add db back in * remove depends on * comment out db * update pe * comment out db * rename dns zone * add users and conf * add dns update * update default value * update var name * add condition * add condition * update rg name * add condition to cname * tidy up comments * tidy up comments * updated for new * feat: added dns for adls * updated * updated * removed databrick provider code * updated for pe --------- Co-authored-by: Rhys Bushnell Co-authored-by: Satenderrathee --- azurerm/modules/azurerm-adb/README.md | Bin 14142 -> 29500 bytes azurerm/modules/azurerm-adb/constraints.tf | 4 - azurerm/modules/azurerm-adb/data.tf | 45 ++++ azurerm/modules/azurerm-adb/load-balancer.tf | 39 ++++ azurerm/modules/azurerm-adb/local.tf | 5 + azurerm/modules/azurerm-adb/main.tf | 57 ++--- azurerm/modules/azurerm-adb/nat.tf | 31 +++ azurerm/modules/azurerm-adb/network.tf | 218 +++++++++++++++++++ azurerm/modules/azurerm-adb/var.tf | 163 ++++++++++++-- 9 files changed, 502 insertions(+), 60 deletions(-) create mode 100644 azurerm/modules/azurerm-adb/data.tf create mode 100644 azurerm/modules/azurerm-adb/load-balancer.tf create mode 100644 azurerm/modules/azurerm-adb/local.tf create mode 100644 azurerm/modules/azurerm-adb/nat.tf create mode 100644 azurerm/modules/azurerm-adb/network.tf diff --git a/azurerm/modules/azurerm-adb/README.md b/azurerm/modules/azurerm-adb/README.md index d51fbf5f8c65ffd11d1c57f1f5e6fb903c5003e6..c0749eed4e69d65c92bd41513adf3acb2cf92dee 100644 GIT binary patch literal 29500 zcmeI5Yj0b}5r)s_0{suH62LJK$qDk!uv5sFq-tzMmE;&f5>R(rjV!669ml!!r?-8d z9S!$#mgFHtxn>d&63KJ+GW*VDXJ&T~|NEai&0m^jy?V{P=3(=Bv(~IP{bsxQO#dG= zUpDufE#0HUr_G<6KQ-^^+eWjgJAYUC`zqO2jn(F{zHy(w2l~9Fdz)%)LnYQ!dcC3 z?a`2yRrRi4iY(D&E6L;v(0DLGe9XUz|~ zI?&&l?pf(`eLL6ZlTbdT>WxZ4X)lcGJe1hi=a-lDd+O~l)IHTFdV8j_v>baq(|s=w zF4ksGC_Gn5C|((kXro2Raq~j;PgLrK{!Z0W+PkAS9%GPZY=?m_M%`=mebIccE9Yu) zKivB;qi#2hg1duI&#-(gs-fUeI35OM+AYN~zE)XihR}#&KFSc>R9pQ0qj@5E-O}H-zHPOncD4CwD1}@4w5L(-i%R-1 z=|IkUdZANWqGVm~e$B6M2NaNc;{_u``b;iOB25;MIzFB0Khk!Q)Nozd6&>~&)Q&|@ zjoV%t4Ovrbl%l^ui#nIJND(q`Qtz$osJ=-DefHJsL4za=3QSyvV~M;Tk_>)_YK8j9 z$mu2jt2s4}U=I=I$20#JIPSIR^Be4#$L2NtGxht185vU}>`Op;?I(5PL}MvZ-4H*H04 zOjF8!NA=ND&y2|0a=mw}$NLk>DxAb?U zIv|g1f(Ba6Y=a!5WdoJk*VW}v_O{B}C@Im?y*=GK4eyaQqt@jxr`;^$Ve_LXKo4kT zRd)EXEXsXZ_BF|>S@kt}6?aw6UcKhqFz$L;JOaFrhmz^LLfBtrtu1N9J?TX0sr1z^ zy?H%&l=sKnfY+1K7E3eMeAMvM^auSk{c_oGYwPyNxwoT|KdEPQ=UKpYN8cPr(^ez{ z%*>`CD_E*h&Pwg`9;1DaGp@}vsqJIV+ST9AeU)$vFut#zC>`G$?TzXwpOyM5>9bNl@ESSrS<`zmnwh(>IYzTn+m}Y-BjIzI zJ*iV)(pC{DcTA1iUh|7ywf^gEAi1wq-fFk0 zs?p+eDb_!qrI&-e5M3Qh_U>w4!EUEs)XGo}PjolcaQc2P#DJTk>0gpZxD=xd^Cc<8 z_|}K|#J8_hj#=8gWH7LD77^P2Uqo&B@^ZcAj@s)Dwe+s^fWF&Q^edcQ636doq`QIh z-&9I`r&;m65iNcz>(EB|rl?_Lp9XvIk+Ax>`6OI(1iDju34RrRAFf_4)|=6??g&zJ63~&t8DpbD{lGR1AWzdMjus{t$D} zh3LaVwe2`o=9X%eQhh7<)cDQt4tsqb5Kd+7mQam;?JeO<^n&k;y~(lTZd)E|V{?v{ zZA5H)O&hrr$r<)|uUP~vpmX+d_EkHbhtS>wy$rE|G=(VPOlTM)ls`6N%uH6N|5? zR88Y8PD5t#%!&J{f!$aHd}l#oHq1eXxb9ha$Fn8dvJy~xxe!>@M=(z`uyo{>$;BXn zWS5aFa_Y=QNU6t&j+~WR1X-)&H_e!5#Tq^KD9!YGUpV8(y_5{5u{fv>B%_OgXZLZ< zh8%uW%n4&BL5Iu(yd*+rb|K2ejvh#x<+6{(JXMAdEcbNZCDrqPFm1Ile*Xc3Vu zM>Z>7<*dCTw3xjY1Cx^a*-)@GVAD*-&+L$1TLdgpdgp^eK7*$1|32|ZW$pKkhOIoc zW1`U>f9bEJ_<3uKCRI*%85xd3;p$pBZI9TpQBvz|{egNdi&@HN?z7n*pn4oOiu>>) zwWy1T`_)mHj}Ba&j$AYT;6=JdkmvX7xx=^zz~uU@@pbsVEDrj?irjVJTBE1FrZL^B zNE}f&8LiQg=41LH%kUy({4W%Dp#8rkl=AUi1v8hI6(MBnDT^vYK9Z}T;n}{pW7+o0 zu>XOTL)HNoOS8s}s2fuGnmBln6R}-35gm4ikjGC>V%5dKw`RR8ro9{B%mfvxsx(ZsUC(bPTuIFY(Bb1J0o`|GT#k-xidi&Un zihdWzM$JoV#;-#}yR$gRj33c78s>7D?vP_{OFfC`liEtnb>LK^W7^op+h|+pQRC@` zRNTq8CeMnz5+@bvOH6W5&T*HXl{`{i_WH1Eni89; z9rwv{M!Dz$jS1s(~DWU9kZ0X zE3GU$C%rVEJz6kt{yq6!-n#7rrTx)$oNH@}3vAam*x6|a_8Gnsf_6U5&Zpw*`2NfE z{7-&z!lftG1Ws#sM9;|zR@B%bA0y_t$C*1fidZ)v>kXBNC!*q6CT4c_iLlGtPVVKs zaF{Bx5OWSO?x~?XylxfV1zrETySd&LxQParX91TFGeJ z>!T^h*xUO^WszylV1ok4gC=>7(O5*p=7EOo$tg9h9H*7SR!>62O3sc6r>nL#`p*M? zahKTJ!lUgOPr`;8$V}>CbU~_C&vDyQn{7M9i9xKCpW|YsH9wu3Tc_HYF1HkU7TwNN zrYro|LcBg-&3$|LvuZvWb6;DZB#*WV@3`KVZXI4qRyQ5p2q=(K*cb9V*Wm5xX3BxV zL`?G1T_iBC;TT=N?B)H&)59j$*NLys%j`L{z5ad<-_GE18W1a-@8hlQ>bLWWrJQ>E z*n_eMn^!o~$)|QNTy4*}T;()wji1-oo6Ews@}!&sVW#3)6{{_5dd*Ya%byM5o(?lx zd5WDULa>vcR(+1-$s##7q72w+4}7|3I*IC5=u}$Cd40b!oj>5XrQTVZJ+l4aWR9VV z{h>$%UehRkr!B#;=+p6y-Q}@0u6TRnxH@Z_&O7sHZ;0~o9x$FH{#R|jvDxA@l*m0x z{Jk2Esn4t5$j%o>>j&h-=eU^My{Jdz4}R zd0SKJ=%NXB3fFwaE=c0-A8VYPFPxvoycQiM+mlXCM0xL`KeX#wQ?|@?lT|;JF^e|M z-b2Yrw_vHR)+49d2@}^lA4}AaN%n7PXgl+s&Ep$wt!L-I?fb@i^1a@ijD0B!+2$!_ z;Y(VY?^eF9KjZ5}+_yvA>9fiMoi*N)CXwfGx*~SW5+3Ke-0$e5?niq0_vvO1R>L>} z%KU}ww`07~T2bPZwgoz)TC;oY^GP)vuiB>L{4~t_X8n1xh{;vjy4m(pKTC;bmT+EK zYxod;<}!K!xfHj(Ah&Yu%)nKn;dD%WaW(Syf%tqRN&ZZ>6Mo+da}xX|LZ8IV?k#oU zWB1mk;h>MOHyk6+Mq}pnQ7#tu@|c!cC*9#RiDP+fv@WmTg-?^(aPB-zl>9f*UckuH;CF*%O5cZNai5^|rQG62S4I*NjOL^Lb zJ--0o982Tby}D0n+43|mOIy?g>HHaR+;M9C*>d*O7i}adP?)RC(NV%6eshR z=#?1S&T^M{TX}FM+d&OJ=XGkxcd6$*Si5W)9;CGR6Ba=wqH26$eU25lh*jOI{CH-c zGM{BEzOmV$eL5AN+MoSl{&l*#*TyuR|G*R9C2gB)d9EV1&pl?Ow1+we=G}|)>w)+I z_VCN{P;SBDn5~zfH$2nvM7R(a+m2aU@a(reSrM;>XX~Z)h!|ez#d8wK??%n&sXv}f zOU?rg;|V#mO#O792v2E42fx=HO6D1KdbZ~Pk|CpR+(*Inzv|kC+U9gO&)!4o?Wu;0 z#zwxXt0#(Yoa?MbuB%S;qHFddmWOuw&grdO8KbFOy17;zGsa#K|Nnj z8J$6>t)d{g*afUjO=2h2ars-nYy%8gcX27)3VbxZO>>CDl5o1 zazqQ0c`#30p_X;oV?BSTv>&5zO6yF6Z(7eWW<`aVFn!(&9$uspTV2O`w7rS#jLHnwZTS4HO{Anm zFxB;GQB3RBElunf)Jra;^&QRW?M&=rI+U-|%J_c1dKNt0<5yztHNopZI$S=Xk?U~> zTn+WZUTztId7&Qnx8_RF)mT6JnD?J?)ch#JM~go;C012|v*9DBKCz zU7mnzuY7ejdL;I=-&Cv0n%;L>$J*rSyyI$O>q{`r3jn`2Jpq(P!nPYEjR@Ovm)Azl@<^JUfN2YL*_*Z%mp|6{($+mj*(^p?%WtOq z*%_j_Bt2ng$N%p1w_haFn`LUrEgneBd#q+6{Vse8fW>ROYP#;#>; zENBRO@!U)JuHqd`H)r{32@!3YiDDNgDAx01UJGf=>v1KmS(D7}wOu~TEZIgvo6D@{ zD8Ew1m)}8&ab~K;uC+Ttan^a>>5ubJsk2@povS7x;d7dgxtyoV@mb!KU8QKo^F;UR zUUNGVw^JE5W0|#ub9n}r+3js0SZ*&so<7+!7wlCLUHl+`#-^^JJ z@Uq{csd2((5nkDBKTmw)iMlUE15b>f{Y;&m!AnoG)lR(m{c(PtdC^9TSqFP6WjjKx z?|O4IY}bLmaaVZS7k9eVb-qT=$xk4*e+?n+p&n=VB^~zcY}S}-y8`Q@$ZMIT rroAzqQE`1<*T)lA#zsZXf4ijgNuZQyl99(>#N;=g9;n~?uVMZV77ga_ delta 264 zcmdnw&EWS)BkBL$)LVs1qnYnZH;U E0FX*KBLDyZ diff --git a/azurerm/modules/azurerm-adb/constraints.tf b/azurerm/modules/azurerm-adb/constraints.tf index 60408b3a..5257ed4f 100644 --- a/azurerm/modules/azurerm-adb/constraints.tf +++ b/azurerm/modules/azurerm-adb/constraints.tf @@ -5,9 +5,5 @@ terraform { source = "hashicorp/azurerm" version = "~> 3.0" } - #TODO: note this is just added right now without any use, can be used In future for databricks provider - databricks = { - source = "databricks/databricks" - } } } diff --git a/azurerm/modules/azurerm-adb/data.tf b/azurerm/modules/azurerm-adb/data.tf new file mode 100644 index 00000000..20fa1ae1 --- /dev/null +++ b/azurerm/modules/azurerm-adb/data.tf @@ -0,0 +1,45 @@ +data "azurerm_client_config" "current" {} + +# data "databricks_current_user" "db" { +# count = var.enable_private_network ? 1 : 0 +# depends_on = [azurerm_databricks_workspace.example] +# } + +data "azurerm_resource_group" "vnet_rg" { + count = var.enable_private_network ? 1 : 0 + name = var.vnet_resource_group +} + +data "azurerm_virtual_network" "vnet" { + count = var.enable_private_network ? 1 : 0 + name = var.vnet_name + resource_group_name = var.vnet_resource_group +} + +data "azurerm_subnet" "public_subnet" { + count = var.enable_private_network == true && var.create_subnets == false && var.managed_vnet == false ? 1 : 0 + name = var.public_subnet_name + virtual_network_name = var.vnet_name + resource_group_name = var.vnet_resource_group +} + +data "azurerm_subnet" "private_subnet" { + count = var.enable_private_network == true && var.create_subnets == false && var.managed_vnet == false ? 1 : 0 + name = var.private_subnet_name + virtual_network_name = var.vnet_name + resource_group_name = var.vnet_resource_group +} + +data "azurerm_subnet" "pe_subnet" { + count = var.enable_private_network == true && var.create_pe_subnet == false && var.managed_vnet == false ? 1 : 0 + + name = var.pe_subnet_name + resource_group_name = var.vnet_resource_group + virtual_network_name = var.vnet_name +} + +data "azurerm_private_dns_zone" "adb_pvt_dns" { + count = var.enable_private_network ? 1 : 0 + name = var.private_dns_zone_name + resource_group_name = var.dns_resource_group_name +} diff --git a/azurerm/modules/azurerm-adb/load-balancer.tf b/azurerm/modules/azurerm-adb/load-balancer.tf new file mode 100644 index 00000000..f0a09ef0 --- /dev/null +++ b/azurerm/modules/azurerm-adb/load-balancer.tf @@ -0,0 +1,39 @@ + +resource "azurerm_lb" "lb" { + count = var.enable_private_network && var.create_lb && var.managed_vnet == false ? 1 : 0 + + name = local.lb_name + location = var.resource_group_location + resource_group_name = var.resource_group_name + + sku = "Standard" + + frontend_ip_configuration { + name = "Databricks-PIP" + public_ip_address_id = azurerm_public_ip.pip[0].id + } +} + +resource "azurerm_lb_outbound_rule" "lb_rule" { + count = var.enable_private_network && var.create_lb && var.managed_vnet == false ? 1 : 0 + + name = "Databricks-LB-Outbound-Rules" + + loadbalancer_id = azurerm_lb.lb[0].id + protocol = "All" + enable_tcp_reset = true + allocated_outbound_ports = 1024 + idle_timeout_in_minutes = 4 + + backend_address_pool_id = azurerm_lb_backend_address_pool.lb_be_pool[0].id + + frontend_ip_configuration { + name = azurerm_lb.lb[0].frontend_ip_configuration[0].name + } +} + +resource "azurerm_lb_backend_address_pool" "lb_be_pool" { + count = var.enable_private_network && var.create_lb && var.managed_vnet == false ? 1 : 0 + loadbalancer_id = azurerm_lb.lb[0].id + name = "Databricks-BE" +} diff --git a/azurerm/modules/azurerm-adb/local.tf b/azurerm/modules/azurerm-adb/local.tf new file mode 100644 index 00000000..d11792be --- /dev/null +++ b/azurerm/modules/azurerm-adb/local.tf @@ -0,0 +1,5 @@ +locals { + public_ip_name = "${var.resource_namer}-pip" + nat_gateway_name = "${var.resource_namer}-nat-gw" + lb_name = "${var.resource_namer}-lb" +} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/main.tf b/azurerm/modules/azurerm-adb/main.tf index eeec33ba..fed92ed8 100644 --- a/azurerm/modules/azurerm-adb/main.tf +++ b/azurerm/modules/azurerm-adb/main.tf @@ -1,9 +1,28 @@ resource "azurerm_databricks_workspace" "example" { - name = var.resource_namer - location = var.resource_group_location - resource_group_name = var.resource_group_name - sku = var.databricks_sku + name = var.resource_namer + location = var.resource_group_location + resource_group_name = var.resource_group_name + sku = var.databricks_sku + public_network_access_enabled = var.public_network_access_enabled + network_security_group_rules_required = var.managed_vnet ? null : var.network_security_group_rules_required + managed_resource_group_name = "databricks-rg-${var.resource_group_name}" + load_balancer_backend_address_pool_id = var.create_lb ? azurerm_lb_backend_address_pool.lb_be_pool[0].id : null + + dynamic "custom_parameters" { + for_each = var.enable_private_network == false ? toset([]) : toset([1]) + content { + no_public_ip = true + public_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.public_subnet[0].name : data.azurerm_subnet.public_subnet[0].name) + private_subnet_name = var.managed_vnet ? null : (var.create_subnets ? azurerm_subnet.private_subnet[0].name : data.azurerm_subnet.private_subnet[0].name) + virtual_network_id = var.managed_vnet ? null : data.azurerm_virtual_network.vnet[0].id + vnet_address_prefix = var.managed_vnet ? null : (var.vnet_address_prefix == "" ? null : var.vnet_address_prefix) + public_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.public[0].id + private_subnet_network_security_group_association_id = var.managed_vnet ? null : azurerm_subnet_network_security_group_association.private[0].id + nat_gateway_name = var.managed_vnet ? null : (var.create_nat ? azurerm_nat_gateway.nat[0].name : null) + public_ip_name = var.managed_vnet ? null : (var.create_nat ? azurerm_public_ip.pip[0].name : null) + } + } tags = var.resource_tags @@ -12,6 +31,7 @@ resource "azurerm_databricks_workspace" "example" { tags, ] } + depends_on = [azurerm_subnet.public_subnet, azurerm_subnet.private_subnet, data.azurerm_subnet.public_subnet, data.azurerm_subnet.private_subnet] } @@ -55,32 +75,3 @@ resource "azurerm_monitor_diagnostic_setting" "databricks_log_analytics" { } } -resource "databricks_workspace_conf" "this" { - count = var.enable_enableDbfsFileBrowser ? 1 : 0 - custom_config = { - - "enableDbfsFileBrowser" : true - - } -} - - -resource "databricks_user" "rbac_users" { - for_each = var.add_rbac_users ? var.rbac_databricks_users : {} - display_name = each.value.display_name - user_name = each.value.user_name - active = each.value.active -} - -resource "databricks_group" "project_users" { - count = var.add_rbac_users ? 1 : 0 - display_name = var.databricks_group_display_name - workspace_access = var.enable_workspace_access - databricks_sql_access = var.enable_sql_access -} - -resource "databricks_group_member" "project_users" { - for_each = var.add_rbac_users ? databricks_user.rbac_users : {} - group_id = databricks_group.project_users[0].id - member_id = each.value.id -} \ No newline at end of file diff --git a/azurerm/modules/azurerm-adb/nat.tf b/azurerm/modules/azurerm-adb/nat.tf new file mode 100644 index 00000000..d403056a --- /dev/null +++ b/azurerm/modules/azurerm-adb/nat.tf @@ -0,0 +1,31 @@ +############################################ +# NAT GATEWAY +############################################ + +resource "azurerm_nat_gateway" "nat" { + count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 + name = local.nat_gateway_name + location = var.resource_group_location + resource_group_name = var.resource_group_name + sku_name = "Standard" + idle_timeout_in_minutes = var.nat_idle_timeout + zones = ["1"] +} + +resource "azurerm_nat_gateway_public_ip_association" "nat_ip" { + count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 + nat_gateway_id = azurerm_nat_gateway.nat[0].id + public_ip_address_id = azurerm_public_ip.pip[0].id +} + +resource "azurerm_subnet_nat_gateway_association" "public_subnet_nat" { + count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id + nat_gateway_id = azurerm_nat_gateway.nat[0].id +} + +resource "azurerm_subnet_nat_gateway_association" "private_subnet_nat" { + count = var.enable_private_network && var.create_nat && var.managed_vnet == false ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id + nat_gateway_id = azurerm_nat_gateway.nat[0].id +} diff --git a/azurerm/modules/azurerm-adb/network.tf b/azurerm/modules/azurerm-adb/network.tf new file mode 100644 index 00000000..f0b761cc --- /dev/null +++ b/azurerm/modules/azurerm-adb/network.tf @@ -0,0 +1,218 @@ +############################################ +# SUBNETS +############################################ + +resource "azurerm_subnet" "public_subnet" { + count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 + + name = var.public_subnet_name + resource_group_name = var.vnet_resource_group + virtual_network_name = var.vnet_name + address_prefixes = var.public_subnet_prefix + + delegation { + name = "databricks" + service_delegation { + name = "Microsoft.Databricks/workspaces" + actions = [ + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action", + "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"] + } + } +} + +resource "azurerm_subnet" "private_subnet" { + count = var.enable_private_network == true && var.create_subnets == true && var.managed_vnet == false ? 1 : 0 + + name = var.private_subnet_name + resource_group_name = var.vnet_resource_group + virtual_network_name = var.vnet_name + address_prefixes = var.private_subnet_prefix + + enforce_private_link_endpoint_network_policies = true + enforce_private_link_service_network_policies = true + + delegation { + name = "databricks" + service_delegation { + name = "Microsoft.Databricks/workspaces" + actions = [ + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action", + "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"] + } + } + + service_endpoints = var.service_endpoints +} + +resource "azurerm_subnet" "pe_subnet" { + count = var.enable_private_network == true && var.create_pe_subnet == true && var.managed_vnet == false ? 1 : 0 + + name = var.pe_subnet_name + resource_group_name = var.vnet_resource_group + virtual_network_name = var.vnet_name + address_prefixes = var.pe_subnet_prefix + enforce_private_link_endpoint_network_policies = true +} + + +############################################ +# NSG +############################################ + +resource "azurerm_network_security_group" "nsg" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "${var.resource_namer}-nsg-databricks" + location = var.resource_group_location + resource_group_name = var.resource_group_name +} + +resource "azurerm_network_security_rule" "nsg_rule" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "adf-db-inbound" + priority = 200 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "DataFactory.WestEurope" + destination_address_prefix = "VirtualNetwork" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.nsg[0].name +} + +resource "azurerm_network_security_rule" "aad" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "AllowAAD" + priority = 200 + direction = "Outbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "AzureActiveDirectory" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.nsg[0].name +} + +resource "azurerm_network_security_rule" "azfrontdoor" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = "AllowAzureFrontDoor" + priority = 201 + direction = "Outbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "AzureFrontDoor.Frontend" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.nsg[0].name +} + + +resource "azurerm_subnet_network_security_group_association" "private" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.private_subnet[0].id : data.azurerm_subnet.private_subnet[0].id + network_security_group_id = azurerm_network_security_group.nsg[0].id +} + +resource "azurerm_subnet_network_security_group_association" "public" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + subnet_id = var.create_subnets ? azurerm_subnet.public_subnet[0].id : data.azurerm_subnet.public_subnet[0].id + network_security_group_id = azurerm_network_security_group.nsg[0].id +} + +############################################ +# PRIVATE ENDPOINT +############################################ + +resource "azurerm_private_endpoint" "databricks" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + + name = "${var.resource_namer}-pe-databricks" + location = var.resource_group_location + resource_group_name = var.resource_group_name + subnet_id = var.create_pe_subnet ? azurerm_subnet.pe_subnet[0].id : data.azurerm_subnet.pe_subnet[0].id + + private_service_connection { + name = "${var.resource_namer}-db-pe" + private_connection_resource_id = azurerm_databricks_workspace.example.id + is_manual_connection = false + subresource_names = ["databricks_ui_api"] + } + + private_dns_zone_group { + + name = "databricks_ui_api" + private_dns_zone_ids = [data.azurerm_private_dns_zone.adb_pvt_dns[0].id] + } + + depends_on = [azurerm_databricks_workspace.example, data.azurerm_private_dns_zone.adb_pvt_dns] +} + +resource "azurerm_private_endpoint" "auth" { + count = var.enable_private_network && var.managed_vnet == false && var.browser_authentication_enabled == true ? 1 : 0 + name = "${var.resource_namer}-pe-databricks-auth" + location = var.resource_group_location + resource_group_name = var.resource_group_name + subnet_id = var.create_pe_subnet ? azurerm_subnet.pe_subnet[0].id : data.azurerm_subnet.pe_subnet[0].id + + private_service_connection { + name = "${var.resource_namer}-db-pe-auth" + private_connection_resource_id = azurerm_databricks_workspace.example.id + is_manual_connection = false + subresource_names = ["browser_authentication"] + } + + private_dns_zone_group { + name = "databricks_auth" + private_dns_zone_ids = [data.azurerm_private_dns_zone.adb_pvt_dns[0].id] + } + + depends_on = [azurerm_databricks_workspace.example] + +} +/* +resource "azurerm_private_dns_zone" "dns" { + count = var.enable_private_network && var.managed_vnet == false && var.create_db_dns_zone ? 1 : 0 + name = "privatelink.azuredatabricks.net" + resource_group_name = var.resource_group_name +} + +data "azurerm_private_dns_zone" "dns" { + count = var.create_db_dns_zone == false ? 1 : 0 + name = "privatelink.azuredatabricks.net" + resource_group_name = var.db_dns_zone_rg +} + +resource "azurerm_private_dns_cname_record" "cname" { + count = var.enable_private_network && var.managed_vnet == false ? 1 : 0 + name = azurerm_databricks_workspace.example.workspace_url + zone_name = var.create_db_dns_zone ? azurerm_private_dns_zone.dns[0].name : data.azurerm_private_dns_zone.dns[0].name + resource_group_name = var.create_db_dns_zone ? var.resource_group_name : var.db_dns_zone_rg + ttl = var.dns_record_ttl + record = "${var.resource_namer}.azuredatabricks.net" +} + +resource "azurerm_private_dns_zone_virtual_network_link" "db_dns_vnet_link" { + count = var.enable_private_network == true && var.managed_vnet == false && var.create_db_dns_zone == true ? 1 : 0 + name = var.resource_namer + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.dns[0].name + virtual_network_id = data.azurerm_virtual_network.vnet[0].id +} +*/ +resource "azurerm_public_ip" "pip" { + count = var.enable_private_network && var.create_pip && var.managed_vnet == false ? 1 : 0 + name = local.public_ip_name + location = var.resource_group_location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" + zones = ["1"] +} diff --git a/azurerm/modules/azurerm-adb/var.tf b/azurerm/modules/azurerm-adb/var.tf index caa4e94b..bfeb6d71 100644 --- a/azurerm/modules/azurerm-adb/var.tf +++ b/azurerm/modules/azurerm-adb/var.tf @@ -66,47 +66,164 @@ variable "data_platform_log_analytics_workspace_id" { # Resource Databricks workspace setting ############################################ -variable "enable_enableDbfsFileBrowser" { +variable "public_network_access_enabled" { + type = bool + default = true + description = "Enables or Disabled Public Access to Databricks Workspace." +} + +variable "network_security_group_rules_required" { + type = string + default = "NoAzureDatabricksRules" + description = " Does the data plane (clusters) to control plane communication happen over private link endpoint only or publicly? Possible values AllRules, NoAzureDatabricksRules or NoAzureServiceRules." +} + +variable "enable_private_network" { type = bool - description = "Whether to enable Dbfs File browser for the Azure Databricks workspace" default = false + description = "Enable Secure Data Platform." } +variable "nat_idle_timeout" { + type = number + default = 10 + description = "Idle timeout period in minutes." +} + +variable "browser_authentication_enabled" { + type = bool + default = false + description = "Specify wether to create to private endpoint for browser authentication, False in Dev and True in Production should be enable in on enviroment." +} ############################################ -# Resource Databricks user +# Network Details ############################################ -variable "add_rbac_users" { - description = "If set to true, the module will create databricks users and group named 'project_users' with the specified users as members, and grant workspace and SQL access to this group. Default is false." +variable "create_subnets" { + type = bool + default = false + description = "Set to true if you need the module to create the subnets for you." +} + +variable "create_pe_subnet" { + type = bool + default = false + description = "Set to true if you need the module to create the private endpoint subnet." +} + +variable "create_db_dns_zone" { type = bool default = true + description = "Create DNS Zone for Azure Databricks." } -variable "rbac_databricks_users" { - type = map(object({ - display_name = string - user_name = string - active = bool - })) - description = "If 'add_rbac_users' set to true then specifies RBAC Databricks users" - default = null +variable "db_dns_zone_rg" { + type = string + default = "value" + description = "Resource Group where DNS is created." +} + +variable "vnet_name" { + type = string + default = "" + description = "Name of the VNET inwhich the Databricks Workspace will be provisioned." +} + +variable "vnet_resource_group" { + type = string + default = "" + description = "The Resource Group which the VNET is provisioned." +} + +variable "public_subnet_name" { + type = string + default = "" + description = "Name of the Public Databricks Subnet." +} + +variable "private_subnet_name" { + type = string + default = "" + description = "Name of the Private Databricks Subnet." } -variable "databricks_group_display_name" { +variable "public_subnet_prefix" { + type = list(string) + default = [] + description = "IP Address Space fo the Public Databricks Subnet." +} + +variable "private_subnet_prefix" { + type = list(string) + default = [] + description = "IP Address Space fo the Private Databricks Subnet." + +} + +variable "pe_subnet_prefix" { + type = list(string) + default = [] + description = "IP Address Space fo the Private Endpoints Databricks Subnet." + +} + +variable "pe_subnet_name" { type = string - description = "If 'add_rbac_users' set to true then specifies databricks group display name" - default = "project_users" + default = "" + description = "Name of the Subnet used to provision Private Endpoints into." } -variable "enable_workspace_access" { +variable "vnet_address_prefix" { + type = string + default = "" + description = "Address Prefix of the VNET." +} + +variable "dns_record_ttl" { + type = number + default = 300 + description = "TTL for DNS Record." +} + +variable "service_endpoints" { + type = list(string) + default = ["Microsoft.AzureActiveDirectory", "Microsoft.KeyVault", "Microsoft.ServiceBus", "Microsoft.Sql", "Microsoft.Storage"] + description = "List of Service Endpoints Enabled on the Subnet." +} + +variable "create_nat" { type = bool - description = "Whether to enable workspace access for the databricks group" - default = true + default = false + description = "Deploy Databricks with a NAT Gateway." } -variable "enable_sql_access" { +variable "create_lb" { type = bool - description = "Whether to enable sql access for the databricks group" - default = true -} \ No newline at end of file + default = false + description = "Deploy Databricks with a Load Balancer." +} + +variable "managed_vnet" { + type = bool + default = false + description = "Used to determine if Databricks is created in a managed vnet configuration." +} + +variable "create_pip" { + type = bool + default = false + description = "Create Databricks with a Public IP." +} + +variable "private_dns_zone_name" { + type = string + default = "privatelink.azuredatabricks.net" + description = "Specifies the Name of the Private DNS Zone Group." +} + +variable "dns_resource_group_name" { + type = string + default = "amido-stacks-euw-de-hub-network" + description = "Name of the resource group where pvt dns is present." +}