From ca1c4e9734b36bbf400ccef9a6c784be9be382a4 Mon Sep 17 00:00:00 2001 From: magreenbaum Date: Mon, 13 Jan 2025 20:09:46 -0500 Subject: [PATCH] support studio encryption_key_arn --- README.md | 14 ++--- examples/private-cluster/README.md | 4 +- examples/private-cluster/versions.tf | 2 +- examples/public-cluster/README.md | 4 +- examples/public-cluster/versions.tf | 2 +- examples/serverless-cluster/README.md | 4 +- examples/serverless-cluster/versions.tf | 2 +- examples/studio/README.md | 7 ++- examples/studio/main.tf | 74 +++++++++++++++++++++++++ examples/studio/versions.tf | 2 +- examples/virtual-cluster/README.md | 4 +- examples/virtual-cluster/versions.tf | 2 +- modules/serverless/README.md | 4 +- modules/serverless/versions.tf | 2 +- modules/studio/README.md | 5 +- modules/studio/main.tf | 1 + modules/studio/variables.tf | 6 ++ modules/studio/versions.tf | 2 +- modules/virtual-cluster/README.md | 4 +- modules/virtual-cluster/versions.tf | 2 +- versions.tf | 2 +- 21 files changed, 117 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 5f287bf..9cfea0b 100644 --- a/README.md +++ b/README.md @@ -330,13 +330,13 @@ Examples codified under the [`examples`](https://github.com/terraform-aws-module | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 5.59 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.59 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules @@ -401,7 +401,7 @@ No modules. | [ec2\_attributes](#input\_ec2\_attributes) | Attributes for the EC2 instances running the job flow | `any` | `{}` | no | | [iam\_instance\_profile\_description](#input\_iam\_instance\_profile\_description) | Description of the EC2 IAM role/instance profile | `string` | `null` | no | | [iam\_instance\_profile\_name](#input\_iam\_instance\_profile\_name) | Name to use on EC2 IAM role/instance profile created | `string` | `null` | no | -| [iam\_instance\_profile\_policies](#input\_iam\_instance\_profile\_policies) | Map of IAM policies to attach to the EC2 IAM role/instance profile | `map(string)` |
{
"AmazonElasticMapReduceforEC2Role": "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
}
| no | +| [iam\_instance\_profile\_policies](#input\_iam\_instance\_profile\_policies) | Map of IAM policies to attach to the EC2 IAM role/instance profile | `map(string)` |
{
"AmazonElasticMapReduceforEC2Role": "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
}
| no | | [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | @@ -419,11 +419,11 @@ No modules. | [master\_instance\_fleet](#input\_master\_instance\_fleet) | Configuration block to use an [Instance Fleet](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-instance-fleet.html) for the master node type. Cannot be specified if any `master_instance_group` configuration blocks are set | `any` | `{}` | no | | [master\_instance\_group](#input\_master\_instance\_group) | Configuration block to use an [Instance Group](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-instance-group-configuration.html#emr-plan-instance-groups) for the [master node type](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-master-core-task-nodes.html#emr-plan-master) | `any` | `{}` | no | | [master\_security\_group\_description](#input\_master\_security\_group\_description) | Description of the security group created | `string` | `"Managed master security group"` | no | -| [master\_security\_group\_rules](#input\_master\_security\_group\_rules) | Security group rules to add to the security group created | `any` |
{
"default": {
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "Allow all egress traffic",
"from_port": 0,
"ipv6_cidr_blocks": [
"::/0"
],
"protocol": "-1",
"to_port": 0,
"type": "egress"
}
}
| no | +| [master\_security\_group\_rules](#input\_master\_security\_group\_rules) | Security group rules to add to the security group created | `any` |
{
"default": {
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "Allow all egress traffic",
"from_port": 0,
"ipv6_cidr_blocks": [
"::/0"
],
"protocol": "-1",
"to_port": 0,
"type": "egress"
}
}
| no | | [name](#input\_name) | Name of the job flow | `string` | `""` | no | | [placement\_group\_config](#input\_placement\_group\_config) | The specified placement group configuration | `any` | `{}` | no | | [release\_label](#input\_release\_label) | Release label for the Amazon EMR release | `string` | `null` | no | -| [release\_label\_filters](#input\_release\_label\_filters) | Map of release label filters use to lookup a release label | `any` |
{
"default": {
"prefix": "emr-6"
}
}
| no | +| [release\_label\_filters](#input\_release\_label\_filters) | Map of release label filters use to lookup a release label | `any` |
{
"default": {
"prefix": "emr-6"
}
}
| no | | [scale\_down\_behavior](#input\_scale\_down\_behavior) | Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an instance group is resized | `string` | `"TERMINATE_AT_TASK_COMPLETION"` | no | | [security\_configuration](#input\_security\_configuration) | Security configuration to create, or attach if `create_security_configuration` is `false`. Only valid for EMR clusters with `release_label` 4.8.0 or greater | `string` | `null` | no | | [security\_configuration\_name](#input\_security\_configuration\_name) | Name of the security configuration to create, or attach if `create_security_configuration` is `false`. Only valid for EMR clusters with `release_label` 4.8.0 or greater | `string` | `null` | no | @@ -431,13 +431,13 @@ No modules. | [service\_iam\_role\_arn](#input\_service\_iam\_role\_arn) | The ARN of an existing IAM role to use for the service | `string` | `null` | no | | [service\_iam\_role\_description](#input\_service\_iam\_role\_description) | Description of the role | `string` | `null` | no | | [service\_iam\_role\_name](#input\_service\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | -| [service\_iam\_role\_policies](#input\_service\_iam\_role\_policies) | Map of IAM policies to attach to the service role | `map(string)` |
{
"AmazonEMRServicePolicy_v2": "arn:aws:iam::aws:policy/service-role/AmazonEMRServicePolicy_v2"
}
| no | +| [service\_iam\_role\_policies](#input\_service\_iam\_role\_policies) | Map of IAM policies to attach to the service role | `map(string)` |
{
"AmazonEMRServicePolicy_v2": "arn:aws:iam::aws:policy/service-role/AmazonEMRServicePolicy_v2"
}
| no | | [service\_pass\_role\_policy\_description](#input\_service\_pass\_role\_policy\_description) | Description of the policy | `string` | `null` | no | | [service\_pass\_role\_policy\_name](#input\_service\_pass\_role\_policy\_name) | Name to use on IAM policy created | `string` | `null` | no | | [service\_security\_group\_description](#input\_service\_security\_group\_description) | Description of the security group created | `string` | `"Managed service access security group"` | no | | [service\_security\_group\_rules](#input\_service\_security\_group\_rules) | Security group rules to add to the security group created | `any` | `{}` | no | | [slave\_security\_group\_description](#input\_slave\_security\_group\_description) | Description of the security group created | `string` | `"Managed slave security group"` | no | -| [slave\_security\_group\_rules](#input\_slave\_security\_group\_rules) | Security group rules to add to the security group created | `any` |
{
"default": {
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "Allow all egress traffic",
"from_port": 0,
"ipv6_cidr_blocks": [
"::/0"
],
"protocol": "-1",
"to_port": 0,
"type": "egress"
}
}
| no | +| [slave\_security\_group\_rules](#input\_slave\_security\_group\_rules) | Security group rules to add to the security group created | `any` |
{
"default": {
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "Allow all egress traffic",
"from_port": 0,
"ipv6_cidr_blocks": [
"::/0"
],
"protocol": "-1",
"to_port": 0,
"type": "egress"
}
}
| no | | [step](#input\_step) | Steps to run when creating the cluster | `any` | `{}` | no | | [step\_concurrency\_level](#input\_step\_concurrency\_level) | Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with `release_label` 5.28.0 or greater (default is 1) | `number` | `null` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | diff --git a/examples/private-cluster/README.md b/examples/private-cluster/README.md index fda227a..193e7af 100644 --- a/examples/private-cluster/README.md +++ b/examples/private-cluster/README.md @@ -28,13 +28,13 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 5.59 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.59 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/examples/private-cluster/versions.tf b/examples/private-cluster/versions.tf index 7ea0158..e0d6884 100644 --- a/examples/private-cluster/versions.tf +++ b/examples/private-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.59" + version = ">= 5.83" } } } diff --git a/examples/public-cluster/README.md b/examples/public-cluster/README.md index 6d78b64..509b42b 100644 --- a/examples/public-cluster/README.md +++ b/examples/public-cluster/README.md @@ -26,13 +26,13 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 5.59 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.59 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/examples/public-cluster/versions.tf b/examples/public-cluster/versions.tf index 7ea0158..e0d6884 100644 --- a/examples/public-cluster/versions.tf +++ b/examples/public-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.59" + version = ">= 5.83" } } } diff --git a/examples/serverless-cluster/README.md b/examples/serverless-cluster/README.md index ee159c7..e19d891 100644 --- a/examples/serverless-cluster/README.md +++ b/examples/serverless-cluster/README.md @@ -26,13 +26,13 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 5.59 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.59 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/examples/serverless-cluster/versions.tf b/examples/serverless-cluster/versions.tf index 7ea0158..e0d6884 100644 --- a/examples/serverless-cluster/versions.tf +++ b/examples/serverless-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.59" + version = ">= 5.83" } } } diff --git a/examples/studio/README.md b/examples/studio/README.md index 8b0b215..044ff57 100644 --- a/examples/studio/README.md +++ b/examples/studio/README.md @@ -22,13 +22,13 @@ $ terraform apply | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 5.59 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.59 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules @@ -38,6 +38,7 @@ $ terraform apply | [emr\_studio\_disabled](#module\_emr\_studio\_disabled) | ../../modules/studio | n/a | | [emr\_studio\_iam](#module\_emr\_studio\_iam) | ../../modules/studio | n/a | | [emr\_studio\_sso](#module\_emr\_studio\_sso) | ../../modules/studio | n/a | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 2.0 | | [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 4.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | @@ -46,7 +47,9 @@ $ terraform apply | Name | Type | |------|------| | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_identitystore_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/identitystore_group) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | | [aws_ssoadmin_instances.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssoadmin_instances) | data source | ## Inputs diff --git a/examples/studio/main.tf b/examples/studio/main.tf index b454235..cd48aba 100644 --- a/examples/studio/main.tf +++ b/examples/studio/main.tf @@ -4,6 +4,10 @@ provider "aws" { data "aws_availability_zones" "available" {} +data "aws_caller_identity" "current" {} + +data "aws_region" "current" {} + locals { name = replace(basename(path.cwd), "-cluster", "") region = "eu-west-1" @@ -139,6 +143,22 @@ module "emr_studio_iam" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets + encryption_key_arn = module.kms.key_arn + + service_role_statements = [ + { + effect = "Allow" + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", + "kms:DescribeKey" + ] + resources = [module.kms.key_arn] + } + ] + tags = local.tags } @@ -198,3 +218,57 @@ module "s3_bucket" { tags = local.tags } + +module "kms" { + source = "terraform-aws-modules/kms/aws" + version = "~> 2.0" + + deletion_window_in_days = 7 + description = "KMS key for ${local.name}." + enable_key_rotation = true + is_enabled = true + key_usage = "ENCRYPT_DECRYPT" + enable_default_policy = true + key_statements = [ + { + sid = "EMRStudio" + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", + "kms:DescribeKey" + ] + resources = ["*"] + + principals = [ + { + type = "AWS" + identifiers = [module.emr_studio_iam.service_iam_role_arn] + } + ] + + conditions = [ + { + test = "StringEquals" + variable = "kms:CallerAccount" + values = [data.aws_caller_identity.current.account_id] + }, + { + test = "StringEquals" + variable = "kms:EncryptionContext:aws:s3:arn" + values = [module.s3_bucket.s3_bucket_arn] + }, + { + test = "StringEquals" + variable = "kms:ViaService" + values = ["s3.${data.aws_region.current.name}.amazonaws.com"] + } + ] + } + ] + + aliases = [local.name] + + tags = local.tags +} diff --git a/examples/studio/versions.tf b/examples/studio/versions.tf index 7ea0158..e0d6884 100644 --- a/examples/studio/versions.tf +++ b/examples/studio/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.59" + version = ">= 5.83" } } } diff --git a/examples/virtual-cluster/README.md b/examples/virtual-cluster/README.md index eb7cd99..f647896 100644 --- a/examples/virtual-cluster/README.md +++ b/examples/virtual-cluster/README.md @@ -45,7 +45,7 @@ aws emr-containers list-virtual-clusters --region us-west-2 --states ARRESTED \ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 5.59 | +| [aws](#requirement\_aws) | >= 5.83 | | [kubernetes](#requirement\_kubernetes) | >= 2.17 | | [null](#requirement\_null) | >= 3.0 | | [time](#requirement\_time) | >= 0.7 | @@ -54,7 +54,7 @@ aws emr-containers list-virtual-clusters --region us-west-2 --states ARRESTED \ | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.59 | +| [aws](#provider\_aws) | >= 5.83 | | [null](#provider\_null) | >= 3.0 | | [time](#provider\_time) | >= 0.7 | diff --git a/examples/virtual-cluster/versions.tf b/examples/virtual-cluster/versions.tf index ba6c676..7aeb5c1 100644 --- a/examples/virtual-cluster/versions.tf +++ b/examples/virtual-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.59" + version = ">= 5.83" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/serverless/README.md b/modules/serverless/README.md index 431dd10..6ed9cd9 100644 --- a/modules/serverless/README.md +++ b/modules/serverless/README.md @@ -134,13 +134,13 @@ Examples codified under the [`examples`](https://github.com/terraform-aws-module | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 5.59 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.59 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules diff --git a/modules/serverless/versions.tf b/modules/serverless/versions.tf index 7ea0158..e0d6884 100644 --- a/modules/serverless/versions.tf +++ b/modules/serverless/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.59" + version = ">= 5.83" } } } diff --git a/modules/studio/README.md b/modules/studio/README.md index 97ec03e..97678ae 100644 --- a/modules/studio/README.md +++ b/modules/studio/README.md @@ -68,13 +68,13 @@ Examples codified under the [`examples`](https://github.com/terraform-aws-module | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 5.59 | +| [aws](#requirement\_aws) | >= 5.83 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.59 | +| [aws](#provider\_aws) | >= 5.83 | ## Modules @@ -119,6 +119,7 @@ No modules. | [create\_user\_role\_policy](#input\_create\_user\_role\_policy) | Determines whether the user IAM role policy should be created | `bool` | `true` | no | | [default\_s3\_location](#input\_default\_s3\_location) | The Amazon S3 location to back up Amazon EMR Studio Workspaces and notebook files | `string` | `""` | no | | [description](#input\_description) | A detailed description of the Amazon EMR Studio | `string` | `null` | no | +| [encryption\_key\_arn](#input\_encryption\_key\_arn) | The AWS KMS key identifier (ARN) used to encrypt Amazon EMR Studio workspace and notebook files when backed up to Amazon S3 | `string` | `null` | no | | [engine\_security\_group\_description](#input\_engine\_security\_group\_description) | Description of the security group created | `string` | `"EMR Studio engine security group"` | no | | [engine\_security\_group\_id](#input\_engine\_security\_group\_id) | The ID of the Amazon EMR Studio Engine security group. The Engine security group allows inbound network traffic from the Workspace security group, and it must be in the same VPC specified by `vpc_id` | `string` | `null` | no | | [engine\_security\_group\_rules](#input\_engine\_security\_group\_rules) | Security group rules to add to the security group created | `any` | `{}` | no | diff --git a/modules/studio/main.tf b/modules/studio/main.tf index d72d566..2d73d0a 100644 --- a/modules/studio/main.tf +++ b/modules/studio/main.tf @@ -22,6 +22,7 @@ resource "aws_emr_studio" "this" { auth_mode = var.auth_mode default_s3_location = var.default_s3_location description = var.description + encryption_key_arn = var.encryption_key_arn engine_security_group_id = local.create_security_groups ? aws_security_group.engine[0].id : var.engine_security_group_id idp_auth_url = var.idp_auth_url idp_relay_state_parameter_name = var.idp_relay_state_parameter_name diff --git a/modules/studio/variables.tf b/modules/studio/variables.tf index 818ad4a..dd6beb9 100644 --- a/modules/studio/variables.tf +++ b/modules/studio/variables.tf @@ -32,6 +32,12 @@ variable "description" { default = null } +variable "encryption_key_arn" { + description = "The AWS KMS key identifier (ARN) used to encrypt Amazon EMR Studio workspace and notebook files when backed up to Amazon S3" + type = string + default = null +} + variable "idp_auth_url" { description = "The authentication endpoint of your identity provider (IdP). Specify this value when you use IAM authentication and want to let federated users log in to a Studio with the Studio URL and credentials from your IdP" type = string diff --git a/modules/studio/versions.tf b/modules/studio/versions.tf index 7ea0158..e0d6884 100644 --- a/modules/studio/versions.tf +++ b/modules/studio/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.59" + version = ">= 5.83" } } } diff --git a/modules/virtual-cluster/README.md b/modules/virtual-cluster/README.md index 329d7f3..06ffc0c 100644 --- a/modules/virtual-cluster/README.md +++ b/modules/virtual-cluster/README.md @@ -82,14 +82,14 @@ Examples codified under the [`examples`](https://github.com/terraform-aws-module | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 5.59 | +| [aws](#requirement\_aws) | >= 5.83 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.59 | +| [aws](#provider\_aws) | >= 5.83 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | ## Modules diff --git a/modules/virtual-cluster/versions.tf b/modules/virtual-cluster/versions.tf index a1c56fc..f88481f 100644 --- a/modules/virtual-cluster/versions.tf +++ b/modules/virtual-cluster/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.59" + version = ">= 5.83" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/versions.tf b/versions.tf index 7ea0158..e0d6884 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.59" + version = ">= 5.83" } } }