diff --git a/README.md b/README.md index 134c033d..e2e43173 100644 --- a/README.md +++ b/README.md @@ -114,20 +114,25 @@ You need the following permissions to run this module. | Name | Source | Version | |------|--------|---------| | [cbr\_rule](#module\_cbr\_rule) | terraform-ibm-modules/cbr/ibm//modules/cbr-rule-module | 1.29.0 | +| [es\_guid\_crn\_parser](#module\_es\_guid\_crn\_parser) | terraform-ibm-modules/common-utilities/ibm//modules/crn-parser | 1.1.0 | ### Resources | Name | Type | |------|------| +| [ibm_event_streams_mirroring_config.es_mirroring_config](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/event_streams_mirroring_config) | resource | | [ibm_event_streams_quota.eventstreams_quotas](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/event_streams_quota) | resource | | [ibm_event_streams_schema.es_schema](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/event_streams_schema) | resource | | [ibm_event_streams_schema_global_rule.es_globalrule](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/event_streams_schema_global_rule) | resource | | [ibm_event_streams_topic.es_topic](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/event_streams_topic) | resource | +| [ibm_iam_authorization_policy.es_s2s_policy](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource | | [ibm_iam_authorization_policy.kms_policy](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource | | [ibm_resource_instance.es_instance](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/resource_instance) | resource | | [ibm_resource_key.service_credentials](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/resource_key) | resource | | [ibm_resource_tag.es_access_tag](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/resource_tag) | resource | | [time_sleep.wait_for_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [time_sleep.wait_for_es_s2s_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [time_sleep.wait_for_kms_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | ### Inputs @@ -138,10 +143,12 @@ You need the following permissions to run this module. | [create\_timeout](#input\_create\_timeout) | The timeout value for creating an Event Streams instance. Specify `3h` for an Enterprise plan instance. Add 1 h for each level of non-default throughput. Add 30 min for each level of non-default storage size. | `string` | `"3h"` | no | | [delete\_timeout](#input\_delete\_timeout) | The timeout value for deleting an Event Streams instance. | `string` | `"15m"` | no | | [es\_name](#input\_es\_name) | The name to give the Event Streams instance created by this module. | `string` | n/a | yes | -| [existing\_kms\_instance\_guid](#input\_existing\_kms\_instance\_guid) | The GUID of the Hyper Protect Crypto Services or Key Protect instance in which the key specified in var.kms\_key\_crn is coming from. Required only if var.kms\_encryption\_enabled is set to true, var.skip\_iam\_authorization\_policy is set to false, and you pass a value for var.kms\_key\_crn. | `string` | `null` | no | +| [existing\_kms\_instance\_guid](#input\_existing\_kms\_instance\_guid) | The GUID of the Hyper Protect Crypto Services or Key Protect instance in which the key specified in var.kms\_key\_crn is coming from. Required only if var.kms\_encryption\_enabled is set to true, var.skip\_kms\_iam\_authorization\_policy is set to false, and you pass a value for var.kms\_key\_crn. | `string` | `null` | no | | [kms\_encryption\_enabled](#input\_kms\_encryption\_enabled) | Set this to true to control the encryption keys used to encrypt the data that you store in IBM Cloud® Databases. If set to false, the data is encrypted by using randomly generated keys. For more info on Key Protect integration, see https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect. For more info on HPCS integration, see https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs | `bool` | `false` | no | | [kms\_key\_crn](#input\_kms\_key\_crn) | The root key CRN of the key management service (Key Protect or Hyper Protect Crypto Services) to use to encrypt the payload data. [Learn more](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-managing_encryption) about integrating Hyper Protect Crypto Services with Event Streams. | `string` | `null` | no | | [metrics](#input\_metrics) | Enhanced metrics to activate, as list of strings. Only allowed for enterprise plans. Allowed values: 'topic', 'partition', 'consumers'. | `list(string)` | `[]` | no | +| [mirroring](#input\_mirroring) | Event Streams mirroring configuration. Required only if creating mirroring instance. For more information on mirroring, see https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-mirroring. |
object({
source_crn = string
source_alias = string
target_alias = string
options = optional(object({
topic_name_transform = object({
type = string
rename = optional(object({
add_prefix = optional(string)
add_suffix = optional(string)
remove_prefix = optional(string)
remove_suffix = optional(string)
}))
})
group_id_transform = object({
type = string
rename = optional(object({
add_prefix = optional(string)
add_suffix = optional(string)
remove_prefix = optional(string)
remove_suffix = optional(string)
}))
})
}))
})
| `null` | no | +| [mirroring\_topic\_patterns](#input\_mirroring\_topic\_patterns) | The list of the topics to set in instance. Required only if creating mirroring instance. | `list(string)` | `null` | no | | [plan](#input\_plan) | The plan for the Event Streams instance. Possible values: `lite`, `standard`, `enterprise-3nodes-2tb`. | `string` | `"standard"` | no | | [quotas](#input\_quotas) | Quotas to be applied to the Event Streams instance. Entity may be 'default' to apply to all users, or an IAM ServiceID for a specific user. Rates are bytes/second, with -1 meaning no quota. |
list(object({
entity = string
producer_byte_rate = optional(number, -1)
consumer_byte_rate = optional(number, -1)
}))
| `[]` | no | | [region](#input\_region) | The region where the Event Streams are created. | `string` | `"us-south"` | no | @@ -150,7 +157,8 @@ You need the following permissions to run this module. | [schemas](#input\_schemas) | The list of schema objects. Include the `schema_id` and the `type` and `name` of the schema in the `schema` object. |
list(object(
{
schema_id = string
schema = object({
type = string
name = string
fields = optional(list(object({
name = string
type = string
})))
})
}
))
| `[]` | no | | [service\_credential\_names](#input\_service\_credential\_names) | The mapping of names and roles for service credentials that you want to create for the Event streams. | `map(string)` | `{}` | no | | [service\_endpoints](#input\_service\_endpoints) | The type of service endpoints. Possible values: 'public', 'private', 'public-and-private'. | `string` | `"public"` | no | -| [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the existing\_kms\_instance\_guid variable. In addition, no policy is created if var.kms\_encryption\_enabled is set to false. | `bool` | `false` | no | +| [skip\_es\_s2s\_iam\_authorization\_policy](#input\_skip\_es\_s2s\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that will allow all Event Streams instances in the given resource group access to read from the mirror source instance. This policy is required when creating a mirroring instance, and will only be created if a value is passed in the mirroring input. | `bool` | `false` | no | +| [skip\_kms\_iam\_authorization\_policy](#input\_skip\_kms\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the `existing_kms_instance_guid` variable. In addition, no policy is created if var.kms\_encryption\_enabled is set to false. | `bool` | `false` | no | | [storage\_size](#input\_storage\_size) | Storage size of the Event Streams in GB. Applies only to Enterprise plan instances. Possible values: `2048`, `4096`, `6144`, `8192`, `10240`, `12288`. Storage capacity cannot be reduced after the instance is created. When the `throughput` input variable is set to `300`, storage size starts at 4096. When `throughput` is `450`, storage size starts starts at `6144`. | `number` | `"2048"` | no | | [tags](#input\_tags) | The list of tags associated with the Event Steams instance. | `list(string)` | `[]` | no | | [throughput](#input\_throughput) | Throughput capacity in MB per second. Applies only to Enterprise plan instances. Possible values: `150`, `300`, `450`. | `number` | `"150"` | no | @@ -167,6 +175,8 @@ You need the following permissions to run this module. | [kafka\_broker\_version](#output\_kafka\_broker\_version) | The Kafka version | | [kafka\_brokers\_sasl](#output\_kafka\_brokers\_sasl) | (Array of Strings) Kafka brokers use for interacting with Kafka native API | | [kafka\_http\_url](#output\_kafka\_http\_url) | The API endpoint to interact with Event Streams REST API | +| [mirroring\_config\_id](#output\_mirroring\_config\_id) | The ID of the mirroring config in CRN format | +| [mirroring\_topic\_patterns](#output\_mirroring\_topic\_patterns) | Mirroring topic patterns | | [service\_credentials\_json](#output\_service\_credentials\_json) | The service credentials JSON map. | | [service\_credentials\_object](#output\_service\_credentials\_object) | The service credentials object. | diff --git a/examples/fscloud/README.md b/examples/fscloud/README.md index c3b05a56..0c28b5b3 100644 --- a/examples/fscloud/README.md +++ b/examples/fscloud/README.md @@ -8,7 +8,7 @@ The example uses the IBM Cloud Terraform provider to create the following infras - An IAM authorization between all Event Stream instances in the given resource group and the Hyper Protect Crypto Services instance that is passed in. - An Event Streams instance that is encrypted with the Hyper Protect Crypto Services root key that is passed in. - A sample virtual private cloud (VPC). -- A context-based restriction (CBR) rule to only allow Event Streams to be accessible from within the VPC. +- A context-based restriction (CBR) rule to only allow Event Streams to be accessible from within the VPC and Schematics. :exclamation: **Important:** In this example, only the Event Streams instance complies with the IBM Cloud Framework for Financial Services. Other parts of the infrastructure do not necessarily comply. diff --git a/examples/fscloud/main.tf b/examples/fscloud/main.tf index c2320ec8..f93f8418 100644 --- a/examples/fscloud/main.tf +++ b/examples/fscloud/main.tf @@ -36,7 +36,7 @@ resource "ibm_is_subnet" "testacc_subnet" { ############################################################################## # Create CBR Zone ############################################################################## -module "cbr_zone" { +module "cbr_vpc_zone" { source = "terraform-ibm-modules/cbr/ibm//modules/cbr-zone-module" version = "1.29.0" name = "${var.prefix}-VPC-network-zone" @@ -48,6 +48,21 @@ module "cbr_zone" { }] } +module "cbr_zone_schematics" { + source = "terraform-ibm-modules/cbr/ibm//modules/cbr-zone-module" + version = "1.29.0" + name = "${var.prefix}-schematics-zone" + zone_description = "CBR Network zone containing Schematics" + account_id = data.ibm_iam_account_settings.iam_account_settings.account_id + addresses = [{ + type = "serviceRef", + ref = { + account_id = data.ibm_iam_account_settings.iam_account_settings.account_id + service_name = "schematics" + } + }] +} + # ############################################################################# # Events-streams-instance @@ -63,6 +78,35 @@ module "event_streams" { topics = var.topics existing_kms_instance_guid = var.existing_kms_instance_guid metrics = ["topic", "partition", "consumers"] + mirroring_topic_patterns = ["topic-1", "topic-2"] + mirroring = { + source_crn = var.event_streams_source_crn # Required for mirroring + source_alias = "source-alias" # Required for mirroring + target_alias = "target-alias" # Required for mirroring + + # 'options' are optional. Valid values for 'type' are 'rename', 'none', or 'use_alias'. + # If 'type' is set to 'rename', then 'rename' object must include the following fields: 'add_prefix', 'add_suffix', 'remove_prefix', and 'remove_suffix'. + options = { + topic_name_transform = { + type = "rename" + rename = { + add_prefix = "add_prefix" + add_suffix = "add_suffix" + remove_prefix = "remove_prefix" + remove_suffix = "remove_suffix" + } + } + group_id_transform = { + type = "rename" + rename = { + add_prefix = "add_prefix" + add_suffix = "add_suffix" + remove_prefix = "remove_prefix" + remove_suffix = "remove_suffix" + } + } + } + } quotas = [ { "entity" = "iam-ServiceId-00000000-0000-0000-0000-000000000000", @@ -78,7 +122,7 @@ module "event_streams" { } cbr_rules = [ { - description = "${var.prefix}-event stream access only from vpc" + description = "${var.prefix}-event streams access from vpc and schematics" enforcement_mode = "enabled" account_id = data.ibm_iam_account_settings.iam_account_settings.account_id rule_contexts = [{ @@ -89,7 +133,17 @@ module "event_streams" { }, { name = "networkZoneId" - value = module.cbr_zone.zone_id + value = module.cbr_vpc_zone.zone_id + }] + }, { + attributes = [ + { + "name" : "endpointType", + "value" : "private" + }, + { + name = "networkZoneId" + value = module.cbr_zone_schematics.zone_id }] }] } diff --git a/examples/fscloud/outputs.tf b/examples/fscloud/outputs.tf index d000e52e..8e53bdba 100644 --- a/examples/fscloud/outputs.tf +++ b/examples/fscloud/outputs.tf @@ -43,3 +43,13 @@ output "service_credentials_object" { value = module.event_streams.service_credentials_object sensitive = true } + +output "mirroring_config_id" { + description = "The ID of the mirroring config in CRN format" + value = module.event_streams.mirroring_config_id +} + +output "mirroring_topic_patterns" { + description = "Mirroring topic patterns" + value = module.event_streams.mirroring_topic_patterns +} diff --git a/examples/fscloud/provider.tf b/examples/fscloud/provider.tf index df45ef50..39b64035 100644 --- a/examples/fscloud/provider.tf +++ b/examples/fscloud/provider.tf @@ -1,4 +1,5 @@ provider "ibm" { ibmcloud_api_key = var.ibmcloud_api_key region = var.region + visibility = "private" } diff --git a/examples/fscloud/variables.tf b/examples/fscloud/variables.tf index b251e010..4ec0733b 100644 --- a/examples/fscloud/variables.tf +++ b/examples/fscloud/variables.tf @@ -63,3 +63,8 @@ variable "kms_key_crn" { type = string description = "The root key CRN of a Hyper Protect Crypto Service (HPCS) that you want to use for disk encryption. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs&interface=ui for more information on integrating HPCS with Event Streams instance." } + +variable "event_streams_source_crn" { + type = string + description = "Source cluster CRN as a string to create mirroring instance." +} diff --git a/main.tf b/main.tf index 2cbdd4d4..452ebe2c 100644 --- a/main.tf +++ b/main.tf @@ -4,32 +4,39 @@ locals { # Validation (approach based on https://github.com/hashicorp/terraform/issues/25609#issuecomment-1057614400) + + # tflint-ignore: terraform_unused_declarations + validate_kms_plan = var.plan != "enterprise-3nodes-2tb" && var.kms_key_crn != null ? tobool("KMS encryption is only supported for enterprise plan.") : true + # tflint-ignore: terraform_unused_declarations + validate_metrics = var.plan != "enterprise-3nodes-2tb" && length(var.metrics) > 0 ? tobool("Metrics are only supported for enterprise plan.") : true + # tflint-ignore: terraform_unused_declarations + validate_quotas = var.plan != "enterprise-3nodes-2tb" && length(var.quotas) > 0 ? tobool("Quotas are only supported for enterprise plan.") : true # tflint-ignore: terraform_unused_declarations - validate_kms_plan = var.kms_key_crn != null && var.plan != "enterprise-3nodes-2tb" ? tobool("kms encryption is only supported for enterprise plan") : true + validate_schema_global_rule = var.plan != "enterprise-3nodes-2tb" && var.schema_global_rule != null ? tobool("Schema global rule is only supported for enterprise plan.") : true + # tflint-ignore: terraform_unused_declarations - validate_kms_values = !var.kms_encryption_enabled && var.kms_key_crn != null ? tobool("When passing values for var.kms_key_crn, you must set var.kms_encryption_enabled to true. Otherwise unset them to use default encryption") : true + validate_kms_values = !var.kms_encryption_enabled && var.kms_key_crn != null ? tobool("When passing values for var.kms_key_crn, you must set var.kms_encryption_enabled to true. Otherwise unset them to use default encryption.") : true # tflint-ignore: terraform_unused_declarations - validate_kms_vars = var.kms_encryption_enabled && var.kms_key_crn == null ? tobool("When setting var.kms_encryption_enabled to true, a value must be passed for var.kms_key_crn and/or var.backup_encryption_key_crn") : true + validate_kms_vars = var.kms_encryption_enabled && var.kms_key_crn == null ? tobool("When setting var.kms_encryption_enabled to true, a value must be passed for var.kms_key_crn and/or var.backup_encryption_key_crn.") : true # tflint-ignore: terraform_unused_declarations - validate_auth_policy = var.kms_encryption_enabled && var.skip_iam_authorization_policy == false && var.existing_kms_instance_guid == null ? tobool("When var.skip_iam_authorization_policy is set to false, and var.kms_encryption_enabled to true, a value must be passed for var.existing_kms_instance_guid in order to create the auth policy.") : true + validate_auth_policy = var.kms_encryption_enabled && var.skip_kms_iam_authorization_policy == false && var.existing_kms_instance_guid == null ? tobool("When var.skip_kms_iam_authorization_policy is set to false, and var.kms_encryption_enabled to true, a value must be passed for var.existing_kms_instance_guid in order to create the auth policy.") : true # tflint-ignore: terraform_unused_declarations validate_throughput_lite_standard = ((var.plan == "lite" || var.plan == "standard") && var.throughput != 150) ? tobool("Throughput value cannot be changed in lite and standard plan. Default value is 150.") : true # tflint-ignore: terraform_unused_declarations validate_storage_size_lite_standard = ((var.plan == "lite" || var.plan == "standard") && var.storage_size != 2048) ? tobool("Storage size value cannot be changed in lite and standard plan. Default value is 2048.") : true # tflint-ignore: terraform_unused_declarations validate_service_end_points_lite_standard = ((var.plan == "lite" || var.plan == "standard") && var.service_endpoints != "public") ? tobool("Service endpoint cannot be changed in lite and standard plan. Default is public.") : true + # tflint-ignore: terraform_unused_declarations + validate_mirroring_topics = var.mirroring == null && var.mirroring_topic_patterns != null ? tobool("When passing values for var.mirroring_topic_patterns, values must also be passed for var.mirroring.") : true + # tflint-ignore: terraform_unused_declarations + validate_mirroring_config = var.mirroring != null && var.mirroring_topic_patterns == null ? tobool("When passing values for var.mirroring, values must also be passed for var.mirroring_topic_patterns.") : true + # Determine what KMS service is being used for database encryption kms_service = var.kms_key_crn != null ? ( can(regex(".*kms.*", var.kms_key_crn)) ? "kms" : ( can(regex(".*hs-crypto.*", var.kms_key_crn)) ? "hs-crypto" : null ) ) : null - # tflint-ignore: terraform_unused_declarations - validate_metrics = var.plan != "enterprise-3nodes-2tb" && length(var.metrics) > 0 ? tobool("metrics are only supported for enterprise plan") : true - # tflint-ignore: terraform_unused_declarations - validate_quotas = var.plan != "enterprise-3nodes-2tb" && length(var.quotas) > 0 ? tobool("quotas are only supported for enterprise plan") : true - # tflint-ignore: terraform_unused_declarations - validate_schema_global_rule = var.plan != "enterprise-3nodes-2tb" && var.schema_global_rule != null ? tobool("schema global rule is only supported for enterprise plan") : true } # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 @@ -40,7 +47,7 @@ resource "time_sleep" "wait_for_authorization_policy" { } resource "ibm_resource_instance" "es_instance" { - depends_on = [time_sleep.wait_for_authorization_policy] + depends_on = [time_sleep.wait_for_kms_authorization_policy, time_sleep.wait_for_es_s2s_policy] name = var.es_name service = "messagehub" plan = var.plan @@ -53,20 +60,20 @@ resource "ibm_resource_instance" "es_instance" { delete = var.delete_timeout } - parameters_json = var.plan != "enterprise-3nodes-2tb" ? null : var.kms_key_crn != null ? jsonencode( + parameters_json = var.plan != "enterprise-3nodes-2tb" ? null : (var.kms_key_crn != null || var.metrics != null || var.mirroring != null) ? jsonencode( { service-endpoints = var.service_endpoints throughput = tostring(var.throughput) storage_size = tostring(var.storage_size) metrics = var.metrics kms_key_crn = var.kms_key_crn + mirroring = var.mirroring } ) : jsonencode( { service-endpoints = var.service_endpoints throughput = tostring(var.throughput) storage_size = tostring(var.storage_size) - metrics = var.metrics } ) } @@ -123,12 +130,12 @@ resource "ibm_event_streams_quota" "eventstreams_quotas" { } ############################################################################## -# IAM Authorization Policy +# IAM Authorization Policies ############################################################################## # Create IAM Authorization Policies to allow messagehub to access kms for the encryption key resource "ibm_iam_authorization_policy" "kms_policy" { - count = var.kms_encryption_enabled == false || var.skip_iam_authorization_policy ? 0 : 1 + count = var.kms_encryption_enabled == false || var.skip_kms_iam_authorization_policy ? 0 : 1 source_service_name = "messagehub" source_resource_group_id = var.resource_group_id target_service_name = local.kms_service @@ -137,6 +144,41 @@ resource "ibm_iam_authorization_policy" "kms_policy" { description = "Allow all Event Streams instances in the resource group ${var.resource_group_id} to read from the ${local.kms_service} instance GUID ${var.existing_kms_instance_guid}" } +# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 +resource "time_sleep" "wait_for_kms_authorization_policy" { + count = var.kms_encryption_enabled == false || var.skip_kms_iam_authorization_policy ? 0 : 1 + depends_on = [ibm_iam_authorization_policy.kms_policy] + + create_duration = "30s" +} + +# Parse GUID from source ES instance +module "es_guid_crn_parser" { + count = var.mirroring != null ? 1 : 0 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.mirroring.source_crn +} + +# Create s2s at service level for provisioning mirroring instance +resource "ibm_iam_authorization_policy" "es_s2s_policy" { + count = var.mirroring == null || var.skip_es_s2s_iam_authorization_policy ? 0 : 1 + source_service_name = "messagehub" + source_resource_group_id = var.resource_group_id + target_service_name = "messagehub" + target_resource_instance_id = module.es_guid_crn_parser[0].service_instance + roles = ["Reader"] + description = "Allow all Event Streams instances in the resource group ${var.resource_group_id} to read from the source Event Streams instance ${module.es_guid_crn_parser[0].service_instance}." +} + +# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 +resource "time_sleep" "wait_for_es_s2s_policy" { + count = var.mirroring == null || var.skip_es_s2s_iam_authorization_policy ? 0 : 1 + depends_on = [ibm_iam_authorization_policy.es_s2s_policy] + + create_duration = "30s" +} + ############################################################################## # Context Based Restrictions ############################################################################## @@ -188,3 +230,9 @@ locals { } } : null } + +resource "ibm_event_streams_mirroring_config" "es_mirroring_config" { + count = var.mirroring != null ? 1 : 0 + resource_instance_id = ibm_resource_instance.es_instance.id + mirroring_topic_patterns = var.mirroring_topic_patterns +} diff --git a/modules/fscloud/README.md b/modules/fscloud/README.md index a88015af..4208639c 100644 --- a/modules/fscloud/README.md +++ b/modules/fscloud/README.md @@ -32,12 +32,16 @@ No resources. | [existing\_kms\_instance\_guid](#input\_existing\_kms\_instance\_guid) | The GUID of the Hyper Protect Crypto service in which the key specified in var.kms\_key\_crn is coming from | `string` | n/a | yes | | [kms\_key\_crn](#input\_kms\_key\_crn) | The root key CRN of the key management service (Key Protect or Hyper Protect Crypto Services) to use to encrypt the payload data. | `string` | n/a | yes | | [metrics](#input\_metrics) | Enhanced metrics to activate, as list of strings. Allowed values: 'topic', 'partition', 'consumers'. | `list(string)` | `[]` | no | +| [mirroring](#input\_mirroring) | Event Streams mirroring configuration. Required only if creating mirroring instance. For more information on mirroring, see https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-mirroring. |
object({
source_crn = string
source_alias = string
target_alias = string
options = optional(object({
topic_name_transform = object({
type = string
rename = optional(object({
add_prefix = optional(string)
add_suffix = optional(string)
remove_prefix = optional(string)
remove_suffix = optional(string)
}))
})
group_id_transform = object({
type = string
rename = optional(object({
add_prefix = optional(string)
add_suffix = optional(string)
remove_prefix = optional(string)
remove_suffix = optional(string)
}))
})
}))
})
| `null` | no | +| [mirroring\_topic\_patterns](#input\_mirroring\_topic\_patterns) | The list of the topics to set in instance. Required only if creating mirroring instance. | `list(string)` | `null` | no | | [quotas](#input\_quotas) | Quotas to be applied to the Event Streams instance. Entity may be 'default' to apply to all users, or an IAM ServiceID for a specific user. Rates are bytes/second, with -1 meaning no quota. |
list(object({
entity = string
producer_byte_rate = optional(number, -1)
consumer_byte_rate = optional(number, -1)
}))
| `[]` | no | | [region](#input\_region) | The region where the Event Streams are created. | `string` | `"us-south"` | no | | [resource\_group\_id](#input\_resource\_group\_id) | The resource group ID where the Event Streams instance is created. | `string` | n/a | yes | | [schema\_global\_rule](#input\_schema\_global\_rule) | Schema global compatibility rule. Allowed values are 'NONE', 'FULL', 'FULL\_TRANSITIVE', 'FORWARD', 'FORWARD\_TRANSITIVE', 'BACKWARD', 'BACKWARD\_TRANSITIVE'. | `string` | `null` | no | | [schemas](#input\_schemas) | The list of schema objects. Include the `schema_id` and the `type` and `name` of the schema in the `schema` object. |
list(object(
{
schema_id = string
schema = object({
type = string
name = string
fields = optional(list(object({
name = string
type = string
})))
})
}
))
| `[]` | no | | [service\_credential\_names](#input\_service\_credential\_names) | The mapping of names and roles for service credentials that you want to create for the Event streams. | `map(string)` | `{}` | no | +| [skip\_es\_s2s\_iam\_authorization\_policy](#input\_skip\_es\_s2s\_iam\_authorization\_policy) | Set to true to skip the creation of an Event Streams s2s IAM authorization policy to provision an Event Streams mirroring instance. This is required to read from the source cluster. This policy is required when creating mirroring instance. | `bool` | `false` | no | +| [skip\_kms\_iam\_authorization\_policy](#input\_skip\_kms\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the existing\_kms\_instance\_guid variable. In addition, no policy is created if var.kms\_encryption\_enabled is set to false. | `bool` | `false` | no | | [tags](#input\_tags) | The list of tags associated with the Event Steams instance. | `list(string)` | `[]` | no | | [topics](#input\_topics) | The list of topics to apply to resources. Only one topic is allowed for Lite plan instances. |
list(object(
{
name = string
partitions = number
config = map(string)
}
))
| `[]` | no | @@ -51,6 +55,8 @@ No resources. | [kafka\_broker\_version](#output\_kafka\_broker\_version) | The Kafka version | | [kafka\_brokers\_sasl](#output\_kafka\_brokers\_sasl) | (Array of Strings) Kafka brokers use for interacting with Kafka native API | | [kafka\_http\_url](#output\_kafka\_http\_url) | The API endpoint to interact with Event Streams REST API | +| [mirroring\_config\_id](#output\_mirroring\_config\_id) | The ID of the mirroring config in CRN format | +| [mirroring\_topic\_patterns](#output\_mirroring\_topic\_patterns) | Mirroring topic patterns | | [service\_credentials\_json](#output\_service\_credentials\_json) | Service credentials json map | | [service\_credentials\_object](#output\_service\_credentials\_object) | Service credentials object | diff --git a/modules/fscloud/main.tf b/modules/fscloud/main.tf index 823f4142..75e89032 100644 --- a/modules/fscloud/main.tf +++ b/modules/fscloud/main.tf @@ -1,20 +1,24 @@ module "event_streams" { - source = "../../" - resource_group_id = var.resource_group_id - es_name = var.es_name - plan = "enterprise-3nodes-2tb" - region = var.region - kms_key_crn = var.kms_key_crn - existing_kms_instance_guid = var.existing_kms_instance_guid - schemas = var.schemas - schema_global_rule = var.schema_global_rule - tags = var.tags - access_tags = var.access_tags - topics = var.topics - service_endpoints = "private" - cbr_rules = var.cbr_rules - service_credential_names = var.service_credential_names - metrics = var.metrics - quotas = var.quotas - kms_encryption_enabled = true + source = "../../" + resource_group_id = var.resource_group_id + es_name = var.es_name + plan = "enterprise-3nodes-2tb" + region = var.region + kms_key_crn = var.kms_key_crn + existing_kms_instance_guid = var.existing_kms_instance_guid + skip_kms_iam_authorization_policy = var.skip_kms_iam_authorization_policy + skip_es_s2s_iam_authorization_policy = var.skip_es_s2s_iam_authorization_policy + schemas = var.schemas + schema_global_rule = var.schema_global_rule + tags = var.tags + access_tags = var.access_tags + topics = var.topics + service_endpoints = "private" + cbr_rules = var.cbr_rules + service_credential_names = var.service_credential_names + metrics = var.metrics + quotas = var.quotas + kms_encryption_enabled = true + mirroring_topic_patterns = var.mirroring_topic_patterns + mirroring = var.mirroring } diff --git a/modules/fscloud/outputs.tf b/modules/fscloud/outputs.tf index 30f93796..034776af 100644 --- a/modules/fscloud/outputs.tf +++ b/modules/fscloud/outputs.tf @@ -43,3 +43,13 @@ output "service_credentials_object" { value = module.event_streams.service_credentials_object sensitive = true } + +output "mirroring_config_id" { + description = "The ID of the mirroring config in CRN format" + value = module.event_streams.mirroring_config_id +} + +output "mirroring_topic_patterns" { + description = "Mirroring topic patterns" + value = module.event_streams.mirroring_topic_patterns +} diff --git a/modules/fscloud/variables.tf b/modules/fscloud/variables.tf index be84db00..22bbe815 100644 --- a/modules/fscloud/variables.tf +++ b/modules/fscloud/variables.tf @@ -64,6 +64,18 @@ variable "topics" { default = [] } +variable "skip_kms_iam_authorization_policy" { + type = bool + description = "Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the existing_kms_instance_guid variable. In addition, no policy is created if var.kms_encryption_enabled is set to false." + default = false +} + +variable "skip_es_s2s_iam_authorization_policy" { + type = bool + description = "Set to true to skip the creation of an Event Streams s2s IAM authorization policy to provision an Event Streams mirroring instance. This is required to read from the source cluster. This policy is required when creating mirroring instance." + default = false +} + variable "existing_kms_instance_guid" { description = "The GUID of the Hyper Protect Crypto service in which the key specified in var.kms_key_crn is coming from" type = string @@ -114,8 +126,44 @@ variable "quotas" { })) description = "Quotas to be applied to the Event Streams instance. Entity may be 'default' to apply to all users, or an IAM ServiceID for a specific user. Rates are bytes/second, with -1 meaning no quota." default = [] - validation { - condition = alltrue([for v in var.quotas : v.entity != "" && (v.producer_byte_rate >= 0 || v.consumer_byte_rate >= 0)]) - error_message = "The quota entity must be defined, and at least one of producer_byte_rate or consumer_byte_rate must be set to a non-negative value" - } +} + +############################################################## +# Mirroring +############################################################## + +variable "mirroring_topic_patterns" { + type = list(string) + description = "The list of the topics to set in instance. Required only if creating mirroring instance." + default = null +} + +variable "mirroring" { + description = "Event Streams mirroring configuration. Required only if creating mirroring instance. For more information on mirroring, see https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-mirroring." + type = object({ + source_crn = string + source_alias = string + target_alias = string + options = optional(object({ + topic_name_transform = object({ + type = string + rename = optional(object({ + add_prefix = optional(string) + add_suffix = optional(string) + remove_prefix = optional(string) + remove_suffix = optional(string) + })) + }) + group_id_transform = object({ + type = string + rename = optional(object({ + add_prefix = optional(string) + add_suffix = optional(string) + remove_prefix = optional(string) + remove_suffix = optional(string) + })) + }) + })) + }) + default = null } diff --git a/outputs.tf b/outputs.tf index 262cd73e..96113a81 100644 --- a/outputs.tf +++ b/outputs.tf @@ -46,3 +46,13 @@ output "service_credentials_object" { value = local.service_credentials_object sensitive = true } + +output "mirroring_config_id" { + description = "The ID of the mirroring config in CRN format" + value = var.mirroring != null ? ibm_event_streams_mirroring_config.es_mirroring_config[0].id : null +} + +output "mirroring_topic_patterns" { + description = "Mirroring topic patterns" + value = var.mirroring != null ? ibm_event_streams_mirroring_config.es_mirroring_config[0].mirroring_topic_patterns : null +} diff --git a/tests/other_test.go b/tests/other_test.go index aa2ed057..658380dc 100644 --- a/tests/other_test.go +++ b/tests/other_test.go @@ -6,11 +6,9 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" ) const basicExampleTerraformDir = "examples/basic" -const fsCloudTerraformDir = "examples/fscloud" func TestRunDefaultExample(t *testing.T) { t.Parallel() @@ -21,28 +19,3 @@ func TestRunDefaultExample(t *testing.T) { assert.Nil(t, err, "This should not have errored") assert.NotNil(t, output, "Expected some output") } - -func TestRunFSCloudExample(t *testing.T) { - t.Parallel() - options := testhelper.TestOptionsDefaultWithVars(&testhelper.TestOptions{ - Testing: t, - TerraformDir: fsCloudTerraformDir, - Prefix: "es-fscloud", - /* - Comment out the 'ResourceGroup' input to force this tests to create a unique resource group to ensure tests do - not clash. This is due to the fact that an auth policy may already exist in this resource group since we are - re-using a permanent HPCS instance. By using a new resource group, the auth policy will not already exist - since this module scopes auth policies by resource group. - */ - //ResourceGroup: resourceGroup, - BestRegionYAMLPath: regionSelectionPath, - TerraformVars: map[string]interface{}{ - "existing_kms_instance_guid": permanentResources["hpcs_south"], - "kms_key_crn": permanentResources["hpcs_south_root_key_crn"], - }, - }) - - output, err := options.RunTestConsistency() - assert.Nil(t, err, "This should not have errored") - assert.NotNil(t, output, "Expected some output") -} diff --git a/tests/pr_test.go b/tests/pr_test.go index 5edb5b5b..d3a7d686 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -9,10 +9,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/common" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" + "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testschematic" ) const completeExampleTerraformDir = "examples/complete" const quickstartSolutionTerraformDir = "solutions/quickstart" +const fsCloudTerraformDir = "examples/fscloud" // Use existing group for tests const resourceGroup = "geretain-test-event-streams" @@ -80,3 +82,40 @@ func TestRunQuickstartSolution(t *testing.T) { assert.Nil(t, err, "This should not have errored") assert.NotNil(t, output, "Expected some output") } + +func TestFSCloudInSchematics(t *testing.T) { + t.Parallel() + + options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ + Testing: t, + Prefix: "es-fscloud", + BestRegionYAMLPath: regionSelectionPath, + TarIncludePatterns: []string{ + "*.tf", + fsCloudTerraformDir + "/*.tf", + "modules/fscloud/*.tf", + }, + /* + Comment out the 'ResourceGroup' input to force this tests to create a unique resource group to ensure tests do + not clash. This is due to the fact that an auth policy may already exist in this resource group since we are + re-using a permanent HPCS instance and a permanent Event Streams instance. By using a new resource group, the auth policy will not already exist + since this module scopes auth policies by resource group. + */ + //ResourceGroup: resourceGroup, + TemplateFolder: fsCloudTerraformDir, + Tags: []string{"test-schematic"}, + DeleteWorkspaceOnFail: false, + WaitJobCompleteMinutes: 180, + }) + + options.TerraformVars = []testschematic.TestSchematicTerraformVar{ + {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, + {Name: "prefix", Value: options.Prefix, DataType: "string"}, + {Name: "existing_kms_instance_guid", Value: permanentResources["hpcs_south"].(string), DataType: "string"}, + {Name: "kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"].(string), DataType: "string"}, + {Name: "event_streams_source_crn", Value: permanentResources["event_streams_us_south_crn"].(string), DataType: "string"}, + } + + err := options.RunSchematicTest() + assert.Nil(t, err, "This should not have errored") +} diff --git a/variables.tf b/variables.tf index 842e557b..cae359b3 100644 --- a/variables.tf +++ b/variables.tf @@ -81,9 +81,15 @@ variable "service_endpoints" { } } -variable "skip_iam_authorization_policy" { +variable "skip_kms_iam_authorization_policy" { type = bool - description = "Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the existing_kms_instance_guid variable. In addition, no policy is created if var.kms_encryption_enabled is set to false." + description = "Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the `existing_kms_instance_guid` variable. In addition, no policy is created if var.kms_encryption_enabled is set to false." + default = false +} + +variable "skip_es_s2s_iam_authorization_policy" { + type = bool + description = "Set to true to skip the creation of an IAM authorization policy that will allow all Event Streams instances in the given resource group access to read from the mirror source instance. This policy is required when creating a mirroring instance, and will only be created if a value is passed in the mirroring input." default = false } @@ -148,7 +154,7 @@ variable "kms_key_crn" { } variable "existing_kms_instance_guid" { - description = "The GUID of the Hyper Protect Crypto Services or Key Protect instance in which the key specified in var.kms_key_crn is coming from. Required only if var.kms_encryption_enabled is set to true, var.skip_iam_authorization_policy is set to false, and you pass a value for var.kms_key_crn." + description = "The GUID of the Hyper Protect Crypto Services or Key Protect instance in which the key specified in var.kms_key_crn is coming from. Required only if var.kms_encryption_enabled is set to true, var.skip_kms_iam_authorization_policy is set to false, and you pass a value for var.kms_key_crn." type = string default = null } @@ -225,3 +231,39 @@ variable "quotas" { error_message = "The quota entity must be defined, and at least one of producer_byte_rate or consumer_byte_rate must be set to a non-negative value" } } + +variable "mirroring_topic_patterns" { + type = list(string) + description = "The list of the topics to set in instance. Required only if creating mirroring instance." + default = null +} + +variable "mirroring" { + description = "Event Streams mirroring configuration. Required only if creating mirroring instance. For more information on mirroring, see https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-mirroring." + type = object({ + source_crn = string + source_alias = string + target_alias = string + options = optional(object({ + topic_name_transform = object({ + type = string + rename = optional(object({ + add_prefix = optional(string) + add_suffix = optional(string) + remove_prefix = optional(string) + remove_suffix = optional(string) + })) + }) + group_id_transform = object({ + type = string + rename = optional(object({ + add_prefix = optional(string) + add_suffix = optional(string) + remove_prefix = optional(string) + remove_suffix = optional(string) + })) + }) + })) + }) + default = null +}