feat: new mongodbatlas_stream_connection resource (#1736)

* feat: new mongodbatlas_stream_connection resource

* define dummy ca cert in external file and embed into variable for testing

* fix content of id

* refactor: avoid depending on existing project for CI tests in search index and stream connection resources (#1737)

* refactor: common cluster config funtion avoids depending on existing project

* define return value as struct and unify comments

* doc: `mongodbatlas_stream_connection` resource docs, examples, and general Atlas Streams guide (#1752)

* docs: stream connection resource docs and examples

* addressing doc review comments

* fix yaml formatting in header + examples mentioning limitations

Author: AgustinBettati

.github/workflows/acceptance-tests.yml

@@ -68,6 +68,7 @@ jobs:
             - 'internal/service/searchdeployment/*.go'
           stream:
             - 'internal/service/streaminstance/*.go'
+            - 'internal/service/streamconnection/*.go'
           generic:
             - 'internal/service/backupcompliancepolicy/*.go'
             - 'internal/service/auditing/*.go'
@@ -528,7 +529,6 @@ jobs:
           MONGODB_ATLAS_PUBLIC_KEY: ${{ secrets.MONGODB_ATLAS_PUBLIC_KEY_CLOUD_DEV }}
           MONGODB_ATLAS_PRIVATE_KEY: ${{ secrets.MONGODB_ATLAS_PRIVATE_KEY_CLOUD_DEV }}
           MONGODB_ATLAS_ORG_ID: ${{ vars.MONGODB_ATLAS_ORG_ID_CLOUD_DEV }}
-          MONGODB_ATLAS_PROJECT_ID: ${{ vars.MONGODB_ATLAS_ACCTEST_SEARCH_INDEX_PROJECT_ID_CLOUD_DEV }}
           TEST_REGEX: "^TestAccSearchIndex"
         run: make testacc
 

examples/atlas-streams/README.md

@@ -0,0 +1,24 @@
+# MongoDB Atlas Provider - Atlas Streams with Terraform
+
+Atlas Stream Processing is composed of multiple components, and users can leverage Terraform to define a subset of these. To obtain more details on each of the components please refer to the [Atlas Stream Processing Documentation](https://www.mongodb.com/docs/atlas/atlas-sp/overview/#atlas-stream-processing-overview).
+
+### Resources supported by Terraform
+
+- `mongodbatlas_stream_instance`: Enables creating, modifying, and deleting Stream Instances. as part of this resource, a computed `hostnames` attribute is available for connecting to the created instance.
+- `mongodbatlas_stream_connection`: Enables creating, modifying, and deleting Stream Instance Connections, which serve as data sources and sinks for your instance.
+
+**NOTE**: To leverage these resources you'll need to set the environment variable `MONGODB_ATLAS_ENABLE_BETA=true` as this functionality is currently in preview.  Also see [Limitations](https://www.mongodb.com/docs/atlas/atlas-sp/limitations/#std-label-atlas-sp-limitations) of Atlas Streams during this preview period.
+
+### Managing Stream Processors
+
+Once a stream instance and its connections have been defined, `Stream Processors` can be created to define how your data will be processed in your instance. There are currently no resources defined in Terraform to provide this configuration. To obtain information on how this can be configured refer to [Manage Stream Processors](https://www.mongodb.com/docs/atlas/atlas-sp/manage-stream-processor/#manage-stream-processors).
+
+Connect to your stream instance defined in terraform using the following code block:
+```
+output "stream_instance_hostname" {
+  value = mongodbatlas_stream_instance.test.hostnames
+}
+```
+
+This value can then be used to connect to the stream instance using `mongosh`, as described in the [Get Started Tutorial](https://www.mongodb.com/docs/atlas/atlas-sp/tutorial/). 
+

examples/atlas-streams/stream-connection/README.md

@@ -0,0 +1,15 @@
+# MongoDB Atlas Provider - Atlas Stream Instance defined in a Project
+
+This example shows how to create Atlas Stream Connections in Terraform. It also creates a stream instance, which is a prerequisite. Both Kafka and Cluster connections types are defined to showcase their usage.
+
+You must set the following variables:
+
+- `public_key`: Atlas public key
+- `private_key`: Atlas  private key
+- `project_id`: Unique 24-hexadecimal digit string that identifies the project where the stream instance will be created.
+- `kafka_username`: Username used for connecting to your external Kafka Cluster. 
+- `kafka_password`: Password used for connecting to your external Kafka Cluster.
+- `kafka_ssl_cert`: String value of public x509 certificate for connecting to Kafka over SSL.
+- `cluster_name`: Name of Cluster that will be used for creating a connection.
+
+To learn more, see the [Stream Instance Connection Registry Documentation](https://www.mongodb.com/docs/atlas/atlas-sp/manage-processing-instance/#view-connections-in-the-connection-registry).

examples/atlas-streams/stream-connection/main.tf

@@ -0,0 +1,55 @@
+resource "mongodbatlas_stream_instance" "example" {
+  project_id    = var.project_id
+  instance_name = "InstanceName"
+  data_process_region = {
+    region         = "VIRGINIA_USA"
+    cloud_provider = "AWS"
+  }
+}
+
+resource "mongodbatlas_stream_connection" "example-cluster" {
+  project_id      = var.project_id
+  instance_name   = mongodbatlas_stream_instance.example.instance_name
+  connection_name = "ClusterConnection"
+  type            = "Cluster"
+  cluster_name    = var.cluster_name
+}
+
+resource "mongodbatlas_stream_connection" "example-kafka-plaintext" {
+  project_id      = var.project_id
+  instance_name   = mongodbatlas_stream_instance.example.instance_name
+  connection_name = "KafkaPlaintextConnection"
+  type            = "Kafka"
+  authentication = {
+    mechanism = "PLAIN"
+    username  = var.kafka_username
+    password  = var.kafka_password
+  }
+  bootstrap_servers = "localhost:9092,localhost:9092"
+  config = {
+    "auto.offset.reset" : "earliest"
+  }
+  security = {
+    protocol = "PLAINTEXT"
+  }
+}
+
+resource "mongodbatlas_stream_connection" "example-kafka-ssl" {
+  project_id      = var.project_id
+  instance_name   = mongodbatlas_stream_instance.example.instance_name
+  connection_name = "KafkaSSLConnection"
+  type            = "Kafka"
+  authentication = {
+    mechanism = "PLAIN"
+    username  = var.kafka_username
+    password  = var.kafka_password
+  }
+  bootstrap_servers = "localhost:9092,localhost:9092"
+  config = {
+    "auto.offset.reset" : "earliest"
+  }
+  security = {
+    broker_public_certificate = var.kafka_ssl_cert
+    protocol                  = "SSL"
+  }
+}

examples/atlas-streams/stream-connection/provider.tf

@@ -0,0 +1,4 @@
+provider "mongodbatlas" {
+  public_key  = var.public_key
+  private_key = var.private_key
+}

examples/atlas-streams/stream-connection/variables.tf

@@ -0,0 +1,32 @@
+variable "public_key" {
+  description = "Public API key to authenticate to Atlas"
+  type        = string
+}
+variable "private_key" {
+  description = "Private API key to authenticate to Atlas"
+  type        = string
+}
+variable "project_id" {
+  description = "Unique 24-hexadecimal digit string that identifies your project"
+  type        = string
+}
+
+variable "kafka_username" {
+  description = "Username for connecting to your Kafka cluster"
+  type        = string
+}
+
+variable "kafka_password" {
+  description = "Password for connecting to your Kafka cluster"
+  type        = string
+}
+
+variable "kafka_ssl_cert" {
+  description = "Public certificate used for SSL configuration to connect to your Kafka cluster"
+  type        = string
+}
+
+variable "cluster_name" {
+  description = "Name of an existing cluster in your project that will be used to create a stream connection"
+  type        = string
+}

examples/atlas-streams/stream-connection/versions.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_providers {
+    mongodbatlas = {
+      source  = "mongodb/mongodbatlas"
+      version = "~> 1.14"
+    }
+  }
+  required_version = ">= 1.0"
+}

internal/provider/provider.go

@@ -33,6 +33,7 @@ import (
 	"github.com/mongodb/terraform-provider-mongodbatlas/internal/service/project"
 	"github.com/mongodb/terraform-provider-mongodbatlas/internal/service/projectipaccesslist"
 	"github.com/mongodb/terraform-provider-mongodbatlas/internal/service/searchdeployment"
+	"github.com/mongodb/terraform-provider-mongodbatlas/internal/service/streamconnection"
 	"github.com/mongodb/terraform-provider-mongodbatlas/internal/service/streaminstance"
 
 	"github.com/mongodb/terraform-provider-mongodbatlas/version"
@@ -435,6 +436,7 @@ func (p *MongodbtlasProvider) Resources(context.Context) []func() resource.Resou
 	}
 	betaResources := []func() resource.Resource{
 		streaminstance.Resource,
+		streamconnection.Resource,
 	}
 	if ProviderEnableBeta {
 		resources = append(resources, betaResources...)