From 5363024bd19aaf2f298b07738f5e2c5263a5f4f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Gniewek-W=C4=99grzyn?= Date: Thu, 19 Sep 2024 11:47:04 +0200 Subject: [PATCH] feat: initial commit of Snowflake Shared Database module --- README.md | 99 +++++++++++++---------- examples/complete/.env.dist | 10 +++ examples/complete/README.md | 77 +++++++++++++++--- examples/complete/create-share.sql | 23 ++++++ examples/complete/fixtures.tfvars | 17 ++-- examples/complete/main.tf | 43 ++++++++-- examples/complete/outputs.tf | 2 +- examples/complete/providers.tf | 4 +- examples/complete/variables.tf | 4 + examples/complete/versions.tf | 9 +-- examples/simple/.env.dist | 10 +++ examples/simple/Makefile | 2 +- examples/simple/README.md | 44 ++++++++--- examples/simple/create-share.sql | 23 ++++++ examples/simple/main.tf | 11 +-- examples/simple/outputs.tf | 2 +- examples/simple/providers.tf | 4 +- examples/simple/variables.tf | 4 + examples/simple/versions.tf | 9 +-- locals.tf | 69 +++++++++++++--- main.tf | 97 +++++++++++++++++------ outputs.tf | 76 +++++++++++++++++- variables.tf | 123 ++++++++++++++++++++++++++--- versions.tf | 11 +-- 24 files changed, 622 insertions(+), 151 deletions(-) create mode 100644 examples/complete/create-share.sql create mode 100644 examples/complete/variables.tf create mode 100644 examples/simple/create-share.sql create mode 100644 examples/simple/variables.tf diff --git a/README.md b/README.md index a5e68b7..23dce30 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,9 @@ -# Terraform Module Template - - -> **Warning**: -> This is a template document. Remember to **remove** all text in _italics_ and **update** Module name, Repo name and links/badges to the actual name of your GitHub repository/module!!! - - - - -![AWS](https://img.shields.io/badge/AWS-%23FF9900.svg?style=for-the-badge&logo=amazon-aws&logoColor=white) - +# Snowflake Database Terraform Module +![Snowflake](https://img.shields.io/badge/-SNOWFLAKE-249edc?style=for-the-badge&logo=snowflake&logoColor=white) ![Terraform](https://img.shields.io/badge/terraform-%235835CC.svg?style=for-the-badge&logo=terraform&logoColor=white) - -![License](https://badgen.net/github/license/getindata/terraform-module-template/) -![Release](https://badgen.net/github/release/getindata/terraform-module-template/) +![License](https://badgen.net/github/license/getindata/terraform-snowflake-shared-database/) +![Release](https://badgen.net/github/release/getindata/terraform-snowflake-shared-database/)

@@ -22,30 +12,27 @@ --- -_Brief Description of MODULE:_ - -* _What it does_ -* _What technologies it uses_ +Terraform module for Snowflake Shared Database management. -> **Warning**: -> _When using "Invicton-Labs/deepmerge/null" module - pin `tflint` version to `v0.41.0` in [pre-commit.yaml](.github/workflows/pre-commit.yml) to avoid failing `tflint` checks_ +* Creates Snowflake Shared database +* Can create custom Snowflake account roles with role-to-role assignments +* Can create a set of default account roles to simplify access management: + * `READONLY` - granted `IMPORTED_PRIVILEGES` privilege on the database ## USAGE -_Example usage of the module - terraform code snippet_ - ```terraform -module "template" { - source = "getindata/template/null" +module "snowflake_shared_database" { + source = "getindata/shared-database/snowflake" # version = "x.x.x" - example_var = "foo" -} -``` + name = "SHARED_DATABASE" + from_share = "" -## NOTES + create_default_roles = true +} -_Additional information that should be made public, for ex. how to solve known issues, additional descriptions/suggestions_ +``` ## EXAMPLES @@ -63,58 +50,90 @@ _Additional information that should be made public, for ex. how to solve known i |------|-------------|------|---------|:--------:| | [additional\_tag\_map](#input\_additional\_tag\_map) | Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`.
This is for some rare cases where resources want additional configuration of tags
and therefore take a list of maps with tag key, value, and additional configuration. | `map(string)` | `{}` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | +| [catalog](#input\_catalog) | The database parameter that specifies the default catalog to use for Iceberg tables | `string` | `null` | no | +| [comment](#input\_comment) | Specifies a comment for the database | `string` | `null` | no | | [context](#input\_context) | Single object for setting entire context at once.
See description of individual variables for details.
Leave string and numeric variables as `null` to use default value.
Individual variable settings (non-null) override settings in context object,
except for attributes, tags, and additional\_tag\_map, which are merged. | `any` | 

{
  "additional_tag_map": {},
  "attributes": [],
  "delimiter": null,
  "descriptor_formats": {},
  "enabled": true,
  "environment": null,
  "id_length_limit": null,
  "label_key_case": null,
  "label_order": [],
  "label_value_case": null,
  "labels_as_tags": [
    "unset"
  ],
  "name": null,
  "namespace": null,
  "regex_replace_chars": null,
  "stage": null,
  "tags": {},
  "tenant": null
}
| no | +| [create\_default\_roles](#input\_create\_default\_roles) | Whether the default roles should be created | `bool` | `false` | no | +| [default\_ddl\_collation](#input\_default\_ddl\_collation) | Specifies a default collation specification for all schemas and tables added to the database. | `string` | `null` | no | | [delimiter](#input\_delimiter) | Delimiter to be used between ID elements.
Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no | | [descriptor\_formats](#input\_descriptor\_formats) | Describe additional descriptors to be output in the `descriptors` output map.
Map of maps. Keys are names of descriptors. Values are maps of the form
`{
format = string
labels = list(string)
}`
(Type is `any` so the map values can later be enhanced to provide additional options.)
`format` is a Terraform format string to be passed to the `format()` function.
`labels` is a list of labels, in order, to pass to `format()` function.
Label values will be normalized before being passed to `format()` so they will be
identical to how they appear in `id`.
Default is `{}` (`descriptors` output will be empty). | `any` | `{}` | no | -| [descriptor\_name](#input\_descriptor\_name) | Name of the descriptor used to form a resource name | `string` | `"resource-type"` | no | +| [descriptor\_name](#input\_descriptor\_name) | Name of the descriptor used to form a resource name | `string` | `"snowflake-database"` | no | +| [enable\_console\_output](#input\_enable\_console\_output) | If true, enables stdout/stderr fast path logging for anonymous stored procedures | `bool` | `null` | no | | [enabled](#input\_enabled) | Set to false to prevent the module from creating any resources | `bool` | `null` | no | | [environment](#input\_environment) | ID element. Usually used for region e.g. 'uw2', 'us-west-2', OR role 'prod', 'staging', 'dev', 'UAT' | `string` | `null` | no | -| [example\_var](#input\_example\_var) | Example variable passed into the module | `string` | n/a | yes | +| [external\_volume](#input\_external\_volume) | The database parameter that specifies the default external volume to use for Iceberg tables | `string` | `null` | no | +| [from\_share](#input\_from\_share) | A fully qualified path to a share from which the database will be created. A fully qualified path follows the format of `..` | `string` | n/a | yes | | [id\_length\_limit](#input\_id\_length\_limit) | Limit `id` to this many characters (minimum 6).
Set to `0` for unlimited length.
Set to `null` for keep the existing setting, which defaults to `0`.
Does not affect `id_full`. | `number` | `null` | no | | [label\_key\_case](#input\_label\_key\_case) | Controls the letter case of the `tags` keys (label names) for tags generated by this module.
Does not affect keys of tags passed in via the `tags` input.
Possible values: `lower`, `title`, `upper`.
Default value: `title`. | `string` | `null` | no | | [label\_order](#input\_label\_order) | The order in which the labels (ID elements) appear in the `id`.
Defaults to ["namespace", "environment", "stage", "name", "attributes"].
You can omit any of the 6 labels ("tenant" is the 6th), but at least one must be present. | `list(string)` | `null` | no | | [label\_value\_case](#input\_label\_value\_case) | Controls the letter case of ID elements (labels) as included in `id`,
set as tag values, and output by this module individually.
Does not affect values of tags passed in via the `tags` input.
Possible values: `lower`, `title`, `upper` and `none` (no transformation).
Set this to `title` and set `delimiter` to `""` to yield Pascal Case IDs.
Default value: `lower`. | `string` | `null` | no | | [labels\_as\_tags](#input\_labels\_as\_tags) | Set of labels (ID elements) to include as tags in the `tags` output.
Default is to include all labels.
Tags with empty values will not be included in the `tags` output.
Set to `[]` to suppress all generated tags.
**Notes:**
The value of the `name` tag, if included, will be the `id`, not the `name`.
Unlike other `null-label` inputs, the initial setting of `labels_as_tags` cannot be
changed in later chained modules. Attempts to change it will be silently ignored. | `set(string)` | 
[
  "default"
]
| no | +| [log\_level](#input\_log\_level) | Specifies the severity level of messages that should be ingested and made available in the active event table. Valid options are: [TRACE DEBUG INFO WARN ERROR FATAL OFF] | `string` | `null` | no | | [name](#input\_name) | ID element. Usually the component or solution name, e.g. 'app' or 'jenkins'.
This is the only ID element not also included as a `tag`.
The "name" tag is set to the full `id` string. There is no tag with the value of the `name` input. | `string` | `null` | no | | [namespace](#input\_namespace) | ID element. Usually an abbreviation of your organization name, e.g. 'eg' or 'cp', to help ensure generated IDs are globally unique | `string` | `null` | no | +| [quoted\_identifiers\_ignore\_case](#input\_quoted\_identifiers\_ignore\_case) | If true, the case of quoted identifiers is ignored | `bool` | `null` | no | | [regex\_replace\_chars](#input\_regex\_replace\_chars) | Terraform regular expression (regex) string.
Characters matching the regex will be removed from the ID elements.
If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. | `string` | `null` | no | +| [replace\_invalid\_characters](#input\_replace\_invalid\_characters) | If true, invalid characters are replaced with the replacement character | `bool` | `null` | no | +| [roles](#input\_roles) | Account roles created on the Shared Database level | 
map(object({
    enabled              = optional(bool, true)
    descriptor_name      = optional(string, "snowflake-role")
    comment              = optional(string)
    role_ownership_grant = optional(string)
    granted_roles        = optional(list(string))
    granted_to_roles     = optional(list(string))
    granted_to_users     = optional(list(string))
    database_grants = optional(object({
      privileges = optional(list(string))
    }))
  }))
| `{}` | no | | [stage](#input\_stage) | ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no | -| [sub\_resource](#input\_sub\_resource) | Some other resource that is part of stack/module | 
object({
    descriptor_name = optional(string, "sub-resource-type")
    example_var     = string
  })
| n/a | yes | +| [storage\_serialization\_policy](#input\_storage\_serialization\_policy) | The storage serialization policy for Iceberg tables that use Snowflake as the catalog. Valid options are: [COMPATIBLE OPTIMIZED] | `string` | `null` | no | +| [suspend\_task\_after\_num\_failures](#input\_suspend\_task\_after\_num\_failures) | How many times a task must fail in a row before it is automatically suspended. 0 disables auto-suspending | `number` | `null` | no | | [tags](#input\_tags) | Additional tags (e.g. `{'BusinessUnit': 'XYZ'}`).
Neither the tag keys nor the tag values will be modified by this module. | `map(string)` | `{}` | no | +| [task\_auto\_retry\_attempts](#input\_task\_auto\_retry\_attempts) | Maximum automatic retries allowed for a user task | `number` | `null` | no | | [tenant](#input\_tenant) | ID element \_(Rarely used, not included by default)\_. A customer identifier, indicating who this instance of a resource is for | `string` | `null` | no | +| [trace\_level](#input\_trace\_level) | Controls how trace events are ingested into the event table. Valid options are: [ALWAYS ON\_EVENT OFF] | `string` | `null` | no | +| [user\_task\_managed\_initial\_warehouse\_size](#input\_user\_task\_managed\_initial\_warehouse\_size) | The initial size of warehouse to use for managed warehouses in the absence of history | `string` | `null` | no | +| [user\_task\_minimum\_trigger\_interval\_in\_seconds](#input\_user\_task\_minimum\_trigger\_interval\_in\_seconds) | Minimum amount of time between Triggered Task executions in seconds | `number` | `null` | no | +| [user\_task\_timeout\_ms](#input\_user\_task\_timeout\_ms) | User task execution timeout in milliseconds | `number` | `null` | no | ## Modules | Name | Source | Version | |------|--------|---------| -| [subresource\_label](#module\_subresource\_label) | cloudposse/label/null | 0.25.0 | +| [database\_label](#module\_database\_label) | cloudposse/label/null | 0.25.0 | +| [roles\_deep\_merge](#module\_roles\_deep\_merge) | Invicton-Labs/deepmerge/null | 0.1.5 | +| [snowflake\_custom\_role](#module\_snowflake\_custom\_role) | getindata/role/snowflake | 2.1.0 | +| [snowflake\_default\_role](#module\_snowflake\_default\_role) | getindata/role/snowflake | 2.1.0 | | [this](#module\_this) | cloudposse/label/null | 0.25.0 | ## Outputs | Name | Description | |------|-------------| -| [example\_output](#output\_example\_output) | Example output of the module | +| [catalog](#output\_catalog) | The database parameter that specifies the default catalog to use for Iceberg tables | +| [default\_ddl\_collation](#output\_default\_ddl\_collation) | Specifies a default collation specification for all schemas and tables added to the database. | +| [enable\_console\_output](#output\_enable\_console\_output) | If true, enables stdout/stderr fast path logging for anonymous stored procedures | +| [external\_volume](#output\_external\_volume) | The database parameter that specifies the default external volume to use for Iceberg tables | +| [log\_level](#output\_log\_level) | Specifies the severity level of messages that should be ingested and made available in the active event table. Valid options are: [TRACE DEBUG INFO WARN ERROR FATAL OFF] | +| [name](#output\_name) | Name of the database | +| [quoted\_identifiers\_ignore\_case](#output\_quoted\_identifiers\_ignore\_case) | If true, the case of quoted identifiers is ignored | +| [roles](#output\_roles) | Snowflake Roles | +| [storage\_serialization\_policy](#output\_storage\_serialization\_policy) | The storage serialization policy for Iceberg tables that use Snowflake as the catalog. Valid options are: [COMPATIBLE OPTIMIZED] | +| [suspend\_task\_after\_num\_failures](#output\_suspend\_task\_after\_num\_failures) | How many times a task must fail in a row before it is automatically suspended. 0 disables auto-suspending | +| [task\_auto\_retry\_attempts](#output\_task\_auto\_retry\_attempts) | Maximum automatic retries allowed for a user task | +| [trace\_level](#output\_trace\_level) | Controls how trace events are ingested into the event table. Valid options are: [ALWAYS ON\_EVENT OFF] | +| [user\_task\_managed\_initial\_warehouse\_size](#output\_user\_task\_managed\_initial\_warehouse\_size) | The initial size of warehouse to use for managed warehouses in the absence of history | +| [user\_task\_minimum\_trigger\_interval\_in\_seconds](#output\_user\_task\_minimum\_trigger\_interval\_in\_seconds) | Minimum amount of time between Triggered Task executions in seconds | +| [user\_task\_timeout\_ms](#output\_user\_task\_timeout\_ms) | User task execution timeout in milliseconds | ## Providers | Name | Version | |------|---------| -| [null](#provider\_null) | 3.1.1 | +| [snowflake](#provider\_snowflake) | >= 0.95 | ## Requirements | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [null](#requirement\_null) | 3.1.1 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [snowflake](#requirement\_snowflake) | >= 0.95 | ## Resources | Name | Type | |------|------| -| [null_resource.output_input](https://registry.terraform.io/providers/hashicorp/null/3.1.1/docs/resources/resource) | resource | -| [null_resource.subresource](https://registry.terraform.io/providers/hashicorp/null/3.1.1/docs/resources/resource) | resource | +| [snowflake_shared_database.this](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/shared_database) | resource | ## CONTRIBUTING @@ -131,7 +150,7 @@ Apache 2 Licensed. See [LICENSE](LICENSE) for full details. - + Made with [contrib.rocks](https://contrib.rocks). diff --git a/examples/complete/.env.dist b/examples/complete/.env.dist index e69de29..5f4ea91 100644 --- a/examples/complete/.env.dist +++ b/examples/complete/.env.dist @@ -0,0 +1,10 @@ +SNOWFLAKE_USER= +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_ROLE="ACCOUNTADMIN" + +# Login using RSA key +# SNOWFLAKE_PRIVATE_KEY_PATH= +# SNOWFLAKE_AUTHENTICATOR="JWT" + +# Login using password +# SNOWFLAKE_PASSWORD= diff --git a/examples/complete/README.md b/examples/complete/README.md index c7da2ed..f7ba08a 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -1,24 +1,71 @@ # Complete Example ```terraform -module "terraform_module_template" { - source = "../../" - context = module.this.context - - example_var = "This is a example value." - sub_resource = { - example_var = "This is a example value of sub resource." +module "snowflake_shared_database" { + source = "../.." + + name = "SHARED_DATABASE" + context = module.this.context + from_share = var.from_share + + descriptor_name = "snowflake-database" + comment = "Sample shared Database" + replace_invalid_characters = true + default_ddl_collation = "UTF8" + log_level = "INFO" + trace_level = "ON_EVENT" + + suspend_task_after_num_failures = 1 + task_auto_retry_attempts = 1 + user_task_managed_initial_warehouse_size = "X-Small" + user_task_minimum_trigger_interval_in_seconds = 300 + user_task_timeout_ms = 200 + quoted_identifiers_ignore_case = true + enable_console_output = true + + create_default_roles = true + + roles = { + readonly = { + comment = "Read-only role" + granted_roles = [resource.snowflake_account_role.this.name] + granted_to_users = [resource.snowflake_user.this.name] + } } } + ``` ## Usage + +1. Configure private sharing on producer Snowflake Account (remember that both producer and consumer accounts have to be deployed on the same Cloud Provider for. ex. AWS and in the same region for. ex. `eu-west-1`) + +This step can be done manually using Snowsight UI or by running [create_share.sql](./create-share.sql) script on producer Snowflake account. + +When using the script, please remember to properly define consumer account details in the last line: + +```sql +ALTER SHARE sample_share ADD ACCOUNT=""; ``` + +2. With share configured in step 1., run terraform on consumer account using below commands + +```shell terraform init terraform plan -var-file fixtures.tfvars -out tfplan terraform apply tfplan ``` +**Please remember to pass share details (from step 1.) to `from_share` variable.** + +```shell +$ terraform plan +var.from_share + A fully qualified path to a share from which the database will be created. A fully qualified path follows the format of `..` + + Enter a value: +``` + @@ -35,6 +82,7 @@ terraform apply tfplan | [descriptor\_formats](#input\_descriptor\_formats) | Describe additional descriptors to be output in the `descriptors` output map.
Map of maps. Keys are names of descriptors. Values are maps of the form
`{
format = string
labels = list(string)
}`
(Type is `any` so the map values can later be enhanced to provide additional options.)
`format` is a Terraform format string to be passed to the `format()` function.
`labels` is a list of labels, in order, to pass to `format()` function.
Label values will be normalized before being passed to `format()` so they will be
identical to how they appear in `id`.
Default is `{}` (`descriptors` output will be empty). | `any` | `{}` | no | | [enabled](#input\_enabled) | Set to false to prevent the module from creating any resources | `bool` | `null` | no | | [environment](#input\_environment) | ID element. Usually used for region e.g. 'uw2', 'us-west-2', OR role 'prod', 'staging', 'dev', 'UAT' | `string` | `null` | no | +| [from\_share](#input\_from\_share) | A fully qualified path to a share from which the database will be created. A fully qualified path follows the format of `..` | `string` | n/a | yes | | [id\_length\_limit](#input\_id\_length\_limit) | Limit `id` to this many characters (minimum 6).
Set to `0` for unlimited length.
Set to `null` for keep the existing setting, which defaults to `0`.
Does not affect `id_full`. | `number` | `null` | no | | [label\_key\_case](#input\_label\_key\_case) | Controls the letter case of the `tags` keys (label names) for tags generated by this module.
Does not affect keys of tags passed in via the `tags` input.
Possible values: `lower`, `title`, `upper`.
Default value: `title`. | `string` | `null` | no | | [label\_order](#input\_label\_order) | The order in which the labels (ID elements) appear in the `id`.
Defaults to ["namespace", "environment", "stage", "name", "attributes"].
You can omit any of the 6 labels ("tenant" is the 6th), but at least one must be present. | `list(string)` | `null` | no | @@ -51,7 +99,7 @@ terraform apply tfplan | Name | Source | Version | |------|--------|---------| -| [terraform\_module\_template](#module\_terraform\_module\_template) | ../../ | n/a | +| [snowflake\_shared\_database](#module\_snowflake\_shared\_database) | ../.. | n/a | | [this](#module\_this) | cloudposse/label/null | 0.25.0 | ## Outputs @@ -62,16 +110,21 @@ terraform apply tfplan ## Providers -No providers. +| Name | Version | +|------|---------| +| [snowflake](#provider\_snowflake) | >= 0.95 | ## Requirements | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [null](#requirement\_null) | 3.1.1 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [snowflake](#requirement\_snowflake) | >= 0.95 | ## Resources -No resources. +| Name | Type | +|------|------| +| [snowflake_account_role.this](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/account_role) | resource | +| [snowflake_user.this](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/user) | resource | diff --git a/examples/complete/create-share.sql b/examples/complete/create-share.sql new file mode 100644 index 0000000..974cbd2 --- /dev/null +++ b/examples/complete/create-share.sql @@ -0,0 +1,23 @@ +CREATE DATABASE sample_shared_db; + +CREATE SCHEMA sample_shared_db.shared_schema; + +CREATE TABLE sample_shared_db.shared_schema.sample_table ( + ID NUMBER, + VALUE TEXT +); + +INSERT INTO sample_shared_db.shared_schema.sample_table + VALUES + (1, 'TEST VALUE'), + (2, 'ANOTHER TEST VALUE'); + +CREATE SHARE sample_share COMMENT = 'Test Share'; + +GRANT USAGE ON DATABASE sample_shared_db TO SHARE sample_share; + +GRANT USAGE ON SCHEMA sample_shared_db.shared_schema TO SHARE sample_share; + +GRANT SELECT ON ALL TABLES IN SCHEMA sample_shared_db.shared_schema TO SHARE sample_share; + +ALTER SHARE sample_share ADD ACCOUNT=""; diff --git a/examples/complete/fixtures.tfvars b/examples/complete/fixtures.tfvars index 11358b0..f575f57 100644 --- a/examples/complete/fixtures.tfvars +++ b/examples/complete/fixtures.tfvars @@ -1,7 +1,14 @@ -descriptor_formats = { - -} +namespace = "gid" +#stage = "example" +#environment = "dev" -tags = { - Terraform = "True" +descriptor_formats = { + snowflake-role = { + labels = ["attributes", "name"] + format = "%v_%v" + } + snowflake-database = { + labels = ["environment", "stage", "name", "attributes"] + format = "%v_%v_%v_%v" + } } diff --git a/examples/complete/main.tf b/examples/complete/main.tf index bd89c41..f390c5f 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -1,9 +1,40 @@ -module "terraform_module_template" { - source = "../../" - context = module.this.context +resource "snowflake_user" "this" { + name = "SAMPLE_USER" +} + +resource "snowflake_account_role" "this" { + name = "SAMPLE_ROLE" +} + +module "snowflake_shared_database" { + source = "../.." + + name = "SHARED_DATABASE" + context = module.this.context + from_share = var.from_share + + descriptor_name = "snowflake-database" + comment = "Sample shared Database" + replace_invalid_characters = true + default_ddl_collation = "UTF8" + log_level = "INFO" + trace_level = "ON_EVENT" + + suspend_task_after_num_failures = 1 + task_auto_retry_attempts = 1 + user_task_managed_initial_warehouse_size = "X-Small" + user_task_minimum_trigger_interval_in_seconds = 300 + user_task_timeout_ms = 200 + quoted_identifiers_ignore_case = true + enable_console_output = true + + create_default_roles = true - example_var = "This is a example value." - sub_resource = { - example_var = "This is a example value of sub resource." + roles = { + readonly = { + comment = "Read-only role" + granted_roles = [resource.snowflake_account_role.this.name] + granted_to_users = [resource.snowflake_user.this.name] + } } } diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index c2007a2..cca6098 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -1,4 +1,4 @@ output "example_output" { description = "Example output of the module" - value = module.terraform_module_template + value = module.snowflake_shared_database } diff --git a/examples/complete/providers.tf b/examples/complete/providers.tf index c793099..d343f0d 100644 --- a/examples/complete/providers.tf +++ b/examples/complete/providers.tf @@ -1,3 +1 @@ -provider "null" { - # Configuration options -} +provider "snowflake" {} diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf new file mode 100644 index 0000000..99be937 --- /dev/null +++ b/examples/complete/variables.tf @@ -0,0 +1,4 @@ +variable "from_share" { + description = "A fully qualified path to a share from which the database will be created. A fully qualified path follows the format of `..`" + type = string +} diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index daa4ce8..659976c 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -1,10 +1,9 @@ terraform { - required_version = ">= 1.3.0" - + required_version = ">= 1.3" required_providers { - null = { - source = "hashicorp/null" - version = "3.1.1" + snowflake = { + source = "Snowflake-Labs/snowflake" + version = ">= 0.95" } } } diff --git a/examples/simple/.env.dist b/examples/simple/.env.dist index e69de29..5f4ea91 100644 --- a/examples/simple/.env.dist +++ b/examples/simple/.env.dist @@ -0,0 +1,10 @@ +SNOWFLAKE_USER= +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_ROLE="ACCOUNTADMIN" + +# Login using RSA key +# SNOWFLAKE_PRIVATE_KEY_PATH= +# SNOWFLAKE_AUTHENTICATOR="JWT" + +# Login using password +# SNOWFLAKE_PASSWORD= diff --git a/examples/simple/Makefile b/examples/simple/Makefile index 9d9205a..2e151d6 100644 --- a/examples/simple/Makefile +++ b/examples/simple/Makefile @@ -8,4 +8,4 @@ apply: terraform apply tfplan destroy: - terraform destroy + terraform destroy diff --git a/examples/simple/README.md b/examples/simple/README.md index 2b7f1b9..9842af3 100644 --- a/examples/simple/README.md +++ b/examples/simple/README.md @@ -1,23 +1,45 @@ # Simple Example ```terraform -module "terraform_module_template" { - source = "../../" +module "snowflake_shared_database" { + source = "../.." - example_var = "This is a example value." - sub_resource = { - example_var = "This is a example value of sub resource." - } + name = "SHARED_DATABASE" + from_share = var.from_share } + ``` ## Usage + +1. Configure private sharing on producer Snowflake Account (remember that both producer and consumer accounts have to be deployed on the same Cloud Provider for. ex. AWS and in the same region for. ex. `eu-west-1`) + +This step can be done manually using Snowsight UI or by running [create_share.sql](./create-share.sql) script on producer Snowflake account. + +When using the script, please remember to properly define consumer account details in the last line: + +```sql +ALTER SHARE sample_share ADD ACCOUNT=""; ``` + +2. With share configured in step 1., run terraform on consumer account using below commands + +```shell terraform init terraform plan -out tfplan terraform apply tfplan ``` +**Please remember to pass share details (from step 1.) to `from_share` variable.** + +```shell +$ terraform plan +var.from_share + A fully qualified path to a share from which the database will be created. A fully qualified path follows the format of `..` + + Enter a value: +``` + @@ -25,13 +47,15 @@ terraform apply tfplan ## Inputs -No inputs. +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [from\_share](#input\_from\_share) | A fully qualified path to a share from which the database will be created. A fully qualified path follows the format of `..` | `string` | n/a | yes | ## Modules | Name | Source | Version | |------|--------|---------| -| [terraform\_module\_template](#module\_terraform\_module\_template) | ../../ | n/a | +| [snowflake\_shared\_database](#module\_snowflake\_shared\_database) | ../.. | n/a | ## Outputs @@ -47,8 +71,8 @@ No providers. | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [null](#requirement\_null) | 3.1.1 | +| [terraform](#requirement\_terraform) | >= 1.3 | +| [snowflake](#requirement\_snowflake) | >= 0.95 | ## Resources diff --git a/examples/simple/create-share.sql b/examples/simple/create-share.sql new file mode 100644 index 0000000..974cbd2 --- /dev/null +++ b/examples/simple/create-share.sql @@ -0,0 +1,23 @@ +CREATE DATABASE sample_shared_db; + +CREATE SCHEMA sample_shared_db.shared_schema; + +CREATE TABLE sample_shared_db.shared_schema.sample_table ( + ID NUMBER, + VALUE TEXT +); + +INSERT INTO sample_shared_db.shared_schema.sample_table + VALUES + (1, 'TEST VALUE'), + (2, 'ANOTHER TEST VALUE'); + +CREATE SHARE sample_share COMMENT = 'Test Share'; + +GRANT USAGE ON DATABASE sample_shared_db TO SHARE sample_share; + +GRANT USAGE ON SCHEMA sample_shared_db.shared_schema TO SHARE sample_share; + +GRANT SELECT ON ALL TABLES IN SCHEMA sample_shared_db.shared_schema TO SHARE sample_share; + +ALTER SHARE sample_share ADD ACCOUNT=""; diff --git a/examples/simple/main.tf b/examples/simple/main.tf index d9a969f..92ddf14 100644 --- a/examples/simple/main.tf +++ b/examples/simple/main.tf @@ -1,9 +1,6 @@ -module "terraform_module_template" { - source = "../../" +module "snowflake_shared_database" { + source = "../.." - example_var = "This is a example value." - - sub_resource = { - example_var = "This is a example value of sub resource." - } + name = "SHARED_DATABASE" + from_share = var.from_share } diff --git a/examples/simple/outputs.tf b/examples/simple/outputs.tf index c2007a2..cca6098 100644 --- a/examples/simple/outputs.tf +++ b/examples/simple/outputs.tf @@ -1,4 +1,4 @@ output "example_output" { description = "Example output of the module" - value = module.terraform_module_template + value = module.snowflake_shared_database } diff --git a/examples/simple/providers.tf b/examples/simple/providers.tf index c793099..d343f0d 100644 --- a/examples/simple/providers.tf +++ b/examples/simple/providers.tf @@ -1,3 +1 @@ -provider "null" { - # Configuration options -} +provider "snowflake" {} diff --git a/examples/simple/variables.tf b/examples/simple/variables.tf new file mode 100644 index 0000000..99be937 --- /dev/null +++ b/examples/simple/variables.tf @@ -0,0 +1,4 @@ +variable "from_share" { + description = "A fully qualified path to a share from which the database will be created. A fully qualified path follows the format of `..`" + type = string +} diff --git a/examples/simple/versions.tf b/examples/simple/versions.tf index daa4ce8..659976c 100644 --- a/examples/simple/versions.tf +++ b/examples/simple/versions.tf @@ -1,10 +1,9 @@ terraform { - required_version = ">= 1.3.0" - + required_version = ">= 1.3" required_providers { - null = { - source = "hashicorp/null" - version = "3.1.1" + snowflake = { + source = "Snowflake-Labs/snowflake" + version = ">= 0.95" } } } diff --git a/locals.tf b/locals.tf index 5778a0d..89dfe89 100644 --- a/locals.tf +++ b/locals.tf @@ -1,15 +1,66 @@ locals { # Get a name from the descriptor. If not available, use default naming convention. # Trim and replace function are used to avoid bare delimiters on both ends of the name and situation of adjacent delimiters. - # - # todo: Build a wrapper module around context module with name from descriptor feature - name_from_descriptor = module.this.enabled ? trim(replace( - lookup(module.this.descriptors, var.descriptor_name, module.this.id), "/${module.this.delimiter}${module.this.delimiter}+/", module.this.delimiter - ), module.this.delimiter) : null + name_from_descriptor = module.database_label.enabled ? trim(replace( + lookup(module.database_label.descriptors, var.descriptor_name, module.database_label.id), "/${module.database_label.delimiter}${module.database_label.delimiter}+/", module.database_label.delimiter + ), module.database_label.delimiter) : null - subresource_name_from_descriptor = module.subresource_label.enabled ? trim(replace( - lookup(module.subresource_label.descriptors, var.sub_resource.descriptor_name, module.subresource_label.id), "/${module.subresource_label.delimiter}${module.subresource_label.delimiter}+/", module.subresource_label.delimiter - ), module.subresource_label.delimiter) : null + create_default_roles = module.this.enabled && var.create_default_roles - enabled = module.this.enabled + #This needs to be the same as an object in roles variable + role_template = { + enabled = true + descriptor_name = "snowflake-role" + comment = null + role_ownership_grant = "SYSADMIN" + granted_roles = [] + granted_to_roles = [] + granted_to_users = [] + database_grants = {} + } + + default_roles_definition = { + readonly = { + database_grants = { + privileges = ["IMPORTED PRIVILEGES"] + } + } + } + + provided_roles = { for role_name, role in var.roles : role_name => { + for k, v in role : k => v + if v != null + } } + + roles_definition = { + for role_name, role in module.roles_deep_merge.merged : role_name => merge( + local.role_template, + role + ) + } + + default_roles = { + for role_name, role in local.roles_definition : role_name => role + if contains(keys(local.default_roles_definition), role_name) + } + + custom_roles = { + for role_name, role in local.roles_definition : role_name => role + if !contains(keys(local.default_roles_definition), role_name) + } + + roles = { + for role_name, role in merge( + var.create_default_roles ? module.snowflake_default_role : {}, + module.snowflake_custom_role + ) : role_name => role + if role_name != null + } +} + +module "roles_deep_merge" { + source = "Invicton-Labs/deepmerge/null" + version = "0.1.5" + + maps = [local.default_roles_definition, local.provided_roles] } diff --git a/main.tf b/main.tf index 740356a..a4200b2 100644 --- a/main.tf +++ b/main.tf @@ -1,36 +1,87 @@ -# Example resource that outputs the input value and -# echoes it's base64 encoded version locally +module "database_label" { + source = "cloudposse/label/null" + version = "0.25.0" + context = module.this.context -resource "null_resource" "output_input" { - count = local.enabled ? 1 : 0 + delimiter = coalesce(module.this.context.delimiter, "_") + regex_replace_chars = coalesce(module.this.context.regex_replace_chars, "/[^_a-zA-Z0-9]/") + label_value_case = coalesce(module.this.context.label_value_case, "upper") +} - triggers = { - name = local.name_from_descriptor - input = var.example_var - } +resource "snowflake_shared_database" "this" { + count = module.this.enabled ? 1 : 0 - provisioner "local-exec" { - command = "echo ${var.example_var} | base64" - } + name = local.name_from_descriptor + from_share = var.from_share + comment = var.comment + + catalog = var.catalog + default_ddl_collation = var.default_ddl_collation + enable_console_output = var.enable_console_output + external_volume = var.external_volume + log_level = var.log_level + quoted_identifiers_ignore_case = var.quoted_identifiers_ignore_case + replace_invalid_characters = var.replace_invalid_characters + storage_serialization_policy = var.storage_serialization_policy + suspend_task_after_num_failures = var.suspend_task_after_num_failures + task_auto_retry_attempts = var.task_auto_retry_attempts + trace_level = var.trace_level + user_task_managed_initial_warehouse_size = var.user_task_managed_initial_warehouse_size + user_task_minimum_trigger_interval_in_seconds = var.user_task_minimum_trigger_interval_in_seconds + user_task_timeout_ms = var.user_task_timeout_ms } -module "subresource_label" { - source = "cloudposse/label/null" - version = "0.25.0" +module "snowflake_default_role" { + for_each = local.default_roles + + source = "getindata/role/snowflake" + version = "2.1.0" context = module.this.context - attributes = ["sub"] -} + name = each.key + comment = lookup(each.value, "comment", null) + enabled = local.create_default_roles && lookup(each.value, "enabled", true) + attributes = [one(snowflake_shared_database.this[*].name)] + descriptor_name = lookup(each.value, "descriptor_name", "snowflake-role") -resource "null_resource" "subresource" { - count = local.enabled ? 1 : 0 + granted_to_roles = lookup(each.value, "granted_to_roles", []) + granted_roles = lookup(each.value, "granted_roles", []) - triggers = { - name = local.subresource_name_from_descriptor - input = var.sub_resource.example_var + account_objects_grants = { + DATABASE = [{ + privileges = each.value.database_grants.privileges + object_name = one(snowflake_shared_database.this[*].name) + }] } - provisioner "local-exec" { - command = "echo ${var.sub_resource.example_var} | base64" + depends_on = [ + snowflake_shared_database.this + ] +} + +module "snowflake_custom_role" { + for_each = local.custom_roles + + source = "getindata/role/snowflake" + version = "2.1.0" + context = module.this.context + + name = each.key + comment = lookup(each.value, "comment", null) + enabled = module.this.enabled && lookup(each.value, "enabled", true) + attributes = [one(snowflake_shared_database.this[*].name)] + descriptor_name = lookup(each.value, "descriptor_name", "snowflake-role") + + granted_to_roles = lookup(each.value, "granted_to_roles", []) + granted_roles = lookup(each.value, "granted_roles", []) + account_objects_grants = { + DATABASE = [{ + privileges = each.value.database_grants.privileges + object_name = one(snowflake_shared_database.this[*].name) + }] } + + depends_on = [ + snowflake_shared_database.this + ] } diff --git a/outputs.tf b/outputs.tf index b1bbafe..e8da423 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,6 +1,74 @@ -# Example output from the module +output "name" { + description = "Name of the database" + value = one(snowflake_shared_database.this[*].name) +} + +output "external_volume" { + description = "The database parameter that specifies the default external volume to use for Iceberg tables" + value = one(snowflake_shared_database.this[*].external_volume) +} + +output "catalog" { + description = "The database parameter that specifies the default catalog to use for Iceberg tables" + value = one(snowflake_shared_database.this[*].catalog) +} + +output "default_ddl_collation" { + description = "Specifies a default collation specification for all schemas and tables added to the database." + value = one(snowflake_shared_database.this[*].default_ddl_collation) +} + +output "storage_serialization_policy" { + description = "The storage serialization policy for Iceberg tables that use Snowflake as the catalog. Valid options are: [COMPATIBLE OPTIMIZED]" + value = one(snowflake_shared_database.this[*].storage_serialization_policy) +} + +output "log_level" { + description = "Specifies the severity level of messages that should be ingested and made available in the active event table. Valid options are: [TRACE DEBUG INFO WARN ERROR FATAL OFF]" + value = one(snowflake_shared_database.this[*].log_level) +} + +output "trace_level" { + description = "Controls how trace events are ingested into the event table. Valid options are: [ALWAYS ON_EVENT OFF]" + value = one(snowflake_shared_database.this[*].trace_level) +} + +output "suspend_task_after_num_failures" { + description = "How many times a task must fail in a row before it is automatically suspended. 0 disables auto-suspending" + value = one(snowflake_shared_database.this[*].suspend_task_after_num_failures) +} + +output "task_auto_retry_attempts" { + description = "Maximum automatic retries allowed for a user task" + value = one(snowflake_shared_database.this[*].task_auto_retry_attempts) +} + +output "user_task_managed_initial_warehouse_size" { + description = "The initial size of warehouse to use for managed warehouses in the absence of history" + value = one(snowflake_shared_database.this[*].user_task_managed_initial_warehouse_size) +} + +output "user_task_minimum_trigger_interval_in_seconds" { + description = "Minimum amount of time between Triggered Task executions in seconds" + value = one(snowflake_shared_database.this[*].user_task_minimum_trigger_interval_in_seconds) +} + +output "user_task_timeout_ms" { + description = "User task execution timeout in milliseconds" + value = one(snowflake_shared_database.this[*].user_task_timeout_ms) +} + +output "quoted_identifiers_ignore_case" { + description = "If true, the case of quoted identifiers is ignored" + value = one(snowflake_shared_database.this[*].quoted_identifiers_ignore_case) +} + +output "enable_console_output" { + description = "If true, enables stdout/stderr fast path logging for anonymous stored procedures" + value = one(snowflake_shared_database.this[*].enable_console_output) +} -output "example_output" { - description = "Example output of the module" - value = one(null_resource.output_input[*].id) +output "roles" { + description = "Snowflake Roles" + value = local.roles } diff --git a/variables.tf b/variables.tf index 291a160..a318aa2 100644 --- a/variables.tf +++ b/variables.tf @@ -1,18 +1,123 @@ -variable "example_var" { - description = "Example variable passed into the module" +variable "from_share" { + description = "A fully qualified path to a share from which the database will be created. A fully qualified path follows the format of `..`" type = string } variable "descriptor_name" { description = "Name of the descriptor used to form a resource name" type = string - default = "resource-type" + default = "snowflake-database" } -variable "sub_resource" { - description = "Some other resource that is part of stack/module" - type = object({ - descriptor_name = optional(string, "sub-resource-type") - example_var = string - }) +variable "comment" { + description = "Specifies a comment for the database" + type = string + default = null +} + +variable "external_volume" { + description = "The database parameter that specifies the default external volume to use for Iceberg tables" + type = string + default = null +} + +variable "catalog" { + description = "The database parameter that specifies the default catalog to use for Iceberg tables" + type = string + default = null +} + +variable "replace_invalid_characters" { + description = "If true, invalid characters are replaced with the replacement character" + type = bool + default = null +} + +variable "default_ddl_collation" { + description = "Specifies a default collation specification for all schemas and tables added to the database." + type = string + default = null +} + +variable "storage_serialization_policy" { + description = "The storage serialization policy for Iceberg tables that use Snowflake as the catalog. Valid options are: [COMPATIBLE OPTIMIZED]" + type = string + default = null +} + +variable "log_level" { + description = "Specifies the severity level of messages that should be ingested and made available in the active event table. Valid options are: [TRACE DEBUG INFO WARN ERROR FATAL OFF]" + type = string + default = null +} + +variable "trace_level" { + description = "Controls how trace events are ingested into the event table. Valid options are: [ALWAYS ON_EVENT OFF]" + type = string + default = null +} + +variable "suspend_task_after_num_failures" { + description = "How many times a task must fail in a row before it is automatically suspended. 0 disables auto-suspending" + type = number + default = null +} + +variable "task_auto_retry_attempts" { + description = "Maximum automatic retries allowed for a user task" + type = number + default = null +} + +variable "user_task_managed_initial_warehouse_size" { + description = "The initial size of warehouse to use for managed warehouses in the absence of history" + type = string + default = null +} + +variable "user_task_minimum_trigger_interval_in_seconds" { + description = "Minimum amount of time between Triggered Task executions in seconds" + type = number + default = null +} + +variable "user_task_timeout_ms" { + description = "User task execution timeout in milliseconds" + type = number + default = null +} + +variable "quoted_identifiers_ignore_case" { + description = "If true, the case of quoted identifiers is ignored" + type = bool + default = null +} + +variable "enable_console_output" { + description = "If true, enables stdout/stderr fast path logging for anonymous stored procedures" + type = bool + default = null +} + +variable "create_default_roles" { + description = "Whether the default roles should be created" + type = bool + default = false +} + +variable "roles" { + description = "Account roles created on the Shared Database level" + type = map(object({ + enabled = optional(bool, true) + descriptor_name = optional(string, "snowflake-role") + comment = optional(string) + role_ownership_grant = optional(string) + granted_roles = optional(list(string)) + granted_to_roles = optional(list(string)) + granted_to_users = optional(list(string)) + database_grants = optional(object({ + privileges = optional(list(string)) + })) + })) + default = {} } diff --git a/versions.tf b/versions.tf index 4114d58..659976c 100644 --- a/versions.tf +++ b/versions.tf @@ -1,12 +1,9 @@ -# Example configuration of terraform providers - terraform { - required_version = ">= 1.3.0" - + required_version = ">= 1.3" required_providers { - null = { - source = "hashicorp/null" - version = "3.1.1" + snowflake = { + source = "Snowflake-Labs/snowflake" + version = ">= 0.95" } } }