From 6bd170968e429b58f85b9197992cd07f4e0c3d8c Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sat, 3 Dec 2022 21:55:33 +0200 Subject: [PATCH 01/10] add args --- internal/service/glue/crawler.go | 348 ++++++++++++++++++------------- 1 file changed, 199 insertions(+), 149 deletions(-) diff --git a/internal/service/glue/crawler.go b/internal/service/glue/crawler.go index 50ba8d091516..540100f70005 100644 --- a/internal/service/glue/crawler.go +++ b/internal/service/glue/crawler.go @@ -21,6 +21,10 @@ import ( "github.com/hashicorp/terraform-provider-aws/internal/verify" ) +func targets() []string { + return []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target", "delta_target"} +} + func ResourceCrawler() *schema.Resource { return &schema.Resource{ Create: resourceCrawlerCreate, @@ -34,90 +38,25 @@ func ResourceCrawler() *schema.Resource { CustomizeDiff: verify.SetTagsDiff, Schema: map[string]*schema.Schema{ - "name": { - Type: schema.TypeString, - ForceNew: true, - Required: true, - ValidateFunc: validation.All( - validation.StringLenBetween(1, 255), - validation.StringMatch(regexp.MustCompile(`[a-zA-Z0-9-_$#\/]+$`), ""), - ), - }, "arn": { Type: schema.TypeString, Computed: true, }, - "database_name": { - Type: schema.TypeString, - ForceNew: true, - Required: true, - }, - "role": { - Type: schema.TypeString, - Required: true, - // Glue API always returns name - DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { - newARN, err := arn.Parse(new) - - if err != nil { - return false - } - - return old == strings.TrimPrefix(newARN.Resource, "role/") - }, - }, - "description": { - Type: schema.TypeString, - Optional: true, - ValidateFunc: validation.StringLenBetween(0, 2048), - }, - "schedule": { - Type: schema.TypeString, - Optional: true, - }, - "classifiers": { - Type: schema.TypeList, - Optional: true, - Elem: &schema.Schema{Type: schema.TypeString}, - }, - "schema_change_policy": { - Type: schema.TypeList, - Optional: true, - DiffSuppressFunc: verify.SuppressMissingOptionalConfigurationBlock, - MaxItems: 1, - Elem: &schema.Resource{ - Schema: map[string]*schema.Schema{ - "delete_behavior": { - Type: schema.TypeString, - Optional: true, - Default: glue.DeleteBehaviorDeprecateInDatabase, - ValidateFunc: validation.StringInSlice(glue.DeleteBehavior_Values(), false), - }, - "update_behavior": { - Type: schema.TypeString, - Optional: true, - Default: glue.UpdateBehaviorUpdateInDatabase, - ValidateFunc: validation.StringInSlice(glue.UpdateBehavior_Values(), false), - }, - }, - }, - }, - "table_prefix": { - Type: schema.TypeString, - Optional: true, - ValidateFunc: validation.StringLenBetween(0, 128), - }, - "s3_target": { + "catalog_target": { Type: schema.TypeList, Optional: true, MinItems: 1, - AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target", "delta_target"}, + AtLeastOneOf: targets(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "connection_name": { Type: schema.TypeString, Optional: true, }, + "database_name": { + Type: schema.TypeString, + Required: true, + }, "dlq_event_queue_arn": { Type: schema.TypeString, Optional: true, @@ -128,58 +67,69 @@ func ResourceCrawler() *schema.Resource { Optional: true, ValidateFunc: verify.ValidARN, }, - "exclusions": { + "tables": { Type: schema.TypeList, - Optional: true, - Elem: &schema.Schema{Type: schema.TypeString}, - }, - "path": { - Type: schema.TypeString, Required: true, - }, - "sample_size": { - Type: schema.TypeInt, - Optional: true, - ValidateFunc: validation.IntBetween(1, 249), + Elem: &schema.Schema{Type: schema.TypeString}, }, }, }, }, - "dynamodb_target": { + "configuration": { + Type: schema.TypeString, + Optional: true, + DiffSuppressFunc: verify.SuppressEquivalentJSONDiffs, + StateFunc: func(v interface{}) string { + json, _ := structure.NormalizeJsonString(v) + return json + }, + ValidateFunc: validation.StringIsJSON, + }, + "classifiers": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "database_name": { + Type: schema.TypeString, + ForceNew: true, + Required: true, + }, + "delta_target": { Type: schema.TypeList, Optional: true, MinItems: 1, - AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target", "delta_target"}, + AtLeastOneOf: targets(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "path": { + "connection_name": { Type: schema.TypeString, + Optional: true, + }, + "delta_tables": { + Type: schema.TypeSet, Required: true, + Elem: &schema.Schema{Type: schema.TypeString}, }, - "scan_all": { + "write_manifest": { Type: schema.TypeBool, - Optional: true, - Default: true, - }, - "scan_rate": { - Type: schema.TypeFloat, - Optional: true, - ValidateFunc: validation.FloatBetween(0.1, 1.5), + Required: true, }, }, }, }, - "mongodb_target": { + "description": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 2048), + }, + "dynamodb_target": { Type: schema.TypeList, Optional: true, MinItems: 1, - AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target", "delta_target"}, + AtLeastOneOf: targets(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "connection_name": { - Type: schema.TypeString, - Required: true, - }, "path": { Type: schema.TypeString, Required: true, @@ -189,6 +139,11 @@ func ResourceCrawler() *schema.Resource { Optional: true, Default: true, }, + "scan_rate": { + Type: schema.TypeFloat, + Optional: true, + ValidateFunc: validation.FloatBetween(0.1, 1.5), + }, }, }, }, @@ -196,105 +151,133 @@ func ResourceCrawler() *schema.Resource { Type: schema.TypeList, Optional: true, MinItems: 1, - AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target", "delta_target"}, + AtLeastOneOf: targets(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "connection_name": { Type: schema.TypeString, Required: true, }, - "path": { - Type: schema.TypeString, - Required: true, + "enable_additional_metadata": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + ValidateFunc: validation.StringInSlice(glue.JdbcMetadataEntry_Values(), false), + }, }, "exclusions": { Type: schema.TypeList, Optional: true, Elem: &schema.Schema{Type: schema.TypeString}, }, + "path": { + Type: schema.TypeString, + Required: true, + }, }, }, }, - "delta_target": { - Type: schema.TypeList, - Optional: true, - MinItems: 1, - AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target", "delta_target"}, + "lineage_configuration": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + DiffSuppressFunc: verify.SuppressMissingOptionalConfigurationBlock, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "connection_name": { - Type: schema.TypeString, - Required: true, - }, - "delta_tables": { - Type: schema.TypeSet, - Required: true, - Elem: &schema.Schema{Type: schema.TypeString}, - }, - "write_manifest": { - Type: schema.TypeBool, - Required: true, + "crawler_lineage_settings": { + Type: schema.TypeString, + Optional: true, + Default: glue.CrawlerLineageSettingsDisable, + ValidateFunc: validation.StringInSlice(glue.CrawlerLineageSettings_Values(), false), }, }, }, }, - "catalog_target": { + "mongodb_target": { Type: schema.TypeList, Optional: true, MinItems: 1, - AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target", "delta_target"}, + AtLeastOneOf: targets(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "database_name": { + "connection_name": { Type: schema.TypeString, Required: true, }, - "tables": { - Type: schema.TypeList, + "path": { + Type: schema.TypeString, Required: true, - Elem: &schema.Schema{Type: schema.TypeString}, + }, + "scan_all": { + Type: schema.TypeBool, + Optional: true, + Default: true, }, }, }, }, - "configuration": { - Type: schema.TypeString, - Optional: true, - DiffSuppressFunc: verify.SuppressEquivalentJSONDiffs, - StateFunc: func(v interface{}) string { - json, _ := structure.NormalizeJsonString(v) - return json - }, - ValidateFunc: validation.StringIsJSON, + "name": { + Type: schema.TypeString, + ForceNew: true, + Required: true, + ValidateFunc: validation.All( + validation.StringLenBetween(1, 255), + validation.StringMatch(regexp.MustCompile(`[a-zA-Z0-9-_$#\/]+$`), ""), + ), }, - "lineage_configuration": { + "recrawl_policy": { Type: schema.TypeList, Optional: true, MaxItems: 1, DiffSuppressFunc: verify.SuppressMissingOptionalConfigurationBlock, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "crawler_lineage_settings": { + "recrawl_behavior": { Type: schema.TypeString, Optional: true, - Default: glue.CrawlerLineageSettingsDisable, - ValidateFunc: validation.StringInSlice(glue.CrawlerLineageSettings_Values(), false), + Default: glue.RecrawlBehaviorCrawlEverything, + ValidateFunc: validation.StringInSlice(glue.RecrawlBehavior_Values(), false), }, }, }, }, - "recrawl_policy": { + "role": { + Type: schema.TypeString, + Required: true, + // Glue API always returns name + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + newARN, err := arn.Parse(new) + + if err != nil { + return false + } + + return old == strings.TrimPrefix(newARN.Resource, "role/") + }, + }, + "schedule": { + Type: schema.TypeString, + Optional: true, + }, + "schema_change_policy": { Type: schema.TypeList, Optional: true, - MaxItems: 1, DiffSuppressFunc: verify.SuppressMissingOptionalConfigurationBlock, + MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "recrawl_behavior": { + "delete_behavior": { Type: schema.TypeString, Optional: true, - Default: glue.RecrawlBehaviorCrawlEverything, - ValidateFunc: validation.StringInSlice(glue.RecrawlBehavior_Values(), false), + Default: glue.DeleteBehaviorDeprecateInDatabase, + ValidateFunc: validation.StringInSlice(glue.DeleteBehavior_Values(), false), + }, + "update_behavior": { + Type: schema.TypeString, + Optional: true, + Default: glue.UpdateBehaviorUpdateInDatabase, + ValidateFunc: validation.StringInSlice(glue.UpdateBehavior_Values(), false), }, }, }, @@ -303,6 +286,49 @@ func ResourceCrawler() *schema.Resource { Type: schema.TypeString, Optional: true, }, + "s3_target": { + Type: schema.TypeList, + Optional: true, + MinItems: 1, + AtLeastOneOf: targets(), + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "connection_name": { + Type: schema.TypeString, + Optional: true, + }, + "dlq_event_queue_arn": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: verify.ValidARN, + }, + "event_queue_arn": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: verify.ValidARN, + }, + "exclusions": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "path": { + Type: schema.TypeString, + Required: true, + }, + "sample_size": { + Type: schema.TypeInt, + Optional: true, + ValidateFunc: validation.IntBetween(1, 249), + }, + }, + }, + }, + "table_prefix": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 128), + }, "tags": tftags.TagsSchema(), "tags_all": tftags.TagsSchemaComputed(), }, @@ -595,9 +621,14 @@ func expandJDBCTarget(cfg map[string]interface{}) *glue.JdbcTarget { ConnectionName: aws.String(cfg["connection_name"].(string)), } - if exclusions, ok := cfg["exclusions"]; ok { - target.Exclusions = flex.ExpandStringList(exclusions.([]interface{})) + if v, ok := cfg["enable_additional_metadata"].([]interface{}); ok { + target.Exclusions = flex.ExpandStringList(v) } + + if v, ok := cfg["exclusions"].([]interface{}); ok { + target.Exclusions = flex.ExpandStringList(v) + } + return target } @@ -620,6 +651,18 @@ func expandCatalogTarget(cfg map[string]interface{}) *glue.CatalogTarget { Tables: flex.ExpandStringList(cfg["tables"].([]interface{})), } + if v, ok := cfg["connection_name"].(string); ok { + target.ConnectionName = aws.String(v) + } + + if v, ok := cfg["dlq_event_queue_arn"].(string); ok { + target.DlqEventQueueArn = aws.String(v) + } + + if v, ok := cfg["event_queue_arn"].(string); ok { + target.EventQueueArn = aws.String(v) + } + return target } @@ -661,9 +704,12 @@ func expandDeltaTargets(targets []interface{}) []*glue.DeltaTarget { func expandDeltaTarget(cfg map[string]interface{}) *glue.DeltaTarget { target := &glue.DeltaTarget{ - ConnectionName: aws.String(cfg["connection_name"].(string)), - DeltaTables: flex.ExpandStringSet(cfg["delta_tables"].(*schema.Set)), - WriteManifest: aws.Bool(cfg["write_manifest"].(bool)), + DeltaTables: flex.ExpandStringSet(cfg["delta_tables"].(*schema.Set)), + WriteManifest: aws.Bool(cfg["write_manifest"].(bool)), + } + + if v, ok := cfg["connection_name"].(string); ok { + target.ConnectionName = aws.String(v) } return target @@ -862,8 +908,11 @@ func flattenCatalogTargets(CatalogTargets []*glue.CatalogTarget) []map[string]in for _, catalogTarget := range CatalogTargets { attrs := make(map[string]interface{}) + attrs["connection_name"] = aws.StringValue(catalogTarget.ConnectionName) attrs["tables"] = flex.FlattenStringList(catalogTarget.Tables) attrs["database_name"] = aws.StringValue(catalogTarget.DatabaseName) + attrs["event_queue_arn"] = aws.StringValue(catalogTarget.EventQueueArn) + attrs["dlq_event_queue_arn"] = aws.StringValue(catalogTarget.DlqEventQueueArn) result = append(result, attrs) } @@ -891,6 +940,7 @@ func flattenJDBCTargets(jdbcTargets []*glue.JdbcTarget) []map[string]interface{} attrs := make(map[string]interface{}) attrs["connection_name"] = aws.StringValue(jdbcTarget.ConnectionName) attrs["exclusions"] = flex.FlattenStringList(jdbcTarget.Exclusions) + attrs["enable_additional_metadata"] = flex.FlattenStringList(jdbcTarget.EnableAdditionalMetadata) attrs["path"] = aws.StringValue(jdbcTarget.Path) result = append(result, attrs) From 5622d341826e407d38acb2fb21f7ba098fc1d926 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sat, 3 Dec 2022 22:01:31 +0200 Subject: [PATCH 02/10] changelog --- .changelog/28156.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changelog/28156.txt diff --git a/.changelog/28156.txt b/.changelog/28156.txt new file mode 100644 index 000000000000..1feb2e96700e --- /dev/null +++ b/.changelog/28156.txt @@ -0,0 +1,7 @@ +```release-note:enhancement +resource/aws_glue_crawler: Add `catalog_target.dlq_event_queue_arn`, `catalog_target.event_queue_arn`, `catalog_target.connection_name`, and `jdbc_target.enable_additional_metadata` arguments +``` + +```release-note:enhancement +resource/aws_glue_crawler: Make `delta_target.connection_name` optional +``` \ No newline at end of file From 05c249a3979da69da28eb61a3d0af12a1d319796 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sat, 3 Dec 2022 22:08:09 +0200 Subject: [PATCH 03/10] catalog dlq --- internal/service/glue/crawler_test.go | 143 +++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/internal/service/glue/crawler_test.go b/internal/service/glue/crawler_test.go index 6cf8d5008a6b..65d5bf4148ac 100644 --- a/internal/service/glue/crawler_test.go +++ b/internal/service/glue/crawler_test.go @@ -752,6 +752,36 @@ func TestAccGlueCrawler_S3Target_eventqueue(t *testing.T) { }) } +func TestAccGlueCrawler_CatalogTarget_dlqeventqueue(t *testing.T) { + var crawler glue.Crawler + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_glue_crawler.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(t) }, + ErrorCheck: acctest.ErrorCheck(t, glue.EndpointsID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckCrawlerDestroy, + Steps: []resource.TestStep{ + { + Config: testAccCrawlerConfig_catalogTargetDlqEventQueue(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckCrawlerExists(resourceName, &crawler), + acctest.CheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("crawler/%s", rName)), + resource.TestCheckResourceAttr(resourceName, "catalog_target.#", "1"), + resource.TestCheckResourceAttrPair(resourceName, "catalog_target.0.event_queue_arn", "aws_sqs_queue.test", "arn"), + resource.TestCheckResourceAttrPair(resourceName, "catalog_target.0.dlq_event_queue_arn", "aws_sqs_queue.test_dql", "arn"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func TestAccGlueCrawler_S3Target_dlqeventqueue(t *testing.T) { var crawler glue.Crawler rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) @@ -769,8 +799,8 @@ func TestAccGlueCrawler_S3Target_dlqeventqueue(t *testing.T) { testAccCheckCrawlerExists(resourceName, &crawler), acctest.CheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("crawler/%s", rName)), resource.TestCheckResourceAttr(resourceName, "s3_target.#", "1"), - acctest.CheckResourceAttrRegionalARN(resourceName, "s3_target.0.event_queue_arn", "sqs", rName), - acctest.CheckResourceAttrRegionalARN(resourceName, "s3_target.0.dlq_event_queue_arn", "sqs", fmt.Sprintf("%sdlq", rName)), + resource.TestCheckResourceAttrPair(resourceName, "s3_target.0.event_queue_arn", "aws_sqs_queue.test", "arn"), + resource.TestCheckResourceAttrPair(resourceName, "s3_target.0.dlq_event_queue_arn", "aws_sqs_queue.test_dql", "arn"), resource.TestCheckResourceAttr(resourceName, "recrawl_policy.0.recrawl_behavior", "CRAWL_EVENT_MODE"), ), }, @@ -2272,6 +2302,115 @@ resource "aws_glue_crawler" "test" { `, rName) } +func testAccCrawlerConfig_catalogTargetDlqEventQueue(rName string) string { + return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` +resource "aws_glue_catalog_database" "test" { + count = 2 + name = "%[1]s_database_${count.index}" +} + +resource "aws_glue_catalog_table" "test" { + count = 2 + database_name = aws_glue_catalog_database.test[count.index].name + name = "%[1]s_table_${count.index}" + table_type = "EXTERNAL_TABLE" + + storage_descriptor { + location = "s3://${aws_s3_bucket.default.bucket}" + } +} + +resource "aws_lakeformation_permissions" "test" { + count = 2 + + permissions = ["ALL"] + principal = aws_iam_role.test.arn + + table { + database_name = aws_glue_catalog_database.test[count.index].name + name = aws_glue_catalog_table.test[count.index].name + } +} + +resource "aws_s3_bucket" "default" { + bucket = %[1]q + force_destroy = true +} + +resource "aws_sqs_queue" "test" { + name = %[1]q + + visibility_timeout_seconds = 3600 +} + +resource "aws_sqs_queue" "test_dlq" { + name = "%[1]sdlq" + + visibility_timeout_seconds = 3600 +} + +resource "aws_iam_role_policy" "test_sqs" { + role = aws_iam_role.test.name + + policy = data.aws_iam_policy_document.role_test_sqs.json +} + +data "aws_iam_policy_document" "role_test_sqs" { + statement { + effect = "Allow" + + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ListDeadLetterSourceQueues", + "sqs:DeleteMessageBatch", + "sqs:ReceiveMessage", + "sqs:GetQueueAttributes", + "sqs:ListQueueTags", + "sqs:SetQueueAttributes", + "sqs:PurgeQueue", + ] + + resources = [ + aws_sqs_queue.test_dlq.arn, + aws_sqs_queue.test.arn, + ] + } +} + +resource "aws_glue_crawler" "test" { + depends_on = [ + aws_iam_role_policy_attachment.test-AWSGlueServiceRole, + aws_iam_role_policy.test_sqs, + ] + + database_name = aws_glue_catalog_database.test.name + name = %[1]q + role = aws_iam_role.test.name + + schema_change_policy { + delete_behavior = "LOG" + } + + catalog_target { + database_name = aws_glue_catalog_database.test.name + tables = flatten([aws_glue_catalog_table.test[*].name]) + event_queue_arn = aws_sqs_queue.test.arn + dlq_event_queue_arn = aws_sqs_queue.test_dlq.arn + } + + configuration = < Date: Sat, 3 Dec 2022 22:16:00 +0200 Subject: [PATCH 04/10] docs --- internal/service/glue/crawler_test.go | 69 ++++++++++------------- website/docs/r/glue_crawler.html.markdown | 8 ++- 2 files changed, 37 insertions(+), 40 deletions(-) diff --git a/internal/service/glue/crawler_test.go b/internal/service/glue/crawler_test.go index 65d5bf4148ac..ad131207f2ff 100644 --- a/internal/service/glue/crawler_test.go +++ b/internal/service/glue/crawler_test.go @@ -2304,39 +2304,6 @@ resource "aws_glue_crawler" "test" { func testAccCrawlerConfig_catalogTargetDlqEventQueue(rName string) string { return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` -resource "aws_glue_catalog_database" "test" { - count = 2 - name = "%[1]s_database_${count.index}" -} - -resource "aws_glue_catalog_table" "test" { - count = 2 - database_name = aws_glue_catalog_database.test[count.index].name - name = "%[1]s_table_${count.index}" - table_type = "EXTERNAL_TABLE" - - storage_descriptor { - location = "s3://${aws_s3_bucket.default.bucket}" - } -} - -resource "aws_lakeformation_permissions" "test" { - count = 2 - - permissions = ["ALL"] - principal = aws_iam_role.test.arn - - table { - database_name = aws_glue_catalog_database.test[count.index].name - name = aws_glue_catalog_table.test[count.index].name - } -} - -resource "aws_s3_bucket" "default" { - bucket = %[1]q - force_destroy = true -} - resource "aws_sqs_queue" "test" { name = %[1]q @@ -2378,11 +2345,37 @@ data "aws_iam_policy_document" "role_test_sqs" { } } +resource "aws_glue_catalog_database" "test" { + name = %[1]q +} + +resource "aws_s3_bucket" "default" { + bucket = %[1]q + force_destroy = true +} + +resource "aws_glue_catalog_table" "test" { + database_name = aws_glue_catalog_database.test.name + name = %[1]q + table_type = "EXTERNAL_TABLE" + + storage_descriptor { + location = "s3://${aws_s3_bucket.default.bucket}" + } +} + +resource "aws_lakeformation_permissions" "test" { + permissions = ["ALL"] + principal = aws_iam_role.test.arn + + table { + database_name = aws_glue_catalog_database.test.name + name = aws_glue_catalog_table.test.name + } +} + resource "aws_glue_crawler" "test" { - depends_on = [ - aws_iam_role_policy_attachment.test-AWSGlueServiceRole, - aws_iam_role_policy.test_sqs, - ] + depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole, aws_iam_role_policy.test] database_name = aws_glue_catalog_database.test.name name = %[1]q @@ -2394,7 +2387,7 @@ resource "aws_glue_crawler" "test" { catalog_target { database_name = aws_glue_catalog_database.test.name - tables = flatten([aws_glue_catalog_table.test[*].name]) + tables = [aws_glue_catalog_table.test.name] event_queue_arn = aws_sqs_queue.test.arn dlq_event_queue_arn = aws_sqs_queue.test_dlq.arn } diff --git a/website/docs/r/glue_crawler.html.markdown b/website/docs/r/glue_crawler.html.markdown index 1703cd190a24..93ae0ebe7d5f 100644 --- a/website/docs/r/glue_crawler.html.markdown +++ b/website/docs/r/glue_crawler.html.markdown @@ -128,7 +128,7 @@ resource "aws_glue_crawler" "events_crawler" { ## Argument Reference -~> **NOTE:** Must specify at least one of `dynamodb_target`, `jdbc_target`, `s3_target` or `catalog_target`. +~> **NOTE:** Must specify at least one of `dynamodb_target`, `jdbc_target`, `s3_target`, `mongodb_target` or `catalog_target`. The following arguments are supported: @@ -161,6 +161,7 @@ The following arguments are supported: * `connection_name` - (Required) The name of the connection to use to connect to the JDBC target. * `path` - (Required) The path of the JDBC target. * `exclusions` - (Optional) A list of glob patterns used to exclude from the crawl. +* `enable_additional_metadata` - (Optional) Specify a value of `RAWTYPES` or `COMMENTS` to enable additional metadata intable responses. `RAWTYPES` provides the native-level datatype. `COMMENTS` provides comments associated with a column or table in the database. ### S3 Target @@ -173,8 +174,11 @@ The following arguments are supported: ### Catalog Target +* `connection_name` - (Optional) The name of the connection for an Amazon S3-backed Data Catalog table to be a target of the crawl when using a Catalog connection type paired with a `NETWORK` Connection type. * `database_name` - (Required) The name of the Glue database to be synchronized. * `tables` - (Required) A list of catalog tables to be synchronized. +* `event_queue_arn` - (Optional) A valid Amazon SQS ARN. +* `dlq_event_queue_arn` - (Optional) A valid Amazon SQS ARN. ~> **Note:** `deletion_behavior` of catalog target doesn't support `DEPRECATE_IN_DATABASE`. @@ -188,7 +192,7 @@ The following arguments are supported: ### Delta Target -* `connection_name` - (Required) The name of the connection to use to connect to the Delta table target. +* `connection_name` - (Optional) The name of the connection to use to connect to the Delta table target. * `delta_tables` - (Required) A list of the Amazon S3 paths to the Delta tables. * `write_manifest` - (Required) Specifies whether to write the manifest files to the Delta table path. From 9ccf8836b90edf2ddcd5c84759eddb72e08966c5 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sat, 3 Dec 2022 22:19:34 +0200 Subject: [PATCH 05/10] fix test --- internal/service/glue/crawler_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/service/glue/crawler_test.go b/internal/service/glue/crawler_test.go index ad131207f2ff..f1a623280fd9 100644 --- a/internal/service/glue/crawler_test.go +++ b/internal/service/glue/crawler_test.go @@ -770,7 +770,7 @@ func TestAccGlueCrawler_CatalogTarget_dlqeventqueue(t *testing.T) { acctest.CheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("crawler/%s", rName)), resource.TestCheckResourceAttr(resourceName, "catalog_target.#", "1"), resource.TestCheckResourceAttrPair(resourceName, "catalog_target.0.event_queue_arn", "aws_sqs_queue.test", "arn"), - resource.TestCheckResourceAttrPair(resourceName, "catalog_target.0.dlq_event_queue_arn", "aws_sqs_queue.test_dql", "arn"), + resource.TestCheckResourceAttrPair(resourceName, "catalog_target.0.dlq_event_queue_arn", "aws_sqs_queue.test_dlq", "arn"), ), }, { @@ -2375,7 +2375,7 @@ resource "aws_lakeformation_permissions" "test" { } resource "aws_glue_crawler" "test" { - depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole, aws_iam_role_policy.test] + depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole, aws_iam_role_policy.test_sqs] database_name = aws_glue_catalog_database.test.name name = %[1]q From 4cba2afa12e611d3ff809e8f8f262c55724b7160 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sat, 3 Dec 2022 23:59:02 +0200 Subject: [PATCH 06/10] use finder --- internal/service/glue/crawler.go | 27 +++++++--------------- internal/service/glue/crawler_test.go | 32 +++++++++------------------ internal/service/glue/find.go | 24 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 41 deletions(-) diff --git a/internal/service/glue/crawler.go b/internal/service/glue/crawler.go index 540100f70005..3e8a466705ba 100644 --- a/internal/service/glue/crawler.go +++ b/internal/service/glue/crawler.go @@ -773,32 +773,21 @@ func resourceCrawlerUpdate(d *schema.ResourceData, meta interface{}) error { } func resourceCrawlerRead(d *schema.ResourceData, meta interface{}) error { - glueConn := meta.(*conns.AWSClient).GlueConn + conn := meta.(*conns.AWSClient).GlueConn defaultTagsConfig := meta.(*conns.AWSClient).DefaultTagsConfig ignoreTagsConfig := meta.(*conns.AWSClient).IgnoreTagsConfig - input := &glue.GetCrawlerInput{ - Name: aws.String(d.Id()), - } - - crawlerOutput, err := glueConn.GetCrawler(input) - if err != nil { - if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { - log.Printf("[WARN] Glue Crawler (%s) not found, removing from state", d.Id()) - d.SetId("") - return nil - } - - return fmt.Errorf("error reading Glue crawler: %w", err) - } - - crawler := crawlerOutput.Crawler - if crawler == nil { + crawler, err := FindCrawlerByName(conn, d.Id()) + if !d.IsNewResource() && tfresource.NotFound(err) { log.Printf("[WARN] Glue Crawler (%s) not found, removing from state", d.Id()) d.SetId("") return nil } + if err != nil { + return fmt.Errorf("error reading Glue Crawler (%s): %w", d.Id(), err) + } + crawlerARN := arn.ARN{ Partition: meta.(*conns.AWSClient).Partition, Service: "glue", @@ -854,7 +843,7 @@ func resourceCrawlerRead(d *schema.ResourceData, meta interface{}) error { } } - tags, err := ListTags(glueConn, crawlerARN) + tags, err := ListTags(conn, crawlerARN) if err != nil { return fmt.Errorf("error listing tags for Glue Crawler (%s): %w", crawlerARN, err) diff --git a/internal/service/glue/crawler_test.go b/internal/service/glue/crawler_test.go index f1a623280fd9..9b67147993ea 100644 --- a/internal/service/glue/crawler_test.go +++ b/internal/service/glue/crawler_test.go @@ -9,13 +9,13 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/glue" - "github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2/tfawserr" sdkacctest "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/hashicorp/terraform-provider-aws/internal/acctest" "github.com/hashicorp/terraform-provider-aws/internal/conns" tfglue "github.com/hashicorp/terraform-provider-aws/internal/service/glue" + "github.com/hashicorp/terraform-provider-aws/internal/tfresource" "github.com/hashicorp/terraform-provider-aws/internal/verify" ) @@ -1546,20 +1546,14 @@ func testAccCheckCrawlerExists(resourceName string, crawler *glue.Crawler) resou return fmt.Errorf("no ID is set") } - glueConn := acctest.Provider.Meta().(*conns.AWSClient).GlueConn - out, err := glueConn.GetCrawler(&glue.GetCrawlerInput{ - Name: aws.String(rs.Primary.ID), - }) + conn := acctest.Provider.Meta().(*conns.AWSClient).GlueConn + output, err := tfglue.FindCrawlerByName(conn, rs.Primary.ID) if err != nil { return err } - if out.Crawler == nil { - return fmt.Errorf("no Glue Crawler found") - } - - *crawler = *out.Crawler + *crawler = *output return nil } @@ -1572,23 +1566,17 @@ func testAccCheckCrawlerDestroy(s *terraform.State) error { } conn := acctest.Provider.Meta().(*conns.AWSClient).GlueConn - output, err := conn.GetCrawler(&glue.GetCrawlerInput{ - Name: aws.String(rs.Primary.ID), - }) + _, err := tfglue.FindCrawlerByName(conn, rs.Primary.ID) - if err != nil { - if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { - return nil - } - return err + if tfresource.NotFound(err) { + continue } - crawler := output.Crawler - if crawler != nil && aws.StringValue(crawler.Name) == rs.Primary.ID { - return fmt.Errorf("Glue Crawler %s still exists", rs.Primary.ID) + if err != nil { + return err } - return nil + return fmt.Errorf("Glue Crawler %s still exists", rs.Primary.ID) } return nil diff --git a/internal/service/glue/find.go b/internal/service/glue/find.go index f38e21dfc0c6..caf4f0539c61 100644 --- a/internal/service/glue/find.go +++ b/internal/service/glue/find.go @@ -261,3 +261,27 @@ func FindClassifierByName(conn *glue.Glue, name string) (*glue.Classifier, error return output.Classifier, nil } + +func FindCrawlerByName(conn *glue.Glue, name string) (*glue.Crawler, error) { + input := &glue.GetCrawlerInput{ + Name: aws.String(name), + } + + output, err := conn.GetCrawler(input) + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { + return nil, &resource.NotFoundError{ + LastError: err, + LastRequest: input, + } + } + + if err != nil { + return nil, err + } + + if output == nil || output.Crawler == nil { + return nil, tfresource.NewEmptyResultError(input) + } + + return output.Crawler, nil +} From 38939eb0ff8f7fd8858a3cb5baab28bee39eaa41 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sun, 4 Dec 2022 00:20:24 +0200 Subject: [PATCH 07/10] lake formation config --- .changelog/28156.txt | 2 +- internal/service/glue/crawler.go | 63 ++++++++++++++++++++++- internal/service/glue/crawler_test.go | 59 +++++++++++++++++++++ website/docs/r/glue_crawler.html.markdown | 6 +++ 4 files changed, 128 insertions(+), 2 deletions(-) diff --git a/.changelog/28156.txt b/.changelog/28156.txt index 1feb2e96700e..855f31f05899 100644 --- a/.changelog/28156.txt +++ b/.changelog/28156.txt @@ -1,5 +1,5 @@ ```release-note:enhancement -resource/aws_glue_crawler: Add `catalog_target.dlq_event_queue_arn`, `catalog_target.event_queue_arn`, `catalog_target.connection_name`, and `jdbc_target.enable_additional_metadata` arguments +resource/aws_glue_crawler: Add `catalog_target.dlq_event_queue_arn`, `catalog_target.event_queue_arn`, `catalog_target.connection_name`, `lake_formation_configuration`, and `jdbc_target.enable_additional_metadata` arguments ``` ```release-note:enhancement diff --git a/internal/service/glue/crawler.go b/internal/service/glue/crawler.go index 3e8a466705ba..7ac3929053b8 100644 --- a/internal/service/glue/crawler.go +++ b/internal/service/glue/crawler.go @@ -178,6 +178,26 @@ func ResourceCrawler() *schema.Resource { }, }, }, + "lake_formation_configuration": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + DiffSuppressFunc: verify.SuppressMissingOptionalConfigurationBlock, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "account_id": { + Type: schema.TypeString, + Optional: true, + Computed: true, + ValidateFunc: verify.ValidAccountID, + }, + "use_lake_formation_credentials": { + Type: schema.TypeBool, + Optional: true, + }, + }, + }, + }, "lineage_configuration": { Type: schema.TypeList, Optional: true, @@ -414,7 +434,7 @@ func createCrawlerInput(d *schema.ResourceData, crawlerName string, defaultTagsC if v, ok := d.GetOk("configuration"); ok { configuration, err := structure.NormalizeJsonString(v) if err != nil { - return nil, fmt.Errorf("Configuration contains an invalid JSON: %v", err) + return nil, fmt.Errorf("configuration contains an invalid JSON: %v", err) } crawlerInput.Configuration = aws.String(configuration) } @@ -427,6 +447,10 @@ func createCrawlerInput(d *schema.ResourceData, crawlerName string, defaultTagsC crawlerInput.LineageConfiguration = expandCrawlerLineageConfiguration(v.([]interface{})) } + if v, ok := d.GetOk("lake_formation_configuration"); ok { + crawlerInput.LakeFormationConfiguration = expandLakeFormationConfiguration(v.([]interface{})) + } + if v, ok := d.GetOk("recrawl_policy"); ok { crawlerInput.RecrawlPolicy = expandCrawlerRecrawlPolicy(v.([]interface{})) } @@ -477,6 +501,10 @@ func updateCrawlerInput(d *schema.ResourceData, crawlerName string) (*glue.Updat crawlerInput.LineageConfiguration = expandCrawlerLineageConfiguration(v.([]interface{})) } + if v, ok := d.GetOk("lake_formation_configuration"); ok { + crawlerInput.LakeFormationConfiguration = expandLakeFormationConfiguration(v.([]interface{})) + } + if v, ok := d.GetOk("recrawl_policy"); ok { crawlerInput.RecrawlPolicy = expandCrawlerRecrawlPolicy(v.([]interface{})) } @@ -864,6 +892,10 @@ func resourceCrawlerRead(d *schema.ResourceData, meta interface{}) error { return fmt.Errorf("error setting lineage_configuration: %w", err) } + if err := d.Set("lake_formation_configuration", flattenLakeFormationConfiguration(crawler.LakeFormationConfiguration)); err != nil { + return fmt.Errorf("error setting lake_formation_configuration: %w", err) + } + if err := d.Set("recrawl_policy", flattenCrawlerRecrawlPolicy(crawler.RecrawlPolicy)); err != nil { return fmt.Errorf("error setting recrawl_policy: %w", err) } @@ -1015,6 +1047,35 @@ func flattenCrawlerLineageConfiguration(cfg *glue.LineageConfiguration) []map[st return []map[string]interface{}{m} } +func expandLakeFormationConfiguration(cfg []interface{}) *glue.LakeFormationConfiguration { + m := cfg[0].(map[string]interface{}) + + target := &glue.LakeFormationConfiguration{} + + if v, ok := m["account_id"].(string); ok { + target.AccountId = aws.String(v) + } + + if v, ok := m["use_lake_formation_credentials"].(bool); ok { + target.UseLakeFormationCredentials = aws.Bool(v) + } + + return target +} + +func flattenLakeFormationConfiguration(cfg *glue.LakeFormationConfiguration) []map[string]interface{} { + if cfg == nil { + return []map[string]interface{}{} + } + + m := map[string]interface{}{ + "account_id": aws.StringValue(cfg.AccountId), + "use_lake_formation_credentials": aws.BoolValue(cfg.UseLakeFormationCredentials), + } + + return []map[string]interface{}{m} +} + func expandCrawlerRecrawlPolicy(cfg []interface{}) *glue.RecrawlPolicy { m := cfg[0].(map[string]interface{}) diff --git a/internal/service/glue/crawler_test.go b/internal/service/glue/crawler_test.go index 9b67147993ea..64844d316f18 100644 --- a/internal/service/glue/crawler_test.go +++ b/internal/service/glue/crawler_test.go @@ -1492,6 +1492,41 @@ func TestAccGlueCrawler_lineage(t *testing.T) { }) } +func TestAccGlueCrawler_lakeformation(t *testing.T) { + var crawler glue.Crawler + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_glue_crawler.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(t) }, + ErrorCheck: acctest.ErrorCheck(t, glue.EndpointsID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckCrawlerDestroy, + Steps: []resource.TestStep{ + { + Config: testAccCrawlerConfig_lakeformation(rName, true), + Check: resource.ComposeTestCheckFunc( + testAccCheckCrawlerExists(resourceName, &crawler), + resource.TestCheckResourceAttr(resourceName, "lake_formation_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "lake_formation_configuration.0.use_lake_formation_credentials", "true"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccCrawlerConfig_lakeformation(rName, false), + Check: resource.ComposeTestCheckFunc( + testAccCheckCrawlerExists(resourceName, &crawler), + resource.TestCheckResourceAttr(resourceName, "lake_formation_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "lake_formation_configuration.0.use_lake_formation_credentials", "false")), + }, + }, + }) +} + func TestAccGlueCrawler_reCrawlPolicy(t *testing.T) { var crawler glue.Crawler rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) @@ -2915,6 +2950,30 @@ resource "aws_glue_crawler" "test" { `, rName, connectionUrl, tableName) } +func testAccCrawlerConfig_lakeformation(rName string, use bool) string { + return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` +resource "aws_glue_catalog_database" "test" { + name = %[1]q +} + +resource "aws_glue_crawler" "test" { + depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole] + + database_name = aws_glue_catalog_database.test.name + name = %[1]q + role = aws_iam_role.test.name + + lake_formation_configuration { + use_lake_formation_credentials = %[2]t + } + + s3_target { + path = "s3://bucket-name" + } +} +`, rName, use) +} + func testAccCrawlerConfig_lineage(rName, lineageConfig string) string { return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { diff --git a/website/docs/r/glue_crawler.html.markdown b/website/docs/r/glue_crawler.html.markdown index 93ae0ebe7d5f..814b466b8994 100644 --- a/website/docs/r/glue_crawler.html.markdown +++ b/website/docs/r/glue_crawler.html.markdown @@ -144,6 +144,7 @@ The following arguments are supported: * `mongodb_target` (Optional) List nested MongoDB target arguments. See [MongoDB Target](#mongodb-target) below. * `schedule` (Optional) A cron expression used to specify the schedule. For more information, see [Time-Based Schedules for Jobs and Crawlers](https://docs.aws.amazon.com/glue/latest/dg/monitor-data-warehouse-schedule.html). For example, to run something every day at 12:15 UTC, you would specify: `cron(15 12 * * ? *)`. * `schema_change_policy` (Optional) Policy for the crawler's update and deletion behavior. See [Schema Change Policy](#schema-change-policy) below. +* `lake_formation_configuration` (Optional) Specifies Lake Formation configuration settings for the crawler. See [Lake Formation Configuration](#lake-formation-configuration) below. * `lineage_configuration` (Optional) Specifies data lineage configuration settings for the crawler. See [Lineage Configuration](#lineage-configuration) below. * `recrawl_policy` (Optional) A policy that specifies whether to crawl the entire dataset again, or to crawl only folders that were added since the last crawler run.. See [Recrawl Policy](#recrawl-policy) below. * `security_configuration` (Optional) The name of Security Configuration to be used by the crawler @@ -201,6 +202,11 @@ The following arguments are supported: * `delete_behavior` - (Optional) The deletion behavior when the crawler finds a deleted object. Valid values: `LOG`, `DELETE_FROM_DATABASE`, or `DEPRECATE_IN_DATABASE`. Defaults to `DEPRECATE_IN_DATABASE`. * `update_behavior` - (Optional) The update behavior when the crawler finds a changed schema. Valid values: `LOG` or `UPDATE_IN_DATABASE`. Defaults to `UPDATE_IN_DATABASE`. +### Lake Formation Configuration + +* `account_id` - (Optional) Required for cross account crawls. For same account crawls as the target data, this can ommited. +* `use_lake_formation_credentials` - (Optional) Specifies whether to use Lake Formation credentials for the crawler instead of the IAM role credentials. + ### Lineage Configuration * `crawler_lineage_settings` - (Optional) Specifies whether data lineage is enabled for the crawler. Valid values are: `ENABLE` and `DISABLE`. Default value is `Disable`. From d82eeb83027c1cbff4a51183a4e042e152da9573 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sun, 4 Dec 2022 00:22:59 +0200 Subject: [PATCH 08/10] lake formation config --- website/docs/r/glue_crawler.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/r/glue_crawler.html.markdown b/website/docs/r/glue_crawler.html.markdown index 814b466b8994..37f076841d86 100644 --- a/website/docs/r/glue_crawler.html.markdown +++ b/website/docs/r/glue_crawler.html.markdown @@ -204,7 +204,7 @@ The following arguments are supported: ### Lake Formation Configuration -* `account_id` - (Optional) Required for cross account crawls. For same account crawls as the target data, this can ommited. +* `account_id` - (Optional) Required for cross account crawls. For same account crawls as the target data, this can omitted. * `use_lake_formation_credentials` - (Optional) Specifies whether to use Lake Formation credentials for the crawler instead of the IAM role credentials. ### Lineage Configuration From c15cff99c70a68fa8891966c34f8986e7cb8d251 Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 7 Dec 2022 09:51:40 -0500 Subject: [PATCH 09/10] Fix typo in 'testAccCrawlerConfig_s3TargetDlqEventQueue'. --- internal/service/glue/crawler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/service/glue/crawler_test.go b/internal/service/glue/crawler_test.go index 64844d316f18..7c9fa9bbc83c 100644 --- a/internal/service/glue/crawler_test.go +++ b/internal/service/glue/crawler_test.go @@ -800,7 +800,7 @@ func TestAccGlueCrawler_S3Target_dlqeventqueue(t *testing.T) { acctest.CheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("crawler/%s", rName)), resource.TestCheckResourceAttr(resourceName, "s3_target.#", "1"), resource.TestCheckResourceAttrPair(resourceName, "s3_target.0.event_queue_arn", "aws_sqs_queue.test", "arn"), - resource.TestCheckResourceAttrPair(resourceName, "s3_target.0.dlq_event_queue_arn", "aws_sqs_queue.test_dql", "arn"), + resource.TestCheckResourceAttrPair(resourceName, "s3_target.0.dlq_event_queue_arn", "aws_sqs_queue.test_dlq", "arn"), resource.TestCheckResourceAttr(resourceName, "recrawl_policy.0.recrawl_behavior", "CRAWL_EVENT_MODE"), ), }, From e5c6e8dd0555736b5201c02a88a0db99ee011e84 Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 7 Dec 2022 10:01:41 -0500 Subject: [PATCH 10/10] r/aws_glue_crawler: Tidy up acceptance test configurations. --- internal/service/glue/crawler_test.go | 203 +++++++++++--------------- 1 file changed, 89 insertions(+), 114 deletions(-) diff --git a/internal/service/glue/crawler_test.go b/internal/service/glue/crawler_test.go index 7c9fa9bbc83c..a4326d7e70a4 100644 --- a/internal/service/glue/crawler_test.go +++ b/internal/service/glue/crawler_test.go @@ -1642,7 +1642,7 @@ func testAccCrawlerConfig_base(rName string) string { data "aws_partition" "current" {} resource "aws_iam_role" "test" { - name = %q + name = %[1]q assume_role_policy = data.aws_iam_policy_document.assume.json } @@ -1689,13 +1689,13 @@ EOF } func testAccCrawlerConfig_classifiersSingle(rName string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { - name = %q + name = %[1]q } resource "aws_glue_classifier" "test1" { - name = %q + name = %[2]q grok_classifier { classification = "example" @@ -1704,7 +1704,7 @@ resource "aws_glue_classifier" "test1" { } resource "aws_glue_classifier" "test2" { - name = %q + name = %[3]q grok_classifier { classification = "example" @@ -1716,7 +1716,7 @@ resource "aws_glue_crawler" "test" { depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole] classifiers = [aws_glue_classifier.test1.id] - name = %q + name = %[4]q database_name = aws_glue_catalog_database.test.name role = aws_iam_role.test.name @@ -1724,17 +1724,17 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket-name" } } -`, rName, rName+"1", rName+"2", rName) +`, rName, rName+"1", rName+"2", rName)) } func testAccCrawlerConfig_classifiersMultiple(rName string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { - name = %q + name = %[1]q } resource "aws_glue_classifier" "test1" { - name = %q + name = %[2]q grok_classifier { classification = "example" @@ -1743,7 +1743,7 @@ resource "aws_glue_classifier" "test1" { } resource "aws_glue_classifier" "test2" { - name = %q + name = %[3]q grok_classifier { classification = "example" @@ -1755,7 +1755,7 @@ resource "aws_glue_crawler" "test" { depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole] classifiers = [aws_glue_classifier.test1.id, aws_glue_classifier.test2.id] - name = %q + name = %[4]q database_name = aws_glue_catalog_database.test.name role = aws_iam_role.test.name @@ -1763,53 +1763,53 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket-name" } } -`, rName, rName+"1", rName+"2", rName) +`, rName, rName+"1", rName+"2", rName)) } func testAccCrawlerConfig_configuration(rName, configuration string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { - name = %q + name = %[1]q } resource "aws_glue_crawler" "test" { depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole] - configuration = %s + configuration = %[2]s database_name = aws_glue_catalog_database.test.name - name = %q + name = %[3]q role = aws_iam_role.test.name s3_target { path = "s3://bucket-name" } } -`, rName, strconv.Quote(configuration), rName) +`, rName, strconv.Quote(configuration), rName)) } func testAccCrawlerConfig_description(rName, description string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { - name = %q + name = %[1]q } resource "aws_glue_crawler" "test" { depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole] database_name = aws_glue_catalog_database.test.name - description = %q - name = %q + description = %[2]q + name = %[3]q role = aws_iam_role.test.name s3_target { path = "s3://bucket-name" } } -`, rName, description, rName) +`, rName, description, rName)) } func testAccCrawlerConfig_dynamoDBTarget(rName, path string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -1825,11 +1825,11 @@ resource "aws_glue_crawler" "test" { path = %[2]q } } -`, rName, path) +`, rName, path)) } func testAccCrawlerConfig_dynamoDBTargetScanAll(rName, path string, scanAll bool) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -1846,11 +1846,11 @@ resource "aws_glue_crawler" "test" { scan_all = %[3]t } } -`, rName, path, scanAll) +`, rName, path, scanAll)) } func testAccCrawlerConfig_dynamoDBTargetScanRate(rName, path string, scanRate float64) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -1867,11 +1867,11 @@ resource "aws_glue_crawler" "test" { scan_rate = %[3]g } } -`, rName, path, scanRate) +`, rName, path, scanRate)) } func testAccCrawlerConfig_jdbcTarget(rName, jdbcConnectionUrl, path string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -1898,11 +1898,11 @@ resource "aws_glue_crawler" "test" { path = %[3]q } } -`, rName, jdbcConnectionUrl, path) +`, rName, jdbcConnectionUrl, path)) } func testAccCrawlerConfig_jdbcTargetExclusions1(rName, jdbcConnectionUrl, exclusion1 string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -1930,11 +1930,11 @@ resource "aws_glue_crawler" "test" { path = "database-name/table1" } } -`, rName, jdbcConnectionUrl, exclusion1) +`, rName, jdbcConnectionUrl, exclusion1)) } func testAccCrawlerConfig_jdbcTargetExclusions2(rName, jdbcConnectionUrl, exclusion1, exclusion2 string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -1962,11 +1962,11 @@ resource "aws_glue_crawler" "test" { path = "database-name/table1" } } -`, rName, jdbcConnectionUrl, exclusion1, exclusion2) +`, rName, jdbcConnectionUrl, exclusion1, exclusion2)) } func testAccCrawlerConfig_jdbcTargetMultiple(rName, jdbcConnectionUrl, path1, path2 string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -1998,11 +1998,11 @@ resource "aws_glue_crawler" "test" { path = %[4]q } } -`, rName, jdbcConnectionUrl, path1, path2) +`, rName, jdbcConnectionUrl, path1, path2)) } func testAccCrawlerConfig_roleARNNoPath(rName string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2018,7 +2018,7 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket-name" } } -`, rName) +`, rName)) } func testAccCrawlerConfig_roleARNPath(rName string) string { @@ -2118,7 +2118,7 @@ resource "aws_glue_crawler" "test" { } func testAccCrawlerConfig_s3Target(rName, path string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2134,11 +2134,11 @@ resource "aws_glue_crawler" "test" { path = %[2]q } } -`, rName, path) +`, rName, path)) } func testAccCrawlerConfig_s3TargetExclusions1(rName, exclusion1 string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2155,30 +2155,13 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket1" } } -`, rName, exclusion1) +`, rName, exclusion1)) } func testAccCrawlerConfig_s3TargetConnectionName(rName string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` -data "aws_availability_zones" "available" { - state = "available" - - filter { - name = "opt-in-status" - values = ["opt-in-not-required"] - } -} - -resource "aws_vpc" "test" { - cidr_block = "10.0.0.0/16" - - tags = { - Name = "terraform-testacc-glue-connection-base" - } -} - + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), acctest.ConfigVPCWithSubnets(rName, 2), fmt.Sprintf(` resource "aws_security_group" "test" { - name = "%[1]s" + name = %[1]q vpc_id = aws_vpc.test.id ingress { @@ -2187,17 +2170,9 @@ resource "aws_security_group" "test" { self = true to_port = 65535 } -} - -resource "aws_subnet" "test" { - count = 2 - - availability_zone = data.aws_availability_zones.available.names[count.index] - cidr_block = "10.0.${count.index}.0/24" - vpc_id = aws_vpc.test.id tags = { - Name = "terraform-testacc-glue-connection-base" + Name = %[1]q } } @@ -2233,11 +2208,11 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket1" } } -`, rName) +`, rName)) } func testAccCrawlerConfig_s3TargetExclusions2(rName, exclusion1, exclusion2 string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2254,11 +2229,11 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket1" } } -`, rName, exclusion1, exclusion2) +`, rName, exclusion1, exclusion2)) } func testAccCrawlerConfig_s3TargetEventQueue(rName string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2322,11 +2297,11 @@ resource "aws_glue_crawler" "test" { recrawl_behavior = "CRAWL_EVENT_MODE" } } -`, rName) +`, rName)) } func testAccCrawlerConfig_catalogTargetDlqEventQueue(rName string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_sqs_queue" "test" { name = %[1]q @@ -2424,11 +2399,11 @@ resource "aws_glue_crawler" "test" { } EOF } -`, rName) +`, rName)) } func testAccCrawlerConfig_s3TargetDlqEventQueue(rName string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2500,11 +2475,11 @@ resource "aws_glue_crawler" "test" { recrawl_behavior = "CRAWL_EVENT_MODE" } } -`, rName) +`, rName)) } func testAccCrawlerConfig_s3TargetMultiple(rName, path1, path2 string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2524,11 +2499,11 @@ resource "aws_glue_crawler" "test" { path = %[3]q } } -`, rName, path1, path2) +`, rName, path1, path2)) } func testAccCrawlerConfig_catalogTarget(rName string, tableCount int) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2587,11 +2562,11 @@ resource "aws_glue_crawler" "test" { } EOF } -`, rName, tableCount) +`, rName, tableCount)) } func testAccCrawlerConfig_catalogTargetMultiple(rName string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { count = 2 name = "%[1]s_database_${count.index}" @@ -2655,11 +2630,11 @@ resource "aws_glue_crawler" "test" { } EOF } -`, rName) +`, rName)) } func testAccCrawlerConfig_schedule(rName, schedule string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2676,11 +2651,11 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket-name" } } -`, rName, schedule) +`, rName, schedule)) } func testAccCrawlerConfig_schemaChangePolicy(rName, deleteBehavior, updateBehavior string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2701,11 +2676,11 @@ resource "aws_glue_crawler" "test" { update_behavior = %[3]q } } -`, rName, deleteBehavior, updateBehavior) +`, rName, deleteBehavior, updateBehavior)) } func testAccCrawlerConfig_tablePrefix(rName, tablePrefix string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2722,11 +2697,11 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket-name" } } -`, rName, tablePrefix) +`, rName, tablePrefix)) } func testAccCrawlerConfig_tags1(rName, tagKey1, tagValue1 string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2747,11 +2722,11 @@ resource "aws_glue_crawler" "test" { %[2]q = %[3]q } } -`, rName, tagKey1, tagValue1) +`, rName, tagKey1, tagValue1)) } func testAccCrawlerConfig_tags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2773,11 +2748,11 @@ resource "aws_glue_crawler" "test" { %[4]q = %[5]q } } -`, rName, tagKey1, tagValue1, tagKey2, tagValue2) +`, rName, tagKey1, tagValue1, tagKey2, tagValue2)) } func testAccCrawlerConfig_securityConfiguration(rName, securityConfiguration string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2812,11 +2787,11 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket-name" } } -`, rName, securityConfiguration) +`, rName, securityConfiguration)) } func testAccCrawlerConfig_mongoDBTarget(rName, connectionUrl, path string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2844,11 +2819,11 @@ resource "aws_glue_crawler" "test" { path = %[3]q } } -`, rName, connectionUrl, path) +`, rName, connectionUrl, path)) } func testAccCrawlerConfig_mongoDBTargetScanAll(rName, connectionUrl string, scan bool) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2877,11 +2852,11 @@ resource "aws_glue_crawler" "test" { scan_all = %[3]t } } -`, rName, connectionUrl, scan) +`, rName, connectionUrl, scan)) } func testAccCrawlerConfig_mongoDBMultiple(rName, connectionUrl, path1, path2 string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2914,11 +2889,11 @@ resource "aws_glue_crawler" "test" { path = %[4]q } } -`, rName, connectionUrl, path1, path2) +`, rName, connectionUrl, path1, path2)) } func testAccCrawlerConfig_deltaTarget(rName, connectionUrl, tableName string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2947,11 +2922,11 @@ resource "aws_glue_crawler" "test" { write_manifest = false } } -`, rName, connectionUrl, tableName) +`, rName, connectionUrl, tableName)) } func testAccCrawlerConfig_lakeformation(rName string, use bool) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2971,11 +2946,11 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket-name" } } -`, rName, use) +`, rName, use)) } func testAccCrawlerConfig_lineage(rName, lineageConfig string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -2995,11 +2970,11 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket-name" } } -`, rName, lineageConfig) +`, rName, lineageConfig)) } func testAccCrawlerConfig_recrawlPolicy(rName, policy string) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -3024,11 +2999,11 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket-name" } } -`, rName, policy) +`, rName, policy)) } func testAccCrawlerConfig_s3TargetSampleSize(rName string, size int) string { - return testAccCrawlerConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccCrawlerConfig_base(rName), fmt.Sprintf(` resource "aws_glue_catalog_database" "test" { name = %[1]q } @@ -3045,5 +3020,5 @@ resource "aws_glue_crawler" "test" { path = "s3://bucket1" } } -`, rName, size) +`, rName, size)) }