From 942b3450f1a3a297f220a1202359643a31b9aac3 Mon Sep 17 00:00:00 2001 From: amanenk Date: Wed, 3 Aug 2022 09:33:48 +0300 Subject: [PATCH 1/9] feat: Added glue table partitions and indexes --- client/mocks/glue.go | 40 +++ client/resolvers.go | 28 ++ client/services.go | 2 + .../tables/aws_glue_database_table_indexes.md | 11 + ...le_partition_storage_descriptor_columns.md | 11 + .../aws_glue_database_table_partitions.md | 36 +++ resources/services/glue/databases.go | 292 +++++++++++++++++- resources/services/glue/databases.hcl | 52 ++++ .../services/glue/databases_mock_test.go | 10 + 9 files changed, 478 insertions(+), 4 deletions(-) create mode 100644 docs/tables/aws_glue_database_table_indexes.md create mode 100644 docs/tables/aws_glue_database_table_partition_storage_descriptor_columns.md create mode 100644 docs/tables/aws_glue_database_table_partitions.md diff --git a/client/mocks/glue.go b/client/mocks/glue.go index 63729991a..a7114fcfb 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -95,6 +95,46 @@ func (mr *MockGlueClientMockRecorder) GetJobs(arg0, arg1 interface{}, arg2 ...in return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetJobs", reflect.TypeOf((*MockGlueClient)(nil).GetJobs), varargs...) } +// GetPartitionIndexes mocks base method. +func (m *MockGlueClient) GetPartitionIndexes(arg0 context.Context, arg1 *glue.GetPartitionIndexesInput, arg2 ...func(*glue.Options)) (*glue.GetPartitionIndexesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetPartitionIndexes", varargs...) + ret0, _ := ret[0].(*glue.GetPartitionIndexesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPartitionIndexes indicates an expected call of GetPartitionIndexes. +func (mr *MockGlueClientMockRecorder) GetPartitionIndexes(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPartitionIndexes", reflect.TypeOf((*MockGlueClient)(nil).GetPartitionIndexes), varargs...) +} + +// GetPartitions mocks base method. +func (m *MockGlueClient) GetPartitions(arg0 context.Context, arg1 *glue.GetPartitionsInput, arg2 ...func(*glue.Options)) (*glue.GetPartitionsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetPartitions", varargs...) + ret0, _ := ret[0].(*glue.GetPartitionsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPartitions indicates an expected call of GetPartitions. +func (mr *MockGlueClientMockRecorder) GetPartitions(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPartitions", reflect.TypeOf((*MockGlueClient)(nil).GetPartitions), varargs...) +} + // GetTables mocks base method. func (m *MockGlueClient) GetTables(arg0 context.Context, arg1 *glue.GetTablesInput, arg2 ...func(*glue.Options)) (*glue.GetTablesOutput, error) { m.ctrl.T.Helper() diff --git a/client/resolvers.go b/client/resolvers.go index 40700af0f..73db5e35e 100644 --- a/client/resolvers.go +++ b/client/resolvers.go @@ -2,6 +2,7 @@ package client import ( "context" + "fmt" "reflect" "time" @@ -82,3 +83,30 @@ func ResolveTimestampField(path string, rfcs ...string) func(_ context.Context, } } } + +func SliceJsonResolver(path, keyPath, valuePath string) schema.ColumnResolver { + return func(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error { + j := make(map[string]interface{}) + field := funk.Get(r.Item, path, funk.WithAllowZero()) + s := reflect.ValueOf(field) + if s.IsNil() { + return nil + } + if reflect.TypeOf(field).Kind() != reflect.Slice { + return diag.WrapError(fmt.Errorf("field: %s is not a slice", path)) + } + for i := 0; i < s.Len(); i++ { + key := funk.Get(s.Index(i).Interface(), keyPath, funk.WithAllowZero()) + value := funk.Get(s.Index(i).Interface(), valuePath, funk.WithAllowZero()) + k := reflect.ValueOf(key) + if k.Kind() == reflect.Ptr { + k = k.Elem() + } + if k.Kind() != reflect.String { + return diag.WrapError(fmt.Errorf("key field: %s is not a string", path)) + } + j[k.String()] = value + } + return r.Set(c.Name, j) + } +} diff --git a/client/services.go b/client/services.go index 2b1b67a09..6196fc174 100644 --- a/client/services.go +++ b/client/services.go @@ -757,6 +757,8 @@ type GlueClient interface { GetJobRuns(ctx context.Context, params *glue.GetJobRunsInput, optFns ...func(*glue.Options)) (*glue.GetJobRunsOutput, error) GetDatabases(ctx context.Context, params *glue.GetDatabasesInput, optFns ...func(*glue.Options)) (*glue.GetDatabasesOutput, error) GetTables(ctx context.Context, params *glue.GetTablesInput, optFns ...func(*glue.Options)) (*glue.GetTablesOutput, error) + GetPartitions(ctx context.Context, params *glue.GetPartitionsInput, optFns ...func(*glue.Options)) (*glue.GetPartitionsOutput, error) + GetPartitionIndexes(ctx context.Context, params *glue.GetPartitionIndexesInput, optFns ...func(*glue.Options)) (*glue.GetPartitionIndexesOutput, error) } //go:generate mockgen -package=mocks -destination=./mocks/kinesis.go . KinesisClient diff --git a/docs/tables/aws_glue_database_table_indexes.md b/docs/tables/aws_glue_database_table_indexes.md new file mode 100644 index 000000000..f183cf5b1 --- /dev/null +++ b/docs/tables/aws_glue_database_table_indexes.md @@ -0,0 +1,11 @@ + +# Table: aws_glue_database_table_indexes +A descriptor for a partition index in a table +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|database_table_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_tables table (FK)| +|index_name|text|The name of the partition index| +|index_status|text|The status of the partition index| +|keys|jsonb|A list of one or more keys, as KeySchemaElement structures, for the partition index| +|backfill_errors|jsonb|A list of errors that can occur when registering partition indexes for an existing table| diff --git a/docs/tables/aws_glue_database_table_partition_storage_descriptor_columns.md b/docs/tables/aws_glue_database_table_partition_storage_descriptor_columns.md new file mode 100644 index 000000000..34ec18a9a --- /dev/null +++ b/docs/tables/aws_glue_database_table_partition_storage_descriptor_columns.md @@ -0,0 +1,11 @@ + +# Table: aws_glue_database_table_partition_storage_descriptor_columns +A column in a Table +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|database_table_partition_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_table_partitions table (FK)| +|name|text|The name of the Column| +|comment|text|A free-form text comment| +|parameters|jsonb|These key-value pairs define properties associated with the column| +|type|text|The data type of the Column| diff --git a/docs/tables/aws_glue_database_table_partitions.md b/docs/tables/aws_glue_database_table_partitions.md new file mode 100644 index 000000000..8317c54d7 --- /dev/null +++ b/docs/tables/aws_glue_database_table_partitions.md @@ -0,0 +1,36 @@ + +# Table: aws_glue_database_table_partitions +Represents a slice of table data +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|database_table_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_tables table (FK)| +|account_id|text|The AWS Account ID of the resource.| +|region|text|The AWS Region of the resource.| +|catalog_id|text|The ID of the Data Catalog in which the partition resides| +|creation_time|timestamp without time zone|The time at which the partition was created| +|database_name|text|The name of the catalog database in which to create the partition| +|last_access_time|timestamp without time zone|The last time at which the partition was accessed| +|last_analyzed_time|timestamp without time zone|The last time at which column statistics were computed for this partition| +|parameters|jsonb|These key-value pairs define partition parameters| +|storage_descriptor_additional_locations|text[]|A list of locations that point to the path where a Delta table is located| +|storage_descriptor_bucket_columns|text[]|A list of reducer grouping columns, clustering columns, and bucketing columns in the table| +|storage_descriptor_compressed|boolean|True if the data in the table is compressed, or False if not| +|storage_descriptor_input_format|text|The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format| +|storage_descriptor_location|text|The physical location of the table| +|storage_descriptor_number_of_buckets|bigint|Must be specified if the table contains any dimension columns| +|storage_descriptor_output_format|text|The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format| +|storage_descriptor_parameters|jsonb|The user-supplied properties in key-value form| +|storage_descriptor_schema_reference_schema_id_registry_name|text|The name of the schema registry that contains the schema| +|storage_descriptor_schema_reference_schema_id_schema_arn|text|The Amazon Resource Name (ARN) of the schema| +|storage_descriptor_schema_reference_schema_id_schema_name|text|The name of the schema| +|storage_descriptor_schema_reference_schema_version_id|text|The unique ID assigned to a version of the schema| +|storage_descriptor_schema_reference_schema_version_number|bigint|The version number of the schema| +|storage_descriptor_serde_info_name|text|Name of the SerDe| +|storage_descriptor_serde_info_parameters|jsonb|These key-value pairs define initialization parameters for the SerDe| +|storage_descriptor_serde_info_serialization_library|text|Usually the class that implements the SerDe| +|storage_descriptor_skewed_info|jsonb|The information about values that appear frequently in a column (skewed values)| +|storage_descriptor_sort_columns|jsonb|A list specifying the sort order of each bucket in the table| +|storage_descriptor_stored_as_sub_directories|boolean|True if the table data is stored in subdirectories, or False if not| +|table_name|text|The name of the database table in which to create the partition| +|values|text[]|The values of the partition| diff --git a/resources/services/glue/databases.go b/resources/services/glue/databases.go index 9abc94c99..c72934a69 100644 --- a/resources/services/glue/databases.go +++ b/resources/services/glue/databases.go @@ -213,7 +213,7 @@ func Databases() *schema.Table { { Name: "aws_glue_database_table_partition_keys", Description: "A column in a Table", - Resolver: fetchGlueDatabaseTablePartitionKeys, + Resolver: schema.PathTableResolver("PartitionKeys"), Columns: []schema.Column{ { Name: "database_table_cq_id", @@ -243,6 +243,257 @@ func Databases() *schema.Table { }, }, }, + { + Name: "aws_glue_database_table_partitions", + Description: "Represents a slice of table data", + Resolver: fetchGlueDatabaseTablePartitions, + Multiplex: client.ServiceAccountRegionMultiplexer("glue"), + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + DeleteFilter: client.DeleteAccountRegionFilter, + Columns: []schema.Column{ + { + Name: "database_table_cq_id", + Description: "Unique CloudQuery ID of aws_glue_database_tables table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "account_id", + Description: "The AWS Account ID of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSAccount, + }, + { + Name: "region", + Description: "The AWS Region of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSRegion, + }, + { + Name: "catalog_id", + Description: "The ID of the Data Catalog in which the partition resides", + Type: schema.TypeString, + }, + { + Name: "creation_time", + Description: "The time at which the partition was created", + Type: schema.TypeTimestamp, + }, + { + Name: "database_name", + Description: "The name of the catalog database in which to create the partition", + Type: schema.TypeString, + }, + { + Name: "last_access_time", + Description: "The last time at which the partition was accessed", + Type: schema.TypeTimestamp, + }, + { + Name: "last_analyzed_time", + Description: "The last time at which column statistics were computed for this partition", + Type: schema.TypeTimestamp, + }, + { + Name: "parameters", + Description: "These key-value pairs define partition parameters", + Type: schema.TypeJSON, + }, + { + Name: "storage_descriptor_additional_locations", + Description: "A list of locations that point to the path where a Delta table is located", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("StorageDescriptor.AdditionalLocations"), + }, + { + Name: "storage_descriptor_bucket_columns", + Description: "A list of reducer grouping columns, clustering columns, and bucketing columns in the table", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("StorageDescriptor.BucketColumns"), + }, + { + Name: "storage_descriptor_compressed", + Description: "True if the data in the table is compressed, or False if not", + Type: schema.TypeBool, + Resolver: schema.PathResolver("StorageDescriptor.Compressed"), + }, + { + Name: "storage_descriptor_input_format", + Description: "The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.InputFormat"), + }, + { + Name: "storage_descriptor_location", + Description: "The physical location of the table", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.Location"), + }, + { + Name: "storage_descriptor_number_of_buckets", + Description: "Must be specified if the table contains any dimension columns", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("StorageDescriptor.NumberOfBuckets"), + }, + { + Name: "storage_descriptor_output_format", + Description: "The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.OutputFormat"), + }, + { + Name: "storage_descriptor_parameters", + Description: "The user-supplied properties in key-value form", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.Parameters"), + }, + { + Name: "storage_descriptor_schema_reference_schema_id_registry_name", + Description: "The name of the schema registry that contains the schema", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaId.RegistryName"), + }, + { + Name: "storage_descriptor_schema_reference_schema_id_schema_arn", + Description: "The Amazon Resource Name (ARN) of the schema", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaId.SchemaArn"), + }, + { + Name: "storage_descriptor_schema_reference_schema_id_schema_name", + Description: "The name of the schema", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaId.SchemaName"), + }, + { + Name: "storage_descriptor_schema_reference_schema_version_id", + Description: "The unique ID assigned to a version of the schema", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaVersionId"), + }, + { + Name: "storage_descriptor_schema_reference_schema_version_number", + Description: "The version number of the schema", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaVersionNumber"), + }, + { + Name: "storage_descriptor_serde_info_name", + Description: "Name of the SerDe", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.SerdeInfo.Name"), + }, + { + Name: "storage_descriptor_serde_info_parameters", + Description: "These key-value pairs define initialization parameters for the SerDe", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.SerdeInfo.Parameters"), + }, + { + Name: "storage_descriptor_serde_info_serialization_library", + Description: "Usually the class that implements the SerDe", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.SerdeInfo.SerializationLibrary"), + }, + { + Name: "storage_descriptor_skewed_info", + Description: "The information about values that appear frequently in a column (skewed values)", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.SkewedInfo"), + }, + { + Name: "storage_descriptor_sort_columns", + Description: "A list specifying the sort order of each bucket in the table", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.SortColumns"), + }, + { + Name: "storage_descriptor_stored_as_sub_directories", + Description: "True if the table data is stored in subdirectories, or False if not", + Type: schema.TypeBool, + Resolver: schema.PathResolver("StorageDescriptor.StoredAsSubDirectories"), + }, + { + Name: "table_name", + Description: "The name of the database table in which to create the partition", + Type: schema.TypeString, + }, + { + Name: "values", + Description: "The values of the partition", + Type: schema.TypeStringArray, + }, + }, + Relations: []*schema.Table{ + { + Name: "aws_glue_database_table_partition_storage_descriptor_columns", + Description: "A column in a Table", + Resolver: schema.PathTableResolver("StorageDescriptor.Columns"), + Columns: []schema.Column{ + { + Name: "database_table_partition_cq_id", + Description: "Unique CloudQuery ID of aws_glue_database_table_partitions table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "name", + Description: "The name of the Column", + Type: schema.TypeString, + }, + { + Name: "comment", + Description: "A free-form text comment", + Type: schema.TypeString, + }, + { + Name: "parameters", + Description: "These key-value pairs define properties associated with the column", + Type: schema.TypeJSON, + }, + { + Name: "type", + Description: "The data type of the Column", + Type: schema.TypeString, + }, + }, + }, + }, + }, + { + Name: "aws_glue_database_table_indexes", + Description: "A descriptor for a partition index in a table", + Resolver: fetchGlueDatabaseTableIndexes, + Columns: []schema.Column{ + { + Name: "database_table_cq_id", + Description: "Unique CloudQuery ID of aws_glue_database_tables table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "index_name", + Description: "The name of the partition index", + Type: schema.TypeString, + }, + { + Name: "index_status", + Description: "The status of the partition index", + Type: schema.TypeString, + }, + { + Name: "keys", + Description: "A list of one or more keys, as KeySchemaElement structures, for the partition index", + Type: schema.TypeJSON, + Resolver: client.SliceJsonResolver("Keys", "Name", "Type"), + }, + { + Name: "backfill_errors", + Description: "A list of errors that can occur when registering partition indexes for an existing table", + Type: schema.TypeJSON, + }, + }, + }, }, }, }, @@ -295,9 +546,42 @@ func fetchGlueDatabaseTables(ctx context.Context, meta schema.ClientMeta, parent } return nil } -func fetchGlueDatabaseTablePartitionKeys(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { - r := parent.Item.(types.Table) - res <- r.PartitionKeys +func fetchGlueDatabaseTablePartitions(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + cl := meta.(*client.Client) + svc := cl.Services().Glue + d := parent.Parent.Item.(types.Database) + t := parent.Item.(types.Table) + input := glue.GetPartitionsInput{DatabaseName: d.Name, CatalogId: d.CatalogId, TableName: t.Name} + for { + result, err := svc.GetPartitions(ctx, &input) + if err != nil { + return diag.WrapError(err) + } + res <- result.Partitions + if aws.ToString(result.NextToken) == "" { + break + } + input.NextToken = result.NextToken + } + return nil +} +func fetchGlueDatabaseTableIndexes(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + cl := meta.(*client.Client) + svc := cl.Services().Glue + d := parent.Parent.Item.(types.Database) + t := parent.Item.(types.Table) + input := glue.GetPartitionIndexesInput{DatabaseName: d.Name, CatalogId: d.CatalogId, TableName: t.Name} + for { + result, err := svc.GetPartitionIndexes(ctx, &input) + if err != nil { + return diag.WrapError(err) + } + res <- result.PartitionIndexDescriptorList + if aws.ToString(result.NextToken) == "" { + break + } + input.NextToken = result.NextToken + } return nil } diff --git a/resources/services/glue/databases.hcl b/resources/services/glue/databases.hcl index acb1f066e..7f588013d 100644 --- a/resources/services/glue/databases.hcl +++ b/resources/services/glue/databases.hcl @@ -47,6 +47,58 @@ resource "aws" "glue" "databases" { column "storage_descriptor" { type = "json" } + + user_relation "aws" "glue" "partitions" { + path = "github.com/aws/aws-sdk-go-v2/service/glue/types.Partition" + ignoreError "IgnoreAccessDenied" { + path = "github.com/cloudquery/cq-provider-aws/client.IgnoreAccessDeniedServiceDisabled" + } + deleteFilter "AccountRegionFilter" { + path = "github.com/cloudquery/cq-provider-aws/client.DeleteAccountRegionFilter" + } + multiplex "AwsAccountRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ServiceAccountRegionMultiplexer" + params = ["glue"] + } + userDefinedColumn "account_id" { + description = "The AWS Account ID of the resource." + type = "string" + resolver "resolveAWSAccount" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSAccount" + } + } + userDefinedColumn "region" { + type = "string" + description = "The AWS Region of the resource." + resolver "resolveAWSRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSRegion" + } + } + + column "storage_descriptor_skewed_info" { + type = "json" + } + + column "storage_descriptor_sort_columns" { + type = "json" + } + } + + user_relation "aws" "glue" "indexes" { + path = "github.com/aws/aws-sdk-go-v2/service/glue/types.PartitionIndexDescriptor" + + column "keys" { + type = "json" + resolver "resolverSliceToJson" { + path = "github.com/cloudquery/cq-provider-aws/client.SliceJsonResolver" + params = ["Keys", "Name", "Type"] + } + } + + column "backfill_errors" { + type = "json" + } + } } column "target_database_database_name" { diff --git a/resources/services/glue/databases_mock_test.go b/resources/services/glue/databases_mock_test.go index 164d3e35b..956250f0e 100644 --- a/resources/services/glue/databases_mock_test.go +++ b/resources/services/glue/databases_mock_test.go @@ -24,6 +24,16 @@ func buildDatabasesMock(t *testing.T, ctrl *gomock.Controller) client.Services { tb.NextToken = nil m.EXPECT().GetTables(gomock.Any(), gomock.Any()).Return(&tb, nil) + p := glue.GetPartitionsOutput{} + require.NoError(t, faker.FakeData(&p)) + p.NextToken = nil + m.EXPECT().GetPartitions(gomock.Any(), gomock.Any()).Return(&p, nil) + + i := glue.GetPartitionIndexesOutput{} + require.NoError(t, faker.FakeData(&i)) + i.NextToken = nil + m.EXPECT().GetPartitionIndexes(gomock.Any(), gomock.Any()).Return(&i, nil) + return client.Services{ Glue: m, } From e02089154357696c97fc517b9a8afe3a66a1c21c Mon Sep 17 00:00:00 2001 From: amanenk Date: Wed, 3 Aug 2022 09:45:32 +0300 Subject: [PATCH 2/9] add comment --- client/resolvers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/resolvers.go b/client/resolvers.go index 73db5e35e..e8c618391 100644 --- a/client/resolvers.go +++ b/client/resolvers.go @@ -84,6 +84,7 @@ func ResolveTimestampField(path string, rfcs ...string) func(_ context.Context, } } +// SliceJsonResolver resolves slice of objects into a map[string]interface{} func SliceJsonResolver(path, keyPath, valuePath string) schema.ColumnResolver { return func(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error { j := make(map[string]interface{}) From c4640719d3a6effc02a4c93cbdda85ecb0b762b4 Mon Sep 17 00:00:00 2001 From: amanenk Date: Thu, 4 Aug 2022 16:57:46 +0300 Subject: [PATCH 3/9] added resolver tests and docs --- client/resolvers.go | 22 ++++++++-- client/resolvers_test.go | 87 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) diff --git a/client/resolvers.go b/client/resolvers.go index e8c618391..b729e50f3 100644 --- a/client/resolvers.go +++ b/client/resolvers.go @@ -84,15 +84,31 @@ func ResolveTimestampField(path string, rfcs ...string) func(_ context.Context, } } -// SliceJsonResolver resolves slice of objects into a map[string]interface{} +/* +SliceJsonResolver resolves slice of objects into a map[string]interface{}. +For example object: SliceJsonStruct{Nested: &SliceJsonStruct{ + Nested: &SliceJsonStruct{ + Value: []types1.Tag{{ + Key: "k1", + Value: "v1", + }, { + Key: "k2", + Value: "v2", + }}, + }, + }} +can be converted to map[string]interface{}{"k1":"v1","k2":"v2"} by setting a resolver with next params: +SliceJsonResolver("Nested.Nested.Value", "Key", "Value") +*/ func SliceJsonResolver(path, keyPath, valuePath string) schema.ColumnResolver { return func(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error { - j := make(map[string]interface{}) + var j map[string]interface{} field := funk.Get(r.Item, path, funk.WithAllowZero()) s := reflect.ValueOf(field) if s.IsNil() { - return nil + return r.Set(c.Name, j) } + j = make(map[string]interface{}) if reflect.TypeOf(field).Kind() != reflect.Slice { return diag.WrapError(fmt.Errorf("field: %s is not a slice", path)) } diff --git a/client/resolvers_test.go b/client/resolvers_test.go index 19631277c..0382806ff 100644 --- a/client/resolvers_test.go +++ b/client/resolvers_test.go @@ -62,3 +62,90 @@ func TestResolveTags(t *testing.T) { assert.Equal(t, tc.ExpectedTags, r.Get(ta.Columns[0].Name)) } } + +type ( + SliceJsonStruct struct { + Value []types1.Tag + Nested *SliceJsonStruct + } +) + +func TestResolveSliceJson(t *testing.T) { + cases := []struct { + InputItem interface{} + ExpectedData map[string]interface{} + path string + keyPath string + valuePath string + }{ + { + InputItem: types1.ListWebhookItem{ // non-ptr + Tags: []types1.Tag{ + { + Key: aws.String("k1"), + Value: aws.String("v1"), + }, + }, + }, + ExpectedData: map[string]interface{}{"k1": aws.String("v1")}, + path: "Tags", + keyPath: "Key", + valuePath: "Value", + }, + { + InputItem: &types2.EventSubscription{ // ptr + Tags: []types2.Tag{ + { + Key: aws.String("k2"), + Value: aws.String("v2"), + }, + }, + }, + ExpectedData: map[string]interface{}{"k2": aws.String("v2")}, + path: "Tags", + keyPath: "Key", + valuePath: "Value", + }, + { + InputItem: SliceJsonStruct{Nested: &SliceJsonStruct{ + Nested: &SliceJsonStruct{ + Value: []types1.Tag{{ + Key: aws.String("k1"), + Value: aws.String("v1"), + }, { + Key: aws.String("k2"), + Value: aws.String("v2"), + }}, + }, + }}, + ExpectedData: map[string]interface{}{"k1": aws.String("v1"), "k2": aws.String("v2")}, + path: "Nested.Nested.Value", + keyPath: "Key", + valuePath: "Value", + }, + { + InputItem: types1.ListWebhookItem{ // non-ptr, nil + Tags: nil, + }, + ExpectedData: nil, + path: "Tags", + keyPath: "Key", + valuePath: "Value", + }, + } + + for _, tc := range cases { + ta := &schema.Table{ + Columns: []schema.Column{ + { + Name: "tags", + Type: schema.TypeJSON, + }, + }, + } + r := schema.NewResourceData(schema.PostgresDialect{}, ta, nil, tc.InputItem, nil, time.Now()) + err := SliceJsonResolver(tc.path, tc.keyPath, tc.valuePath)(context.Background(), nil, r, ta.Columns[0]) + assert.NoError(t, err) + assert.Equal(t, tc.ExpectedData, r.Get(ta.Columns[0].Name)) + } +} From 22751592dfc38fe25c368a8f4d5b8782c879936e Mon Sep 17 00:00:00 2001 From: amanenk Date: Thu, 4 Aug 2022 19:07:33 +0300 Subject: [PATCH 4/9] structure adjusted --- client/mocks/glue.go | 40 ++++ .../tables/aws_glue_database_table_columns.md | 11 + ..._glue_database_table_partition_columns.md} | 2 +- .../aws_glue_database_table_partitions.md | 38 ++-- docs/tables/aws_glue_database_tables.md | 20 +- resources/services/glue/databases.go | 188 +++++++++++++----- resources/services/glue/databases.hcl | 32 ++- 7 files changed, 261 insertions(+), 70 deletions(-) create mode 100644 docs/tables/aws_glue_database_table_columns.md rename docs/tables/{aws_glue_database_table_partition_storage_descriptor_columns.md => aws_glue_database_table_partition_columns.md} (86%) diff --git a/client/mocks/glue.go b/client/mocks/glue.go index 450ee3dea..35a3edb5e 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -195,6 +195,46 @@ func (mr *MockGlueClientMockRecorder) GetMLTransforms(arg0, arg1 interface{}, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMLTransforms", reflect.TypeOf((*MockGlueClient)(nil).GetMLTransforms), varargs...) } +// GetPartitionIndexes mocks base method. +func (m *MockGlueClient) GetPartitionIndexes(arg0 context.Context, arg1 *glue.GetPartitionIndexesInput, arg2 ...func(*glue.Options)) (*glue.GetPartitionIndexesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetPartitionIndexes", varargs...) + ret0, _ := ret[0].(*glue.GetPartitionIndexesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPartitionIndexes indicates an expected call of GetPartitionIndexes. +func (mr *MockGlueClientMockRecorder) GetPartitionIndexes(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPartitionIndexes", reflect.TypeOf((*MockGlueClient)(nil).GetPartitionIndexes), varargs...) +} + +// GetPartitions mocks base method. +func (m *MockGlueClient) GetPartitions(arg0 context.Context, arg1 *glue.GetPartitionsInput, arg2 ...func(*glue.Options)) (*glue.GetPartitionsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetPartitions", varargs...) + ret0, _ := ret[0].(*glue.GetPartitionsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPartitions indicates an expected call of GetPartitions. +func (mr *MockGlueClientMockRecorder) GetPartitions(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPartitions", reflect.TypeOf((*MockGlueClient)(nil).GetPartitions), varargs...) +} + // GetSecurityConfigurations mocks base method. func (m *MockGlueClient) GetSecurityConfigurations(arg0 context.Context, arg1 *glue.GetSecurityConfigurationsInput, arg2 ...func(*glue.Options)) (*glue.GetSecurityConfigurationsOutput, error) { m.ctrl.T.Helper() diff --git a/docs/tables/aws_glue_database_table_columns.md b/docs/tables/aws_glue_database_table_columns.md new file mode 100644 index 000000000..d499c368a --- /dev/null +++ b/docs/tables/aws_glue_database_table_columns.md @@ -0,0 +1,11 @@ + +# Table: aws_glue_database_table_columns +A column in a Table +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|database_table_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_tables table (FK)| +|name|text|The name of the Column| +|comment|text|A free-form text comment| +|parameters|jsonb|These key-value pairs define properties associated with the column| +|type|text|The data type of the Column| diff --git a/docs/tables/aws_glue_database_table_partition_storage_descriptor_columns.md b/docs/tables/aws_glue_database_table_partition_columns.md similarity index 86% rename from docs/tables/aws_glue_database_table_partition_storage_descriptor_columns.md rename to docs/tables/aws_glue_database_table_partition_columns.md index 34ec18a9a..ebdd9cdd4 100644 --- a/docs/tables/aws_glue_database_table_partition_storage_descriptor_columns.md +++ b/docs/tables/aws_glue_database_table_partition_columns.md @@ -1,5 +1,5 @@ -# Table: aws_glue_database_table_partition_storage_descriptor_columns +# Table: aws_glue_database_table_partition_columns A column in a Table ## Columns | Name | Type | Description | diff --git a/docs/tables/aws_glue_database_table_partitions.md b/docs/tables/aws_glue_database_table_partitions.md index 8317c54d7..764150052 100644 --- a/docs/tables/aws_glue_database_table_partitions.md +++ b/docs/tables/aws_glue_database_table_partitions.md @@ -13,24 +13,24 @@ Represents a slice of table data |last_access_time|timestamp without time zone|The last time at which the partition was accessed| |last_analyzed_time|timestamp without time zone|The last time at which column statistics were computed for this partition| |parameters|jsonb|These key-value pairs define partition parameters| -|storage_descriptor_additional_locations|text[]|A list of locations that point to the path where a Delta table is located| -|storage_descriptor_bucket_columns|text[]|A list of reducer grouping columns, clustering columns, and bucketing columns in the table| -|storage_descriptor_compressed|boolean|True if the data in the table is compressed, or False if not| -|storage_descriptor_input_format|text|The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format| -|storage_descriptor_location|text|The physical location of the table| -|storage_descriptor_number_of_buckets|bigint|Must be specified if the table contains any dimension columns| -|storage_descriptor_output_format|text|The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format| -|storage_descriptor_parameters|jsonb|The user-supplied properties in key-value form| -|storage_descriptor_schema_reference_schema_id_registry_name|text|The name of the schema registry that contains the schema| -|storage_descriptor_schema_reference_schema_id_schema_arn|text|The Amazon Resource Name (ARN) of the schema| -|storage_descriptor_schema_reference_schema_id_schema_name|text|The name of the schema| -|storage_descriptor_schema_reference_schema_version_id|text|The unique ID assigned to a version of the schema| -|storage_descriptor_schema_reference_schema_version_number|bigint|The version number of the schema| -|storage_descriptor_serde_info_name|text|Name of the SerDe| -|storage_descriptor_serde_info_parameters|jsonb|These key-value pairs define initialization parameters for the SerDe| -|storage_descriptor_serde_info_serialization_library|text|Usually the class that implements the SerDe| -|storage_descriptor_skewed_info|jsonb|The information about values that appear frequently in a column (skewed values)| -|storage_descriptor_sort_columns|jsonb|A list specifying the sort order of each bucket in the table| -|storage_descriptor_stored_as_sub_directories|boolean|True if the table data is stored in subdirectories, or False if not| +|additional_locations|text[]|A list of locations that point to the path where a Delta table is located| +|bucket_columns|text[]|A list of reducer grouping columns, clustering columns, and bucketing columns in the table| +|compressed|boolean|True if the data in the table is compressed, or False if not| +|input_format|text|The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format| +|location|text|The physical location of the table| +|number_of_buckets|bigint|Must be specified if the table contains any dimension columns| +|output_format|text|The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format| +|parameters|jsonb|The user-supplied properties in key-value form| +|schema_reference_schema_id_registry_name|text|The name of the schema registry that contains the schema| +|schema_reference_schema_id_schema_arn|text|The Amazon Resource Name (ARN) of the schema| +|schema_reference_schema_id_schema_name|text|The name of the schema| +|schema_reference_schema_version_id|text|The unique ID assigned to a version of the schema| +|schema_reference_schema_version_number|bigint|The version number of the schema| +|serde_info_name|text|Name of the SerDe| +|serde_info_parameters|jsonb|These key-value pairs define initialization parameters for the SerDe| +|serde_info_serialization_library|text|Usually the class that implements the SerDe| +|skewed_info|jsonb|The information about values that appear frequently in a column (skewed values)| +|sort_columns|jsonb|A list specifying the sort order of each bucket in the table| +|stored_as_sub_directories|boolean|True if the table data is stored in subdirectories, or False if not| |table_name|text|The name of the database table in which to create the partition| |values|text[]|The values of the partition| diff --git a/docs/tables/aws_glue_database_tables.md b/docs/tables/aws_glue_database_tables.md index e218c65b4..b527b86fe 100644 --- a/docs/tables/aws_glue_database_tables.md +++ b/docs/tables/aws_glue_database_tables.md @@ -17,7 +17,25 @@ Represents a collection of related data organized in columns and rows |owner|text|The owner of the table| |parameters|jsonb|These key-value pairs define properties associated with the table| |retention|bigint|The retention time for this table| -|storage_descriptor|jsonb|A storage descriptor containing information about the physical storage of this table| +|additional_locations|text[]|A list of locations that point to the path where a Delta table is located| +|bucket_columns|text[]|A list of reducer grouping columns, clustering columns, and bucketing columns in the table| +|compressed|boolean|True if the data in the table is compressed, or False if not| +|input_format|text|The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format| +|location|text|The physical location of the table| +|number_of_buckets|bigint|Must be specified if the table contains any dimension columns| +|output_format|text|The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format| +|parameters|jsonb|The user-supplied properties in key-value form| +|schema_reference_schema_id_registry_name|text|The name of the schema registry that contains the schema| +|schema_reference_schema_id_schema_arn|text|The Amazon Resource Name (ARN) of the schema| +|schema_reference_schema_id_schema_name|text|The name of the schema| +|schema_reference_schema_version_id|text|The unique ID assigned to a version of the schema| +|schema_reference_schema_version_number|bigint|The version number of the schema| +|serde_info_name|text|Name of the SerDe| +|serde_info_parameters|jsonb|These key-value pairs define initialization parameters for the SerDe| +|serde_info_serialization_library|text|Usually the class that implements the SerDe| +|skewed_info|jsonb|The information about values that appear frequently in a column (skewed values)| +|sort_columns|jsonb|A list specifying the sort order of each bucket in the table| +|stored_as_sub_directories|boolean|True if the table data is stored in subdirectories, or False if not| |table_type|text|The type of this table (EXTERNAL_TABLE, VIRTUAL_VIEW, etc)| |target_table_catalog_id|text|The ID of the Data Catalog in which the table resides| |target_table_database_name|text|The name of the catalog database that contains the target table| diff --git a/resources/services/glue/databases.go b/resources/services/glue/databases.go index c72934a69..de4f5f987 100644 --- a/resources/services/glue/databases.go +++ b/resources/services/glue/databases.go @@ -161,9 +161,94 @@ func Databases() *schema.Table { Type: schema.TypeBigInt, }, { - Name: "storage_descriptor", - Description: "A storage descriptor containing information about the physical storage of this table", + Name: "additional_locations", + Description: "A list of locations that point to the path where a Delta table is located", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("StorageDescriptor.AdditionalLocations"), + }, + { + Name: "bucket_columns", + Description: "A list of reducer grouping columns, clustering columns, and bucketing columns in the table", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("StorageDescriptor.BucketColumns"), + }, + { + Name: "compressed", + Description: "True if the data in the table is compressed, or False if not", + Type: schema.TypeBool, + Resolver: schema.PathResolver("StorageDescriptor.Compressed"), + }, + { + Name: "input_format", + Description: "The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.InputFormat"), + }, + { + Name: "location", + Description: "The physical location of the table", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.Location"), + }, + { + Name: "number_of_buckets", + Description: "Must be specified if the table contains any dimension columns", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("StorageDescriptor.NumberOfBuckets"), + }, + { + Name: "output_format", + Description: "The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.OutputFormat"), + }, + { + Name: "parameters", + Description: "The user-supplied properties in key-value form", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.Parameters"), + }, + { + Name: "schema_reference_schema_id", + Description: "A structure that contains schema identity fields", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaId"), + }, + { + Name: "schema_reference_schema_version_id", + Description: "The unique ID assigned to a version of the schema", + Type: schema.TypeString, + Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaVersionId"), + }, + { + Name: "schema_reference_schema_version_number", + Description: "The version number of the schema", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaVersionNumber"), + }, + { + Name: "serde_info", + Description: "The serialization/deserialization (SerDe) information", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.SerdeInfo"), + }, + { + Name: "skewed_info", + Description: "The information about values that appear frequently in a column (skewed values)", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.SkewedInfo"), + }, + { + Name: "sort_columns", + Description: "A list specifying the sort order of each bucket in the table", Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.SortColumns"), + }, + { + Name: "stored_as_sub_directories", + Description: "True if the table data is stored in subdirectories, or False if not", + Type: schema.TypeBool, + Resolver: schema.PathResolver("StorageDescriptor.StoredAsSubDirectories"), }, { Name: "table_type", @@ -243,6 +328,39 @@ func Databases() *schema.Table { }, }, }, + { + Name: "aws_glue_database_table_columns", + Description: "A column in a Table", + Resolver: schema.PathTableResolver("StorageDescriptor.Columns"), + Columns: []schema.Column{ + { + Name: "database_table_cq_id", + Description: "Unique CloudQuery ID of aws_glue_database_tables table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "name", + Description: "The name of the Column", + Type: schema.TypeString, + }, + { + Name: "comment", + Description: "A free-form text comment", + Type: schema.TypeString, + }, + { + Name: "parameters", + Description: "These key-value pairs define properties associated with the column", + Type: schema.TypeJSON, + }, + { + Name: "type", + Description: "The data type of the Column", + Type: schema.TypeString, + }, + }, + }, { Name: "aws_glue_database_table_partitions", Description: "Represents a slice of table data", @@ -300,115 +418,91 @@ func Databases() *schema.Table { Type: schema.TypeJSON, }, { - Name: "storage_descriptor_additional_locations", + Name: "additional_locations", Description: "A list of locations that point to the path where a Delta table is located", Type: schema.TypeStringArray, Resolver: schema.PathResolver("StorageDescriptor.AdditionalLocations"), }, { - Name: "storage_descriptor_bucket_columns", + Name: "bucket_columns", Description: "A list of reducer grouping columns, clustering columns, and bucketing columns in the table", Type: schema.TypeStringArray, Resolver: schema.PathResolver("StorageDescriptor.BucketColumns"), }, { - Name: "storage_descriptor_compressed", + Name: "compressed", Description: "True if the data in the table is compressed, or False if not", Type: schema.TypeBool, Resolver: schema.PathResolver("StorageDescriptor.Compressed"), }, { - Name: "storage_descriptor_input_format", + Name: "input_format", Description: "The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format", Type: schema.TypeString, Resolver: schema.PathResolver("StorageDescriptor.InputFormat"), }, { - Name: "storage_descriptor_location", + Name: "location", Description: "The physical location of the table", Type: schema.TypeString, Resolver: schema.PathResolver("StorageDescriptor.Location"), }, { - Name: "storage_descriptor_number_of_buckets", + Name: "number_of_buckets", Description: "Must be specified if the table contains any dimension columns", Type: schema.TypeBigInt, Resolver: schema.PathResolver("StorageDescriptor.NumberOfBuckets"), }, { - Name: "storage_descriptor_output_format", + Name: "output_format", Description: "The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format", Type: schema.TypeString, Resolver: schema.PathResolver("StorageDescriptor.OutputFormat"), }, { - Name: "storage_descriptor_parameters", + Name: "parameters", Description: "The user-supplied properties in key-value form", Type: schema.TypeJSON, Resolver: schema.PathResolver("StorageDescriptor.Parameters"), }, { - Name: "storage_descriptor_schema_reference_schema_id_registry_name", - Description: "The name of the schema registry that contains the schema", - Type: schema.TypeString, - Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaId.RegistryName"), - }, - { - Name: "storage_descriptor_schema_reference_schema_id_schema_arn", - Description: "The Amazon Resource Name (ARN) of the schema", - Type: schema.TypeString, - Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaId.SchemaArn"), - }, - { - Name: "storage_descriptor_schema_reference_schema_id_schema_name", - Description: "The name of the schema", - Type: schema.TypeString, - Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaId.SchemaName"), + Name: "schema_reference_schema_id", + Description: "A structure that contains schema identity fields", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaId"), }, { - Name: "storage_descriptor_schema_reference_schema_version_id", + Name: "schema_reference_schema_version_id", Description: "The unique ID assigned to a version of the schema", Type: schema.TypeString, Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaVersionId"), }, { - Name: "storage_descriptor_schema_reference_schema_version_number", + Name: "schema_reference_schema_version_number", Description: "The version number of the schema", Type: schema.TypeBigInt, Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaVersionNumber"), }, { - Name: "storage_descriptor_serde_info_name", - Description: "Name of the SerDe", - Type: schema.TypeString, - Resolver: schema.PathResolver("StorageDescriptor.SerdeInfo.Name"), - }, - { - Name: "storage_descriptor_serde_info_parameters", - Description: "These key-value pairs define initialization parameters for the SerDe", + Name: "serde_info", + Description: "The serialization/deserialization (SerDe) information", Type: schema.TypeJSON, - Resolver: schema.PathResolver("StorageDescriptor.SerdeInfo.Parameters"), - }, - { - Name: "storage_descriptor_serde_info_serialization_library", - Description: "Usually the class that implements the SerDe", - Type: schema.TypeString, - Resolver: schema.PathResolver("StorageDescriptor.SerdeInfo.SerializationLibrary"), + Resolver: schema.PathResolver("StorageDescriptor.SerdeInfo"), }, { - Name: "storage_descriptor_skewed_info", + Name: "skewed_info", Description: "The information about values that appear frequently in a column (skewed values)", Type: schema.TypeJSON, Resolver: schema.PathResolver("StorageDescriptor.SkewedInfo"), }, { - Name: "storage_descriptor_sort_columns", + Name: "sort_columns", Description: "A list specifying the sort order of each bucket in the table", Type: schema.TypeJSON, Resolver: schema.PathResolver("StorageDescriptor.SortColumns"), }, { - Name: "storage_descriptor_stored_as_sub_directories", + Name: "stored_as_sub_directories", Description: "True if the table data is stored in subdirectories, or False if not", Type: schema.TypeBool, Resolver: schema.PathResolver("StorageDescriptor.StoredAsSubDirectories"), @@ -426,7 +520,7 @@ func Databases() *schema.Table { }, Relations: []*schema.Table{ { - Name: "aws_glue_database_table_partition_storage_descriptor_columns", + Name: "aws_glue_database_table_partition_columns", Description: "A column in a Table", Resolver: schema.PathTableResolver("StorageDescriptor.Columns"), Columns: []schema.Column{ diff --git a/resources/services/glue/databases.hcl b/resources/services/glue/databases.hcl index 7f588013d..fd47075b6 100644 --- a/resources/services/glue/databases.hcl +++ b/resources/services/glue/databases.hcl @@ -45,6 +45,22 @@ resource "aws" "glue" "databases" { path = "github.com/aws/aws-sdk-go-v2/service/glue/types.Table" column "storage_descriptor" { + skip_prefix = true + } + + column "skewed_info" { + type = "json" + } + + column "sort_columns" { + type = "json" + } + + column "schema_reference_schema_id" { + type = "json" + } + + column "serde_info" { type = "json" } @@ -75,11 +91,23 @@ resource "aws" "glue" "databases" { } } - column "storage_descriptor_skewed_info" { + column "storage_descriptor" { + skip_prefix = true + } + + column "skewed_info" { + type = "json" + } + + column "sort_columns" { + type = "json" + } + + column "schema_reference_schema_id" { type = "json" } - column "storage_descriptor_sort_columns" { + column "serde_info" { type = "json" } } From 6285b2add9b47128ff43f59ec11c869a1f38b57f Mon Sep 17 00:00:00 2001 From: amanenk Date: Thu, 4 Aug 2022 19:13:30 +0300 Subject: [PATCH 5/9] linter --- client/resolvers_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/client/resolvers_test.go b/client/resolvers_test.go index 0382806ff..62701c650 100644 --- a/client/resolvers_test.go +++ b/client/resolvers_test.go @@ -12,6 +12,11 @@ import ( "github.com/stretchr/testify/assert" ) +type SliceJsonStruct struct { + Value []types1.Tag + Nested *SliceJsonStruct +} + func TestResolveTags(t *testing.T) { cases := []struct { InputItem interface{} @@ -63,13 +68,6 @@ func TestResolveTags(t *testing.T) { } } -type ( - SliceJsonStruct struct { - Value []types1.Tag - Nested *SliceJsonStruct - } -) - func TestResolveSliceJson(t *testing.T) { cases := []struct { InputItem interface{} From c7903e9a77132a226e43be766d6893cadccd98f2 Mon Sep 17 00:00:00 2001 From: amanenk Date: Thu, 4 Aug 2022 19:39:18 +0300 Subject: [PATCH 6/9] fix --- .../aws_glue_database_table_partitions.md | 10 +---- docs/tables/aws_glue_database_tables.md | 10 +---- resources/services/glue/databases.go | 42 +++++++++---------- resources/services/glue/databases.hcl | 41 ++++++++++++++++++ 4 files changed, 65 insertions(+), 38 deletions(-) diff --git a/docs/tables/aws_glue_database_table_partitions.md b/docs/tables/aws_glue_database_table_partitions.md index 764150052..1bb86c5d9 100644 --- a/docs/tables/aws_glue_database_table_partitions.md +++ b/docs/tables/aws_glue_database_table_partitions.md @@ -12,7 +12,6 @@ Represents a slice of table data |database_name|text|The name of the catalog database in which to create the partition| |last_access_time|timestamp without time zone|The last time at which the partition was accessed| |last_analyzed_time|timestamp without time zone|The last time at which column statistics were computed for this partition| -|parameters|jsonb|These key-value pairs define partition parameters| |additional_locations|text[]|A list of locations that point to the path where a Delta table is located| |bucket_columns|text[]|A list of reducer grouping columns, clustering columns, and bucketing columns in the table| |compressed|boolean|True if the data in the table is compressed, or False if not| @@ -20,15 +19,10 @@ Represents a slice of table data |location|text|The physical location of the table| |number_of_buckets|bigint|Must be specified if the table contains any dimension columns| |output_format|text|The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format| -|parameters|jsonb|The user-supplied properties in key-value form| -|schema_reference_schema_id_registry_name|text|The name of the schema registry that contains the schema| -|schema_reference_schema_id_schema_arn|text|The Amazon Resource Name (ARN) of the schema| -|schema_reference_schema_id_schema_name|text|The name of the schema| +|schema_reference_schema_id|jsonb|A structure that contains schema identity fields| |schema_reference_schema_version_id|text|The unique ID assigned to a version of the schema| |schema_reference_schema_version_number|bigint|The version number of the schema| -|serde_info_name|text|Name of the SerDe| -|serde_info_parameters|jsonb|These key-value pairs define initialization parameters for the SerDe| -|serde_info_serialization_library|text|Usually the class that implements the SerDe| +|serde_info|jsonb|The serialization/deserialization (SerDe) information| |skewed_info|jsonb|The information about values that appear frequently in a column (skewed values)| |sort_columns|jsonb|A list specifying the sort order of each bucket in the table| |stored_as_sub_directories|boolean|True if the table data is stored in subdirectories, or False if not| diff --git a/docs/tables/aws_glue_database_tables.md b/docs/tables/aws_glue_database_tables.md index b527b86fe..0a1b21856 100644 --- a/docs/tables/aws_glue_database_tables.md +++ b/docs/tables/aws_glue_database_tables.md @@ -15,7 +15,6 @@ Represents a collection of related data organized in columns and rows |last_access_time|timestamp without time zone|The last time that the table was accessed| |last_analyzed_time|timestamp without time zone|The last time that column statistics were computed for this table| |owner|text|The owner of the table| -|parameters|jsonb|These key-value pairs define properties associated with the table| |retention|bigint|The retention time for this table| |additional_locations|text[]|A list of locations that point to the path where a Delta table is located| |bucket_columns|text[]|A list of reducer grouping columns, clustering columns, and bucketing columns in the table| @@ -24,15 +23,10 @@ Represents a collection of related data organized in columns and rows |location|text|The physical location of the table| |number_of_buckets|bigint|Must be specified if the table contains any dimension columns| |output_format|text|The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format| -|parameters|jsonb|The user-supplied properties in key-value form| -|schema_reference_schema_id_registry_name|text|The name of the schema registry that contains the schema| -|schema_reference_schema_id_schema_arn|text|The Amazon Resource Name (ARN) of the schema| -|schema_reference_schema_id_schema_name|text|The name of the schema| +|schema_reference_schema_id|jsonb|A structure that contains schema identity fields| |schema_reference_schema_version_id|text|The unique ID assigned to a version of the schema| |schema_reference_schema_version_number|bigint|The version number of the schema| -|serde_info_name|text|Name of the SerDe| -|serde_info_parameters|jsonb|These key-value pairs define initialization parameters for the SerDe| -|serde_info_serialization_library|text|Usually the class that implements the SerDe| +|serde_info|jsonb|The serialization/deserialization (SerDe) information| |skewed_info|jsonb|The information about values that appear frequently in a column (skewed values)| |sort_columns|jsonb|A list specifying the sort order of each bucket in the table| |stored_as_sub_directories|boolean|True if the table data is stored in subdirectories, or False if not| diff --git a/resources/services/glue/databases.go b/resources/services/glue/databases.go index de4f5f987..0ae357f4c 100644 --- a/resources/services/glue/databases.go +++ b/resources/services/glue/databases.go @@ -100,6 +100,16 @@ func Databases() *schema.Table { Type: schema.TypeUUID, Resolver: schema.ParentIdResolver, }, + { + Name: "parameters", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("Parameters"), + }, + { + Name: "storage_parameters", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.Parameters"), + }, { Name: "name", Description: "The table name", @@ -150,11 +160,6 @@ func Databases() *schema.Table { Description: "The owner of the table", Type: schema.TypeString, }, - { - Name: "parameters", - Description: "These key-value pairs define properties associated with the table", - Type: schema.TypeJSON, - }, { Name: "retention", Description: "The retention time for this table", @@ -202,12 +207,6 @@ func Databases() *schema.Table { Type: schema.TypeString, Resolver: schema.PathResolver("StorageDescriptor.OutputFormat"), }, - { - Name: "parameters", - Description: "The user-supplied properties in key-value form", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("StorageDescriptor.Parameters"), - }, { Name: "schema_reference_schema_id", Description: "A structure that contains schema identity fields", @@ -387,6 +386,16 @@ func Databases() *schema.Table { Type: schema.TypeString, Resolver: client.ResolveAWSRegion, }, + { + Name: "parameters", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("Parameters"), + }, + { + Name: "storage_parameters", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("StorageDescriptor.Parameters"), + }, { Name: "catalog_id", Description: "The ID of the Data Catalog in which the partition resides", @@ -412,11 +421,6 @@ func Databases() *schema.Table { Description: "The last time at which column statistics were computed for this partition", Type: schema.TypeTimestamp, }, - { - Name: "parameters", - Description: "These key-value pairs define partition parameters", - Type: schema.TypeJSON, - }, { Name: "additional_locations", Description: "A list of locations that point to the path where a Delta table is located", @@ -459,12 +463,6 @@ func Databases() *schema.Table { Type: schema.TypeString, Resolver: schema.PathResolver("StorageDescriptor.OutputFormat"), }, - { - Name: "parameters", - Description: "The user-supplied properties in key-value form", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("StorageDescriptor.Parameters"), - }, { Name: "schema_reference_schema_id", Description: "A structure that contains schema identity fields", diff --git a/resources/services/glue/databases.hcl b/resources/services/glue/databases.hcl index fd47075b6..24aa251a3 100644 --- a/resources/services/glue/databases.hcl +++ b/resources/services/glue/databases.hcl @@ -64,6 +64,26 @@ resource "aws" "glue" "databases" { type = "json" } + column "parameters" { + skip = true + } + + userDefinedColumn "parameters" { + type = "json" + resolver "resolverSliceToJson" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["Parameters"] + } + } + + userDefinedColumn "storage_parameters" { + type = "json" + resolver "resolverSliceToJson" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["StorageDescriptor.Parameters"] + } + } + user_relation "aws" "glue" "partitions" { path = "github.com/aws/aws-sdk-go-v2/service/glue/types.Partition" ignoreError "IgnoreAccessDenied" { @@ -110,6 +130,27 @@ resource "aws" "glue" "databases" { column "serde_info" { type = "json" } + + column "parameters" { + skip = true + } + + userDefinedColumn "parameters" { + type = "json" + resolver "resolverSliceToJson" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["Parameters"] + } + } + + userDefinedColumn "storage_parameters" { + type = "json" + resolver "resolverSliceToJson" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["StorageDescriptor.Parameters"] + } + } + } user_relation "aws" "glue" "indexes" { From 92dff76369ddfb1d6adc564c13b1032bb38e6f5f Mon Sep 17 00:00:00 2001 From: amanenk Date: Thu, 4 Aug 2022 19:41:03 +0300 Subject: [PATCH 7/9] docs --- docs/tables/aws_glue_database_table_partitions.md | 2 ++ docs/tables/aws_glue_database_tables.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/tables/aws_glue_database_table_partitions.md b/docs/tables/aws_glue_database_table_partitions.md index 1bb86c5d9..807f983ad 100644 --- a/docs/tables/aws_glue_database_table_partitions.md +++ b/docs/tables/aws_glue_database_table_partitions.md @@ -7,6 +7,8 @@ Represents a slice of table data |database_table_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_tables table (FK)| |account_id|text|The AWS Account ID of the resource.| |region|text|The AWS Region of the resource.| +|parameters|jsonb|| +|storage_parameters|jsonb|| |catalog_id|text|The ID of the Data Catalog in which the partition resides| |creation_time|timestamp without time zone|The time at which the partition was created| |database_name|text|The name of the catalog database in which to create the partition| diff --git a/docs/tables/aws_glue_database_tables.md b/docs/tables/aws_glue_database_tables.md index 0a1b21856..2c740dd49 100644 --- a/docs/tables/aws_glue_database_tables.md +++ b/docs/tables/aws_glue_database_tables.md @@ -5,6 +5,8 @@ Represents a collection of related data organized in columns and rows | Name | Type | Description | | ------------- | ------------- | ----- | |database_cq_id|uuid|Unique CloudQuery ID of aws_glue_databases table (FK)| +|parameters|jsonb|| +|storage_parameters|jsonb|| |name|text|The table name| |catalog_id|text|The ID of the Data Catalog in which the table resides| |create_time|timestamp without time zone|The time when the table definition was created in the Data Catalog| From 02b8592854beb34203a36e9513ec4dc34ba4da43 Mon Sep 17 00:00:00 2001 From: amanenk Date: Tue, 9 Aug 2022 11:35:14 +0300 Subject: [PATCH 8/9] removed partitions --- ...s_glue_database_table_partition_columns.md | 11 - .../aws_glue_database_table_partitions.md | 32 --- resources/services/glue/databases.go | 211 ------------------ resources/services/glue/databases.hcl | 69 ------ 4 files changed, 323 deletions(-) delete mode 100644 docs/tables/aws_glue_database_table_partition_columns.md delete mode 100644 docs/tables/aws_glue_database_table_partitions.md diff --git a/docs/tables/aws_glue_database_table_partition_columns.md b/docs/tables/aws_glue_database_table_partition_columns.md deleted file mode 100644 index ebdd9cdd4..000000000 --- a/docs/tables/aws_glue_database_table_partition_columns.md +++ /dev/null @@ -1,11 +0,0 @@ - -# Table: aws_glue_database_table_partition_columns -A column in a Table -## Columns -| Name | Type | Description | -| ------------- | ------------- | ----- | -|database_table_partition_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_table_partitions table (FK)| -|name|text|The name of the Column| -|comment|text|A free-form text comment| -|parameters|jsonb|These key-value pairs define properties associated with the column| -|type|text|The data type of the Column| diff --git a/docs/tables/aws_glue_database_table_partitions.md b/docs/tables/aws_glue_database_table_partitions.md deleted file mode 100644 index 807f983ad..000000000 --- a/docs/tables/aws_glue_database_table_partitions.md +++ /dev/null @@ -1,32 +0,0 @@ - -# Table: aws_glue_database_table_partitions -Represents a slice of table data -## Columns -| Name | Type | Description | -| ------------- | ------------- | ----- | -|database_table_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_tables table (FK)| -|account_id|text|The AWS Account ID of the resource.| -|region|text|The AWS Region of the resource.| -|parameters|jsonb|| -|storage_parameters|jsonb|| -|catalog_id|text|The ID of the Data Catalog in which the partition resides| -|creation_time|timestamp without time zone|The time at which the partition was created| -|database_name|text|The name of the catalog database in which to create the partition| -|last_access_time|timestamp without time zone|The last time at which the partition was accessed| -|last_analyzed_time|timestamp without time zone|The last time at which column statistics were computed for this partition| -|additional_locations|text[]|A list of locations that point to the path where a Delta table is located| -|bucket_columns|text[]|A list of reducer grouping columns, clustering columns, and bucketing columns in the table| -|compressed|boolean|True if the data in the table is compressed, or False if not| -|input_format|text|The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format| -|location|text|The physical location of the table| -|number_of_buckets|bigint|Must be specified if the table contains any dimension columns| -|output_format|text|The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format| -|schema_reference_schema_id|jsonb|A structure that contains schema identity fields| -|schema_reference_schema_version_id|text|The unique ID assigned to a version of the schema| -|schema_reference_schema_version_number|bigint|The version number of the schema| -|serde_info|jsonb|The serialization/deserialization (SerDe) information| -|skewed_info|jsonb|The information about values that appear frequently in a column (skewed values)| -|sort_columns|jsonb|A list specifying the sort order of each bucket in the table| -|stored_as_sub_directories|boolean|True if the table data is stored in subdirectories, or False if not| -|table_name|text|The name of the database table in which to create the partition| -|values|text[]|The values of the partition| diff --git a/resources/services/glue/databases.go b/resources/services/glue/databases.go index 0ae357f4c..9a961abec 100644 --- a/resources/services/glue/databases.go +++ b/resources/services/glue/databases.go @@ -360,198 +360,6 @@ func Databases() *schema.Table { }, }, }, - { - Name: "aws_glue_database_table_partitions", - Description: "Represents a slice of table data", - Resolver: fetchGlueDatabaseTablePartitions, - Multiplex: client.ServiceAccountRegionMultiplexer("glue"), - IgnoreError: client.IgnoreAccessDeniedServiceDisabled, - DeleteFilter: client.DeleteAccountRegionFilter, - Columns: []schema.Column{ - { - Name: "database_table_cq_id", - Description: "Unique CloudQuery ID of aws_glue_database_tables table (FK)", - Type: schema.TypeUUID, - Resolver: schema.ParentIdResolver, - }, - { - Name: "account_id", - Description: "The AWS Account ID of the resource.", - Type: schema.TypeString, - Resolver: client.ResolveAWSAccount, - }, - { - Name: "region", - Description: "The AWS Region of the resource.", - Type: schema.TypeString, - Resolver: client.ResolveAWSRegion, - }, - { - Name: "parameters", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("Parameters"), - }, - { - Name: "storage_parameters", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("StorageDescriptor.Parameters"), - }, - { - Name: "catalog_id", - Description: "The ID of the Data Catalog in which the partition resides", - Type: schema.TypeString, - }, - { - Name: "creation_time", - Description: "The time at which the partition was created", - Type: schema.TypeTimestamp, - }, - { - Name: "database_name", - Description: "The name of the catalog database in which to create the partition", - Type: schema.TypeString, - }, - { - Name: "last_access_time", - Description: "The last time at which the partition was accessed", - Type: schema.TypeTimestamp, - }, - { - Name: "last_analyzed_time", - Description: "The last time at which column statistics were computed for this partition", - Type: schema.TypeTimestamp, - }, - { - Name: "additional_locations", - Description: "A list of locations that point to the path where a Delta table is located", - Type: schema.TypeStringArray, - Resolver: schema.PathResolver("StorageDescriptor.AdditionalLocations"), - }, - { - Name: "bucket_columns", - Description: "A list of reducer grouping columns, clustering columns, and bucketing columns in the table", - Type: schema.TypeStringArray, - Resolver: schema.PathResolver("StorageDescriptor.BucketColumns"), - }, - { - Name: "compressed", - Description: "True if the data in the table is compressed, or False if not", - Type: schema.TypeBool, - Resolver: schema.PathResolver("StorageDescriptor.Compressed"), - }, - { - Name: "input_format", - Description: "The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format", - Type: schema.TypeString, - Resolver: schema.PathResolver("StorageDescriptor.InputFormat"), - }, - { - Name: "location", - Description: "The physical location of the table", - Type: schema.TypeString, - Resolver: schema.PathResolver("StorageDescriptor.Location"), - }, - { - Name: "number_of_buckets", - Description: "Must be specified if the table contains any dimension columns", - Type: schema.TypeBigInt, - Resolver: schema.PathResolver("StorageDescriptor.NumberOfBuckets"), - }, - { - Name: "output_format", - Description: "The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format", - Type: schema.TypeString, - Resolver: schema.PathResolver("StorageDescriptor.OutputFormat"), - }, - { - Name: "schema_reference_schema_id", - Description: "A structure that contains schema identity fields", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaId"), - }, - { - Name: "schema_reference_schema_version_id", - Description: "The unique ID assigned to a version of the schema", - Type: schema.TypeString, - Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaVersionId"), - }, - { - Name: "schema_reference_schema_version_number", - Description: "The version number of the schema", - Type: schema.TypeBigInt, - Resolver: schema.PathResolver("StorageDescriptor.SchemaReference.SchemaVersionNumber"), - }, - { - Name: "serde_info", - Description: "The serialization/deserialization (SerDe) information", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("StorageDescriptor.SerdeInfo"), - }, - { - Name: "skewed_info", - Description: "The information about values that appear frequently in a column (skewed values)", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("StorageDescriptor.SkewedInfo"), - }, - { - Name: "sort_columns", - Description: "A list specifying the sort order of each bucket in the table", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("StorageDescriptor.SortColumns"), - }, - { - Name: "stored_as_sub_directories", - Description: "True if the table data is stored in subdirectories, or False if not", - Type: schema.TypeBool, - Resolver: schema.PathResolver("StorageDescriptor.StoredAsSubDirectories"), - }, - { - Name: "table_name", - Description: "The name of the database table in which to create the partition", - Type: schema.TypeString, - }, - { - Name: "values", - Description: "The values of the partition", - Type: schema.TypeStringArray, - }, - }, - Relations: []*schema.Table{ - { - Name: "aws_glue_database_table_partition_columns", - Description: "A column in a Table", - Resolver: schema.PathTableResolver("StorageDescriptor.Columns"), - Columns: []schema.Column{ - { - Name: "database_table_partition_cq_id", - Description: "Unique CloudQuery ID of aws_glue_database_table_partitions table (FK)", - Type: schema.TypeUUID, - Resolver: schema.ParentIdResolver, - }, - { - Name: "name", - Description: "The name of the Column", - Type: schema.TypeString, - }, - { - Name: "comment", - Description: "A free-form text comment", - Type: schema.TypeString, - }, - { - Name: "parameters", - Description: "These key-value pairs define properties associated with the column", - Type: schema.TypeJSON, - }, - { - Name: "type", - Description: "The data type of the Column", - Type: schema.TypeString, - }, - }, - }, - }, - }, { Name: "aws_glue_database_table_indexes", Description: "A descriptor for a partition index in a table", @@ -638,25 +446,6 @@ func fetchGlueDatabaseTables(ctx context.Context, meta schema.ClientMeta, parent } return nil } -func fetchGlueDatabaseTablePartitions(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { - cl := meta.(*client.Client) - svc := cl.Services().Glue - d := parent.Parent.Item.(types.Database) - t := parent.Item.(types.Table) - input := glue.GetPartitionsInput{DatabaseName: d.Name, CatalogId: d.CatalogId, TableName: t.Name} - for { - result, err := svc.GetPartitions(ctx, &input) - if err != nil { - return diag.WrapError(err) - } - res <- result.Partitions - if aws.ToString(result.NextToken) == "" { - break - } - input.NextToken = result.NextToken - } - return nil -} func fetchGlueDatabaseTableIndexes(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { cl := meta.(*client.Client) svc := cl.Services().Glue diff --git a/resources/services/glue/databases.hcl b/resources/services/glue/databases.hcl index 24aa251a3..6d908b3a9 100644 --- a/resources/services/glue/databases.hcl +++ b/resources/services/glue/databases.hcl @@ -84,75 +84,6 @@ resource "aws" "glue" "databases" { } } - user_relation "aws" "glue" "partitions" { - path = "github.com/aws/aws-sdk-go-v2/service/glue/types.Partition" - ignoreError "IgnoreAccessDenied" { - path = "github.com/cloudquery/cq-provider-aws/client.IgnoreAccessDeniedServiceDisabled" - } - deleteFilter "AccountRegionFilter" { - path = "github.com/cloudquery/cq-provider-aws/client.DeleteAccountRegionFilter" - } - multiplex "AwsAccountRegion" { - path = "github.com/cloudquery/cq-provider-aws/client.ServiceAccountRegionMultiplexer" - params = ["glue"] - } - userDefinedColumn "account_id" { - description = "The AWS Account ID of the resource." - type = "string" - resolver "resolveAWSAccount" { - path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSAccount" - } - } - userDefinedColumn "region" { - type = "string" - description = "The AWS Region of the resource." - resolver "resolveAWSRegion" { - path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSRegion" - } - } - - column "storage_descriptor" { - skip_prefix = true - } - - column "skewed_info" { - type = "json" - } - - column "sort_columns" { - type = "json" - } - - column "schema_reference_schema_id" { - type = "json" - } - - column "serde_info" { - type = "json" - } - - column "parameters" { - skip = true - } - - userDefinedColumn "parameters" { - type = "json" - resolver "resolverSliceToJson" { - path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" - params = ["Parameters"] - } - } - - userDefinedColumn "storage_parameters" { - type = "json" - resolver "resolverSliceToJson" { - path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" - params = ["StorageDescriptor.Parameters"] - } - } - - } - user_relation "aws" "glue" "indexes" { path = "github.com/aws/aws-sdk-go-v2/service/glue/types.PartitionIndexDescriptor" From b501b2b50037fe28d2bab15dd653fa32ed1ade97 Mon Sep 17 00:00:00 2001 From: amanenk Date: Tue, 9 Aug 2022 11:44:06 +0300 Subject: [PATCH 9/9] mock generation --- client/mocks/glue.go | 60 +++++++------------ client/services.go | 2 + .../services/glue/databases_mock_test.go | 5 -- 3 files changed, 22 insertions(+), 45 deletions(-) diff --git a/client/mocks/glue.go b/client/mocks/glue.go index c323289cc..df6e14f08 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -235,6 +235,26 @@ func (mr *MockGlueClientMockRecorder) GetMLTransforms(arg0, arg1 interface{}, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMLTransforms", reflect.TypeOf((*MockGlueClient)(nil).GetMLTransforms), varargs...) } +// GetPartitionIndexes mocks base method. +func (m *MockGlueClient) GetPartitionIndexes(arg0 context.Context, arg1 *glue.GetPartitionIndexesInput, arg2 ...func(*glue.Options)) (*glue.GetPartitionIndexesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetPartitionIndexes", varargs...) + ret0, _ := ret[0].(*glue.GetPartitionIndexesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPartitionIndexes indicates an expected call of GetPartitionIndexes. +func (mr *MockGlueClientMockRecorder) GetPartitionIndexes(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPartitionIndexes", reflect.TypeOf((*MockGlueClient)(nil).GetPartitionIndexes), varargs...) +} + // GetSchema mocks base method. func (m *MockGlueClient) GetSchema(arg0 context.Context, arg1 *glue.GetSchemaInput, arg2 ...func(*glue.Options)) (*glue.GetSchemaOutput, error) { m.ctrl.T.Helper() @@ -275,46 +295,6 @@ func (mr *MockGlueClientMockRecorder) GetSchemaVersion(arg0, arg1 interface{}, a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaVersion", reflect.TypeOf((*MockGlueClient)(nil).GetSchemaVersion), varargs...) } -// GetPartitionIndexes mocks base method. -func (m *MockGlueClient) GetPartitionIndexes(arg0 context.Context, arg1 *glue.GetPartitionIndexesInput, arg2 ...func(*glue.Options)) (*glue.GetPartitionIndexesOutput, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "GetPartitionIndexes", varargs...) - ret0, _ := ret[0].(*glue.GetPartitionIndexesOutput) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetPartitionIndexes indicates an expected call of GetPartitionIndexes. -func (mr *MockGlueClientMockRecorder) GetPartitionIndexes(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPartitionIndexes", reflect.TypeOf((*MockGlueClient)(nil).GetPartitionIndexes), varargs...) -} - -// GetPartitions mocks base method. -func (m *MockGlueClient) GetPartitions(arg0 context.Context, arg1 *glue.GetPartitionsInput, arg2 ...func(*glue.Options)) (*glue.GetPartitionsOutput, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "GetPartitions", varargs...) - ret0, _ := ret[0].(*glue.GetPartitionsOutput) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetPartitions indicates an expected call of GetPartitions. -func (mr *MockGlueClientMockRecorder) GetPartitions(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPartitions", reflect.TypeOf((*MockGlueClient)(nil).GetPartitions), varargs...) -} - // GetSecurityConfigurations mocks base method. func (m *MockGlueClient) GetSecurityConfigurations(arg0 context.Context, arg1 *glue.GetSecurityConfigurationsInput, arg2 ...func(*glue.Options)) (*glue.GetSecurityConfigurationsOutput, error) { m.ctrl.T.Helper() diff --git a/client/services.go b/client/services.go index 5a051d336..b2c53a9b0 100644 --- a/client/services.go +++ b/client/services.go @@ -775,6 +775,8 @@ type GlueClient interface { GetCrawlers(ctx context.Context, params *glue.GetCrawlersInput, optFns ...func(*glue.Options)) (*glue.GetCrawlersOutput, error) GetSecurityConfigurations(ctx context.Context, params *glue.GetSecurityConfigurationsInput, optFns ...func(*glue.Options)) (*glue.GetSecurityConfigurationsOutput, error) GetPartitionIndexes(ctx context.Context, params *glue.GetPartitionIndexesInput, optFns ...func(*glue.Options)) (*glue.GetPartitionIndexesOutput, error) + GetClassifiers(ctx context.Context, params *glue.GetClassifiersInput, optFns ...func(*glue.Options)) (*glue.GetClassifiersOutput, error) + GetConnections(ctx context.Context, params *glue.GetConnectionsInput, optFns ...func(*glue.Options)) (*glue.GetConnectionsOutput, error) } //go:generate mockgen -package=mocks -destination=./mocks/kinesis.go . KinesisClient diff --git a/resources/services/glue/databases_mock_test.go b/resources/services/glue/databases_mock_test.go index 956250f0e..55528a70d 100644 --- a/resources/services/glue/databases_mock_test.go +++ b/resources/services/glue/databases_mock_test.go @@ -24,11 +24,6 @@ func buildDatabasesMock(t *testing.T, ctrl *gomock.Controller) client.Services { tb.NextToken = nil m.EXPECT().GetTables(gomock.Any(), gomock.Any()).Return(&tb, nil) - p := glue.GetPartitionsOutput{} - require.NoError(t, faker.FakeData(&p)) - p.NextToken = nil - m.EXPECT().GetPartitions(gomock.Any(), gomock.Any()).Return(&p, nil) - i := glue.GetPartitionIndexesOutput{} require.NoError(t, faker.FakeData(&i)) i.NextToken = nil