From 088f4a8373342e37102b7e62b27050c77605887d Mon Sep 17 00:00:00 2001 From: The Magician Date: Wed, 28 Aug 2024 10:35:16 -0700 Subject: [PATCH] Add type support (#11252) (#8069) [upstream:5c149d738ad2b3e6327127f3871c55a508d9f036] Signed-off-by: Modular Magician --- .changelog/11252.txt | 3 + .../bigtable/resource_bigtable_table.go | 191 +++++++++++++++--- .../bigtable/resource_bigtable_table_test.go | 91 ++++++++- website/docs/r/bigtable_table.html.markdown | 20 ++ 4 files changed, 276 insertions(+), 29 deletions(-) create mode 100644 .changelog/11252.txt diff --git a/.changelog/11252.txt b/.changelog/11252.txt new file mode 100644 index 0000000000..0ca77602f9 --- /dev/null +++ b/.changelog/11252.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +bigtable: added support for `column_family.type` in `google_bigtable_table` +``` \ No newline at end of file diff --git a/google-beta/services/bigtable/resource_bigtable_table.go b/google-beta/services/bigtable/resource_bigtable_table.go index e960a6431b..4aa334ddd7 100644 --- a/google-beta/services/bigtable/resource_bigtable_table.go +++ b/google-beta/services/bigtable/resource_bigtable_table.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "log" + "strings" "time" "cloud.google.com/go/bigtable" @@ -18,6 +19,24 @@ import ( "github.com/hashicorp/terraform-provider-google-beta/google-beta/verify" ) +func familyHash(v interface{}) int { + m := v.(map[string]interface{}) + cf := m["family"].(string) + t, err := getType(m["type"]) + if err != nil { + panic(err) + } + if t == nil { + // no specified type. + return tpgresource.Hashcode(cf) + } + b, err := bigtable.MarshalJSON(t) + if err != nil { + panic(err) + } + return tpgresource.Hashcode(cf + string(b)) +} + func ResourceBigtableTable() *schema.Resource { return &schema.Resource{ Create: resourceBigtableTableCreate, @@ -61,8 +80,15 @@ func ResourceBigtableTable() *schema.Resource { Required: true, Description: `The name of the column family.`, }, + "type": { + Type: schema.TypeString, + Optional: true, + Description: `The type of the column family.`, + DiffSuppressFunc: typeDiffFunc, + }, }, }, + Set: familyHash, }, "instance_name": { @@ -135,6 +161,18 @@ func ResourceBigtableTable() *schema.Resource { } } +func typeDiffFunc(k, oldValue, newValue string, d *schema.ResourceData) bool { + old, err := getType(oldValue) + if err != nil { + panic(fmt.Sprintf("old error: %v", err)) + } + new, err := getType(newValue) + if err != nil { + panic(fmt.Sprintf("new error: %v", err)) + } + return bigtable.Equal(old, new) +} + func resourceBigtableTableCreate(d *schema.ResourceData, meta interface{}) error { config := meta.(*transport_tpg.Config) userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) @@ -214,7 +252,7 @@ func resourceBigtableTableCreate(d *schema.ResourceData, meta interface{}) error } // Set the column families if given. - columnFamilies := make(map[string]bigtable.GCPolicy) + columnFamilies := make(map[string]bigtable.Family) if d.Get("column_family.#").(int) > 0 { columns := d.Get("column_family").(*schema.Set).List() @@ -222,12 +260,19 @@ func resourceBigtableTableCreate(d *schema.ResourceData, meta interface{}) error column := co.(map[string]interface{}) if v, ok := column["family"]; ok { - // By default, there is no GC rules. - columnFamilies[v.(string)] = bigtable.NoGcPolicy() + valueType, err := getType(column["type"]) + if err != nil { + return err + } + columnFamilies[v.(string)] = bigtable.Family{ + // By default, there is no GC rules. + GCPolicy: bigtable.NoGcPolicy(), + ValueType: valueType, + } } } } - tblConf.Families = columnFamilies + tblConf.ColumnFamilies = columnFamilies // This method may return before the table's creation is complete - we may need to wait until // it exists in the future. @@ -283,7 +328,11 @@ func resourceBigtableTableRead(d *schema.ResourceData, meta interface{}) error { if err := d.Set("project", project); err != nil { return fmt.Errorf("Error setting project: %s", err) } - if err := d.Set("column_family", FlattenColumnFamily(table.Families)); err != nil { + families, err := FlattenColumnFamily(table.FamilyInfos) + if err != nil { + return fmt.Errorf("Error flatenning column families: %v", err) + } + if err := d.Set("column_family", families); err != nil { return fmt.Errorf("Error setting column_family: %s", err) } @@ -330,6 +379,48 @@ func resourceBigtableTableRead(d *schema.ResourceData, meta interface{}) error { return nil } +func toFamilyMap(set *schema.Set) (map[string]bigtable.Family, error) { + result := map[string]bigtable.Family{} + for _, item := range set.List() { + column := item.(map[string]interface{}) + + if v, ok := column["family"]; ok && v != "" { + valueType, err := getType(column["type"]) + if err != nil { + return nil, err + } + result[v.(string)] = bigtable.Family{ + ValueType: valueType, + } + } + } + return result, nil +} + +// familyMapDiffKeys returns a new map that is the result of a-b, comparing keys +func familyMapDiffKeys(a, b map[string]bigtable.Family) map[string]bigtable.Family { + result := map[string]bigtable.Family{} + for k, v := range a { + if _, ok := b[k]; !ok { + result[k] = v + } + } + return result +} + +// familyMapDiffValueTypes returns a new map that is the result of a-b, where a and b share keys but have different value types +func familyMapDiffValueTypes(a, b map[string]bigtable.Family) map[string]bigtable.Family { + result := map[string]bigtable.Family{} + for k, va := range a { + if vb, ok := b[k]; ok { + if !bigtable.Equal(va.ValueType, vb.ValueType) { + result[k] = va + } + } + } + return result +} + func resourceBigtableTableUpdate(d *schema.ResourceData, meta interface{}) error { config := meta.(*transport_tpg.Config) userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) @@ -351,31 +442,33 @@ func resourceBigtableTableUpdate(d *schema.ResourceData, meta interface{}) error defer c.Close() o, n := d.GetChange("column_family") - oSet := o.(*schema.Set) - nSet := n.(*schema.Set) name := d.Get("name").(string) - // Add column families that are in new but not in old - for _, new := range nSet.Difference(oSet).List() { - column := new.(map[string]interface{}) + oMap, err := toFamilyMap(o.(*schema.Set)) + if err != nil { + return err + } + nMap, err := toFamilyMap(n.(*schema.Set)) + if err != nil { + return err + } - if v, ok := column["family"]; ok { - log.Printf("[DEBUG] adding column family %q", v) - if err := c.CreateColumnFamily(ctx, name, v.(string)); err != nil { - return fmt.Errorf("Error creating column family %q: %s", v, err) - } + for cfn, cf := range familyMapDiffKeys(nMap, oMap) { + log.Printf("[DEBUG] adding column family %q", cfn) + if err := c.CreateColumnFamilyWithConfig(ctx, name, cfn, cf); err != nil { + return fmt.Errorf("Error creating column family %q: %s", cfn, err) } } - - // Remove column families that are in old but not in new - for _, old := range oSet.Difference(nSet).List() { - column := old.(map[string]interface{}) - - if v, ok := column["family"]; ok { - log.Printf("[DEBUG] removing column family %q", v) - if err := c.DeleteColumnFamily(ctx, name, v.(string)); err != nil { - return fmt.Errorf("Error deleting column family %q: %s", v, err) - } + for cfn, _ := range familyMapDiffKeys(oMap, nMap) { + log.Printf("[DEBUG] removing column family %q", cfn) + if err := c.DeleteColumnFamily(ctx, name, cfn); err != nil { + return fmt.Errorf("Error deleting column family %q: %s", cfn, err) + } + } + for cfn, cf := range familyMapDiffValueTypes(nMap, oMap) { + log.Printf("[DEBUG] updating column family: %q", cfn) + if err := c.UpdateFamily(ctx, name, cfn, cf); err != nil { + return fmt.Errorf("Error update column family %q: %s", cfn, err) } } @@ -487,16 +580,23 @@ func resourceBigtableTableDestroy(d *schema.ResourceData, meta interface{}) erro return nil } -func FlattenColumnFamily(families []string) []map[string]interface{} { +func FlattenColumnFamily(families []bigtable.FamilyInfo) ([]map[string]interface{}, error) { result := make([]map[string]interface{}, 0, len(families)) for _, f := range families { data := make(map[string]interface{}) - data["family"] = f + data["family"] = f.Name + if _, ok := f.ValueType.(bigtable.AggregateType); ok { + marshalled, err := bigtable.MarshalJSON(f.ValueType) + if err != nil { + return nil, err + } + data["type"] = string(marshalled) + } result = append(result, data) } - return result + return result, nil } // TODO(rileykarson): Fix the stored import format after rebasing 3.0.0 @@ -519,3 +619,38 @@ func resourceBigtableTableImport(d *schema.ResourceData, meta interface{}) ([]*s return []*schema.ResourceData{d}, nil } + +func getType(input interface{}) (bigtable.Type, error) { + if input == nil || input.(string) == "" { + return nil, nil + } + inputType := strings.TrimSuffix(input.(string), "\n") + switch inputType { + case "intsum": + return bigtable.AggregateType{ + Input: bigtable.Int64Type{}, + Aggregator: bigtable.SumAggregator{}, + }, nil + case "intmin": + return bigtable.AggregateType{ + Input: bigtable.Int64Type{}, + Aggregator: bigtable.MinAggregator{}, + }, nil + case "intmax": + return bigtable.AggregateType{ + Input: bigtable.Int64Type{}, + Aggregator: bigtable.MaxAggregator{}, + }, nil + case "inthll": + return bigtable.AggregateType{ + Input: bigtable.Int64Type{}, + Aggregator: bigtable.HllppUniqueCountAggregator{}, + }, nil + } + + output, err := bigtable.UnmarshalJSON([]byte(inputType)) + if err != nil { + return nil, err + } + return output, nil +} diff --git a/google-beta/services/bigtable/resource_bigtable_table_test.go b/google-beta/services/bigtable/resource_bigtable_table_test.go index f5ddc634a0..4273e0f711 100644 --- a/google-beta/services/bigtable/resource_bigtable_table_test.go +++ b/google-beta/services/bigtable/resource_bigtable_table_test.go @@ -92,6 +92,63 @@ func TestAccBigtableTable_family(t *testing.T) { }) } +func TestAccBigtableTable_familyType(t *testing.T) { + // bigtable instance does not use the shared HTTP client, this test creates an instance + acctest.SkipIfVcr(t) + t.Parallel() + + instanceName := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) + tableName := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) + family := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckBigtableTableDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccBigtableTable_familyType(instanceName, tableName, family, "intmax"), + }, + { + ResourceName: "google_bigtable_table.table", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccBigtableTable_familyType(instanceName, tableName, family, `{ + "aggregateType": { + "max": {}, + "inputType": { + "int64Type": { + "encoding": { + "bigEndianBytes": {} + } + } + } + } + }`), + }, + { + ResourceName: "google_bigtable_table.table", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccBigtableTable_familyType(instanceName, tableName, family, "intmax"), + }, + { + ResourceName: "google_bigtable_table.table", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccBigtableTable_familyType(instanceName, tableName, family, "intmin"), + ExpectError: regexp.MustCompile(".*Immutable fields 'value_type' cannot be updated.*"), + }, + }, + }) +} + func TestAccBigtableTable_deletion_protection_protected(t *testing.T) { // bigtable instance does not use the shared HTTP client, this test creates an instance acctest.SkipIfVcr(t) @@ -459,7 +516,11 @@ func testAccBigtableColumnFamilyExists(t *testing.T, table_name_space, family st if err != nil { return fmt.Errorf("Error retrieving table. Could not find %s in %s.", rs.Primary.Attributes["name"], rs.Primary.Attributes["instance_name"]) } - for _, data := range bigtable.FlattenColumnFamily(table.Families) { + families, err := bigtable.FlattenColumnFamily(table.FamilyInfos) + if err != nil { + return fmt.Errorf("Error flattening column families: %v", err) + } + for _, data := range families { if data["family"] != family { return fmt.Errorf("Error checking column family. Could not find column family %s in %s.", family, rs.Primary.Attributes["name"]) } @@ -592,6 +653,34 @@ resource "google_bigtable_table" "table" { `, instanceName, instanceName, tableName, family) } +func testAccBigtableTable_familyType(instanceName, tableName, family, familyType string) string { + return fmt.Sprintf(` +resource "google_bigtable_instance" "instance" { + name = "%s" + + cluster { + cluster_id = "%s" + zone = "us-central1-b" + } + + instance_type = "DEVELOPMENT" + deletion_protection = false +} + +resource "google_bigtable_table" "table" { + name = "%s" + instance_name = google_bigtable_instance.instance.name + + column_family { + family = "%s" + type = <