Add type support (#11252) (#8069)

[upstream:5c149d738ad2b3e6327127f3871c55a508d9f036]

Signed-off-by: Modular Magician <[email protected]>

.changelog/11252.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+bigtable: added support for `column_family.type` in `google_bigtable_table`
+```

google-beta/services/bigtable/resource_bigtable_table.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	"cloud.google.com/go/bigtable"
@@ -18,6 +19,24 @@ import (
 	"github.com/hashicorp/terraform-provider-google-beta/google-beta/verify"
 )
 
+func familyHash(v interface{}) int {
+	m := v.(map[string]interface{})
+	cf := m["family"].(string)
+	t, err := getType(m["type"])
+	if err != nil {
+		panic(err)
+	}
+	if t == nil {
+		// no specified type.
+		return tpgresource.Hashcode(cf)
+	}
+	b, err := bigtable.MarshalJSON(t)
+	if err != nil {
+		panic(err)
+	}
+	return tpgresource.Hashcode(cf + string(b))
+}
+
 func ResourceBigtableTable() *schema.Resource {
 	return &schema.Resource{
 		Create: resourceBigtableTableCreate,
@@ -61,8 +80,15 @@ func ResourceBigtableTable() *schema.Resource {
 							Required:    true,
 							Description: `The name of the column family.`,
 						},
+						"type": {
+							Type:             schema.TypeString,
+							Optional:         true,
+							Description:      `The type of the column family.`,
+							DiffSuppressFunc: typeDiffFunc,
+						},
 					},
 				},
+				Set: familyHash,
 			},
 
 			"instance_name": {
@@ -135,6 +161,18 @@ func ResourceBigtableTable() *schema.Resource {
 	}
 }
 
+func typeDiffFunc(k, oldValue, newValue string, d *schema.ResourceData) bool {
+	old, err := getType(oldValue)
+	if err != nil {
+		panic(fmt.Sprintf("old error: %v", err))
+	}
+	new, err := getType(newValue)
+	if err != nil {
+		panic(fmt.Sprintf("new error: %v", err))
+	}
+	return bigtable.Equal(old, new)
+}
+
 func resourceBigtableTableCreate(d *schema.ResourceData, meta interface{}) error {
 	config := meta.(*transport_tpg.Config)
 	userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent)
@@ -214,20 +252,27 @@ func resourceBigtableTableCreate(d *schema.ResourceData, meta interface{}) error
 	}
 
 	// Set the column families if given.
-	columnFamilies := make(map[string]bigtable.GCPolicy)
+	columnFamilies := make(map[string]bigtable.Family)
 	if d.Get("column_family.#").(int) > 0 {
 		columns := d.Get("column_family").(*schema.Set).List()
 
 		for _, co := range columns {
 			column := co.(map[string]interface{})
 
 			if v, ok := column["family"]; ok {
-				// By default, there is no GC rules.
-				columnFamilies[v.(string)] = bigtable.NoGcPolicy()
+				valueType, err := getType(column["type"])
+				if err != nil {
+					return err
+				}
+				columnFamilies[v.(string)] = bigtable.Family{
+					// By default, there is no GC rules.
+					GCPolicy:  bigtable.NoGcPolicy(),
+					ValueType: valueType,
+				}
 			}
 		}
 	}
-	tblConf.Families = columnFamilies
+	tblConf.ColumnFamilies = columnFamilies
 
 	// This method may return before the table's creation is complete - we may need to wait until
 	// it exists in the future.
@@ -283,7 +328,11 @@ func resourceBigtableTableRead(d *schema.ResourceData, meta interface{}) error {
 	if err := d.Set("project", project); err != nil {
 		return fmt.Errorf("Error setting project: %s", err)
 	}
-	if err := d.Set("column_family", FlattenColumnFamily(table.Families)); err != nil {
+	families, err := FlattenColumnFamily(table.FamilyInfos)
+	if err != nil {
+		return fmt.Errorf("Error flatenning column families: %v", err)
+	}
+	if err := d.Set("column_family", families); err != nil {
 		return fmt.Errorf("Error setting column_family: %s", err)
 	}
 
@@ -330,6 +379,48 @@ func resourceBigtableTableRead(d *schema.ResourceData, meta interface{}) error {
 	return nil
 }
 
+func toFamilyMap(set *schema.Set) (map[string]bigtable.Family, error) {
+	result := map[string]bigtable.Family{}
+	for _, item := range set.List() {
+		column := item.(map[string]interface{})
+
+		if v, ok := column["family"]; ok && v != "" {
+			valueType, err := getType(column["type"])
+			if err != nil {
+				return nil, err
+			}
+			result[v.(string)] = bigtable.Family{
+				ValueType: valueType,
+			}
+		}
+	}
+	return result, nil
+}
+
+// familyMapDiffKeys returns a new map that is the result of a-b, comparing keys
+func familyMapDiffKeys(a, b map[string]bigtable.Family) map[string]bigtable.Family {
+	result := map[string]bigtable.Family{}
+	for k, v := range a {
+		if _, ok := b[k]; !ok {
+			result[k] = v
+		}
+	}
+	return result
+}
+
+// familyMapDiffValueTypes returns a new map that is the result of a-b, where a and b share keys but have different value types
+func familyMapDiffValueTypes(a, b map[string]bigtable.Family) map[string]bigtable.Family {
+	result := map[string]bigtable.Family{}
+	for k, va := range a {
+		if vb, ok := b[k]; ok {
+			if !bigtable.Equal(va.ValueType, vb.ValueType) {
+				result[k] = va
+			}
+		}
+	}
+	return result
+}
+
 func resourceBigtableTableUpdate(d *schema.ResourceData, meta interface{}) error {
 	config := meta.(*transport_tpg.Config)
 	userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent)
@@ -351,31 +442,33 @@ func resourceBigtableTableUpdate(d *schema.ResourceData, meta interface{}) error
 	defer c.Close()
 
 	o, n := d.GetChange("column_family")
-	oSet := o.(*schema.Set)
-	nSet := n.(*schema.Set)
 	name := d.Get("name").(string)
 
-	// Add column families that are in new but not in old
-	for _, new := range nSet.Difference(oSet).List() {
-		column := new.(map[string]interface{})
+	oMap, err := toFamilyMap(o.(*schema.Set))
+	if err != nil {
+		return err
+	}
+	nMap, err := toFamilyMap(n.(*schema.Set))
+	if err != nil {
+		return err
+	}
 
-		if v, ok := column["family"]; ok {
-			log.Printf("[DEBUG] adding column family %q", v)
-			if err := c.CreateColumnFamily(ctx, name, v.(string)); err != nil {
-				return fmt.Errorf("Error creating column family %q: %s", v, err)
-			}
+	for cfn, cf := range familyMapDiffKeys(nMap, oMap) {
+		log.Printf("[DEBUG] adding column family %q", cfn)
+		if err := c.CreateColumnFamilyWithConfig(ctx, name, cfn, cf); err != nil {
+			return fmt.Errorf("Error creating column family %q: %s", cfn, err)
 		}
 	}
-
-	// Remove column families that are in old but not in new
-	for _, old := range oSet.Difference(nSet).List() {
-		column := old.(map[string]interface{})
-
-		if v, ok := column["family"]; ok {
-			log.Printf("[DEBUG] removing column family %q", v)
-			if err := c.DeleteColumnFamily(ctx, name, v.(string)); err != nil {
-				return fmt.Errorf("Error deleting column family %q: %s", v, err)
-			}
+	for cfn, _ := range familyMapDiffKeys(oMap, nMap) {
+		log.Printf("[DEBUG] removing column family %q", cfn)
+		if err := c.DeleteColumnFamily(ctx, name, cfn); err != nil {
+			return fmt.Errorf("Error deleting column family %q: %s", cfn, err)
+		}
+	}
+	for cfn, cf := range familyMapDiffValueTypes(nMap, oMap) {
+		log.Printf("[DEBUG] updating column family: %q", cfn)
+		if err := c.UpdateFamily(ctx, name, cfn, cf); err != nil {
+			return fmt.Errorf("Error update column family %q: %s", cfn, err)
 		}
 	}
 
@@ -487,16 +580,23 @@ func resourceBigtableTableDestroy(d *schema.ResourceData, meta interface{}) erro
 	return nil
 }
 
-func FlattenColumnFamily(families []string) []map[string]interface{} {
+func FlattenColumnFamily(families []bigtable.FamilyInfo) ([]map[string]interface{}, error) {
 	result := make([]map[string]interface{}, 0, len(families))
 
 	for _, f := range families {
 		data := make(map[string]interface{})
-		data["family"] = f
+		data["family"] = f.Name
+		if _, ok := f.ValueType.(bigtable.AggregateType); ok {
+			marshalled, err := bigtable.MarshalJSON(f.ValueType)
+			if err != nil {
+				return nil, err
+			}
+			data["type"] = string(marshalled)
+		}
 		result = append(result, data)
 	}
 
-	return result
+	return result, nil
 }
 
 // TODO(rileykarson): Fix the stored import format after rebasing 3.0.0
@@ -519,3 +619,38 @@ func resourceBigtableTableImport(d *schema.ResourceData, meta interface{}) ([]*s
 
 	return []*schema.ResourceData{d}, nil
 }
+
+func getType(input interface{}) (bigtable.Type, error) {
+	if input == nil || input.(string) == "" {
+		return nil, nil
+	}
+	inputType := strings.TrimSuffix(input.(string), "\n")
+	switch inputType {
+	case "intsum":
+		return bigtable.AggregateType{
+			Input:      bigtable.Int64Type{},
+			Aggregator: bigtable.SumAggregator{},
+		}, nil
+	case "intmin":
+		return bigtable.AggregateType{
+			Input:      bigtable.Int64Type{},
+			Aggregator: bigtable.MinAggregator{},
+		}, nil
+	case "intmax":
+		return bigtable.AggregateType{
+			Input:      bigtable.Int64Type{},
+			Aggregator: bigtable.MaxAggregator{},
+		}, nil
+	case "inthll":
+		return bigtable.AggregateType{
+			Input:      bigtable.Int64Type{},
+			Aggregator: bigtable.HllppUniqueCountAggregator{},
+		}, nil
+	}
+
+	output, err := bigtable.UnmarshalJSON([]byte(inputType))
+	if err != nil {
+		return nil, err
+	}
+	return output, nil
+}

google-beta/services/bigtable/resource_bigtable_table_test.go

@@ -92,6 +92,63 @@ func TestAccBigtableTable_family(t *testing.T) {
 	})
 }
 
+func TestAccBigtableTable_familyType(t *testing.T) {
+	// bigtable instance does not use the shared HTTP client, this test creates an instance
+	acctest.SkipIfVcr(t)
+	t.Parallel()
+
+	instanceName := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10))
+	tableName := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10))
+	family := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10))
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckBigtableTableDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccBigtableTable_familyType(instanceName, tableName, family, "intmax"),
+			},
+			{
+				ResourceName:      "google_bigtable_table.table",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			{
+				Config: testAccBigtableTable_familyType(instanceName, tableName, family, `{
+					"aggregateType": {
+						"max": {},
+						"inputType": {
+							"int64Type": {
+								"encoding": {
+									"bigEndianBytes": {}
+								}
+							}
+						}
+					}
+				}`),
+			},
+			{
+				ResourceName:      "google_bigtable_table.table",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			{
+				Config: testAccBigtableTable_familyType(instanceName, tableName, family, "intmax"),
+			},
+			{
+				ResourceName:      "google_bigtable_table.table",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			{
+				Config:      testAccBigtableTable_familyType(instanceName, tableName, family, "intmin"),
+				ExpectError: regexp.MustCompile(".*Immutable fields 'value_type' cannot be updated.*"),
+			},
+		},
+	})
+}
+
 func TestAccBigtableTable_deletion_protection_protected(t *testing.T) {
 	// bigtable instance does not use the shared HTTP client, this test creates an instance
 	acctest.SkipIfVcr(t)
@@ -459,7 +516,11 @@ func testAccBigtableColumnFamilyExists(t *testing.T, table_name_space, family st
 		if err != nil {
 			return fmt.Errorf("Error retrieving table. Could not find %s in %s.", rs.Primary.Attributes["name"], rs.Primary.Attributes["instance_name"])
 		}
-		for _, data := range bigtable.FlattenColumnFamily(table.Families) {
+		families, err := bigtable.FlattenColumnFamily(table.FamilyInfos)
+		if err != nil {
+			return fmt.Errorf("Error flattening column families: %v", err)
+		}
+		for _, data := range families {
 			if data["family"] != family {
 				return fmt.Errorf("Error checking column family. Could not find column family %s in %s.", family, rs.Primary.Attributes["name"])
 			}
@@ -592,6 +653,34 @@ resource "google_bigtable_table" "table" {
 `, instanceName, instanceName, tableName, family)
 }
 
+func testAccBigtableTable_familyType(instanceName, tableName, family, familyType string) string {
+	return fmt.Sprintf(`
+resource "google_bigtable_instance" "instance" {
+  name = "%s"
+
+  cluster {
+    cluster_id = "%s"
+    zone       = "us-central1-b"
+  }
+
+  instance_type = "DEVELOPMENT"
+  deletion_protection = false
+}
+
+resource "google_bigtable_table" "table" {
+  name          = "%s"
+  instance_name = google_bigtable_instance.instance.name
+
+  column_family {
+    family = "%s"
+	type =  <<EOF
+%s
+EOF
+  }
+}
+`, instanceName, instanceName, tableName, family, familyType)
+}
+
 func testAccBigtableTable_deletion_protection(instanceName, tableName, deletionProtection, family string) string {
 	return fmt.Sprintf(`
 resource "google_bigtable_instance" "instance" {