Added resource set validation cloudquery#7

Added TableValidation to enforce column/table name longer than 63 char cloudquery#9

provider/migrator.go

@@ -2,6 +2,8 @@ package provider
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"reflect"
 	"runtime"
 	"strconv"
@@ -20,14 +22,27 @@ const queryTableColumns = `SELECT array_agg(column_name::text) as columns FROM i
 
 const addColumnToTable = `ALTER TABLE %s ADD COLUMN IF NOT EXISTS %v %v;`
 
+const maxTableName = 63
+
+const maxColumnName = 63
+
 // Migrator handles creation of schema.Table in database if they don't exist
 type Migrator struct {
-	db  schema.Database
-	log hclog.Logger
+	db         schema.Database
+	log        hclog.Logger
+	validators []TableValidator
 }
 
 func NewMigrator(db schema.Database, log hclog.Logger) Migrator {
-	return Migrator{db, log}
+	return Migrator{
+		db,
+		log,
+		[]TableValidator{
+			LengthTableValidator{
+				log: log,
+			},
+		},
+	}
 }
 
 func (m Migrator) upgradeTable(ctx context.Context, t *schema.Table) error {
@@ -109,3 +124,33 @@ func (m Migrator) buildColumns(ctb *sqlbuilder.CreateTableBuilder, cc []schema.C
 		ctb.Define(defs...)
 	}
 }
+
+func (m Migrator) ValidateTable(t *schema.Table) error {
+	for _, validator := range m.validators {
+		return validator.Validate(t)
+	}
+	return nil
+}
+
+type TableValidator interface {
+	Validate(t *schema.Table) error
+}
+
+type LengthTableValidator struct {
+	log hclog.Logger
+}
+
+func (tv LengthTableValidator) Validate(t *schema.Table) error {
+	if len(t.Name) > maxTableName {
+		tv.log.Error("table name has exceeded max length", "table", t.Name)
+		return errors.New("table name has exceeded max length")
+	}
+
+	for _, col := range t.ColumnNames() {
+		if len(col) > maxColumnName {
+			tv.log.Error("column name has exceeded max length", "column", col)
+			return errors.New(fmt.Sprintf("column name %s has exceeded max length", col))
+		}
+	}
+	return nil
+}

provider/migrator_test.go

@@ -0,0 +1,51 @@
+package provider
+
+import (
+	"github.com/cloudquery/cq-provider-sdk/logging"
+	"github.com/cloudquery/cq-provider-sdk/provider/schema"
+	"github.com/hashicorp/go-hclog"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+var testTable = schema.Table{
+	Name: "test_table_validator",
+	Columns: []schema.Column{
+		{
+			Name: "zero_bool",
+			Type: schema.TypeBool,
+		},
+		{
+			Name: "zero_int",
+			Type: schema.TypeBigInt,
+		},
+		{
+			Name: "not_zero_bool",
+			Type: schema.TypeBool,
+		},
+	},
+}
+
+func TestMigratorTableValidators(t *testing.T) {
+	logger := logging.New(&hclog.LoggerOptions{
+		Level:      hclog.Trace,
+		JSONFormat: true,
+	})
+	m := NewMigrator(nil, logger)
+
+	// table has passed all validators
+	err := m.ValidateTable(&testTable)
+	assert.Nil(t, err)
+
+	// table name is too long
+	tableWithLongName := testTable
+	tableWithLongName.Name = "WithLongNametableWithLongNametableWithLongNametableWithLongNamet"
+	err = m.ValidateTable(&tableWithLongName)
+	assert.Error(t, err)
+
+	// column name is too long
+	tableWithLongColumnName := testTable
+	tableWithLongName.Columns[0].Name = "tableWithLongColumnNametableWithLongColumnNametableWithLongColumnName"
+	err = m.ValidateTable(&tableWithLongColumnName)
+	assert.Error(t, err)
+}

provider/provider.go

@@ -60,6 +60,14 @@ func (p *Provider) Init(_ string, dsn string, _ bool) error {
 	// Create tables
 	m := NewMigrator(p.db, p.Logger)
 	for _, t := range p.ResourceMap {
+
+		// validate table
+		validationErr := m.ValidateTable(t)
+		if validationErr != nil {
+			p.Logger.Error("table validation failed", "table", t.Name, "error", err)
+			return err
+		}
+
 		err := m.CreateTable(context.Background(), t, nil)
 		if err != nil {
 			p.Logger.Error("failed to create table", "table", t.Name, "error", err)

provider/schema/resource.go

@@ -1,6 +1,8 @@
 package schema
 
 import (
+	"fmt"
+	"github.com/cloudquery/go-funk"
 	"github.com/google/uuid"
 )
 
@@ -32,6 +34,10 @@ func (r Resource) Get(key string) interface{} {
 }
 
 func (r Resource) Set(key string, value interface{}) {
+	columnExists := funk.ContainsString(r.table.ColumnNames(), key)
+	if !columnExists {
+		panic(fmt.Sprintf("column %s does not exits", key))
+	}
 	r.data[key] = value
 }
 

provider/schema/resources_test.go

@@ -75,6 +75,9 @@ func TestResourceColumns(t *testing.T) {
 	v, err = r.Values()
 	assert.Nil(t, err)
 	assert.Equal(t, v, []interface{}{r.id, "test", "name_no_prefix", "prefix_name"})
+
+	// check non existing col
+	assert.Panics(t, func() { r.Set("non_exist_col", "test") })
 }
 
 func TestResourceResolveColumns(t *testing.T) {