Fixed CR

Added resource set validation cloudquery#7

Added TableValidation to enforce column/table name longer than 63 char cloudquery#9

provider/migrator.go

@@ -2,8 +2,6 @@ package provider
 
 import (
 	"context"
-	"errors"
-	"fmt"
 	"reflect"
 	"runtime"
 	"strconv"
@@ -22,26 +20,16 @@ const queryTableColumns = `SELECT array_agg(column_name::text) as columns FROM i
 
 const addColumnToTable = `ALTER TABLE %s ADD COLUMN IF NOT EXISTS %v %v;`
 
-const maxTableName = 63
-
-const maxColumnName = 63
-
 // Migrator handles creation of schema.Table in database if they don't exist
 type Migrator struct {
-	db         schema.Database
-	log        hclog.Logger
-	validators []TableValidator
+	db  schema.Database
+	log hclog.Logger
 }
 
 func NewMigrator(db schema.Database, log hclog.Logger) Migrator {
 	return Migrator{
 		db,
 		log,
-		[]TableValidator{
-			LengthTableValidator{
-				log: log,
-			},
-		},
 	}
 }
 
@@ -124,33 +112,3 @@ func (m Migrator) buildColumns(ctb *sqlbuilder.CreateTableBuilder, cc []schema.C
 		ctb.Define(defs...)
 	}
 }
-
-func (m Migrator) ValidateTable(t *schema.Table) error {
-	for _, validator := range m.validators {
-		return validator.Validate(t)
-	}
-	return nil
-}
-
-type TableValidator interface {
-	Validate(t *schema.Table) error
-}
-
-type LengthTableValidator struct {
-	log hclog.Logger
-}
-
-func (tv LengthTableValidator) Validate(t *schema.Table) error {
-	if len(t.Name) > maxTableName {
-		tv.log.Error("table name has exceeded max length", "table", t.Name)
-		return errors.New("table name has exceeded max length")
-	}
-
-	for _, col := range t.ColumnNames() {
-		if len(col) > maxColumnName {
-			tv.log.Error("column name has exceeded max length", "column", col)
-			return errors.New(fmt.Sprintf("column name %s has exceeded max length", col))
-		}
-	}
-	return nil
-}

provider/provider.go

@@ -62,7 +62,7 @@ func (p *Provider) Init(_ string, dsn string, _ bool) error {
 	for _, t := range p.ResourceMap {
 
 		// validate table
-		validationErr := m.ValidateTable(t)
+		validationErr := schema.ValidateTable(t)
 		if validationErr != nil {
 			p.Logger.Error("table validation failed", "table", t.Name, "error", err)
 			return err

provider/schema/execution.go

@@ -155,7 +155,10 @@ func (e ExecutionData) resolveColumns(ctx context.Context, meta ClientMeta, reso
 		if v == nil {
 			v = c.Default
 		}
-		resource.Set(c.Name, v)
+		err := resource.Set(c.Name, v)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

provider/schema/execution_test.go

@@ -152,7 +152,8 @@ func TestExecutionData_ResolveTable(t *testing.T) {
 		testTable.Resolver = dataReturningSingleResolver
 		var expectedResource *Resource
 		testTable.PostResourceResolver = func(ctx context.Context, meta ClientMeta, parent *Resource) error {
-			parent.Set("name", "other")
+			err := parent.Set("name", "other")
+			assert.Nil(t, err)
 			expectedResource = parent
 			return nil
 		}

provider/schema/resolvers.go

@@ -9,13 +9,19 @@ import (
 
 func PathResolver(path string) ColumnResolver {
 	return func(_ context.Context, meta ClientMeta, r *Resource, c Column) error {
-		r.Set(c.Name, funk.GetAllowZero(r.Item, path))
+		err := r.Set(c.Name, funk.GetAllowZero(r.Item, path))
+		if err != nil {
+			return err
+		}
 		return nil
 	}
 }
 
 func ParentIdResolver(_ context.Context, _ ClientMeta, r *Resource, c Column) error {
-	r.Set(c.Name, r.Parent.Id())
+	err := r.Set(c.Name, r.Parent.Id())
+	if err != nil {
+		return err
+	}
 	return nil
 }
 

provider/schema/resolvers_test.go

@@ -24,6 +24,22 @@ func TestPathResolver(t *testing.T) {
 	resource := &Resource{
 		Item: testStruct{Inner: innerStruct{Value: "bla"}, Value: 5, unexported: false},
 		data: map[string]interface{}{},
+		table: &Table{
+			Columns: []Column{
+				{
+					Name: "test",
+					Type: TypeString,
+				},
+				{
+					Name: "int_value",
+					Type: TypeInt,
+				},
+				{
+					Name: "unexported",
+					Type: TypeBool,
+				},
+			},
+		},
 	}
 	err := r1(context.TODO(), nil, resource, Column{Name: "test"})
 

provider/schema/resource.go

@@ -2,6 +2,7 @@ package schema
 
 import (
 	"fmt"
+
 	"github.com/cloudquery/go-funk"
 	"github.com/google/uuid"
 )
@@ -33,12 +34,13 @@ func (r Resource) Get(key string) interface{} {
 	return r.data[key]
 }
 
-func (r Resource) Set(key string, value interface{}) {
+func (r Resource) Set(key string, value interface{}) error {
 	columnExists := funk.ContainsString(r.table.ColumnNames(), key)
 	if !columnExists {
-		panic(fmt.Sprintf("column %s does not exits", key))
+		return fmt.Errorf("column %s does not exits", key)
 	}
 	r.data[key] = value
+	return nil
 }
 
 func (r Resource) Id() uuid.UUID {

provider/schema/resources_test.go

@@ -57,27 +57,33 @@ type zeroValuedStruct struct {
 func TestResourceColumns(t *testing.T) {
 
 	r := NewResourceData(testTable, nil, nil)
-	r.Set("name", "test")
+	errf := r.Set("name", "test")
+	assert.Nil(t, errf)
 	assert.Equal(t, r.Get("name"), "test")
 	v, err := r.Values()
 	assert.Nil(t, err)
 	assert.Equal(t, v, []interface{}{r.id, "test", nil, nil})
 	// Set invalid type to resource
-	r.Set("name", 5)
+	errf = r.Set("name", 5)
+	assert.Nil(t, errf)
 	v, err = r.Values()
 	assert.Error(t, err)
 	assert.Nil(t, v)
 
 	// Set resource fully
-	r.Set("name", "test")
-	r.Set("name_no_prefix", "name_no_prefix")
-	r.Set("prefix_name", "prefix_name")
+	errf = r.Set("name", "test")
+	assert.Nil(t, errf)
+	errf = r.Set("name_no_prefix", "name_no_prefix")
+	assert.Nil(t, errf)
+	errf = r.Set("prefix_name", "prefix_name")
+	assert.Nil(t, errf)
 	v, err = r.Values()
 	assert.Nil(t, err)
 	assert.Equal(t, v, []interface{}{r.id, "test", "name_no_prefix", "prefix_name"})
 
 	// check non existing col
-	assert.Panics(t, func() { r.Set("non_exist_col", "test") })
+	err = r.Set("non_exist_col", "test")
+	assert.Error(t, err)
 }
 
 func TestResourceResolveColumns(t *testing.T) {

provider/schema/validators.go

@@ -0,0 +1,54 @@
+package schema
+
+import (
+	"errors"
+	"fmt"
+)
+
+const maxTableName = 63
+
+const maxColumnName = 63
+
+var defaultValidators = []TableValidator{
+	LengthTableValidator{},
+}
+
+func ValidateTable(t *Table) error {
+	for _, validator := range defaultValidators {
+		return validator.Validate(t)
+	}
+	return nil
+}
+
+type TableValidator interface {
+	Validate(t *Table) error
+}
+
+type LengthTableValidator struct{}
+
+func validateTableAttributesNameLength(t *Table) error {
+	// validate table name
+	if len(t.Name) > maxTableName {
+		return errors.New("table name has exceeded max length")
+	}
+
+	// validate table columns
+	for _, col := range t.ColumnNames() {
+		if len(col) > maxColumnName {
+			return fmt.Errorf("column name %s has exceeded max length", col)
+		}
+	}
+
+	// validate table relations
+	for _, rel := range t.Relations {
+		err := validateTableAttributesNameLength(rel)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (tv LengthTableValidator) Validate(t *Table) error {
+	return validateTableAttributesNameLength(t)
+}

provider/schema/validators_test.go

@@ -0,0 +1,43 @@
+package schema
+
+import (
+	"github.com/stretchr/testify/assert"
+
+	"testing"
+)
+
+var testTableValidators = Table{
+	Name: "test_table_validator",
+	Columns: []Column{
+		{
+			Name: "zero_bool",
+			Type: TypeBool,
+		},
+		{
+			Name: "zero_int",
+			Type: TypeBigInt,
+		},
+		{
+			Name: "not_zero_bool",
+			Type: TypeBool,
+		},
+	},
+}
+
+func TestTableValidators(t *testing.T) {
+	// table has passed all validators
+	err := ValidateTable(&testTableValidators)
+	assert.Nil(t, err)
+
+	// table name is too long
+	tableWithLongName := testTableValidators
+	tableWithLongName.Name = "WithLongNametableWithLongNametableWithLongNametableWithLongNamet"
+	err = ValidateTable(&tableWithLongName)
+	assert.Error(t, err)
+
+	// column name is too long
+	tableWithLongColumnName := testTableValidators
+	tableWithLongName.Columns[0].Name = "tableWithLongColumnNametableWithLongColumnNametableWithLongColumnName"
+	err = ValidateTable(&tableWithLongColumnName)
+	assert.Error(t, err)
+}