feat!: IgnoreError Recursively for tables and columns (#323)

* feat: IgnoreError Recursively for tables and columns

* feat: Remove IgnoreError for column field

* add tests

* remove column ignore error per review

* fixed bug

* more fix

* move IgnoreError recursion to TableExecutor

provider/execution/execution.go

@@ -28,6 +28,8 @@ const executionJitter = -1 * time.Minute
 type TableExecutor struct {
 	// ResourceName name of top-level resource associated with table
 	ResourceName string
+	// ParentExecutor is the parent executor, useful for nested tables to propagate up and use IgnoreError and so forth.
+	ParentExecutor *TableExecutor
 	// Table this execution is associated with
 	Table *schema.Table
 	// Database connection to insert data into
@@ -92,6 +94,7 @@ func (e TableExecutor) withTable(t *schema.Table, kv ...interface{}) *TableExecu
 	var c [2]schema.ColumnList
 	c[0], c[1] = e.Db.Dialect().Columns(t).Sift()
 	cpy := e
+	cpy.ParentExecutor = &e
 	cpy.Table = t
 	cpy.Logger = cpy.Logger.With(kv...)
 	cpy.columns = c
@@ -236,7 +239,7 @@ func (e TableExecutor) callTableResolve(ctx context.Context, client schema.Clien
 			close(res)
 		}()
 		if err := e.Table.Resolver(ctx, client, parent, res); err != nil {
-			if e.Table.IgnoreError != nil && e.Table.IgnoreError(err) {
+			if e.IgnoreError(err) {
 				e.Logger.Debug("ignored an error", "err", err)
 				err = diag.NewBaseError(err, diag.RESOLVING, diag.WithSeverity(diag.IGNORE), diag.WithSummary("table %q resolver ignored error", e.Table.Name))
 			}
@@ -412,12 +415,7 @@ func (e TableExecutor) resolveColumns(ctx context.Context, meta schema.ClientMet
 			if funk.ContainsString(e.Db.Dialect().PrimaryKeys(e.Table), c.Name) {
 				return diags.Add(ClassifyError(err, diag.WithResourceName(e.ResourceName), WithResource(resource), diag.WithSummary("failed to resolve column %s@%s", e.Table.Name, c.Name)))
 			}
-			// check if column resolver defined an IgnoreError function, if it does check if ignore should be ignored.
-			if c.IgnoreError != nil && c.IgnoreError(err) {
-				diags = diags.Add(e.handleResolveError(meta, resource, err, diag.WithSeverity(diag.IGNORE), diag.WithSummary("column resolver %q failed for table %q", c.Name, e.Table.Name)))
-			} else {
-				diags = diags.Add(e.handleResolveError(meta, resource, err, diag.WithSummary("column resolver %q failed for table %q", c.Name, e.Table.Name)))
-			}
+			diags = diags.Add(e.handleResolveError(meta, resource, err, diag.WithSummary("column resolver %q failed for table %q", c.Name, e.Table.Name)))
 			continue
 		}
 		e.Logger.Trace("resolving column value with path", "column", c.Name)
@@ -459,6 +457,21 @@ func (e TableExecutor) handleResolveError(meta schema.ClientMeta, r *schema.Reso
 	return errAsDiags
 }
 
+// IsIgnoreError returns true if the error is ignored via the current table IgnoreError function or in any other parent table (in that ordered)
+// it stops checking the moment one of them exists and not until it returns true or fals
+func (e TableExecutor) IgnoreError(err error) bool {
+	// first priority is to check the tables IgnoreError function
+	if e.Table.IgnoreError != nil {
+		return e.Table.IgnoreError(err)
+	}
+	// secondy priority is to check the parent tables IgnoreError recursively
+	if e.ParentExecutor != nil {
+		return e.ParentExecutor.IgnoreError(err)
+	}
+
+	return false
+}
+
 func identifyClient(meta schema.ClientMeta) string {
 	ider, ok := meta.(schema.ClientIdentifier)
 	if ok {

provider/execution/execution_test.go

@@ -613,6 +613,33 @@ func TestTableExecutor_Resolve(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "ignore_error_recursive",
+			Table: &schema.Table{
+				Name:        "simple",
+				Resolver:    returnValueResolver,
+				IgnoreError: func(err error) bool { return true },
+				Columns:     commonColumns,
+				Relations: []*schema.Table{
+					{
+						Name:     "simple",
+						Resolver: returnErrorResolver,
+						Columns:  commonColumns,
+					},
+				},
+			},
+			ErrorExpected:         true,
+			ExpectedResourceCount: 1,
+			ExpectedDiags: []diag.FlatDiag{
+				{
+					Err:      "some error",
+					Resource: "ignore_error_recursive",
+					Severity: diag.IGNORE,
+					Summary:  `table "simple" resolver ignored error: some error`,
+					Type:     diag.RESOLVING,
+				},
+			},
+		},
 	}
 
 	executionClient := executionClient{testlog.New(t)}
@@ -622,6 +649,9 @@ func TestTableExecutor_Resolve(t *testing.T) {
 			if tc.SetupStorage != nil {
 				storage = tc.SetupStorage(t)
 			}
+			if tc.Name == "ignore_error_recursive" {
+				fmt.Println("debug")
+			}
 			limiter := semaphore.NewWeighted(int64(limit.GetMaxGoRoutines()))
 			exec := NewTableExecutor(tc.Name, storage, testlog.New(t), tc.Table, tc.ExtraFields, nil, nil, limiter, 10*time.Second)
 			count, diags := exec.Resolve(context.Background(), executionClient)

provider/schema/column.go

@@ -51,18 +51,14 @@ type Column struct {
 	Description string
 	// Column Resolver allows to set you own data based on resolving this can be an API call or setting multiple embedded values etc'
 	Resolver ColumnResolver
-	// Ignore errors checks if returned error from column resolver should be ignored.
-	IgnoreError IgnoreErrorFunc
 	// Creation options allow modifying how column is defined when table is created
 	CreationOptions ColumnCreationOptions
-
 	// IgnoreInTests is used to skip verifying the column is non-nil in integration tests.
 	// By default, integration tests perform a fetch for all resources in cloudquery's test account, and
 	// verify all columns are non-nil.
 	// If IgnoreInTests is true, verification is skipped for this column.
 	// Used when it is hard to create a reproducible environment with this column being non-nil (e.g. various error columns).
 	IgnoreInTests bool
-
 	// internal is true if this column is managed by the SDK
 	internal bool
 	// meta holds serializable information about the column's resolvers and functions
@@ -305,7 +301,7 @@ func (c Column) Meta() *ColumnMeta {
 	if c.Resolver == nil {
 		return &ColumnMeta{
 			Resolver:     nil,
-			IgnoreExists: c.IgnoreError != nil,
+			IgnoreExists: false,
 		}
 	}
 	fnName := runtime.FuncForPC(reflect.ValueOf(c.Resolver).Pointer()).Name()
@@ -314,7 +310,7 @@ func (c Column) Meta() *ColumnMeta {
 			Name:    strings.TrimPrefix(fnName, "github.com/cloudquery/cq-provider-sdk/provider/"),
 			Builtin: strings.HasPrefix(fnName, "github.com/cloudquery/cq-provider-sdk/"),
 		},
-		IgnoreExists: c.IgnoreError != nil,
+		IgnoreExists: false,
 	}
 }