Provider refeactor for new CQ sdk (#6)

* Provider refeactor for new CQ sdk

* Update cloudquery v0.10.0

.gitignore

@@ -11,3 +11,5 @@ vendor/
 bin/
 .release-env
 cq-provider-aws
+*.exe
+*.log

go.mod

@@ -28,10 +28,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sns v1.1.1
 	github.com/aws/aws-sdk-go-v2/service/sts v1.1.1
 	github.com/aws/smithy-go v1.1.0
-	github.com/cloudquery/cloudquery v0.9.6
+	github.com/cloudquery/cloudquery v0.10.0
 	github.com/gocarina/gocsv v0.0.0-20201208093247-67c824bc04d4
+	github.com/hashicorp/go-hclog v0.15.0
 	github.com/mitchellh/mapstructure v1.3.3
-	go.uber.org/zap v1.16.0
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

main.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"github.com/cloudquery/cloudquery/sdk"
+	"github.com/cloudquery/cq-provider-aws/provider"
+	"github.com/hashicorp/go-hclog"
+)
+
+func main() {
+	logger := hclog.New(&hclog.LoggerOptions{
+		Level:      hclog.Trace,
+		JSONFormat: true,
+	})
+
+	p := &provider.Provider{Logger: logger.Named("aws")}
+
+	sdk.ServePlugin(sdk.ServeOpts{
+		Name:                "aws",
+		Provider:            p,
+		Logger:              logger,
+		NoLogOutputOverride: false,
+	})
+}

config.go → provider/config.go

@@ -1,4 +1,4 @@
-package main
+package provider
 
 const configYaml = `
   - name: aws

provider.go → provider/provider.go

@@ -1,4 +1,4 @@
-package main
+package provider
 
 import (
 	"context"
@@ -8,49 +8,45 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/aws/smithy-go"
-	"github.com/cloudquery/cq-provider-aws/cloudwatch"
-	"github.com/cloudquery/cq-provider-aws/cloudwatchlogs"
-	"github.com/cloudquery/cq-provider-aws/directconnect"
-	"github.com/cloudquery/cq-provider-aws/ec2"
-	"github.com/cloudquery/cq-provider-aws/ecr"
-	"github.com/cloudquery/cq-provider-aws/ecs"
-	"github.com/cloudquery/cq-provider-aws/efs"
-	"github.com/cloudquery/cq-provider-aws/elasticbeanstalk"
-	"github.com/cloudquery/cq-provider-aws/elbv2"
-	"github.com/cloudquery/cq-provider-aws/emr"
-	"github.com/cloudquery/cq-provider-aws/fsx"
-	"github.com/cloudquery/cq-provider-aws/iam"
-	"github.com/cloudquery/cq-provider-aws/kms"
-	"github.com/cloudquery/cq-provider-aws/organizations"
-	"github.com/cloudquery/cq-provider-aws/rds"
-	"github.com/cloudquery/cq-provider-aws/redshift"
-	"github.com/cloudquery/cq-provider-aws/s3"
-	"github.com/cloudquery/cq-provider-aws/sns"
+	"github.com/cloudquery/cq-provider-aws/resources/autoscaling"
+	"github.com/cloudquery/cq-provider-aws/resources/cloudtrail"
+	"github.com/cloudquery/cq-provider-aws/resources/cloudwatch"
+	"github.com/cloudquery/cq-provider-aws/resources/cloudwatchlogs"
+	"github.com/cloudquery/cq-provider-aws/resources/directconnect"
+	"github.com/cloudquery/cq-provider-aws/resources/ec2"
+	"github.com/cloudquery/cq-provider-aws/resources/ecr"
+	"github.com/cloudquery/cq-provider-aws/resources/ecs"
+	"github.com/cloudquery/cq-provider-aws/resources/efs"
+	"github.com/cloudquery/cq-provider-aws/resources/elasticbeanstalk"
+	"github.com/cloudquery/cq-provider-aws/resources/elbv2"
+	"github.com/cloudquery/cq-provider-aws/resources/emr"
+	"github.com/cloudquery/cq-provider-aws/resources/fsx"
+	"github.com/cloudquery/cq-provider-aws/resources/iam"
+	"github.com/cloudquery/cq-provider-aws/resources/kms"
+	"github.com/cloudquery/cq-provider-aws/resources/organizations"
+	"github.com/cloudquery/cq-provider-aws/resources/rds"
+	"github.com/cloudquery/cq-provider-aws/resources/redshift"
+	"github.com/cloudquery/cq-provider-aws/resources/s3"
+	"github.com/cloudquery/cq-provider-aws/resources/sns"
+	"github.com/hashicorp/go-hclog"
 	"golang.org/x/sync/errgroup"
-
-	"github.com/cloudquery/cloudquery/cqlog"
-	"github.com/cloudquery/cloudquery/sdk"
-	"github.com/cloudquery/cq-provider-aws/autoscaling"
-	"github.com/cloudquery/cq-provider-aws/cloudtrail"
 	"strings"
 	"sync"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/cloudquery/cloudquery/database"
-	"github.com/cloudquery/cq-provider-aws/resource"
-	"go.uber.org/zap"
+	"github.com/cloudquery/cq-provider-aws/resources/resource"
 	"gopkg.in/yaml.v3"
 )
 
 type Provider struct {
-	cfg				aws.Config
+	cfg             aws.Config
 	region          string
 	db              *database.Database
 	config          Config
 	accountID       string
 	resourceClients map[string]resource.ClientInterface
-	log             *zap.Logger
-	//logLevel        aws.LogLevelType
+	Logger          hclog.Logger
 }
 
 type Account struct {
@@ -71,7 +67,7 @@ type Config struct {
 
 var globalCollectedResources = map[string]bool{}
 
-type ServiceNewFunction func(awsConfig aws.Config, db *database.Database, log *zap.Logger, accountID string, region string) resource.ClientInterface
+type ServiceNewFunction func(awsConfig aws.Config, db *database.Database, log hclog.Logger, accountID string, region string) resource.ClientInterface
 
 var globalServices = map[string]ServiceNewFunction{
 	"iam":           iam.NewClient,
@@ -151,10 +147,8 @@ func (p *Provider) Init(driver string, dsn string, verbose bool) error {
 		return err
 	}
 
-	zapLogger, err := cqlog.NewLogger(verbose)
-	p.log = zapLogger
 	p.resourceClients = map[string]resource.ClientInterface{}
-	p.log.Info("Creating tables if needed")
+	p.Logger.Info("Creating tables if needed")
 	for _, tables := range tablesArr {
 		err := p.db.AutoMigrate(tables...)
 		if err != nil {
@@ -168,27 +162,7 @@ func (p *Provider) GenConfig() (string, error) {
 	return configYaml, nil
 }
 
-//func (p *Provider) parseLogLevel() {
-//	if p.config.LogLevel == nil {
-//		return
-//	}
-//	switch *p.config.LogLevel {
-//	case "debug", "debug_with_signing":
-//		p.logLevel = aws.LogDebug
-//	case "debug_with_http_body":
-//		p.logLevel = aws.LogDebugWithSigning
-//	case "debug_with_request_retries":
-//		p.logLevel = aws.LogDebugWithRequestRetries
-//	case "debug_with_request_error":
-//		p.logLevel = aws.LogDebugWithRequestErrors
-//	case "debug_with_event_stream_body":
-//		p.logLevel = aws.LogDebugWithEventStreamBody
-//	default:
-//		log.Fatalf("unknown log_level %s", *p.config.LogLevel)
-//	}
-//}
-
-var allRegions = []string {
+var allRegions = []string{
 	"us-east-1",
 	"us-east-2",
 	"us-west-1",
@@ -222,13 +196,13 @@ func (p *Provider) Fetch(data []byte) error {
 	}
 
 	if len(p.config.Resources) == 0 {
-		p.log.Info("no resources specified. See available resources: see: https://docs.cloudquery.io/aws/tables-reference")
+		p.Logger.Info("no resources specified. See available resources: see: https://docs.cloudquery.io/aws/tables-reference")
 		return nil
 	}
 	regions := p.config.Regions
 	if len(regions) == 0 {
 		regions = allRegions
-		p.log.Info(fmt.Sprintf("No regions specified in config.yml. Assuming all %d regions", len(regions)))
+		p.Logger.Info(fmt.Sprintf("No regions specified in config.yml. Assuming all %d regions", len(regions)))
 	}
 
 	if len(p.config.Accounts) == 0 {
@@ -268,7 +242,6 @@ func (p *Provider) Fetch(data []byte) error {
 		}
 		p.accountID = *output.Account
 
-
 		for _, region := range regions {
 			p.region = region
 
@@ -278,7 +251,7 @@ func (p *Provider) Fetch(data []byte) error {
 			})
 			if err != nil {
 				if errors.As(err, &ae) && (ae.ErrorCode() == "InvalidClientTokenId" || ae.ErrorCode() == "OptInRequired") {
-					p.log.Info("region disabled. skipping...", zap.String("region", region))
+					p.Logger.Info("region disabled. skipping...", "region", region)
 					continue
 				}
 				return err
@@ -306,10 +279,10 @@ func (p *Provider) Fetch(data []byte) error {
 }
 
 func (p *Provider) initRegionalClients() {
-	zapLog := p.log.With(zap.String("account_id", p.accountID), zap.String("region", p.region))
+	innerLog := p.Logger.With("account_id", p.accountID, "region", p.region)
 	for serviceName, newFunc := range regionalServices {
 		p.resourceClients[serviceName] = newFunc(p.cfg,
-			p.db, zapLog, p.accountID, p.region)
+			p.db, innerLog, p.accountID, p.region)
 	}
 }
 
@@ -331,9 +304,9 @@ func (p *Provider) collectResource(fullResourceName string, config interface{})
 		}
 		globalCollectedResources[fullResourceName] = true
 		if p.resourceClients[service] == nil {
-			zapLog := p.log.With(zap.String("account_id", p.accountID))
+			innerLogger := p.Logger.With("account_id", p.accountID)
 			p.resourceClients[service] = globalServices[service](p.cfg,
-				p.db, zapLog, p.accountID, p.region)
+				p.db, innerLogger, p.accountID, p.region)
 		}
 		lock.Unlock()
 	}
@@ -348,18 +321,15 @@ func (p *Provider) collectResource(fullResourceName string, config interface{})
 		if errors.As(err, &ae) {
 			switch ae.ErrorCode() {
 			case "AccessDenied", "AccessDeniedException", "UnauthorizedOperation":
-				p.log.Info("Skipping resource. Access denied", zap.String("account_id", p.accountID), zap.String("region", p.region), zap.String("resource", fullResourceName), zap.Error(err))
+				p.Logger.Info("Skipping resource. Access denied", "account_id", p.accountID, "region", p.region, "resource", fullResourceName, "error", err)
 				return nil
 			case "OptInRequired", "SubscriptionRequiredException":
-				p.log.Info("Skipping resource. Service disabled", zap.String("account_id", p.accountID), zap.String("region", p.region), zap.String("resource", fullResourceName), zap.Error(err))
+				p.Logger.Info("Skipping resource. Service disabled", "account_id", p.accountID, "region", p.region, "resource", fullResourceName, "error", err)
+
 				return nil
 			}
 		}
 		return err
 	}
 	return nil
 }
-
-func main() {
-	sdk.ServePlugin(&Provider{})
-}

autoscaling/client.go → resources/autoscaling/client.go

@@ -5,19 +5,19 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/autoscaling"
 	"github.com/cloudquery/cloudquery/database"
-	"github.com/cloudquery/cq-provider-aws/resource"
-	"go.uber.org/zap"
+	"github.com/cloudquery/cq-provider-aws/resources/resource"
+	"github.com/hashicorp/go-hclog"
 )
 
 type Client struct {
 	db        *database.Database
-	log       *zap.Logger
+	log  hclog.Logger
 	accountID string
 	region    string
 	svc       *autoscaling.Client
 }
 
-func NewClient(awsConfig aws.Config, db *database.Database, log *zap.Logger,
+func NewClient(awsConfig aws.Config, db *database.Database, log hclog.Logger,
 	accountID string, region string) resource.ClientInterface {
 	return &Client{
 		db:        db,

autoscaling/launch_configurations.go → resources/autoscaling/launch_configurations.go

@@ -7,7 +7,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
 	//"github.com/cloudquery/cq-provider-aws/common"
 	"github.com/mitchellh/mapstructure"
-	"go.uber.org/zap"
 	"time"
 )
 
@@ -165,7 +164,7 @@ func (c *Client) launchConfigurations(gConfig interface{}) error {
 			return err
 		}
 		c.db.ChunkedCreate(c.transformLaunchConfigurations(output.LaunchConfigurations))
-		c.log.Info("Fetched resources", zap.String("resource", "auto_scaling.launch_configurations"), zap.Int("count", len(output.LaunchConfigurations)))
+		c.log.Info("Fetched resources", "resource", "auto_scaling.launch_configurations", "count", len(output.LaunchConfigurations))
 		if aws.ToString(output.NextToken) == "" {
 			break
 		}

cloudtrail/client.go → resources/cloudtrail/client.go

@@ -5,19 +5,19 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/cloudtrail"
 	"github.com/cloudquery/cloudquery/database"
-	"github.com/cloudquery/cq-provider-aws/resource"
-	"go.uber.org/zap"
+	"github.com/cloudquery/cq-provider-aws/resources/resource"
+	"github.com/hashicorp/go-hclog"
 )
 
 type Client struct {
 	db        *database.Database
-	log       *zap.Logger
+	log  hclog.Logger
 	accountID string
 	region    string
 	svc       *cloudtrail.Client
 }
 
-func NewClient(awsConfig aws.Config, db *database.Database, log *zap.Logger,
+func NewClient(awsConfig aws.Config, db *database.Database, log hclog.Logger,
 	accountID string, region string) resource.ClientInterface {
 	return &Client{
 		db:        db,

cloudtrail/event_selectors.go → resources/cloudtrail/event_selectors.go

@@ -22,7 +22,7 @@ func (EventSelector) TableName() string {
 func (c *Client) transformEventSelectors(values []types.EventSelector) []*EventSelector {
 	var tValues []*EventSelector
 	for _, value := range values {
-		tValue := EventSelector {
+		tValue := EventSelector{
 			AccountID: c.accountID,
 			Region: c.region,
 			IncludeManagementEvents: value.IncludeManagementEvents,

cloudtrail/trails.go → resources/cloudtrail/trails.go

@@ -5,7 +5,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/cloudtrail"
 	"github.com/aws/aws-sdk-go-v2/service/cloudtrail/types"
 	"github.com/mitchellh/mapstructure"
-	"go.uber.org/zap"
 	"regexp"
 	"time"
 )
@@ -31,7 +30,7 @@ type Trail struct {
 	S3KeyPrefix                *string
 	SnsTopicARN                *string
 	SnsTopicName               *string
-	TrailARN                   *string `neo:"unique"`
+	TrailARN                   *string            `neo:"unique"`
 	EventSelectors 			   []*EventSelector `gorm:"constraint:OnDelete:CASCADE;"`
 	IsLogging *bool
 
@@ -68,7 +67,7 @@ func (c *Client) transformTrails(ctx context.Context, values []types.Trail) ([]*
 		if value.CloudWatchLogsLogGroupArn != nil {
 			matches := groupNameRegex.FindStringSubmatch(*value.CloudWatchLogsLogGroupArn)
 			if len(matches) < 2 {
-				c.log.Warn("CloudWatchLogsLogGroupARN doesn't fit standard regex", zap.String("arn", *value.CloudWatchLogsLogGroupArn))
+				c.log.Warn("CloudWatchLogsLogGroupARN doesn't fit standard regex", "arn", *value.CloudWatchLogsLogGroupArn)
 			} else {
 				groupName = matches[1]
 			}
@@ -155,7 +154,7 @@ func (c *Client) trails(gConfig interface{}) error {
 		return err
 	}
 	c.db.ChunkedCreate(tValues)
-	c.log.Info("Fetched resources", zap.String("resource", "cloudtrail.trails"), zap.Int("count", len(output.TrailList)))
+	c.log.Info("Fetched resources", "resource", "cloudtrail.trails", "count", len(output.TrailList))
 
 	return nil
 }

cloudwatch/alarms.go → resources/cloudwatch/alarms.go

@@ -6,7 +6,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
 	"github.com/mitchellh/mapstructure"
-	"go.uber.org/zap"
 	"time"
 )
 
@@ -84,7 +83,7 @@ func (MetricAlarmMetric) TableName() string {
 func (c *Client) transformMetricAlarms(values *[]types.MetricAlarm) []*MetricAlarm {
 	var tValues []*MetricAlarm
 	for _, value := range *values {
-		tValue := MetricAlarm {
+		tValue := MetricAlarm{
 			AccountID: c.accountID,
 			Region: c.region,
 			ActionsEnabled: value.ActionsEnabled,
@@ -179,7 +178,7 @@ func (c *Client)alarms(gConfig interface{}) error {
 			return err
 		}
 		c.db.ChunkedCreate(c.transformMetricAlarms(&output.MetricAlarms))
-		c.log.Info("Fetched resources", zap.String("resource", "cloudwatch.alarms"), zap.Int("count", len(output.MetricAlarms)))
+		c.log.Info("Fetched resources", "resource", "cloudwatch.alarms", "count", len(output.MetricAlarms))
 		if aws.ToString(output.NextToken) == "" {
 			break
 		}