From b0d88c98d98e68e92699c2b5da81a4ca09f02154 Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Sun, 24 Sep 2023 16:20:44 -0500 Subject: [PATCH 1/2] feat: group comamnds --- .goreleaser.yml | 36 ++++++++++++++++++++++++++---------- cmd/gen.go | 2 +- cmd/info.go | 8 ++++---- cmd/lint.go | 2 +- cmd/list.go | 8 ++++---- cmd/new.go | 4 ++-- cmd/root.go | 2 +- cmd/run.go | 2 +- 8 files changed, 40 insertions(+), 24 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 9a1bd3917..52fb9695e 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -2,13 +2,20 @@ project_name: meteor release: prerelease: auto - draft: true before: hooks: - go mod tidy - make clean +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - "^build:" + builds: - main: ./main.go id: "linux" @@ -59,14 +66,6 @@ checksum: snapshot: name_template: "{{ .Tag }}-next" -changelog: - sort: asc - filters: - exclude: - - "^docs:" - - "^test:" - - "^build:" - dockers: - goos: linux goarch: amd64 @@ -78,6 +77,24 @@ dockers: - "docker.io/raystack/{{.ProjectName}}:{{ .Version }}" - "docker.io/raystack/{{.ProjectName}}:{{ .Version }}-amd64" +nfpms: + - maintainer: Raystack + description: "Metadata collection tool." + homepage: https://github.com/raystack/meteor + license: Apache 2.0 + formats: + - deb + - rpm + - apk + +scoops: + - homepage: "https://github.com/raystack/meteor" + description: "Metadata collection tool." + license: Apache 2.0 + bucket: + owner: raystack + name: scoop-bucket + brews: - name: meteor homepage: "https://github.com/raystack/meteor" @@ -87,7 +104,6 @@ brews: name: homebrew-tap license: "Apache 2.0" folder: Formula - dependencies: - name: git install: |- diff --git a/cmd/gen.go b/cmd/gen.go index 1e1ebfaf8..9e1612a45 100644 --- a/cmd/gen.go +++ b/cmd/gen.go @@ -31,7 +31,7 @@ func GenCmd() *cobra.Command { $ meteor gen my-template.yaml -o ./output-dir -d ./data.yaml `), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, RunE: func(cmd *cobra.Command, args []string) error { templatePath := args[0] diff --git a/cmd/info.go b/cmd/info.go index 0285071ea..fbbeca590 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -17,7 +17,7 @@ func InfoCmd() *cobra.Command { Use: "info ", Short: "Display plugin information", Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, } cmd.AddCommand(InfoSinkCmd()) @@ -41,7 +41,7 @@ func InfoSinkCmd() *cobra.Command { `), Args: cobra.MaximumNArgs(1), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, RunE: func(cmd *cobra.Command, args []string) error { var sinks []string @@ -86,7 +86,7 @@ func InfoExtCmd() *cobra.Command { `), Args: cobra.MaximumNArgs(1), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, RunE: func(cmd *cobra.Command, args []string) error { var extractors []string @@ -130,7 +130,7 @@ func InfoProccCmd() *cobra.Command { `), Args: cobra.MaximumNArgs(1), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, RunE: func(cmd *cobra.Command, args []string) error { var processors []string diff --git a/cmd/lint.go b/cmd/lint.go index d81c3fcc5..261fe9cea 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -45,7 +45,7 @@ func LintCmd() *cobra.Command { $ meteor lint . `), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, RunE: func(cmd *cobra.Command, args []string) error { cfg, err := config.Load("./meteor.yaml") diff --git a/cmd/list.go b/cmd/list.go index 75bb75ce4..4ecd09a35 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -19,7 +19,7 @@ func ListCmd() *cobra.Command { Use: "list ", Short: "List available plugins", Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, } cmd.AddCommand(ListExtCmd()) @@ -49,7 +49,7 @@ func ListExtCmd() *cobra.Command { $ meteor list e `), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, Run: func(cmd *cobra.Command, args []string) { extractors := registry.Extractors.List() @@ -89,7 +89,7 @@ func ListSinksCmd() *cobra.Command { $ meteor list s `), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, Run: func(cmd *cobra.Command, args []string) { sinks := registry.Sinks.List() @@ -128,7 +128,7 @@ func ListProccCmd() *cobra.Command { $ meteor list p `), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, Run: func(cmd *cobra.Command, args []string) { processors := registry.Processors.List() diff --git a/cmd/new.go b/cmd/new.go index b4eca8005..8715c683b 100644 --- a/cmd/new.go +++ b/cmd/new.go @@ -18,7 +18,7 @@ func NewCmd() *cobra.Command { Use: "new", Short: "Bootstrap new recipes", Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, } @@ -58,7 +58,7 @@ func NewRecipeCmd() *cobra.Command { $ meteor new recipe sample -e bigquery -s compass > recipe.yaml `), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, RunE: func(cmd *cobra.Command, args []string) error { var sinkList []string diff --git a/cmd/root.go b/cmd/root.go index f88b27f53..0621f8b3d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,7 +20,7 @@ func New() *cobra.Command { $ meteor gen recipe --extractor=date --sink console `), Annotations: map[string]string{ - "group:core": "true", + "group": "core", "help:learn": heredoc.Doc(` Use 'meteor --help' for more information about a command. Read the manual at https://raystack.github.io/meteor/ diff --git a/cmd/run.go b/cmd/run.go index 37f16177a..d544f0296 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -55,7 +55,7 @@ func RunCmd() *cobra.Command { `), Args: cobra.ExactArgs(1), Annotations: map[string]string{ - "group:core": "true", + "group": "core", }, RunE: func(cmd *cobra.Command, args []string) error { cfg, err := config.Load(configFile) From 32d76ec473e2858e6f0c8785f657ad8667517bbd Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Wed, 29 May 2024 12:12:59 -0500 Subject: [PATCH 2/2] feat(bigquery): adds sql query for VIEW and MATERIALIZED VIEW in asset for BQ --- plugins/extractors/bigquery/bigquery.go | 57 +++++++++++++------------ 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/plugins/extractors/bigquery/bigquery.go b/plugins/extractors/bigquery/bigquery.go index fc4448758..27047644e 100755 --- a/plugins/extractors/bigquery/bigquery.go +++ b/plugins/extractors/bigquery/bigquery.go @@ -415,44 +415,47 @@ func (e *Extractor) buildAsset(ctx context.Context, t *bigquery.Table, md *bigqu } } - table, err := anypb.New(&v1beta2.Table{ - Columns: e.buildColumns(ctx, md.Schema, md), - PreviewFields: previewFields, - PreviewRows: previewRows, - Profile: tableProfile, - Attributes: utils.TryParseMapToProto(map[string]interface{}{ - "full_qualified_name": tableFQN, - "dataset": t.DatasetID, - "project": t.ProjectID, - "type": string(md.Type), - "partition_data": partitionData, - "clustering_fields": clusteringFields, - }), - CreateTime: timestamppb.New(md.CreationTime), - UpdateTime: timestamppb.New(md.LastModifiedTime), - }) - if err != nil { - e.logger.Warn("error creating Any struct", "error", err) - } - asset := &v1beta2.Asset{ Urn: tableURN, Name: t.TableID, Type: "table", Description: md.Description, Service: "bigquery", - Data: table, - Labels: md.Labels, + // Data: table, + Labels: md.Labels, + } + attributesData := map[string]interface{}{ + "full_qualified_name": tableFQN, + "dataset": t.DatasetID, + "project": t.ProjectID, + "type": string(md.Type), + "partition_data": partitionData, + "clustering_fields": clusteringFields, } - if e.config.BuildViewLineage && (md.Type == bigquery.ViewTable || md.Type == bigquery.MaterializedView) { + if md.Type == bigquery.ViewTable || md.Type == bigquery.MaterializedView { query := getViewQuery(md) - upstreamResources := getUpstreamResources(query) - asset.Lineage = &v1beta2.Lineage{ - Upstreams: upstreamResources, + attributesData["sql"] = query + if e.config.BuildViewLineage { + upstreamResources := getUpstreamResources(query) + asset.Lineage = &v1beta2.Lineage{ + Upstreams: upstreamResources, + } } } - + table, err := anypb.New(&v1beta2.Table{ + Columns: e.buildColumns(ctx, md.Schema, md), + PreviewFields: previewFields, + PreviewRows: previewRows, + Profile: tableProfile, + Attributes: utils.TryParseMapToProto(attributesData), + CreateTime: timestamppb.New(md.CreationTime), + UpdateTime: timestamppb.New(md.LastModifiedTime), + }) + if err != nil { + e.logger.Warn("error creating Any struct", "error", err) + } + asset.Data = table return asset, nil }