diff --git a/.goreleaser.yml b/.goreleaser.yml
index 522349d91..512916d60 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -13,21 +13,13 @@ builds:
- main: ./main.go
id: "meteor"
binary: meteor
- flags:
- - -a
+ flags: [-a]
ldflags:
- -X github.com/odpf/meteor/cmd.Version={{.Tag}}
- -X github.com/odpf/meteor/cmd.BuildCommit={{.FullCommit}}
- -X github.com/odpf/meteor/cmd.BuildDate={{.Date}}
- goos:
- - darwin
- - linux
- - windows
- goarch:
- - amd64
- - 386
- - arm
- - arm64
+ goos: [darwin, linux, windows]
+ goarch: [amd64, 386, arm, arm64]
env:
- CGO_ENABLED=0
@@ -70,16 +62,19 @@ dockers:
brews:
- name: meteor
+ homepage: "https://github.com/odpf/meteor"
+ description: "Metadata collection tool."
tap:
owner: odpf
name: homebrew-taps
license: "Apache 2.0"
folder: Formula
+
dependencies:
- name: git
install: |-
bin.install "meteor"
commit_author:
- name: Stewart Jingga
- email: stewart_jingga@yahoo.com
+ name: Ravi Suhag
+ email: ravi.suhag@gmail.com
\ No newline at end of file
diff --git a/README.md b/README.md
index c1bdd259a..3d50ff49f 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
Meteor is a plugin driven agent for collecting metadata. Meteor has plugins to source metadata from a variety of data stores, services and message queues. It also has sink plugins to send metadata to variety of third party APIs and catalog services.
-
+
## Key Features
diff --git a/cmd/gen.go b/cmd/gen.go
index 2c145af64..ca93b00ee 100644
--- a/cmd/gen.go
+++ b/cmd/gen.go
@@ -42,8 +42,7 @@ func GenRecipeCmd() *cobra.Command {
The recipe will be printed on standard output.
Specify recipe name with the first argument without extension.
- Use commma to separate multiple sinks and processors.
- `),
+ Use commma to separate multiple sinks and processors.`),
Example: heredoc.Doc(`
# generate a recipe with a bigquery extractor and a console sink
$ meteor gen recipe sample -e bigquery -s console
@@ -61,7 +60,6 @@ func GenRecipeCmd() *cobra.Command {
sinkList := strings.Split(sinks, ",")
procList := strings.Split(processors, ",")
- // TODO: validate extractor and sink names
return generator.Recipe(args[0], extractor, sinkList, procList)
},
}
diff --git a/cmd/info.go b/cmd/info.go
index 8461163df..3d7de8c39 100644
--- a/cmd/info.go
+++ b/cmd/info.go
@@ -34,10 +34,10 @@ func InfoSinkCmd() *cobra.Command {
Long: heredoc.Doc(`
Display sink information.
- The list of supported sinks is available via the 'meteor list sinks' command.
- `),
+ The list of supported sinks is available via the 'meteor list sinks' command.`),
Example: heredoc.Doc(`
$ meteor info sink console
+ $ meteor info sink columbus
`),
Args: cobra.ExactArgs(1),
Annotations: map[string]string{
@@ -68,6 +68,7 @@ func InfoExtCmd() *cobra.Command {
`),
Example: heredoc.Doc(`
$ meteor info extractor postgres
+ $ meteor info extractor bigquery
`),
Args: cobra.ExactArgs(1),
Annotations: map[string]string{
diff --git a/cmd/lint.go b/cmd/lint.go
index 210891b5e..25e062140 100644
--- a/cmd/lint.go
+++ b/cmd/lint.go
@@ -2,6 +2,7 @@ package cmd
import (
"fmt"
+ "os"
"github.com/MakeNowJust/heredoc"
"github.com/odpf/meteor/agent"
@@ -9,6 +10,7 @@ import (
"github.com/odpf/meteor/recipe"
"github.com/odpf/meteor/registry"
"github.com/odpf/salt/log"
+ "github.com/odpf/salt/printer"
"github.com/odpf/salt/term"
"github.com/spf13/cobra"
@@ -25,8 +27,7 @@ func LintCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command {
Check for issues specified recipes.
Linters are run on the recipe files in the specified path.
- If no path is specified, the current directory is used.
- `),
+ If no path is specified, the current directory is used.`),
Example: heredoc.Doc(`
$ meteor lint recipe.yml
@@ -50,32 +51,40 @@ func LintCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command {
}
if len(recipes) == 0 {
- fmt.Println(cs.Yellowf("no recipe found in [%s]", args[0]))
+ fmt.Println(cs.Yellowf("No recipe found in [%s]", args[0]))
fmt.Println(cs.Blue("\nUse 'meteor gen recipe' to generate a new recipe."))
return nil
}
- report := []string{""}
+ report := [][]string{}
var success = 0
var failures = 0
+
+ // Run linters and generate report
for _, recipe := range recipes {
errs := runner.Validate(recipe)
+ var row = []string{}
if len(errs) > 0 {
for _, err := range errs {
lg.Error(err.Error())
}
- report = append(report, fmt.Sprint(cs.FailureIcon(), cs.Redf(" %d errors found is recipe %s", len(errs), recipe.Name)))
+ row = []string{fmt.Sprintf("%s %s", cs.FailureIcon(), recipe.Name), cs.Greyf("(%d errors, 0 warnings)", len(errs))}
failures++
- continue
+ } else {
+ row = []string{fmt.Sprintf("%s %s", cs.SuccessIcon(), recipe.Name), cs.Greyf("(%d errors, 0 warnings)", len(errs))}
+ success++
}
- report = append(report, fmt.Sprint(cs.SuccessIcon(), cs.Greenf(" 0 error found in recipe %s", recipe.Name)))
- success++
+ report = append(report, row)
}
- for _, line := range report {
- fmt.Println(line)
+ // Print the report
+ if failures > 0 {
+ fmt.Println("\nSome checks were not successful")
+ } else {
+ fmt.Println("\nAll checks were successful")
}
- fmt.Printf("Total: %d, Success: %d, Failures: %d\n", len(recipes), success, failures)
+ fmt.Printf("%d failing, %d successful, and %d total\n\n", failures, success, len(recipes))
+ printer.Table(os.Stdout, report)
return nil
},
}
diff --git a/cmd/list.go b/cmd/list.go
index 1c17379d8..0a7e910dd 100644
--- a/cmd/list.go
+++ b/cmd/list.go
@@ -3,11 +3,13 @@ package cmd
import (
"fmt"
"os"
+ "strings"
"github.com/MakeNowJust/heredoc"
"github.com/odpf/meteor/registry"
"github.com/odpf/salt/log"
"github.com/odpf/salt/printer"
+ "github.com/odpf/salt/term"
"github.com/spf13/cobra"
)
@@ -51,9 +53,19 @@ func ListExtCmd() *cobra.Command {
"group:core": "true",
},
Run: func(cmd *cobra.Command, args []string) {
+ cs := term.NewColorScheme()
+
extractors := registry.Extractors.List()
fmt.Printf(" \nShowing %d of %d extractors\n \n", len(extractors), len(extractors))
- printer.Table(os.Stdout, extractors)
+
+ report := [][]string{}
+ index := 1
+
+ for n, i := range extractors {
+ report = append(report, []string{cs.Greenf("#%02d", index), n, i.Description, cs.Greyf(" (%s)", strings.Join(i.Tags, ", "))})
+ index++
+ }
+ printer.Table(os.Stdout, report)
},
}
return cmd
@@ -70,8 +82,7 @@ func ListSinksCmd() *cobra.Command {
This command lists all available sinks.
Sinks are used to send data to a target.
- For example, you can use a sink to send metadata to standard output.
- `),
+ For example, you can use a sink to send metadata to standard output.`),
Example: heredoc.Doc(`
$ meteor list sinks
@@ -82,9 +93,18 @@ func ListSinksCmd() *cobra.Command {
"group:core": "true",
},
Run: func(cmd *cobra.Command, args []string) {
+ cs := term.NewColorScheme()
+
sinks := registry.Sinks.List()
fmt.Printf(" \nShowing %d of %d sinks\n \n", len(sinks), len(sinks))
- printer.Table(os.Stdout, sinks)
+
+ report := [][]string{}
+ index := 1
+ for n, i := range sinks {
+ report = append(report, []string{cs.Greenf("#%02d", index), n, i.Description, cs.Greyf(" (%s)", strings.Join(i.Tags, ", "))})
+ index++
+ }
+ printer.Table(os.Stdout, report)
},
}
return cmd
@@ -101,8 +121,7 @@ func ListProccCmd() *cobra.Command {
This command lists all available processors.
Processors are used to transform data before it is sent to a sink.
- For example, you can use a processor to enrich custom attributes.
- `),
+ For example, you can use a processor to enrich custom attributes.`),
Example: heredoc.Doc(`
$ meteor list processors
@@ -113,9 +132,19 @@ func ListProccCmd() *cobra.Command {
"group:core": "true",
},
Run: func(cmd *cobra.Command, args []string) {
+ cs := term.NewColorScheme()
+
processors := registry.Processors.List()
fmt.Printf(" \nShowing %d of %d processors\n \n", len(processors), len(processors))
- printer.Table(os.Stdout, processors)
+
+ report := [][]string{}
+ index := 1
+
+ for n, i := range processors {
+ report = append(report, []string{cs.Greenf("#%02d", index), n, i.Description, cs.Greyf(" (%s)", strings.Join(i.Tags, ", "))})
+ index++
+ }
+ printer.Table(os.Stdout, report)
},
}
return cmd
diff --git a/cmd/root.go b/cmd/root.go
index 03aa6486f..d7172e6f0 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -19,17 +19,20 @@ func New(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command {
Meteor is a plugin driven agent for collecting metadata.
Meteor has plugins to source metadata from a variety of data stores,
services and message queues. It also has sink plugins to send metadata
- to variety of third party APIs and catalog services.
- `),
+ to variety of third party APIs and catalog services.`),
SilenceErrors: true,
SilenceUsage: false,
Example: heredoc.Doc(`
$ meteor list extractors
$ meteor run recipe.yaml
- $ meteor gen recipe --extractor=date
+ $ meteor gen recipe --extractor=date --sink console
`),
Annotations: map[string]string{
"group:core": "true",
+ "help:learn": heredoc.Doc(`
+ Use 'meteor --help' for more information about a command.
+ Read the manual at https://odpf.gitbook.io/meteor/
+ `),
"help:feedback": heredoc.Doc(`
Open an issue here https://github.com/odpf/meteor/issues
`),
diff --git a/cmd/run.go b/cmd/run.go
index 01fbda342..126b47e2d 100644
--- a/cmd/run.go
+++ b/cmd/run.go
@@ -27,11 +27,8 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command {
A recipe is a set of instructions and configurations defined by user,
and in Meteor they are used to define how metadata will be collected.
- If a recipe file is provided, recipe will be
- executed as a single recipe.
- If a recipe directory is provided, recipes will
- be executed as a group of recipes.
- `),
+ If a recipe file is provided, recipe will be executed as a single recipe.
+ If a recipe directory is provided, recipes will be executed as a group of recipes.`),
Example: heredoc.Doc(`
$ meteor run recipe.yml
@@ -56,36 +53,38 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command {
}
if len(recipes) == 0 {
- fmt.Println(cs.WarningIcon(), cs.Yellowf(" no recipe found in [%s]", args[0]))
+ fmt.Println(cs.WarningIcon(), cs.Yellowf("No recipe found in [%s]", args[0]))
return nil
}
+ report := [][]string{}
var success = 0
var failures = 0
- tabular_report := [][]string{}
- tabular_report = append(tabular_report, []string{"Recipe", "Source", "Status", "Duration(ms)"})
+
+ // Run recipes and collect results
runs := runner.RunMultiple(recipes)
for _, run := range runs {
lg.Debug("recipe details", "recipe", run.Recipe)
- report_row := []string{run.Recipe.Name, run.Recipe.Source.Type}
-
+ row := []string{}
if run.Error != nil {
lg.Error(run.Error.Error(), "recipe")
failures++
- report_row = append(report_row, cs.FailureIcon()+cs.Redf("failure"))
+ row = append(row, fmt.Sprintf("%s %s", cs.FailureIcon(), run.Recipe.Name), cs.Grey(run.Recipe.Source.Type), cs.Greyf("%v ms", strconv.Itoa(run.DurationInMs)))
} else {
success++
- report_row = append(report_row, cs.SuccessIcon()+cs.Greenf("successful"))
+ row = append(row, fmt.Sprintf("%s %s", cs.SuccessIcon(), run.Recipe.Name), cs.Grey(run.Recipe.Source.Type), cs.Greyf("%v ms", strconv.Itoa(run.DurationInMs)))
}
-
- report_row = append(report_row, strconv.Itoa(run.DurationInMs))
- tabular_report = append(tabular_report, report_row)
+ report = append(report, row)
}
- fmt.Print("\n\n")
- printer.Table(os.Stdout, tabular_report)
- fmt.Printf("Total: %d, Success: %d, Failures: %d\n", len(recipes), success, failures)
-
+ // Print the report
+ if failures > 0 {
+ fmt.Println("\nSome recipes were not successful")
+ } else {
+ fmt.Println("\nAll recipes ran successful")
+ }
+ fmt.Printf("%d failing, %d successful, and %d total\n\n", failures, success, len(recipes))
+ printer.Table(os.Stdout, report)
return nil
},
}
diff --git a/docs/assets/.DS_Store b/docs/assets/.DS_Store
new file mode 100644
index 000000000..5008ddfcf
Binary files /dev/null and b/docs/assets/.DS_Store differ
diff --git a/docs/assets/overview_4.svg b/docs/assets/overview_4.svg
new file mode 100644
index 000000000..b442096da
--- /dev/null
+++ b/docs/assets/overview_4.svg
@@ -0,0 +1,142 @@
+
+
\ No newline at end of file
diff --git a/go.mod b/go.mod
index 5716c8ad0..b3434a9ba 100644
--- a/go.mod
+++ b/go.mod
@@ -31,7 +31,7 @@ require (
github.com/mitchellh/mapstructure v1.4.1
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/muesli/reflow v0.3.0
- github.com/odpf/salt v0.0.0-20210910080317-2357c9c5287c
+ github.com/odpf/salt v0.0.0-20210919015538-3fd8ab22acea
github.com/opencontainers/runc v1.0.1 // indirect
github.com/ory/dockertest/v3 v3.7.0
github.com/pkg/errors v0.9.1
diff --git a/go.sum b/go.sum
index 26ca954d9..674450a09 100644
--- a/go.sum
+++ b/go.sum
@@ -461,6 +461,10 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/odpf/salt v0.0.0-20210910080317-2357c9c5287c h1:jaekBiaCyiRd1qhq7a3zOsifM5uOtQeUlQOt7QAnPHY=
github.com/odpf/salt v0.0.0-20210910080317-2357c9c5287c/go.mod h1:/KxVYLrR36QFVSlaS3IL1b/hZHmLdXuZu+cukpyvxok=
+github.com/odpf/salt v0.0.0-20210919001227-5e300ea77aea h1:6+eJn/KpecACT39vMKDeLcg2oHjfh1e2tfClKEXA2Z8=
+github.com/odpf/salt v0.0.0-20210919001227-5e300ea77aea/go.mod h1:a0e+Px6w5350jq9THbrOFuRf7ooL9BHXF3r3tBBq1co=
+github.com/odpf/salt v0.0.0-20210919015538-3fd8ab22acea h1:/wywsQrCWV76QE5otGicnB8XojTXyLvg8QMfbnk/7qQ=
+github.com/odpf/salt v0.0.0-20210919015538-3fd8ab22acea/go.mod h1:a0e+Px6w5350jq9THbrOFuRf7ooL9BHXF3r3tBBq1co=
github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
diff --git a/plugins/extractors/bigquery/bigquery.go b/plugins/extractors/bigquery/bigquery.go
index 54d0f85fa..c49f642ac 100644
--- a/plugins/extractors/bigquery/bigquery.go
+++ b/plugins/extractors/bigquery/bigquery.go
@@ -74,7 +74,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Big Query table metadata and metrics",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"gcp,table,extractor"},
+ Tags: []string{"gcp", "table", "extractor"},
}
}
diff --git a/plugins/extractors/bigtable/bigtable.go b/plugins/extractors/bigtable/bigtable.go
index 59584d4c7..78e7f59d4 100644
--- a/plugins/extractors/bigtable/bigtable.go
+++ b/plugins/extractors/bigtable/bigtable.go
@@ -59,7 +59,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Compressed, high-performance, proprietary data storage system.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"gcp,extractor"},
+ Tags: []string{"gcp", "extractor"},
}
}
diff --git a/plugins/extractors/cassandra/cassandra.go b/plugins/extractors/cassandra/cassandra.go
index 978652727..ab14f21a5 100644
--- a/plugins/extractors/cassandra/cassandra.go
+++ b/plugins/extractors/cassandra/cassandra.go
@@ -4,6 +4,7 @@ import (
"context"
_ "embed" // used to print the embedded assets
"fmt"
+
"github.com/gocql/gocql"
"github.com/odpf/meteor/models"
_ "github.com/odpf/meteor/models"
@@ -33,7 +34,7 @@ type Config struct {
UserID string `mapstructure:"user_id" validate:"required"`
Password string `mapstructure:"password" validate:"required"`
Host string `mapstructure:"host" validate:"required"`
- Port int `mapstructure:"port" validate:"required"`
+ Port int `mapstructure:"port" validate:"required"`
}
var sampleConfig = `
@@ -46,10 +47,10 @@ var sampleConfig = `
// Extractor manages the extraction of data from cassandra
type Extractor struct {
excludedKeyspaces map[string]bool
- logger log.Logger
- config Config
- session *gocql.Session
- emit plugins.Emit
+ logger log.Logger
+ config Config
+ session *gocql.Session
+ emit plugins.Emit
}
// New returns a pointer to an initialized Extractor Object
@@ -65,7 +66,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Table metadata from cassandra server.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"oss,extractor"},
+ Tags: []string{"oss", "extractor"},
}
}
@@ -193,11 +194,11 @@ func (e *Extractor) extractColumns(keyspace string, tableName string) (columns [
}
columns = append(columns, &facets.Column{
- Name: fieldName,
- DataType: dataType,
+ Name: fieldName,
+ DataType: dataType,
})
}
-
+
return
}
diff --git a/plugins/extractors/clickhouse/clickhouse.go b/plugins/extractors/clickhouse/clickhouse.go
index e7901b771..81286aa76 100644
--- a/plugins/extractors/clickhouse/clickhouse.go
+++ b/plugins/extractors/clickhouse/clickhouse.go
@@ -53,7 +53,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Column-oriented DBMS for online analytical processing.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"oss,extractor"},
+ Tags: []string{"oss", "extractor"},
}
}
diff --git a/plugins/extractors/csv/csv.go b/plugins/extractors/csv/csv.go
index 196c90e43..ac683b563 100644
--- a/plugins/extractors/csv/csv.go
+++ b/plugins/extractors/csv/csv.go
@@ -55,7 +55,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Comma separated file",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"file,extractor"},
+ Tags: []string{"file", "extractor"},
}
}
diff --git a/plugins/extractors/date/date.go b/plugins/extractors/date/date.go
index 82de109ba..cf79c648b 100644
--- a/plugins/extractors/date/date.go
+++ b/plugins/extractors/date/date.go
@@ -37,7 +37,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Print current date from system",
SampleConfig: "",
Summary: summary,
- Tags: []string{"system,extractor"},
+ Tags: []string{"system", "extractor"},
}
}
diff --git a/plugins/extractors/elastic/elastic.go b/plugins/extractors/elastic/elastic.go
index 89c08fbd2..ac8e988ec 100644
--- a/plugins/extractors/elastic/elastic.go
+++ b/plugins/extractors/elastic/elastic.go
@@ -50,7 +50,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Search engine based on the Lucene library.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"oss,extractor"},
+ Tags: []string{"oss", "extractor"},
}
}
diff --git a/plugins/extractors/gcs/gcs.go b/plugins/extractors/gcs/gcs.go
index 34cdbc97b..5090bf9f2 100644
--- a/plugins/extractors/gcs/gcs.go
+++ b/plugins/extractors/gcs/gcs.go
@@ -69,7 +69,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Online file storage web service for storing and accessing data.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"gcp,extractor"},
+ Tags: []string{"gcp", "extractor"},
}
}
diff --git a/plugins/extractors/github/github.go b/plugins/extractors/github/github.go
index b56cec78c..e15b64de4 100644
--- a/plugins/extractors/github/github.go
+++ b/plugins/extractors/github/github.go
@@ -41,7 +41,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "User list from Github organisation.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"platform,extractor"},
+ Tags: []string{"platform", "extractor"},
}
}
diff --git a/plugins/extractors/grafana/grafana.go b/plugins/extractors/grafana/grafana.go
index 826f6e652..e002c53a7 100644
--- a/plugins/extractors/grafana/grafana.go
+++ b/plugins/extractors/grafana/grafana.go
@@ -48,7 +48,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Dashboard list from Grafana server.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"oss,extractor"},
+ Tags: []string{"oss", "extractor"},
}
}
diff --git a/plugins/extractors/kafka/kafka.go b/plugins/extractors/kafka/kafka.go
index 0cf15b8d7..18961b0d3 100644
--- a/plugins/extractors/kafka/kafka.go
+++ b/plugins/extractors/kafka/kafka.go
@@ -49,7 +49,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Topic list from Apache Kafka.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"oss,extractor"},
+ Tags: []string{"oss", "extractor"},
}
}
diff --git a/plugins/extractors/metabase/metabase.go b/plugins/extractors/metabase/metabase.go
index a69d7a5bd..b07b35bab 100644
--- a/plugins/extractors/metabase/metabase.go
+++ b/plugins/extractors/metabase/metabase.go
@@ -58,7 +58,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Dashboard list from Metabase server.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"oss,extractor"},
+ Tags: []string{"oss", "extractor"},
}
}
diff --git a/plugins/extractors/mongodb/mongodb.go b/plugins/extractors/mongodb/mongodb.go
index 702df411b..c97185093 100644
--- a/plugins/extractors/mongodb/mongodb.go
+++ b/plugins/extractors/mongodb/mongodb.go
@@ -63,7 +63,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Collection metadata from MongoDB Server",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"oss,extractor"},
+ Tags: []string{"oss", "extractor"},
}
}
diff --git a/plugins/extractors/mssql/mssql.go b/plugins/extractors/mssql/mssql.go
index 99c99634c..f8c4439a7 100644
--- a/plugins/extractors/mssql/mssql.go
+++ b/plugins/extractors/mssql/mssql.go
@@ -8,7 +8,7 @@ import (
"github.com/odpf/salt/log"
- _ "github.com/denisenkom/go-mssqldb"
+ _ "github.com/denisenkom/go-mssqldb"
"github.com/odpf/meteor/models"
"github.com/odpf/meteor/plugins"
"github.com/odpf/meteor/registry"
@@ -63,7 +63,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Table metdata from MSSQL server",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"microsoft,extractor"},
+ Tags: []string{"microsoft", "extractor"},
}
}
diff --git a/plugins/extractors/mysql/mysql.go b/plugins/extractors/mysql/mysql.go
index 4519ae8a3..b605a3d08 100644
--- a/plugins/extractors/mysql/mysql.go
+++ b/plugins/extractors/mysql/mysql.go
@@ -6,11 +6,11 @@ import (
_ "embed" // used to print the embedded assets
"fmt"
+ _ "github.com/go-sql-driver/mysql"
"github.com/odpf/meteor/models"
"github.com/odpf/meteor/models/odpf/assets"
"github.com/odpf/meteor/models/odpf/assets/common"
"github.com/odpf/meteor/models/odpf/assets/facets"
- _ "github.com/go-sql-driver/mysql"
"github.com/odpf/meteor/plugins"
"github.com/odpf/meteor/registry"
"github.com/odpf/meteor/utils"
@@ -61,7 +61,7 @@ func (e *Extractor) Info() plugins.Info {
Description: "Table metadata from MySQL server.",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"oss,extractor"},
+ Tags: []string{"oss", "extractor"},
}
}
diff --git a/plugins/processors/enrich/processor.go b/plugins/processors/enrich/processor.go
index b787c9c7f..751148459 100644
--- a/plugins/processors/enrich/processor.go
+++ b/plugins/processors/enrich/processor.go
@@ -35,7 +35,7 @@ func (p *Processor) Info() plugins.Info {
Description: "Append custom fields to records",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"processor"},
+ Tags: []string{"processor", "transform"},
}
}
diff --git a/plugins/sinks/columbus/sink.go b/plugins/sinks/columbus/sink.go
index e65dee9c8..694d0704f 100644
--- a/plugins/sinks/columbus/sink.go
+++ b/plugins/sinks/columbus/sink.go
@@ -56,7 +56,7 @@ func (s *Sink) Info() plugins.Info {
Description: "Send metadata to columbus http service",
SampleConfig: sampleConfig,
Summary: summary,
- Tags: []string{"http,sink"},
+ Tags: []string{"http", "sink"},
}
}
diff --git a/registry/extractors.go b/registry/extractors.go
index 548a7d6fd..f883794b5 100644
--- a/registry/extractors.go
+++ b/registry/extractors.go
@@ -1,8 +1,6 @@
package registry
import (
- "strings"
-
"github.com/odpf/meteor/plugins"
"github.com/pkg/errors"
)
@@ -30,12 +28,13 @@ func (f *ExtractorFactory) Info(name string) (plugins.Info, error) {
}
// List returns a list of registered Extractors.
-func (f *ExtractorFactory) List() (names [][]string) {
+func (f *ExtractorFactory) List() map[string]plugins.Info {
+ list := make(map[string]plugins.Info)
for name := range f.fnStore {
info, _ := f.Info(name)
- names = append(names, []string{name, info.Description, strings.Join(info.Tags, ",")})
+ list[name] = info
}
- return
+ return list
}
// Register registers an Extractor.
diff --git a/registry/processors.go b/registry/processors.go
index d70b1e481..03024baa4 100644
--- a/registry/processors.go
+++ b/registry/processors.go
@@ -1,8 +1,6 @@
package registry
import (
- "strings"
-
"github.com/odpf/meteor/plugins"
"github.com/pkg/errors"
)
@@ -30,12 +28,13 @@ func (f *ProcessorFactory) Info(name string) (info plugins.Info, err error) {
}
// List returns a list of registered processors.
-func (f *ProcessorFactory) List() (names [][]string) {
+func (f *ProcessorFactory) List() map[string]plugins.Info {
+ list := make(map[string]plugins.Info)
for name := range f.fnStore {
info, _ := f.Info(name)
- names = append(names, []string{name, info.Description, strings.Join(info.Tags, ",")})
+ list[name] = info
}
- return
+ return list
}
// Register registers a Processor.
diff --git a/registry/sinks.go b/registry/sinks.go
index 105f97caa..a29913241 100644
--- a/registry/sinks.go
+++ b/registry/sinks.go
@@ -1,8 +1,6 @@
package registry
import (
- "strings"
-
"github.com/odpf/meteor/plugins"
"github.com/pkg/errors"
)
@@ -30,12 +28,13 @@ func (f *SinkFactory) Info(name string) (info plugins.Info, err error) {
}
// List returns a list of registered Sinks.
-func (f *SinkFactory) List() (names [][]string) {
+func (f *SinkFactory) List() map[string]plugins.Info {
+ list := make(map[string]plugins.Info)
for name := range f.fnStore {
info, _ := f.Info(name)
- names = append(names, []string{name, info.Description, strings.Join(info.Tags, ",")})
+ list[name] = info
}
- return
+ return list
}
// Register registers a Sink.