From 3c8f088e7add33d74ab552c0f253da76e7825f44 Mon Sep 17 00:00:00 2001 From: someshkoli Date: Thu, 13 May 2021 01:44:18 +0530 Subject: [PATCH 1/9] Adds csv, tsv printer and updates table printer Signed-off-by: someshkoli --- cmd/thanos/tools_bucket.go | 61 ++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index cad437f2b9..1d70c73f28 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -6,6 +6,7 @@ package main import ( "context" "crypto/rand" + "encoding/csv" "encoding/json" "fmt" "io/ioutil" @@ -72,6 +73,14 @@ var ( inspectColumns = []string{"ULID", "FROM", "UNTIL", "RANGE", "UNTIL-DOWN", "#SERIES", "#SAMPLES", "#CHUNKS", "COMP-LEVEL", "COMP-FAILED", "LABELS", "RESOLUTION", "SOURCE"} ) +type outputType string + +const ( + TABLE outputType = "table" + CSV outputType = "csv" + TSV outputType = "tsv" +) + func registerBucket(app extkingpin.AppClause) { cmd := app.Command("bucket", "Bucket utility commands") @@ -273,6 +282,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat sortBy := cmd.Flag("sort-by", "Sort by columns. It's also possible to sort by multiple columns, e.g. '--sort-by FROM --sort-by UNTIL'. I.e., if the 'FROM' value is equal the rows are then further sorted by the 'UNTIL' value."). Default("FROM", "UNTIL").Enums(inspectColumns...) timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() + output := cmd.Flag("output", "Output format for result. Currently supports table, cvs, and, tsv.").Default("table").String() cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error { @@ -316,7 +326,13 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat blockMetas = append(blockMetas, meta) } - return printTable(blockMetas, selectorLabels, *sortBy) + if outputType(*output) == CSV { + return printBlockData(blockMetas, selectorLabels, *sortBy, printCSV) + } + if outputType(*output) == TSV { + return printBlockData(blockMetas, selectorLabels, *sortBy, printTSV) + } + return printBlockData(blockMetas, selectorLabels, *sortBy, printTable) }) } @@ -619,7 +635,36 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat }) } -func printTable(blockMetas []*metadata.Meta, selectorLabels labels.Labels, sortBy []string) error { +type tablePrinter func(t Table) + +func printTable(t Table) { + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader(t.Header) + table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) + table.SetCenterSeparator("|") + table.SetAutoWrapText(false) + table.SetReflowDuringAutoWrap(false) + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.AppendBulk(t.Lines) + table.Render() +} + +func printCSV(t Table) { + csv := csv.NewWriter(os.Stdout) + csv.Write(t.Header) + csv.WriteAll(t.Lines) + csv.Flush() +} + +func printTSV(t Table) { + csv := csv.NewWriter(os.Stdout) + csv.Comma = '\t' + csv.Write(t.Header) + csv.WriteAll(t.Lines) + csv.Flush() +} + +func printBlockData(blockMetas []*metadata.Meta, selectorLabels labels.Labels, sortBy []string, printer tablePrinter) error { header := inspectColumns var lines [][]string @@ -671,17 +716,7 @@ func printTable(blockMetas []*metadata.Meta, selectorLabels labels.Labels, sortB t := Table{Header: header, Lines: lines, SortIndices: sortByColNum} sort.Sort(t) - - table := tablewriter.NewWriter(os.Stdout) - table.SetHeader(t.Header) - table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) - table.SetCenterSeparator("|") - table.SetAutoWrapText(false) - table.SetReflowDuringAutoWrap(false) - table.SetAlignment(tablewriter.ALIGN_LEFT) - table.AppendBulk(t.Lines) - table.Render() - + printer(t) return nil } From b69c0a7d9e3e66037a50abbb1a647c3774fada0c Mon Sep 17 00:00:00 2001 From: someshkoli Date: Mon, 5 Jul 2021 21:13:55 +0530 Subject: [PATCH 2/9] updates docs Signed-off-by: someshkoli --- docs/components/tools.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/components/tools.md b/docs/components/tools.md index 86bd401f89..b6a28a22ec 100644 --- a/docs/components/tools.md +++ b/docs/components/tools.md @@ -431,7 +431,9 @@ Flags: Path to YAML file that contains object store configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration - -l, --selector==\"\" ... + --output="table" Output format for result. Currently supports table, + cvs, and, tsv. + -l, --selector==\"\" ... Selects blocks based on label, e.g. '-l key1=\"value1\" -l key2=\"value2\"'. All key value pairs must match. From 73e3a28f614c950f50eaef194db8808eeab5e41c Mon Sep 17 00:00:00 2001 From: someshkoli Date: Thu, 13 May 2021 02:01:39 +0530 Subject: [PATCH 3/9] adds err handling to csv write Signed-off-by: someshkoli --- cmd/thanos/tools_bucket.go | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 1d70c73f28..67f0bcbb3c 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -635,9 +635,9 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat }) } -type tablePrinter func(t Table) +type tablePrinter func(t Table) error -func printTable(t Table) { +func printTable(t Table) error { table := tablewriter.NewWriter(os.Stdout) table.SetHeader(t.Header) table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) @@ -647,21 +647,36 @@ func printTable(t Table) { table.SetAlignment(tablewriter.ALIGN_LEFT) table.AppendBulk(t.Lines) table.Render() + return nil } -func printCSV(t Table) { +func printCSV(t Table) error { csv := csv.NewWriter(os.Stdout) - csv.Write(t.Header) - csv.WriteAll(t.Lines) + err := csv.Write(t.Header) + if err != nil { + return err + } + err = csv.WriteAll(t.Lines) + if err != nil { + return err + } csv.Flush() + return nil } -func printTSV(t Table) { +func printTSV(t Table) error { csv := csv.NewWriter(os.Stdout) csv.Comma = '\t' - csv.Write(t.Header) - csv.WriteAll(t.Lines) + err := csv.Write(t.Header) + if err != nil { + return err + } + err = csv.WriteAll(t.Lines) + if err != nil { + return err + } csv.Flush() + return nil } func printBlockData(blockMetas []*metadata.Meta, selectorLabels labels.Labels, sortBy []string, printer tablePrinter) error { @@ -716,7 +731,10 @@ func printBlockData(blockMetas []*metadata.Meta, selectorLabels labels.Labels, s t := Table{Header: header, Lines: lines, SortIndices: sortByColNum} sort.Sort(t) - printer(t) + err := printer(t) + if err != nil { + return errors.Errorf("unable to write output.") + } return nil } From 8a1df540234c85718ea37b1a66ba1d6c75f3daed Mon Sep 17 00:00:00 2001 From: someshkoli Date: Thu, 13 May 2021 02:07:21 +0530 Subject: [PATCH 4/9] adds err handling to invalid outputtype Signed-off-by: someshkoli --- cmd/thanos/tools_bucket.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 67f0bcbb3c..34b33c7f24 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -326,13 +326,16 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat blockMetas = append(blockMetas, meta) } - if outputType(*output) == CSV { + switch o := outputType(*output); o { + case CSV: return printBlockData(blockMetas, selectorLabels, *sortBy, printCSV) - } - if outputType(*output) == TSV { + case TSV: return printBlockData(blockMetas, selectorLabels, *sortBy, printTSV) + case TABLE: + return printBlockData(blockMetas, selectorLabels, *sortBy, printTable) + default: + return fmt.Errorf("Invalid output type %s", *output) } - return printBlockData(blockMetas, selectorLabels, *sortBy, printTable) }) } From ebb77f2256ed02ce00fe5d028655f8cad3b7927d Mon Sep 17 00:00:00 2001 From: someshkoli Date: Thu, 13 May 2021 02:32:46 +0530 Subject: [PATCH 5/9] adds err handling to invalid outputtype Signed-off-by: someshkoli --- cmd/thanos/tools_bucket.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 34b33c7f24..1f962f97c4 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -282,7 +282,9 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat sortBy := cmd.Flag("sort-by", "Sort by columns. It's also possible to sort by multiple columns, e.g. '--sort-by FROM --sort-by UNTIL'. I.e., if the 'FROM' value is equal the rows are then further sorted by the 'UNTIL' value."). Default("FROM", "UNTIL").Enums(inspectColumns...) timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() + output := cmd.Flag("output", "Output format for result. Currently supports table, cvs, and, tsv.").Default("table").String() + opType := outputType(*output) cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error { @@ -325,16 +327,14 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat for _, meta := range metas { blockMetas = append(blockMetas, meta) } - - switch o := outputType(*output); o { - case CSV: - return printBlockData(blockMetas, selectorLabels, *sortBy, printCSV) - case TSV: - return printBlockData(blockMetas, selectorLabels, *sortBy, printTSV) - case TABLE: + if opType == TABLE { return printBlockData(blockMetas, selectorLabels, *sortBy, printTable) - default: - return fmt.Errorf("Invalid output type %s", *output) + } else if opType == TSV { + return printBlockData(blockMetas, selectorLabels, *sortBy, printTSV) + } else if opType == CSV { + return printBlockData(blockMetas, selectorLabels, *sortBy, printCSV) + } else { + return fmt.Errorf("Invalid output type %s.", *output) } }) } From f50d2ba633f86f21e0b40b2362a5e5f0733fa721 Mon Sep 17 00:00:00 2001 From: someshkoli Date: Thu, 13 May 2021 16:15:26 +0530 Subject: [PATCH 6/9] adds new tsv writer Signed-off-by: someshkoli --- cmd/thanos/tools_bucket.go | 39 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 1f962f97c4..6bbd441888 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -9,6 +9,7 @@ import ( "encoding/csv" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "os" @@ -71,6 +72,7 @@ var ( }, } inspectColumns = []string{"ULID", "FROM", "UNTIL", "RANGE", "UNTIL-DOWN", "#SERIES", "#SAMPLES", "#CHUNKS", "COMP-LEVEL", "COMP-FAILED", "LABELS", "RESOLUTION", "SOURCE"} + outputTypes = []string{"table", "tsv", "csv"} ) type outputType string @@ -283,8 +285,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat Default("FROM", "UNTIL").Enums(inspectColumns...) timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() - output := cmd.Flag("output", "Output format for result. Currently supports table, cvs, and, tsv.").Default("table").String() - opType := outputType(*output) + output := cmd.Flag("output", "Output format for result. Currently supports table, cvs, and, tsv.").Default("table").Enum(outputTypes...) cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error { @@ -327,15 +328,18 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat for _, meta := range metas { blockMetas = append(blockMetas, meta) } - if opType == TABLE { - return printBlockData(blockMetas, selectorLabels, *sortBy, printTable) - } else if opType == TSV { - return printBlockData(blockMetas, selectorLabels, *sortBy, printTSV) - } else if opType == CSV { - return printBlockData(blockMetas, selectorLabels, *sortBy, printCSV) - } else { - return fmt.Errorf("Invalid output type %s.", *output) + + var opPrinter tablePrinter + op := outputType(*output) + switch op { + case TABLE: + opPrinter = printTable + case TSV: + opPrinter = printTSV + case CSV: + opPrinter = printCSV } + return printBlockData(blockMetas, selectorLabels, *sortBy, opPrinter) }) } @@ -667,18 +671,23 @@ func printCSV(t Table) error { return nil } +func newTSVWriter(w io.Writer) *csv.Writer { + writer := csv.NewWriter(w) + writer.Comma = rune('\t') + return writer +} + func printTSV(t Table) error { - csv := csv.NewWriter(os.Stdout) - csv.Comma = '\t' - err := csv.Write(t.Header) + tsv := newTSVWriter(os.Stdout) + err := tsv.Write(t.Header) if err != nil { return err } - err = csv.WriteAll(t.Lines) + err = tsv.WriteAll(t.Lines) if err != nil { return err } - csv.Flush() + tsv.Flush() return nil } From 7be769638e9a595a982cf211fad4dea81e01f0bc Mon Sep 17 00:00:00 2001 From: someshkoli Date: Thu, 13 May 2021 23:48:10 +0530 Subject: [PATCH 7/9] updates docs Signed-off-by: someshkoli --- cmd/thanos/tools_bucket.go | 2 +- docs/components/tools.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 6bbd441888..816393141e 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -285,7 +285,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat Default("FROM", "UNTIL").Enums(inspectColumns...) timeout := cmd.Flag("timeout", "Timeout to download metadata from remote storage").Default("5m").Duration() - output := cmd.Flag("output", "Output format for result. Currently supports table, cvs, and, tsv.").Default("table").Enum(outputTypes...) + output := cmd.Flag("output", "Output format for result. Currently supports table, cvs, tsv.").Default("table").Enum(outputTypes...) cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error { diff --git a/docs/components/tools.md b/docs/components/tools.md index b6a28a22ec..2b0fe04311 100644 --- a/docs/components/tools.md +++ b/docs/components/tools.md @@ -431,8 +431,8 @@ Flags: Path to YAML file that contains object store configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration - --output="table" Output format for result. Currently supports table, - cvs, and, tsv. + --output=table Output format for result. Currently supports table, + cvs, tsv. -l, --selector==\"\" ... Selects blocks based on label, e.g. '-l key1=\"value1\" -l key2=\"value2\"'. All key value From 639c8ccf63f1d7dc07f1e07bb3aaff8cd2cba08b Mon Sep 17 00:00:00 2001 From: someshkoli Date: Fri, 14 May 2021 03:14:37 +0530 Subject: [PATCH 8/9] added additional prram w to writer Signed-off-by: someshkoli --- cmd/thanos/tools_bucket.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 816393141e..97fb866c7b 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -642,10 +642,10 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat }) } -type tablePrinter func(t Table) error +type tablePrinter func(w io.Writer, t Table) error -func printTable(t Table) error { - table := tablewriter.NewWriter(os.Stdout) +func printTable(w io.Writer, t Table) error { + table := tablewriter.NewWriter(w) table.SetHeader(t.Header) table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) table.SetCenterSeparator("|") @@ -657,8 +657,8 @@ func printTable(t Table) error { return nil } -func printCSV(t Table) error { - csv := csv.NewWriter(os.Stdout) +func printCSV(w io.Writer, t Table) error { + csv := csv.NewWriter(w) err := csv.Write(t.Header) if err != nil { return err @@ -677,8 +677,8 @@ func newTSVWriter(w io.Writer) *csv.Writer { return writer } -func printTSV(t Table) error { - tsv := newTSVWriter(os.Stdout) +func printTSV(w io.Writer, t Table) error { + tsv := newTSVWriter(w) err := tsv.Write(t.Header) if err != nil { return err @@ -743,7 +743,7 @@ func printBlockData(blockMetas []*metadata.Meta, selectorLabels labels.Labels, s t := Table{Header: header, Lines: lines, SortIndices: sortByColNum} sort.Sort(t) - err := printer(t) + err := printer(os.Stdout, t) if err != nil { return errors.Errorf("unable to write output.") } From b040d7457cf7138b0573732e28de655f3c40a9ec Mon Sep 17 00:00:00 2001 From: someshkoli Date: Tue, 9 Nov 2021 02:15:01 +0530 Subject: [PATCH 9/9] update docs and changelog Signed-off-by: someshkoli --- CHANGELOG.md | 1 + docs/components/tools.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faea109d71..38793728b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Added +- [#4228](https://github.com/thanos-io/thanos/pull/4228) Tools `thanos bucket inspect`: Add flag `--output` to provide output method (table,csv,tsv). - [#4394](https://github.com/thanos-io/thanos/pull/4394) Add error logs to receiver when write request rejected with invalid replica - [#4403](https://github.com/thanos-io/thanos/pull/4403) UI: Add sorting and filtering to flags page - [#4299](https://github.com/thanos-io/thanos/pull/4299) Tracing: Add tracing to exemplar APIs. diff --git a/docs/components/tools.md b/docs/components/tools.md index 2b0fe04311..2f92bcbda3 100644 --- a/docs/components/tools.md +++ b/docs/components/tools.md @@ -433,7 +433,7 @@ Flags: https://thanos.io/tip/thanos/storage.md/#configuration --output=table Output format for result. Currently supports table, cvs, tsv. - -l, --selector==\"\" ... + -l, --selector==\"\" ... Selects blocks based on label, e.g. '-l key1=\"value1\" -l key2=\"value2\"'. All key value pairs must match.