From 2b349e70acb1dc6b56d4c6e670e6805844d542dd Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Mon, 22 Apr 2024 16:43:21 -0600 Subject: [PATCH 1/4] feat: add detected-fields command to logcli --- cmd/logcli/main.go | 74 +++++++++++++++++++++++++++++++++++ pkg/logcli/client/client.go | 53 +++++++++++++++++++------ pkg/logcli/client/file.go | 17 ++++++-- pkg/logcli/detected/fields.go | 46 ++++++++++++++++++++++ pkg/loghttp/detected.go | 14 +++++++ 5 files changed, 190 insertions(+), 14 deletions(-) create mode 100644 pkg/logcli/detected/fields.go create mode 100644 pkg/loghttp/detected.go diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go index 3d2aa85297b3f..a53e2ea3cfd56 100644 --- a/cmd/logcli/main.go +++ b/cmd/logcli/main.go @@ -16,6 +16,7 @@ import ( "gopkg.in/alecthomas/kingpin.v2" "github.com/grafana/loki/v3/pkg/logcli/client" + "github.com/grafana/loki/v3/pkg/logcli/detected" "github.com/grafana/loki/v3/pkg/logcli/index" "github.com/grafana/loki/v3/pkg/logcli/labelquery" "github.com/grafana/loki/v3/pkg/logcli/output" @@ -253,6 +254,39 @@ Example: 'my-query' `) volumeRangeQuery = newVolumeQuery(true, volumeRangeCmd) + + detectedFieldsCmd = app.Command("detected-fields", `Run a query for detected fields.. + +The "detected-fields" command will return information about fields detected using either +the "logfmt" or "json" parser against the log lines returned by the provided query for the +provided time range. + +The "detected-fields" command will output extra information about the query +and its results, such as the API URL, set of common labels, and set +of excluded labels. This extra information can be suppressed with the +--quiet flag. + +By default we look over the last hour of data; use --since to modify +or provide specific start and end times with --from and --to respectively. + +Notice that when using --from and --to then ensure to use RFC3339Nano +time format, but without timezone at the end. The local timezone will be added +automatically or if using --timezone flag. + +Example: + + logcli detected-fields + --timezone=UTC + --from="2021-01-19T10:00:00Z" + --to="2021-01-19T20:00:00Z" + --output=jsonl + 'my-query' + +The output is limited to 100 fields by default; use --field-limit to increase. +The query is limited to processing 1000 lines per subquery; use --line-limit to increase. +`) + + detectedFieldsQuery = newDetectedFieldsQuery(detectedFieldsCmd) ) func main() { @@ -388,6 +422,8 @@ func main() { } else { index.GetVolume(volumeQuery, queryClient, out, *statistics) } + case detectedFieldsCmd.FullCommand(): + detectedFieldsQuery.Do(queryClient, *statistics) } } @@ -652,3 +688,41 @@ func newVolumeQuery(rangeQuery bool, cmd *kingpin.CmdClause) *volume.Query { return q } + +func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.DetectedFieldsQuery { + // calculate query range from cli params + var from, to string + var since time.Duration + + q := &detected.DetectedFieldsQuery{} + + // executed after all command flags are parsed + cmd.Action(func(c *kingpin.ParseContext) error { + defaultEnd := time.Now() + defaultStart := defaultEnd.Add(-since) + + q.Start = mustParse(from, defaultStart) + q.End = mustParse(to, defaultEnd) + + q.Quiet = *quiet + + return nil + }) + + cmd.Flag("field-limit", "Limit on number of fields to return."). + Default("100"). + IntVar(&q.FieldLimit) + cmd.Flag("line-limit", "Limit the number of lines each subquery is allowed to process."). + Default("1000"). + IntVar(&q.LineLimit) + cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'"). + Required(). + StringVar(&q.QueryString) + cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since) + cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from) + cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to) + cmd.Flag("step", "Query resolution step width, for metric queries. Evaluate the query at the specified step over the time range."). + DurationVar(&q.Step) + + return q +} diff --git a/pkg/logcli/client/client.go b/pkg/logcli/client/client.go index 73ddccd7efd17..8eda13bbdd12d 100644 --- a/pkg/logcli/client/client.go +++ b/pkg/logcli/client/client.go @@ -28,16 +28,17 @@ import ( ) const ( - queryPath = "/loki/api/v1/query" - queryRangePath = "/loki/api/v1/query_range" - labelsPath = "/loki/api/v1/labels" - labelValuesPath = "/loki/api/v1/label/%s/values" - seriesPath = "/loki/api/v1/series" - tailPath = "/loki/api/v1/tail" - statsPath = "/loki/api/v1/index/stats" - volumePath = "/loki/api/v1/index/volume" - volumeRangePath = "/loki/api/v1/index/volume_range" - defaultAuthHeader = "Authorization" + queryPath = "/loki/api/v1/query" + queryRangePath = "/loki/api/v1/query_range" + labelsPath = "/loki/api/v1/labels" + labelValuesPath = "/loki/api/v1/label/%s/values" + seriesPath = "/loki/api/v1/series" + tailPath = "/loki/api/v1/tail" + statsPath = "/loki/api/v1/index/stats" + volumePath = "/loki/api/v1/index/volume" + volumeRangePath = "/loki/api/v1/index/volume_range" + detectedFieldsPath = "/loki/api/v1/detected_fields" + defaultAuthHeader = "Authorization" ) var userAgent = fmt.Sprintf("loki-logcli/%s", build.Version) @@ -54,6 +55,7 @@ type Client interface { GetStats(queryStr string, start, end time.Time, quiet bool) (*logproto.IndexStatsResponse, error) GetVolume(query *volume.Query) (*loghttp.QueryResponse, error) GetVolumeRange(query *volume.Query) (*loghttp.QueryResponse, error) + GetDetectedFields(queryStr string, fieldLimit, lineLimit int, start, end time.Time, step time.Duration, quiet bool) (*loghttp.DetectedFieldsResponse, error) } // Tripperware can wrap a roundtripper. @@ -224,7 +226,36 @@ func (c *DefaultClient) getVolume(path string, query *volume.Query) (*loghttp.Qu return &resp, nil } -func (c *DefaultClient) doQuery(path string, query string, quiet bool) (*loghttp.QueryResponse, error) { +func (c *DefaultClient) GetDetectedFields( + queryStr string, + fieldLimit, lineLimit int, + start, end time.Time, + step time.Duration, + quiet bool, +) (*loghttp.DetectedFieldsResponse, error) { + qsb := util.NewQueryStringBuilder() + qsb.SetString("query", queryStr) + qsb.SetInt("field_limit", int64(fieldLimit)) + qsb.SetInt("line_limit", int64(lineLimit)) + qsb.SetInt("start", start.UnixNano()) + qsb.SetInt("end", end.UnixNano()) + qsb.SetString("step", step.String()) + + var err error + var r loghttp.DetectedFieldsResponse + + if err = c.doRequest(detectedFieldsPath, qsb.Encode(), quiet, &r); err != nil { + return nil, err + } + + return &r, nil +} + +func (c *DefaultClient) doQuery( + path string, + query string, + quiet bool, +) (*loghttp.QueryResponse, error) { var err error var r loghttp.QueryResponse diff --git a/pkg/logcli/client/file.go b/pkg/logcli/client/file.go index dd0432a79e172..34b76422d4bc2 100644 --- a/pkg/logcli/client/file.go +++ b/pkg/logcli/client/file.go @@ -190,17 +190,28 @@ func (f *FileClient) GetOrgID() string { } func (f *FileClient) GetStats(_ string, _, _ time.Time, _ bool) (*logproto.IndexStatsResponse, error) { - // TODO(trevorwhitney): could we teach logcli to read from an actual index file? + // TODO(twhitney): could we teach logcli to read from an actual index file? return nil, ErrNotSupported } func (f *FileClient) GetVolume(_ *volume.Query) (*loghttp.QueryResponse, error) { - // TODO(trevorwhitney): could we teach logcli to read from an actual index file? + // TODO(twhitney): could we teach logcli to read from an actual index file? return nil, ErrNotSupported } func (f *FileClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryResponse, error) { - // TODO(trevorwhitney): could we teach logcli to read from an actual index file? + // TODO(twhitney): could we teach logcli to read from an actual index file? + return nil, ErrNotSupported +} + +func (f *FileClient) GetDetectedFields( + _ string, + _, _ int, + _, _ time.Time, + _ time.Duration, + _ bool, +) (*loghttp.DetectedFieldsResponse, error) { + // TODO(twhitney): could we teach logcli to do this? return nil, ErrNotSupported } diff --git a/pkg/logcli/detected/fields.go b/pkg/logcli/detected/fields.go new file mode 100644 index 0000000000000..61ad6b3e8aaf4 --- /dev/null +++ b/pkg/logcli/detected/fields.go @@ -0,0 +1,46 @@ +package detected + +import ( + "fmt" + "log" + "slices" + "strings" + "time" + + "github.com/fatih/color" + "github.com/grafana/loki/v3/pkg/logcli/client" + "github.com/grafana/loki/v3/pkg/loghttp" +) + +type DetectedFieldsQuery struct { + QueryString string + Start time.Time + End time.Time + FieldLimit int + LineLimit int + Step time.Duration + Quiet bool + ColoredOutput bool +} + +// DoQuery executes the query and prints out the results +func (q *DetectedFieldsQuery) Do(c client.Client, statistics bool) { + var resp *loghttp.DetectedFieldsResponse + var err error + + resp, err = c.GetDetectedFields(q.QueryString, q.FieldLimit, q.LineLimit, q.Start, q.End, q.Step, q.Quiet) + if err != nil { + log.Fatalf("Error doing request: %+v", err) + } + + output := make([]string, len(resp.Fields)) + for i, field := range resp.Fields { + bold := color.New(color.Bold) + output[i] = fmt.Sprintf("label: %s\t\t", bold.Sprintf("%s", field.Label)) + + fmt.Sprintf("type: %s\t\t", bold.Sprintf("%s", field.Type)) + + fmt.Sprintf("cardinality: %s", bold.Sprintf("%d", field.Cardinality)) + } + + slices.Sort(output) + fmt.Println(strings.Join(output, "\n")) +} diff --git a/pkg/loghttp/detected.go b/pkg/loghttp/detected.go new file mode 100644 index 0000000000000..724bacf6167b7 --- /dev/null +++ b/pkg/loghttp/detected.go @@ -0,0 +1,14 @@ +package loghttp + +import "github.com/grafana/loki/v3/pkg/logproto" + +// LabelResponse represents the http json response to a label query +type DetectedFieldsResponse struct { + Fields []DetectedField`json:"fields,omitempty"` +} + +type DetectedField struct { + Label string `json:"label,omitempty"` + Type logproto.DetectedFieldType `json:"type,omitempty"` + Cardinality uint64 `json:"cardinality,omitempty"` +} From d1a9c05d99e268aed537324396d95393d5c1d239 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Mon, 22 Apr 2024 16:54:09 -0600 Subject: [PATCH 2/4] feat: add raw output mode to detected fields query --- cmd/logcli/main.go | 2 +- pkg/logcli/detected/fields.go | 30 ++++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go index a53e2ea3cfd56..7985e54e86586 100644 --- a/cmd/logcli/main.go +++ b/cmd/logcli/main.go @@ -423,7 +423,7 @@ func main() { index.GetVolume(volumeQuery, queryClient, out, *statistics) } case detectedFieldsCmd.FullCommand(): - detectedFieldsQuery.Do(queryClient, *statistics) + detectedFieldsQuery.Do(queryClient, *statistics, *outputMode) } } diff --git a/pkg/logcli/detected/fields.go b/pkg/logcli/detected/fields.go index 61ad6b3e8aaf4..99d5a53c07b43 100644 --- a/pkg/logcli/detected/fields.go +++ b/pkg/logcli/detected/fields.go @@ -1,6 +1,7 @@ package detected import ( + "encoding/json" "fmt" "log" "slices" @@ -24,7 +25,7 @@ type DetectedFieldsQuery struct { } // DoQuery executes the query and prints out the results -func (q *DetectedFieldsQuery) Do(c client.Client, statistics bool) { +func (q *DetectedFieldsQuery) Do(c client.Client, statistics bool, outputMode string) { var resp *loghttp.DetectedFieldsResponse var err error @@ -33,14 +34,23 @@ func (q *DetectedFieldsQuery) Do(c client.Client, statistics bool) { log.Fatalf("Error doing request: %+v", err) } - output := make([]string, len(resp.Fields)) - for i, field := range resp.Fields { - bold := color.New(color.Bold) - output[i] = fmt.Sprintf("label: %s\t\t", bold.Sprintf("%s", field.Label)) + - fmt.Sprintf("type: %s\t\t", bold.Sprintf("%s", field.Type)) + - fmt.Sprintf("cardinality: %s", bold.Sprintf("%d", field.Cardinality)) - } + switch outputMode { + case "raw": + out, err := json.Marshal(resp) + if err != nil { + log.Fatalf("Error marshalling response: %+v", err) + } + fmt.Println(string(out)) + default: + output := make([]string, len(resp.Fields)) + for i, field := range resp.Fields { + bold := color.New(color.Bold) + output[i] = fmt.Sprintf("label: %s\t\t", bold.Sprintf("%s", field.Label)) + + fmt.Sprintf("type: %s\t\t", bold.Sprintf("%s", field.Type)) + + fmt.Sprintf("cardinality: %s", bold.Sprintf("%d", field.Cardinality)) + } - slices.Sort(output) - fmt.Println(strings.Join(output, "\n")) + slices.Sort(output) + fmt.Println(strings.Join(output, "\n")) + } } From 9c1ec87381cd7389e8f6bbda064bc366738cd70a Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Mon, 22 Apr 2024 17:14:44 -0600 Subject: [PATCH 3/4] ci: fix linting --- cmd/logcli/main.go | 6 +++--- pkg/logcli/client/client.go | 2 +- pkg/logcli/detected/fields.go | 4 ++-- pkg/logcli/query/query_test.go | 10 ++++++++++ pkg/loghttp/detected.go | 6 +++--- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go index 7985e54e86586..e4e04da8d6657 100644 --- a/cmd/logcli/main.go +++ b/cmd/logcli/main.go @@ -423,7 +423,7 @@ func main() { index.GetVolume(volumeQuery, queryClient, out, *statistics) } case detectedFieldsCmd.FullCommand(): - detectedFieldsQuery.Do(queryClient, *statistics, *outputMode) + detectedFieldsQuery.Do(queryClient, *outputMode) } } @@ -689,12 +689,12 @@ func newVolumeQuery(rangeQuery bool, cmd *kingpin.CmdClause) *volume.Query { return q } -func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.DetectedFieldsQuery { +func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.FieldsQuery { // calculate query range from cli params var from, to string var since time.Duration - q := &detected.DetectedFieldsQuery{} + q := &detected.FieldsQuery{} // executed after all command flags are parsed cmd.Action(func(c *kingpin.ParseContext) error { diff --git a/pkg/logcli/client/client.go b/pkg/logcli/client/client.go index 8eda13bbdd12d..e417ccfa3ce52 100644 --- a/pkg/logcli/client/client.go +++ b/pkg/logcli/client/client.go @@ -230,7 +230,7 @@ func (c *DefaultClient) GetDetectedFields( queryStr string, fieldLimit, lineLimit int, start, end time.Time, - step time.Duration, + step time.Duration, quiet bool, ) (*loghttp.DetectedFieldsResponse, error) { qsb := util.NewQueryStringBuilder() diff --git a/pkg/logcli/detected/fields.go b/pkg/logcli/detected/fields.go index 99d5a53c07b43..ff95160a859a7 100644 --- a/pkg/logcli/detected/fields.go +++ b/pkg/logcli/detected/fields.go @@ -13,7 +13,7 @@ import ( "github.com/grafana/loki/v3/pkg/loghttp" ) -type DetectedFieldsQuery struct { +type FieldsQuery struct { QueryString string Start time.Time End time.Time @@ -25,7 +25,7 @@ type DetectedFieldsQuery struct { } // DoQuery executes the query and prints out the results -func (q *DetectedFieldsQuery) Do(c client.Client, statistics bool, outputMode string) { +func (q *FieldsQuery) Do(c client.Client, outputMode string) { var resp *loghttp.DetectedFieldsResponse var err error diff --git a/pkg/logcli/query/query_test.go b/pkg/logcli/query/query_test.go index c7543fa2288de..fffdcf15444b6 100644 --- a/pkg/logcli/query/query_test.go +++ b/pkg/logcli/query/query_test.go @@ -485,6 +485,16 @@ func (t *testQueryClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryRespons panic("not implemented") } +func (t *testQueryClient) GetDetectedFields( + _ string, + _, _ int, + _, _ time.Time, + _ time.Duration, + _ bool, +) (*loghttp.DetectedFieldsResponse, error) { + panic("not implemented") +} + var legacySchemaConfigContents = `schema_config: configs: - from: 2020-05-15 diff --git a/pkg/loghttp/detected.go b/pkg/loghttp/detected.go index 724bacf6167b7..d255bf6124a75 100644 --- a/pkg/loghttp/detected.go +++ b/pkg/loghttp/detected.go @@ -4,11 +4,11 @@ import "github.com/grafana/loki/v3/pkg/logproto" // LabelResponse represents the http json response to a label query type DetectedFieldsResponse struct { - Fields []DetectedField`json:"fields,omitempty"` + Fields []DetectedField `json:"fields,omitempty"` } type DetectedField struct { - Label string `json:"label,omitempty"` + Label string `json:"label,omitempty"` Type logproto.DetectedFieldType `json:"type,omitempty"` - Cardinality uint64 `json:"cardinality,omitempty"` + Cardinality uint64 `json:"cardinality,omitempty"` } From c7a5d77bbd75758f848c56a7e58a1e0c2ee4f867 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Tue, 23 Apr 2024 08:59:39 -0600 Subject: [PATCH 4/4] fix: format --- pkg/logcli/detected/fields.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/logcli/detected/fields.go b/pkg/logcli/detected/fields.go index ff95160a859a7..f8ba585ea2a00 100644 --- a/pkg/logcli/detected/fields.go +++ b/pkg/logcli/detected/fields.go @@ -9,6 +9,7 @@ import ( "time" "github.com/fatih/color" + "github.com/grafana/loki/v3/pkg/logcli/client" "github.com/grafana/loki/v3/pkg/loghttp" )