Skip to content

Commit 7983f94

Browse files
trevorwhitneytrevorwhitney
authored
feat: detected field values (#14350)
1 parent e9efcb5 commit 7983f94

31 files changed

+909
-415
lines changed

‎cmd/logcli/main.go

+7-3
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ func newVolumeQuery(rangeQuery bool, cmd *kingpin.CmdClause) *volume.Query {
692692

693693
func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.FieldsQuery {
694694
// calculate query range from cli params
695-
var from, to string
695+
var fieldName, from, to string
696696
var since time.Duration
697697

698698
q := &detected.FieldsQuery{}
@@ -705,24 +705,28 @@ func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.FieldsQuery {
705705
q.Start = mustParse(from, defaultStart)
706706
q.End = mustParse(to, defaultEnd)
707707

708+
q.FieldName = fieldName
709+
708710
q.Quiet = *quiet
709711

710712
return nil
711713
})
712714

713-
cmd.Flag("field-limit", "Limit on number of fields to return.").
715+
cmd.Flag("limit", "Limit on number of fields or values to return.").
714716
Default("100").
715-
IntVar(&q.FieldLimit)
717+
IntVar(&q.Limit)
716718
cmd.Flag("line-limit", "Limit the number of lines each subquery is allowed to process.").
717719
Default("1000").
718720
IntVar(&q.LineLimit)
719721
cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'").
720722
Required().
721723
StringVar(&q.QueryString)
724+
cmd.Arg("field", "The name of the field.").Default("").StringVar(&fieldName)
722725
cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since)
723726
cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from)
724727
cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to)
725728
cmd.Flag("step", "Query resolution step width, for metric queries. Evaluate the query at the specified step over the time range.").
729+
Default("10s").
726730
DurationVar(&q.Step)
727731

728732
return q

‎docs/sources/setup/install/helm/reference.md

+2
Original file line numberDiff line numberDiff line change
@@ -5640,6 +5640,7 @@ null
56405640
"/loki/api/v1/index/volume",
56415641
"/loki/api/v1/index/volume_range",
56425642
"/loki/api/v1/format_query",
5643+
"/loki/api/v1/detected_field",
56435644
"/loki/api/v1/detected_fields",
56445645
"/loki/api/v1/detected_labels",
56455646
"/loki/api/v1/patterns"
@@ -5702,6 +5703,7 @@ null
57025703
"/loki/api/v1/index/volume",
57035704
"/loki/api/v1/index/volume_range",
57045705
"/loki/api/v1/format_query",
5706+
"/loki/api/v1/detected_field",
57055707
"/loki/api/v1/detected_fields",
57065708
"/loki/api/v1/detected_labels",
57075709
"/loki/api/v1/patterns"

‎pkg/ingester-rf1/ingester.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -881,6 +881,6 @@ func (i *Ingester) GetDetectedFields(_ context.Context, r *logproto.DetectedFiel
881881
Cardinality: 1,
882882
},
883883
},
884-
FieldLimit: r.GetFieldLimit(),
884+
Limit: r.GetLimit(),
885885
}, nil
886886
}

‎pkg/ingester/ingester.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1595,7 +1595,7 @@ func (i *Ingester) GetDetectedFields(_ context.Context, r *logproto.DetectedFiel
15951595
Cardinality: 1,
15961596
},
15971597
},
1598-
FieldLimit: r.GetFieldLimit(),
1598+
Limit: r.GetLimit(),
15991599
}, nil
16001600
}
16011601

‎pkg/logcli/client/client.go

+23-16
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,18 @@ import (
2828
)
2929

3030
const (
31-
queryPath = "/loki/api/v1/query"
32-
queryRangePath = "/loki/api/v1/query_range"
33-
labelsPath = "/loki/api/v1/labels"
34-
labelValuesPath = "/loki/api/v1/label/%s/values"
35-
seriesPath = "/loki/api/v1/series"
36-
tailPath = "/loki/api/v1/tail"
37-
statsPath = "/loki/api/v1/index/stats"
38-
volumePath = "/loki/api/v1/index/volume"
39-
volumeRangePath = "/loki/api/v1/index/volume_range"
40-
detectedFieldsPath = "/loki/api/v1/detected_fields"
41-
defaultAuthHeader = "Authorization"
31+
queryPath = "/loki/api/v1/query"
32+
queryRangePath = "/loki/api/v1/query_range"
33+
labelsPath = "/loki/api/v1/labels"
34+
labelValuesPath = "/loki/api/v1/label/%s/values"
35+
seriesPath = "/loki/api/v1/series"
36+
tailPath = "/loki/api/v1/tail"
37+
statsPath = "/loki/api/v1/index/stats"
38+
volumePath = "/loki/api/v1/index/volume"
39+
volumeRangePath = "/loki/api/v1/index/volume_range"
40+
detectedFieldsPath = "/loki/api/v1/detected_fields"
41+
detectedFieldValuesPath = "/loki/api/v1/detected_field/%s/values"
42+
defaultAuthHeader = "Authorization"
4243

4344
// HTTP header keys
4445
HTTPScopeOrgID = "X-Scope-OrgID"
@@ -61,7 +62,7 @@ type Client interface {
6162
GetStats(queryStr string, start, end time.Time, quiet bool) (*logproto.IndexStatsResponse, error)
6263
GetVolume(query *volume.Query) (*loghttp.QueryResponse, error)
6364
GetVolumeRange(query *volume.Query) (*loghttp.QueryResponse, error)
64-
GetDetectedFields(queryStr string, fieldLimit, lineLimit int, start, end time.Time, step time.Duration, quiet bool) (*loghttp.DetectedFieldsResponse, error)
65+
GetDetectedFields(queryStr, fieldName string, fieldLimit, lineLimit int, start, end time.Time, step time.Duration, quiet bool) (*loghttp.DetectedFieldsResponse, error)
6566
}
6667

6768
// Tripperware can wrap a roundtripper.
@@ -234,15 +235,16 @@ func (c *DefaultClient) getVolume(path string, query *volume.Query) (*loghttp.Qu
234235
}
235236

236237
func (c *DefaultClient) GetDetectedFields(
237-
queryStr string,
238-
fieldLimit, lineLimit int,
238+
queryStr, fieldName string,
239+
limit, lineLimit int,
239240
start, end time.Time,
240241
step time.Duration,
241242
quiet bool,
242243
) (*loghttp.DetectedFieldsResponse, error) {
244+
243245
qsb := util.NewQueryStringBuilder()
244246
qsb.SetString("query", queryStr)
245-
qsb.SetInt("field_limit", int64(fieldLimit))
247+
qsb.SetInt("limit", int64(limit))
246248
qsb.SetInt("line_limit", int64(lineLimit))
247249
qsb.SetInt("start", start.UnixNano())
248250
qsb.SetInt("end", end.UnixNano())
@@ -251,7 +253,12 @@ func (c *DefaultClient) GetDetectedFields(
251253
var err error
252254
var r loghttp.DetectedFieldsResponse
253255

254-
if err = c.doRequest(detectedFieldsPath, qsb.Encode(), quiet, &r); err != nil {
256+
path := detectedFieldsPath
257+
if fieldName != "" {
258+
path = fmt.Sprintf(detectedFieldValuesPath, url.PathEscape(fieldName))
259+
}
260+
261+
if err = c.doRequest(path, qsb.Encode(), quiet, &r); err != nil {
255262
return nil, err
256263
}
257264

‎pkg/logcli/client/file.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ func (f *FileClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryResponse, er
207207
}
208208

209209
func (f *FileClient) GetDetectedFields(
210-
_ string,
210+
_, _ string,
211211
_, _ int,
212212
_, _ time.Time,
213213
_ time.Duration,

‎pkg/logcli/detected/fields.go

+23-8
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ type FieldsQuery struct {
1818
QueryString string
1919
Start time.Time
2020
End time.Time
21-
FieldLimit int
21+
Limit int
2222
LineLimit int
2323
Step time.Duration
2424
Quiet bool
25+
FieldName string
2526
ColoredOutput bool
2627
}
2728

@@ -30,7 +31,16 @@ func (q *FieldsQuery) Do(c client.Client, outputMode string) {
3031
var resp *loghttp.DetectedFieldsResponse
3132
var err error
3233

33-
resp, err = c.GetDetectedFields(q.QueryString, q.FieldLimit, q.LineLimit, q.Start, q.End, q.Step, q.Quiet)
34+
resp, err = c.GetDetectedFields(
35+
q.QueryString,
36+
q.FieldName,
37+
q.Limit,
38+
q.LineLimit,
39+
q.Start,
40+
q.End,
41+
q.Step,
42+
q.Quiet,
43+
)
3444
if err != nil {
3545
log.Fatalf("Error doing request: %+v", err)
3646
}
@@ -43,12 +53,17 @@ func (q *FieldsQuery) Do(c client.Client, outputMode string) {
4353
}
4454
fmt.Println(string(out))
4555
default:
46-
output := make([]string, len(resp.Fields))
47-
for i, field := range resp.Fields {
48-
bold := color.New(color.Bold)
49-
output[i] = fmt.Sprintf("label: %s\t\t", bold.Sprintf("%s", field.Label)) +
50-
fmt.Sprintf("type: %s\t\t", bold.Sprintf("%s", field.Type)) +
51-
fmt.Sprintf("cardinality: %s", bold.Sprintf("%d", field.Cardinality))
56+
var output []string
57+
if len(resp.Fields) > 0 {
58+
output = make([]string, len(resp.Fields))
59+
for i, field := range resp.Fields {
60+
bold := color.New(color.Bold)
61+
output[i] = fmt.Sprintf("label: %s\t\t", bold.Sprintf("%s", field.Label)) +
62+
fmt.Sprintf("type: %s\t\t", bold.Sprintf("%s", field.Type)) +
63+
fmt.Sprintf("cardinality: %s", bold.Sprintf("%d", field.Cardinality))
64+
}
65+
} else if len(resp.Values) > 0 {
66+
output = resp.Values
5267
}
5368

5469
slices.Sort(output)

‎pkg/logcli/query/query_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,7 @@ func (t *testQueryClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryRespons
486486
}
487487

488488
func (t *testQueryClient) GetDetectedFields(
489-
_ string,
489+
_, _ string,
490490
_, _ int,
491491
_, _ time.Time,
492492
_ time.Duration,

‎pkg/loghttp/detected.go

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import "github.com/grafana/loki/v3/pkg/logproto"
55
// LabelResponse represents the http json response to a label query
66
type DetectedFieldsResponse struct {
77
Fields []DetectedField `json:"fields,omitempty"`
8+
Values []string `json:"values,omitempty"`
89
}
910

1011
type DetectedField struct {

‎pkg/loghttp/params.go

+10-3
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import (
1919

2020
const (
2121
defaultQueryLimit = 100
22-
defaultFieldLimit = 1000
22+
defaultLimit = 1000
2323
defaultSince = 1 * time.Hour
2424
defaultDirection = logproto.BACKWARD
2525
)
@@ -46,11 +46,18 @@ func lineLimit(r *http.Request) (uint32, error) {
4646
return uint32(l), nil
4747
}
4848

49-
func fieldLimit(r *http.Request) (uint32, error) {
50-
l, err := parseInt(r.Form.Get("field_limit"), defaultFieldLimit)
49+
func detectedFieldsLimit(r *http.Request) (uint32, error) {
50+
limit := r.Form.Get("limit")
51+
if limit == "" {
52+
// for backwards compatability
53+
limit = r.Form.Get("field_limit")
54+
}
55+
56+
l, err := parseInt(limit, defaultLimit)
5157
if err != nil {
5258
return 0, err
5359
}
60+
5461
if l <= 0 {
5562
return 0, errors.New("limit must be a positive value")
5663
}

‎pkg/loghttp/query.go

+9-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"unsafe"
1010

1111
"github.com/c2h5oh/datasize"
12+
"github.com/gorilla/mux"
1213
"github.com/grafana/jsonparser"
1314
json "github.com/json-iterator/go"
1415
"github.com/prometheus/common/model"
@@ -650,6 +651,7 @@ func ParseDetectedFieldsQuery(r *http.Request) (*logproto.DetectedFieldsRequest,
650651
result := &logproto.DetectedFieldsRequest{}
651652

652653
result.Query = query(r)
654+
result.Values, result.Name = values(r)
653655
result.Start, result.End, err = bounds(r)
654656
if err != nil {
655657
return nil, err
@@ -664,7 +666,7 @@ func ParseDetectedFieldsQuery(r *http.Request) (*logproto.DetectedFieldsRequest,
664666
return nil, err
665667
}
666668

667-
result.FieldLimit, err = fieldLimit(r)
669+
result.Limit, err = detectedFieldsLimit(r)
668670
if err != nil {
669671
return nil, err
670672
}
@@ -684,9 +686,15 @@ func ParseDetectedFieldsQuery(r *http.Request) (*logproto.DetectedFieldsRequest,
684686
if (result.End.Sub(result.Start) / step) > 11000 {
685687
return nil, errStepTooSmall
686688
}
689+
687690
return result, nil
688691
}
689692

693+
func values(r *http.Request) (bool, string) {
694+
name, ok := mux.Vars(r)["name"]
695+
return ok, name
696+
}
697+
690698
func targetLabels(r *http.Request) []string {
691699
lbls := strings.Split(r.Form.Get("targetLabels"), ",")
692700
if (len(lbls) == 1 && lbls[0] == "") || len(lbls) == 0 {

‎pkg/logproto/compat.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ func (m *DetectedFieldsRequest) LogToSpan(sp opentracing.Span) {
524524
otlog.String("start", m.Start.String()),
525525
otlog.String("end", m.End.String()),
526526
otlog.String("step", time.Duration(m.Step).String()),
527-
otlog.String("field_limit", fmt.Sprintf("%d", m.FieldLimit)),
527+
otlog.String("field_limit", fmt.Sprintf("%d", m.Limit)),
528528
otlog.String("line_limit", fmt.Sprintf("%d", m.LineLimit)),
529529
}
530530
sp.LogFields(fields...)

0 commit comments

Comments
 (0)