Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add LogQL support for Loki v3.1.0 #723

Merged
merged 1 commit into from
Jul 11, 2024
Merged

Add LogQL support for Loki v3.1.0 #723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 84 additions & 12 deletions logql/v2/ast.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,9 @@ type LogFilterExpr struct {
filter string
filterOp string
value string
isNested bool
chainOp string
right []*LogFilterExpr
}

func (LogFilterExpr) logQLExpr() {}
Expand All @@ -124,11 +127,22 @@ func newLogFilterExpr(filter, filterOp, value string) *LogFilterExpr {
return &LogFilterExpr{filter: filter, filterOp: filterOp, value: value}
}

func (l *LogFilterExpr) chain(op string, expr *LogFilterExpr) *LogFilterExpr {
expr.isNested = true
expr.chainOp = op
l.right = append(l.right, expr)

return l
}

func (l *LogFilterExpr) String() string {
var sb strings.Builder

sb.WriteString(l.filter)
sb.WriteString(" ")
// Render filter only on first filter and not nested or filters
if !l.isNested {
sb.WriteString(l.filter)
sb.WriteString(" ")
}

if l.filterOp != "" {
sb.WriteString(l.filterOp)
Expand All @@ -139,25 +153,85 @@ func (l *LogFilterExpr) String() string {
sb.WriteString(strconv.Quote(l.value))
}

for i, r := range l.right {
switch r.chainOp {
case "or":
sb.WriteString(" ")
sb.WriteString(r.chainOp)
sb.WriteString(" ")
sb.WriteString(r.String())
}

if i == len(l.right) {
sb.WriteString(" ")
}
}

return sb.String()
}

func (l *LogFilterExpr) Walk(fn WalkFn) {
fn(l)
}

type FilterValueType string

const (
TypeNumber FilterValueType = "number"
TypeDuration FilterValueType = "duration"
TypeText FilterValueType = "text"
)

type LogLabelFilterValue struct {
filterType FilterValueType
numberVal *LogNumberExpr
strVal string
durVal time.Duration
}

func newLogLabelFilterValue(t FilterValueType, numberVal *LogNumberExpr, strVal string, durVal time.Duration) *LogLabelFilterValue {
switch t {
case TypeNumber:
return &LogLabelFilterValue{filterType: t, numberVal: numberVal}
case TypeDuration:
return &LogLabelFilterValue{filterType: t, durVal: durVal}
case TypeText:
return &LogLabelFilterValue{filterType: t, strVal: strVal}
default:
return &LogLabelFilterValue{}
}
}

func (l *LogLabelFilterValue) String() string {
var sb strings.Builder
switch l.filterType {
case TypeNumber:
sb.WriteString(l.numberVal.String())
case TypeDuration:
sb.WriteString(l.durVal.String())
case TypeText:
sb.WriteString(`"`)
sb.WriteString(l.strVal)
sb.WriteString(`"`)
default:
return ""
}

return sb.String()
}

type LogLabelFilterExpr struct {
defaultLogQLExpr // nolint:unused
labelName string
comparisonOp string
filterOp string
labelValue string
labelValue *LogLabelFilterValue
isNested bool
chainOp string
right []*LogLabelFilterExpr
}

func newLogLabelFilter(identifier, comparisonOp, filterOp, value string) *LogLabelFilterExpr {
func newLogLabelFilter(identifier, comparisonOp, filterOp string, value *LogLabelFilterValue) *LogLabelFilterExpr {
return &LogLabelFilterExpr{labelName: identifier, comparisonOp: comparisonOp, filterOp: filterOp, labelValue: value}
}

Expand All @@ -181,19 +255,17 @@ func (l *LogLabelFilterExpr) String() string {
}

sb.WriteString(l.labelName)
sb.WriteString(" ")
sb.WriteString(l.comparisonOp)
sb.WriteString(" ")

if l.filterOp != "" {
sb.WriteString(l.filterOp)
sb.WriteString("(")
sb.WriteString(`"`)
sb.WriteString(l.labelValue)
sb.WriteString(`"`)
sb.WriteString(l.labelValue.String())
sb.WriteString(")")
} else {
sb.WriteString(`"`)
sb.WriteString(l.labelValue)
sb.WriteString(`"`)
sb.WriteString(l.labelValue.String())
}

for i, r := range l.right {
Expand Down Expand Up @@ -535,13 +607,13 @@ func (l *LogQueryExpr) AppendPipelineMatchers(matchers []*labels.Matcher, chainO
matchersFilter := LogLabelFilterExpr{
labelName: matchers[0].Name,
comparisonOp: matchers[0].Type.String(),
labelValue: matchers[0].Value,
labelValue: newLogLabelFilterValue(TypeText, nil, matchers[0].Value, 0),
}
for _, m := range matchers[1:] {
matchersFilter.right = append(matchersFilter.right, &LogLabelFilterExpr{
labelName: m.Name,
comparisonOp: m.Type.String(),
labelValue: m.Value,
labelValue: newLogLabelFilterValue(TypeText, nil, m.Value, 0),
isNested: true,
chainOp: chainOp,
})
Expand Down
16 changes: 8 additions & 8 deletions logql/v2/ast_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ func Test_AstWalker_AppendMatcher(t *testing.T) {
output: `{first="value", second="next"} |= "other" |= ip("8.8.8.8")`,
},
{
input: `{ first = "value" }|logfmt|addr>=ip("1.1.1.1")`,
output: `{first="value", second="next"} | logfmt | addr>=ip("1.1.1.1")`,
input: `{ first = "value" }|logfmt|addr >= ip("1.1.1.1")`,
output: `{first="value", second="next"} | logfmt | addr >= ip("1.1.1.1")`,
},
// log metric expressions
{
Expand Down Expand Up @@ -169,24 +169,24 @@ func Test_AstWalker_AppendORMatcher(t *testing.T) {
// log selector expressions
{
input: `{first="value"}`,
output: `{first="value"} | second=~"foo|bar" or third=~"foo|bar"`,
output: `{first="value"} | second =~ "foo|bar" or third =~ "foo|bar"`,
},
{
input: `{first="value"} |= "other" |= ip("8.8.8.8")`,
output: `{first="value"} | second=~"foo|bar" or third=~"foo|bar" |= "other" |= ip("8.8.8.8")`,
output: `{first="value"} | second =~ "foo|bar" or third =~ "foo|bar" |= "other" |= ip("8.8.8.8")`,
},
{
input: `{ first = "value" }|logfmt|addr>=ip("1.1.1.1")`,
output: `{first="value"} | second=~"foo|bar" or third=~"foo|bar" | logfmt | addr>=ip("1.1.1.1")`,
input: `{ first = "value" }|logfmt|addr >= ip("1.1.1.1")`,
output: `{first="value"} | second =~ "foo|bar" or third =~ "foo|bar" | logfmt | addr >= ip("1.1.1.1")`,
},
// log metric expressions
{
input: `sum(rate({first="value"}[5m]))`,
output: `sum(rate({first="value"}[5m] | second=~"foo|bar" or third=~"foo|bar"))`,
output: `sum(rate({first="value"}[5m] | second =~ "foo|bar" or third =~ "foo|bar"))`,
},
{
input: `max without (second) (count_over_time({first="value"}[5h]))`,
output: `max without(second) (count_over_time({first="value"}[5h] | second=~"foo|bar" or third=~"foo|bar"))`,
output: `max without(second) (count_over_time({first="value"}[5h] | second =~ "foo|bar" or third =~ "foo|bar"))`,
},
}
for _, tc := range tc {
Expand Down
30 changes: 18 additions & 12 deletions logql/v2/expr.y
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ import (
%token <duration> DURATION
%token <val> MATCHERS LABELS EQ RE NRE OPEN_BRACE CLOSE_BRACE OPEN_BRACKET CLOSE_BRACKET COMMA DOT
OPEN_PARENTHESIS CLOSE_PARENTHESIS COUNT_OVER_TIME RATE RATE_COUNTER SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK SORT SORT_DESC
BYTES_OVER_TIME BYTES_RATE BOOL JSON REGEXP LOGFMT PIPE_MATCH PIPE_EXACT PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME
BYTES_OVER_TIME BYTES_RATE BOOL JSON REGEXP LOGFMT PIPE_MATCH PIPE_EXACT PIPE_MATCH_PATTERN PIPE_NOT_MATCH_PATTERN PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME
MAX_OVER_TIME STDVAR_OVER_TIME STDDEV_OVER_TIME QUANTILE_OVER_TIME FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME
BY WITHOUT VECTOR LABEL_REPLACE IP UNPACK PATTERN OFFSET BYTES_CONV DURATION_CONV DURATION_SECONDS_CONV ON IGNORING GROUP_LEFT GROUP_RIGHT
DECOLORIZE DROP KEEP
Expand Down Expand Up @@ -132,16 +132,20 @@ logStageExpr:
;

logFilterExpr:
filter STRING { $$ = newLogFilterExpr($1, "", $2) }
| filter IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogFilterExpr($1, OpIP, $4) }
filter STRING { $$ = newLogFilterExpr($1, "", $2) }
| filter IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogFilterExpr($1, OpIP, $4) }
| filter STRING OR STRING { $$ = newLogFilterExpr($1, "", $2).chain("or", newLogFilterExpr($1, "", $4)) }
| filter STRING OR IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogFilterExpr($1, "", $2).chain("or", newLogFilterExpr($1, OpIP, $6)) }
;

logLabelFilterExpr:
IDENTIFIER comparisonOp STRING { $$ = newLogLabelFilter($1, $2, "", $3) }
| IDENTIFIER comparisonOp IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogLabelFilter($1, $2, OpIP, $5) }
| logLabelFilterExpr AND logLabelFilterExpr { $$ = $1.chain("and", $3) }
| logLabelFilterExpr OR logLabelFilterExpr { $$ = $1.chain("or", $3) }
| logLabelFilterExpr COMMA logLabelFilterExpr { $$ = $1.chain(",", $3) }
IDENTIFIER comparisonOp STRING { $$ = newLogLabelFilter($1, $2, "", newLogLabelFilterValue(TypeText, nil, $3, 0)) }
| IDENTIFIER comparisonOp DURATION { $$ = newLogLabelFilter($1, $2, "", newLogLabelFilterValue(TypeDuration, nil, "", $3)) }
| IDENTIFIER comparisonOp logNumberExpr { $$ = newLogLabelFilter($1, $2, "", newLogLabelFilterValue(TypeNumber, &$3, "", 0)) }
| IDENTIFIER comparisonOp IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogLabelFilter($1, $2, OpIP, newLogLabelFilterValue(TypeText, nil, $5, 0)) }
| logLabelFilterExpr AND logLabelFilterExpr { $$ = $1.chain("and", $3) }
| logLabelFilterExpr OR logLabelFilterExpr { $$ = $1.chain("or", $3) }
| logLabelFilterExpr COMMA logLabelFilterExpr { $$ = $1.chain(",", $3) }
;

logFormatExpr:
Expand Down Expand Up @@ -396,10 +400,12 @@ metricOp:
;

filter:
PIPE_MATCH { $$ = "|~" }
| PIPE_EXACT { $$ = "|=" }
| NRE { $$ = "!~" }
| NEQ { $$ = "!=" }
PIPE_MATCH { $$ = "|~" }
| PIPE_EXACT { $$ = "|=" }
| PIPE_MATCH_PATTERN { $$ = "|>" }
| PIPE_NOT_MATCH_PATTERN { $$ = "!>" }
| NRE { $$ = "!~" }
| NEQ { $$ = "!=" }
;

comparisonOp:
Expand Down
Loading