-
Notifications
You must be signed in to change notification settings - Fork 3.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(promtail): Support of RFC3164 aka BSD Syslog #12810
Changes from 5 commits
1a2ebf7
0320106
1426356
10f1033
d6818a9
e64f8a9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
@@ -174,6 +174,15 @@ type JournalTargetConfig struct { | |||||||||||||
Matches string `yaml:"matches"` | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
type SyslogFormat string | ||||||||||||||
|
||||||||||||||
const ( | ||||||||||||||
// A modern Syslog RFC | ||||||||||||||
SyslogFormatRFC5424 = "rfc5424" | ||||||||||||||
// A legacy Syslog RFC also known as BSD-syslog | ||||||||||||||
SyslogFormatRFC3164 = "rfc3164" | ||||||||||||||
) | ||||||||||||||
|
||||||||||||||
// SyslogTargetConfig describes a scrape config that listens for log lines over syslog. | ||||||||||||||
type SyslogTargetConfig struct { | ||||||||||||||
// ListenAddress is the address to listen on for syslog messages. | ||||||||||||||
|
@@ -202,12 +211,19 @@ type SyslogTargetConfig struct { | |||||||||||||
// message should be pushed to Loki | ||||||||||||||
UseRFC5424Message bool `yaml:"use_rfc5424_message"` | ||||||||||||||
|
||||||||||||||
// Syslog format used at the target. | ||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we should clarify the valid options here, and that There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we could implement the default in the parser or like this: loki/clients/pkg/promtail/targets/syslog/syslogtarget.go Lines 233 to 238 in efdae3d
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It looks like we don't have a dedicated config parsing function on this path, so maybe in |
||||||||||||||
SyslogFormat SyslogFormat `yaml:"syslog_format"` | ||||||||||||||
|
||||||||||||||
// MaxMessageLength sets the maximum limit to the length of syslog messages | ||||||||||||||
MaxMessageLength int `yaml:"max_message_length"` | ||||||||||||||
|
||||||||||||||
TLSConfig promconfig.TLSConfig `yaml:"tls_config,omitempty"` | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
func (config SyslogTargetConfig) IsRFC3164Message() bool { | ||||||||||||||
return config.SyslogFormat == SyslogFormatRFC3164 | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
// WindowsEventsTargetConfig describes a scrape config that listen for windows event logs. | ||||||||||||||
type WindowsEventsTargetConfig struct { | ||||||||||||||
|
||||||||||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,8 +9,9 @@ import ( | |
|
||
"github.com/go-kit/log" | ||
"github.com/go-kit/log/level" | ||
"github.com/influxdata/go-syslog/v3" | ||
"github.com/influxdata/go-syslog/v3/rfc5424" | ||
"github.com/leodido/go-syslog/v4" | ||
"github.com/leodido/go-syslog/v4/rfc3164" | ||
"github.com/leodido/go-syslog/v4/rfc5424" | ||
"github.com/prometheus/common/model" | ||
"github.com/prometheus/prometheus/model/labels" | ||
"github.com/prometheus/prometheus/model/relabel" | ||
|
@@ -106,7 +107,7 @@ func (t *SyslogTarget) handleMessageError(err error) { | |
t.metrics.syslogParsingErrors.Inc() | ||
} | ||
|
||
func (t *SyslogTarget) handleMessage(connLabels labels.Labels, msg syslog.Message) { | ||
func (t *SyslogTarget) handleMessageRFC5424(connLabels labels.Labels, msg syslog.Message) { | ||
rfc5424Msg := msg.(*rfc5424.SyslogMessage) | ||
|
||
if rfc5424Msg.Message == nil { | ||
|
@@ -173,6 +174,64 @@ func (t *SyslogTarget) handleMessage(connLabels labels.Labels, msg syslog.Messag | |
t.messages <- message{filtered, m, timestamp} | ||
} | ||
|
||
func (t *SyslogTarget) handleMessageRFC3164(connLabels labels.Labels, msg syslog.Message) { | ||
rfc3164Msg := msg.(*rfc3164.SyslogMessage) | ||
|
||
if rfc3164Msg.Message == nil { | ||
t.metrics.syslogEmptyMessages.Inc() | ||
return | ||
} | ||
|
||
lb := labels.NewBuilder(connLabels) | ||
if v := rfc3164Msg.SeverityLevel(); v != nil { | ||
lb.Set("__syslog_message_severity", *v) | ||
} | ||
if v := rfc3164Msg.FacilityLevel(); v != nil { | ||
lb.Set("__syslog_message_facility", *v) | ||
} | ||
if v := rfc3164Msg.Hostname; v != nil { | ||
lb.Set("__syslog_message_hostname", *v) | ||
} | ||
if v := rfc3164Msg.Appname; v != nil { | ||
lb.Set("__syslog_message_app_name", *v) | ||
} | ||
if v := rfc3164Msg.ProcID; v != nil { | ||
lb.Set("__syslog_message_proc_id", *v) | ||
} | ||
if v := rfc3164Msg.MsgID; v != nil { | ||
lb.Set("__syslog_message_msg_id", *v) | ||
} | ||
|
||
processed, _ := relabel.Process(lb.Labels(), t.relabelConfig...) | ||
|
||
filtered := make(model.LabelSet) | ||
for _, lbl := range processed { | ||
if strings.HasPrefix(lbl.Name, "__") { | ||
continue | ||
} | ||
filtered[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value) | ||
} | ||
|
||
var timestamp time.Time | ||
if t.config.UseIncomingTimestamp && rfc3164Msg.Timestamp != nil { | ||
timestamp = *rfc3164Msg.Timestamp | ||
} else { | ||
timestamp = time.Now() | ||
} | ||
|
||
m := *rfc3164Msg.Message | ||
|
||
t.messages <- message{filtered, m, timestamp} | ||
} | ||
|
||
func (t *SyslogTarget) handleMessage(connLabels labels.Labels, msg syslog.Message) { | ||
if t.config.IsRFC3164Message() { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think switching on the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Here possible, but it is using in few more places, and it leads to So, I've decided to use trivial method which can be refactored. Anyway, here only two RFC for Syslog, and I doubt that someone will make 3rd soon enough. |
||
t.handleMessageRFC3164(connLabels, msg) | ||
} else { | ||
t.handleMessageRFC5424(connLabels, msg) | ||
} | ||
} | ||
|
||
func (t *SyslogTarget) messageSender(entries chan<- api.Entry) { | ||
for msg := range t.messages { | ||
entries <- api.Entry{ | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think we wanted a second new config option
PushFullSyslogMessage
? we still need to keep this one as well, and mark it as deprecatedThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I can't do it, because RFC3164 message has lack of
String()
method. See implementation for RFC5424: https://github.com/leodido/go-syslog/blob/v4.1.0/rfc5424/builder.go#L9348-L9408The issue that RFC3164 isn't well structured and quite flexible, and rebuild original message is quite a challenge.
Sorry to misslead you.