Skip to content

Commit

Permalink
Fix Fluent-Bit plugin encoding array values as Base64
Browse files Browse the repository at this point in the history
This issue is caused by the fact that the Fluent-Bit decoder
sends strings as []byte. There was a partial fix for this in
place already that recursed into maps and convertined []byte
to string. The fix was incomplete as it didn't handle []interface{}.
Hence, strings encoded as []byte inside slices weren't converted.

This issue is almost exactly the same as the one in Loki:
grafana/loki#1889

Hence, the fix is also almost the same.
  • Loading branch information
Photonios committed May 19, 2021
1 parent b39839b commit 4dfdcd4
Showing 1 changed file with 35 additions and 9 deletions.
44 changes: 35 additions & 9 deletions integrations/fluent-bit/plugin/out_coralogix.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func FLBPluginFlushCtx(ctx, data unsafe.Pointer, length C.int, tag *C.char) int
}

// Convert record to JSON
jsonRecord, err := jsoniter.MarshalToString(encodeJSON(record))
jsonRecord, err := jsoniter.MarshalToString(toStringMap(record))
if err != nil {
log.Printf(" ERROR: %v\n", err)
continue
Expand Down Expand Up @@ -194,22 +194,48 @@ func FLBPluginExitCtx(ctx unsafe.Pointer) int {
return output.FLB_OK
}

// encodeJSON encodes record to UTF-8
func encodeJSON(record map[interface{}]interface{}) map[string]interface{} {
// toStringMap recursively goes through the slice and converts []byte
// to string so that jsonitor.MarshalToString/json.Marshal don't
// encode []byte to Base64.
func toStringSlice(slice []interface{}) []interface{} {
var s []interface{}
for _, v := range slice {
switch t := v.(type) {
case []byte:
s = append(s, string(t))
case map[interface{}]interface{}:
s = append(s, toStringMap(t))
case []interface{}:
s = append(s, toStringSlice(t))
default:
s = append(s, t)
}
}
return s
}

// toStringMap recursively goes through the map and converts []byte
// to string so that jsonitor.MarshalToString/json.Marshal don't
// encode []byte to Base64.
func toStringMap(record map[interface{}]interface{}) map[string]interface{} {
m := make(map[string]interface{})
for k, v := range record {
key, ok := k.(string)
if !ok {
continue
}
switch t := v.(type) {
case []byte:
// prevent encoding to base64
m[k.(string)] = string(t)
m[key] = string(t)
case map[interface{}]interface{}:
if nextValue, ok := record[k].(map[interface{}]interface{}); ok {
m[k.(string)] = encodeJSON(nextValue)
}
m[key] = toStringMap(t)
case []interface{}:
m[key] = toStringSlice(t)
default:
m[k.(string)] = v
m[key] = v
}
}

return m
}

Expand Down

0 comments on commit 4dfdcd4

Please sign in to comment.