Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OTLP: Increase valid created timestamp threshold to 5 minutes #706

Merged
merged 5 commits into from
Oct 3, 2024
Merged

OTLP: Increase valid created timestamp threshold to 5 minutes #706

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ce7e117
OTLP: Increase valid created timestamp threshold to 5 minutes
jesusvazquez Oct 2, 2024
fb62222
Add unit tests
jesusvazquez Oct 2, 2024
5c80cbb
Allow to override the valid interval from TSDB config
jesusvazquez Oct 3, 2024
d145d09
Fix renaming
jesusvazquez Oct 3, 2024
d2e6c5b
Separate our arguments from the others
jesusvazquez Oct 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Allow to override the valid interval from TSDB config 
Signed-off-by: Jesus Vazquez <[email protected]>
  • Loading branch information
@jesusvazquez
jesusvazquez committed Oct 3, 2024
commit 5c80cbb133d9742538b1fdede39024eafcc92e4b
18 changes: 9 additions & 9 deletions storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,15 @@ import (
)

type Settings struct {
ExternalLabels map[string]string
Namespace string
PromoteResourceAttributes []string
ValidIntervalForStartTimestamps time.Duration
DisableTargetInfo bool
ExportCreatedMetric bool
AddMetricSuffixes bool
SendMetadata bool
EnableCreatedTimestampZeroIngestion bool
ExternalLabels map[string]string
Namespace string
PromoteResourceAttributes []string
DisableTargetInfo bool
ExportCreatedMetric bool
AddMetricSuffixes bool
SendMetadata bool
EnableCreatedTimestampZeroIngestion bool
ValidIntervalCreatedTimestampZeroIngestion time.Duration
}

type StartTsAndTs struct {
Expand Down
27 changes: 15 additions & 12 deletions storage/remote/write_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"github.com/prometheus/prometheus/prompb"
writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
"github.com/prometheus/prometheus/storage"
otlptranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite"

Check failure on line 41 in storage/remote/write_handler.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

could not import github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite (-: # github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite

Check failure on line 41 in storage/remote/write_handler.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

could not import github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite (-: # github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite
)

type writeHandler struct {
Expand Down Expand Up @@ -482,25 +482,27 @@

// NewOTLPWriteHandler creates a http.Handler that accepts OTLP write requests and
// writes them to the provided appendable.
func NewOTLPWriteHandler(logger log.Logger, appendable storage.Appendable, configFunc func() config.Config, enableCTZeroIngestion bool) http.Handler {
func NewOTLPWriteHandler(logger log.Logger, appendable storage.Appendable, configFunc func() config.Config, enableCTZeroIngestion bool, validIntervalCTZeroIngestion time.Duration) http.Handler {
rwHandler := &writeHandler{
logger: logger,
appendable: appendable,
}

return &otlpWriteHandler{
logger: logger,
rwHandler: rwHandler,
configFunc: configFunc,
enableCTZeroIngestion: enableCTZeroIngestion,
logger: logger,
rwHandler: rwHandler,
configFunc: configFunc,
enableCTZeroIngestion: enableCTZeroIngestion,
validIntervalCTZeroIngestion: validIntervalCTZeroIngestion,
}
}

type otlpWriteHandler struct {
logger log.Logger
rwHandler *writeHandler
configFunc func() config.Config
enableCTZeroIngestion bool
logger log.Logger
rwHandler *writeHandler
configFunc func() config.Config
enableCTZeroIngestion bool
validIntervalCTZeroIngestion time.Duration
}

func (h *otlpWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
Expand All @@ -515,9 +517,10 @@

converter := otlptranslator.NewPrometheusConverter()
annots, err := converter.FromMetrics(r.Context(), req.Metrics(), otlptranslator.Settings{
AddMetricSuffixes: true,
PromoteResourceAttributes: otlpCfg.PromoteResourceAttributes,
EnableCreatedTimestampZeroIngestion: h.enableCTZeroIngestion,
AddMetricSuffixes: true,
PromoteResourceAttributes: otlpCfg.PromoteResourceAttributes,
EnableCreatedTimestampZeroIngestion: h.enableCTZeroIngestion,
ValidIntervalCreatedTimestampZeroIngestion: h.validIntervalCTZeroIngestion,
}, h.logger)
if err != nil {
level.Warn(h.logger).Log("msg", "Error translating OTLP metrics to Prometheus write request", "err", err)
Expand Down
2 changes: 1 addition & 1 deletion storage/remote/write_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ func TestOTLPWriteHandler(t *testing.T) {
return config.Config{
OTLPConfig: config.DefaultOTLPConfig,
}
}, false)
}, false, 0)

recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, req)
Expand Down
3 changes: 2 additions & 1 deletion web/api/v1/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ func NewAPI(
acceptRemoteWriteProtoMsgs []config.RemoteWriteProtoMsg,
otlpEnabled bool,
enableCTZeroIngestion bool,
validIntervalCTZeroIngestion time.Duration,
) *API {
a := &API{
QueryEngine: qe,
Expand Down Expand Up @@ -296,7 +297,7 @@ func NewAPI(
a.remoteWriteHandler = remote.NewWriteHandler(logger, registerer, ap, acceptRemoteWriteProtoMsgs)
}
if otlpEnabled {
a.otlpWriteHandler = remote.NewOTLPWriteHandler(logger, ap, configFunc, enableCTZeroIngestion)
a.otlpWriteHandler = remote.NewOTLPWriteHandler(logger, ap, configFunc, enableCTZeroIngestion, validIntervalCTZeroIngestion)
}

return a
Expand Down
1 change: 1 addition & 0 deletions web/api/v1/errors_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ func createPrometheusAPI(t *testing.T, q storage.SampleAndChunkQueryable) *route
config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV1, config.RemoteWriteProtoMsgV2},
false,
false,
0,
)

promRouter := route.New().WithPrefix("/api/v1")
Expand Down
44 changes: 23 additions & 21 deletions web/web.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,27 +244,28 @@ type Options struct {
Version *PrometheusVersion
Flags map[string]string

ListenAddresses []string
CORSOrigin *regexp.Regexp
ReadTimeout time.Duration
MaxConnections int
ExternalURL *url.URL
RoutePrefix string
UseLocalAssets bool
UserAssetsPath string
ConsoleTemplatesPath string
ConsoleLibrariesPath string
EnableLifecycle bool
EnableAdminAPI bool
PageTitle string
RemoteReadSampleLimit int
RemoteReadConcurrencyLimit int
RemoteReadBytesInFrame int
EnableRemoteWriteReceiver bool
EnableOTLPWriteReceiver bool
EnableCreatedTimestampZeroIngestion bool
IsAgent bool
AppName string
ListenAddresses []string
CORSOrigin *regexp.Regexp
ReadTimeout time.Duration
MaxConnections int
ExternalURL *url.URL
RoutePrefix string
UseLocalAssets bool
UserAssetsPath string
ConsoleTemplatesPath string
ConsoleLibrariesPath string
EnableLifecycle bool
EnableAdminAPI bool
PageTitle string
RemoteReadSampleLimit int
RemoteReadConcurrencyLimit int
RemoteReadBytesInFrame int
EnableRemoteWriteReceiver bool
EnableOTLPWriteReceiver bool
EnableCreatedTimestampZeroIngestion bool
ValidIntervalCreatedTimestampZeroIngestion time.Duration
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: these two cause conflicts when updating from upstream due to the indenting. Would be nice to eventually clean these up by adding our additions at the end after empty line

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You mean something like:

	RemoteReadBytesInFrame                     int
	EnableRemoteWriteReceiver                  bool
	EnableOTLPWriteReceiver                    bool

        // Our additions
	EnableCreatedTimestampZeroIngestion        bool
	ValidIntervalCreatedTimestampZeroIngestion time.Duration

IsAgent bool
AppName string

AcceptRemoteWriteProtoMsgs []config.RemoteWriteProtoMsg

Expand Down Expand Up @@ -359,6 +360,7 @@ func New(logger log.Logger, o *Options) *Handler {
o.AcceptRemoteWriteProtoMsgs,
o.EnableOTLPWriteReceiver,
o.EnableCreatedTimestampZeroIngestion,
o.ValidIntervalCreatedTimestampZeroIngestion,
)

if o.RoutePrefix != "/" {
Expand Down
Loading