Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Promtail: Add new target for receiving GCP PubSub Push messages #6777

Merged
merged 29 commits into from
Aug 2, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
7c1d9dc
WIP
thepalbi Jul 21, 2022
f541e8f
wip test running
thepalbi Jul 25, 2022
3cfaf34
one test passing
thepalbi Jul 25, 2022
dd4977e
some fixes and testing
thepalbi Jul 26, 2022
89eddfc
refactor out some shared http target components
thepalbi Jul 26, 2022
3b5207e
fix linting
thepalbi Jul 26, 2022
07f47eb
correcting heroku copied code
thepalbi Jul 26, 2022
d6329ce
add x-scope-orgid header test
thepalbi Jul 26, 2022
668402e
Using just one GCPLog target with multiple subs types
thepalbi Jul 27, 2022
3863f04
Comments and simplfying switch
thepalbi Jul 27, 2022
1b7d7fe
Added target creator tests
thepalbi Jul 27, 2022
31e6a5b
added NewGCPLog target tests
thepalbi Jul 27, 2022
6f9548d
Address github comments
thepalbi Jul 28, 2022
ef23e23
make actual target impls package private
thepalbi Jul 28, 2022
031844b
fix lint
thepalbi Jul 28, 2022
b79387d
Update clients/pkg/promtail/targets/gcplog/target.go
thepalbi Jul 28, 2022
258a3a8
fix test
thepalbi Jul 28, 2022
5b4b5a8
added config section
thepalbi Jul 28, 2022
69a2bec
added scraping docs
thepalbi Jul 28, 2022
4938dd8
Make everything as backwards compatible as possible
thepalbi Jul 28, 2022
1020b26
Update docs/sources/clients/promtail/configuration.md
thepalbi Aug 1, 2022
8232428
Update docs/sources/clients/promtail/configuration.md
thepalbi Aug 1, 2022
91dc153
Update docs/sources/clients/promtail/configuration.md
thepalbi Aug 1, 2022
6052172
Update docs/sources/clients/promtail/configuration.md
thepalbi Aug 1, 2022
d97ec67
Update docs/sources/clients/promtail/configuration.md
thepalbi Aug 1, 2022
2803829
Update docs/sources/clients/promtail/configuration.md
thepalbi Aug 1, 2022
eb77eb2
Update docs/sources/clients/promtail/scraping.md
thepalbi Aug 1, 2022
1dfa0dd
Update docs/sources/clients/promtail/scraping.md
thepalbi Aug 1, 2022
979cdea
Update docs/sources/clients/promtail/configuration.md
thepalbi Aug 2, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Address github comments
  • Loading branch information
thepalbi committed Aug 2, 2022
commit 6f9548da664eff513c675e0ecfd527e28eb5024d
33 changes: 9 additions & 24 deletions clients/pkg/promtail/targets/gcplog/pull_target.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,28 +57,7 @@ func newPullTarget(
return nil, err
}

target := newGcplogTarget(metrics, logger, handler, relabel, jobName, config, ps, ctx, cancel)

go func() {
_ = target.run()
}()

return target, nil
}

// nolint:revive
func newGcplogTarget(
metrics *Metrics,
logger log.Logger,
handler api.EntryHandler,
relabel []*relabel.Config,
jobName string,
config *scrapeconfig.GCPLogTargetConfig,
pubsubClient *pubsub.Client,
ctx context.Context,
cancel func(),
) *PullTarget {
return &PullTarget{
target := &PullTarget{
metrics: metrics,
logger: logger,
handler: handler,
Expand All @@ -87,9 +66,15 @@ func newGcplogTarget(
jobName: jobName,
ctx: ctx,
cancel: cancel,
ps: pubsubClient,
ps: ps,
msgs: make(chan *pubsub.Message),
}

go func() {
_ = target.run()
}()

return target, nil
}

func (t *PullTarget) run() error {
Expand Down Expand Up @@ -133,7 +118,7 @@ func (t *PullTarget) run() error {
}

func (t *PullTarget) Type() target.TargetType {
return target.GcplogTargetType
return target.GCPLogTargetType
}

func (t *PullTarget) Ready() bool {
Expand Down
25 changes: 14 additions & 11 deletions clients/pkg/promtail/targets/gcplog/pull_target_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package gcplog

import (
"context"
"github.com/grafana/loki/clients/pkg/promtail/api"
"sync"
"testing"
"time"
Expand Down Expand Up @@ -123,17 +124,19 @@ func testPullTarget(t *testing.T) (*PullTarget, *fake.Client, *pubsub.Client, fu

fakeClient := fake.New(func() {})

target := newGcplogTarget(
NewMetrics(prometheus.NewRegistry()),
log.NewNopLogger(),
fakeClient,
nil,
"job-test-gcplogtarget",
testConfig,
mockpubsubClient,
ctx,
cancel,
)
var handler api.EntryHandler = fakeClient
target := &PullTarget{
metrics: NewMetrics(prometheus.NewRegistry()),
logger: log.NewNopLogger(),
handler: handler,
relabelConfig: nil,
config: testConfig,
jobName: "job-test-gcplogtarget",
ctx: ctx,
cancel: cancel,
ps: mockpubsubClient,
msgs: make(chan *pubsub.Message),
}

// cleanup
return target, fakeClient, mockpubsubClient, func() {
Expand Down
2 changes: 1 addition & 1 deletion clients/pkg/promtail/targets/gcplog/push_target.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ func (h *PushTarget) push(w http.ResponseWriter, r *http.Request) {
}

func (h *PushTarget) Type() target.TargetType {
return target.GCPPushTargetType
return target.GCPLogTargetType
}

func (h *PushTarget) DiscoveredLabels() model.LabelSet {
Expand Down
7 changes: 2 additions & 5 deletions clients/pkg/promtail/targets/target/target.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ const (
// SyslogTargetType is a syslog target
SyslogTargetType = TargetType("Syslog")

// GcplogTargetType is a target where log entries are pulled from pubsub topic.
GcplogTargetType = TargetType("Gcplog")
// GCPLogTargetType is a target where log entries are pulled from pubsub topic.
GCPLogTargetType = TargetType("GCPLog")

// DroppedTargetType is a target that's been dropped.
DroppedTargetType = TargetType("dropped")
Expand All @@ -44,9 +44,6 @@ const (

// HerokuDrainTargetType is a Heroku Logs target
HerokuDrainTargetType = TargetType("HerokuDrain")

// GCPPushTargetType is a GCP PubSub Push target
GCPPushTargetType = TargetType("GCPPush")
)

// Target is a promtail scrape target
Expand Down