From 2f69169aea142d374465054b163d660c48a6ef32 Mon Sep 17 00:00:00 2001 From: Michael Parker Date: Wed, 2 Aug 2023 10:44:34 -0500 Subject: [PATCH] feat: create datadog notification provider Signed-off-by: Michael Parker --- api/v1beta2/provider_types.go | 3 +- ...ification.toolkit.fluxcd.io_providers.yaml | 1 + docs/spec/v1beta2/providers.md | 57 ++++++ go.mod | 3 + go.sum | 6 + internal/notifier/datadog.go | 166 ++++++++++++++++++ internal/notifier/datadog_fuzz_test.go | 53 ++++++ internal/notifier/datadog_test.go | 75 ++++++++ internal/notifier/factory.go | 2 + internal/notifier/util.go | 8 + 10 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 internal/notifier/datadog.go create mode 100644 internal/notifier/datadog_fuzz_test.go create mode 100644 internal/notifier/datadog_test.go diff --git a/api/v1beta2/provider_types.go b/api/v1beta2/provider_types.go index 62ef05404..178f733d4 100644 --- a/api/v1beta2/provider_types.go +++ b/api/v1beta2/provider_types.go @@ -49,12 +49,13 @@ const ( OpsgenieProvider string = "opsgenie" AlertManagerProvider string = "alertmanager" PagerDutyProvider string = "pagerduty" + DataDogProvider string = "datadog" ) // ProviderSpec defines the desired state of the Provider. type ProviderSpec struct { // Type specifies which Provider implementation to use. - // +kubebuilder:validation:Enum=slack;discord;msteams;rocket;generic;generic-hmac;github;gitlab;gitea;bitbucket;azuredevops;googlechat;googlepubsub;webex;sentry;azureeventhub;telegram;lark;matrix;opsgenie;alertmanager;grafana;githubdispatch;pagerduty + // +kubebuilder:validation:Enum=slack;discord;msteams;rocket;generic;generic-hmac;github;gitlab;gitea;bitbucket;azuredevops;googlechat;googlepubsub;webex;sentry;azureeventhub;telegram;lark;matrix;opsgenie;alertmanager;grafana;githubdispatch;pagerduty;datadog // +required Type string `json:"type"` diff --git a/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml b/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml index d2f26b390..a8b7ce2f1 100644 --- a/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml +++ b/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml @@ -305,6 +305,7 @@ spec: - grafana - githubdispatch - pagerduty + - datadog type: string username: description: Username specifies the name under which events are posted. diff --git a/docs/spec/v1beta2/providers.md b/docs/spec/v1beta2/providers.md index 5a3e934aa..1f60fd213 100644 --- a/docs/spec/v1beta2/providers.md +++ b/docs/spec/v1beta2/providers.md @@ -109,6 +109,7 @@ The supported alerting providers are: | [Generic webhook](#generic-webhook) | `generic` | | [Generic webhook with HMAC](#generic-webhook-with-hmac) | `generic-hmac` | | [Azure Event Hub](#azure-event-hub) | `azureeventhub` | +| [DataDog](#datadog) | `datadog` | | [Discord](#discord) | `discord` | | [GitHub dispatch](#github-dispatch) | `githubdispatch` | | [Google Chat](#google-chat) | `googlechat` | @@ -405,6 +406,62 @@ stringData: address: "https://xxx.webhook.office.com/..." ``` +##### DataDog + +When `.spec.type` is set to `datadog`, the controller will send a payload for +an [Event](events.md#event-structure) to the provided DataDog API [Address](#address). + +The Event will be formatted into a [DataDog Event](https://docs.datadoghq.com/api/latest/events/#post-an-event) and sent to the +API endpoint of the provided DataDog [Address](#address). + +This Provider type supports the configuration of a [proxy URL](#https-proxy) +and/or [TLS certificates](#tls-certificates). + +The metadata of the Event is included in the DataDog event as extra tags. + +###### DataDog example + +To configure a Provider for DataDog, create a Secret with [the `token`](#token-example) +set to a [DataDog API key](https://docs.datadoghq.com/account_management/api-app-keys/#api-keys) +(not an application key!) and a `datadog` Provider with a [Secret reference](#secret-reference). + +```yaml +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta2 +kind: Provider +metadata: + name: datadog + namespace: default +spec: + type: datadog + address: https://api.datadoghq.com # DataDog Site US1 + secretRef: + name: datadog-secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: datadog-secret + namespace: default +stringData: + token: +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta1 +kind: Alert +metadata: + name: datadog-info + namespace: default +spec: + eventSeverity: info + eventSources: + - kind: HelmRelease + name: "*" + providerRef: + name: datadog + eventMetadata: + env: my-k8s-cluster # example of adding a custom `env` tag to the event +``` + ##### Discord When `.spec.type` is set to `discord`, the controller will send a payload for diff --git a/go.mod b/go.mod index b3a70b38a..adc074bdc 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 github.com/Azure/azure-amqp-common-go/v4 v4.2.0 github.com/Azure/azure-event-hubs-go/v3 v3.6.0 + github.com/DataDog/datadog-api-client-go/v2 v2.15.0 github.com/PagerDuty/go-pagerduty v1.7.0 github.com/containrrr/shoutrrr v0.7.1 github.com/fluxcd/notification-controller/api v1.0.0 @@ -62,6 +63,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/DataDog/zstd v1.5.2 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230619160724-3fbb1f12458c // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -83,6 +85,7 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/go.sum b/go.sum index e0d9c8b51..3b7da9425 100644 --- a/go.sum +++ b/go.sum @@ -232,7 +232,11 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-api-client-go/v2 v2.15.0 h1:5UVON1xs6Lul4d6R5TmLDqqSJxOkunkm/UdM/fjm+zc= +github.com/DataDog/datadog-api-client-go/v2 v2.15.0/go.mod h1:ZG8wS+y2rUmkRDJZQq7Og7EAPFPage+7vXcmuah2I9o= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -376,6 +380,8 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/internal/notifier/datadog.go b/internal/notifier/datadog.go new file mode 100644 index 000000000..3a3c3c6c6 --- /dev/null +++ b/internal/notifier/datadog.go @@ -0,0 +1,166 @@ +/* +Copyright 2023 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package notifier + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadog" + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV1" + + eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" +) + +type DataDog struct { + apiClient *datadog.APIClient + eventsApi *datadogV1.EventsApi + apiKey string +} + +// NewDataDog creates a new DataDog provider by mapping the notification provider API to sensible values for the DataDog API. +// url: The DataDog API endpoint to use. Examples: https://api.datadoghq.com, https://api.datadoghq.eu, etc. +// token: The DataDog API key (not the application key). +// headers: A map of extra tags to add to the event +func NewDataDog(address string, proxyUrl string, certPool *x509.CertPool, token string) (*DataDog, error) { + conf := datadog.NewConfiguration() + + if token == "" { + return nil, fmt.Errorf("token cannot be empty") + } + + baseUrl, err := url.Parse(address) + if err != nil { + return nil, fmt.Errorf("failed to parse address %q: %w", address, err) + } + + conf.Host = baseUrl.Host + conf.Scheme = baseUrl.Scheme + + if proxyUrl != "" || certPool != nil { + transport := &http.Transport{} + + if proxyUrl != "" { + proxy, err := url.Parse(proxyUrl) + if err != nil { + return nil, fmt.Errorf("failed to parse proxy URL %q: %w", proxyUrl, err) + } + + transport.Proxy = http.ProxyURL(proxy) + } + + if certPool != nil { + transport.TLSClientConfig = &tls.Config{ + RootCAs: certPool, + } + } + + conf.HTTPClient = &http.Client{ + Transport: transport, + } + } + + apiClient := datadog.NewAPIClient(conf) + eventsApi := datadogV1.NewEventsApi(apiClient) + + return &DataDog{ + apiClient: apiClient, + eventsApi: eventsApi, + apiKey: token, + }, nil +} + +func (d *DataDog) Post(ctx context.Context, event eventv1.Event) error { + dataDogEvent := d.toDataDogEvent(&event) + + _, _, err := d.eventsApi.CreateEvent(d.dataDogCtx(ctx), dataDogEvent) + if err != nil { + return fmt.Errorf("failed to post event to DataDog: %w", err) + } + + return nil +} + +// dataDogCtx returns a context with the DataDog API key set. +// This is one way to authenticate with the DataDog API. +func (d *DataDog) dataDogCtx(ctx context.Context) context.Context { + return context.WithValue(ctx, datadog.ContextAPIKeys, map[string]datadog.APIKey{ + "apiKeyAuth": { + Key: d.apiKey, + }, + }) +} + +// toDataDogEvent converts an eventv1.Event to a datadogV1.EventCreateRequest. +func (d *DataDog) toDataDogEvent(event *eventv1.Event) datadogV1.EventCreateRequest { + return datadogV1.EventCreateRequest{ + // Note: Title's printf format matches other events from datadog's kubernetes integration + Title: fmt.Sprintf("Events from the %s %s/%s", event.InvolvedObject.Kind, event.InvolvedObject.Name, event.InvolvedObject.Namespace), + Text: event.Message, + Tags: d.toDataDogTags(event), + // fluxcd matches the name datadog picked for their flux integration: https://docs.datadoghq.com/integrations/fluxcd/ + SourceTypeName: strPtr("fluxcd"), + DateHappened: int64Ptr(event.Timestamp.Unix()), + AlertType: toDataDogAlertType(event), + } +} + +// toDataDogTags parses an eventv1.Event to return a slice of tags. +// We set kind, name, and namespace to the appropriate values of the involved object. +func (d *DataDog) toDataDogTags(event *eventv1.Event) []string { + // Note: Datadog's built in kubernetes tagging is documented here: https://docs.datadoghq.com/containers/kubernetes/tag/?tab=containerizedagent#out-of-the-box-tags + tags := []string{ + fmt.Sprintf("flux_reporting_controller:%s", event.ReportingController), + fmt.Sprintf("flux_reason:%s", event.Reason), + // Note: DataDog standardizes kubernetes tags as "kube_*": https://github.com/DataDog/datadog-agent/blob/82dc933aa86de037c70fe960384aa06a62e457a8/pkg/collector/corechecks/cluster/kubernetesapiserver/events_common.go#L48 + fmt.Sprintf("kube_kind:%s", event.InvolvedObject.Kind), + fmt.Sprintf("kube_name:%s", event.InvolvedObject.Name), + fmt.Sprintf("kube_namespace:%s", event.InvolvedObject.Namespace), + } + + // add extra tags from event metadata + for k, v := range event.Metadata { + tags = append(tags, fmt.Sprintf("%s:%s", k, v)) + } + + // Note: https://docs.datadoghq.com/getting_started/tagging/ + // "Tags are converted to lowercase" + // To keep the events consistent, we run toLower on all input strings. + for idx := range tags { + tags[idx] = strings.ToLower(tags[idx]) + } + + return tags +} + +// toDataDogAlertType parses an eventv1.Event to return a datadogV1.EventAlertType. +func toDataDogAlertType(event *eventv1.Event) *datadogV1.EventAlertType { + if event.Severity == eventv1.EventSeverityError { + return dataDogEventAlertTypePtr(datadogV1.EVENTALERTTYPE_ERROR) + } + + return dataDogEventAlertTypePtr(datadogV1.EVENTALERTTYPE_INFO) +} + +func dataDogEventAlertTypePtr(t datadogV1.EventAlertType) *datadogV1.EventAlertType { + return &t +} diff --git a/internal/notifier/datadog_fuzz_test.go b/internal/notifier/datadog_fuzz_test.go new file mode 100644 index 000000000..e1d20a570 --- /dev/null +++ b/internal/notifier/datadog_fuzz_test.go @@ -0,0 +1,53 @@ +package notifier + +import ( + "context" + "crypto/x509" + "io" + "net/http" + "net/http/httptest" + "testing" + + fuzz "github.com/AdaLogics/go-fuzz-headers" + eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" + "github.com/stretchr/testify/require" +) + +func Fuzz_DataDog(f *testing.F) { + f.Add("token", "error", "", []byte{}, []byte{}) + f.Add("token", "info", "", []byte{}, []byte{}) + + f.Fuzz(func(t *testing.T, + apiKey, severity, message string, seed, response []byte) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/events", func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write(response) + require.NoError(t, err) + _, err = io.Copy(io.Discard, r.Body) + require.NoError(t, err) + require.NoError(t, r.Body.Close()) + }) + ts := httptest.NewServer(mux) + defer ts.Close() + + var cert x509.CertPool + _ = fuzz.NewConsumer(seed).GenerateStruct(&cert) + + dd, err := NewDataDog(ts.URL, "", &cert, apiKey) + if err != nil { + return + } + + event := eventv1.Event{} + _ = fuzz.NewConsumer(seed).GenerateStruct(&event) + + if event.Metadata == nil { + event.Metadata = map[string]string{} + } + + event.Message = message + event.Severity = severity + + _ = dd.Post(context.TODO(), event) + }) +} diff --git a/internal/notifier/datadog_test.go b/internal/notifier/datadog_test.go new file mode 100644 index 000000000..e72132133 --- /dev/null +++ b/internal/notifier/datadog_test.go @@ -0,0 +1,75 @@ +package notifier + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV1" + "github.com/stretchr/testify/require" +) + +func TestDataDogPost(t *testing.T) { + thisRun := func(expectedToFail bool) func(t *testing.T) { + return func(t *testing.T) { + ddApiKey := "sdfsdf" + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/events", func(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + var payload datadogV1.EventCreateRequest + err = json.Unmarshal(b, &payload) + require.NoError(t, err) + if expectedToFail { + w.WriteHeader(http.StatusForbidden) + } + }) + ts := httptest.NewServer(mux) + defer ts.Close() + + dd, err := NewDataDog(ts.URL, "", nil, ddApiKey) + require.NoError(t, err) + + err = dd.Post(context.Background(), testEvent()) + if expectedToFail { + require.Error(t, err) + } else { + require.NoError(t, err) + } + } + } + t.Run("working", thisRun(false)) + t.Run("failing", thisRun(true)) +} + +func TestDataDogProviderErrors(t *testing.T) { + _, err := NewDataDog("https://api.datadoghq.com", "", nil, "") + require.Error(t, err) + require.Equal(t, "token cannot be empty", err.Error()) + + _, err = NewDataDog("https://bad url :)", "", nil, "token") + require.Error(t, err) + require.Contains(t, err.Error(), "failed to parse address") +} + +func TestToDataDogTags(t *testing.T) { + dd, err := NewDataDog("https://api.datadoghq.com", "", nil, "token") + require.NoError(t, err) + + event := testEvent() + + tags := dd.toDataDogTags(&event) + + require.Contains(t, tags, "test:metadata") + require.Contains(t, tags, fmt.Sprintf("kube_kind:%s", strings.ToLower(event.InvolvedObject.Kind))) + require.Contains(t, tags, fmt.Sprintf("kube_namespace:%s", event.InvolvedObject.Namespace)) + require.Contains(t, tags, fmt.Sprintf("kube_name:%s", strings.ToLower(event.InvolvedObject.Name))) + require.Contains(t, tags, fmt.Sprintf("flux_reporting_controller:%s", strings.ToLower(event.ReportingController))) + require.Contains(t, tags, fmt.Sprintf("flux_reason:%s", strings.ToLower(event.Reason))) + +} diff --git a/internal/notifier/factory.go b/internal/notifier/factory.go index 1dafb198c..62458b68f 100644 --- a/internal/notifier/factory.go +++ b/internal/notifier/factory.go @@ -113,6 +113,8 @@ func (f Factory) Notifier(provider string) (Interface, error) { n, err = NewGrafana(f.URL, f.ProxyURL, f.Token, f.CertPool, f.Username, f.Password) case apiv1.PagerDutyProvider: n, err = NewPagerDuty(f.URL, f.ProxyURL, f.CertPool, f.Channel) + case apiv1.DataDogProvider: + n, err = NewDataDog(f.URL, f.ProxyURL, f.CertPool, f.Token) default: err = fmt.Errorf("provider %s not supported", provider) } diff --git a/internal/notifier/util.go b/internal/notifier/util.go index 9462e6f2c..c7b59bde7 100644 --- a/internal/notifier/util.go +++ b/internal/notifier/util.go @@ -125,3 +125,11 @@ func basicAuth(username, password string) string { auth := username + ":" + password return base64.StdEncoding.EncodeToString([]byte(auth)) } + +func strPtr(s string) *string { + return &s +} + +func int64Ptr(i int64) *int64 { + return &i +}