diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index dc22b18..b4cedc2 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Code generated by controller-gen. DO NOT EDIT. +// autogenerated by controller-gen object, do not modify manually package v1alpha1 diff --git a/controllers/dbfsblock_controller_databricks.go b/controllers/dbfsblock_controller_databricks.go index 82e18e1..c68c7a8 100644 --- a/controllers/dbfsblock_controller_databricks.go +++ b/controllers/dbfsblock_controller_databricks.go @@ -34,7 +34,10 @@ func (r *DbfsBlockReconciler) submit(instance *databricksv1alpha1.DbfsBlock) err } // Open handler + execution := NewExecution("dbfsblocks", "create") createResponse, err := r.APIClient.Dbfs().Create(instance.Spec.Path, true) + execution.Finish(err) + if err != nil { return err } @@ -42,18 +45,26 @@ func (r *DbfsBlockReconciler) submit(instance *databricksv1alpha1.DbfsBlock) err // DataBricks limits the AddBlock size to be 1024KB var g = 1000 for i := 0; i < len(data); i += g { + execution = NewExecution("dbfsblocks", "add_block") + if i+g <= len(data) { err = r.APIClient.Dbfs().AddBlock(createResponse.Handle, data[i:i+g]) } else { err = r.APIClient.Dbfs().AddBlock(createResponse.Handle, data[i:]) } + + execution.Finish(err) + if err != nil { return err } } // Close handler + execution = NewExecution("dbfsblocks", "close") err = r.APIClient.Dbfs().Close(createResponse.Handle) + execution.Finish(err) + if err != nil { return err } @@ -61,7 +72,10 @@ func (r *DbfsBlockReconciler) submit(instance *databricksv1alpha1.DbfsBlock) err time.Sleep(1 * time.Second) // Refresh info + execution = NewExecution("dbfsblocks", "get_status") fileInfo, err := r.APIClient.Dbfs().GetStatus(instance.Spec.Path) + execution.Finish(err) + if err != nil { return err } @@ -83,5 +97,8 @@ func (r *DbfsBlockReconciler) delete(instance *databricksv1alpha1.DbfsBlock) err path := instance.Status.FileInfo.Path - return r.APIClient.Dbfs().Delete(path, true) + execution := NewExecution("dbfsblocks", "delete") + err := r.APIClient.Dbfs().Delete(path, true) + execution.Finish(err) + return err } diff --git a/controllers/dcluster_controller_databricks.go b/controllers/dcluster_controller_databricks.go index 654fc72..5f45ae9 100644 --- a/controllers/dcluster_controller_databricks.go +++ b/controllers/dcluster_controller_databricks.go @@ -22,7 +22,6 @@ import ( "reflect" databricksv1alpha1 "github.com/microsoft/azure-databricks-operator/api/v1alpha1" - "github.com/prometheus/client_golang/prometheus" dbmodels "github.com/xinsnake/databricks-sdk-golang/azure/models" ) @@ -80,31 +79,22 @@ func (r *DclusterReconciler) delete(instance *databricksv1alpha1.Dcluster) error return nil } - return trackExecutionTime(dclusterDeleteDuration, func() error { - err := r.APIClient.Clusters().PermanentDelete(instance.Status.ClusterInfo.ClusterID) - trackSuccessFailure(err, dclusterCounterVec, "delete") - return err - }) + execution := NewExecution("dclusters", "delete") + err := r.APIClient.Clusters().PermanentDelete(instance.Status.ClusterInfo.ClusterID) + execution.Finish(err) + return err } func (r *DclusterReconciler) getCluster(clusterID string) (cluster dbmodels.ClusterInfo, err error) { - timer := prometheus.NewTimer(dclusterGetDuration) - defer timer.ObserveDuration() - + execution := NewExecution("dclusters", "get") cluster, err = r.APIClient.Clusters().Get(clusterID) - - trackSuccessFailure(err, dclusterCounterVec, "get") - + execution.Finish(err) return cluster, err } func (r *DclusterReconciler) createCluster(instance *databricksv1alpha1.Dcluster) (cluster dbmodels.ClusterInfo, err error) { - timer := prometheus.NewTimer(dclusterCreateDuration) - defer timer.ObserveDuration() - + execution := NewExecution("dclusters", "create") cluster, err = r.APIClient.Clusters().Create(*instance.Spec) - - trackSuccessFailure(err, dclusterCounterVec, "create") - + execution.Finish(err) return cluster, err } diff --git a/controllers/dcluster_metrics.go b/controllers/dcluster_metrics.go deleted file mode 100644 index dc381f7..0000000 --- a/controllers/dcluster_metrics.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2019 microsoft. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "github.com/prometheus/client_golang/prometheus" - "sigs.k8s.io/controller-runtime/pkg/metrics" -) - -var ( - dclusterCounterVec = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: metricPrefix + "dcluster_total", - Help: "Counter related to the dCluster CRD partitioned by status and method invoked. Status = success/fail and method indicates REST endpoint", - }, - []string{"status", "method"}, - ) - - dclusterCreateDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "dcluster_creation_request_duration_seconds", - Help: "Duration of DB api dcluster create calls.", - }) - - dclusterGetDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "dcluster_get_request_duration_seconds", - Help: "Duration of DB api dcluster get calls.", - }) - - dclusterDeleteDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "dcluster_delete_request_duration_seconds", - Help: "Duration of DB api dcluster delete calls.", - }) -) - -func init() { - // Register custom metrics with the global prometheus registry - metrics.Registry.MustRegister(dclusterCounterVec, - dclusterCreateDuration, dclusterGetDuration, dclusterDeleteDuration) -} diff --git a/controllers/djob_controller_databricks.go b/controllers/djob_controller_databricks.go index 4288808..2709921 100644 --- a/controllers/djob_controller_databricks.go +++ b/controllers/djob_controller_databricks.go @@ -23,7 +23,6 @@ import ( "strings" databricksv1alpha1 "github.com/microsoft/azure-databricks-operator/api/v1alpha1" - "github.com/prometheus/client_golang/prometheus" dbmodels "github.com/xinsnake/databricks-sdk-golang/azure/models" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -71,7 +70,8 @@ func (r *DjobReconciler) submit(instance *databricksv1alpha1.Djob) error { } instance.ObjectMeta.SetOwnerReferences(references) } - job, err := r.createJob(instance) + jobSettings := databricksv1alpha1.ToDatabricksJobSettings(instance.Spec) + job, err := r.createJob(jobSettings) if err != nil { return err @@ -138,32 +138,22 @@ func (r *DjobReconciler) delete(instance *databricksv1alpha1.Djob) error { return err } - return trackExecutionTime(djobDeleteDuration, func() error { - err := r.APIClient.Jobs().Delete(jobID) - trackSuccessFailure(err, djobCounterVec, "delete") - return err - }) + execution := NewExecution("djobs", "delete") + err := r.APIClient.Jobs().Delete(jobID) + execution.Finish(err) + return err } func (r *DjobReconciler) getJob(jobID int64) (job dbmodels.Job, err error) { - timer := prometheus.NewTimer(djobGetDuration) - defer timer.ObserveDuration() - + execution := NewExecution("djobs", "get") job, err = r.APIClient.Jobs().Get(jobID) - - trackSuccessFailure(err, djobCounterVec, "get") - + execution.Finish(err) return job, err } -func (r *DjobReconciler) createJob(instance *databricksv1alpha1.Djob) (job dbmodels.Job, err error) { - timer := prometheus.NewTimer(djobCreateDuration) - defer timer.ObserveDuration() - - jobSettings := databricksv1alpha1.ToDatabricksJobSettings(instance.Spec) +func (r *DjobReconciler) createJob(jobSettings dbmodels.JobSettings) (job dbmodels.Job, err error) { + execution := NewExecution("djobs", "create") job, err = r.APIClient.Jobs().Create(jobSettings) - - trackSuccessFailure(err, djobCounterVec, "create") - + execution.Finish(err) return job, err } diff --git a/controllers/djob_metrics.go b/controllers/djob_metrics.go deleted file mode 100644 index 801dfd9..0000000 --- a/controllers/djob_metrics.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2019 microsoft. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "github.com/prometheus/client_golang/prometheus" - "sigs.k8s.io/controller-runtime/pkg/metrics" -) - -var ( - djobCounterVec = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: metricPrefix + "djob_total", - Help: "Counter related to the dJob CRD partitioned by status and method invoked. Status = success/fail and method indicates REST endpoint", - }, - []string{"status", "method"}, - ) - - djobCreateDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "djob_creation_request_duration_seconds", - Help: "Duration of DB api djob create calls.", - }) - - djobGetDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "djob_get_request_duration_seconds", - Help: "Duration of DB api djob get calls.", - }) - - djobDeleteDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "djob_delete_request_duration_seconds", - Help: "Duration of DB api djob delete calls.", - }) -) - -func init() { - // Register custom metrics with the global prometheus registry - metrics.Registry.MustRegister(djobCounterVec, - djobCreateDuration, djobGetDuration, djobDeleteDuration) -} diff --git a/controllers/metrics.go b/controllers/metrics.go index 8a2d2c2..38621ec 100644 --- a/controllers/metrics.go +++ b/controllers/metrics.go @@ -17,25 +17,48 @@ limitations under the License. package controllers import ( + "time" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" ) const ( - metricPrefix = "databricks_" successMetric = "success" failureMetric = "failure" ) -func trackExecutionTime(histogram prometheus.Histogram, f func() error) error { - timer := prometheus.NewTimer(histogram) - defer timer.ObserveDuration() - return f() +var databricksRequestHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "databricks_request_duration_seconds", + Help: "Duration of upstream calls to Databricks REST service endpoints", +}, []string{"object_type", "action", "outcome"}) + +func init() { + // Register custom metrics with the global prometheus registry + metrics.Registry.MustRegister(databricksRequestHistogram) +} + +// NewExecution creates an Execution instance and starts the timer +func NewExecution(objectType string, action string) Execution { + return Execution{ + begin: time.Now(), + labels: prometheus.Labels{"object_type": objectType, "action": action}, + } +} + +// Execution tracks state for an API execution for emitting metrics +type Execution struct { + begin time.Time + labels prometheus.Labels } -func trackSuccessFailure(err error, counterVec *prometheus.CounterVec, method string) { +// Finish is used to log duration and success/failure +func (e *Execution) Finish(err error) { if err == nil { - counterVec.With(prometheus.Labels{"status": successMetric, "method": method}).Inc() + e.labels["outcome"] = successMetric } else { - counterVec.With(prometheus.Labels{"status": failureMetric, "method": method}).Inc() + e.labels["outcome"] = failureMetric } + duration := time.Since(e.begin) + databricksRequestHistogram.With(e.labels).Observe(duration.Seconds()) } diff --git a/controllers/run_controller_databricks.go b/controllers/run_controller_databricks.go index 962c24e..9d139df 100644 --- a/controllers/run_controller_databricks.go +++ b/controllers/run_controller_databricks.go @@ -24,7 +24,6 @@ import ( "time" databricksv1alpha1 "github.com/microsoft/azure-databricks-operator/api/v1alpha1" - "github.com/prometheus/client_golang/prometheus" "github.com/xinsnake/databricks-sdk-golang/azure" dbmodels "github.com/xinsnake/databricks-sdk-golang/azure/models" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -111,21 +110,20 @@ func (r *RunReconciler) delete(instance *databricksv1alpha1.Run) error { // We will not check for error when cancelling a job, // if it fails just let it be - r.APIClient.Jobs().RunsCancel(runID) //nolint:errcheck + execution := NewExecution("runs", "cancel") + err := r.APIClient.Jobs().RunsCancel(runID) + execution.Finish(err) // It takes time for DataBricks to cancel a run time.Sleep(15 * time.Second) - return trackExecutionTime(runDeleteDuration, func() error { - err := r.APIClient.Jobs().RunsDelete(runID) - trackSuccessFailure(err, runCounterVec, "delete") - return err - }) + execution = NewExecution("runs", "delete") + err = r.APIClient.Jobs().RunsDelete(runID) + execution.Finish(err) + return err } func (r *RunReconciler) runUsingRunNow(instance *databricksv1alpha1.Run) (*dbmodels.Run, bool, error) { - timer := prometheus.NewTimer(runNowDuration) - defer timer.ObserveDuration() runParameters := dbmodels.RunParameters{ JarParams: instance.Spec.JarParams, @@ -154,15 +152,13 @@ func (r *RunReconciler) runUsingRunNow(instance *databricksv1alpha1.Run) (*dbmod return nil, true, fmt.Errorf("Run references Djob that is not yet submitted") } + execution := NewExecution("runs", "run_now") run, err := r.APIClient.Jobs().RunNow(k8sJob.Status.JobStatus.JobID, runParameters) - trackSuccessFailure(err, runCounterVec, "runsnow") + execution.Finish(err) return &run, false, err } func (r *RunReconciler) runUsingRunsSubmit(instance *databricksv1alpha1.Run) (*dbmodels.Run, error) { - timer := prometheus.NewTimer(runSubmitDuration) - defer timer.ObserveDuration() - clusterSpec := dbmodels.ClusterSpec{ NewCluster: instance.Spec.NewCluster, ExistingClusterID: instance.Spec.ExistingClusterID, @@ -175,29 +171,24 @@ func (r *RunReconciler) runUsingRunsSubmit(instance *databricksv1alpha1.Run) (*d SparkSubmitTask: instance.Spec.SparkSubmitTask, } + execution := NewExecution("runs", "run_submit") run, err := r.APIClient.Jobs().RunsSubmit(instance.Spec.RunName, clusterSpec, jobTask, instance.Spec.TimeoutSeconds) - trackSuccessFailure(err, runCounterVec, "runssubmit") + execution.Finish(err) return &run, err } func (r *RunReconciler) getRun(runID int64) (dbmodels.Run, error) { - timer := prometheus.NewTimer(runGetDuration) - defer timer.ObserveDuration() - + execution := NewExecution("runs", "get") runOutput, err := r.APIClient.Jobs().RunsGet(runID) - - trackSuccessFailure(err, runCounterVec, "get") + execution.Finish(err) return runOutput, err } func (r *RunReconciler) getRunOutput(runID int64) (azure.JobsRunsGetOutputResponse, error) { - timer := prometheus.NewTimer(runGetOutputDuration) - defer timer.ObserveDuration() - + execution := NewExecution("runs", "run_get_output") runOutput, err := r.APIClient.Jobs().RunsGetOutput(runID) - - trackSuccessFailure(err, runCounterVec, "getoutput") + execution.Finish(err) return runOutput, err } diff --git a/controllers/run_metrics.go b/controllers/run_metrics.go deleted file mode 100644 index 903cd2a..0000000 --- a/controllers/run_metrics.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2019 microsoft. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "github.com/prometheus/client_golang/prometheus" - "sigs.k8s.io/controller-runtime/pkg/metrics" -) - -var ( - runCounterVec = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: metricPrefix + "run_total", - Help: "Counter related to the Run CRD partitioned by status and method invoked. Status = success/fail and method indicates REST endpoint", - }, - []string{"status", "method"}, - ) - - runSubmitDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "run_submit_request_duration_seconds", - Help: "Duration of DB api run submit calls.", - }) - - runNowDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "run_now_request_duration_seconds", - Help: "Duration of DB api run now calls.", - }) - - runGetDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "run_get_request_duration_seconds", - Help: "Duration of DB api run get calls.", - }) - - runGetOutputDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "run_get_output_request_duration_seconds", - Help: "Duration of DB api run get output calls.", - }) - - runDeleteDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: metricPrefix + "run_delete_request_duration_seconds", - Help: "Duration of DB api run delete calls.", - }) -) - -func init() { - // Register custom metrics with the global prometheus registry - metrics.Registry.MustRegister(runCounterVec, runSubmitDuration, - runNowDuration, runGetDuration, runGetOutputDuration, runDeleteDuration) -} diff --git a/controllers/secretscope_controller_databricks.go b/controllers/secretscope_controller_databricks.go index 380ecbf..2404519 100644 --- a/controllers/secretscope_controller_databricks.go +++ b/controllers/secretscope_controller_databricks.go @@ -29,7 +29,9 @@ import ( ) func (r *SecretScopeReconciler) get(scope string) (*dbmodels.SecretScope, error) { + execution := NewExecution("secretscopes", "list_secret_scops") scopes, err := r.APIClient.Secrets().ListSecretScopes() + execution.Finish(err) if err != nil { return nil, err } @@ -51,7 +53,9 @@ func (r *SecretScopeReconciler) get(scope string) (*dbmodels.SecretScope, error) func (r *SecretScopeReconciler) submitSecrets(instance *databricksv1alpha1.SecretScope) error { scope := instance.ObjectMeta.Name namespace := instance.Namespace + execution := NewExecution("secretscopes", "list_secrets") scopeSecrets, err := r.APIClient.Secrets().ListSecrets(scope) + execution.Finish(err) if err != nil { return err } @@ -60,7 +64,9 @@ func (r *SecretScopeReconciler) submitSecrets(instance *databricksv1alpha1.Secre // therefore, we delete all then create all if len(scopeSecrets) > 0 { for _, existingSecret := range scopeSecrets { + execution := NewExecution("secretscopes", "delete_secret") err = r.APIClient.Secrets().DeleteSecret(scope, existingSecret.Key) + execution.Finish(err) if err != nil { return err } @@ -69,7 +75,9 @@ func (r *SecretScopeReconciler) submitSecrets(instance *databricksv1alpha1.Secre for _, secret := range instance.Spec.SecretScopeSecrets { if secret.StringValue != "" { + execution := NewExecution("secretscopes", "put_secret_string") err = r.APIClient.Secrets().PutSecretString(secret.StringValue, scope, secret.Key) + execution.Finish(err) if err != nil { return err } @@ -78,7 +86,9 @@ func (r *SecretScopeReconciler) submitSecrets(instance *databricksv1alpha1.Secre if err != nil { return err } + execution := NewExecution("secretscopes", "put_secret") err = r.APIClient.Secrets().PutSecret(v, scope, secret.Key) + execution.Finish(err) if err != nil { return err } @@ -88,7 +98,9 @@ func (r *SecretScopeReconciler) submitSecrets(instance *databricksv1alpha1.Secre return err } + execution := NewExecution("secretscopes", "put_secret_string") err = r.APIClient.Secrets().PutSecretString(value, scope, secret.Key) + execution.Finish(err) if err != nil { return err } @@ -116,14 +128,18 @@ func (r *SecretScopeReconciler) getSecretValueFrom(namespace string, scopeSecret func (r *SecretScopeReconciler) submitACLs(instance *databricksv1alpha1.SecretScope) error { scope := instance.ObjectMeta.Name + execution := NewExecution("secretscopes", "list_secret_acls") scopeSecretACLs, err := r.APIClient.Secrets().ListSecretACLs(scope) + execution.Finish(err) if err != nil { return err } if len(scopeSecretACLs) > 0 { for _, existingACL := range scopeSecretACLs { + execution := NewExecution("secretscopes", "delete_secret_acl") err = r.APIClient.Secrets().DeleteSecretACL(scope, existingACL.Principal) + execution.Finish(err) if err != nil { return err } @@ -146,7 +162,9 @@ func (r *SecretScopeReconciler) submitACLs(instance *databricksv1alpha1.SecretSc return err } + execution := NewExecution("secretscopes", "put_secret_acl") err = r.APIClient.Secrets().PutSecretACL(scope, acl.Principal, permission) + execution.Finish(err) } return nil @@ -173,7 +191,9 @@ func (r *SecretScopeReconciler) submit(instance *databricksv1alpha1.SecretScope) scope := instance.ObjectMeta.Name initialManagePrincipal := instance.Spec.InitialManagePrincipal + execution := NewExecution("secretscopes", "create_secret_scope") err = r.APIClient.Secrets().CreateSecretScope(scope, initialManagePrincipal) + execution.Finish(err) if err != nil { return } @@ -205,7 +225,9 @@ func (r *SecretScopeReconciler) delete(instance *databricksv1alpha1.SecretScope) if instance.Status.SecretScope != nil { scope := instance.Status.SecretScope.Name + execution := NewExecution("secretscopes", "delete_secret_scope") err := r.APIClient.Secrets().DeleteSecretScope(scope) + execution.Finish(err) if err != nil && !strings.Contains(err.Error(), "does not exist") { return err } diff --git a/controllers/workspaceitem_controller_databricks.go b/controllers/workspaceitem_controller_databricks.go index 3873bc0..2dac7f4 100644 --- a/controllers/workspaceitem_controller_databricks.go +++ b/controllers/workspaceitem_controller_databricks.go @@ -36,7 +36,9 @@ func (r *WorkspaceItemReconciler) submit(instance *databricksv1alpha1.WorkspaceI return err } + execution := NewExecution("workspaceitems", "import") err = r.APIClient.Workspace().Import(instance.Spec.Path, instance.Spec.Format, instance.Spec.Language, data, true) + execution.Finish(err) if err != nil { return err } @@ -44,7 +46,9 @@ func (r *WorkspaceItemReconciler) submit(instance *databricksv1alpha1.WorkspaceI time.Sleep(1 * time.Second) // Refresh info + execution = NewExecution("workspaceitems", "get_status") objectInfo, err := r.APIClient.Workspace().GetStatus(instance.Spec.Path) + execution.Finish(err) if err != nil { return err } @@ -66,5 +70,8 @@ func (r *WorkspaceItemReconciler) delete(instance *databricksv1alpha1.WorkspaceI path := instance.Status.ObjectInfo.Path - return r.APIClient.Workspace().Delete(path, true) + execution := NewExecution("workspaceitems", "import") + err := r.APIClient.Workspace().Delete(path, true) + execution.Finish(err) + return err } diff --git a/docs/resources.md b/docs/resources.md index 60a7f9d..a5c8037 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -60,13 +60,13 @@ More info: - Open another terminal and curl request the metric endpoint: `curl localhost:8080/metrics` ### Counter metrics -Counter metrics take the format `databricks_[x]_total` where: -- x: Object being maniputlated; example: `dcluster` -Counter metrics have labels that show breakdown by: -- status (success | failure) -- method (the action being performed via REST call example: get, create, delete) +In addition to the standard metrics that kubebuilder provides, the following custom metrics have been added. -Histogram metrics take the format `databricks_[x]_[action]_request_duration_seconds` where: -- x: Object being maniputlated; example: `dcluster` -- action: Action being performed; example: `create` +The `databricks_request_duration_seconds` histogram provides metrics on the duration of calls via the databricks SDK and has the following labels: + +|Name|Description| +|-|-| +|`object_type`|The type of object that the call relatest to, e.g. `dcluster`| +|`action`| The action being performed, e.g. `get`, `create`| +|`outcome`| `success` or `failure`| diff --git a/go.mod b/go.mod index 6ddaaca..b35f33c 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/onsi/ginkgo v1.10.3 github.com/onsi/gomega v1.7.0 github.com/prometheus/client_golang v0.9.2 + github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 github.com/spf13/pflag v1.0.5 // indirect github.com/xinsnake/databricks-sdk-golang v0.1.2 golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 // indirect