From f0dff420b982b2f618089ef2015f3a5d032a2a18 Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Wed, 11 Sep 2019 11:10:59 -0600 Subject: [PATCH] tweaks to sqlserver and database to allow more stable declarative deploy --- api/v1/sqldatabase_types.go | 2 +- api/v1/sqlserver_types.go | 2 +- controllers/sqldatabase_controller.go | 45 +++++++---- controllers/sqlserver_controller.go | 110 ++++++++++++++++---------- pkg/errhelp/errors.go | 25 ++++-- 5 files changed, 119 insertions(+), 65 deletions(-) diff --git a/api/v1/sqldatabase_types.go b/api/v1/sqldatabase_types.go index bb89e1619d1..19ce0cdaf88 100644 --- a/api/v1/sqldatabase_types.go +++ b/api/v1/sqldatabase_types.go @@ -65,5 +65,5 @@ func init() { } func (s *SqlDatabase) IsSubmitted() bool { - return s.Status.Provisioning || s.Status.Provisioned + return s.Status.Provisioned } diff --git a/api/v1/sqlserver_types.go b/api/v1/sqlserver_types.go index 1baf4f89978..ed4e7d10cca 100644 --- a/api/v1/sqlserver_types.go +++ b/api/v1/sqlserver_types.go @@ -66,5 +66,5 @@ func init() { } func (s *SqlServer) IsSubmitted() bool { - return s.Status.Provisioning || s.Status.Provisioned + return s.Status.Provisioned || s.Status.Provisioning } diff --git a/controllers/sqldatabase_controller.go b/controllers/sqldatabase_controller.go index 1aac41f567f..cd2ed8a7434 100644 --- a/controllers/sqldatabase_controller.go +++ b/controllers/sqldatabase_controller.go @@ -85,9 +85,18 @@ func (r *SqlDatabaseReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) if !instance.IsSubmitted() { r.Recorder.Event(&instance, "Normal", "Submitting", "starting resource reconciliation for SqlDatabase") if err := r.reconcileExternal(&instance); err != nil { - if errhelp.IsAsynchronousOperationNotComplete(err) || errhelp.IsGroupNotFound(err) { - r.Recorder.Event(&instance, "Normal", "Provisioning", "async op still running") - return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil + + catch := []string{ + errhelp.ParentNotFoundErrorCode, + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.NotFoundErrorCode, + errhelp.AsyncOpIncompleteError, + } + if azerr, ok := err.(*errhelp.AzureError); ok { + if helpers.ContainsString(catch, azerr.Type) { + log.Info("Got ignorable error", "type", azerr.Type) + return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil + } } return ctrl.Result{}, fmt.Errorf("error reconciling sql database in azure: %v", err) } @@ -127,26 +136,29 @@ func (r *SqlDatabaseReconciler) reconcileExternal(instance *azurev1.SqlDatabase) } r.Log.Info("Calling createorupdate SQL database") - instance.Status.Provisioning = true + // instance.Status.Provisioning = true - // write information back to instance - if updateerr := r.Status().Update(ctx, instance); updateerr != nil { - r.Recorder.Event(instance, "Warning", "Failed", "Unable to update instance") - } + // // write information back to instance + // if updateerr := r.Status().Update(ctx, instance); updateerr != nil { + // r.Recorder.Event(instance, "Warning", "Failed", "Unable to update instance") + // } _, err := sdkClient.CreateOrUpdateDB(sqlDatabaseProperties) if err != nil { if errhelp.IsAsynchronousOperationNotComplete(err) || errhelp.IsGroupNotFound(err) { r.Log.Info("Async operation not complete or group not found") - return err - } - r.Recorder.Event(instance, "Warning", "Failed", "Couldn't create resource in azure") - instance.Status.Provisioning = false - errUpdate := r.Status().Update(ctx, instance) - if errUpdate != nil { - r.Recorder.Event(instance, "Warning", "Failed", "Unable to update instance") + instance.Status.Provisioning = true + if errup := r.Status().Update(ctx, instance); errup != nil { + r.Recorder.Event(instance, "Warning", "Failed", "Unable to update instance") + } } - return err + + return errhelp.NewAzureError(err) + } + + _, err = sdkClient.GetDB(dbName) + if err != nil { + return errhelp.NewAzureError(err) } instance.Status.Provisioning = false @@ -174,6 +186,7 @@ func (r *SqlDatabaseReconciler) deleteExternal(instance *azurev1.SqlDatabase) er Location: location, } + r.Log.Info(fmt.Sprintf("deleting external resource: group/%s/server/%s/database/%s"+groupName, server, dbname)) _, err := sdk.DeleteDB(dbname) if err != nil { if errhelp.IsStatusCode204(err) { diff --git a/controllers/sqlserver_controller.go b/controllers/sqlserver_controller.go index e1814029649..e0e8ac6d34e 100644 --- a/controllers/sqlserver_controller.go +++ b/controllers/sqlserver_controller.go @@ -54,7 +54,6 @@ func (r *SqlServerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ctx := context.Background() log := r.Log.WithValues("sqlserver", req.NamespacedName) - // your logic here var instance azurev1.SqlServer if err := r.Get(ctx, req.NamespacedName, &instance); err != nil { @@ -68,7 +67,17 @@ func (r *SqlServerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { if helpers.IsBeingDeleted(&instance) { if helpers.HasFinalizer(&instance, SQLServerFinalizerName) { if err := r.deleteExternal(&instance); err != nil { + catch := []string{ + errhelp.AsyncOpIncompleteError, + } + if azerr, ok := err.(*errhelp.AzureError); ok { + if helpers.ContainsString(catch, azerr.Type) { + log.Info("Got ignorable error", "type", azerr.Type) + return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil + } + } log.Info("Delete SqlServer failed with ", "error", err.Error()) + return ctrl.Result{}, err } @@ -90,14 +99,27 @@ func (r *SqlServerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { if !instance.IsSubmitted() { r.Recorder.Event(&instance, "Normal", "Submitting", "starting resource reconciliation") if err := r.reconcileExternal(&instance); err != nil { - if strings.Contains(err.Error(), "asynchronous operation has not completed") { - r.Recorder.Event(&instance, "Normal", "Provisioning", "async op still running") - return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil + // if strings.Contains(err.Error(), "asynchronous operation has not completed") { + // r.Recorder.Event(&instance, "Normal", "Provisioning", "async op still running") + // return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil + // } + catch := []string{ + errhelp.ParentNotFoundErrorCode, + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.NotFoundErrorCode, + errhelp.AsyncOpIncompleteError, + } + if azerr, ok := err.(*errhelp.AzureError); ok { + if helpers.ContainsString(catch, azerr.Type) { + log.Info("Got ignorable error", "type", azerr.Type) + return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil + } } return ctrl.Result{}, fmt.Errorf("error reconciling sql server in azure: %v", err) } - // if the request was just sent to azure, the resource probably isn't ready yet - return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil + // give azure some time to catch up + log.Info("waiting for provision to take effect") + return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil } if err := r.verifyExternal(&instance); err != nil { @@ -105,6 +127,7 @@ func (r *SqlServerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { errhelp.ResourceGroupNotFoundErrorCode, errhelp.NotFoundErrorCode, errhelp.ResourceNotFound, + errhelp.AsyncOpIncompleteError, } if azerr, ok := err.(*errhelp.AzureError); ok { if helpers.ContainsString(catch, azerr.Type) { @@ -138,33 +161,22 @@ func (r *SqlServerReconciler) reconcileExternal(instance *azurev1.SqlServer) err Location: location, } - sqlServerProperties := sql.SQLServerProperties{ - AdministratorLogin: to.StringPtr(generateRandomString(8)), - AdministratorLoginPassword: to.StringPtr(generateRandomString(16)), - } - // Check to see if secret already exists for admin username/password - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: instance.Namespace, - }, - Data: map[string][]byte{ - "username": []byte(*sqlServerProperties.AdministratorLogin), - "password": []byte(*sqlServerProperties.AdministratorLoginPassword), - "sqlservernamespace": []byte(instance.Namespace), - "sqlservername": []byte(name), - }, - Type: "Opaque", + secret := r.GetOrPrepareSecret(instance) + sqlServerProperties := sql.SQLServerProperties{ + AdministratorLogin: to.StringPtr(string(secret.Data["username"])), + AdministratorLoginPassword: to.StringPtr(string(secret.Data["password"])), } - // If secret doesn't exist, generate creds - // Note: sql server enforces password policy. Details can be found here: - // https://docs.microsoft.com/en-us/sql/relational-databases/security/password-policy?view=sql-server-2017 - if err := r.Get(context.Background(), types.NamespacedName{Name: name, Namespace: instance.Namespace}, secret); err == nil { - r.Log.Info("secret already exists, pulling creds now") - sqlServerProperties.AdministratorLogin = to.StringPtr(string(secret.Data["username"])) - sqlServerProperties.AdministratorLoginPassword = to.StringPtr(string(secret.Data["password"])) + // create the sql server + //instance.Status.Provisioning = true + if _, err := sdkClient.CreateOrUpdateSQLServer(sqlServerProperties); err != nil { + if !strings.Contains(err.Error(), "not complete") { + r.Recorder.Event(instance, "Warning", "Failed", "Unable to provision or update instance") + return errhelp.NewAzureError(err) + } + } else { + r.Recorder.Event(instance, "Normal", "Provisioned", "resource request successfully dubmitted to Azure") } _, createOrUpdateSecretErr := controllerutil.CreateOrUpdate(context.Background(), r.Client, secret, func() error { @@ -179,23 +191,13 @@ func (r *SqlServerReconciler) reconcileExternal(instance *azurev1.SqlServer) err return createOrUpdateSecretErr } - // create the sql server instance.Status.Provisioning = true - _, err := sdkClient.CreateOrUpdateSQLServer(sqlServerProperties) - if err != nil { - r.Recorder.Event(instance, "Warning", "Failed", "Unable to provision or update instance") - instance.Status.Provisioning = false - err = errhelp.NewAzureError(err) - } else { - r.Recorder.Event(instance, "Normal", "Provisioned", "resource request successfully dubmitted to Azure") - } - // write information back to instance if updateerr := r.Status().Update(ctx, instance); updateerr != nil { r.Recorder.Event(instance, "Warning", "Failed", "Unable to update instance") } - return err + return nil } func (r *SqlServerReconciler) verifyExternal(instance *azurev1.SqlServer) error { @@ -264,13 +266,37 @@ func (r *SqlServerReconciler) deleteExternal(instance *azurev1.SqlServer) error _, err := sdkClient.DeleteSQLServer() if err != nil { r.Recorder.Event(instance, "Warning", "Failed", "Couldn't delete resouce in azure") - return err + return errhelp.NewAzureError(err) } r.Recorder.Event(instance, "Normal", "Deleted", name+" deleted") return nil } +func (r *SqlServerReconciler) GetOrPrepareSecret(instance *azurev1.SqlServer) *v1.Secret { + name := instance.ObjectMeta.Name + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: instance.Namespace, + }, + Data: map[string][]byte{ + "username": []byte(generateRandomString(8)), + "password": []byte(generateRandomString(16)), + "sqlservernamespace": []byte(instance.Namespace), + "sqlservername": []byte(name), + }, + Type: "Opaque", + } + + if err := r.Get(context.Background(), types.NamespacedName{Name: name, Namespace: instance.Namespace}, secret); err == nil { + r.Log.Info("secret already exists, pulling creds now") + } + + return secret +} + // helper function to generate username/password for secrets func generateRandomString(n int) string { rand.Seed(time.Now().UnixNano()) diff --git a/pkg/errhelp/errors.go b/pkg/errhelp/errors.go index 7b148af36a7..c64b554aa86 100644 --- a/pkg/errhelp/errors.go +++ b/pkg/errhelp/errors.go @@ -1,6 +1,8 @@ package errhelp import ( + "encoding/json" + "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" "k8s.io/apimachinery/pkg/api/errors" @@ -11,20 +13,20 @@ const ( ResourceGroupNotFoundErrorCode = "ResourceGroupNotFound" NotFoundErrorCode = "NotFound" ResourceNotFound = "ResourceNotFound" + AsyncOpIncompleteError = "AsyncOpIncomplete" ) func NewAzureError(err error) error { - + var kind, reason string if err == nil { return nil } ae := AzureError{ Original: err, } - //det := err.(autorest.DetailedError) if det, ok := err.(autorest.DetailedError); ok { - var kind, reason string + ae.Code = det.StatusCode.(int) if e, ok := det.Original.(*azure.RequestError); ok { kind = e.ServiceError.Code @@ -32,13 +34,26 @@ func NewAzureError(err error) error { } else if e, ok := det.Original.(*azure.ServiceError); ok { kind = e.Code reason = e.Message + if e.Code == "Failed" && len(e.AdditionalInfo) == 1 { + if v, ok := e.AdditionalInfo[0]["code"]; ok { + kind = v.(string) + } + } } else if _, ok := det.Original.(*errors.StatusError); ok { kind = "StatusError" reason = "StatusError" + } else if _, ok := det.Original.(*json.UnmarshalTypeError); ok { + kind = NotFoundErrorCode + reason = NotFoundErrorCode } - ae.Reason = reason - ae.Type = kind + + } else if _, ok := err.(azure.AsyncOpIncompleteError); ok { + kind = "AsyncOpIncomplete" + reason = "AsyncOpIncomplete" } + ae.Reason = reason + ae.Type = kind + return &ae }