Add AzureSQL short term retention policies

api/v1beta1/azuresqldatabase_types.go

@@ -40,6 +40,13 @@ type SqlDatabaseSku struct {
 	Capacity *int32 `json:"capacity,omitempty"`
 }
 
+type SQLDatabaseShortTermRetentionPolicy struct {
+	// RetentionDays is the backup retention period in days. This is how many days
+	// Point-in-Time Restore will be supported.
+	// +kubebuilder:validation:Required
+	RetentionDays int32 `json:"retentionDays"`
+}
+
 // AzureSqlDatabaseSpec defines the desired state of AzureSqlDatabase
 type AzureSqlDatabaseSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
@@ -59,14 +66,15 @@ type AzureSqlDatabaseSpec struct {
 	Server string `json:"server"`
 
 	// +kubebuilder:validation:Optional
-	Edition          DBEdition          `json:"edition"`       // TODO: Remove this in v1beta2
-	Sku              *SqlDatabaseSku    `json:"sku,omitempty"` // TODO: make this required in v1beta2
-	MaxSize          *resource.Quantity `json:"maxSize,omitempty"`
-	DbName           string             `json:"dbName,omitempty"`
-	WeeklyRetention  string             `json:"weeklyRetention,omitempty"`
-	MonthlyRetention string             `json:"monthlyRetention,omitempty"`
-	YearlyRetention  string             `json:"yearlyRetention,omitempty"`
-	WeekOfYear       int32              `json:"weekOfYear,omitempty"`
+	Edition                  DBEdition                            `json:"edition"`       // TODO: Remove this in v1beta2
+	Sku                      *SqlDatabaseSku                      `json:"sku,omitempty"` // TODO: make this required in v1beta2
+	MaxSize                  *resource.Quantity                   `json:"maxSize,omitempty"`
+	DbName                   string                               `json:"dbName,omitempty"`
+	WeeklyRetention          string                               `json:"weeklyRetention,omitempty"`
+	MonthlyRetention         string                               `json:"monthlyRetention,omitempty"`
+	YearlyRetention          string                               `json:"yearlyRetention,omitempty"`
+	WeekOfYear               int32                                `json:"weekOfYear,omitempty"`
+	ShortTermRetentionPolicy *SQLDatabaseShortTermRetentionPolicy `json:"shortTermRetentionPolicy,omitempty"`
 }
 
 // AzureSqlDatabase is the Schema for the azuresqldatabases API

config/samples/azure_v1beta1_azuresqldatabase.yaml

@@ -28,4 +28,9 @@ spec:
 
   # The week of year to take the yearly backup, valid values [1, 52]
   # weekOfYear: 16
-
+
+  # The short term retention policy to use
+  # shortTermRetentionPolicy:
+      # RetentionDays is the backup retention period in days. This is how many days
+      #	Point-in-Time Restore will be supported.
+  #   retentionDays: 21

controllers/azuresql_combined_test.go

@@ -11,17 +11,18 @@ import (
 	"strings"
 	"testing"
 
-	azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1"
-	"github.com/Azure/azure-service-operator/api/v1beta1"
 	"github.com/stretchr/testify/assert"
-
-	helpers "github.com/Azure/azure-service-operator/pkg/helpers"
-	"github.com/Azure/azure-service-operator/pkg/resourcemanager/config"
-	kvsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+
+	azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1"
+	"github.com/Azure/azure-service-operator/api/v1beta1"
+	"github.com/Azure/azure-service-operator/pkg/errhelp"
+	helpers "github.com/Azure/azure-service-operator/pkg/helpers"
+	"github.com/Azure/azure-service-operator/pkg/resourcemanager/config"
+	kvsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault"
 )
 
 func TestAzureSqlServerCombinedHappyPath(t *testing.T) {
@@ -65,8 +66,10 @@ func TestAzureSqlServerCombinedHappyPath(t *testing.T) {
 
 	sqlDatabaseName1 := GenerateTestResourceNameWithRandom("sqldatabase", 10)
 	sqlDatabaseName2 := GenerateTestResourceNameWithRandom("sqldatabase", 10)
+	sqlDatabaseName3 := GenerateTestResourceNameWithRandom("sqldatabase", 10)
 	var sqlDatabaseInstance1 *v1beta1.AzureSqlDatabase
 	var sqlDatabaseInstance2 *v1beta1.AzureSqlDatabase
+	var sqlDatabaseInstance3 *v1beta1.AzureSqlDatabase
 
 	sqlFirewallRuleNamespacedNameLocal := types.NamespacedName{
 		Name:      GenerateTestResourceNameWithRandom("sqlfwr-local", 10),
@@ -190,6 +193,47 @@ func TestAzureSqlServerCombinedHappyPath(t *testing.T) {
 			assert.Equal(true, db.Status.Provisioned)
 		})
 
+		// Create a database in the new server
+		t.Run("set up database with short and long term retention", func(t *testing.T) {
+			t.Parallel()
+
+			// Create the SqlDatabase object and expect the Reconcile to be created
+			sqlDatabaseInstance3 = &v1beta1.AzureSqlDatabase{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      sqlDatabaseName3,
+					Namespace: "default",
+				},
+				Spec: v1beta1.AzureSqlDatabaseSpec{
+					Location:      rgLocation,
+					ResourceGroup: rgName,
+					Server:        sqlServerName,
+					Sku: &v1beta1.SqlDatabaseSku{
+						Name: "S0",
+						Tier: "Standard",
+					},
+					WeeklyRetention: "P3W",
+					ShortTermRetentionPolicy: &v1beta1.SQLDatabaseShortTermRetentionPolicy{
+						RetentionDays: 3,
+					},
+				},
+			}
+
+			EnsureInstance(ctx, t, tc, sqlDatabaseInstance3)
+
+			// Now update with an invalid retention policy
+			sqlDatabaseInstance3.Spec.ShortTermRetentionPolicy.RetentionDays = -1
+			err = tc.k8sClient.Update(ctx, sqlDatabaseInstance3)
+			assert.Equal(nil, err, "updating sql database in k8s")
+
+			namespacedName := types.NamespacedName{Name: sqlDatabaseName3, Namespace: "default"}
+			assert.Eventually(func() bool {
+				db := &v1beta1.AzureSqlDatabase{}
+				err = tc.k8sClient.Get(ctx, namespacedName, db)
+				assert.Equal(nil, err, "err getting DB from k8s")
+				return db.Status.Provisioned == false && strings.Contains(db.Status.Message, errhelp.BackupRetentionPolicyInvalid)
+			}, tc.timeout, tc.retry, "wait for sql database to be updated in k8s")
+		})
+
 		// Create FirewallRules ---------------------------------------
 
 		t.Run("set up wide range firewall rule in primary server", func(t *testing.T) {
@@ -494,6 +538,7 @@ func TestAzureSqlServerCombinedHappyPath(t *testing.T) {
 			t.Parallel()
 			EnsureDelete(ctx, t, tc, sqlDatabaseInstance1)
 			EnsureDelete(ctx, t, tc, sqlDatabaseInstance2)
+			EnsureDelete(ctx, t, tc, sqlDatabaseInstance3)
 		})
 
 	})

controllers/keyvault_controller_test.go

@@ -407,8 +407,8 @@ func TestKeyvaultControllerBadAccessPolicy(t *testing.T) {
 			Namespace: "default",
 		},
 		Spec: azurev1alpha1.KeyVaultSpec{
-			Location:      keyVaultLocation,
-			ResourceGroup: tc.resourceGroupName,
+			Location:       keyVaultLocation,
+			ResourceGroup:  tc.resourceGroupName,
 			AccessPolicies: &accessPolicies,
 		},
 	}

pkg/errhelp/errors.go

@@ -66,6 +66,7 @@ const (
 	FeatureNotSupportedForEdition                  = "FeatureNotSupportedForEdition"
 	VirtualNetworkRuleBadRequest                   = "VirtualNetworkRuleBadRequest"
 	LongTermRetentionPolicyInvalid                 = "LongTermRetentionPolicyInvalid"
+	BackupRetentionPolicyInvalid                   = "InvalidBackupRetentionPeriod"
 	OperationIdNotFound                            = "OperationIdNotFound"
 )
 

pkg/resourcemanager/azuresql/azuresqldb/azuresqldb.go

@@ -6,10 +6,11 @@ package azuresqldb
 import (
 	"context"
 	"fmt"
-	"net/http"
 
 	"github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/v3.0/sql"
-	sql3 "github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/v3.0/sql"
+	"github.com/pkg/errors"
+
+	"github.com/Azure/azure-service-operator/api/v1beta1"
 	"github.com/Azure/azure-service-operator/pkg/errhelp"
 	"github.com/Azure/azure-service-operator/pkg/helpers"
 	azuresqlshared "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared"
@@ -139,28 +140,29 @@ func (m *AzureSqlDbManager) CreateOrUpdateDB(
 }
 
 // AddLongTermRetention enables / disables long term retention
-func (m *AzureSqlDbManager) AddLongTermRetention(ctx context.Context, resourceGroupName string, serverName string, databaseName string, weeklyRetention string, monthlyRetention string, yearlyRetention string, weekOfYear int32) (*http.Response, error) {
+func (m *AzureSqlDbManager) AddLongTermRetention(
+	ctx context.Context,
+	resourceGroupName string,
+	serverName string,
+	databaseName string,
+	weeklyRetention string,
+	monthlyRetention string,
+	yearlyRetention string,
+	weekOfYear int32) (*sql.BackupLongTermRetentionPoliciesCreateOrUpdateFuture, error) {
 
 	longTermClient, err := azuresqlshared.GetBackupLongTermRetentionPoliciesClient(m.creds)
-	// TODO: Probably shouldn't return a response at all in the err case here (all through this function)
 	if err != nil {
-		return &http.Response{
-			StatusCode: 0,
-		}, err
+		return nil, err
 	}
 
 	// validate the input and exit if nothing needs to happen - this is ok!
 	if weeklyRetention == "" && monthlyRetention == "" && yearlyRetention == "" {
-		return &http.Response{
-			StatusCode: 200,
-		}, nil
+		return nil, nil
 	}
 
 	// validate the pairing of yearly retention and week of year
 	if yearlyRetention != "" && (weekOfYear <= 0 || weekOfYear > 52) {
-		return &http.Response{
-			StatusCode: 500,
-		}, fmt.Errorf("weekOfYear must be greater than 0 and less or equal to 52 when yearlyRetention is used")
+		return nil, fmt.Errorf("weekOfYear must be greater than 0 and less or equal to 52 when yearlyRetention is used")
 	}
 
 	// create pointers so that we can pass nils if needed
@@ -184,8 +186,8 @@ func (m *AzureSqlDbManager) AddLongTermRetention(ctx context.Context, resourceGr
 		resourceGroupName,
 		serverName,
 		databaseName,
-		sql3.BackupLongTermRetentionPolicy{
-			LongTermRetentionPolicyProperties: &sql3.LongTermRetentionPolicyProperties{
+		sql.BackupLongTermRetentionPolicy{
+			LongTermRetentionPolicyProperties: &sql.LongTermRetentionPolicyProperties{
 				WeeklyRetention:  pWeeklyRetention,
 				MonthlyRetention: pMonthlyRetention,
 				YearlyRetention:  pYearlyRetention,
@@ -195,12 +197,57 @@ func (m *AzureSqlDbManager) AddLongTermRetention(ctx context.Context, resourceGr
 	)
 
 	if err != nil {
-		return &http.Response{
-			StatusCode: 500,
-		}, nil
+		return nil, err
+	}
+
+	return &future, err
+}
+
+func (m *AzureSqlDbManager) AddShortTermRetention(
+	ctx context.Context,
+	resourceGroupName string,
+	serverName string,
+	databaseName string,
+	policy *v1beta1.SQLDatabaseShortTermRetentionPolicy) (*sql.BackupShortTermRetentionPoliciesCreateOrUpdateFuture, error) {
+
+	client, err := azuresqlshared.GetBackupShortTermRetentionPoliciesClient(m.creds)
+	if err != nil {
+		return nil, errors.Wrapf(err, "couldn't create BackupShortTermRetentionPoliciesClient")
+	}
+
+	var policyProperties *sql.BackupShortTermRetentionPolicyProperties
+	if policy == nil {
+		// If policy is nil we're in a bit of an awkward situation since we cannot know if the customer has mutated
+		// the retention policy in a previous reconciliation loop and then subsequently removed it. If they have,
+		// "doing nothing" here is wrong because that leaves them in the previous modified state (but with no reflection
+		// of that fact in the Spec).
+		// Unfortunately you cannot update the retention policy to nil, nor can you delete it, so we must awkwardly
+		// set it back to its default configuration.
+		// Note: There are risks here, such as if the default on the server and the default in our code drift apart
+		// at some point in the future.
+		policyProperties = &sql.BackupShortTermRetentionPolicyProperties{
+			RetentionDays: to.Int32Ptr(7), // 7 is the magical default as of Jan 2021
+		}
+	} else {
+		policyProperties = &sql.BackupShortTermRetentionPolicyProperties{
+			RetentionDays: to.Int32Ptr(policy.RetentionDays),
+		}
+	}
+
+	future, err := client.CreateOrUpdate(
+		ctx,
+		resourceGroupName,
+		serverName,
+		databaseName,
+		sql.BackupShortTermRetentionPolicy{
+			BackupShortTermRetentionPolicyProperties: policyProperties,
+		})
+
+	if err != nil {
+		return nil, err
 	}
 
-	return future.Response(), err
+	return &future, err
 }
 
 var goneCodes = []string{

pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_manager.go

@@ -5,9 +5,9 @@ package azuresqldb
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/v3.0/sql"
+
 	azuresqlshared "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared"
 
 	"github.com/Azure/azure-service-operator/pkg/resourcemanager"
@@ -42,7 +42,7 @@ type SqlDbManager interface {
 		weeklyRetention string,
 		monthlyRetention string,
 		yearlyRetention string,
-		weekOfYear int32) (*http.Response, error)
+		weekOfYear int32) (*sql.BackupLongTermRetentionPoliciesCreateOrUpdateFuture, error)
 
 	resourcemanager.ARMClient
 }

pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go

@@ -133,6 +133,27 @@ func (db *AzureSqlDbManager) Ensure(ctx context.Context, obj runtime.Object, opt
 				}
 			}
 
+			_, err = db.AddShortTermRetention(
+				ctx,
+				groupName,
+				server,
+				dbName,
+				instance.Spec.ShortTermRetentionPolicy)
+			if err != nil {
+				failureErrors := []string{
+					errhelp.BackupRetentionPolicyInvalid,
+				}
+				instance.Status.Message = fmt.Sprintf("Azure DB short-term retention policy error: %s", errhelp.StripErrorIDs(err))
+				azerr := errhelp.NewAzureError(err)
+				if helpers.ContainsString(failureErrors, azerr.Type) {
+					// Leave message the same as above
+					instance.Status.SetFailedProvisioning(instance.Status.Message)
+					return true, nil
+				} else {
+					return false, err
+				}
+			}
+
 			// db exists, we have successfully provisioned everything
 			instance.Status.SetProvisioned(resourcemanager.SuccessMsg)
 			instance.Status.State = string(dbGet.Status)

pkg/resourcemanager/azuresql/azuresqlshared/getgoclients.go

@@ -61,36 +61,48 @@ func GetGoFirewallClient(creds config.Credentials) (sql.FirewallRulesClient, err
 
 // GetGoVNetRulesClient retrieves a VirtualNetworkRulesClient
 func GetGoVNetRulesClient(creds config.Credentials) (sql.VirtualNetworkRulesClient, error) {
-	VNetRulesClient := sql.NewVirtualNetworkRulesClientWithBaseURI(config.BaseURI(), creds.SubscriptionID())
+	vnetRulesClient := sql.NewVirtualNetworkRulesClientWithBaseURI(config.BaseURI(), creds.SubscriptionID())
 	a, err := iam.GetResourceManagementAuthorizer(creds)
 	if err != nil {
 		return sql.VirtualNetworkRulesClient{}, err
 	}
-	VNetRulesClient.Authorizer = a
-	VNetRulesClient.AddToUserAgent(config.UserAgent())
-	return VNetRulesClient, nil
+	vnetRulesClient.Authorizer = a
+	vnetRulesClient.AddToUserAgent(config.UserAgent())
+	return vnetRulesClient, nil
 }
 
 // GetNetworkSubnetClient retrieves a Subnetclient
 func GetGoNetworkSubnetClient(creds config.Credentials, subscription string) (network.SubnetsClient, error) {
-	SubnetsClient := network.NewSubnetsClientWithBaseURI(config.BaseURI(), subscription)
+	subnetsClient := network.NewSubnetsClientWithBaseURI(config.BaseURI(), subscription)
 	a, err := iam.GetResourceManagementAuthorizer(creds)
 	if err != nil {
 		return network.SubnetsClient{}, err
 	}
-	SubnetsClient.Authorizer = a
-	SubnetsClient.AddToUserAgent(config.UserAgent())
-	return SubnetsClient, nil
+	subnetsClient.Authorizer = a
+	subnetsClient.AddToUserAgent(config.UserAgent())
+	return subnetsClient, nil
 }
 
-// GetBackupLongTermRetentionPoliciesClient retrieves a Subnetclient
+// GetBackupLongTermRetentionPoliciesClient retrieves a BackupLongTermRetentionPoliciesClient
 func GetBackupLongTermRetentionPoliciesClient(creds config.Credentials) (sql3.BackupLongTermRetentionPoliciesClient, error) {
-	BackupClient := sql3.NewBackupLongTermRetentionPoliciesClientWithBaseURI(config.BaseURI(), creds.SubscriptionID())
+	backupClient := sql3.NewBackupLongTermRetentionPoliciesClientWithBaseURI(config.BaseURI(), creds.SubscriptionID())
 	a, err := iam.GetResourceManagementAuthorizer(creds)
 	if err != nil {
 		return sql3.BackupLongTermRetentionPoliciesClient{}, err
 	}
-	BackupClient.Authorizer = a
-	BackupClient.AddToUserAgent(config.UserAgent())
-	return BackupClient, nil
+	backupClient.Authorizer = a
+	backupClient.AddToUserAgent(config.UserAgent())
+	return backupClient, nil
+}
+
+// GetBackupShortTermRetentionPoliciesClient retrieves a BackupShortTermRetentionPoliciesClient
+func GetBackupShortTermRetentionPoliciesClient(creds config.Credentials) (sql3.BackupShortTermRetentionPoliciesClient, error) {
+	backupClient := sql3.NewBackupShortTermRetentionPoliciesClientWithBaseURI(config.BaseURI(), creds.SubscriptionID())
+	a, err := iam.GetResourceManagementAuthorizer(creds)
+	if err != nil {
+		return sql3.BackupShortTermRetentionPoliciesClient{}, err
+	}
+	backupClient.Authorizer = a
+	backupClient.AddToUserAgent(config.UserAgent())
+	return backupClient, nil
 }