Add configurable expiry time for local docs, defaults to 90 days (#2850)

* Add configurable expiry time for local docs, defaults to 90 days

* Updates based on review feedback

* refactor to export LocalDocExpirySecs

* Moved DefaultLocalDocExpirySecs to base.constants.go

Formatted rest/server_context.go

base/constants.go

@@ -52,6 +52,8 @@ const (
 
 	DefaultUseXattrs = false // Whether Sync Gateway uses xattrs for metadata storage, if not specified in the config
 
+	// Default value of _local document expiry
+	DefaultLocalDocExpirySecs = uint32(60 * 60 * 24 * 90) //90 days in seconds
 )
 
 func UnitTestUrl() string {

db/database.go

@@ -94,7 +94,8 @@ type DatabaseContextOptions struct {
 	OIDCOptions           *auth.OIDCOptions
 	DBOnlineCallback      DBOnlineCallback // Callback function to take the DB back online
 	ImportOptions         ImportOptions
-	EnableXattr           bool // Use xattr for _sync
+	EnableXattr           bool   // Use xattr for _sync
+	LocalDocExpirySecs    uint32 //The _local doc expiry time in seconds
 }
 
 type OidcTestProviderOptions struct {

db/revision.go

@@ -70,8 +70,8 @@ func (body Body) ImmutableAttachmentsCopy() Body {
 // Returns the expiry as uint32 (using getExpiry), and removes the _exp property from the body
 func (body Body) extractExpiry() (uint32, error) {
 
-	exp, err := body.getExpiry()
-	if err != nil {
+	exp, present, err := body.getExpiry()
+	if !present || err != nil {
 		return exp, err
 	}
 	delete(body, "_exp")
@@ -84,33 +84,33 @@ func (body Body) extractExpiry() (uint32, error) {
 //   2. String JSON values that are numbers are converted to int32 and returned as-is
 //   3. String JSON values that are ISO-8601 dates are converted to UNIX time and returned
 //   4. Null JSON values return 0
-func (body Body) getExpiry() (uint32, error) {
+func (body Body) getExpiry() (uint32, bool, error) {
 	rawExpiry, ok := body["_exp"]
 	if !ok {
-		return 0, nil
+		return 0, false, nil //_exp not present
 	}
 	switch expiry := rawExpiry.(type) {
 	case float64:
-		return uint32(expiry), nil
+		return uint32(expiry), true, nil
 	case string:
 		// First check if it's a numeric string
 		expInt, err := strconv.ParseInt(expiry, 10, 32)
 		if err == nil {
-			return uint32(expInt), nil
+			return uint32(expInt), true, nil
 		}
 		// Check if it's an ISO-8601 date
 		expRFC3339, err := time.Parse(time.RFC3339, expiry)
 		if err == nil {
-			return uint32(expRFC3339.Unix()), nil
+			return uint32(expRFC3339.Unix()), true, nil
 		} else {
-			return 0, fmt.Errorf("Unable to parse expiry %s as either numeric or date expiry:%v", err)
+			return 0, true, fmt.Errorf("Unable to parse expiry %s as either numeric or date expiry:%v", err)
 		}
 	case nil:
 		// Leave as zero/empty expiry
-		return 0, nil
+		return 0, true, nil
 	}
 
-	return 0, nil
+	return 0, true, nil
 }
 
 // nonJSONPrefix is used to ensure old revision bodies aren't hidden from N1QL/Views.

db/shadower.go

@@ -104,10 +104,11 @@ func (s *Shadower) pullDocument(key string, value []byte, isDeletion bool, cas u
 	}
 
 	db, _ := CreateDatabase(s.context)
-	expiry, err := body.getExpiry()
+	expiry, _, err := body.getExpiry()
 	if err != nil {
 		return base.HTTPErrorf(http.StatusBadRequest, "Invalid expiry: %v", err)
 	}
+
 	_, err = db.updateDoc(key, false, expiry, func(doc *document) (Body, AttachmentData, error) {
 		// (Be careful: this block can be invoked multiple times if there are races!)
 		if doc.UpstreamCAS != nil && *doc.UpstreamCAS == cas {

db/special_docs.go

@@ -24,10 +24,21 @@ func (db *Database) GetSpecial(doctype string, docid string) (Body, error) {
 	}
 
 	body := Body{}
-	_, err := db.Bucket.Get(key, &body)
-	if err != nil {
-		return nil, err
+
+	if doctype == "local" && db.DatabaseContext.Options.LocalDocExpirySecs > 0 {
+		rawDocBytes, _, err := db.Bucket.GetAndTouchRaw(key, base.SecondsToCbsExpiry(int(db.DatabaseContext.Options.LocalDocExpirySecs)))
+		if err != nil {
+			return nil, err
+		}
+		if err := json.Unmarshal(rawDocBytes, &body); err != nil {
+			return nil, err
+		}
+	} else {
+		if _, err := db.Bucket.Get(key, &body); err != nil {
+			return nil, err
+		}
 	}
+
 	return body, nil
 }
 
@@ -39,10 +50,14 @@ func (db *Database) putSpecial(doctype string, docid string, matchRev string, bo
 	}
 	var revid string
 
-	expiry, err := body.getExpiry()
+	expiry, expPresent, err := body.getExpiry()
 	if err != nil {
 		return "", base.HTTPErrorf(http.StatusBadRequest, "Invalid expiry: %v", err)
 	}
+
+	if expPresent && expiry == 0 && doctype == "local" {
+		expiry = uint32(base.SecondsToCbsExpiry(int(db.DatabaseContext.Options.LocalDocExpirySecs)))
+	}
 	err = db.Bucket.Update(key, int(expiry), func(value []byte) ([]byte, error) {
 		if len(value) == 0 {
 			if matchRev != "" || body == nil {

rest/config.go

@@ -130,6 +130,7 @@ type DbConfig struct {
 	Unsupported          db.UnsupportedOptions          `json:"unsupported,omitempty"`                 // Config for unsupported features
 	OIDCConfig           *auth.OIDCOptions              `json:"oidc,omitempty"`                        // Config properties for OpenID Connect authentication
 	ViewQueryTimeoutSecs *uint32                        `json:"view_query_timeout_secs,omitempty"`     // The view query timeout in seconds
+	LocalDocExpirySecs   *uint32                        `json:"local_doc_expiry_secs,omitempty"`       // The _local doc expiry time in seconds
 	EnableXattrs         *bool                          `json:"enable_shared_bucket_access,omitempty"` // Whether to use extended attributes to store _sync metadata
 }
 

rest/server_context.go

@@ -284,6 +284,11 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(config *DbConfig, useExisti
 		viewQueryTimeoutSecs = config.ViewQueryTimeoutSecs
 	}
 
+	localDocExpirySecs := base.DefaultLocalDocExpirySecs
+	if config.LocalDocExpirySecs != nil && *config.LocalDocExpirySecs >= 0 {
+		localDocExpirySecs = *config.LocalDocExpirySecs
+	}
+
 	if sc.databases_[dbName] != nil {
 		if useExisting {
 			return sc.databases_[dbName], nil
@@ -505,6 +510,7 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(config *DbConfig, useExisti
 		IndexOptions:          channelIndexOptions,
 		SequenceHashOptions:   sequenceHashOptions,
 		RevisionCacheCapacity: revCacheSize,
+		LocalDocExpirySecs:    localDocExpirySecs,
 		AdminInterface:        sc.config.AdminInterface,
 		UnsupportedOptions:    config.Unsupported,
 		TrackDocs:             trackDocs,