From ebddb81bdd1dc7996610fc59ee0a7e8249c14434 Mon Sep 17 00:00:00 2001 From: Andrew Reslan Date: Thu, 14 Sep 2017 13:56:00 +0100 Subject: [PATCH 1/3] Added support for custom TTL on repaired docs and default value for RepairBucketParams.ViewQueryPageSize when not set --- db/repair_bucket.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/db/repair_bucket.go b/db/repair_bucket.go index 7256f2f703..1270ca6278 100644 --- a/db/repair_bucket.go +++ b/db/repair_bucket.go @@ -6,11 +6,14 @@ import ( "github.com/couchbase/go-couchbase" "github.com/couchbase/sync_gateway/base" + "time" ) // Enum for the different repair jobs (eg, repairing rev tree cycles) type RepairJobType string +const kDefaultRepairedFileTTL = 60 * 60 * 24 * time.Second // 24 hours + const ( RepairRevTreeCycles = RepairJobType("RepairRevTreeCycles") ) @@ -18,7 +21,7 @@ const ( // Params suitable for external (eg, HTTP) invocations to describe a RepairBucket operation type RepairBucketParams struct { DryRun bool `json:"dry_run"` - ViewQueryPageSize int `json:"view_query_page_size"` + ViewQueryPageSize *int `json:"view_query_page_size"` RepairJobs []RepairJobParams `json:"repair_jobs"` } @@ -43,6 +46,7 @@ type DocTransformer func(docId string, originalCBDoc []byte) (transformedCBDoc [ // A RepairBucket struct is the main API entrypoint to call for repairing documents in buckets type RepairBucket struct { DryRun bool // If true, will only output what changes it *would* have made, but not make any changes + RepairedFileTTL *int ViewQueryPageSize int Bucket base.Bucket RepairJobs []DocTransformer @@ -68,7 +72,10 @@ func (r *RepairBucket) AddRepairJob(repairJob DocTransformer) *RepairBucket { func (r *RepairBucket) InitFrom(params RepairBucketParams) *RepairBucket { r.SetDryRun(params.DryRun) - r.ViewQueryPageSize = params.ViewQueryPageSize + if params.ViewQueryPageSize != nil && *params.ViewQueryPageSize > 0 { + r.ViewQueryPageSize = *params.ViewQueryPageSize + } + for _, repairJobParams := range params.RepairJobs { switch repairJobParams.RepairJobType { case RepairRevTreeCycles: @@ -259,9 +266,19 @@ func (r RepairBucket) WriteRepairedDocsToBucket(docId string, originalDoc, updat return backupOrDryRunDocId, fmt.Errorf("Error unmarshalling updated/original doc. Err: %v", err) } - expirySeconds := 60 * 60 * 24 // 24 hours + expirySeconds := kDefaultRepairedFileTTL + + //If the RepairedFileTTL is explicitly set to 0 then don't write the doc at all + if r.RepairedFileTTL != nil { + if *r.RepairedFileTTL == 0 { + base.LogTo("CRUD", "Repair Doc: Doc %v repaired, TTL set to 0, doc will not be written to bucket", backupOrDryRunDocId) + return backupOrDryRunDocId, nil + } else { + expirySeconds = time.Duration(*r.RepairedFileTTL) * time.Second + } + } - if err := r.Bucket.Set(backupOrDryRunDocId, expirySeconds, doc); err != nil { + if err := r.Bucket.Set(backupOrDryRunDocId, base.DurationToCbsExpiry(time.Duration(expirySeconds)), doc); err != nil { return backupOrDryRunDocId, err } From 0a52482d93d9dcd0fee01ed217417f7ee97af31b Mon Sep 17 00:00:00 2001 From: Andrew Reslan Date: Thu, 14 Sep 2017 14:09:33 +0100 Subject: [PATCH 2/3] Added custom TTL for Repaired docs --- db/repair_bucket.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/db/repair_bucket.go b/db/repair_bucket.go index 1270ca6278..4c620cf97e 100644 --- a/db/repair_bucket.go +++ b/db/repair_bucket.go @@ -22,6 +22,7 @@ const ( type RepairBucketParams struct { DryRun bool `json:"dry_run"` ViewQueryPageSize *int `json:"view_query_page_size"` + RepairedFileTTL *int `json:"repaired_file_ttl_seconds"` RepairJobs []RepairJobParams `json:"repair_jobs"` } @@ -46,7 +47,7 @@ type DocTransformer func(docId string, originalCBDoc []byte) (transformedCBDoc [ // A RepairBucket struct is the main API entrypoint to call for repairing documents in buckets type RepairBucket struct { DryRun bool // If true, will only output what changes it *would* have made, but not make any changes - RepairedFileTTL *int + RepairedFileTTL time.Duration ViewQueryPageSize int Bucket base.Bucket RepairJobs []DocTransformer @@ -56,6 +57,7 @@ func NewRepairBucket(bucket base.Bucket) *RepairBucket { return &RepairBucket{ Bucket: bucket, ViewQueryPageSize: base.DefaultViewQueryPageSize, + RepairedFileTTL: kDefaultRepairedFileTTL, } } @@ -76,6 +78,10 @@ func (r *RepairBucket) InitFrom(params RepairBucketParams) *RepairBucket { r.ViewQueryPageSize = *params.ViewQueryPageSize } + if params.RepairedFileTTL != nil && *params.RepairedFileTTL >= 0 { + r.RepairedFileTTL = time.Duration(*params.ViewQueryPageSize) * time.Second + } + for _, repairJobParams := range params.RepairJobs { switch repairJobParams.RepairJobType { case RepairRevTreeCycles: @@ -266,19 +272,13 @@ func (r RepairBucket) WriteRepairedDocsToBucket(docId string, originalDoc, updat return backupOrDryRunDocId, fmt.Errorf("Error unmarshalling updated/original doc. Err: %v", err) } - expirySeconds := kDefaultRepairedFileTTL - //If the RepairedFileTTL is explicitly set to 0 then don't write the doc at all - if r.RepairedFileTTL != nil { - if *r.RepairedFileTTL == 0 { - base.LogTo("CRUD", "Repair Doc: Doc %v repaired, TTL set to 0, doc will not be written to bucket", backupOrDryRunDocId) - return backupOrDryRunDocId, nil - } else { - expirySeconds = time.Duration(*r.RepairedFileTTL) * time.Second - } + if int(r.RepairedFileTTL.Seconds()) == 0 { + base.LogTo("CRUD", "Repair Doc: Doc %v repaired, TTL set to 0, doc will not be written to bucket", backupOrDryRunDocId) + return backupOrDryRunDocId, nil } - if err := r.Bucket.Set(backupOrDryRunDocId, base.DurationToCbsExpiry(time.Duration(expirySeconds)), doc); err != nil { + if err := r.Bucket.Set(backupOrDryRunDocId, base.DurationToCbsExpiry(r.RepairedFileTTL), doc); err != nil { return backupOrDryRunDocId, err } From aeb9b147397b54ad6f3f63ea2913364b9e72ed7b Mon Sep 17 00:00:00 2001 From: Andrew Reslan Date: Thu, 14 Sep 2017 17:29:41 +0100 Subject: [PATCH 3/3] Renamed config parameter to "repair_output_ttl_seconds" --- db/repair_bucket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/repair_bucket.go b/db/repair_bucket.go index 4c620cf97e..7cbc50ffb5 100644 --- a/db/repair_bucket.go +++ b/db/repair_bucket.go @@ -22,7 +22,7 @@ const ( type RepairBucketParams struct { DryRun bool `json:"dry_run"` ViewQueryPageSize *int `json:"view_query_page_size"` - RepairedFileTTL *int `json:"repaired_file_ttl_seconds"` + RepairedFileTTL *int `json:"repair_output_ttl_seconds"` RepairJobs []RepairJobParams `json:"repair_jobs"` }