From 67d07931f0d61e36c2b427fe2c88814448461f3a Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Fri, 3 Jul 2020 16:45:16 +0800
Subject: [PATCH 1/8] add compression type
---
go.mod | 2 ++
go.sum | 2 ++
pkg/backup/client.go | 25 ++++++++++++++-----------
pkg/task/backup.go | 28 ++++++++++++++++++++++++++++
pkg/task/backup_raw.go | 27 ++++++++++++++++++---------
5 files changed, 64 insertions(+), 20 deletions(-)
diff --git a/go.mod b/go.mod
index 2fb2f69be..acf439f58 100644
--- a/go.mod
+++ b/go.mod
@@ -35,3 +35,5 @@ require (
google.golang.org/api v0.15.1
google.golang.org/grpc v1.26.0
)
+
+replace github.com/pingcap/kvproto => github.com/glorv/kvproto v0.0.0-20200702084207-a2a6737837f6
diff --git a/go.sum b/go.sum
index abe4575b9..ee77dd6b8 100644
--- a/go.sum
+++ b/go.sum
@@ -167,6 +167,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y=
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
+github.com/glorv/kvproto v0.0.0-20200702084207-a2a6737837f6 h1:WA/xq+rYQCi8TpdUnzAA7QSUCDmr4ndimybY7A5ate8=
+github.com/glorv/kvproto v0.0.0-20200702084207-a2a6737837f6/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I=
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
diff --git a/pkg/backup/client.go b/pkg/backup/client.go
index c0651d7a7..4c7d1c070 100644
--- a/pkg/backup/client.go
+++ b/pkg/backup/client.go
@@ -481,8 +481,8 @@ func (bc *Client) BackupRange(
// Find and backup remaining ranges.
// TODO: test fine grained backup.
err = bc.fineGrainedBackup(
- ctx, startKey, endKey, req.StartVersion,
- req.EndVersion, req.RateLimit, req.Concurrency, results, updateCh)
+ ctx, startKey, endKey, req.StartVersion, req.EndVersion, req.CompressionType,
+ req.RateLimit, req.Concurrency, results, updateCh)
if err != nil {
return err
}
@@ -546,6 +546,7 @@ func (bc *Client) fineGrainedBackup(
startKey, endKey []byte,
lastBackupTS uint64,
backupTS uint64,
+ compressType kvproto.BackupRequest_CompressionType,
rateLimit uint64,
concurrency uint32,
rangeTree rtree.RangeTree,
@@ -576,7 +577,7 @@ func (bc *Client) fineGrainedBackup(
defer wg.Done()
for rg := range retry {
backoffMs, err :=
- bc.handleFineGrained(ctx, boFork, rg, lastBackupTS, backupTS, rateLimit, concurrency, respCh)
+ bc.handleFineGrained(ctx, boFork, rg, lastBackupTS, backupTS, compressType, rateLimit, concurrency, respCh)
if err != nil {
errCh <- err
return
@@ -710,6 +711,7 @@ func (bc *Client) handleFineGrained(
rg rtree.Range,
lastBackupTS uint64,
backupTS uint64,
+ compressType kvproto.BackupRequest_CompressionType,
rateLimit uint64,
concurrency uint32,
respCh chan<- *kvproto.BackupResponse,
@@ -722,14 +724,15 @@ func (bc *Client) handleFineGrained(
max := 0
req := kvproto.BackupRequest{
- ClusterId: bc.clusterID,
- StartKey: rg.StartKey, // TODO: the range may cross region.
- EndKey: rg.EndKey,
- StartVersion: lastBackupTS,
- EndVersion: backupTS,
- StorageBackend: bc.backend,
- RateLimit: rateLimit,
- Concurrency: concurrency,
+ ClusterId: bc.clusterID,
+ StartKey: rg.StartKey, // TODO: the range may cross region.
+ EndKey: rg.EndKey,
+ StartVersion: lastBackupTS,
+ EndVersion: backupTS,
+ StorageBackend: bc.backend,
+ RateLimit: rateLimit,
+ Concurrency: concurrency,
+ CompressionType: compressType,
}
lockResolver := bc.mgr.GetLockResolver()
client, err := bc.mgr.GetBackupClient(ctx, storeID)
diff --git a/pkg/task/backup.go b/pkg/task/backup.go
index 8e17296e5..ce4b007ac 100644
--- a/pkg/task/backup.go
+++ b/pkg/task/backup.go
@@ -32,6 +32,7 @@ const (
flagBackupTimeago = "timeago"
flagBackupTS = "backupts"
flagLastBackupTS = "lastbackupts"
+ flagCompressionType = "compression-type"
flagGCTTL = "gcttl"
@@ -46,6 +47,7 @@ type BackupConfig struct {
BackupTS uint64 `json:"backup-ts" toml:"backup-ts"`
LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"`
GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"`
+ CompressionType kvproto.BackupRequest_CompressionType `json:"compression-type" toml:"compression-type"`
}
// DefineBackupFlags defines common flags for the backup command.
@@ -60,6 +62,7 @@ func DefineBackupFlags(flags *pflag.FlagSet) {
flags.String(flagBackupTS, "", "the backup ts support TSO or datetime,"+
" e.g. '400036290571534337', '2018-05-11 01:42:23'")
flags.Int64(flagGCTTL, backup.DefaultBRGCSafePointTTL, "the TTL (in seconds) that PD holds for BR's GC safepoint")
+ flags.String(flagCompressionType, "", "backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'")
}
// ParseFromFlags parses the backup-related flags from the flag set.
@@ -90,12 +93,19 @@ func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet) error {
}
cfg.GCTTL = gcTTL
+ compressionType, err := parseCompressionType(flagCompressionType)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ cfg.CompressionType = compressionType
+
if err = cfg.Config.ParseFromFlags(flags); err != nil {
return errors.Trace(err)
}
if cfg.Config.Concurrency == 0 {
cfg.Config.Concurrency = defaultBackupConcurrency
}
+
return nil
}
@@ -184,6 +194,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
EndVersion: backupTS,
RateLimit: cfg.RateLimit,
Concurrency: cfg.Concurrency,
+ CompressionType: cfg.CompressionType,
}
err = client.BackupRanges(
ctx, ranges, req, updateCh)
@@ -279,3 +290,20 @@ func parseTSString(ts string) (uint64, error) {
}
return variable.GoTimeToTS(t1), nil
}
+
+func parseCompressionType(s string) (kvproto.BackupRequest_CompressionType, error) {
+ var ct kvproto.BackupRequest_CompressionType
+ switch s {
+ case "lz4":
+ ct = kvproto.BackupRequest_LZ4
+ case "snappy":
+ ct = kvproto.BackupRequest_SNAPPY
+ case "zstd":
+ ct = kvproto.BackupRequest_ZSTD
+ case "":
+ ct = kvproto.BackupRequest_UNKNOWN
+ default:
+ return kvproto.BackupRequest_UNKNOWN, errors.Errorf("invalid compression type '%s'", s)
+ }
+ return ct, nil
+}
diff --git a/pkg/task/backup_raw.go b/pkg/task/backup_raw.go
index aacd61627..ce9d42a54 100644
--- a/pkg/task/backup_raw.go
+++ b/pkg/task/backup_raw.go
@@ -31,9 +31,10 @@ const (
type RawKvConfig struct {
Config
- StartKey []byte `json:"start-key" toml:"start-key"`
- EndKey []byte `json:"end-key" toml:"end-key"`
- CF string `json:"cf" toml:"cf"`
+ StartKey []byte `json:"start-key" toml:"start-key"`
+ EndKey []byte `json:"end-key" toml:"end-key"`
+ CF string `json:"cf" toml:"cf"`
+ CompressionType kvproto.BackupRequest_CompressionType `json:"compression-type" toml:"compression-type"`
}
// DefineRawBackupFlags defines common flags for the backup command.
@@ -42,6 +43,7 @@ func DefineRawBackupFlags(command *cobra.Command) {
command.Flags().StringP(flagTiKVColumnFamily, "", "default", "backup specify cf, correspond to tikv cf")
command.Flags().StringP(flagStartKey, "", "", "backup raw kv start key, key is inclusive")
command.Flags().StringP(flagEndKey, "", "", "backup raw kv end key, key is exclusive")
+ command.Flags().String(flagCompressionType, "", "backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'")
}
// ParseFromFlags parses the backup-related flags from the flag set.
@@ -78,6 +80,12 @@ func (cfg *RawKvConfig) ParseFromFlags(flags *pflag.FlagSet) error {
if err = cfg.Config.ParseFromFlags(flags); err != nil {
return errors.Trace(err)
}
+
+ compressionType, err := parseCompressionType(flagCompressionType)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ cfg.CompressionType = compressionType
return nil
}
@@ -121,12 +129,13 @@ func RunBackupRaw(c context.Context, g glue.Glue, cmdName string, cfg *RawKvConf
ctx, cmdName, int64(approximateRegions), !cfg.LogProgress)
req := kvproto.BackupRequest{
- StartVersion: 0,
- EndVersion: 0,
- RateLimit: cfg.RateLimit,
- Concurrency: cfg.Concurrency,
- IsRawKv: true,
- Cf: cfg.CF,
+ StartVersion: 0,
+ EndVersion: 0,
+ RateLimit: cfg.RateLimit,
+ Concurrency: cfg.Concurrency,
+ IsRawKv: true,
+ Cf: cfg.CF,
+ CompressionType: cfg.CompressionType,
}
err = client.BackupRange(ctx, backupRange.StartKey, backupRange.EndKey, req, updateCh)
From 3591bb1d41f7974bff6e8b2e2fdfc8671216330f Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Fri, 3 Jul 2020 17:41:44 +0800
Subject: [PATCH 2/8] fix
---
pkg/task/backup.go | 28 ++++++++++++++++------------
pkg/task/backup_raw.go | 6 +++++-
2 files changed, 21 insertions(+), 13 deletions(-)
diff --git a/pkg/task/backup.go b/pkg/task/backup.go
index ce4b007ac..22b1330d3 100644
--- a/pkg/task/backup.go
+++ b/pkg/task/backup.go
@@ -29,9 +29,9 @@ import (
)
const (
- flagBackupTimeago = "timeago"
- flagBackupTS = "backupts"
- flagLastBackupTS = "lastbackupts"
+ flagBackupTimeago = "timeago"
+ flagBackupTS = "backupts"
+ flagLastBackupTS = "lastbackupts"
flagCompressionType = "compression-type"
flagGCTTL = "gcttl"
@@ -43,10 +43,10 @@ const (
type BackupConfig struct {
Config
- TimeAgo time.Duration `json:"time-ago" toml:"time-ago"`
- BackupTS uint64 `json:"backup-ts" toml:"backup-ts"`
- LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"`
- GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"`
+ TimeAgo time.Duration `json:"time-ago" toml:"time-ago"`
+ BackupTS uint64 `json:"backup-ts" toml:"backup-ts"`
+ LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"`
+ GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"`
CompressionType kvproto.BackupRequest_CompressionType `json:"compression-type" toml:"compression-type"`
}
@@ -93,7 +93,11 @@ func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet) error {
}
cfg.GCTTL = gcTTL
- compressionType, err := parseCompressionType(flagCompressionType)
+ compressionStr, err := flags.GetString(flagCompressionType)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ compressionType, err := parseCompressionType(compressionStr)
if err != nil {
return errors.Trace(err)
}
@@ -190,10 +194,10 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
ctx, cmdName, int64(approximateRegions), !cfg.LogProgress)
req := kvproto.BackupRequest{
- StartVersion: cfg.LastBackupTS,
- EndVersion: backupTS,
- RateLimit: cfg.RateLimit,
- Concurrency: cfg.Concurrency,
+ StartVersion: cfg.LastBackupTS,
+ EndVersion: backupTS,
+ RateLimit: cfg.RateLimit,
+ Concurrency: cfg.Concurrency,
CompressionType: cfg.CompressionType,
}
err = client.BackupRanges(
diff --git a/pkg/task/backup_raw.go b/pkg/task/backup_raw.go
index ce9d42a54..40dd72053 100644
--- a/pkg/task/backup_raw.go
+++ b/pkg/task/backup_raw.go
@@ -81,7 +81,11 @@ func (cfg *RawKvConfig) ParseFromFlags(flags *pflag.FlagSet) error {
return errors.Trace(err)
}
- compressionType, err := parseCompressionType(flagCompressionType)
+ compressionStr, err := flags.GetString(flagCompressionType)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ compressionType, err := parseCompressionType(compressionStr)
if err != nil {
return errors.Trace(err)
}
From 5cdd6d041cef127f20a6d62c0ef94ceeeeceb42e Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Tue, 7 Jul 2020 19:38:05 +0800
Subject: [PATCH 3/8] apply compresssion
---
go.mod | 4 +---
go.sum | 4 ++--
pkg/backup/client.go | 4 ++--
pkg/task/backup.go | 24 ++++++++++++------------
pkg/task/backup_raw.go | 8 ++++----
5 files changed, 21 insertions(+), 23 deletions(-)
diff --git a/go.mod b/go.mod
index acf439f58..5c9244284 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,7 @@ require (
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
github.com/pingcap/failpoint v0.0.0-20200506114213-c17f16071c53
- github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29
+ github.com/pingcap/kvproto v0.0.0-20200706115936-1e0910aabe6c
github.com/pingcap/log v0.0.0-20200511115504-543df19646ad
github.com/pingcap/parser v0.0.0-20200518090819-ec1e13b948b1
github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200520083007-2c251bd8f181
@@ -35,5 +35,3 @@ require (
google.golang.org/api v0.15.1
google.golang.org/grpc v1.26.0
)
-
-replace github.com/pingcap/kvproto => github.com/glorv/kvproto v0.0.0-20200702084207-a2a6737837f6
diff --git a/go.sum b/go.sum
index ee77dd6b8..ef9b219d5 100644
--- a/go.sum
+++ b/go.sum
@@ -167,8 +167,6 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y=
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
-github.com/glorv/kvproto v0.0.0-20200702084207-a2a6737837f6 h1:WA/xq+rYQCi8TpdUnzAA7QSUCDmr4ndimybY7A5ate8=
-github.com/glorv/kvproto v0.0.0-20200702084207-a2a6737837f6/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I=
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -482,6 +480,8 @@ github.com/pingcap/kvproto v0.0.0-20200424032552-6650270c39c3/go.mod h1:IOdRDPLy
github.com/pingcap/kvproto v0.0.0-20200428135407-0f5ffe459677/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29 h1:NpW1OuYrIl+IQrSsVbtyHpHpazmSCHy+ysrOixY0xY4=
github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
+github.com/pingcap/kvproto v0.0.0-20200706115936-1e0910aabe6c h1:VnLpCAxMAeDxc7HXTetwDQB+/MtDQjHAOBsd4QnGVwA=
+github.com/pingcap/kvproto v0.0.0-20200706115936-1e0910aabe6c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=
diff --git a/pkg/backup/client.go b/pkg/backup/client.go
index 4c7d1c070..482699ffc 100644
--- a/pkg/backup/client.go
+++ b/pkg/backup/client.go
@@ -546,7 +546,7 @@ func (bc *Client) fineGrainedBackup(
startKey, endKey []byte,
lastBackupTS uint64,
backupTS uint64,
- compressType kvproto.BackupRequest_CompressionType,
+ compressType kvproto.CompressionType,
rateLimit uint64,
concurrency uint32,
rangeTree rtree.RangeTree,
@@ -711,7 +711,7 @@ func (bc *Client) handleFineGrained(
rg rtree.Range,
lastBackupTS uint64,
backupTS uint64,
- compressType kvproto.BackupRequest_CompressionType,
+ compressType kvproto.CompressionType,
rateLimit uint64,
concurrency uint32,
respCh chan<- *kvproto.BackupResponse,
diff --git a/pkg/task/backup.go b/pkg/task/backup.go
index 22b1330d3..d42133480 100644
--- a/pkg/task/backup.go
+++ b/pkg/task/backup.go
@@ -43,11 +43,11 @@ const (
type BackupConfig struct {
Config
- TimeAgo time.Duration `json:"time-ago" toml:"time-ago"`
- BackupTS uint64 `json:"backup-ts" toml:"backup-ts"`
- LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"`
- GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"`
- CompressionType kvproto.BackupRequest_CompressionType `json:"compression-type" toml:"compression-type"`
+ TimeAgo time.Duration `json:"time-ago" toml:"time-ago"`
+ BackupTS uint64 `json:"backup-ts" toml:"backup-ts"`
+ LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"`
+ GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"`
+ CompressionType kvproto.CompressionType `json:"compression-type" toml:"compression-type"`
}
// DefineBackupFlags defines common flags for the backup command.
@@ -295,19 +295,19 @@ func parseTSString(ts string) (uint64, error) {
return variable.GoTimeToTS(t1), nil
}
-func parseCompressionType(s string) (kvproto.BackupRequest_CompressionType, error) {
- var ct kvproto.BackupRequest_CompressionType
+func parseCompressionType(s string) (kvproto.CompressionType, error) {
+ var ct kvproto.CompressionType
switch s {
case "lz4":
- ct = kvproto.BackupRequest_LZ4
+ ct = kvproto.CompressionType_LZ4
case "snappy":
- ct = kvproto.BackupRequest_SNAPPY
+ ct = kvproto.CompressionType_SNAPPY
case "zstd":
- ct = kvproto.BackupRequest_ZSTD
+ ct = kvproto.CompressionType_ZSTD
case "":
- ct = kvproto.BackupRequest_UNKNOWN
+ ct = kvproto.CompressionType_UNKNOWN
default:
- return kvproto.BackupRequest_UNKNOWN, errors.Errorf("invalid compression type '%s'", s)
+ return kvproto.CompressionType_UNKNOWN, errors.Errorf("invalid compression type '%s'", s)
}
return ct, nil
}
diff --git a/pkg/task/backup_raw.go b/pkg/task/backup_raw.go
index 40dd72053..6b583cdea 100644
--- a/pkg/task/backup_raw.go
+++ b/pkg/task/backup_raw.go
@@ -31,10 +31,10 @@ const (
type RawKvConfig struct {
Config
- StartKey []byte `json:"start-key" toml:"start-key"`
- EndKey []byte `json:"end-key" toml:"end-key"`
- CF string `json:"cf" toml:"cf"`
- CompressionType kvproto.BackupRequest_CompressionType `json:"compression-type" toml:"compression-type"`
+ StartKey []byte `json:"start-key" toml:"start-key"`
+ EndKey []byte `json:"end-key" toml:"end-key"`
+ CF string `json:"cf" toml:"cf"`
+ CompressionType kvproto.CompressionType `json:"compression-type" toml:"compression-type"`
}
// DefineRawBackupFlags defines common flags for the backup command.
From 28003a70f88bf35cb3669d91546741626d8ecf49 Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Tue, 7 Jul 2020 20:25:19 +0800
Subject: [PATCH 4/8] fix lint
---
pkg/task/backup_raw.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pkg/task/backup_raw.go b/pkg/task/backup_raw.go
index 4a2a8be4e..0bf10f37a 100644
--- a/pkg/task/backup_raw.go
+++ b/pkg/task/backup_raw.go
@@ -43,7 +43,8 @@ func DefineRawBackupFlags(command *cobra.Command) {
command.Flags().StringP(flagTiKVColumnFamily, "", "default", "backup specify cf, correspond to tikv cf")
command.Flags().StringP(flagStartKey, "", "", "backup raw kv start key, key is inclusive")
command.Flags().StringP(flagEndKey, "", "", "backup raw kv end key, key is exclusive")
- command.Flags().String(flagCompressionType, "", "backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'")
+ command.Flags().String(flagCompressionType, "",
+ "backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'")
}
// ParseFromFlags parses the backup-related flags from the flag set.
From c02c5e8c9e7cddede2dad129c56f3039f835b8b3 Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Wed, 8 Jul 2020 11:16:13 +0800
Subject: [PATCH 5/8] fix restore flag
---
cmd/backup.go | 2 +-
pkg/task/backup_raw.go | 11 ++++++++++-
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/cmd/backup.go b/cmd/backup.go
index 971d4ef08..948249671 100644
--- a/cmd/backup.go
+++ b/cmd/backup.go
@@ -30,7 +30,7 @@ func runBackupCommand(command *cobra.Command, cmdName string) error {
func runBackupRawCommand(command *cobra.Command, cmdName string) error {
cfg := task.RawKvConfig{Config: task.Config{LogProgress: HasLogFile()}}
- if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+ if err := cfg.ParseBackupConfigFromFlags(command.Flags()); err != nil {
command.SilenceUsage = false
return err
}
diff --git a/pkg/task/backup_raw.go b/pkg/task/backup_raw.go
index 0bf10f37a..f681a0f62 100644
--- a/pkg/task/backup_raw.go
+++ b/pkg/task/backup_raw.go
@@ -47,7 +47,7 @@ func DefineRawBackupFlags(command *cobra.Command) {
"backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'")
}
-// ParseFromFlags parses the backup-related flags from the flag set.
+// ParseFromFlags parses the raw kv backup&restore common flags from the flag set.
func (cfg *RawKvConfig) ParseFromFlags(flags *pflag.FlagSet) error {
format, err := flags.GetString(flagKeyFormat)
if err != nil {
@@ -81,6 +81,15 @@ func (cfg *RawKvConfig) ParseFromFlags(flags *pflag.FlagSet) error {
if err = cfg.Config.ParseFromFlags(flags); err != nil {
return errors.Trace(err)
}
+ return nil
+}
+
+// ParseBackupConfigFromFlags parses the backup-related flags from the flag set.
+func (cfg *RawKvConfig) ParseBackupConfigFromFlags(flags *pflag.FlagSet) error {
+ err := cfg.ParseFromFlags(flags)
+ if err != nil {
+ return err
+ }
compressionStr, err := flags.GetString(flagCompressionType)
if err != nil {
From 02ef27010ca1f43b4bf61f302fe00a3279f41566 Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Wed, 8 Jul 2020 14:58:49 +0800
Subject: [PATCH 6/8] provide default value for flag compression
---
pkg/task/backup.go | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/pkg/task/backup.go b/pkg/task/backup.go
index 541d67026..4f8b1bf3a 100644
--- a/pkg/task/backup.go
+++ b/pkg/task/backup.go
@@ -31,7 +31,7 @@ const (
flagBackupTimeago = "timeago"
flagBackupTS = "backupts"
flagLastBackupTS = "lastbackupts"
- flagCompressionType = "compression-type"
+ flagCompressionType = "compression"
flagGCTTL = "gcttl"
@@ -62,7 +62,8 @@ func DefineBackupFlags(flags *pflag.FlagSet) {
flags.String(flagBackupTS, "", "the backup ts support TSO or datetime,"+
" e.g. '400036290571534337', '2018-05-11 01:42:23'")
flags.Int64(flagGCTTL, backup.DefaultBRGCSafePointTTL, "the TTL (in seconds) that PD holds for BR's GC safepoint")
- flags.String(flagCompressionType, "", "backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'")
+ flags.String(flagCompressionType, "lz4",
+ "backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'")
}
// ParseFromFlags parses the backup-related flags from the flag set.
@@ -319,8 +320,6 @@ func parseCompressionType(s string) (kvproto.CompressionType, error) {
ct = kvproto.CompressionType_SNAPPY
case "zstd":
ct = kvproto.CompressionType_ZSTD
- case "":
- ct = kvproto.CompressionType_UNKNOWN
default:
return kvproto.CompressionType_UNKNOWN, errors.Errorf("invalid compression type '%s'", s)
}
From 60a5befe1e0e321674c294a70f96bd2ce2ca18b2 Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Wed, 8 Jul 2020 19:41:36 +0800
Subject: [PATCH 7/8] change default backup compression type from 'lz4' to
'zstd' in favor of better performance
---
pkg/task/backup.go | 2 +-
pkg/task/backup_raw.go | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pkg/task/backup.go b/pkg/task/backup.go
index 9278b29f0..b79f33d35 100644
--- a/pkg/task/backup.go
+++ b/pkg/task/backup.go
@@ -62,7 +62,7 @@ func DefineBackupFlags(flags *pflag.FlagSet) {
flags.String(flagBackupTS, "", "the backup ts support TSO or datetime,"+
" e.g. '400036290571534337', '2018-05-11 01:42:23'")
flags.Int64(flagGCTTL, backup.DefaultBRGCSafePointTTL, "the TTL (in seconds) that PD holds for BR's GC safepoint")
- flags.String(flagCompressionType, "lz4",
+ flags.String(flagCompressionType, "zstd",
"backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'")
}
diff --git a/pkg/task/backup_raw.go b/pkg/task/backup_raw.go
index f681a0f62..4d237848b 100644
--- a/pkg/task/backup_raw.go
+++ b/pkg/task/backup_raw.go
@@ -43,7 +43,7 @@ func DefineRawBackupFlags(command *cobra.Command) {
command.Flags().StringP(flagTiKVColumnFamily, "", "default", "backup specify cf, correspond to tikv cf")
command.Flags().StringP(flagStartKey, "", "", "backup raw kv start key, key is inclusive")
command.Flags().StringP(flagEndKey, "", "", "backup raw kv end key, key is exclusive")
- command.Flags().String(flagCompressionType, "",
+ command.Flags().String(flagCompressionType, "zstd",
"backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'")
}
From 550b8683ed6816eceeb6cd76a494db70e959ca17 Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Wed, 8 Jul 2020 20:39:27 +0800
Subject: [PATCH 8/8] fix a log bug
---
pkg/backup/client.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/backup/client.go b/pkg/backup/client.go
index 9643449fa..a86801434 100644
--- a/pkg/backup/client.go
+++ b/pkg/backup/client.go
@@ -815,7 +815,7 @@ func SendBackup(
// TODO: handle errors in the resp.
log.Info("range backuped",
zap.Stringer("StartKey", utils.WrapKey(resp.GetStartKey())),
- zap.Stringer("EndKey", utils.WrapKey(req.GetEndKey())))
+ zap.Stringer("EndKey", utils.WrapKey(resp.GetEndKey())))
err = respFn(resp)
if err != nil {
return err