diff --git a/br/pkg/redact/BUILD.bazel b/br/pkg/redact/BUILD.bazel index 9dfe9995d6ebd..bc103956b5084 100644 --- a/br/pkg/redact/BUILD.bazel +++ b/br/pkg/redact/BUILD.bazel @@ -5,7 +5,10 @@ go_library( srcs = ["redact.go"], importpath = "github.com/pingcap/tidb/br/pkg/redact", visibility = ["//visibility:public"], - deps = ["@com_github_pingcap_errors//:errors"], + deps = [ + "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_kvproto//pkg/brpb", + ], ) go_test( diff --git a/br/pkg/redact/redact.go b/br/pkg/redact/redact.go index 495bceb08c44d..d298aadb548c9 100644 --- a/br/pkg/redact/redact.go +++ b/br/pkg/redact/redact.go @@ -4,9 +4,20 @@ package redact import ( "encoding/hex" + "regexp" "strings" "github.com/pingcap/errors" + backup "github.com/pingcap/kvproto/pkg/brpb" +) + +var ( + reAccessKey = regexp.MustCompile(`access_key:\"[^\"]*\"`) + reSecretAccessKey = regexp.MustCompile(`secret_access_key:\"[^\"]*\"`) + reSharedKey = regexp.MustCompile(`shared_key:\"[^\"]*\"`) + reCredentialsBlob = regexp.MustCompile(`credentials_blob:\"[^\"]*\"`) + reAccessSig = regexp.MustCompile(`access_sig:\"[^\"]*\"`) + reEncryptKey = regexp.MustCompile(`encryption_key:<.*?>`) ) // InitRedact inits the enableRedactLog @@ -34,3 +45,25 @@ func Key(key []byte) string { } return strings.ToUpper(hex.EncodeToString(key)) } + +// TaskInfoRedacted is a wrapper of backup.StreamBackupTaskInfo to redact sensitive information +type TaskInfoRedacted struct { + Info *backup.StreamBackupTaskInfo +} + +func (TaskInfoRedacted) redact(input string) string { + // Replace the matched fields with redacted versions + output := reAccessKey.ReplaceAllString(input, `access_key:"[REDACTED]"`) + output = reSecretAccessKey.ReplaceAllString(output, `secret_access_key:"[REDACTED]"`) + output = reSharedKey.ReplaceAllString(output, `shared_key:"[REDACTED]"`) + output = reCredentialsBlob.ReplaceAllString(output, `CredentialsBlob:"[REDACTED]"`) + output = reAccessSig.ReplaceAllString(output, `access_sig:"[REDACTED]"`) + output = reEncryptKey.ReplaceAllString(output, `encryption_key:<[REDACTED]>`) + + return output +} + +// String returns the redacted string of the task info +func (t TaskInfoRedacted) String() string { + return t.redact(t.Info.String()) +} diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel index 15a63c1b9a46e..db040bfb1515b 100644 --- a/br/pkg/streamhelper/BUILD.bazel +++ b/br/pkg/streamhelper/BUILD.bazel @@ -68,7 +68,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 28, + shard_count = 29, deps = [ ":streamhelper", "//br/pkg/errors", diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go index 83485d0d1c9ba..7588c6abd51d3 100644 --- a/br/pkg/streamhelper/advancer.go +++ b/br/pkg/streamhelper/advancer.go @@ -17,6 +17,7 @@ import ( backuppb "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/log" "github.com/pingcap/tidb/br/pkg/logutil" + "github.com/pingcap/tidb/br/pkg/redact" "github.com/pingcap/tidb/br/pkg/streamhelper/config" "github.com/pingcap/tidb/br/pkg/streamhelper/spans" "github.com/pingcap/tidb/br/pkg/utils" @@ -438,7 +439,7 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error } log.Info("get global checkpoint", zap.Uint64("checkpoint", globalCheckpointTs)) c.lastCheckpoint = newCheckpointWithTS(globalCheckpointTs) - log.Info("added event", zap.Stringer("task", e.Info), + log.Info("added event", zap.Stringer("task", redact.TaskInfoRedacted{Info: e.Info}), zap.Stringer("ranges", logutil.StringifyKeys(c.taskRange))) case EventDel: utils.LogBackupTaskCountDec() diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go index a9f52927b3684..d1b4ed7aac431 100644 --- a/br/pkg/streamhelper/advancer_test.go +++ b/br/pkg/streamhelper/advancer_test.go @@ -15,6 +15,7 @@ import ( backup "github.com/pingcap/kvproto/pkg/brpb" logbackup "github.com/pingcap/kvproto/pkg/logbackuppb" "github.com/pingcap/log" + "github.com/pingcap/tidb/br/pkg/redact" "github.com/pingcap/tidb/br/pkg/streamhelper" "github.com/pingcap/tidb/br/pkg/streamhelper/config" "github.com/pingcap/tidb/br/pkg/streamhelper/spans" @@ -801,3 +802,35 @@ func TestSubscriptionPanic(t *testing.T) { cancel() wg.Wait() } + +func TestRedactBackend(t *testing.T) { + info := new(backup.StreamBackupTaskInfo) + info.Name = "test" + info.Storage = &backup.StorageBackend{ + Backend: &backup.StorageBackend_S3{ + S3: &backup.S3{ + Endpoint: "http://", + Bucket: "test", + Prefix: "test", + AccessKey: "12abCD!@#[]{}?/\\", + SecretAccessKey: "12abCD!@#[]{}?/\\", + }, + }, + } + + redacted := redact.TaskInfoRedacted{Info: info} + require.Equal(t, "storage: > name:\"test\" ", redacted.String()) + + info.Storage = &backup.StorageBackend{ + Backend: &backup.StorageBackend_Gcs{ + Gcs: &backup.GCS{ + Endpoint: "http://", + Bucket: "test", + Prefix: "test", + CredentialsBlob: "12abCD!@#[]{}?/\\", + }, + }, + } + redacted = redact.TaskInfoRedacted{Info: info} + require.Equal(t, "storage: > name:\"test\" ", redacted.String()) +}