From 222b9d858a326b76fefbdea4ea5be6ee41aaa2be Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Mon, 20 Jan 2025 14:12:06 +0530 Subject: [PATCH] Fix: Do not redact password for conn pool connection uri while connecting to DB (#2203) Fixing the bug where import data always fails to connect to DB via the connection pool with password authentication as the connection URLs are redacted while passing to the connection pool. Deep copying the connection params for redacting while logging using the package - https://github.com/tiendc/go-deepcopy. --- .github/workflows/misc-migtests.yml | 2 +- .github/workflows/mysql-migtests.yml | 2 +- .github/workflows/pg-13-migtests.yml | 2 +- .github/workflows/pg-17-migtests.yml | 2 +- .github/workflows/pg-9-migtests.yml | 2 +- yb-voyager/go.mod | 3 ++- yb-voyager/go.sum | 2 ++ yb-voyager/src/tgtdb/yugabytedb.go | 9 ++++++++- 8 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index 336202848..053de031e 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -100,7 +100,7 @@ jobs: docker run -d --name yugabytedb-$version \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:$version \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 echo "Test YugabyteDB connection" diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml index 1dfda76a6..224286187 100644 --- a/.github/workflows/mysql-migtests.yml +++ b/.github/workflows/mysql-migtests.yml @@ -63,7 +63,7 @@ jobs: docker run -d --name yugabytedb \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:${{ matrix.version }} \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 - name: Test YugabyteDB connection diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml index d6b03ec16..bcb78f433 100644 --- a/.github/workflows/pg-13-migtests.yml +++ b/.github/workflows/pg-13-migtests.yml @@ -92,7 +92,7 @@ jobs: docker run -d --name yugabytedb \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:${{ matrix.version }} \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 - name: Test YugabyteDB connection diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml index 27aacc0c6..771d20f8d 100644 --- a/.github/workflows/pg-17-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -92,7 +92,7 @@ jobs: docker run -d --name yugabytedb \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:${{ matrix.version }} \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 - name: Test YugabyteDB connection diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml index 8fa3251d1..7fb10ef6e 100644 --- a/.github/workflows/pg-9-migtests.yml +++ b/.github/workflows/pg-9-migtests.yml @@ -85,7 +85,7 @@ jobs: docker run -d --name yugabytedb \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:${{ matrix.version }} \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 - name: Test YugabyteDB connection diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index 7df52aa40..228330bfa 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -37,6 +37,7 @@ require ( github.com/tebeka/atexit v0.3.0 github.com/testcontainers/testcontainers-go v0.34.0 github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.34.0 + github.com/tiendc/go-deepcopy v1.2.0 github.com/vbauerster/mpb/v8 v8.4.0 gocloud.dev v0.29.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa @@ -168,7 +169,7 @@ require ( golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.16.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index 1c06c76f9..61e548512 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -1932,6 +1932,8 @@ github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNu github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.34.0 h1:9wIqSZJwBr4s8Q7R3S+rhe1J2zqHHxH0S1bN17ld+CI= github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.34.0/go.mod h1:bgHrbdYjpNPSstf8HfxChUxc6XztBCSoqDR0syb1Oeg= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tiendc/go-deepcopy v1.2.0 h1:6vCCs+qdLQHzFqY1fcPirsAWOmrLbuccilfp8UzD1Qo= +github.com/tiendc/go-deepcopy v1.2.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index fc3a392ae..8989190ee 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -37,6 +37,7 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" "github.com/samber/lo" log "github.com/sirupsen/logrus" + "github.com/tiendc/go-deepcopy" "golang.org/x/exp/slices" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" @@ -243,7 +244,13 @@ func (yb *TargetYugabyteDB) InitConnPool() error { SessionInitScript: getYBSessionInitScript(yb.tconf), } yb.connPool = NewConnectionPool(params) - redactedParams := params + redactedParams := &ConnectionParams{} + //Whenever adding new fields to CONNECTION PARAMS check if that needs to be redacted while logging + err := deepcopy.Copy(redactedParams, params) + if err!= nil { + log.Errorf("couldn't get the copy of connection params for logging: %v", err) + return nil + } redactedParams.ConnUriList = utils.GetRedactedURLs(redactedParams.ConnUriList) log.Info("Initialized connection pool with settings: ", spew.Sdump(redactedParams)) return nil