From a83acb2474310d469325cde29b2c54295d805709 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Fri, 17 Jan 2025 16:19:39 +0530 Subject: [PATCH 1/7] add md5 conf for passwords for yb cluster --- .github/workflows/misc-migtests.yml | 2 +- .github/workflows/mysql-migtests.yml | 2 +- .github/workflows/pg-13-migtests.yml | 2 +- .github/workflows/pg-17-migtests.yml | 2 +- .github/workflows/pg-9-migtests.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index 3362028481..053de031e4 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -100,7 +100,7 @@ jobs: docker run -d --name yugabytedb-$version \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:$version \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 echo "Test YugabyteDB connection" diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml index 1dfda76a62..2242861873 100644 --- a/.github/workflows/mysql-migtests.yml +++ b/.github/workflows/mysql-migtests.yml @@ -63,7 +63,7 @@ jobs: docker run -d --name yugabytedb \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:${{ matrix.version }} \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 - name: Test YugabyteDB connection diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml index d6b03ec165..bcb78f4338 100644 --- a/.github/workflows/pg-13-migtests.yml +++ b/.github/workflows/pg-13-migtests.yml @@ -92,7 +92,7 @@ jobs: docker run -d --name yugabytedb \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:${{ matrix.version }} \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 - name: Test YugabyteDB connection diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml index 27aacc0c68..771d20f8d1 100644 --- a/.github/workflows/pg-17-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -92,7 +92,7 @@ jobs: docker run -d --name yugabytedb \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:${{ matrix.version }} \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 - name: Test YugabyteDB connection diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml index 8fa3251d1b..7fb10ef6e2 100644 --- a/.github/workflows/pg-9-migtests.yml +++ b/.github/workflows/pg-9-migtests.yml @@ -85,7 +85,7 @@ jobs: docker run -d --name yugabytedb \ -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ yugabytedb/yugabyte:${{ matrix.version }} \ - bin/yugabyted start --background=false --ui=false + bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false sleep 20 - name: Test YugabyteDB connection From 00ada303ac477549fa16ee995e63a1422ebd3048 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Fri, 17 Jan 2025 16:30:21 +0530 Subject: [PATCH 2/7] fix for not redacting connection uris for conn pool --- yb-voyager/src/tgtdb/yugabytedb.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index fc3a392ae9..0091cb6881 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -243,8 +243,12 @@ func (yb *TargetYugabyteDB) InitConnPool() error { SessionInitScript: getYBSessionInitScript(yb.tconf), } yb.connPool = NewConnectionPool(params) - redactedParams := params - redactedParams.ConnUriList = utils.GetRedactedURLs(redactedParams.ConnUriList) + redactedParams := ConnectionParams{ + NumConnections: yb.tconf.Parallelism, + NumMaxConnections: yb.tconf.MaxParallelism, + ConnUriList: utils.GetRedactedURLs(targetUriList), + SessionInitScript: getYBSessionInitScript(yb.tconf), + } log.Info("Initialized connection pool with settings: ", spew.Sdump(redactedParams)) return nil } From b54c79a77ab914849a1df628614267a4c7b46ea0 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Fri, 17 Jan 2025 17:27:10 +0530 Subject: [PATCH 3/7] review comment --- yb-voyager/src/tgtdb/yugabytedb.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index 0091cb6881..4259ddc3c2 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -243,12 +243,7 @@ func (yb *TargetYugabyteDB) InitConnPool() error { SessionInitScript: getYBSessionInitScript(yb.tconf), } yb.connPool = NewConnectionPool(params) - redactedParams := ConnectionParams{ - NumConnections: yb.tconf.Parallelism, - NumMaxConnections: yb.tconf.MaxParallelism, - ConnUriList: utils.GetRedactedURLs(targetUriList), - SessionInitScript: getYBSessionInitScript(yb.tconf), - } + redactedParams := *params log.Info("Initialized connection pool with settings: ", spew.Sdump(redactedParams)) return nil } From 4b36842dad65332f01a48e9360c284162deae4fe Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Sat, 18 Jan 2025 12:44:23 +0530 Subject: [PATCH 4/7] Revert "review comment" This reverts commit b54c79a77ab914849a1df628614267a4c7b46ea0. --- yb-voyager/src/tgtdb/yugabytedb.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index 4259ddc3c2..0091cb6881 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -243,7 +243,12 @@ func (yb *TargetYugabyteDB) InitConnPool() error { SessionInitScript: getYBSessionInitScript(yb.tconf), } yb.connPool = NewConnectionPool(params) - redactedParams := *params + redactedParams := ConnectionParams{ + NumConnections: yb.tconf.Parallelism, + NumMaxConnections: yb.tconf.MaxParallelism, + ConnUriList: utils.GetRedactedURLs(targetUriList), + SessionInitScript: getYBSessionInitScript(yb.tconf), + } log.Info("Initialized connection pool with settings: ", spew.Sdump(redactedParams)) return nil } From b73842600ffe957d15cafc3308eafc5793aad861 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Mon, 20 Jan 2025 11:56:25 +0530 Subject: [PATCH 5/7] deepcopy --- yb-voyager/go.mod | 3 ++- yb-voyager/go.sum | 2 ++ yb-voyager/src/tgtdb/yugabytedb.go | 11 +++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index 7df52aa40d..228330bfa1 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -37,6 +37,7 @@ require ( github.com/tebeka/atexit v0.3.0 github.com/testcontainers/testcontainers-go v0.34.0 github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.34.0 + github.com/tiendc/go-deepcopy v1.2.0 github.com/vbauerster/mpb/v8 v8.4.0 gocloud.dev v0.29.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa @@ -168,7 +169,7 @@ require ( golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.16.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index 1c06c76f90..61e5485128 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -1932,6 +1932,8 @@ github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNu github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.34.0 h1:9wIqSZJwBr4s8Q7R3S+rhe1J2zqHHxH0S1bN17ld+CI= github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.34.0/go.mod h1:bgHrbdYjpNPSstf8HfxChUxc6XztBCSoqDR0syb1Oeg= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tiendc/go-deepcopy v1.2.0 h1:6vCCs+qdLQHzFqY1fcPirsAWOmrLbuccilfp8UzD1Qo= +github.com/tiendc/go-deepcopy v1.2.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index 0091cb6881..197f687f01 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -37,6 +37,7 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" "github.com/samber/lo" log "github.com/sirupsen/logrus" + "github.com/tiendc/go-deepcopy" "golang.org/x/exp/slices" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" @@ -243,12 +244,10 @@ func (yb *TargetYugabyteDB) InitConnPool() error { SessionInitScript: getYBSessionInitScript(yb.tconf), } yb.connPool = NewConnectionPool(params) - redactedParams := ConnectionParams{ - NumConnections: yb.tconf.Parallelism, - NumMaxConnections: yb.tconf.MaxParallelism, - ConnUriList: utils.GetRedactedURLs(targetUriList), - SessionInitScript: getYBSessionInitScript(yb.tconf), - } + redactedParams := &ConnectionParams{} + //Whenever adding new fields to CONNECTION PARAMS check if that needs to be redacted while logging + deepcopy.Copy(redactedParams, params) + redactedParams.ConnUriList = utils.GetRedactedURLs(redactedParams.ConnUriList) log.Info("Initialized connection pool with settings: ", spew.Sdump(redactedParams)) return nil } From f91cd7aa48e77c2a989a813a9e95ed563860b4ec Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Mon, 20 Jan 2025 12:03:21 +0530 Subject: [PATCH 6/7] add 2.25 to supported preview versions --- yb-voyager/src/ybversion/constants.go | 1 + yb-voyager/src/ybversion/yb_version.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/yb-voyager/src/ybversion/constants.go b/yb-voyager/src/ybversion/constants.go index 94bbfffa94..afa2c125f8 100644 --- a/yb-voyager/src/ybversion/constants.go +++ b/yb-voyager/src/ybversion/constants.go @@ -23,6 +23,7 @@ const ( SERIES_2024_2 = "2024.2" SERIES_2_21 = "2.21" SERIES_2_23 = "2.23" + SERIES_2_25 = "2.25" ) var LatestStable *YBVersion diff --git a/yb-voyager/src/ybversion/yb_version.go b/yb-voyager/src/ybversion/yb_version.go index fad15e0863..087de2d730 100644 --- a/yb-voyager/src/ybversion/yb_version.go +++ b/yb-voyager/src/ybversion/yb_version.go @@ -29,7 +29,7 @@ import ( // Reference - https://docs.yugabyte.com/preview/releases/ybdb-releases/ var supportedYBVersionStableSeriesOld = []string{SERIES_2_14, SERIES_2_18, SERIES_2_20} var supportedYBVersionStableSeries = []string{SERIES_2024_1, SERIES_2024_2} -var supportedYBVersionPreviewSeries = []string{SERIES_2_21, SERIES_2_23} +var supportedYBVersionPreviewSeries = []string{SERIES_2_21, SERIES_2_23, SERIES_2_25} var allSupportedYBVersionSeries = lo.Flatten([][]string{supportedYBVersionStableSeries, supportedYBVersionPreviewSeries, supportedYBVersionStableSeriesOld}) var ErrUnsupportedSeries = fmt.Errorf("unsupported YB version series. Supported YB version series = %v", allSupportedYBVersionSeries) From 979c379c166ab31385af505f0b59bc75ba279c17 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Mon, 20 Jan 2025 13:39:36 +0530 Subject: [PATCH 7/7] handle error --- yb-voyager/src/tgtdb/yugabytedb.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/yb-voyager/src/tgtdb/yugabytedb.go b/yb-voyager/src/tgtdb/yugabytedb.go index 197f687f01..8989190ee4 100644 --- a/yb-voyager/src/tgtdb/yugabytedb.go +++ b/yb-voyager/src/tgtdb/yugabytedb.go @@ -246,7 +246,11 @@ func (yb *TargetYugabyteDB) InitConnPool() error { yb.connPool = NewConnectionPool(params) redactedParams := &ConnectionParams{} //Whenever adding new fields to CONNECTION PARAMS check if that needs to be redacted while logging - deepcopy.Copy(redactedParams, params) + err := deepcopy.Copy(redactedParams, params) + if err!= nil { + log.Errorf("couldn't get the copy of connection params for logging: %v", err) + return nil + } redactedParams.ConnUriList = utils.GetRedactedURLs(redactedParams.ConnUriList) log.Info("Initialized connection pool with settings: ", spew.Sdump(redactedParams)) return nil