diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 96658551e3..ea81fd4e7c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,7 +8,7 @@ on: jobs: - build: + build-and-test: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -26,7 +26,7 @@ jobs: - name: Test run: | cd yb-voyager - go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... + go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' - name: Vet run: | diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 0000000000..b6b018be21 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,53 @@ +name: Go + +on: + push: + branches: ['main', '*.*-dev', '*.*.*-dev'] + pull_request: + branches: [main] + +env: + ORACLE_INSTANT_CLIENT_VERSION: "21.5.0.0.0-1" + +jobs: + integration-tests: + strategy: + fail-fast: false + + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.23.1" + + - name: Build + run: | + cd yb-voyager + go build -v ./... + + # required by godror driver used in the tests + - name: Install Oracle Instant Clients + run: | + # Download and install the YB APT repository package + wget https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/repos/reporpms/yb-apt-repo_1.0.0_all.deb + sudo apt-get install -y ./yb-apt-repo_1.0.0_all.deb + sudo apt-get update -y + + # Install Oracle Instant Client packages using the defined version + sudo apt-get install -y oracle-instantclient-tools=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-basic=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-devel=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-jdbc=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-sqlplus=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + + # Clean up the YB APT repository package + sudo apt-get remove -y yb-apt-repo + rm -f yb-apt-repo_1.0.0_all.deb + + - name: Run Integration Tests + run: | + cd yb-voyager + go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags 'integration' diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml index 2c965c5427..7db2e6260c 100644 --- a/.github/workflows/issue-tests.yml +++ b/.github/workflows/issue-tests.yml @@ -47,5 +47,4 @@ jobs: - name: Test Issues Against YB Version run: | cd yb-voyager - go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... - + go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' diff --git a/yb-voyager/cmd/common_test.go b/yb-voyager/cmd/common_test.go index d0046ea7ad..6a5c462365 100644 --- a/yb-voyager/cmd/common_test.go +++ b/yb-voyager/cmd/common_test.go @@ -8,9 +8,9 @@ import ( "testing" "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestAssessmentReportStructs(t *testing.T) { diff --git a/yb-voyager/cmd/exportDataStatus_test.go b/yb-voyager/cmd/exportDataStatus_test.go index beec85005c..692710aace 100644 --- a/yb-voyager/cmd/exportDataStatus_test.go +++ b/yb-voyager/cmd/exportDataStatus_test.go @@ -6,8 +6,8 @@ import ( "reflect" "testing" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestExportSnapshotStatusStructs(t *testing.T) { diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index a8a306dbcb..8cf7a703f2 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -22,8 +22,8 @@ require ( github.com/gosuri/uilive v0.0.4 github.com/gosuri/uitable v0.0.4 github.com/hashicorp/go-version v1.7.0 - github.com/jackc/pgconn v1.13.0 - github.com/jackc/pgx/v4 v4.17.2 + github.com/jackc/pgconn v1.14.3 + github.com/jackc/pgx/v4 v4.18.3 github.com/jackc/pgx/v5 v5.0.3 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 @@ -45,6 +45,7 @@ require ( golang.org/x/term v0.24.0 google.golang.org/api v0.169.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gotest.tools v2.2.0+incompatible ) require ( @@ -143,9 +144,9 @@ require ( github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pglogrepl v0.0.0-20231111135425-1627ab1b5780 github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.1 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.12.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/magiconair/properties v1.8.7 github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index 3b156d1c04..e68839fdc2 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -1361,8 +1361,9 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pglogrepl v0.0.0-20231111135425-1627ab1b5780 h1:pNK2AKKIRC1MMMvpa6UiNtdtOebpiIloX7q2JZDkfsk= @@ -1380,22 +1381,26 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/pgx/v5 v5.0.3 h1:4flM5ecR/555F0EcnjdaZa6MhBU+nr0QbZIo5vaKjuM= github.com/jackc/pgx/v5 v5.0.3/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= diff --git a/yb-voyager/src/callhome/diagnostics_test.go b/yb-voyager/src/callhome/diagnostics_test.go index a212004a05..9d63b562b5 100644 --- a/yb-voyager/src/callhome/diagnostics_test.go +++ b/yb-voyager/src/callhome/diagnostics_test.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/google/uuid" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestCallhomeStructs(t *testing.T) { diff --git a/yb-voyager/src/cp/yugabyted/yugabyted_test.go b/yb-voyager/src/cp/yugabyted/yugabyted_test.go index 1c6690999d..90bf0c5e25 100644 --- a/yb-voyager/src/cp/yugabyted/yugabyted_test.go +++ b/yb-voyager/src/cp/yugabyted/yugabyted_test.go @@ -14,29 +14,33 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v4/pgxpool" - _ "github.com/lib/pq" // PostgreSQL driver + _ "github.com/jackc/pgx/v5/stdlib" "github.com/stretchr/testify/assert" controlPlane "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" - "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + testcontainers "github.com/yugabyte/yb-voyager/yb-voyager/test/containers" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestYugabyteDTableSchema(t *testing.T) { ctx := context.Background() - // Start a YugabyteDB container - ybContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.YUGABYTEDB) + yugabyteDBContainer := testcontainers.NewTestContainer("yugabytedb", nil) + err := yugabyteDBContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start yugabytedb container: %v", err) + } + defer testcontainers.TerminateAllContainers() assert.NoError(t, err, "Failed to start YugabyteDB container") - defer ybContainer.Terminate(ctx) // Connect to the database - dsn := fmt.Sprintf("host=%s port=%s user=yugabyte password=yugabyte dbname=yugabyte sslmode=disable", host, port.Port()) - db, err := sql.Open("postgres", dsn) + dsn := yugabyteDBContainer.GetConnectionString() + db, err := sql.Open("pgx", dsn) assert.NoError(t, err) defer db.Close() // Wait for the database to be ready - err = testcontainers.WaitForDBToBeReady(db) + err = testutils.WaitForDBToBeReady(db) assert.NoError(t, err) // Export the database connection string to env variable YUGABYTED_DB_CONN_STRING err = os.Setenv("YUGABYTED_DB_CONN_STRING", dsn) diff --git a/yb-voyager/src/datafile/descriptor_test.go b/yb-voyager/src/datafile/descriptor_test.go index dc617d0909..6092aba4c3 100644 --- a/yb-voyager/src/datafile/descriptor_test.go +++ b/yb-voyager/src/datafile/descriptor_test.go @@ -6,7 +6,7 @@ import ( "reflect" "testing" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestDescriptorStructs(t *testing.T) { diff --git a/yb-voyager/src/dbzm/status_test.go b/yb-voyager/src/dbzm/status_test.go index 9b384e9ad4..753188dfd8 100644 --- a/yb-voyager/src/dbzm/status_test.go +++ b/yb-voyager/src/dbzm/status_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestExportStatusStructs(t *testing.T) { diff --git a/yb-voyager/src/metadb/metadataDB_test.go b/yb-voyager/src/metadb/metadataDB_test.go index 89648a26b7..b6c0126927 100644 --- a/yb-voyager/src/metadb/metadataDB_test.go +++ b/yb-voyager/src/metadb/metadataDB_test.go @@ -7,7 +7,7 @@ import ( "testing" _ "github.com/mattn/go-sqlite3" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) // Test the initMetaDB function diff --git a/yb-voyager/src/migassessment/assessmentDB_test.go b/yb-voyager/src/migassessment/assessmentDB_test.go index 843d1cca06..854b52c8fd 100644 --- a/yb-voyager/src/migassessment/assessmentDB_test.go +++ b/yb-voyager/src/migassessment/assessmentDB_test.go @@ -7,7 +7,7 @@ import ( "testing" _ "github.com/mattn/go-sqlite3" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestInitAssessmentDB(t *testing.T) { diff --git a/yb-voyager/src/namereg/namereg_test.go b/yb-voyager/src/namereg/namereg_test.go index 9d6b16c523..c785dbff02 100644 --- a/yb-voyager/src/namereg/namereg_test.go +++ b/yb-voyager/src/namereg/namereg_test.go @@ -13,8 +13,8 @@ import ( "github.com/stretchr/testify/require" "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) var oracleToYBNameRegistry = &NameRegistry{ diff --git a/yb-voyager/src/srcdb/main_test.go b/yb-voyager/src/srcdb/main_test.go new file mode 100644 index 0000000000..770855ad60 --- /dev/null +++ b/yb-voyager/src/srcdb/main_test.go @@ -0,0 +1,176 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "context" + "os" + "testing" + + _ "github.com/godror/godror" + _ "github.com/jackc/pgx/v5/stdlib" + log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + testcontainers "github.com/yugabyte/yb-voyager/yb-voyager/test/containers" +) + +type TestDB struct { + testcontainers.TestContainer + *Source +} + +var ( + testPostgresSource *TestDB + testOracleSource *TestDB + testMySQLSource *TestDB + testYugabyteDBSource *TestDB +) + +func TestMain(m *testing.M) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + postgresContainer := testcontainers.NewTestContainer("postgresql", nil) + err := postgresContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start postgres container: %v", err) + } + host, port, err := postgresContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testPostgresSource = &TestDB{ + TestContainer: postgresContainer, + Source: &Source{ + DBType: "postgresql", + DBVersion: postgresContainer.GetConfig().DBVersion, + User: postgresContainer.GetConfig().User, + Password: postgresContainer.GetConfig().Password, + Schema: postgresContainer.GetConfig().Schema, + DBName: postgresContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }, + } + err = testPostgresSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to postgres database: %w", err) + } + defer testPostgresSource.DB().Disconnect() + + oracleContainer := testcontainers.NewTestContainer("oracle", nil) + err = oracleContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start oracle container: %v", err) + } + host, port, err = oracleContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + + testOracleSource = &TestDB{ + TestContainer: oracleContainer, + Source: &Source{ + DBType: "oracle", + DBVersion: oracleContainer.GetConfig().DBVersion, + User: oracleContainer.GetConfig().User, + Password: oracleContainer.GetConfig().Password, + Schema: oracleContainer.GetConfig().Schema, + DBName: oracleContainer.GetConfig().DBName, + Host: host, + Port: port, + }, + } + + err = testOracleSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to oracle database: %w", err) + } + defer testOracleSource.DB().Disconnect() + + mysqlContainer := testcontainers.NewTestContainer("mysql", nil) + err = mysqlContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start mysql container: %v", err) + } + host, port, err = mysqlContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testMySQLSource = &TestDB{ + TestContainer: mysqlContainer, + Source: &Source{ + DBType: "mysql", + DBVersion: mysqlContainer.GetConfig().DBVersion, + User: mysqlContainer.GetConfig().User, + Password: mysqlContainer.GetConfig().Password, + Schema: mysqlContainer.GetConfig().Schema, + DBName: mysqlContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }, + } + + err = testMySQLSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to mysql database: %w", err) + } + defer testMySQLSource.DB().Disconnect() + + yugabytedbContainer := testcontainers.NewTestContainer("yugabytedb", nil) + err = yugabytedbContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start yugabytedb container: %v", err) + } + host, port, err = yugabytedbContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testYugabyteDBSource = &TestDB{ + TestContainer: yugabytedbContainer, + Source: &Source{ + DBType: "yugabytedb", + DBVersion: yugabytedbContainer.GetConfig().DBVersion, + User: yugabytedbContainer.GetConfig().User, + Password: yugabytedbContainer.GetConfig().Password, + Schema: yugabytedbContainer.GetConfig().Schema, + DBName: yugabytedbContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }, + } + + err = testYugabyteDBSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to yugabytedb database: %w", err) + } + defer testYugabyteDBSource.DB().Disconnect() + + // to avoid info level logs flooding the test output + log.SetLevel(log.WarnLevel) + + exitCode := m.Run() + + // cleanig up all the running containers + testcontainers.TerminateAllContainers() + + os.Exit(exitCode) +} diff --git a/yb-voyager/src/srcdb/mysql_test.go b/yb-voyager/src/srcdb/mysql_test.go new file mode 100644 index 0000000000..db74724162 --- /dev/null +++ b/yb-voyager/src/srcdb/mysql_test.go @@ -0,0 +1,89 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestMysqlGetAllTableNames(t *testing.T) { + testMySQLSource.ExecuteSqls( + `CREATE DATABASE test;`, + `CREATE TABLE test.foo ( + id INT PRIMARY KEY, + name VARCHAR(255) + );`, + `CREATE TABLE test.bar ( + id INT PRIMARY KEY, + name VARCHAR(255) + );`, + `CREATE TABLE test.non_pk1( + id INT, + name VARCHAR(255) + );`) + defer testMySQLSource.ExecuteSqls(`DROP DATABASE test;`) + + sqlname.SourceDBType = "mysql" + testMySQLSource.Source.DBName = "test" // used in query of GetAllTableNames() + + // Test GetAllTableNames + actualTables := testMySQLSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("test", "foo"), + sqlname.NewSourceName("test", "bar"), + sqlname.NewSourceName("test", "non_pk1"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +// TODO: Seems like a Bug somwhere, because now mysql.GetAllNonPkTables() as it is returning all the tables created in this test +// func TestMySQLGetNonPKTables(t *testing.T) { +// testMySQLSource.ExecuteSqls( +// `CREATE DATABASE test;`, +// `CREATE TABLE test.table1 ( +// id INT AUTO_INCREMENT PRIMARY KEY, +// name VARCHAR(100) +// );`, +// `CREATE TABLE test.table2 ( +// id INT AUTO_INCREMENT PRIMARY KEY, +// email VARCHAR(100) +// );`, +// `CREATE TABLE test.non_pk1( +// id INT, +// name VARCHAR(255) +// );`, +// `CREATE TABLE test.non_pk2( +// id INT, +// name VARCHAR(255) +// );`) +// defer testMySQLSource.ExecuteSqls(`DROP DATABASE test;`) + +// testMySQLSource.Source.DBName = "test" +// actualTables, err := testMySQLSource.DB().GetNonPKTables() +// assert.NilError(t, err, "Expected nil but non nil error: %v", err) + +// expectedTables := []string{"test.non_pk1", "test.non_pk2"} + +// testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +// } diff --git a/yb-voyager/src/srcdb/oracle_test.go b/yb-voyager/src/srcdb/oracle_test.go new file mode 100644 index 0000000000..9369b82575 --- /dev/null +++ b/yb-voyager/src/srcdb/oracle_test.go @@ -0,0 +1,80 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestOracleGetAllTableNames(t *testing.T) { + sqlname.SourceDBType = "oracle" + + // Test GetAllTableNames + actualTables := testOracleSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("YBVOYAGER", "foo"), + sqlname.NewSourceName("YBVOYAGER", "bar"), + sqlname.NewSourceName("YBVOYAGER", "table1"), + sqlname.NewSourceName("YBVOYAGER", "table2"), + sqlname.NewSourceName("YBVOYAGER", "unique_table"), + sqlname.NewSourceName("YBVOYAGER", "non_pk1"), + sqlname.NewSourceName("YBVOYAGER", "non_pk2"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +func TestOracleGetTableToUniqueKeyColumnsMap(t *testing.T) { + objectName := sqlname.NewObjectName("oracle", "YBVOYAGER", "YBVOYAGER", "UNIQUE_TABLE") + + // Test GetTableToUniqueKeyColumnsMap + tableList := []sqlname.NameTuple{ + {CurrentName: objectName}, + } + uniqueKeys, err := testOracleSource.DB().GetTableToUniqueKeyColumnsMap(tableList) + if err != nil { + t.Fatalf("Error retrieving unique keys: %v", err) + } + + expectedKeys := map[string][]string{ + "UNIQUE_TABLE": {"EMAIL", "PHONE", "ADDRESS"}, + } + + // Compare the maps by iterating over each table and asserting the columns list + for table, expectedColumns := range expectedKeys { + actualColumns, exists := uniqueKeys[table] + if !exists { + t.Errorf("Expected table %s not found in uniqueKeys", table) + } + + testutils.AssertEqualStringSlices(t, expectedColumns, actualColumns) + } +} + +func TestOracleGetNonPKTables(t *testing.T) { + actualTables, err := testOracleSource.DB().GetNonPKTables() + assert.NilError(t, err, "Expected nil but non nil error: %v", err) + + expectedTables := []string{`YBVOYAGER."NON_PK1"`, `YBVOYAGER."NON_PK2"`} + testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 4c90ffb5e6..ba2e21b720 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -940,6 +940,7 @@ var PG_QUERY_TO_CHECK_IF_TABLE_HAS_PK = `SELECT nspname AS schema_name, relname FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_constraint con ON con.conrelid = c.oid AND con.contype = 'p' +WHERE c.relkind = 'r' OR c.relkind = 'p' -- Only consider table objects GROUP BY schema_name, table_name HAVING nspname IN (%s);` func (pg *PostgreSQL) GetNonPKTables() ([]string, error) { @@ -964,8 +965,9 @@ func (pg *PostgreSQL) GetNonPKTables() ([]string, error) { if err != nil { return nil, fmt.Errorf("error in scanning query rows for primary key: %v", err) } - table := sqlname.NewSourceName(schemaName, fmt.Sprintf(`"%s"`, tableName)) + if pkCount == 0 { + table := sqlname.NewSourceName(schemaName, fmt.Sprintf(`"%s"`, tableName)) nonPKTables = append(nonPKTables, table.Qualified.Quoted) } } diff --git a/yb-voyager/src/srcdb/postgres_test.go b/yb-voyager/src/srcdb/postgres_test.go new file mode 100644 index 0000000000..41ac55d5ac --- /dev/null +++ b/yb-voyager/src/srcdb/postgres_test.go @@ -0,0 +1,136 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestPostgresGetAllTableNames(t *testing.T) { + testPostgresSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`) + defer testPostgresSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + sqlname.SourceDBType = "postgresql" + testPostgresSource.Source.Schema = "test_schema" // used in query of GetAllTableNames() + + // Test GetAllTableNames + actualTables := testPostgresSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("test_schema", "foo"), + sqlname.NewSourceName("test_schema", "bar"), + sqlname.NewSourceName("test_schema", "non_pk1"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +func TestPostgresGetTableToUniqueKeyColumnsMap(t *testing.T) { + testPostgresSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE, + phone VARCHAR(20) UNIQUE, + address VARCHAR(255) UNIQUE + );`, + `INSERT INTO test_schema.unique_table (email, phone, address) VALUES + ('john@example.com', '1234567890', '123 Elm Street'), + ('jane@example.com', '0987654321', '456 Oak Avenue');`, + `CREATE TABLE test_schema.another_unique_table ( + user_id SERIAL PRIMARY KEY, + username VARCHAR(50) UNIQUE, + age INT + );`, + `CREATE UNIQUE INDEX idx_age ON test_schema.another_unique_table(age);`, + `INSERT INTO test_schema.another_unique_table (username, age) VALUES + ('user1', 30), + ('user2', 40);`) + defer testPostgresSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + uniqueTablesList := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "another_unique_table")}, + } + + actualUniqKeys, err := testPostgresSource.DB().GetTableToUniqueKeyColumnsMap(uniqueTablesList) + if err != nil { + t.Fatalf("Error retrieving unique keys: %v", err) + } + + expectedUniqKeys := map[string][]string{ + "test_schema.unique_table": {"email", "phone", "address"}, + "test_schema.another_unique_table": {"username", "age"}, + } + + // Compare the maps by iterating over each table and asserting the columns list + for table, expectedColumns := range expectedUniqKeys { + actualColumns, exists := actualUniqKeys[table] + if !exists { + t.Errorf("Expected table %s not found in uniqueKeys", table) + } + + testutils.AssertEqualStringSlices(t, expectedColumns, actualColumns) + } +} + +func TestPostgresGetNonPKTables(t *testing.T) { + testPostgresSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testPostgresSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + actualTables, err := testPostgresSource.DB().GetNonPKTables() + assert.NilError(t, err, "Expected nil but non nil error: %v", err) + + expectedTables := []string{`test_schema."non_pk2"`, `test_schema."non_pk1"`} // func returns table.Qualified.Quoted + testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/srcdb/yugbaytedb_test.go b/yb-voyager/src/srcdb/yugbaytedb_test.go new file mode 100644 index 0000000000..4a7cf8e6f9 --- /dev/null +++ b/yb-voyager/src/srcdb/yugbaytedb_test.go @@ -0,0 +1,136 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestYugabyteGetAllTableNames(t *testing.T) { + testYugabyteDBSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`) + defer testYugabyteDBSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + sqlname.SourceDBType = "postgresql" + testYugabyteDBSource.Source.Schema = "test_schema" + + // Test GetAllTableNames + actualTables := testYugabyteDBSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("test_schema", "foo"), + sqlname.NewSourceName("test_schema", "bar"), + sqlname.NewSourceName("test_schema", "non_pk1"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +func TestYugabyteGetTableToUniqueKeyColumnsMap(t *testing.T) { + testYugabyteDBSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE, + phone VARCHAR(20) UNIQUE, + address VARCHAR(255) UNIQUE + );`, + `INSERT INTO test_schema.unique_table (email, phone, address) VALUES + ('john@example.com', '1234567890', '123 Elm Street'), + ('jane@example.com', '0987654321', '456 Oak Avenue');`, + `CREATE TABLE test_schema.another_unique_table ( + user_id SERIAL PRIMARY KEY, + username VARCHAR(50) UNIQUE, + age INT + );`, + `CREATE UNIQUE INDEX idx_age ON test_schema.another_unique_table(age);`, + `INSERT INTO test_schema.another_unique_table (username, age) VALUES + ('user1', 30), + ('user2', 40);`) + defer testYugabyteDBSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + uniqueTablesList := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "another_unique_table")}, + } + + actualUniqKeys, err := testYugabyteDBSource.DB().GetTableToUniqueKeyColumnsMap(uniqueTablesList) + if err != nil { + t.Fatalf("Error retrieving unique keys: %v", err) + } + + expectedUniqKeys := map[string][]string{ + "test_schema.unique_table": {"email", "phone", "address"}, + "test_schema.another_unique_table": {"username", "age"}, + } + + // Compare the maps by iterating over each table and asserting the columns list + for table, expectedColumns := range expectedUniqKeys { + actualColumns, exists := actualUniqKeys[table] + if !exists { + t.Errorf("Expected table %s not found in uniqueKeys", table) + } + + testutils.AssertEqualStringSlices(t, expectedColumns, actualColumns) + } +} + +func TestYugabyteGetNonPKTables(t *testing.T) { + testYugabyteDBSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testYugabyteDBSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + actualTables, err := testYugabyteDBSource.DB().GetNonPKTables() + assert.NilError(t, err, "Expected nil but non nil error: %v", err) + + expectedTables := []string{`test_schema."non_pk2"`, `test_schema."non_pk1"`} // func returns table.Qualified.Quoted + testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/tgtdb/conn_pool_test.go b/yb-voyager/src/tgtdb/conn_pool_test.go index 59fd4dad44..7d4ce43805 100644 --- a/yb-voyager/src/tgtdb/conn_pool_test.go +++ b/yb-voyager/src/tgtdb/conn_pool_test.go @@ -23,44 +23,18 @@ import ( "time" "github.com/jackc/pgx/v4" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - - embeddedpostgres "github.com/fergusstrange/embedded-postgres" ) -// var postgres *embeddedpostgres.EmbeddedPostgres - -func setupPostgres(t *testing.T) *embeddedpostgres.EmbeddedPostgres { - postgres := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig(). - Username("postgres"). - Password("postgres"). - Database("test"). - Port(9876). - StartTimeout(30 * time.Second)) - err := postgres.Start() - if err != nil { - t.Fatal(err) - } - return postgres -} - -func shutdownPostgres(postgres *embeddedpostgres.EmbeddedPostgres, t *testing.T) { - err := postgres.Stop() - if err != nil { - t.Fatal(err) - } -} - func TestBasic(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10. size := 10 connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: size, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -88,8 +62,6 @@ func dummyProcess(pool *ConnectionPool, milliseconds int, wg *sync.WaitGroup) { } func TestIncreaseConnectionsUptoMax(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10, with max 20 connections. size := 10 maxSize := 20 @@ -97,7 +69,7 @@ func TestIncreaseConnectionsUptoMax(t *testing.T) { connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: maxSize, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -105,7 +77,7 @@ func TestIncreaseConnectionsUptoMax(t *testing.T) { // WHEN: multiple goroutines acquire connection, perform some operation // and release connection back to pool - // WHEN: we keep increasing the connnections upto the max.. + // WHEN: we keep increasing the connections upto the max.. var wg sync.WaitGroup wg.Add(1) @@ -133,8 +105,6 @@ func TestIncreaseConnectionsUptoMax(t *testing.T) { } func TestDecreaseConnectionsUptoMin(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10, with max 20 connections. size := 10 maxSize := 20 @@ -142,7 +112,7 @@ func TestDecreaseConnectionsUptoMin(t *testing.T) { connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: maxSize, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -178,8 +148,6 @@ func TestDecreaseConnectionsUptoMin(t *testing.T) { } func TestUpdateConnectionsRandom(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10, with max 20 connections. size := 10 maxSize := 20 @@ -187,7 +155,7 @@ func TestUpdateConnectionsRandom(t *testing.T) { connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: maxSize, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -207,7 +175,7 @@ func TestUpdateConnectionsRandom(t *testing.T) { if pool.size+randomNumber < 1 || (pool.size+randomNumber > pool.params.NumMaxConnections) { continue } - fmt.Printf("i=%d, updating by %d. New pool size expected = %d\n", i, randomNumber, *expectedFinalSize+randomNumber) + log.Infof("i=%d, updating by %d. New pool size expected = %d\n", i, randomNumber, *expectedFinalSize+randomNumber) err := pool.UpdateNumConnections(randomNumber) assert.NoError(t, err) time.Sleep(10 * time.Millisecond) diff --git a/yb-voyager/src/tgtdb/main_test.go b/yb-voyager/src/tgtdb/main_test.go new file mode 100644 index 0000000000..46cc2150e2 --- /dev/null +++ b/yb-voyager/src/tgtdb/main_test.go @@ -0,0 +1,139 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package tgtdb + +import ( + "context" + "os" + "testing" + + _ "github.com/godror/godror" + _ "github.com/jackc/pgx/v5/stdlib" + log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + testcontainers "github.com/yugabyte/yb-voyager/yb-voyager/test/containers" +) + +type TestDB struct { + testcontainers.TestContainer + TargetDB +} + +var ( + testPostgresTarget *TestDB + testOracleTarget *TestDB + testYugabyteDBTarget *TestDB +) + +func TestMain(m *testing.M) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + postgresContainer := testcontainers.NewTestContainer("postgresql", nil) + err := postgresContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start postgres container: %v", err) + } + host, port, err := postgresContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testPostgresTarget = &TestDB{ + TestContainer: postgresContainer, + TargetDB: NewTargetDB(&TargetConf{ + TargetDBType: "postgresql", + DBVersion: postgresContainer.GetConfig().DBVersion, + User: postgresContainer.GetConfig().User, + Password: postgresContainer.GetConfig().Password, + Schema: postgresContainer.GetConfig().Schema, + DBName: postgresContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }), + } + + err = testPostgresTarget.Init() + if err != nil { + utils.ErrExit("Failed to connect to postgres database: %w", err) + } + defer testPostgresTarget.Finalize() + + // oracleContainer := testcontainers.NewTestContainer("oracle", nil) + // _ = oracleContainer.Start(ctx) + // host, port, err = oracleContainer.GetHostPort() + // if err != nil { + // utils.ErrExit("%v", err) + // } + // testOracleTarget = &TestDB2{ + // Container: oracleContainer, + // TargetDB: NewTargetDB(&TargetConf{ + // TargetDBType: "oracle", + // DBVersion: oracleContainer.GetConfig().DBVersion, + // User: oracleContainer.GetConfig().User, + // Password: oracleContainer.GetConfig().Password, + // Schema: oracleContainer.GetConfig().Schema, + // DBName: oracleContainer.GetConfig().DBName, + // Host: host, + // Port: port, + // }), + // } + + // err = testOracleTarget.Init() + // if err != nil { + // utils.ErrExit("Failed to connect to oracle database: %w", err) + // } + // defer testOracleTarget.Finalize() + + yugabytedbContainer := testcontainers.NewTestContainer("yugabytedb", nil) + err = yugabytedbContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start yugabytedb container: %v", err) + } + host, port, err = yugabytedbContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testYugabyteDBTarget = &TestDB{ + TestContainer: yugabytedbContainer, + TargetDB: NewTargetDB(&TargetConf{ + TargetDBType: "yugabytedb", + DBVersion: yugabytedbContainer.GetConfig().DBVersion, + User: yugabytedbContainer.GetConfig().User, + Password: yugabytedbContainer.GetConfig().Password, + Schema: yugabytedbContainer.GetConfig().Schema, + DBName: yugabytedbContainer.GetConfig().DBName, + Host: host, + Port: port, + }), + } + + err = testYugabyteDBTarget.Init() + if err != nil { + utils.ErrExit("Failed to connect to yugabytedb database: %w", err) + } + defer testYugabyteDBTarget.Finalize() + + // to avoid info level logs flooding the test output + log.SetLevel(log.WarnLevel) + + exitCode := m.Run() + + // cleaning up all the running containers + testcontainers.TerminateAllContainers() + + os.Exit(exitCode) +} diff --git a/yb-voyager/src/tgtdb/postgres_test.go b/yb-voyager/src/tgtdb/postgres_test.go index 251ced5a63..e0e9bd60c5 100644 --- a/yb-voyager/src/tgtdb/postgres_test.go +++ b/yb-voyager/src/tgtdb/postgres_test.go @@ -1,33 +1,39 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package tgtdb import ( - "context" "database/sql" "fmt" "strings" "testing" + _ "github.com/jackc/pgx/v5/stdlib" "github.com/stretchr/testify/assert" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" - "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestCreateVoyagerSchemaPG(t *testing.T) { - ctx := context.Background() - - // Start a PostgreSQL container - pgContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.POSTGRESQL) - assert.NoError(t, err, "Failed to start PostgreSQL container") - defer pgContainer.Terminate(ctx) - - // Connect to the database - dsn := fmt.Sprintf("host=%s port=%s user=testuser password=testpassword dbname=testdb sslmode=disable", host, port.Port()) - db, err := sql.Open("postgres", dsn) + db, err := sql.Open("pgx", testPostgresTarget.GetConnectionString()) assert.NoError(t, err) defer db.Close() // Wait for the database to be ready - err = testcontainers.WaitForDBToBeReady(db) + err = testutils.WaitForDBToBeReady(db) assert.NoError(t, err) // Initialize the TargetYugabyteDB instance @@ -80,3 +86,60 @@ func TestCreateVoyagerSchemaPG(t *testing.T) { }) } } + +func TestPostgresGetNonEmptyTables(t *testing.T) { + testPostgresTarget.ExecuteSqls( + `CREATE SCHEMA test_schema`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(100), + phone VARCHAR(100), + address VARCHAR(255), + UNIQUE (email, phone) -- Unique constraint on combination of columns + );`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testPostgresTarget.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + tables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "bar")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "table1")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "table2")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "non_pk1")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "non_pk2")}, + } + + expectedTables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "bar")}, + } + + actualTables := testPostgresTarget.GetNonEmptyTables(tables) + testutils.AssertEqualNameTuplesSlice(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/tgtdb/yugabytedb_test.go b/yb-voyager/src/tgtdb/yugabytedb_test.go index af22664460..755ed46d67 100644 --- a/yb-voyager/src/tgtdb/yugabytedb_test.go +++ b/yb-voyager/src/tgtdb/yugabytedb_test.go @@ -1,33 +1,40 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package tgtdb import ( - "context" "database/sql" "fmt" "strings" "testing" + _ "github.com/jackc/pgx/v5/stdlib" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" - "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestCreateVoyagerSchemaYB(t *testing.T) { - ctx := context.Background() - - // Start a YugabyteDB container - ybContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.YUGABYTEDB) - assert.NoError(t, err, "Failed to start YugabyteDB container") - defer ybContainer.Terminate(ctx) - - // Connect to the database - dsn := fmt.Sprintf("host=%s port=%s user=yugabyte password=yugabyte dbname=yugabyte sslmode=disable", host, port.Port()) - db, err := sql.Open("postgres", dsn) + db, err := sql.Open("pgx", testYugabyteDBTarget.GetConnectionString()) assert.NoError(t, err) defer db.Close() // Wait for the database to be ready - err = testcontainers.WaitForDBToBeReady(db) + err = testutils.WaitForDBToBeReady(db) assert.NoError(t, err) // Initialize the TargetYugabyteDB instance @@ -80,3 +87,61 @@ func TestCreateVoyagerSchemaYB(t *testing.T) { }) } } + +func TestYugabyteGetNonEmptyTables(t *testing.T) { + testYugabyteDBTarget.ExecuteSqls( + `CREATE SCHEMA test_schema`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(100), + phone VARCHAR(100), + address VARCHAR(255), + UNIQUE (email, phone) + );`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testYugabyteDBTarget.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + tables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "bar")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "table1")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "table2")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "non_pk1")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "non_pk2")}, + } + + expectedTables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "bar")}, + } + + actualTables := testYugabyteDBTarget.GetNonEmptyTables(tables) + log.Infof("non empty tables: %+v\n", actualTables) + testutils.AssertEqualNameTuplesSlice(t, expectedTables, actualTables) +} diff --git a/yb-voyager/test/containers/helpers.go b/yb-voyager/test/containers/helpers.go new file mode 100644 index 0000000000..cd3afee88d --- /dev/null +++ b/yb-voyager/test/containers/helpers.go @@ -0,0 +1,60 @@ +package testcontainers + +import ( + "context" + _ "embed" + "fmt" + "io" + + log "github.com/sirupsen/logrus" + + "github.com/testcontainers/testcontainers-go" +) + +const ( + DEFAULT_PG_PORT = "5432" + DEFAULT_YB_PORT = "5433" + DEFAULT_ORACLE_PORT = "1521" + DEFAULT_MYSQL_PORT = "3306" + + POSTGRESQL = "postgresql" + YUGABYTEDB = "yugabytedb" + ORACLE = "oracle" + MYSQL = "mysql" +) + +//go:embed test_schemas/postgresql_schema.sql +var postgresInitSchemaFile []byte + +//go:embed test_schemas/oracle_schema.sql +var oracleInitSchemaFile []byte + +//go:embed test_schemas/mysql_schema.sql +var mysqlInitSchemaFile []byte + +//go:embed test_schemas/yugabytedb_schema.sql +var yugabytedbInitSchemaFile []byte + +func printContainerLogs(container testcontainers.Container) { + if container == nil { + log.Printf("Cannot fetch logs: container is nil") + return + } + + containerID := container.GetContainerID() + logs, err := container.Logs(context.Background()) + if err != nil { + log.Printf("Error fetching logs for container %s: %v", containerID, err) + return + } + defer logs.Close() + + // Read the logs + logData, err := io.ReadAll(logs) + if err != nil { + log.Printf("Error reading logs for container %s: %v", containerID, err) + return + } + + fmt.Printf("=== Logs for container %s ===\n%s\n=== End of Logs for container %s ===\n", containerID, string(logData), containerID) +} diff --git a/yb-voyager/test/containers/mysql_container.go b/yb-voyager/test/containers/mysql_container.go new file mode 100644 index 0000000000..c2d8930cff --- /dev/null +++ b/yb-voyager/test/containers/mysql_container.go @@ -0,0 +1,149 @@ +package testcontainers + +import ( + "context" + "database/sql" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type MysqlContainer struct { + ContainerConfig + container testcontainers.Container + db *sql.DB +} + +func (ms *MysqlContainer) Start(ctx context.Context) (err error) { + if ms.container != nil { + utils.PrintAndLog("Mysql-%s container already running", ms.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "mysql_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(mysqlInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + req := testcontainers.ContainerRequest{ + // TODO: verify the docker images being used are the correct/certified ones + Image: fmt.Sprintf("mysql:%s", ms.DBVersion), + ExposedPorts: []string{"3306/tcp"}, + Env: map[string]string{ + "MYSQL_ROOT_PASSWORD": ms.Password, + "MYSQL_USER": ms.User, + "MYSQL_PASSWORD": ms.Password, + "MYSQL_DATABASE": ms.DBName, + }, + WaitingFor: wait.ForListeningPort("3306/tcp").WithStartupTimeout(2 * time.Minute).WithPollInterval(5 * time.Second), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "docker-entrypoint-initdb.d/mysql_schema.sql", + FileMode: 0755, + }, + }, + } + + ms.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + printContainerLogs(ms.container) + if err != nil { + return err + } + + dsn := ms.GetConnectionString() + db, err := sql.Open("mysql", dsn) + if err != nil { + return fmt.Errorf("failed to open mysql connection: %w", err) + } + + if err = db.Ping(); err != nil { + db.Close() + return fmt.Errorf("failed to ping mysql after connection: %w", err) + } + + // Store the DB connection for reuse + ms.db = db + + return nil +} + +func (ms *MysqlContainer) Terminate(ctx context.Context) { + if ms == nil { + return + } + + // Close the DB connection if it exists + if ms.db != nil { + if err := ms.db.Close(); err != nil { + log.Errorf("failed to close mysql db connection: %v", err) + } + } + + err := ms.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate mysql container: %v", err) + } +} + +func (ms *MysqlContainer) GetHostPort() (string, int, error) { + if ms.container == nil { + return "", -1, fmt.Errorf("mysql container is not started: nil") + } + + ctx := context.Background() + host, err := ms.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for mysql container: %w", err) + } + + port, err := ms.container.MappedPort(ctx, nat.Port(DEFAULT_MYSQL_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for mysql container: %w", err) + } + + return host, port.Int(), nil +} + +func (ms *MysqlContainer) GetConfig() ContainerConfig { + return ms.ContainerConfig +} + +func (ms *MysqlContainer) GetConnectionString() string { + host, port, err := ms.GetHostPort() + if err != nil { + utils.ErrExit("failed to get host port for mysql connection string: %v", err) + } + + // DSN format: user:password@tcp(host:port)/dbname + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", + ms.User, ms.Password, host, port, ms.DBName) +} + +func (ms *MysqlContainer) ExecuteSqls(sqls ...string) { + if ms.db == nil { + utils.ErrExit("db connection not initialized for mysql container") + } + + for _, sqlStmt := range sqls { + _, err := ms.db.Exec(sqlStmt) + if err != nil { + utils.ErrExit("failed to execute sql '%s': %w", sqlStmt, err) + } + } +} diff --git a/yb-voyager/test/containers/oracle_container.go b/yb-voyager/test/containers/oracle_container.go new file mode 100644 index 0000000000..8fb8218de2 --- /dev/null +++ b/yb-voyager/test/containers/oracle_container.go @@ -0,0 +1,107 @@ +package testcontainers + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type OracleContainer struct { + ContainerConfig + container testcontainers.Container +} + +func (ora *OracleContainer) Start(ctx context.Context) (err error) { + if ora.container != nil { + utils.PrintAndLog("Oracle-%s container already running", ora.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "oracle_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(oracleInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + // refer: https://hub.docker.com/r/gvenzl/oracle-xe + req := testcontainers.ContainerRequest{ + // TODO: verify the docker images being used are the correct/certified ones (No license issue) + Image: fmt.Sprintf("gvenzl/oracle-xe:%s", ora.DBVersion), + ExposedPorts: []string{"1521/tcp"}, + Env: map[string]string{ + "ORACLE_PASSWORD": ora.Password, // for SYS user + "ORACLE_DATABASE": ora.DBName, + "APP_USER": ora.User, + "APP_USER_PASSWORD": ora.Password, + }, + WaitingFor: wait.ForLog("DATABASE IS READY TO USE").WithStartupTimeout(2 * time.Minute).WithPollInterval(5 * time.Second), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "docker-entrypoint-initdb.d/oracle_schema.sql", + FileMode: 0755, + }, + }, + } + + ora.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + printContainerLogs(ora.container) + return err +} + +func (ora *OracleContainer) Terminate(ctx context.Context) { + if ora == nil { + return + } + + err := ora.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate oracle container: %v", err) + } +} + +func (ora *OracleContainer) GetHostPort() (string, int, error) { + if ora.container == nil { + return "", -1, fmt.Errorf("oracle container is not started: nil") + } + + ctx := context.Background() + host, err := ora.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for oracle container: %w", err) + } + + port, err := ora.container.MappedPort(ctx, nat.Port(DEFAULT_ORACLE_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for oracle container: %w", err) + } + + return host, port.Int(), nil +} + +func (ora *OracleContainer) GetConfig() ContainerConfig { + return ora.ContainerConfig +} + +func (ora *OracleContainer) GetConnectionString() string { + panic("GetConnectionString() not implemented yet for oracle") +} + +func (ora *OracleContainer) ExecuteSqls(sqls ...string) { + +} diff --git a/yb-voyager/test/containers/postgres_container.go b/yb-voyager/test/containers/postgres_container.go new file mode 100644 index 0000000000..539763f7de --- /dev/null +++ b/yb-voyager/test/containers/postgres_container.go @@ -0,0 +1,151 @@ +package testcontainers + +import ( + "context" + "database/sql" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type PostgresContainer struct { + ContainerConfig + container testcontainers.Container + db *sql.DB +} + +func (pg *PostgresContainer) Start(ctx context.Context) (err error) { + if pg.container != nil { + utils.PrintAndLog("Postgres-%s container already running", pg.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "postgresql_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(postgresInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + req := testcontainers.ContainerRequest{ + // TODO: verify the docker images being used are the correct/certified ones + Image: fmt.Sprintf("postgres:%s", pg.DBVersion), + ExposedPorts: []string{"5432/tcp"}, + Env: map[string]string{ + "POSTGRES_USER": pg.User, + "POSTGRES_PASSWORD": pg.Password, + "POSTGRES_DB": pg.DBName, // NOTE: PG image makes the database with same name as user if not specific + }, + WaitingFor: wait.ForAll( + wait.ForListeningPort("5432/tcp").WithStartupTimeout(2*time.Minute).WithPollInterval(5*time.Second), + wait.ForLog("database system is ready to accept connections").WithStartupTimeout(3*time.Minute).WithPollInterval(5*time.Second), + ), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "docker-entrypoint-initdb.d/postgresql_schema.sql", + FileMode: 0755, + }, + }, + } + + pg.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + + printContainerLogs(pg.container) + if err != nil { + return err + } + + dsn := pg.GetConnectionString() + db, err := sql.Open("pgx", dsn) + if err != nil { + return fmt.Errorf("failed to open postgres connection: %w", err) + } + + if err := db.Ping(); err != nil { + db.Close() + pg.container.Terminate(ctx) + return fmt.Errorf("failed to ping postgres after connection: %w", err) + } + + // Store the DB connection for reuse + pg.db = db + return nil +} + +func (pg *PostgresContainer) Terminate(ctx context.Context) { + if pg == nil { + return + } + + // Close the DB connection if it exists + if pg.db != nil { + if err := pg.db.Close(); err != nil { + log.Errorf("failed to close postgres db connection: %v", err) + } + } + + err := pg.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate postgres container: %v", err) + } +} + +func (pg *PostgresContainer) GetHostPort() (string, int, error) { + if pg.container == nil { + return "", -1, fmt.Errorf("postgres container is not started: nil") + } + + ctx := context.Background() + host, err := pg.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for postgres container: %w", err) + } + + port, err := pg.container.MappedPort(ctx, nat.Port(DEFAULT_PG_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for postgres container: %w", err) + } + + return host, port.Int(), nil +} + +func (pg *PostgresContainer) GetConfig() ContainerConfig { + return pg.ContainerConfig +} + +func (pg *PostgresContainer) GetConnectionString() string { + config := pg.GetConfig() + host, port, err := pg.GetHostPort() + if err != nil { + utils.ErrExit("failed to get host port for postgres connection string: %v", err) + } + + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", config.User, config.Password, host, port, config.DBName) +} + +func (pg *PostgresContainer) ExecuteSqls(sqls ...string) { + if pg.db == nil { + utils.ErrExit("db connection not initialized for postgres container") + } + + for _, sqlStmt := range sqls { + _, err := pg.db.Exec(sqlStmt) + if err != nil { + utils.ErrExit("failed to execute sql '%s': %w", sqlStmt, err) + } + } +} diff --git a/yb-voyager/test/containers/test_schemas/mysql_schema.sql b/yb-voyager/test/containers/test_schemas/mysql_schema.sql new file mode 100644 index 0000000000..e72f4bcb17 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/mysql_schema.sql @@ -0,0 +1,6 @@ +-- TODO: create user as per User creation steps in docs and use that in tests + +-- Grant CREATE, ALTER, DROP privileges globally to 'ybvoyager' +GRANT CREATE, ALTER, DROP ON *.* TO 'ybvoyager'@'%' WITH GRANT OPTION; +-- Apply the changes +FLUSH PRIVILEGES; \ No newline at end of file diff --git a/yb-voyager/test/containers/test_schemas/oracle_schema.sql b/yb-voyager/test/containers/test_schemas/oracle_schema.sql new file mode 100644 index 0000000000..95b3a3a9b4 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/oracle_schema.sql @@ -0,0 +1,47 @@ +-- TODO: create user as per User creation steps in docs and use that in tests + +-- Used ORACLE_DATABASE=DMS i.e. pluggable database to create APP_USER +ALTER SESSION SET CONTAINER = "DMS"; + + +-- creating tables under YBVOYAGER schema, same as APP_USER +CREATE TABLE YBVOYAGER.foo ( + id NUMBER PRIMARY KEY, + name VARCHAR2(255) +); + +CREATE TABLE YBVOYAGER.bar ( + id NUMBER PRIMARY KEY, + name VARCHAR2(255) +); + +CREATE TABLE YBVOYAGER.unique_table ( + id NUMBER PRIMARY KEY, + email VARCHAR2(100), + phone VARCHAR2(100), + address VARCHAR2(255), + CONSTRAINT email_phone_unq UNIQUE (email, phone) +); + +CREATE UNIQUE INDEX YBVOYAGER.unique_address_idx ON YBVOYAGER.unique_table (address); + +CREATE TABLE YBVOYAGER.table1 ( + id NUMBER PRIMARY KEY, + name VARCHAR2(100) +); + +CREATE TABLE YBVOYAGER.table2 ( + id NUMBER PRIMARY KEY, + email VARCHAR2(100) +); + + +CREATE TABLE YBVOYAGER.non_pk1 ( + id NUMBER, + name VARCHAR2(10) +); + +CREATE TABLE YBVOYAGER.non_pk2 ( + id NUMBER, + name VARCHAR2(10) +); \ No newline at end of file diff --git a/yb-voyager/test/containers/test_schemas/postgresql_schema.sql b/yb-voyager/test/containers/test_schemas/postgresql_schema.sql new file mode 100644 index 0000000000..36bda657a5 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/postgresql_schema.sql @@ -0,0 +1 @@ +-- TODO: create source migration user as per User creation steps in docs and use that in tests diff --git a/yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql b/yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql new file mode 100644 index 0000000000..c36ddc5b93 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql @@ -0,0 +1 @@ +-- TODO: create user as per User creation steps in docs and use that in tests diff --git a/yb-voyager/test/containers/testcontainers.go b/yb-voyager/test/containers/testcontainers.go new file mode 100644 index 0000000000..c9fce4b15c --- /dev/null +++ b/yb-voyager/test/containers/testcontainers.go @@ -0,0 +1,143 @@ +package testcontainers + +import ( + "context" + "fmt" + "sync" + + "github.com/samber/lo" + log "github.com/sirupsen/logrus" +) + +// containerRegistry to ensure one container per database(dbtype+version) [Singleton Pattern] +// Limitation - go test spawns different process for running tests of each package, hence the containers won't be shared across packages. +var ( + containerRegistry = make(map[string]TestContainer) + registryMutex sync.Mutex +) + +type TestContainer interface { + Start(ctx context.Context) error + Terminate(ctx context.Context) + GetHostPort() (string, int, error) + GetConfig() ContainerConfig + GetConnectionString() string + /* + TODOs + // Function to run sql script for a specific test case + SetupSqlScript(scriptName string, dbName string) error + + // Add Capability to run multiple versions of a dbtype parallely + */ + ExecuteSqls(sqls ...string) +} + +type ContainerConfig struct { + DBVersion string + User string + Password string + DBName string + Schema string +} + +func NewTestContainer(dbType string, containerConfig *ContainerConfig) TestContainer { + registryMutex.Lock() + defer registryMutex.Unlock() + + // initialise containerConfig struct if nothing is provided + if containerConfig == nil { + containerConfig = &ContainerConfig{} + } + setContainerConfigDefaultsIfNotProvided(dbType, containerConfig) + + // check if container is already created after fetching default configs + containerName := fmt.Sprintf("%s-%s", dbType, containerConfig.DBVersion) + if container, exists := containerRegistry[containerName]; exists { + log.Infof("container '%s' already exists in the registry", containerName) + return container + } + + var testContainer TestContainer + switch dbType { + case POSTGRESQL: + testContainer = &PostgresContainer{ + ContainerConfig: *containerConfig, + } + case YUGABYTEDB: + testContainer = &YugabyteDBContainer{ + ContainerConfig: *containerConfig, + } + case ORACLE: + testContainer = &OracleContainer{ + ContainerConfig: *containerConfig, + } + case MYSQL: + testContainer = &MysqlContainer{ + ContainerConfig: *containerConfig, + } + default: + panic(fmt.Sprintf("unsupported db type '%q' for creating test container\n", dbType)) + } + + containerRegistry[containerName] = testContainer + return testContainer +} + +/* +Challenges in golang for running this a teardown step +1. In golang when you execute go test in the top level folder it executes all the tests one by one. +2. Where each defined package, can have its TestMain() which can control the setup and teardown steps for that package +3. There is no way to run these before/after the tests of first/last package in codebase + +Potential solution: Implement a counter(total=number_of_package) based logic to execute teardown(i.e. TerminateAllContainers() in our case) +Figure out the best solution. + +For now we can rely on TestContainer ryuk(the container repear), which terminates all the containers after the process exits. +But the test framework should have capability of terminating all containers at the end. +*/ +func TerminateAllContainers() { + registryMutex.Lock() + defer registryMutex.Unlock() + + ctx := context.Background() + for name, container := range containerRegistry { + log.Infof("terminating the container '%s'", name) + container.Terminate(ctx) + } +} + +func setContainerConfigDefaultsIfNotProvided(dbType string, config *ContainerConfig) { + // TODO: discuss and decide the default DBVersion values for each dbtype + + switch dbType { + case POSTGRESQL: + config.User = lo.Ternary(config.User == "", "ybvoyager", config.User) + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "11", config.DBVersion) + config.Schema = lo.Ternary(config.Schema == "", "public", config.Schema) + config.DBName = lo.Ternary(config.DBName == "", "postgres", config.DBName) + + case YUGABYTEDB: + config.User = lo.Ternary(config.User == "", "yugabyte", config.User) // ybdb docker doesn't create specified user + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "2.20.7.1-b10", config.DBVersion) + config.Schema = lo.Ternary(config.Schema == "", "public", config.Schema) + config.DBName = lo.Ternary(config.DBName == "", "yugabyte", config.DBName) + + case ORACLE: + config.User = lo.Ternary(config.User == "", "ybvoyager", config.User) + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "21", config.DBVersion) + config.Schema = lo.Ternary(config.Schema == "", "YBVOYAGER", config.Schema) + config.DBName = lo.Ternary(config.DBName == "", "DMS", config.DBName) + + case MYSQL: + config.User = lo.Ternary(config.User == "", "ybvoyager", config.User) + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "8.4", config.DBVersion) + config.DBName = lo.Ternary(config.DBName == "", "dms", config.DBName) + + default: + panic(fmt.Sprintf("unsupported db type '%q' for creating test container\n", dbType)) + } +} diff --git a/yb-voyager/test/containers/yugabytedb_container.go b/yb-voyager/test/containers/yugabytedb_container.go new file mode 100644 index 0000000000..822a7f6945 --- /dev/null +++ b/yb-voyager/test/containers/yugabytedb_container.go @@ -0,0 +1,129 @@ +package testcontainers + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + "github.com/jackc/pgx/v5" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type YugabyteDBContainer struct { + ContainerConfig + container testcontainers.Container +} + +func (yb *YugabyteDBContainer) Start(ctx context.Context) (err error) { + if yb.container != nil { + utils.PrintAndLog("YugabyteDB-%s container already running", yb.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "yugabytedb_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(yugabytedbInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + // this will create a 1 Node RF-1 cluster + req := testcontainers.ContainerRequest{ + Image: fmt.Sprintf("yugabytedb/yugabyte:%s", yb.DBVersion), + ExposedPorts: []string{"5433/tcp", "15433/tcp", "7000/tcp", "9000/tcp", "9042/tcp"}, + Cmd: []string{ + "bin/yugabyted", + "start", + "--daemon=false", + "--ui=false", + "--initial_scripts_dir=/home/yugabyte/initial-scripts", + }, + WaitingFor: wait.ForAll( + wait.ForListeningPort("5433/tcp").WithStartupTimeout(2*time.Minute).WithPollInterval(5*time.Second), + wait.ForLog("Data placement constraint successfully verified").WithStartupTimeout(3*time.Minute).WithPollInterval(1*time.Second), + ), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "/home/yugabyte/initial-scripts/yugabytedb_schema.sql", + FileMode: 0755, + }, + }, + } + + yb.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + printContainerLogs(yb.container) + return err +} + +func (yb *YugabyteDBContainer) Terminate(ctx context.Context) { + if yb == nil { + return + } + + err := yb.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate yugabytedb container: %v", err) + } +} + +func (yb *YugabyteDBContainer) GetHostPort() (string, int, error) { + if yb.container == nil { + return "", -1, fmt.Errorf("yugabytedb container is not started: nil") + } + + ctx := context.Background() + host, err := yb.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for yugabytedb container: %w", err) + } + + port, err := yb.container.MappedPort(ctx, nat.Port(DEFAULT_YB_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for yugabytedb container: %w", err) + } + + return host, port.Int(), nil +} + +func (yb *YugabyteDBContainer) GetConfig() ContainerConfig { + return yb.ContainerConfig +} + +func (yb *YugabyteDBContainer) GetConnectionString() string { + config := yb.GetConfig() + host, port, err := yb.GetHostPort() + if err != nil { + utils.ErrExit("failed to get host port for yugabytedb connection string: %v", err) + } + + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", config.User, config.Password, host, port, config.DBName) +} + +func (yb *YugabyteDBContainer) ExecuteSqls(sqls ...string) { + connStr := yb.GetConnectionString() + conn, err := pgx.Connect(context.Background(), connStr) + if err != nil { + utils.ErrExit("failed to connect postgres for executing sqls: %w", err) + } + defer conn.Close(context.Background()) + + for _, sql := range sqls { + _, err := conn.Exec(context.Background(), sql) + if err != nil { + utils.ErrExit("failed to execute sql '%s': %w", sql, err) + } + } +} diff --git a/yb-voyager/src/testutils/testing_utils.go b/yb-voyager/test/utils/testutils.go similarity index 89% rename from yb-voyager/src/testutils/testing_utils.go rename to yb-voyager/test/utils/testutils.go index 2a10007dee..74c30bc446 100644 --- a/yb-voyager/src/testutils/testing_utils.go +++ b/yb-voyager/test/utils/testutils.go @@ -6,11 +6,14 @@ import ( "os" "reflect" "regexp" + "sort" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" ) type ColumnPropertiesSqlite struct { @@ -201,10 +204,10 @@ func CheckTableStructurePG(t *testing.T, db *sql.DB, schema, table string, expec func checkPrimaryKeyOfTablePG(t *testing.T, db *sql.DB, schema, table string, expectedColumns map[string]ColumnPropertiesPG) { // Validate primary keys queryPrimaryKeys := ` - SELECT conrelid::regclass AS table_name, - conname AS primary_key, + SELECT conrelid::regclass AS table_name, + conname AS primary_key, pg_get_constraintdef(oid) - FROM pg_constraint + FROM pg_constraint WHERE contype = 'p' -- 'p' indicates primary key AND conrelid::regclass::text = $1 ORDER BY conrelid::regclass::text, contype DESC;` @@ -350,3 +353,50 @@ func CheckTableExistencePG(t *testing.T, db *sql.DB, schema string, expectedTabl } } } + +// === assertion helper functions +func AssertEqualStringSlices(t *testing.T, expected, actual []string) { + t.Helper() + if len(expected) != len(actual) { + t.Errorf("Mismatch in slice length. Expected: %v, Actual: %v", expected, actual) + } + + sort.Strings(expected) + sort.Strings(actual) + assert.Equal(t, expected, actual) +} + +func AssertEqualSourceNameSlices(t *testing.T, expected, actual []*sqlname.SourceName) { + SortSourceNames(expected) + SortSourceNames(actual) + assert.Equal(t, expected, actual) +} + +func SortSourceNames(tables []*sqlname.SourceName) { + sort.Slice(tables, func(i, j int) bool { + return tables[i].Qualified.MinQuoted < tables[j].Qualified.MinQuoted + }) +} + +func AssertEqualNameTuplesSlice(t *testing.T, expected, actual []sqlname.NameTuple) { + sortNameTuples(expected) + sortNameTuples(actual) + assert.Equal(t, expected, actual) +} + +func sortNameTuples(tables []sqlname.NameTuple) { + sort.Slice(tables, func(i, j int) bool { + return tables[i].ForOutput() < tables[j].ForOutput() + }) +} + +// waitForDBConnection waits until the database is ready for connections. +func WaitForDBToBeReady(db *sql.DB) error { + for i := 0; i < 12; i++ { + if err := db.Ping(); err == nil { + return nil + } + time.Sleep(5 * time.Second) + } + return fmt.Errorf("database did not become ready in time") +} diff --git a/yb-voyager/testcontainers/testcontainers.go b/yb-voyager/testcontainers/testcontainers.go deleted file mode 100644 index 336d7b2e30..0000000000 --- a/yb-voyager/testcontainers/testcontainers.go +++ /dev/null @@ -1,108 +0,0 @@ -package testcontainers - -import ( - "context" - "database/sql" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const ( - POSTGRESQL = "postgresql" - YUGABYTEDB = "yugabytedb" -) - -func StartDBContainer(ctx context.Context, dbType string) (testcontainers.Container, string, nat.Port, error) { - switch dbType { - case POSTGRESQL: - return startPostgresContainer(ctx) - case YUGABYTEDB: - return startYugabyteDBContainer(ctx) - default: - return nil, "", "", fmt.Errorf("unsupported database type: %s", dbType) - } -} - -func startPostgresContainer(ctx context.Context) (container testcontainers.Container, host string, port nat.Port, err error) { - // Create a PostgreSQL TestContainer - req := testcontainers.ContainerRequest{ - Image: "postgres:latest", // Use the latest PostgreSQL image - ExposedPorts: []string{"5432/tcp"}, - Env: map[string]string{ - "POSTGRES_USER": "testuser", // Set PostgreSQL username - "POSTGRES_PASSWORD": "testpassword", // Set PostgreSQL password - "POSTGRES_DB": "testdb", // Set PostgreSQL database name - }, - WaitingFor: wait.ForListeningPort("5432/tcp").WithStartupTimeout(30 * 1e9), // Wait for PostgreSQL to be ready - } - - // Start the container - pgContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - - if err != nil { - return nil, "", "", err - } - - // Get the container's host and port - host, err = pgContainer.Host(ctx) - if err != nil { - return nil, "", "", err - } - - port, err = pgContainer.MappedPort(ctx, "5432") - if err != nil { - return nil, "", "", err - } - - return pgContainer, host, port, nil -} - -func startYugabyteDBContainer(ctx context.Context) (container testcontainers.Container, host string, port nat.Port, err error) { - // Create a YugabyteDB TestContainer - req := testcontainers.ContainerRequest{ - Image: "yugabytedb/yugabyte:latest", - ExposedPorts: []string{"5433/tcp"}, - WaitingFor: wait.ForListeningPort("5433/tcp"), - Cmd: []string{"bin/yugabyted", "start", "--daemon=false"}, - } - - // Start the container - ybContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - return nil, "", "", err - } - - // Get the container's host and port - host, err = ybContainer.Host(ctx) - if err != nil { - return nil, "", "", err - } - - port, err = ybContainer.MappedPort(ctx, "5433") - if err != nil { - return nil, "", "", err - } - - return ybContainer, host, port, nil -} - -// waitForDBConnection waits until the database is ready for connections. -func WaitForDBToBeReady(db *sql.DB) error { - for i := 0; i < 12; i++ { - if err := db.Ping(); err == nil { - return nil - } - time.Sleep(5 * time.Second) - } - return fmt.Errorf("database did not become ready in time") -}