diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml index 81c7ff2ae8..1dfda76a62 100644 --- a/.github/workflows/mysql-migtests.yml +++ b/.github/workflows/mysql-migtests.yml @@ -137,6 +137,7 @@ jobs: # Placeholder for now so that a basic test can run - name: Create the live migration user + if: always() run: | mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';' diff --git a/migtests/lib/yb.py b/migtests/lib/yb.py index 854de33fe4..c9351928d8 100644 --- a/migtests/lib/yb.py +++ b/migtests/lib/yb.py @@ -52,7 +52,8 @@ def new_source_db(): def verify_colocation(tgt, source_db_type): print("Verifying the colocation of the tables") - json_file = "export-dir/assessment/reports/migration_assessment_report.json" + export_dir = os.getenv("EXPORT_DIR", "export-dir") + json_file = f"{export_dir}/assessment/reports/migration_assessment_report.json" sharded_tables, colocated_tables = fetch_sharded_and_colocated_tables(json_file) diff --git a/migtests/scripts/add-pk-from-alter-to-create b/migtests/scripts/add-pk-from-alter-to-create index 83d0ef3eb2..77f1ab5b7b 100755 --- a/migtests/scripts/add-pk-from-alter-to-create +++ b/migtests/scripts/add-pk-from-alter-to-create @@ -5,7 +5,7 @@ import os import re import shutil -table_file_path = os.getenv('TEST_DIR')+'/export-dir/schema/tables/table.sql' +table_file_path = os.getenv('EXPORT_DIR')+'/schema/tables/table.sql' #copy the table_file_path to table_file_path+'.old' src = table_file_path diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 18f3ff9a28..49275ad445 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -134,7 +134,8 @@ grant_user_permission_oracle(){ */ GRANT FLASHBACK ANY TABLE TO ybvoyager; EOF - run_sqlplus_as_sys ${db_name} "oracle-inputs.sql" + run_sqlplus_as_sys ${db_name} "oracle-inputs.sql" + rm oracle-inputs.sql } @@ -149,6 +150,8 @@ EOF run_sqlplus_as_sys ${pdb_name} "create-pdb-tablespace.sql" cp ${SCRIPTS}/oracle/live-grants.sql oracle-inputs.sql run_sqlplus_as_sys ${cdb_name} "oracle-inputs.sql" + rm create-pdb-tablespace.sql + rm oracle-inputs.sql } grant_permissions_for_live_migration_pg() { @@ -191,7 +194,7 @@ run_sqlplus_as_sys() { run_sqlplus_as_schema_owner() { db_name=$1 sql=$2 - conn_string="${SOURCE_DB_USER_SCHEMA_OWNER}/${SOURCE_DB_USER_SCHEMA_OWNER_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}" + conn_string="${SOURCE_DB_SCHEMA}/${SOURCE_DB_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}" echo exit | sqlplus -f "${conn_string}" @"${sql}" } @@ -623,16 +626,6 @@ get_value_from_msr(){ echo $val } -create_ff_schema(){ - db_name=$1 - - cat > create-ff-schema.sql << EOF - CREATE USER FF_SCHEMA IDENTIFIED BY "password"; - GRANT all privileges to FF_SCHEMA; -EOF - run_sqlplus_as_sys ${db_name} "create-ff-schema.sql" -} - set_replica_identity(){ db_schema=$1 cat > alter_replica_identity.sql < $TEMP_SCRIPT + + run_sqlplus_as_sys ${SOURCE_DB_NAME} $TEMP_SCRIPT + + # Clean up the temporary file after execution + rm -f $TEMP_SCRIPT elif [ "${SOURCE_DB_TYPE}" = "postgresql" ]; then conn_string="postgresql://${SOURCE_DB_ADMIN_USER}:${SOURCE_DB_ADMIN_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${SOURCE_DB_NAME}" psql "${conn_string}" -v voyager_user="${SOURCE_DB_USER}" -v schema_list="${SOURCE_DB_SCHEMA}" -v replication_group='replication_group' -v is_live_migration=1 -v is_live_migration_fall_back=1 -f /opt/yb-voyager/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql @@ -1044,3 +1046,60 @@ cutover_to_target() { yb-voyager initiate cutover to target ${args} $* } + +create_source_db() { + source_db=$1 + case ${SOURCE_DB_TYPE} in + postgresql) + run_psql postgres "DROP DATABASE IF EXISTS ${source_db};" + run_psql postgres "CREATE DATABASE ${source_db};" + ;; + mysql) + run_mysql mysql "DROP DATABASE IF EXISTS ${source_db};" + run_mysql mysql "CREATE DATABASE ${source_db};" + ;; + oracle) + cat > create-oracle-schema.sql << EOF + CREATE USER ${source_db} IDENTIFIED BY "password"; + GRANT all privileges to ${source_db}; +EOF + run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-oracle-schema.sql" + rm create-oracle-schema.sql + ;; + *) + echo "ERROR: Source DB not created for ${SOURCE_DB_TYPE}" + exit 1 + ;; + esac +} + +normalize_and_export_vars() { + local test_suffix=$1 + + # Normalize TEST_NAME + # Keeping the full name for PG and MySQL to test out large schema/export dir names + export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" + + # Set EXPORT_DIR + export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_${test_suffix}_export-dir"} + if [ -n "${SOURCE_DB_SSL_MODE}" ]; then + EXPORT_DIR="${EXPORT_DIR}_ssl" + fi + + # Set database-specific variables + case "${SOURCE_DB_TYPE}" in + postgresql|mysql) + export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_${test_suffix}"} + ;; + oracle) + # Limit schema name to 10 characters for Oracle/Debezium due to 30 character limit + # Since test_suffix is the unique identifying factor, we need to add it post all the normalization + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_${test_suffix}"} + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} + ;; + *) + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 + ;; + esac +} diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index fc4fb886a5..bac1a1e968 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -18,9 +18,7 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} export QUEUE_SEGMENT_MAX_BYTES=400 - export PYTHONPATH="${REPO_ROOT}/migtests/lib" # Order of env.sh import matters. @@ -37,10 +35,12 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -source ${SCRIPTS}/yugabytedb/env.sh - source ${SCRIPTS}/functions.sh +normalize_and_export_vars "fallb" + +source ${SCRIPTS}/yugabytedb/env.sh + main() { echo "Deleting the parent export-dir present in the test directory" @@ -56,6 +56,10 @@ main() { pushd ${TEST_DIR} step "Initialise source database." + if [ "${SOURCE_DB_TYPE}" = "oracle" ] + then + create_source_db ${SOURCE_DB_SCHEMA} + fi ./init-db step "Grant source database user permissions for live migration" @@ -169,7 +173,11 @@ main() { import_schema --post-snapshot-import true --refresh-mviews=true step "Run snapshot validations." - "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true' + "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true' || { + tail_log_file "yb-voyager-import-data.log" + tail_log_file "yb-voyager-export-data-from-source.log" + exit 1 + } step "Inserting new events" run_sql_file source_delta.sql @@ -264,7 +272,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index 175630f70a..54dbe7bd78 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -18,7 +18,6 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} export PYTHONPATH="${REPO_ROOT}/migtests/lib" export PATH="${PATH}:/usr/lib/oracle/21/client64/bin" @@ -38,12 +37,14 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi +source ${SCRIPTS}/functions.sh + +normalize_and_export_vars "fallf" + source ${SCRIPTS}/${SOURCE_DB_TYPE}/ff_env.sh source ${SCRIPTS}/yugabytedb/env.sh -source ${SCRIPTS}/functions.sh - main() { echo "Deleting the parent export-dir present in the test directory" @@ -62,7 +63,9 @@ main() { if [ "${SOURCE_DB_TYPE}" = "oracle" ] then - create_ff_schema ${SOURCE_REPLICA_DB_NAME} + create_source_db ${SOURCE_DB_SCHEMA} + # TODO: Add dynamic Fall Forward schema creation. Currently using the same name for all tests. + create_source_db ${SOURCE_REPLICA_DB_SCHEMA} run_sqlplus_as_sys ${SOURCE_REPLICA_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql fi ./init-db @@ -206,7 +209,12 @@ main() { import_schema --post-snapshot-import true --refresh-mviews true step "Run snapshot validations." - "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false' + "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false' || { + tail_log_file "yb-voyager-import-data.log" + tail_log_file "yb-voyager-export-data-from-source.log" + tail_log_file "yb-voyager-import-data-to-source-replica.log" + exit 1 + } step "Inserting new events to source" run_sql_file source_delta.sql @@ -286,7 +294,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/live-migration-run-test.sh b/migtests/scripts/live-migration-run-test.sh index 1594f1449c..98f99e6beb 100755 --- a/migtests/scripts/live-migration-run-test.sh +++ b/migtests/scripts/live-migration-run-test.sh @@ -18,7 +18,6 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} export QUEUE_SEGMENT_MAX_BYTES=400 export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -37,10 +36,12 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -source ${SCRIPTS}/yugabytedb/env.sh - source ${SCRIPTS}/functions.sh +normalize_and_export_vars "live" + +source ${SCRIPTS}/yugabytedb/env.sh + main() { echo "Deleting the parent export-dir present in the test directory" @@ -56,6 +57,10 @@ main() { pushd ${TEST_DIR} step "Initialise source database." + if [ "${SOURCE_DB_TYPE}" = "oracle" ] + then + create_source_db ${SOURCE_DB_SCHEMA} + fi ./init-db step "Grant source database user permissions for live migration" @@ -218,7 +223,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-schema-migration.sh b/migtests/scripts/run-schema-migration.sh index 0f37bab501..8a142cf152 100755 --- a/migtests/scripts/run-schema-migration.sh +++ b/migtests/scripts/run-schema-migration.sh @@ -139,7 +139,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-test-export-data.sh b/migtests/scripts/run-test-export-data.sh index 04e84bdd22..dcbeecd362 100755 --- a/migtests/scripts/run-test-export-data.sh +++ b/migtests/scripts/run-test-export-data.sh @@ -74,7 +74,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index f0d9408cf8..e40cc60d10 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -17,7 +17,6 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -33,11 +32,16 @@ then else source ${TEST_DIR}/env.sh fi + source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh -source ${SCRIPTS}/yugabytedb/env.sh + source ${SCRIPTS}/functions.sh +normalize_and_export_vars "offline" + +source ${SCRIPTS}/yugabytedb/env.sh + main() { echo "Deleting the parent export-dir present in the test directory" rm -rf ${EXPORT_DIR} @@ -52,6 +56,18 @@ main() { pushd ${TEST_DIR} step "Initialise source database." + if [[ "${SKIP_DB_CREATION}" != "true" ]]; then + if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then + create_source_db "${SOURCE_DB_NAME}" + elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then + create_source_db "${SOURCE_DB_SCHEMA}" + else + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 + fi + else + echo "Skipping database creation as SKIP_DB_CREATION is set to true." + fi ./init-db step "Grant source database user permissions" @@ -190,7 +206,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-validate-assessment-report.sh b/migtests/scripts/run-validate-assessment-report.sh index 5cdac1d392..75e3d8875c 100755 --- a/migtests/scripts/run-validate-assessment-report.sh +++ b/migtests/scripts/run-validate-assessment-report.sh @@ -94,7 +94,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" } main diff --git a/migtests/scripts/run-validate-bulk-assessment-report.sh b/migtests/scripts/run-validate-bulk-assessment-report.sh index 05b43c87ab..ac67773b1d 100755 --- a/migtests/scripts/run-validate-bulk-assessment-report.sh +++ b/migtests/scripts/run-validate-bulk-assessment-report.sh @@ -54,6 +54,7 @@ main() { echo "Assigning permissions to the export-dir to execute init-db, cleanup-db scripts" chmod +x ${TEST_DIR}/init-db + chmod +x ${TEST_DIR}/cleanup-db step "START: ${TEST_NAME}" print_env @@ -109,6 +110,8 @@ main() { fi step "Clean up" + ./cleanup-db + rm -rf "${BULK_ASSESSMENT_DIR}" } diff --git a/migtests/scripts/yugabytedb/env.sh b/migtests/scripts/yugabytedb/env.sh index a288b14ffc..10b9e70e5f 100644 --- a/migtests/scripts/yugabytedb/env.sh +++ b/migtests/scripts/yugabytedb/env.sh @@ -7,7 +7,10 @@ export TARGET_DB_ADMIN_PASSWORD=${TARGET_DB_ADMIN_PASSWORD:-''} export TARGET_DB_SCHEMA=${TARGET_DB_SCHEMA:-'public'} # The PG driver, used to connect to YB, is case-sensitive about database name. -if [ "${TARGET_DB_NAME}" == "" ] -then - export TARGET_DB_NAME=`echo ${SOURCE_DB_NAME} | tr [A-Z] [a-z]` +if [ "${TARGET_DB_NAME}" == "" ]; then + if [ "${SOURCE_DB_TYPE}" == "oracle" ]; then + export TARGET_DB_NAME=$(echo ${SOURCE_DB_SCHEMA} | tr [A-Z] [a-z]) + else + export TARGET_DB_NAME=$(echo ${SOURCE_DB_NAME} | tr [A-Z] [a-z]) + fi fi diff --git a/migtests/tests/mysql/basic-live-test/env.sh b/migtests/tests/mysql/basic-live-test/env.sh index dfe82e6660..62ddf80b40 100644 --- a/migtests/tests/mysql/basic-live-test/env.sh +++ b/migtests/tests/mysql/basic-live-test/env.sh @@ -1,2 +1 @@ -export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"live_test"} \ No newline at end of file +export SOURCE_DB_TYPE="mysql" \ No newline at end of file diff --git a/migtests/tests/mysql/chinook/Chinook.sql b/migtests/tests/mysql/chinook/Chinook.sql index c761f0d041..f3309ff09b 100644 --- a/migtests/tests/mysql/chinook/Chinook.sql +++ b/migtests/tests/mysql/chinook/Chinook.sql @@ -15829,4 +15829,3 @@ INSERT INTO `PlaylistTrack` (`PlaylistId`, `TrackId`) VALUES (17, 2096); INSERT INTO `PlaylistTrack` (`PlaylistId`, `TrackId`) VALUES (17, 3290); INSERT INTO `PlaylistTrack` (`PlaylistId`, `TrackId`) VALUES (18, 597); - diff --git a/migtests/tests/mysql/chinook/env.sh b/migtests/tests/mysql/chinook/env.sh index ff571a1c0e..c640e171cd 100644 --- a/migtests/tests/mysql/chinook/env.sh +++ b/migtests/tests/mysql/chinook/env.sh @@ -1,3 +1,5 @@ export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"Chinook"} -export TARGET_DB_SCHEMA="TEST_SAMPLE_CHINOOK" \ No newline at end of file +export SOURCE_DB_NAME="Chinook" +export TARGET_DB_SCHEMA="TEST_SAMPLE_CHINOOK" +export SKIP_DB_CREATION="true" + diff --git a/migtests/tests/mysql/datatypes/env.sh b/migtests/tests/mysql/datatypes/env.sh index 1d4759f090..aedc7cffd6 100644 --- a/migtests/tests/mysql/datatypes/env.sh +++ b/migtests/tests/mysql/datatypes/env.sh @@ -1,2 +1 @@ export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"datatypes"} diff --git a/migtests/tests/mysql/sakila/env.sh b/migtests/tests/mysql/sakila/env.sh index 620640ea43..5fa4dc4acb 100644 --- a/migtests/tests/mysql/sakila/env.sh +++ b/migtests/tests/mysql/sakila/env.sh @@ -1,2 +1,3 @@ export SOURCE_DB_TYPE="mysql" export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"sakila"} +export SKIP_DB_CREATION="true" \ No newline at end of file diff --git a/migtests/tests/mysql/sakila/init-db b/migtests/tests/mysql/sakila/init-db index f6eb9c5bd6..b783d909f1 100755 --- a/migtests/tests/mysql/sakila/init-db +++ b/migtests/tests/mysql/sakila/init-db @@ -23,4 +23,4 @@ run_mysql ${SOURCE_DB_NAME} "SOURCE sakila-db/sakila-data.sql;" run_mysql ${SOURCE_DB_NAME} "ALTER TABLE address DROP COLUMN location;" echo "Check source database." -run_mysql ${SOURCE_DB_NAME} "SELECT count(*) FROM payment;" +run_mysql ${SOURCE_DB_NAME} "SELECT count(*) FROM payment;" \ No newline at end of file diff --git a/migtests/tests/oracle/basic-live-test/env.sh b/migtests/tests/oracle/basic-live-test/env.sh index 722796b6a8..998f6081f6 100644 --- a/migtests/tests/oracle/basic-live-test/env.sh +++ b/migtests/tests/oracle/basic-live-test/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"} export TARGET_DB_SCHEMA="test_schema" diff --git a/migtests/tests/oracle/bulk-assessment-test/cleanup-db b/migtests/tests/oracle/bulk-assessment-test/cleanup-db new file mode 100644 index 0000000000..07ebc07a8e --- /dev/null +++ b/migtests/tests/oracle/bulk-assessment-test/cleanup-db @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e +set -x + +source ${SCRIPTS}/functions.sh + +for schema in TEST_SCHEMA TEST_SCHEMA2; do + export SOURCE_DB_SCHEMA=${schema} + run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema +done \ No newline at end of file diff --git a/migtests/tests/oracle/bulk-assessment-test/init-db b/migtests/tests/oracle/bulk-assessment-test/init-db index b8acab5175..0cd3340a0d 100755 --- a/migtests/tests/oracle/bulk-assessment-test/init-db +++ b/migtests/tests/oracle/bulk-assessment-test/init-db @@ -5,21 +5,20 @@ set -x source ${SCRIPTS}/functions.sh -echo "Deleting existing data in source database" -run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema +export SOURCE_DB_SCHEMA="TEST_SCHEMA" + +create_source_db ${SOURCE_DB_SCHEMA} + +./cleanup-db echo "Initialising source database 1 & inserting data" run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../assessment-report-test/oracle_assessment_report.sql echo "Initialising source database 2 & inserting data" -cat > create-schema2.sql << EOF - CREATE USER TEST_SCHEMA2 identified by "password"; - GRANT ALL PRIVILEGES TO TEST_SCHEMA2; -EOF - run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-schema2.sql" +export SOURCE_DB_SCHEMA="TEST_SCHEMA2" -export SOURCE_DB_USER_SCHEMA_OWNER="TEST_SCHEMA2" +create_source_db ${SOURCE_DB_SCHEMA} run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../partitions/partition_schema.sql run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../partitions/partition_data.sql diff --git a/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh b/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh index 4f724929c1..429f19fb3a 100644 --- a/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh +++ b/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"} export TARGET_DB_SCHEMA="test_schema2" diff --git a/migtests/tests/oracle/co-db/env.sh b/migtests/tests/oracle/co-db/env.sh index f0f431a669..15d6f15d90 100644 --- a/migtests/tests/oracle/co-db/env.sh +++ b/migtests/tests/oracle/co-db/env.sh @@ -1,4 +1,5 @@ export SOURCE_DB_TYPE="oracle" export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"CO"} export TARGET_DB_NAME=${TARGET_DB_NAME:-"co_db_test"} +export SKIP_DB_CREATION="true" diff --git a/migtests/tests/oracle/partitions/env.sh b/migtests/tests/oracle/partitions/env.sh index 05489d8a99..515cf7abbe 100644 --- a/migtests/tests/oracle/partitions/env.sh +++ b/migtests/tests/oracle/partitions/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"partition_test"} diff --git a/migtests/tests/oracle/sequences/env.sh b/migtests/tests/oracle/sequences/env.sh index ef3b4aa439..0357fe1d0e 100644 --- a/migtests/tests/oracle/sequences/env.sh +++ b/migtests/tests/oracle/sequences/env.sh @@ -1,3 +1 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"oracle_sequences_test"} diff --git a/migtests/tests/oracle/sequences/sequence_schema.sql b/migtests/tests/oracle/sequences/sequence_schema.sql index 1baac1c7fc..310edc2501 100644 --- a/migtests/tests/oracle/sequences/sequence_schema.sql +++ b/migtests/tests/oracle/sequences/sequence_schema.sql @@ -61,7 +61,7 @@ CREATE TABLE empty_identity_def ( description VARCHAR2(100) not null ); - drop table "Case_Sensitive_always"; +drop table "Case_Sensitive_always"; CREATE TABLE "Case_Sensitive_always" ( id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE PRIMARY KEY, diff --git a/migtests/tests/oracle/sequences/validateAfterChanges b/migtests/tests/oracle/sequences/validateAfterChanges index cc6c2705e3..95f0832da3 100755 --- a/migtests/tests/oracle/sequences/validateAfterChanges +++ b/migtests/tests/oracle/sequences/validateAfterChanges @@ -166,14 +166,14 @@ def migration_completed_checks_yb(): def migration_completed_checks_ff(): print("Running tests on Oracle source replica") global db_schema - db_schema = os.environ.get("SOURCE_REPLICA_DB_SCHEMA") + db_schema = os.environ.get("SOURCE_REPLICA_DB_SCHEMA").upper() change_expected_values_ff_fb() oracle.run_checks(migration_completed_checks, db_type="source_replica") def migration_completed_checks_fb(): print("Running tests on Oracle source") global db_schema - db_schema = os.environ.get("SOURCE_DB_SCHEMA") + db_schema = os.environ.get("SOURCE_DB_SCHEMA").upper() change_expected_values_ff_fb() oracle.run_checks(migration_completed_checks, db_type="source") diff --git a/migtests/tests/oracle/unique-key-conflicts-test/env.sh b/migtests/tests/oracle/unique-key-conflicts-test/env.sh index 722796b6a8..998f6081f6 100644 --- a/migtests/tests/oracle/unique-key-conflicts-test/env.sh +++ b/migtests/tests/oracle/unique-key-conflicts-test/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"} export TARGET_DB_SCHEMA="test_schema" diff --git a/migtests/tests/pg/datatypes/env.sh b/migtests/tests/pg/datatypes/env.sh index 328c0615f3..3606e020cc 100644 --- a/migtests/tests/pg/datatypes/env.sh +++ b/migtests/tests/pg/datatypes/env.sh @@ -1,3 +1,2 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"pg_datatypes"} export SOURCE_DB_SCHEMA="public" diff --git a/migtests/tests/pg/partitions-with-indexes/env.sh b/migtests/tests/pg/partitions-with-indexes/env.sh index 6b22305323..7f216dc8da 100644 --- a/migtests/tests/pg/partitions-with-indexes/env.sh +++ b/migtests/tests/pg/partitions-with-indexes/env.sh @@ -1,3 +1,3 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions"} +export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions_with_index"} export SOURCE_DB_SCHEMA="public,p1,p2" diff --git a/migtests/tests/pg/partitions/env.sh b/migtests/tests/pg/partitions/env.sh index 9ba7757697..bbca0e5017 100644 --- a/migtests/tests/pg/partitions/env.sh +++ b/migtests/tests/pg/partitions/env.sh @@ -1,4 +1,3 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions"} export SOURCE_DB_SCHEMA="public,p1,p2" export MOVE_PK_FROM_ALTER_TO_CREATE="true" \ No newline at end of file diff --git a/migtests/tests/pg/partitions/fix-schema b/migtests/tests/pg/partitions/fix-schema index 57007db3f5..8a341cc602 100755 --- a/migtests/tests/pg/partitions/fix-schema +++ b/migtests/tests/pg/partitions/fix-schema @@ -3,15 +3,15 @@ set -e set -x -sed -i 's/p2\.boston/p2.boston_region/g' $TEST_DIR/export-dir/schema/tables/table.sql -sed -i 's/p2\.london/p2.london_region/g' $TEST_DIR/export-dir/schema/tables/table.sql -sed -i 's/p2\.sydney/p2.sydney_region/g' $TEST_DIR/export-dir/schema/tables/table.sql +sed -i 's/p2\.boston/p2.boston_region/g' ${EXPORT_DIR}/schema/tables/table.sql +sed -i 's/p2\.london/p2.london_region/g' ${EXPORT_DIR}/schema/tables/table.sql +sed -i 's/p2\.sydney/p2.sydney_region/g' ${EXPORT_DIR}/schema/tables/table.sql -sed -i 's/p2\.boston/p2.boston_region/g' $TEST_DIR/export-dir/schema/tables/INDEXES_table.sql -sed -i 's/p2\.london/p2.london_region/g' $TEST_DIR/export-dir/schema/tables/INDEXES_table.sql -sed -i 's/p2\.sydney/p2.sydney_region/g' $TEST_DIR/export-dir/schema/tables/INDEXES_table.sql +sed -i 's/p2\.boston/p2.boston_region/g' ${EXPORT_DIR}/schema/tables/INDEXES_table.sql +sed -i 's/p2\.london/p2.london_region/g' ${EXPORT_DIR}/schema/tables/INDEXES_table.sql +sed -i 's/p2\.sydney/p2.sydney_region/g' ${EXPORT_DIR}/schema/tables/INDEXES_table.sql # Added so that the validations work(to check the recommendations in target YB applied or not) -sed -i 's/p2\.boston/p2.boston_region/g' $TEST_DIR/export-dir/assessment/reports/migration_assessment_report.json -sed -i 's/p2\.london/p2.london_region/g' $TEST_DIR/export-dir/assessment/reports/migration_assessment_report.json -sed -i 's/p2\.sydney/p2.sydney_region/g' $TEST_DIR/export-dir/assessment/reports/migration_assessment_report.json \ No newline at end of file +sed -i 's/p2\.boston/p2.boston_region/g' ${EXPORT_DIR}/assessment/reports/migration_assessment_report.json +sed -i 's/p2\.london/p2.london_region/g' ${EXPORT_DIR}/assessment/reports/migration_assessment_report.json +sed -i 's/p2\.sydney/p2.sydney_region/g' ${EXPORT_DIR}/assessment/reports/migration_assessment_report.json \ No newline at end of file diff --git a/migtests/tests/pg/unique-key-conflicts-test/env.sh b/migtests/tests/pg/unique-key-conflicts-test/env.sh index f812fc984c..4615850a1d 100644 --- a/migtests/tests/pg/unique-key-conflicts-test/env.sh +++ b/migtests/tests/pg/unique-key-conflicts-test/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"unique_key_conflict_cases"} export SOURCE_DB_SCHEMA="public,non_public" -export SOURCE_REPLICA_DB_NAME="unique_key_conflict_cases_replica" \ No newline at end of file diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh index 7550d8f765..5f30e93410 100755 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh @@ -269,7 +269,7 @@ main() { run_command "$ora2pg_cmd" done - ora2pg_report_cmd="ora2pg -t show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet --quiet > $assessment_metadata_dir/schema/ora2pg_report.csv" + ora2pg_report_cmd="ora2pg -t show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet --quiet > $assessment_metadata_dir/schema/ora2pg_report.csv" log "INFO" "executing ora2pg command for report: $ora2pg_report_cmd" run_command "$ora2pg_report_cmd"