From 221383febb897b3c4702ab3ef912b41150244add Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 15 Oct 2024 15:08:06 +0000 Subject: [PATCH 1/8] Added large file test --- migtests/scripts/functions.sh | 2 +- migtests/tests/import-file/run-import-file-test | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 217b3eb3c5..92679d76e4 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -460,7 +460,7 @@ import_data_file() { --target-db-name ${TARGET_DB_NAME} \ --disable-pb true \ --send-diagnostics=false \ - --parallel-jobs 3 \ + --parallel-jobs 1 \ $* || { cat ${EXPORT_DIR}/metainfo/dataFileDescriptor.json exit 1 diff --git a/migtests/tests/import-file/run-import-file-test b/migtests/tests/import-file/run-import-file-test index a71ad90a25..470a915b82 100755 --- a/migtests/tests/import-file/run-import-file-test +++ b/migtests/tests/import-file/run-import-file-test @@ -239,6 +239,14 @@ main() { step "Import data file from GCS (csv): t1_quote_escape_char1.csv -> gcs_quote_escape_char1" import_data_file --data-dir "gs://voyager-automation-data" --format csv --delimiter '|' \ --file-table-map "t1_quote_escape_char1.csv:gcs_quote_escape_char1" --quote-char="'" --escape-char="'" + + + if [ "${RUN_LARGE_IMPORT_DATA_FILE_TEST}" = true ] ; then + + step "Run large sized import data file test" + import_data_file --data-dir "s3://yb-voyager-test-data" --delimiter "\t" --format "text" --file-table-map "accounts_350m_data.sql:accounts" --start-clean true + + fi step "Run validations." "${TEST_DIR}/validate" From da07ae637885749c4d84323000ce73699eb2d7b1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 15 Oct 2024 18:20:12 +0000 Subject: [PATCH 2/8] Test --- migtests/tests/import-file/run-import-file-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migtests/tests/import-file/run-import-file-test b/migtests/tests/import-file/run-import-file-test index 470a915b82..ec89fbe96a 100755 --- a/migtests/tests/import-file/run-import-file-test +++ b/migtests/tests/import-file/run-import-file-test @@ -244,7 +244,7 @@ main() { if [ "${RUN_LARGE_IMPORT_DATA_FILE_TEST}" = true ] ; then step "Run large sized import data file test" - import_data_file --data-dir "s3://yb-voyager-test-data" --delimiter "\t" --format "text" --file-table-map "accounts_350m_data.sql:accounts" --start-clean true + import_data_file --data-dir "s3://yb-voyager-test-data" --delimiter "\t" --format "text" --file-table-map "accounts_350m_data.sql:accounts" --start-clean true --yes fi From 1074d9a093bca304788d25fb30d9d56f76950fbb Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 15 Oct 2024 18:35:43 +0000 Subject: [PATCH 3/8] Added separate table --- migtests/tests/import-file/run-import-file-test | 2 +- migtests/tests/import-file/schema.sql | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/migtests/tests/import-file/run-import-file-test b/migtests/tests/import-file/run-import-file-test index ec89fbe96a..971bcc47ad 100755 --- a/migtests/tests/import-file/run-import-file-test +++ b/migtests/tests/import-file/run-import-file-test @@ -244,7 +244,7 @@ main() { if [ "${RUN_LARGE_IMPORT_DATA_FILE_TEST}" = true ] ; then step "Run large sized import data file test" - import_data_file --data-dir "s3://yb-voyager-test-data" --delimiter "\t" --format "text" --file-table-map "accounts_350m_data.sql:accounts" --start-clean true --yes + import_data_file --data-dir "s3://yb-voyager-test-data" --delimiter "\t" --format "text" --file-table-map "accounts_350m_data.sql:accounts_large" --start-clean true --yes fi diff --git a/migtests/tests/import-file/schema.sql b/migtests/tests/import-file/schema.sql index 06be909fb9..eb82d2e9cb 100644 --- a/migtests/tests/import-file/schema.sql +++ b/migtests/tests/import-file/schema.sql @@ -68,6 +68,19 @@ CREATE TABLE public.accounts ( PRIMARY KEY (block, address) ); +CREATE TABLE public.accounts_large ( + block bigint NOT NULL, + address text NOT NULL, + dc_balance bigint DEFAULT 0 NOT NULL, + dc_nonce bigint DEFAULT 0 NOT NULL, + security_balance bigint DEFAULT 0 NOT NULL, + security_nonce bigint DEFAULT 0 NOT NULL, + balance bigint DEFAULT 0 NOT NULL, + nonce bigint DEFAULT 0 NOT NULL, + staked_balance bigint, + PRIMARY KEY (block, address) +); + create table t1_quote_char (i int, j timestamp, k bigint, l varchar(30)); create table t1_quote_escape_char1 (i int, j timestamp, k bigint, l varchar(30)); From f5f450a02ec4726d991db7fa63a80690cf98b8e0 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 16 Oct 2024 02:30:47 +0000 Subject: [PATCH 4/8] Added the vaalidation for accounts_large --- migtests/tests/import-file/validate | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/migtests/tests/import-file/validate b/migtests/tests/import-file/validate index cae8cbb458..3dd55f9674 100755 --- a/migtests/tests/import-file/validate +++ b/migtests/tests/import-file/validate @@ -51,7 +51,12 @@ EXPECTED = { "gcs_csv_with_header": 3, "gcs_volume": 1000000, "gcs_quote_escape_char1": 3, + "accounts_large": 0 } + +if os.getenv("RUN_LARGE_IMPORT_DATA_FILE_TEST") == "true": + EXPECTED["accounts_large"] = 350000000 + EXPECTED_NON_PUBLIC = { "smsa": 60, } From 52c0675034adbaf23839f928d090bf8b5682db6b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 16 Oct 2024 05:51:38 +0000 Subject: [PATCH 5/8] Added the vaalidation for accounts_large --- migtests/tests/import-file/validate | 1 + 1 file changed, 1 insertion(+) diff --git a/migtests/tests/import-file/validate b/migtests/tests/import-file/validate index 3dd55f9674..9932be9409 100755 --- a/migtests/tests/import-file/validate +++ b/migtests/tests/import-file/validate @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import yb +import os def main(): yb.run_checks(file_import_done_checks) From c2bfd34aaea3d0eb9e33fd9567643764c8456b7a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 16 Oct 2024 13:13:17 +0000 Subject: [PATCH 6/8] Removed parallel jobs from import data file tests --- migtests/scripts/functions.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 92679d76e4..c37023e7d4 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -460,7 +460,6 @@ import_data_file() { --target-db-name ${TARGET_DB_NAME} \ --disable-pb true \ --send-diagnostics=false \ - --parallel-jobs 1 \ $* || { cat ${EXPORT_DIR}/metainfo/dataFileDescriptor.json exit 1 From d79e9312a0856ff13acd27bbbc09beba77491869 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 16 Oct 2024 14:34:59 +0000 Subject: [PATCH 7/8] Test using adaptive parallelism --- migtests/scripts/functions.sh | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index c37023e7d4..93efc108f6 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -451,21 +451,27 @@ import_data_to_source_replica() { } import_data_file() { - yb-voyager import data file --export-dir ${EXPORT_DIR} \ - --target-db-host ${TARGET_DB_HOST} \ - --target-db-port ${TARGET_DB_PORT} \ - --target-db-user ${TARGET_DB_USER} \ - --target-db-password ${TARGET_DB_PASSWORD:-''} \ - --target-db-schema ${TARGET_DB_SCHEMA:-''} \ - --target-db-name ${TARGET_DB_NAME} \ - --disable-pb true \ - --send-diagnostics=false \ - $* || { - cat ${EXPORT_DIR}/metainfo/dataFileDescriptor.json - exit 1 - } + args=" + --export-dir ${EXPORT_DIR} + --target-db-host ${TARGET_DB_HOST} + --target-db-port ${TARGET_DB_PORT} + --target-db-user ${TARGET_DB_USER} + --target-db-password ${TARGET_DB_PASSWORD:-''} + --target-db-schema ${TARGET_DB_SCHEMA:-''} + --target-db-name ${TARGET_DB_NAME} + --disable-pb true + --send-diagnostics=false + " + + # Check if RUN_USING_ADAPTIVE_PARALLELISM is true + if [ "${RUN_USING_ADAPTIVE_PARALLELISM}" = "true" ]; then + args="${args} --enable-adaptive-parallelism true" + fi + + yb-voyager import data file ${args} $* } + archive_changes() { ENABLE=$(shuf -i 0-1 -n 1) echo "archive changes ENABLE=${ENABLE}" From 894e93b881414621b57d3cd3dbf432cf7f6d2f2a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 17 Oct 2024 09:48:49 +0000 Subject: [PATCH 8/8] Added adaptive parallelism to import data as well --- migtests/scripts/functions.sh | 5 +++++ migtests/scripts/live-migration-fallf-run-test.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 93efc108f6..aed7ebea1e 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -422,6 +422,11 @@ import_data() { args="${args} --exclude-table-list-file-path ${IMPORT_EX_TABLE_LIST_FILE_PATH}" fi + # Check if RUN_USING_ADAPTIVE_PARALLELISM is true + if [ "${RUN_USING_ADAPTIVE_PARALLELISM}" = "true" ]; then + args="${args} --enable-adaptive-parallelism true" + fi + yb-voyager import data ${args} $* } diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index afa4a95ba6..175630f70a 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -158,7 +158,7 @@ main() { cat ${EXPORT_DIR}/metainfo/dataFileDescriptor.json step "Import data." - import_data --parallel-jobs 3 || { + import_data || { tail_log_file "yb-voyager-import-data.log" for i in {1..3}; do sleep 5