diff --git a/docs/config/_default/menus.toml b/docs/config/_default/menus.toml index 28f679e9b3bf..63d188ec8fdd 100644 --- a/docs/config/_default/menus.toml +++ b/docs/config/_default/menus.toml @@ -930,15 +930,6 @@ showSection = true hideLink = true -[[preview_yugabyte-voyager]] - name = "Monitor" - parent = "yugabytedb-voyager" - weight = 103 - identifier = "monitor-voyager" - [preview_yugabyte-voyager.params] - showSection = true - hideLink = true - ########## Menus for stable [[stable]] diff --git a/docs/content/preview/deploy/public-clouds/aws/manual-deployment.md b/docs/content/preview/deploy/public-clouds/aws/manual-deployment.md index ca132b09d986..f9fd6b4af2c4 100644 --- a/docs/content/preview/deploy/public-clouds/aws/manual-deployment.md +++ b/docs/content/preview/deploy/public-clouds/aws/manual-deployment.md @@ -666,7 +666,7 @@ From any node, execute the following command. ```sh $ cd ~/tserver -$ ./bin/ysqlsh +$ ./bin/ysqlsh -h ``` ```sql diff --git a/docs/content/preview/yugabyte-voyager/airgapped.md b/docs/content/preview/yugabyte-voyager/airgapped.md index 748f5ce0b931..ba8e59d2c768 100644 --- a/docs/content/preview/yugabyte-voyager/airgapped.md +++ b/docs/content/preview/yugabyte-voyager/airgapped.md @@ -57,7 +57,7 @@ You can perform an airgapped installation on RHEL 8 and CentOS 8. 1. Download the airgapped bundle: ```sh - wget https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/repos/airgapped/yb-voyager-1.8.1-0-rhel-8-x86_64.tar.gz + wget https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/repos/airgapped/yb-voyager-1.8.2-0-rhel-8-x86_64.tar.gz ``` 1. Extract the bundle. @@ -171,7 +171,7 @@ You can perform an airgapped installation on Ubuntu 22 and later. 1. Download the airgapped bundle: ```sh - wget https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/repos/airgapped/yb-voyager-1.8.1_0_debian.tar.gz + wget https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/repos/airgapped/yb-voyager-1.8.2_0_debian.tar.gz ``` 1. Extract the bundle. diff --git a/docs/content/preview/yugabyte-voyager/known-issues/_index.md b/docs/content/preview/yugabyte-voyager/known-issues/_index.md index f2f082ad23eb..b4df5632b264 100644 --- a/docs/content/preview/yugabyte-voyager/known-issues/_index.md +++ b/docs/content/preview/yugabyte-voyager/known-issues/_index.md @@ -1,12 +1,15 @@ --- -title: Manual review guideline for YugabyteDB Voyager -headerTitle: Manual review guideline -linkTitle: Manual review guideline +title: Schema review workarounds for YugabyteDB Voyager +headerTitle: Schema review workarounds +linkTitle: Schema review workarounds image: /images/section_icons/develop/learn.png headcontent: What to watch out for when migrating data using YugabyteDB Voyager description: What to watch out for when migrating data using YugabyteDB Voyager. type: indexpage showRightNav: true +aliases: + - /preview/yugabyte-voyager/known-issues/general-issues/ + - /preview/yugabyte-voyager/known-issues/mysql-oracle/ menu: preview_yugabyte-voyager: identifier: known-issues @@ -28,22 +31,10 @@ Currently, yb-voyager doesn't support the following features: | Unsupported data types | Data migration is unsupported for some data types, such as BLOB and XML. For others such as ANY and BFile, both schema and data migration is unsupported. Refer to [datatype mapping](../reference/datatype-mapping-oracle/) for the detailed list of data types. | | | Unsupported PostgreSQL features | Yugabyte currently doesn't support the PostgreSQL features listed in [PostgreSQL compatibility](../../explore/ysql-language-features/postgresql-compatibility/#unsupported-postgresql-features). If such schema clauses are encountered, Voyager results in an error. | | -## Manual review +## Schema review {{}} - {{}} - - {{}} - {{}} {{}} {{}} diff --git a/docs/content/preview/yugabyte-voyager/known-issues/general-issues.md b/docs/content/preview/yugabyte-voyager/known-issues/general-issues.md deleted file mode 100644 index a35321e89ae5..000000000000 --- a/docs/content/preview/yugabyte-voyager/known-issues/general-issues.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -title: General guidelines -linkTitle: General -headcontent: What to watch out for when migrating data from MySQL, Oracle, or PostgreSQL -description: Review limitations and suggested workarounds for migrating data from MySQL, Oracle, or PostgreSQL. -menu: - preview_yugabyte-voyager: - identifier: general-issues - parent: known-issues - weight: 100 -type: docs -rightNav: - hideH3: true ---- - -Review limitations and implement suggested workarounds to successfully migrate data from MySQL, Oracle, or PostgreSQL to YugabyteDB. - -## Contents - -- [Index on timestamp column should be imported as ASC (Range) index to avoid sequential scans](#index-on-timestamp-column-should-be-imported-as-asc-range-index-to-avoid-sequential-scans) - -- [Exporting data with names for tables/functions/procedures using special characters/whitespaces fails](#exporting-data-with-names-for-tables-functions-procedures-using-special-characters-whitespaces-fails) - -- [Importing with case-sensitive schema names](#importing-with-case-sensitive-schema-names) - -### Index on timestamp column should be imported as ASC (Range) index to avoid sequential scans - -**GitHub**: [Issue #49](https://github.com/yugabyte/yb-voyager/issues/49) - -**Description**: If there is an index on a timestamp column, the index should be imported as a range index automatically, as most queries relying on timestamp columns use range predicates. This avoids sequential scans and makes indexed scans accessible. - -**Workaround**: Manually add the ASC (range) clause to the exported files. - -**Example** - -An example schema on the source database is as follows: - -```sql -CREATE INDEX ON timestamp_demo (ts); -``` - -Suggested change to the schema is to add the `ASC` clause as follows: - -```sql -CREATE INDEX ON timestamp_demo (ts ASC); -``` - ---- - -### Exporting data with names for tables/functions/procedures using special characters/whitespaces fails - -**GitHub**: [Issue #636](https://github.com/yugabyte/yb-voyager/issues/636), [Issue #688](https://github.com/yugabyte/yb-voyager/issues/688), [Issue #702](https://github.com/yugabyte/yb-voyager/issues/702) - -**Description**: If you define complex names for your source database tables/functions/procedures using backticks or double quotes for example, \`abc xyz\` , \`abc@xyz\`, or "abc@123", the migration hangs during the export data step. - -**Workaround**: Rename the objects (tables/functions/procedures) on the source database to a name without special characters. - -**Example** - -An example schema on the source MySQL database is as follows: - -```sql -CREATE TABLE `xyz abc`(id int); -INSERT INTO `xyz abc` VALUES(1); -INSERT INTO `xyz abc` VALUES(2); -INSERT INTO `xyz abc` VALUES(3); -``` - -The exported schema is as follows: - -```sql -CREATE TABLE "xyz abc" (id bigint); -``` - -The preceding example may hang or result in an error. - ---- - -### Importing with case-sensitive schema names - -**GitHub**: [Issue #422](https://github.com/yugabyte/yb-voyager/issues/422) - -**Description**: If you migrate your database using a case-sensitive schema name, the migration will fail with a "no schema has been selected" or "schema already exists" error(s). - -**Workaround**: Currently, yb-voyager does not support case-sensitive schema names; all schema names are assumed to be case-insensitive (lower-case). If required, you may alter the schema names to a case-sensitive alternative post-migration using the ALTER SCHEMA command. - -**Example** - -An example yb-voyager import-schema command with a case-sensitive schema name is as follows: - -```sh -yb-voyager import schema --target-db-name voyager - --target-db-hostlocalhost - --export-dir . - --target-db-password password - --target-db-user yugabyte - --target-db-schema "\"Test\"" -``` - -The preceding example will result in an error as follows: - -```output -ERROR: no schema has been selected to create in (SQLSTATE 3F000) -``` - -Suggested changes to the schema can be done using the following steps: - -1. Change the case sensitive schema name during schema migration as follows: - - ```sh - yb-voyager import schema --target-db-name voyager - --target-db-hostlocalhost - --export-dir . - --target-db-password password - --target-db-user yugabyte - --target-db-schema test - ``` - -1. Alter the schema name post migration as follows: - - ```sh - ALTER SCHEMA "test" RENAME TO "Test"; - ``` diff --git a/docs/content/preview/yugabyte-voyager/known-issues/mysql-oracle.md b/docs/content/preview/yugabyte-voyager/known-issues/mysql-oracle.md deleted file mode 100644 index 866942f8d5d3..000000000000 --- a/docs/content/preview/yugabyte-voyager/known-issues/mysql-oracle.md +++ /dev/null @@ -1,128 +0,0 @@ ---- -title: MySQL and Oracle source databases -linkTitle: MySQL and Oracle -headcontent: What to watch out for when migrating data from MySQL or Oracle -description: Review limitations and suggested workarounds for migrating data from MySQL or Oracle. -menu: - preview_yugabyte-voyager: - identifier: mysql-oracle-issues - parent: known-issues - weight: 103 -type: docs -rightNav: - hideH3: true ---- - -This page documents known issues you may encounter and suggested workarounds when migrating data from MySQL or Oracle to YugabyteDB. - -## Contents - -- [Tables partitioned with expressions cannot contain primary/unique keys](#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys) -- [Multi-column partition by list is not supported](#multi-column-partition-by-list-is-not-supported) - -### Tables partitioned with expressions cannot contain primary/unique keys - -**GitHub**: [Issue#698](https://github.com/yugabyte/yb-voyager/issues/698) - -**Description**: If you have a table in the source database which is partitioned using any expression/function, that table cannot have a primary or unique key on any of its columns, as it is an invalid syntax in YugabyteDB. - -**Workaround**: Remove any primary/unique keys from exported schemas. - -An example schema on the MySQL source database with primary key is as follows: - -```sql -/* Table definition */ - -CREATE TABLE Sales ( - cust_id INT NOT NULL, - name VARCHAR(40), - store_id VARCHAR(20) NOT NULL, - bill_no INT NOT NULL, - bill_date DATE NOT NULL, - amount DECIMAL(8,2) NOT NULL, - PRIMARY KEY (bill_no,bill_date) -) -PARTITION BY RANGE (year(bill_date))( - PARTITION p0 VALUES LESS THAN (2016), - PARTITION p1 VALUES LESS THAN (2017), - PARTITION p2 VALUES LESS THAN (2018), - PARTITION p3 VALUES LESS THAN (2020) -); -``` - -The exported schema is as follows: - -```sql -/* Table definition */ -CREATE TABLE sales ( - cust_id bigint NOT NULL, - name varchar(40), - store_id varchar(20) NOT NULL, - bill_no bigint NOT NULL, - bill_date timestamp NOT NULL, - amount decimal(8,2) NOT NULL, - PRIMARY KEY (bill_no,bill_date) -) PARTITION BY RANGE ((extract(year from date(bill_date)))) ; -``` - -Suggested change to the schema is to remove the primary/unique key from the exported schema as follows: - -```sql -CREATE TABLE sales ( - cust_id bigint NOT NULL, - name varchar(40), - store_id varchar(20) NOT NULL, - bill_no bigint NOT NULL, - bill_date timestamp NOT NULL, - amount decimal(8,2) NOT NULL -) PARTITION BY RANGE ((extract(year from date(bill_date)))) ; -``` - ---- - -### Multi-column partition by list is not supported - -**GitHub**: [Issue#699](https://github.com/yugabyte/yb-voyager/issues/699) - -**Description**: In YugabyteDB, you cannot perform a partition by list on multiple columns and exporting the schema results in an error. - -**Workaround**: Make the partition a single column partition by list by making suitable changes or choose other supported partitioning methods. - -**Example** - -An example schema on the Oracle source database is as follows: - -```sql -CREATE TABLE test ( - id NUMBER, - country_code VARCHAR2(3), - record_type VARCHAR2(5), - descriptions VARCHAR2(50), - CONSTRAINT t1_pk PRIMARY KEY (id) -) -PARTITION BY LIST (country_code, record_type) -( - PARTITION part_gbr_abc VALUES (('GBR','A'), ('GBR','B'), ('GBR','C')), - PARTITION part_ire_ab VALUES (('IRE','A'), ('IRE','B')), - PARTITION part_usa_a VALUES (('USA','A')), - PARTITION part_others VALUES (DEFAULT) -); -``` - -The exported schema is as follows: - -```sql -CREATE TABLE test ( - id numeric NOT NULL, - country_code varchar(3), - record_type varchar(5), - descriptions varchar(50), - PRIMARY KEY (id) -) PARTITION BY LIST (country_code, record_type) ; -``` - -The preceding schema example will result in an error as follows: - -```output -ERROR: cannot use "list" partition strategy with more than one column (SQLSTATE 42P17) -``` diff --git a/docs/content/preview/yugabyte-voyager/known-issues/mysql.md b/docs/content/preview/yugabyte-voyager/known-issues/mysql.md index 3074b89e09b1..a8fd163a7b19 100644 --- a/docs/content/preview/yugabyte-voyager/known-issues/mysql.md +++ b/docs/content/preview/yugabyte-voyager/known-issues/mysql.md @@ -7,7 +7,7 @@ menu: preview_yugabyte-voyager: identifier: mysql-issues parent: known-issues - weight: 101 + weight: 103 type: docs rightNav: hideH3: true @@ -33,6 +33,11 @@ Review limitations and implement suggested workarounds to successfully migrate d - [DOUBLE UNSIGNED and FLOAT UNSIGNED data types are not supported](#double-unsigned-and-float-unsigned-data-types-are-not-supported) - [Foreign key referenced column cannot be a subset of a unique/primary key](#foreign-key-referenced-column-cannot-be-a-subset-of-a-unique-primary-key) - [Multiple indexes on the same column of a table errors during import](#multiple-indexes-on-the-same-column-of-a-table-errors-during-import) +- [Index on timestamp column should be imported as ASC (Range) index to avoid sequential scans](#index-on-timestamp-column-should-be-imported-as-asc-range-index-to-avoid-sequential-scans) +- [Exporting data with names for tables/functions/procedures using special characters/whitespaces fails](#exporting-data-with-names-for-tables-functions-procedures-using-special-characters-whitespaces-fails) +- [Importing with case-sensitive schema names](#importing-with-case-sensitive-schema-names) +- [Tables partitioned with expressions cannot contain primary/unique keys](#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys) +- [Multi-column partition by list is not supported](#multi-column-partition-by-list-is-not-supported) ### Approaching MAX/MIN double precision values are not imported @@ -742,3 +747,213 @@ Suggested changes: 1. Drop index1 or index2 on the source before exporting. --- + +### Index on timestamp column should be imported as ASC (Range) index to avoid sequential scans + +**GitHub**: [Issue #49](https://github.com/yugabyte/yb-voyager/issues/49) + +**Description**: If there is an index on a timestamp column, the index should be imported as a range index automatically, as most queries relying on timestamp columns use range predicates. This avoids sequential scans and makes indexed scans accessible. + +**Workaround**: Manually add the ASC (range) clause to the exported files. + +**Example** + +An example schema on the source database is as follows: + +```sql +CREATE INDEX ON timestamp_demo (ts); +``` + +Suggested change to the schema is to add the `ASC` clause as follows: + +```sql +CREATE INDEX ON timestamp_demo (ts ASC); +``` + +--- + +### Exporting data with names for tables/functions/procedures using special characters/whitespaces fails + +**GitHub**: [Issue #636](https://github.com/yugabyte/yb-voyager/issues/636), [Issue #688](https://github.com/yugabyte/yb-voyager/issues/688), [Issue #702](https://github.com/yugabyte/yb-voyager/issues/702) + +**Description**: If you define complex names for your source database tables/functions/procedures using backticks or double quotes for example, \`abc xyz\` , \`abc@xyz\`, or "abc@123", the migration hangs during the export data step. + +**Workaround**: Rename the objects (tables/functions/procedures) on the source database to a name without special characters. + +**Example** + +An example schema on the source MySQL database is as follows: + +```sql +CREATE TABLE `xyz abc`(id int); +INSERT INTO `xyz abc` VALUES(1); +INSERT INTO `xyz abc` VALUES(2); +INSERT INTO `xyz abc` VALUES(3); +``` + +The exported schema is as follows: + +```sql +CREATE TABLE "xyz abc" (id bigint); +``` + +The preceding example may hang or result in an error. + +--- + +### Importing with case-sensitive schema names + +**GitHub**: [Issue #422](https://github.com/yugabyte/yb-voyager/issues/422) + +**Description**: If you migrate your database using a case-sensitive schema name, the migration will fail with a "no schema has been selected" or "schema already exists" error(s). + +**Workaround**: Currently, yb-voyager does not support case-sensitive schema names; all schema names are assumed to be case-insensitive (lower-case). If required, you may alter the schema names to a case-sensitive alternative post-migration using the ALTER SCHEMA command. + +**Example** + +An example yb-voyager import-schema command with a case-sensitive schema name is as follows: + +```sh +yb-voyager import schema --target-db-name voyager + --target-db-hostlocalhost + --export-dir . + --target-db-password password + --target-db-user yugabyte + --target-db-schema "\"Test\"" +``` + +The preceding example will result in an error as follows: + +```output +ERROR: no schema has been selected to create in (SQLSTATE 3F000) +``` + +Suggested changes to the schema can be done using the following steps: + +1. Change the case sensitive schema name during schema migration as follows: + + ```sh + yb-voyager import schema --target-db-name voyager + --target-db-hostlocalhost + --export-dir . + --target-db-password password + --target-db-user yugabyte + --target-db-schema test + ``` + +1. Alter the schema name post migration as follows: + + ```sh + ALTER SCHEMA "test" RENAME TO "Test"; + ``` + +--- + +### Tables partitioned with expressions cannot contain primary/unique keys + +**GitHub**: [Issue#698](https://github.com/yugabyte/yb-voyager/issues/698) + +**Description**: If you have a table in the source database which is partitioned using any expression/function, that table cannot have a primary or unique key on any of its columns, as it is an invalid syntax in YugabyteDB. + +**Workaround**: Remove any primary/unique keys from exported schemas. + +An example schema on the MySQL source database with primary key is as follows: + +```sql +/* Table definition */ + +CREATE TABLE Sales ( + cust_id INT NOT NULL, + name VARCHAR(40), + store_id VARCHAR(20) NOT NULL, + bill_no INT NOT NULL, + bill_date DATE NOT NULL, + amount DECIMAL(8,2) NOT NULL, + PRIMARY KEY (bill_no,bill_date) +) +PARTITION BY RANGE (year(bill_date))( + PARTITION p0 VALUES LESS THAN (2016), + PARTITION p1 VALUES LESS THAN (2017), + PARTITION p2 VALUES LESS THAN (2018), + PARTITION p3 VALUES LESS THAN (2020) +); +``` + +The exported schema is as follows: + +```sql +/* Table definition */ +CREATE TABLE sales ( + cust_id bigint NOT NULL, + name varchar(40), + store_id varchar(20) NOT NULL, + bill_no bigint NOT NULL, + bill_date timestamp NOT NULL, + amount decimal(8,2) NOT NULL, + PRIMARY KEY (bill_no,bill_date) +) PARTITION BY RANGE ((extract(year from date(bill_date)))) ; +``` + +Suggested change to the schema is to remove the primary/unique key from the exported schema as follows: + +```sql +CREATE TABLE sales ( + cust_id bigint NOT NULL, + name varchar(40), + store_id varchar(20) NOT NULL, + bill_no bigint NOT NULL, + bill_date timestamp NOT NULL, + amount decimal(8,2) NOT NULL +) PARTITION BY RANGE ((extract(year from date(bill_date)))) ; +``` + +--- + +### Multi-column partition by list is not supported + +**GitHub**: [Issue#699](https://github.com/yugabyte/yb-voyager/issues/699) + +**Description**: In YugabyteDB, you cannot perform a partition by list on multiple columns and exporting the schema results in an error. + +**Workaround**: Make the partition a single column partition by list by making suitable changes or choose other supported partitioning methods. + +**Example** + +An example schema on the Oracle source database is as follows: + +```sql +CREATE TABLE test ( + id NUMBER, + country_code VARCHAR2(3), + record_type VARCHAR2(5), + descriptions VARCHAR2(50), + CONSTRAINT t1_pk PRIMARY KEY (id) +) +PARTITION BY LIST (country_code, record_type) +( + PARTITION part_gbr_abc VALUES (('GBR','A'), ('GBR','B'), ('GBR','C')), + PARTITION part_ire_ab VALUES (('IRE','A'), ('IRE','B')), + PARTITION part_usa_a VALUES (('USA','A')), + PARTITION part_others VALUES (DEFAULT) +); +``` + +The exported schema is as follows: + +```sql +CREATE TABLE test ( + id numeric NOT NULL, + country_code varchar(3), + record_type varchar(5), + descriptions varchar(50), + PRIMARY KEY (id) +) PARTITION BY LIST (country_code, record_type) ; +``` + +The preceding schema example will result in an error as follows: + +```output +ERROR: cannot use "list" partition strategy with more than one column (SQLSTATE 42P17) +``` + +--- diff --git a/docs/content/preview/yugabyte-voyager/known-issues/oracle.md b/docs/content/preview/yugabyte-voyager/known-issues/oracle.md index 207a7d5fe546..f02c05c3254c 100644 --- a/docs/content/preview/yugabyte-voyager/known-issues/oracle.md +++ b/docs/content/preview/yugabyte-voyager/known-issues/oracle.md @@ -7,7 +7,7 @@ menu: preview_yugabyte-voyager: identifier: oracle-issues parent: known-issues - weight: 101 + weight: 102 type: docs rightNav: hideH3: true @@ -32,6 +32,11 @@ Cluster, Domain, Bitmap join, IOT indexes, and reverse indexes are not exported. - [%TYPE syntax is unsupported](#type-syntax-is-unsupported) - [TRANSLATE USING is unsupported](#translate-using-is-unsupported) - [Issue with importing data from tables with reserved keyword datatypes matching table names](#issue-with-importing-data-from-tables-with-reserved-keyword-datatypes-matching-table-names) +- [Index on timestamp column should be imported as ASC (Range) index to avoid sequential scans](#index-on-timestamp-column-should-be-imported-as-asc-range-index-to-avoid-sequential-scans) +- [Exporting data with names for tables/functions/procedures using special characters/whitespaces fails](#exporting-data-with-names-for-tables-functions-procedures-using-special-characters-whitespaces-fails) +- [Importing with case-sensitive schema names](#importing-with-case-sensitive-schema-names) +- [Tables partitioned with expressions cannot contain primary/unique keys](#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys) +- [Multi-column partition by list is not supported](#multi-column-partition-by-list-is-not-supported) ### Some numeric types are not exported @@ -444,3 +449,213 @@ COPY number (id) FROM STDIN; ``` --- + +### Index on timestamp column should be imported as ASC (Range) index to avoid sequential scans + +**GitHub**: [Issue #49](https://github.com/yugabyte/yb-voyager/issues/49) + +**Description**: If there is an index on a timestamp column, the index should be imported as a range index automatically, as most queries relying on timestamp columns use range predicates. This avoids sequential scans and makes indexed scans accessible. + +**Workaround**: Manually add the ASC (range) clause to the exported files. + +**Example** + +An example schema on the source database is as follows: + +```sql +CREATE INDEX ON timestamp_demo (ts); +``` + +Suggested change to the schema is to add the `ASC` clause as follows: + +```sql +CREATE INDEX ON timestamp_demo (ts ASC); +``` + +--- + +### Exporting data with names for tables/functions/procedures using special characters/whitespaces fails + +**GitHub**: [Issue #636](https://github.com/yugabyte/yb-voyager/issues/636), [Issue #688](https://github.com/yugabyte/yb-voyager/issues/688), [Issue #702](https://github.com/yugabyte/yb-voyager/issues/702) + +**Description**: If you define complex names for your source database tables/functions/procedures using backticks or double quotes for example, \`abc xyz\` , \`abc@xyz\`, or "abc@123", the migration hangs during the export data step. + +**Workaround**: Rename the objects (tables/functions/procedures) on the source database to a name without special characters. + +**Example** + +An example schema on the source MySQL database is as follows: + +```sql +CREATE TABLE `xyz abc`(id int); +INSERT INTO `xyz abc` VALUES(1); +INSERT INTO `xyz abc` VALUES(2); +INSERT INTO `xyz abc` VALUES(3); +``` + +The exported schema is as follows: + +```sql +CREATE TABLE "xyz abc" (id bigint); +``` + +The preceding example may hang or result in an error. + +--- + +### Importing with case-sensitive schema names + +**GitHub**: [Issue #422](https://github.com/yugabyte/yb-voyager/issues/422) + +**Description**: If you migrate your database using a case-sensitive schema name, the migration will fail with a "no schema has been selected" or "schema already exists" error(s). + +**Workaround**: Currently, yb-voyager does not support case-sensitive schema names; all schema names are assumed to be case-insensitive (lower-case). If required, you may alter the schema names to a case-sensitive alternative post-migration using the ALTER SCHEMA command. + +**Example** + +An example yb-voyager import-schema command with a case-sensitive schema name is as follows: + +```sh +yb-voyager import schema --target-db-name voyager + --target-db-hostlocalhost + --export-dir . + --target-db-password password + --target-db-user yugabyte + --target-db-schema "\"Test\"" +``` + +The preceding example will result in an error as follows: + +```output +ERROR: no schema has been selected to create in (SQLSTATE 3F000) +``` + +Suggested changes to the schema can be done using the following steps: + +1. Change the case sensitive schema name during schema migration as follows: + + ```sh + yb-voyager import schema --target-db-name voyager + --target-db-hostlocalhost + --export-dir . + --target-db-password password + --target-db-user yugabyte + --target-db-schema test + ``` + +1. Alter the schema name post migration as follows: + + ```sh + ALTER SCHEMA "test" RENAME TO "Test"; + ``` + +--- + +### Tables partitioned with expressions cannot contain primary/unique keys + +**GitHub**: [Issue#698](https://github.com/yugabyte/yb-voyager/issues/698) + +**Description**: If you have a table in the source database which is partitioned using any expression/function, that table cannot have a primary or unique key on any of its columns, as it is an invalid syntax in YugabyteDB. + +**Workaround**: Remove any primary/unique keys from exported schemas. + +An example schema on the MySQL source database with primary key is as follows: + +```sql +/* Table definition */ + +CREATE TABLE Sales ( + cust_id INT NOT NULL, + name VARCHAR(40), + store_id VARCHAR(20) NOT NULL, + bill_no INT NOT NULL, + bill_date DATE NOT NULL, + amount DECIMAL(8,2) NOT NULL, + PRIMARY KEY (bill_no,bill_date) +) +PARTITION BY RANGE (year(bill_date))( + PARTITION p0 VALUES LESS THAN (2016), + PARTITION p1 VALUES LESS THAN (2017), + PARTITION p2 VALUES LESS THAN (2018), + PARTITION p3 VALUES LESS THAN (2020) +); +``` + +The exported schema is as follows: + +```sql +/* Table definition */ +CREATE TABLE sales ( + cust_id bigint NOT NULL, + name varchar(40), + store_id varchar(20) NOT NULL, + bill_no bigint NOT NULL, + bill_date timestamp NOT NULL, + amount decimal(8,2) NOT NULL, + PRIMARY KEY (bill_no,bill_date) +) PARTITION BY RANGE ((extract(year from date(bill_date)))) ; +``` + +Suggested change to the schema is to remove the primary/unique key from the exported schema as follows: + +```sql +CREATE TABLE sales ( + cust_id bigint NOT NULL, + name varchar(40), + store_id varchar(20) NOT NULL, + bill_no bigint NOT NULL, + bill_date timestamp NOT NULL, + amount decimal(8,2) NOT NULL +) PARTITION BY RANGE ((extract(year from date(bill_date)))) ; +``` + +--- + +### Multi-column partition by list is not supported + +**GitHub**: [Issue#699](https://github.com/yugabyte/yb-voyager/issues/699) + +**Description**: In YugabyteDB, you cannot perform a partition by list on multiple columns and exporting the schema results in an error. + +**Workaround**: Make the partition a single column partition by list by making suitable changes or choose other supported partitioning methods. + +**Example** + +An example schema on the Oracle source database is as follows: + +```sql +CREATE TABLE test ( + id NUMBER, + country_code VARCHAR2(3), + record_type VARCHAR2(5), + descriptions VARCHAR2(50), + CONSTRAINT t1_pk PRIMARY KEY (id) +) +PARTITION BY LIST (country_code, record_type) +( + PARTITION part_gbr_abc VALUES (('GBR','A'), ('GBR','B'), ('GBR','C')), + PARTITION part_ire_ab VALUES (('IRE','A'), ('IRE','B')), + PARTITION part_usa_a VALUES (('USA','A')), + PARTITION part_others VALUES (DEFAULT) +); +``` + +The exported schema is as follows: + +```sql +CREATE TABLE test ( + id numeric NOT NULL, + country_code varchar(3), + record_type varchar(5), + descriptions varchar(50), + PRIMARY KEY (id) +) PARTITION BY LIST (country_code, record_type) ; +``` + +The preceding schema example will result in an error as follows: + +```output +ERROR: cannot use "list" partition strategy with more than one column (SQLSTATE 42P17) +``` + +--- diff --git a/docs/content/preview/yugabyte-voyager/known-issues/postgresql.md b/docs/content/preview/yugabyte-voyager/known-issues/postgresql.md index c5a61c69d046..195ed9f011e8 100644 --- a/docs/content/preview/yugabyte-voyager/known-issues/postgresql.md +++ b/docs/content/preview/yugabyte-voyager/known-issues/postgresql.md @@ -7,7 +7,7 @@ menu: preview_yugabyte-voyager: identifier: postgresql-issues parent: known-issues - weight: 102 + weight: 101 type: docs rightNav: hideH3: true @@ -38,6 +38,9 @@ Review limitations and implement suggested workarounds to successfully migrate d - [Policies on users in source require manual user creation](#policies-on-users-in-source-require-manual-user-creation) - [VIEW WITH CHECK OPTION is not supported](#view-with-check-option-is-not-supported) - [UNLOGGED table is not supported](#unlogged-table-is-not-supported) +- [Index on timestamp column should be imported as ASC (Range) index to avoid sequential scans](#index-on-timestamp-column-should-be-imported-as-asc-range-index-to-avoid-sequential-scans) +- [Exporting data with names for tables/functions/procedures using special characters/whitespaces fails](#exporting-data-with-names-for-tables-functions-procedures-using-special-characters-whitespaces-fails) +- [Importing with case-sensitive schema names](#importing-with-case-sensitive-schema-names) ### Adding primary key to a partitioned table results in an error @@ -556,7 +559,7 @@ CREATE INDEX gist_idx ON public.ts_query_table USING gist (query); ### Indexes on some complex data types are not supported -**GitHub**: [Issue #9698](https://github.com/yugabyte/yugabyte-db/issues/9698), [Issue #23829](https://github.com/yugabyte/yugabyte-db/issues/23829), [Issue #17017](https://github.com/yugabyte/yugabyte-db/issues/17017) +**GitHub**: [Issue #9698](https://github.com/yugabyte/yugabyte-db/issues/9698), [Issue #23829](https://github.com/yugabyte/yugabyte-db/issues/23829), [Issue #17017](https://github.com/yugabyte/yugabyte-db/issues/17017) **Description**: If you have indexes on some complex types such as TSQUERY, TSVECTOR, JSONB, ARRAYs, INET, UDTs, citext, and so on, those will error out in import schema phase with the following error: @@ -883,7 +886,7 @@ CREATE TRIGGER trigger_modify_employee_12000 **GitHub**: [Issue #1129](https://github.com/yugabyte/yugabyte-db/issues/1129) -**Description**: If there are UNLOGGED tables in the source schema, they will error out during the import schema with the following error as it is not supported in target YugabyteDB. +**Description**: If there are UNLOGGED tables in the source schema, they will error out during the import schema with the following error as it is not supported in target YugabyteDB. ```output ERROR: UNLOGGED database object not supported yet @@ -912,3 +915,104 @@ CREATE TABLE tbl_unlogged ( ``` --- + +### Index on timestamp column should be imported as ASC (Range) index to avoid sequential scans + +**GitHub**: [Issue #49](https://github.com/yugabyte/yb-voyager/issues/49) + +**Description**: If there is an index on a timestamp column, the index should be imported as a range index automatically, as most queries relying on timestamp columns use range predicates. This avoids sequential scans and makes indexed scans accessible. + +**Workaround**: Manually add the ASC (range) clause to the exported files. + +**Example** + +An example schema on the source database is as follows: + +```sql +CREATE INDEX ON timestamp_demo (ts); +``` + +Suggested change to the schema is to add the `ASC` clause as follows: + +```sql +CREATE INDEX ON timestamp_demo (ts ASC); +``` + +--- + +### Exporting data with names for tables/functions/procedures using special characters/whitespaces fails + +**GitHub**: [Issue #636](https://github.com/yugabyte/yb-voyager/issues/636), [Issue #688](https://github.com/yugabyte/yb-voyager/issues/688), [Issue #702](https://github.com/yugabyte/yb-voyager/issues/702) + +**Description**: If you define complex names for your source database tables/functions/procedures using backticks or double quotes for example, \`abc xyz\` , \`abc@xyz\`, or "abc@123", the migration hangs during the export data step. + +**Workaround**: Rename the objects (tables/functions/procedures) on the source database to a name without special characters. + +**Example** + +An example schema on the source MySQL database is as follows: + +```sql +CREATE TABLE `xyz abc`(id int); +INSERT INTO `xyz abc` VALUES(1); +INSERT INTO `xyz abc` VALUES(2); +INSERT INTO `xyz abc` VALUES(3); +``` + +The exported schema is as follows: + +```sql +CREATE TABLE "xyz abc" (id bigint); +``` + +The preceding example may hang or result in an error. + +--- + +### Importing with case-sensitive schema names + +**GitHub**: [Issue #422](https://github.com/yugabyte/yb-voyager/issues/422) + +**Description**: If you migrate your database using a case-sensitive schema name, the migration will fail with a "no schema has been selected" or "schema already exists" error(s). + +**Workaround**: Currently, yb-voyager does not support case-sensitive schema names; all schema names are assumed to be case-insensitive (lower-case). If required, you may alter the schema names to a case-sensitive alternative post-migration using the ALTER SCHEMA command. + +**Example** + +An example yb-voyager import-schema command with a case-sensitive schema name is as follows: + +```sh +yb-voyager import schema --target-db-name voyager + --target-db-hostlocalhost + --export-dir . + --target-db-password password + --target-db-user yugabyte + --target-db-schema "\"Test\"" +``` + +The preceding example will result in an error as follows: + +```output +ERROR: no schema has been selected to create in (SQLSTATE 3F000) +``` + +Suggested changes to the schema can be done using the following steps: + +1. Change the case sensitive schema name during schema migration as follows: + + ```sh + yb-voyager import schema --target-db-name voyager + --target-db-hostlocalhost + --export-dir . + --target-db-password password + --target-db-user yugabyte + --target-db-schema test + ``` + +1. Alter the schema name post migration as follows: + + ```sh + ALTER SCHEMA "test" RENAME TO "Test"; + ``` + +--- \ No newline at end of file diff --git a/docs/content/preview/yugabyte-voyager/migrate/bulk-data-load.md b/docs/content/preview/yugabyte-voyager/migrate/bulk-data-load.md index 7ade8268762b..bb17882a716b 100644 --- a/docs/content/preview/yugabyte-voyager/migrate/bulk-data-load.md +++ b/docs/content/preview/yugabyte-voyager/migrate/bulk-data-load.md @@ -78,6 +78,15 @@ yb-voyager import data file --export-dir \ Refer to [import data file](../../reference/bulk-data-load/import-data-file/) for details about the arguments. +{{< note title= "Migrating data files with large size rows" >}} +For CSV file format, the import data file has a default row size limit of 32MB. If a row exceeds this limit but is smaller than the `batch-size * max-row-size`, you can increase the limit for the import data file process by setting the following environment variable: + +```sh +export CSV_READER_MAX_BUFFER_SIZE_BYTES = +``` + +{{< /note >}} + ### Import data status Run the `yb-voyager import data status --export-dir ` command to get an overall progress of the data import operation. diff --git a/docs/content/preview/yugabyte-voyager/migrate/live-fall-back.md b/docs/content/preview/yugabyte-voyager/migrate/live-fall-back.md index 5db054a57d86..f9a9f2c5a339 100644 --- a/docs/content/preview/yugabyte-voyager/migrate/live-fall-back.md +++ b/docs/content/preview/yugabyte-voyager/migrate/live-fall-back.md @@ -934,6 +934,10 @@ yb-voyager export data from source --export-dir \ For PostgreSQL, make sure that no other processes are running on the source database that can try to take locks; with more than one parallel job, Voyager will not be able to take locks to dump the data. {{< /note >}} +{{< note title= "Migrating source databases with large row sizes" >}} +If the size of a source database table row is too large and exceeds the default [RPC message size](../../../reference/configuration/all-flags-yb-master/#rpc-max-message-size), import data will fail with the error `ERROR: Sending too long RPC message..`. Migrate those tables separately after removing the large rows. +{{< /note >}} + The export data from source command first ensures that it exports a snapshot of the data already present on the source database. Next, you start a streaming phase (CDC phase) where you begin capturing new changes made to the data on the source after the migration has started. Some important metrics such as the number of events, export rate, and so on, is displayed during the CDC phase similar to the following: ```output @@ -1016,6 +1020,15 @@ If the `yb-voyager import data to target` command terminates before completing t {{< /tip >}} +{{< note title= "Migrating Oracle source databases with large row sizes" >}} +When migrating from Oracle source, when the snapshot import process, the default row size limit for data import is 32MB. If a row exceeds this limit but is smaller than the `batch-size * max-row-size`, you can increase the limit by setting the following environment variable: + +```sh +export CSV_READER_MAX_BUFFER_SIZE_BYTES = +``` + +{{< /note >}} + #### get data-migration-report Run the following command with required arguments to get a consolidated report of the overall progress of data migration concerning all the databases involved (source and target). @@ -1311,6 +1324,6 @@ Refer to [end migration](../../reference/end-migration/) for more details on the In addition to the Live migration [limitations](../live-migrate/#limitations), the following additional limitations apply to the fall-back feature: - Fall-back is unsupported with a YugabyteDB cluster running on YugabyteDB Aeon. -- SSL Connectivity is unsupported for export or streaming events from YugabyteDB during `export data from target`. +- [SSL Connectivity](../../reference/yb-voyager-cli/#ssl-connectivity) is partially supported for export or streaming events from YugabyteDB during `export data from target`. Basic SSL and server authentication via root certificate is supported. Client authentication is not supported. - In the fall-back phase, you need to manually disable (and subsequently re-enable if required) constraints/indexes/triggers on the source database. - [Export data from target](../../reference/data-migration/export-data/#export-data-from-target) supports DECIMAL/NUMERIC datatypes for YugabyteDB versions 2.20.1.1 and later. diff --git a/docs/content/preview/yugabyte-voyager/migrate/live-fall-forward.md b/docs/content/preview/yugabyte-voyager/migrate/live-fall-forward.md index ab491b862b62..21d7f2191450 100644 --- a/docs/content/preview/yugabyte-voyager/migrate/live-fall-forward.md +++ b/docs/content/preview/yugabyte-voyager/migrate/live-fall-forward.md @@ -941,6 +941,10 @@ yb-voyager export data from source --export-dir \ For PostgreSQL, make sure that no other processes are running on the source database that can try to take locks; with more than one parallel job, Voyager will not be able to take locks to dump the data. {{< /note >}} +{{< note title= "Migrating source databases with large row sizes" >}} +If a table's row size on the source database is too large, and exceeds the default [RPC message size](../../../reference/configuration/all-flags-yb-master/#rpc-max-message-size), import data will fail with the error `ERROR: Sending too long RPC message..`. So, you need to migrate those tables separately after removing the large rows. +{{< /note >}} + The export data from source command first ensures that it exports a snapshot of the data already present on the source database. Next, you start a streaming phase (CDC phase) where you begin capturing new changes made to the data on the source after the migration has started. Some important metrics such as number of events, export rate, and so on will be displayed during the CDC phase similar to the following: ```output @@ -1027,6 +1031,15 @@ If the `yb-voyager import data to target` command terminates before completing t {{< /tip >}} +{{< note title= "Migrating Oracle source databases with large row sizes" >}} +When migrating from Oracle source, during the snapshot import process, the default row size limit for data import is 32MB. If a row exceeds this limit but is smaller than the `batch-size * max-row-size`, you can increase the limit for the import data process by setting the following an environment variable to handle such rows: + +```sh +export CSV_READER_MAX_BUFFER_SIZE_BYTES = +``` + +{{< /note >}} + #### get data-migration-report Run the following command with required arguments to get a consolidated report of the overall progress of data migration concerning all the databases involved (source, target, and source-replica). diff --git a/docs/content/preview/yugabyte-voyager/migrate/live-migrate.md b/docs/content/preview/yugabyte-voyager/migrate/live-migrate.md index f917bfcdafd0..a07b257d09c0 100644 --- a/docs/content/preview/yugabyte-voyager/migrate/live-migrate.md +++ b/docs/content/preview/yugabyte-voyager/migrate/live-migrate.md @@ -785,6 +785,10 @@ yb-voyager export data from source --export-dir \ For PostgreSQL, make sure that no other processes are running on the source database that can try to take locks; with more than one parallel job, Voyager will not be able to take locks to dump the data. {{< /note >}} +{{< note title= "Migrating source databases with large row sizes" >}} +If a table's row size on the source database is too large, and exceeds the default [RPC message size](../../../reference/configuration/all-flags-yb-master/#rpc-max-message-size), import data will fail with the error `ERROR: Sending too long RPC message..`. So, you need to migrate those tables separately after removing the large rows. +{{< /note >}} + The export data command first ensures that it exports a snapshot of the data already present on the source database. Next, you start a streaming phase (CDC phase) where you begin capturing new changes made to the data on the source after the migration has started. Some important metrics such as the number of events, export rate, and so on, is displayed during the CDC phase similar to the following: ```output @@ -871,6 +875,15 @@ If the `yb-voyager import data to target` command terminates before completing t {{< /tip >}} +{{< note title= "Migrating Oracle source databases with large row sizes" >}} +When migrating from Oracle source, when the snapshot import process, the default row size limit for data import is 32MB. If a row exceeds this limit but is smaller than the `batch-size * max-row-size`, you can increase the limit for the import data process by setting the following an environment variable to handle such rows: + +```sh +export CSV_READER_MAX_BUFFER_SIZE_BYTES = +``` + +{{< /note >}} + #### get data-migration-report Run the following command with required arguments to get a consolidated report of the overall progress of data migration concerning all the databases involved (source or target). diff --git a/docs/content/preview/yugabyte-voyager/migrate/migrate-steps.md b/docs/content/preview/yugabyte-voyager/migrate/migrate-steps.md index d7dd0785e35e..8e80f25005cc 100644 --- a/docs/content/preview/yugabyte-voyager/migrate/migrate-steps.md +++ b/docs/content/preview/yugabyte-voyager/migrate/migrate-steps.md @@ -253,6 +253,10 @@ Note that there are some special cases involving sequences such as the following {{< /note >}} +{{< note title= "Migrating source databases with large row sizes" >}} +If a table's row size on the source database is too large, and exceeds the default [RPC message size](../../../reference/configuration/all-flags-yb-master/#rpc-max-message-size), import data will fail with the error `ERROR: Sending too long RPC message..`. So, you need to migrate those tables separately after removing the large rows. +{{< /note >}} + #### Export data status Run the `yb-voyager export data status --export-dir ` command to get an overall progress of the export data operation. @@ -327,6 +331,15 @@ If the `yb-voyager import data` command terminates before completing the data in {{< /tip >}} +{{< note title= "Migrating source databases with large row sizes" >}} +When exporting data using the `BETA_FAST_DATA_EXPORT` flag, the import data process has a default row size limit of 32MB. If a row exceeds this limit but is smaller than the `batch-size * max-row-size`, you can increase the limit for the import data process by setting the following an environment variable to handle such rows: + +```sh +export CSV_READER_MAX_BUFFER_SIZE_BYTES = +``` + +{{< /note >}} + #### Import data status Run the `yb-voyager import data status --export-dir ` command to get an overall progress of the data import operation. diff --git a/docs/content/preview/yugabyte-voyager/reference/_index.md b/docs/content/preview/yugabyte-voyager/reference/_index.md index ef64f547f221..09a71cde814e 100644 --- a/docs/content/preview/yugabyte-voyager/reference/_index.md +++ b/docs/content/preview/yugabyte-voyager/reference/_index.md @@ -14,30 +14,30 @@ menu: weight: 105 --- -
+{{}} -
- -
- -
yb-voyager CLI
-
-
- yb-voyager CLI commands, arguments, and SSL connectivity. -
- -
+ {{}} -
- -
- -
Datatype mapping
-
-
- Data type mapping from different source databases to YugabyteDB. -
- -
+ {{}} -
+ {{}} + + {{}} + +{{}} diff --git a/docs/content/preview/yugabyte-voyager/diagnostics-report.md b/docs/content/preview/yugabyte-voyager/reference/diagnostics-report.md similarity index 50% rename from docs/content/preview/yugabyte-voyager/diagnostics-report.md rename to docs/content/preview/yugabyte-voyager/reference/diagnostics-report.md index 41005bda3678..497a6ede1b49 100644 --- a/docs/content/preview/yugabyte-voyager/diagnostics-report.md +++ b/docs/content/preview/yugabyte-voyager/reference/diagnostics-report.md @@ -6,7 +6,7 @@ menu: preview_yugabyte-voyager: identifier: diagnostics-report parent: reference-voyager - weight: 104 + weight: 105 aliases: - /preview/yugabyte-voyager/diagnostics-report/ type: docs @@ -28,12 +28,10 @@ When enabled, the following data is collected while running yb-voyager: - Migration type (Offline, live migration (with fall-foward or fall-back), or bulk data load) - YugabyteDB Voyager version (the version of yb-voyager used for the migration) - Source database details - - Host IP/name - Database type (PostgreSQL, Oracle, and MySQL) - Database version - Total size of the database migrating to YugabyteDB - Target YugabyteDB database details - - Host IP/name - Database version - Node counts on the target YugabyteDB cluster - Total cores on the target YugabyteDB cluster @@ -45,12 +43,12 @@ The payload for each migration phase is described in the following table. Note t | Migration phase | Data collected | | :---- | :------------- | -|
[assess-migration](../reference/assess-migration/) |
  • Unsupported features (reported in the assessment report)
  • Unsupported datatypes (reported in the assessment report)
  • Error message while running the assessment if any
  • Table sizing statistics (table name, reads/writes per second and size in bytes)
  • Index sizing statistics (index name, reads/writes per second, and size in bytes)
  • Schema summary (reported in the assessment report)
  • Source connectivity (whether assessment is run with source connectivity)
  • Command line arguments changed during this phase
| -|
[export-schema](../reference/schema-migration/export-schema/) |
  • Whether sharding recommendations from the assessment report were applied
  • Command line arguments changed during this phase
| -|
[analyze-schema](../reference/schema-migration/analyze-schema/) |
  • Issues (issues reported by analyze-schema without the SQL statements)
  • Database objects (objects migrated in the migration)
| -|
[import-schema](../reference/schema-migration/import-schema/) |
  • Errors (if there are errors while running the import schema)
  • Command line arguments changed during this phase
| -|
[export-data](../reference/data-migration/export-data/#export-data)
[export-data-from-source](../reference/data-migration/export-data/#export-data)
[export-data-from-target](../reference/data-migration/export-data/#export-data-from-target/) |
  • Parallel jobs used
  • Total rows (total number of rows of all tables exported)
  • Largest table rows (total number of rows of all tables exported)
  • Phase (Snapshot, Streaming, or Cutover)
  • Total exported events (in case of live migration these are the total exported events)
  • Events export rate (in case of live migration this is the rate of events exported for the last 3 minutes)
  • Live migration workflow type (Normal, with fall-forward or fall-back)
  • Command line arguments changed during this phase
| -|
[import-data](../reference/data-migration/import-data/#import-data)
[import-data-to-target](../reference/data-migration/import-data/#import-data)
[import‑data‑to‑source‑replica](../reference/data-migration/import-data/#import-data-to-source-replica)
[import-data-to-source](../reference/data-migration/import-data/#import-data-to-source) |
  • Parallel jobs used
  • Total rows (total number of rows of all tables imported)
  • Largest table rows (total number of rows of all tables imported)
  • Phase (Snapshot, Streaming, or Cutover)
  • Total imported events (for live migration these are the total imported events)
  • Events import rate (for live migration this the rate of events imported for the last 3 minutes)
  • Live migration workflow type (Normal, with fall-forward or fall-back)
  • Command line arguments changed during this phase
| +|
[assess-migration](../reference/assess-migration/) |
  • Unsupported features (reported in the assessment report without any object names or SQL statements; only count of object per feature)
  • Unsupported datatypes (reported in the assessment report without any table name or column name; only the datatype)
  • Error message while running the assessment if any
  • Table sizing statistics (reads/writes per second, and size in bytes)
  • Index sizing statistics (reads/writes per second, and size in bytes)
  • Schema summary (reported in the assessment report, without object names for each object type)
  • Source connectivity (whether assessment is run with source connectivity)
| +|
[export-schema](../reference/schema-migration/export-schema/) |
  • Whether sharding recommendations from the assessment report were applied
| +|
[analyze-schema](../reference/schema-migration/analyze-schema/) |
  • Issues (issues reported by analyze-schema without the SQL statements)
  • Database objects (objects migrated in the migration without the object names per object type)
| +|
[import-schema](../reference/schema-migration/import-schema/) |
  • Error count (if there are errors while running the import schema).
  • Flag values if passed or not:
    • Continue on error
    • Enable orafce
    • Ignore exist
    • Refresh mviews
    • Post snapshot import
| +|
[export-data](../reference/data-migration/export-data/#export-data)
[export-data-from-source](../reference/data-migration/export-data/#export-data)
[export-data-from-target](../reference/data-migration/export-data/#export-data-from-target/) |
  • Parallel jobs used
  • Total rows (total number of rows of all tables exported)
  • Largest table rows (total number of rows of all tables exported)
  • Phase (Snapshot, Streaming, or Cutover)
  • Total exported events (in case of live migration these are the total exported events)
  • Events export rate (in case of live migration this is the rate of events exported for the last 3 minutes)
  • Live migration workflow type (Normal, with fall-forward or fall-back)
| +|
[import-data](../reference/data-migration/import-data/#import-data)
[import-data-to-target](../reference/data-migration/import-data/#import-data)
[import‑data‑to‑source‑replica](../reference/data-migration/import-data/#import-data-to-source-replica)
[import-data-to-source](../reference/data-migration/import-data/#import-data-to-source) |
  • Parallel jobs used
  • Total rows (total number of rows of all tables imported)
  • Largest table rows (total number of rows of all tables imported)
  • Phase (Snapshot, Streaming, or Cutover)
  • Total imported events (for live migration these are the total imported events)
  • Events import rate (for live migration this the rate of events imported for the last 3 minutes)
  • Live migration workflow type (Normal, with fall-forward or fall-back)
| |
[import-data-file](../reference/bulk-data-load/import-data-file/) (bulk data load) |
  • Parallel jobs used
  • Total size (total size of all tables bulk imported)
  • Largest table size (largest size of tables imported)
  • File storage type (Local, AWS S3, GCS, or Azure blob storage)
  • Data file format parameters (for example, null-string, format type, and so on)
| ## Example payloads @@ -65,7 +63,7 @@ The following are examples of payloads that are collected during some migration "phase_start_time": "2024-07-08 16:30:21.237453", "collected_at": "2024-07-08 16:30:22.69088", "source_db_details": { - "host": "127.0.0.1", + "host": "", "db_type": "postgresql", "db_version": "14.1 (Debian 14.1-1.pgdg110+1)", "total_db_size_bytes": 31391744 @@ -77,180 +75,51 @@ The following are examples of payloads that are collected during some migration "unsupported_features": [ { "FeatureName": "GIST indexes", - "ObjectNames": [ - "documents_tsv_idx", - "locations_geom_idx", - "locations_geom_idx1" - ] + "ObjectCount": 3 }, { "FeatureName": "Constraint triggers", - "ObjectNames": [] + "ObjectCount": 0 }, { "FeatureName": "Inherited tables", - "ObjectNames": [] + "ObjectCount": 0 }, { "FeatureName": "Tables with Stored generated columns", - "ObjectNames": [] - } - ], - "unsupported_datatypes": [ - { - "SchemaName": "public", - "TableName": "locations", - "ColumnName": "point3", - "DataType": "point" - }, - { - "SchemaName": "public", - "TableName": "locations", - "ColumnName": "geom", - "DataType": "point" + "ObjectCount": 0 } ], + "unsupported_datatypes": ["point", "point"], "table_sizing_stats": [ - { - "schema_name": "public", - "object_name": "test", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 0 - }, - { - "schema_name": "public", - "object_name": "documents", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 8192 - }, - { - "schema_name": "public", - "object_name": "locations", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 8192 - }, - { - "schema_name": "public", - "object_name": "customers", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 1245184 - }, - { - "schema_name": "public", - "object_name": "cities", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 0 - }, - { - "schema_name": "public", - "object_name": "test_partitions_sequences_l", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 3055616 - }, - { - "schema_name": "public", - "object_name": "test_partitions_sequences_s", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 4825088 - }, - { - "schema_name": "public", - "object_name": "test_partitions_sequences_b", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 2998272 - } + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 0 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 8192 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 8192 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 1245184 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 0 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 3055616 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 4825088 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 2998272 } ], "index_sizing_stats": [ - { - "schema_name": "public", - "object_name": "idx_test", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 8192 - }, - { - "schema_name": "public", - "object_name": "idx_1", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 8192 - }, - { - "schema_name": "public", - "object_name": "locations_geom_idx", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 8192 - }, - { - "schema_name": "public", - "object_name": "documents_tsv_idx", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 8192 - }, - { - "schema_name": "public", - "object_name": "locations_geom_idx1", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 8192 - }, - { - "schema_name": "public", - "object_name": "test_partitions_sequences_b_id_amount_idx", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 1130496 - }, - { - "schema_name": "public", - "object_name": "test_partitions_sequences_l_id_amount_idx", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 1130496 - }, - { - "schema_name": "public", - "object_name": "test_partitions_sequences_s_id_amount_idx", - "reads_per_second": 0, - "writes_per_second": 0, - "size_in_bytes": 1130496 - } + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 8192 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 8192 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 8192 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 8192 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 8192 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 1130496 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 1130496 }, + { "object_name": "XXX", "reads_per_second": 0, "writes_per_second": 0, "size_in_bytes": 1130496 } ], "schema_summary": { - "SchemaNames": [ - "public" - ], "DatabaseObjects": [ - { - "ObjectType": "SCHEMA", - "TotalCount": 1 - }, - { - "ObjectType": "SEQUENCE", - "TotalCount": 3 - }, - { - "ObjectType": "TABLE", - "TotalCount": 9 - }, - { - "ObjectType": "INDEX", - "TotalCount": 9 - } + { "ObjectType": "SCHEMA", "TotalCount": 1 }, + { "ObjectType": "SEQUENCE", "TotalCount": 3 }, + { "ObjectType": "TABLE", "TotalCount": 9 }, + { "ObjectType": "INDEX", "TotalCount": 9 } ] }, - "source_connectivity": true, - "command_line_args": "--iops-capture-interval=0 --send-diagnostics=true --source-db-host=127.0.0.1 --source-db-name=test_callhome --source-db-port=5432 --source-db-schema=public --source-db-type=postgresql --source-db-user=postgres --start-clean=true" + "source_connectivity": true }, "migration_type": "", "time_taken_sec": 2, @@ -262,30 +131,29 @@ The following are examples of payloads that are collected during some migration ```output.json { - "migration_uuid": "8d9d678c-f6d4-4253-b24f-e10e4a49cc31", - "phase_start_time": "2024-07-08 16:28:59.642321", - "collected_at": "2024-07-08 16:29:05.986381", - "source_db_details": { - "host": "127.0.0.1", - "db_type": "postgresql", - "db_version": "14.1 (Debian 14.1-1.pgdg110+1)", - "total_db_size_bytes": 31391744 - }, - "target_db_details": "", - "yb_voyager_version": "1.7.2", - "migration_phase": "export-data", - "phase_payload": { - "parallel_jobs": 4, - "total_rows_exported": 173641, - "largest_table_rows_exported": 149000, - "start_clean": true, - "export_snapshot_mechanism": "pg_dump", - "phase": "SNAPSHOT", - "command_line_args": "--send-diagnostics=true --source-db-host=127.0.0.1 --source-db-name=test_callhome --source-db-schema=public --source-db-type=postgresql --source-db-user=postgres --start-clean=true" - }, - "migration_type": "offline", - "time_taken_sec": 7, - "status": "COMPLETE" + "migration_uuid": "8d9d678c-f6d4-4253-b24f-e10e4a49cc31", + "phase_start_time": "2024-07-08 16:28:59.642321", + "collected_at": "2024-07-08 16:29:05.986381", + "source_db_details": { + "host": "", + "db_type": "postgresql", + "db_version": "14.1 (Debian 14.1-1.pgdg110+1)", + "total_db_size_bytes": 31391744 + }, + "target_db_details": "", + "yb_voyager_version": "1.7.2", + "migration_phase": "export-data", + "phase_payload": { + "parallel_jobs": 4, + "total_rows_exported": 173641, + "largest_table_rows_exported": 149000, + "start_clean": true, + "export_snapshot_mechanism": "pg_dump", + "phase": "SNAPSHOT" + }, + "migration_type": "offline", + "time_taken_sec": 7, + "status": "COMPLETE" } ``` @@ -298,8 +166,8 @@ The following are examples of payloads that are collected during some migration "collected_at": "2024-07-08 16:42:03.363395", "source_db_details": "", "target_db_details": { - "host": "10.9.14.120", - "db_version": "11.2-YB-{{}}-b0", + "host": "", + "db_version": "11.2-YB-2.18.2.1-b0", "node_count": 1, "total_cores": 8 }, @@ -310,8 +178,7 @@ The following are examples of payloads that are collected during some migration "total_rows_imported": 173641, "largest_table_rows_imported": 149000, "start_clean": true, - "phase": "SNAPSHOT", - "command_line_args": "--enable-upsert=true --send-diagnostics=true --start-clean=true --target-db-host=10.9.14.120 --target-db-name=test_callhome --target-db-user=yugabyte" + "phase": "SNAPSHOT" }, "migration_type": "offline", "time_taken_sec": 77, diff --git a/docs/content/preview/yugabyte-voyager/monitor/performance.md b/docs/content/preview/yugabyte-voyager/reference/performance.md similarity index 98% rename from docs/content/preview/yugabyte-voyager/monitor/performance.md rename to docs/content/preview/yugabyte-voyager/reference/performance.md index afd93a11512b..c315aaeb4062 100644 --- a/docs/content/preview/yugabyte-voyager/monitor/performance.md +++ b/docs/content/preview/yugabyte-voyager/reference/performance.md @@ -5,10 +5,11 @@ description: Performance menu: preview_yugabyte-voyager: identifier: performance - parent: monitor-voyager - weight: 103 + parent: reference-voyager + weight: 104 aliases: - /preview/yugabyte-voyager/performance/ + - /preview/yugabyte-voyager/monitor/performance/ type: docs --- diff --git a/docs/content/preview/yugabyte-voyager/reference/schema-migration/analyze-schema.md b/docs/content/preview/yugabyte-voyager/reference/schema-migration/analyze-schema.md index 4f8eb699c71b..694c5cbb5585 100644 --- a/docs/content/preview/yugabyte-voyager/reference/schema-migration/analyze-schema.md +++ b/docs/content/preview/yugabyte-voyager/reference/schema-migration/analyze-schema.md @@ -27,7 +27,7 @@ The valid *arguments* for analyze schema are described in the following table: | :------- | :------------------------ | | -e, --export-dir | Path to the export directory. This directory is a workspace used to store exported schema DDL files, export data files, migration state, and a log file.| | -h, --help | Command line help. | -| --output-format | Format in which the report file is generated. One of `html`, `txt`, `json`, or `xml`. | +| --output-format | Format in which the report file is generated. One of `html`, `txt`, `json`, or `xml`. If not provided, reports are generated in both `json` and `html` formats by default. | | ‑‑send‑diagnostics | Enable or disable sending [diagnostics](../../../diagnostics-report/) information to Yugabyte.
Default: true
Accepted parameters: true, false, yes, no, 0, 1 | | -y, --yes | Answer yes to all prompts during the export schema operation.
Default: false
Accepted parameters: true, false, yes, no, 0, 1 | diff --git a/docs/content/preview/yugabyte-voyager/release-notes.md b/docs/content/preview/yugabyte-voyager/release-notes.md index 0b1d28614a20..02c6e66616cf 100644 --- a/docs/content/preview/yugabyte-voyager/release-notes.md +++ b/docs/content/preview/yugabyte-voyager/release-notes.md @@ -13,6 +13,24 @@ type: docs What follows are the release notes for the YugabyteDB Voyager v1 release series. Content will be added as new notable features and changes are available in the patch releases of the YugabyteDB v1 series. +## v1.8.2 - October 1, 2024 + +### Enhancements + +- The [assess-migration](../reference/assess-migration/) and [analyze schema](../reference/schema-migration/analyze-schema/) commands now report issues with indexes on UDT columns under the [Index on complex data types](../known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported) PostgreSQL unsupported feature. +- Improved logic for cluster sizing recommendations in assessment reports: + + - Tables with more than two indexes will now be recommended for sharding. + - Materialized views and their associated indexes are now considered when sizing clusters from a PostgreSQL source. + +- The analyze-schema command now generates both HTML and JSON format reports by default unless the `output-format` flag is used, in which case only the specified format will be generated. +- Voyager's [diagnostic service](../reference/diagnostics-report/) (callhome) now uses a secure port for enhanced data security. +- The installer script now installs the `jq` package on all supported operating systems and verifies that the installed Java version is between 17 and 19 (inclusive). + +### Bug fix + +- Fixed an issue during the streaming phase of the import data command, where the process would hang on resumption if conflicts arose due to unique constraints between events. + ## v1.8.1 - September 17, 2024 ### Enhancements @@ -23,7 +41,7 @@ What follows are the release notes for the YugabyteDB Voyager v1 release series. - Changed assessment report file names from "assessmentReport" to "migration_assessment_report", and "bulkAssessmentReport" to "bulk_assessment_report" . - Improved UI output of [assess-migration](../reference/assess-migration/) by removing the progress bar from ora2pg when fetching the assessment report. -### Bug Fixes +### Bug fixes - Fixed an issue in streaming phase of [import data](../reference/data-migration/import-data/) where conflicts between events that could cause unique constraint errors were not being detected properly. - Fixed an issue in [analyze schema](../reference/schema-migration/analyze-schema/) where multi-column GIN indexes were incorrectly reported as an issue due to a regex misidentifying expression indexes that combine two columns. diff --git a/docs/content/stable/deploy/public-clouds/aws/manual-deployment.md b/docs/content/stable/deploy/public-clouds/aws/manual-deployment.md index 4507aa9f088c..465ae496f95e 100644 --- a/docs/content/stable/deploy/public-clouds/aws/manual-deployment.md +++ b/docs/content/stable/deploy/public-clouds/aws/manual-deployment.md @@ -666,7 +666,7 @@ From any node, execute the following command. ```sh $ cd ~/tserver -$ ./bin/ysqlsh +$ ./bin/ysqlsh -h ``` ```sql diff --git a/docs/content/v2.14/deploy/public-clouds/aws/manual-deployment.md b/docs/content/v2.14/deploy/public-clouds/aws/manual-deployment.md index c2a1c7db9823..c182b911b952 100644 --- a/docs/content/v2.14/deploy/public-clouds/aws/manual-deployment.md +++ b/docs/content/v2.14/deploy/public-clouds/aws/manual-deployment.md @@ -669,7 +669,7 @@ From any node, execute the following command. ```sh $ cd ~/tserver -$ ./bin/ysqlsh +$ ./bin/ysqlsh -h ``` ```sql diff --git a/docs/content/v2.18/deploy/public-clouds/aws/manual-deployment.md b/docs/content/v2.18/deploy/public-clouds/aws/manual-deployment.md index 79b40758ff26..496aa0e0c012 100644 --- a/docs/content/v2.18/deploy/public-clouds/aws/manual-deployment.md +++ b/docs/content/v2.18/deploy/public-clouds/aws/manual-deployment.md @@ -669,7 +669,7 @@ From any node, execute the following command. ```sh $ cd ~/tserver -$ ./bin/ysqlsh +$ ./bin/ysqlsh -h ``` ```sql diff --git a/docs/content/v2.20/deploy/public-clouds/aws/manual-deployment.md b/docs/content/v2.20/deploy/public-clouds/aws/manual-deployment.md index bf2a72ea85da..91e3f008e9fb 100644 --- a/docs/content/v2.20/deploy/public-clouds/aws/manual-deployment.md +++ b/docs/content/v2.20/deploy/public-clouds/aws/manual-deployment.md @@ -669,7 +669,7 @@ From any node, execute the following command. ```sh $ cd ~/tserver -$ ./bin/ysqlsh +$ ./bin/ysqlsh -h ``` ```sql