Merge branch 'main' into aneesh/fetch-with-ties-issue

.github/workflows/go.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
 
-  build:
+  build-and-test:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
@@ -26,7 +26,7 @@ jobs:
     - name: Test
       run: |
         cd yb-voyager
-        go test -v  -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... 
+        go test -v ./... -tags 'unit'
         
     - name: Vet
       run: |

.github/workflows/integration-tests.yml

@@ -0,0 +1,53 @@
+name: Go
+
+on:
+  push:
+    branches: ['main', '*.*-dev', '*.*.*-dev']
+  pull_request:
+    branches: [main]
+
+env:
+  ORACLE_INSTANT_CLIENT_VERSION: "21.5.0.0.0-1"
+
+jobs:
+  integration-tests:
+    strategy:
+      fail-fast: false
+
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: "1.23.1"
+
+    - name: Build
+      run: |
+        cd yb-voyager
+        go build -v ./...
+
+    # required by godror driver used in the tests
+    - name: Install Oracle Instant Clients
+      run: |
+        # Download and install the YB APT repository package
+        wget https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/repos/reporpms/yb-apt-repo_1.0.0_all.deb
+        sudo apt-get install -y ./yb-apt-repo_1.0.0_all.deb
+        sudo apt-get update -y
+
+        # Install Oracle Instant Client packages using the defined version
+        sudo apt-get install -y oracle-instantclient-tools=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
+        sudo apt-get install -y oracle-instantclient-basic=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
+        sudo apt-get install -y oracle-instantclient-devel=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
+        sudo apt-get install -y oracle-instantclient-jdbc=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
+        sudo apt-get install -y oracle-instantclient-sqlplus=${{ env.ORACLE_INSTANT_CLIENT_VERSION }}
+
+        # Clean up the YB APT repository package
+        sudo apt-get remove -y yb-apt-repo
+        rm -f yb-apt-repo_1.0.0_all.deb
+
+    - name: Run Integration Tests
+      run: |
+        cd yb-voyager
+        go test -v ./... -tags 'integration'

.github/workflows/issue-tests.yml

@@ -47,5 +47,4 @@ jobs:
     - name: Test Issues Against YB Version
       run: |
         cd yb-voyager
-        go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./...
-        
+        go test -v ./... -tags 'issues_integration'

migtests/scripts/functions.sh

@@ -932,7 +932,7 @@ compare_files() {
         return 0
     else
         echo "Data does not match expected report."
-        diff_output=$(diff "$file1" "$file2")
+        diff_output=$(diff --context "$file1" "$file2")
         echo "$diff_output"
         return 1
     fi

migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql

@@ -386,4 +386,37 @@ CREATE TABLE public.locations (
 
 CREATE TABLE image (title text, raster lo);
 
-CREATE TABLE employees (id INT PRIMARY KEY, salary INT);
+CREATE TABLE employees (id INT PRIMARY KEY, salary INT);
+-- create table with multirange data types
+
+-- Create tables with primary keys directly
+CREATE TABLE bigint_multirange_table (
+    id integer PRIMARY KEY,
+    value_ranges int8multirange
+);
+
+CREATE TABLE date_multirange_table (
+    id integer PRIMARY KEY,
+    project_dates datemultirange
+);
+
+CREATE TABLE int_multirange_table (
+    id integer PRIMARY KEY,
+    value_ranges int4multirange
+);
+
+CREATE TABLE numeric_multirange_table (
+    id integer PRIMARY KEY,
+    price_ranges nummultirange
+);
+
+CREATE TABLE timestamp_multirange_table (
+    id integer PRIMARY KEY,
+    event_times tsmultirange
+);
+
+CREATE TABLE timestamptz_multirange_table (
+    id integer PRIMARY KEY,
+    global_event_times tstzmultirange
+);
+

migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql

@@ -38,4 +38,14 @@ CREATE VIEW top_employees_view AS SELECT * FROM (
 			SELECT * FROM employees
 			ORDER BY salary DESC
 			FETCH FIRST 2 ROWS WITH TIES
-		) AS top_employees;
+		) AS top_employees;
+CREATE VIEW public.my_films_view AS 
+SELECT jt.* FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites[*]'
+   COLUMNS (
+    id FOR ORDINALITY,
+    kind text PATH '$.kind',
+    NESTED PATH '$.films[*]' COLUMNS (
+      title text FORMAT JSON PATH '$.title' OMIT QUOTES,
+      director text PATH '$.director' KEEP QUOTES))) AS jt;

migtests/tests/analyze-schema/expected_issues.json

@@ -20,6 +20,36 @@
             "GH": "https://github.com/yugabyte/yb-voyager/issues/1542",
             "MinimumVersionsFixedIn": null
         },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "VIEW",
+            "ObjectName": "public.my_films_view",
+            "Reason": "Json Query Functions",
+            "SqlStatement": "CREATE VIEW public.my_films_view AS\nSELECT jt.* FROM\n my_films,\n JSON_TABLE ( js, '$.favorites[*]'\n   COLUMNS (\n    id FOR ORDINALITY,\n    kind text PATH '$.kind',\n    NESTED PATH '$.films[*]' COLUMNS (\n      title text FORMAT JSON PATH '$.title' OMIT QUOTES,\n      director text PATH '$.director' KEEP QUOTES))) AS jt;",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "VIEW",
+            "ObjectName": "test",
+            "Reason": "Json Constructor Functions",
+            "SqlStatement": "CREATE OR REPLACE view test AS (\n                            select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n                            FROM test1\n                            where t = '1DAY' group by x\n                            );",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "MVIEW",
+            "ObjectName": "test",
+            "Reason": "Json Constructor Functions",
+            "SqlStatement": "CREATE MATERIALIZED VIEW test AS (\n                            select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n                            FROM test1\n                            where t = '1DAY' group by x\n                            );",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
         {
             "IssueType": "unsupported_features",
             "ObjectType": "TABLE",
@@ -1910,5 +1940,65 @@
             "Suggestion": "No workaround available right now",
             "GH": "",
             "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_datatypes",
+            "ObjectType": "TABLE",
+            "ObjectName": "bigint_multirange_table",
+            "Reason": "Unsupported datatype - int8multirange on column - value_ranges",
+            "SqlStatement": "CREATE TABLE bigint_multirange_table (\n    id integer PRIMARY KEY,\n    value_ranges int8multirange\n);",
+            "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_datatypes",
+            "ObjectType": "TABLE",
+            "ObjectName": "date_multirange_table",
+            "Reason": "Unsupported datatype - datemultirange on column - project_dates",
+            "SqlStatement": "CREATE TABLE date_multirange_table (\n    id integer PRIMARY KEY,\n    project_dates datemultirange\n);",
+            "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_datatypes",
+            "ObjectType": "TABLE",
+            "ObjectName": "int_multirange_table",
+            "Reason": "Unsupported datatype - int4multirange on column - value_ranges",
+            "SqlStatement": "CREATE TABLE int_multirange_table (\n    id integer PRIMARY KEY,\n    value_ranges int4multirange\n);",
+            "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_datatypes",
+            "ObjectType": "TABLE",
+            "ObjectName": "numeric_multirange_table",
+            "Reason": "Unsupported datatype - nummultirange on column - price_ranges",
+            "SqlStatement": "CREATE TABLE numeric_multirange_table (\n    id integer PRIMARY KEY,\n    price_ranges nummultirange\n);",
+            "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_datatypes",
+            "ObjectType": "TABLE",
+            "ObjectName": "timestamp_multirange_table",
+            "Reason": "Unsupported datatype - tsmultirange on column - event_times",
+            "SqlStatement": "CREATE TABLE timestamp_multirange_table (\n    id integer PRIMARY KEY,\n    event_times tsmultirange\n);",
+            "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_datatypes",
+            "ObjectType": "TABLE",
+            "ObjectName": "timestamptz_multirange_table",
+            "Reason": "Unsupported datatype - tstzmultirange on column - global_event_times",
+            "SqlStatement": "CREATE TABLE timestamptz_multirange_table (\n    id integer PRIMARY KEY,\n    global_event_times tstzmultirange\n);",
+            "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
         }
 ]

migtests/tests/analyze-schema/summary.json

@@ -26,9 +26,10 @@
             },
             {
                 "ObjectType": "TABLE",
-                "TotalCount": 52,
-                "InvalidCount": 43,
-                "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr"            },
+                "TotalCount": 58,
+                "InvalidCount": 49,
+                "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table"            },
+
             {
                 "ObjectType": "INDEX",
                 "TotalCount": 43,
@@ -50,9 +51,9 @@
             },
             {
                 "ObjectType": "VIEW",
-                "TotalCount": 6,
-                "InvalidCount": 6,
-                "ObjectNames": "v1, v2, test, public.orders_view, view_name, top_employees_view"
+                "TotalCount": 7,
+                "InvalidCount": 7,
+                "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name, top_employees_view"
             },
             {
                 "ObjectType": "TRIGGER",
@@ -91,4 +92,4 @@
                 "ObjectNames": "\u003c%"
             }
         ]
-}
+}

migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql

@@ -41,4 +41,8 @@ INSERT INTO hr.departments (department_name, location) VALUES ('Engineering', 'B
 INSERT INTO hr.departments (department_name, location) VALUES ('Sales', 'Building B');
 INSERT INTO public.employees (name, department_id) VALUES ('Alice', 1), ('Bob', 1), ('Charlie', 2);
 INSERT INTO sales.orders (customer_id, amount) VALUES (101, 500.00), (102, 1200.00);
-INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRate', 0.023), ('ChurnRate', 0.05);
+INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRate', 0.023), ('ChurnRate', 0.05);
+
+create view sales.employ_depart_view AS  SELECT
+        any_value(name) AS any_employee
+    FROM employees;

migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql

@@ -20,4 +20,9 @@ FROM public.employees;
 -- 4) Advisory locks (analytics schema)
 SELECT metric_name, pg_advisory_lock(metric_id)
 FROM analytics.metrics
-WHERE metric_value > 0.02;
+WHERE metric_value > 0.02;
+
+-- Aggregate functions UQC NOT REPORTING as it need PG16 upgarde in pipeline from PG15
+SELECT
+        any_value(name) AS any_employee
+    FROM employees;