From bd4737592e481c03ce7393204fc160c6ab22ce90 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Wed, 22 Jan 2025 13:20:41 +0530 Subject: [PATCH] Reporting CTE Materialized clause and Sql body in Function (#2165) Reports the CTE statements with the Materialized clause as not supported. https://yugabyte.atlassian.net/browse/DB-14227 Reports the presence of SQL body in Functions - PG15 docs https://yugabyte.atlassian.net/browse/DB-14234 Added end-to-end tests --- .../schema/functions/function.sql | 16 ++- .../tests/analyze-schema/expected_issues.json | 33 +++++ migtests/tests/analyze-schema/summary.json | 6 +- .../expectedAssessmentReport.json | 33 ++++- .../pg_assessment_report_uqc.sql | 9 +- .../unsupported_query_constructs.sql | 18 +++ .../expectedAssessmentReport.json | 29 ++++- .../pg_assessment_report.sql | 13 ++ yb-voyager/cmd/assessMigrationCommand.go | 2 + yb-voyager/src/query/queryissue/constants.go | 5 + yb-voyager/src/query/queryissue/detectors.go | 39 ++++++ .../src/query/queryissue/detectors_ddl.go | 21 +++ yb-voyager/src/query/queryissue/issues_ddl.go | 14 ++ .../src/query/queryissue/issues_ddl_test.go | 28 ++++ yb-voyager/src/query/queryissue/issues_dml.go | 14 ++ .../src/query/queryissue/issues_dml_test.go | 31 +++++ .../query/queryissue/parser_issue_detector.go | 1 + .../queryissue/parser_issue_detector_test.go | 120 ++++++++++++++++++ yb-voyager/src/query/queryparser/constants.go | 104 +++++++++++++++ .../src/query/queryparser/ddl_processor.go | 37 +----- .../src/query/queryparser/helpers_protomsg.go | 37 ++---- .../src/query/queryparser/helpers_struct.go | 8 +- .../query/queryparser/traversal_plpgsql.go | 13 -- .../src/query/queryparser/traversal_proto.go | 35 +---- 24 files changed, 543 insertions(+), 123 deletions(-) create mode 100644 yb-voyager/src/query/queryparser/constants.go diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql index fc5c894f6..972728848 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/functions/function.sql @@ -173,4 +173,18 @@ BEGIN RAISE NOTICE 'High earner salary: %', temp_salary; END LOOP; END; -$$ LANGUAGE plpgsql; \ No newline at end of file +$$ LANGUAGE plpgsql; + +CREATE FUNCTION public.asterisks(n integer) RETURNS SETOF text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE + BEGIN ATOMIC + SELECT repeat('*'::text, g.g) AS repeat + FROM generate_series(1, asterisks.n) g(g); +END; +--TODO fix our parser for this case - splitting it into two "CREATE FUNCTION ...FROM generate_series(1, asterisks.n) g(g);" "END; + +CREATE FUNCTION add(int, int) RETURNS int IMMUTABLE PARALLEL SAFE BEGIN ATOMIC; SELECT $1 + $2; END; + +CREATE FUNCTION public.asterisks1(n integer) RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE + RETURN repeat('*'::text, n); \ No newline at end of file diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index c0c5b2278..3d7c83fd3 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -2067,6 +2067,39 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "FUNCTION", + "ObjectName": "public.asterisks1", + "Reason": "SQL Body for sql languages in function statement is not supported in YugabyteDB", + "SqlStatement": "CREATE FUNCTION public.asterisks1(n integer) RETURNS text\n LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n RETURN repeat('*'::text, n);", + "Suggestion": "No workaround available", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "FUNCTION", + "ObjectName": "public.asterisks", + "Reason": "SQL Body for sql languages in function statement is not supported in YugabyteDB", + "SqlStatement": "CREATE FUNCTION public.asterisks(n integer) RETURNS SETOF text\n LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n BEGIN ATOMIC\n SELECT repeat('*'::text, g.g) AS repeat\n FROM generate_series(1, asterisks.n) g(g);\nEND;", + "Suggestion": "No workaround available", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "FUNCTION", + "ObjectName": "add", + "Reason": "SQL Body for sql languages in function statement is not supported in YugabyteDB", + "SqlStatement": "CREATE FUNCTION add(int, int) RETURNS int IMMUTABLE PARALLEL SAFE BEGIN ATOMIC; SELECT $1 + $2; END;", + "Suggestion": "No workaround available", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_datatypes", "ObjectType": "TABLE", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 80dee631a..77d7c992d 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -44,9 +44,9 @@ }, { "ObjectType": "FUNCTION", - "TotalCount": 7, - "InvalidCount": 7, - "ObjectNames": "create_and_populate_tables, public.get_employeee_salary, get_employee_details, calculate_tax, log_salary_change, list_high_earners, copy_high_earners" + "TotalCount": 10, + "InvalidCount": 10, + "ObjectNames": "public.asterisks, add, public.asterisks1, create_and_populate_tables, public.get_employeee_salary, get_employee_details, calculate_tax, log_salary_change, list_high_earners, copy_high_earners" }, { "ObjectType": "PROCEDURE", diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index 29b451e19..490dcfd35 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -25,9 +25,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 7, + "TotalCount": 8, "InvalidCount": 2, - "ObjectNames": "analytics.metrics, sales.orders, sales.test_json_chk, sales.events, sales.json_data, sales.customer_account, sales.recent_transactions" + "ObjectNames": "sales.big_table, analytics.metrics, sales.orders, sales.test_json_chk, sales.events, sales.json_data, sales.customer_account, sales.recent_transactions" }, { "ObjectType": "SEQUENCE", @@ -53,6 +53,7 @@ "Sizing": { "SizingRecommendation": { "ColocatedTables": [ + "sales.big_table", "sales.orders", "analytics.metrics", "sales.customer_account", @@ -61,7 +62,7 @@ "sales.json_data", "sales.test_json_chk" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 7 objects (7 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 8 objects (8 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": null, "NumNodes": 3, "VCPUsPerInstance": 4, @@ -141,6 +142,20 @@ "ParentTableName": null, "SizeInBytes": 8192 }, + { + "SchemaName": "sales", + "ObjectName": "big_table", + "RowCount": 0, + "ColumnCount": 4, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "sales", "ObjectName": "recent_transactions", @@ -289,6 +304,18 @@ "Query": "SELECT * \nFROM sales.json_data\nWHERE array_column IS JSON ARRAY", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "CTE with MATERIALIZE clause", + "Query": "WITH w AS NOT MATERIALIZED ( \n SELECT * FROM sales.big_table\n) \nSELECT * FROM w AS w1 JOIN w AS w2 ON w1.key = w2.ref\nWHERE w2.key = $1", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", + "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "CTE with MATERIALIZE clause", + "Query": "WITH w AS MATERIALIZED ( \n SELECT * FROM sales.big_table\n) \nSELECT * FROM w AS w1 JOIN w AS w2 ON w1.key = w2.ref\nWHERE w2.key = $1", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": [ diff --git a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql index bb31768d8..d0b6c268d 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql @@ -135,7 +135,7 @@ CREATE TABLE sales.json_data ( unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS) ); -INSERT INTO public.json_data ( +INSERT INTO sales.json_data ( id, data_column, object_column, array_column, scalar_column, unique_keys_column ) VALUES ( 1, '{"key": "value"}', @@ -144,3 +144,10 @@ INSERT INTO public.json_data ( 4, '"hello"', 5, '{"uniqueKey1": "value1", "uniqueKey2": "value2"}' ); + +CREATE TABLE sales.big_table ( + key INT, + ref INT, + other_column_1 TEXT, + other_column_2 TEXT +); \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql index 9133844f1..150a24f18 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql @@ -59,3 +59,21 @@ FROM sales.events; SELECT * FROM sales.json_data WHERE array_column IS JSON ARRAY; + +WITH w AS NOT MATERIALIZED ( + SELECT * FROM sales.big_table +) +SELECT * FROM w AS w1 JOIN w AS w2 ON w1.key = w2.ref +WHERE w2.key = 123; + +WITH w AS MATERIALIZED ( + SELECT * FROM sales.big_table +) +SELECT * FROM w AS w1 JOIN w AS w2 ON w1.key = w2.ref +WHERE w2.key = 123; + +WITH w AS ( + SELECT * FROM sales.big_table +) +SELECT * FROM w AS w1 JOIN w AS w2 ON w1.key = w2.ref +WHERE w2.key = 123; \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 8977b993c..4583b9abb 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -56,9 +56,9 @@ }, { "ObjectType": "FUNCTION", - "TotalCount": 11, - "InvalidCount": 4, - "ObjectNames": "public.manage_large_object, public.auditlogfunc, public.check_sales_region, public.prevent_update_shipped_without_date, public.process_combined_tbl, public.process_order, public.total, schema2.auditlogfunc, schema2.prevent_update_shipped_without_date, schema2.process_order, schema2.total" }, + "TotalCount": 15, + "InvalidCount": 8, + "ObjectNames": "public.asterisks, schema2.asterisks, public.asterisks1, schema2.asterisks1, public.manage_large_object, public.auditlogfunc, public.check_sales_region, public.prevent_update_shipped_without_date, public.process_combined_tbl, public.process_order, public.total, schema2.auditlogfunc, schema2.prevent_update_shipped_without_date, schema2.process_order, schema2.total" }, { "ObjectType": "AGGREGATE", "TotalCount": 2, @@ -503,6 +503,29 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", "MinimumVersionsFixedIn": null }, + { + "FeatureName": "SQL Body in function", + "Objects": [ + { + "ObjectName": "public.asterisks", + "SqlStatement": "CREATE FUNCTION public.asterisks(n integer) RETURNS SETOF text\n LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n BEGIN ATOMIC\n SELECT repeat('*'::text, g.g) AS repeat\n FROM generate_series(1, asterisks.n) g(g);\nEND;" + }, + { + "ObjectName": "public.asterisks1", + "SqlStatement": "CREATE FUNCTION public.asterisks1(n integer) RETURNS text\n LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n RETURN repeat('*'::text, n);" + }, + { + "ObjectName": "schema2.asterisks", + "SqlStatement": "CREATE FUNCTION schema2.asterisks(n integer) RETURNS SETOF text\n LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n BEGIN ATOMIC\n SELECT repeat('*'::text, g.g) AS repeat\n FROM generate_series(1, asterisks.n) g(g);\nEND;" + }, + { + "ObjectName": "schema2.asterisks1", + "SqlStatement": "CREATE FUNCTION schema2.asterisks1(n integer) RETURNS text\n LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE\n RETURN repeat('*'::text, n);" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", + "MinimumVersionsFixedIn": null + }, { "FeatureName": "View with check option", "Objects": [ diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index d19486231..37db10002 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -478,3 +478,16 @@ CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email NULLS NOT DISTINCT; + +CREATE OR REPLACE FUNCTION asterisks(n int) + RETURNS SETOF text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC +SELECT repeat('*', g) FROM generate_series (1, n) g; +END; +-- BEGIN ATOMIC syntax is not working with regex parser we have for functions TODO: fix + +CREATE OR REPLACE FUNCTION asterisks1(n int) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +RETURN repeat('*', n); \ No newline at end of file diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 8a11962cb..944fd9f26 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1071,6 +1071,8 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSONB_SUBSCRIPTING_NAME, "", queryissue.JSONB_SUBSCRIPTING, schemaAnalysisReport, false)) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false)) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false)) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SQL_BODY_IN_FUNCTION_NAME, "", queryissue.SQL_BODY_IN_FUNCTION, schemaAnalysisReport, false,)) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.CTE_WITH_MATERIALIZED_CLAUSE_NAME, "", queryissue.CTE_WITH_MATERIALIZED_CLAUSE, schemaAnalysisReport, false)) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { return len(f.Objects) > 0 diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 832258a53..eb652831d 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -87,6 +87,11 @@ const ( FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE = "FOREIGN_KEY_REFERENCED_PARTITIONED_TABLE" FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME = "Foreign key constraint references partitioned table" + CTE_WITH_MATERIALIZED_CLAUSE = "CTE_WITH_MATERIALIZED_CLAUSE" + CTE_WITH_MATERIALIZED_CLAUSE_NAME = "CTE with MATERIALIZE clause" + + SQL_BODY_IN_FUNCTION = "SQL_BODY_IN_FUNCTION" + SQL_BODY_IN_FUNCTION_NAME = "SQL Body in function" UNIQUE_NULLS_NOT_DISTINCT = "UNIQUE_NULLS_NOT_DISTINCT" UNIQUE_NULLS_NOT_DISTINCT_NAME = "Unique Nulls Not Distinct" ) diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index c62f473ed..7f6c895d0 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -554,3 +554,42 @@ func (j *JsonPredicateExprDetector) GetIssues() []QueryIssue { } return issues } + +type CommonTableExpressionDetector struct { + query string + materializedClauseDetected bool +} + +func NewCommonTableExpressionDetector(query string) *CommonTableExpressionDetector { + return &CommonTableExpressionDetector{ + query: query, + } +} + +func (c *CommonTableExpressionDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) != queryparser.PG_QUERY_CTE_NODE { + return nil + } + /* + with_clause:{ctes:{common_table_expr:{ctename:"cte" ctematerialized:CTEMaterializeNever + ctequery:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}} location:939}} location:939}} from_clause:{range_var:{relname:"a" inh:true relpersistence:"p" location:946}} limit_option:LIMIT_OPTION_DEFAULT op:SETOP_NONE}} location:906}} location:901} op:SETOP_NONE}} stmt_location:898 + */ + cteNode, err := queryparser.ProtoAsCTENode(msg) + if err != nil { + return err + } + if cteNode.Ctematerialized != queryparser.CTE_MATERIALIZED_DEFAULT { + //MATERIALIZED / NOT MATERIALIZED clauses in CTE is not supported in YB + c.materializedClauseDetected = true + } + return nil +} + +func (c *CommonTableExpressionDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if c.materializedClauseDetected { + issues = append(issues, NewCTEWithMaterializedIssue(DML_QUERY_OBJECT_TYPE, "", c.query)) + } + return issues +} + diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index 09c250445..738d068b3 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -603,6 +603,25 @@ func (v *ViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssu return issues, nil } +// ================FUNCTION ISSUE DETECTOR ================== + +type FunctionIssueDetector struct{} + +func (v *FunctionIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { + function, ok := obj.(*queryparser.Function) + if !ok { + return nil, fmt.Errorf("invalid object type: expected View") + } + var issues []QueryIssue + + if function.HasSqlBody { + //https://www.postgresql.org/docs/15/sql-createfunction.html#:~:text=a%20new%20session.-,sql_body,-The%20body%20of + issues = append(issues, NewSqlBodyInFunctionIssue(function.GetObjectType(), function.GetObjectName(), "")) + } + + return issues, nil +} + // ==============MVIEW ISSUE DETECTOR ====================== type MViewIssueDetector struct{} @@ -667,6 +686,8 @@ func (p *ParserIssueDetector) GetDDLDetector(obj queryparser.DDLObject) (DDLIssu return &ViewIssueDetector{}, nil case *queryparser.MView: return &MViewIssueDetector{}, nil + case *queryparser.Function: + return &FunctionIssueDetector{}, nil case *queryparser.Collation: return &CollationIssueDetector{}, nil default: diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index b3f49895e..32839b5f6 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -563,6 +563,20 @@ func NewForeignKeyReferencesPartitionedTableIssue(objectType string, objectName return newQueryIssue(foreignKeyReferencesPartitionedTableIssue, objectType, objectName, SqlStatement, details) } +var sqlBodyInFunctionIssue = issue.Issue{ + Type: SQL_BODY_IN_FUNCTION, + Name: SQL_BODY_IN_FUNCTION_NAME, + Impact: constants.IMPACT_LEVEL_1, + Description: "SQL Body for sql languages in function statement is not supported in YugabyteDB", + Suggestion: "No workaround available", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", +} + +func NewSqlBodyInFunctionIssue(objectType string, objectName string, SqlStatement string) QueryIssue { + return newQueryIssue(sqlBodyInFunctionIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} + var uniqueNullsNotDistinctIssue = issue.Issue{ Type: UNIQUE_NULLS_NOT_DISTINCT, Name: UNIQUE_NULLS_NOT_DISTINCT_NAME, diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 01f90aa0f..1eb15b9d0 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -299,6 +299,31 @@ func testForeignKeyReferencesPartitionedTableIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `cannot reference partitioned table "abc1"`, foreignKeyReferencesPartitionedTableIssue) } +func testSQLBodyInFunctionIssue(t *testing.T) { + sqls := map[string]string{ + `CREATE OR REPLACE FUNCTION asterisks(n int) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +RETURN repeat('*', n);`: `syntax error at or near "RETURN"`, + `CREATE OR REPLACE FUNCTION asterisks(n int) + RETURNS SETOF text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC +SELECT repeat('*', g) FROM generate_series (1, n) g; +END;`: `syntax error at or near "BEGIN"`, + } + for sql, errMsg := range sqls { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, sql) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, errMsg, sqlBodyInFunctionIssue) + } +} + func testUniqueNullsNotDistinctIssue(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -381,6 +406,9 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "foreign key referenced partitioned table", ybVersion), testForeignKeyReferencesPartitionedTableIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "sql body in function", ybVersion), testSQLBodyInFunctionIssue) + assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "unique nulls not distinct", ybVersion), testUniqueNullsNotDistinctIssue) assert.True(t, success) } diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 8f15698d0..b896e3325 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -220,6 +220,20 @@ func NewFetchWithTiesIssue(objectType string, objectName string, sqlStatement st return newQueryIssue(fetchWithTiesIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } +var cteWithMaterializedIssue = issue.Issue{ + Type: CTE_WITH_MATERIALIZED_CLAUSE, + Name: CTE_WITH_MATERIALIZED_CLAUSE_NAME, + Impact: constants.IMPACT_LEVEL_2, + Description: "Modifying the materialization of CTE is not supported yet in YugabyteDB.", + Suggestion: "No workaround available right now", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25575", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", +} + +func NewCTEWithMaterializedIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(cteWithMaterializedIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + var mergeStatementIssue = issue.Issue{ Type: MERGE_STATEMENT, Name: "Merge Statement", diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index c18352fc7..5e86ebfda 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -267,6 +267,34 @@ FROM events;`, } } +func testCTEWithMaterializedIssue(t *testing.T) { + sqls := map[string]string{`WITH w AS NOT MATERIALIZED ( + SELECT * FROM big_table + ) + SELECT * FROM w AS w1 JOIN w AS w2 ON w1.key = w2.ref + WHERE w2.key = 123;`: `syntax error at or near "NOT"`, + `WITH moved_rows AS MATERIALIZED ( + DELETE FROM products + WHERE + "date" >= '2010-10-01' AND + "date" < '2010-11-01' + RETURNING * + ) + INSERT INTO products_log + SELECT * FROM moved_rows;`: `syntax error at or near "MATERIALIZED"`, + } + for sql, errMsg := range sqls { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, sql) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, errMsg, cteWithMaterializedIssue) + } +} + func TestDMLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -323,4 +351,7 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "json type predicate", ybVersion), testJsonPredicateIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "cte with materialized cluase", ybVersion), testCTEWithMaterializedIssue) + assert.True(t, success) + } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 19c84fe90..de3450f20 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -397,6 +397,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewJsonbSubscriptingDetector(query, p.jsonbColumns, p.getJsonbReturnTypeFunctions()), NewUniqueNullsNotDistinctDetector(query), NewJsonPredicateExprDetector(query), + NewCommonTableExpressionDetector(query), } processor := func(msg protoreflect.Message) error { diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 744abc25f..6ad5e0eda 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -1040,3 +1040,123 @@ REFERENCES schema1.abc (id); } } } + +func TestCTEIssues(t *testing.T) { + sqls := []string{ + `WITH w AS ( + SELECT key, very_expensive_function(val) as f FROM some_table +) +SELECT * FROM w AS w1 JOIN w AS w2 ON w1.f = w2.f;`, + `WITH w AS NOT MATERIALIZED ( + SELECT * FROM big_table +) +SELECT * FROM w AS w1 JOIN w AS w2 ON w1.key = w2.ref +WHERE w2.key = 123;`, + `WITH moved_rows AS MATERIALIZED ( + DELETE FROM products + WHERE + "date" >= '2010-10-01' AND + "date" < '2010-11-01' + RETURNING * +) +INSERT INTO products_log +SELECT * FROM moved_rows;`, +`CREATE VIEW view1 AS +WITH data_cte AS NOT MATERIALIZED ( + SELECT + generate_series(1, 5) AS id, + 'Name ' || generate_series(1, 5) AS name +) +SELECT * FROM data_cte;`, +`CREATE VIEW view2 AS +WITH data_cte AS MATERIALIZED ( + SELECT + generate_series(1, 5) AS id, + 'Name ' || generate_series(1, 5) AS name +) +SELECT * FROM data_cte;`, +`CREATE VIEW view3 AS +WITH data_cte AS ( + SELECT + generate_series(1, 5) AS id, + 'Name ' || generate_series(1, 5) AS name +) +SELECT * FROM data_cte;`, + } + + stmtsWithExpectedIssues := map[string][]QueryIssue{ + sqls[0]: []QueryIssue{}, + sqls[1]: []QueryIssue{ + NewCTEWithMaterializedIssue(DML_QUERY_OBJECT_TYPE, "", sqls[1]), + }, + sqls[2]: []QueryIssue{ + NewCTEWithMaterializedIssue(DML_QUERY_OBJECT_TYPE, "", sqls[2]), + }, + sqls[3]: []QueryIssue{ + NewCTEWithMaterializedIssue("VIEW", "view1", sqls[3]), + }, + sqls[4]: []QueryIssue{ + NewCTEWithMaterializedIssue("VIEW", "view2", sqls[4]), + }, + sqls[5]: []QueryIssue{}, + } + + parserIssueDetector := NewParserIssueDetector() + for stmt, expectedIssues := range stmtsWithExpectedIssues { + issues, err := parserIssueDetector.GetAllIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } + +} + +func TestSQLBodyIssues(t *testing.T) { + sqls := []string{ + `CREATE OR REPLACE FUNCTION asterisks(n int) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +RETURN repeat('*', n);`, + `CREATE OR REPLACE FUNCTION asterisks(n int) + RETURNS SETOF text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC +SELECT repeat('*', g) FROM generate_series (1, n) g; +END;`, + `CREATE OR REPLACE FUNCTION asterisks(n int) + RETURNS SETOF text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS +$func$ +SELECT repeat('*', g) FROM generate_series (1, n) g; +$func$;`, + } + + stmtsWithExpectedIssues := map[string][]QueryIssue{ + sqls[0]: []QueryIssue{ + NewSqlBodyInFunctionIssue("FUNCTION", "asterisks", sqls[0]), + }, + sqls[1]: []QueryIssue{ + NewSqlBodyInFunctionIssue("FUNCTION", "asterisks", sqls[1]), + }, + sqls[2]: []QueryIssue{}, + } + parserIssueDetector := NewParserIssueDetector() + for stmt, expectedIssues := range stmtsWithExpectedIssues { + issues, err := parserIssueDetector.GetDDLIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} diff --git a/yb-voyager/src/query/queryparser/constants.go b/yb-voyager/src/query/queryparser/constants.go new file mode 100644 index 000000000..d6cb6a757 --- /dev/null +++ b/yb-voyager/src/query/queryparser/constants.go @@ -0,0 +1,104 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package queryparser + +import pg_query "github.com/pganalyze/pg_query_go/v6" + +const ( + PLPGSQL_EXPR = "PLpgSQL_expr" + QUERY = "query" + + ACTION = "action" + DATUMS = "datums" + PLPGSQL_VAR = "PLpgSQL_var" + DATATYPE = "datatype" + TYPENAME = "typname" + PLPGSQL_TYPE = "PLpgSQL_type" + PLPGSQL_FUNCTION = "PLpgSQL_function" + + TABLE_OBJECT_TYPE = "TABLE" + TYPE_OBJECT_TYPE = "TYPE" + VIEW_OBJECT_TYPE = "VIEW" + MVIEW_OBJECT_TYPE = "MVIEW" + FOREIGN_TABLE_OBJECT_TYPE = "FOREIGN TABLE" + FUNCTION_OBJECT_TYPE = "FUNCTION" + PROCEDURE_OBJECT_TYPE = "PROCEDURE" + INDEX_OBJECT_TYPE = "INDEX" + POLICY_OBJECT_TYPE = "POLICY" + TRIGGER_OBJECT_TYPE = "TRIGGER" + COLLATION_OBJECT_TYPE = "COLLATION" + PG_QUERY_CREATE_STMT = "pg_query.CreateStmt" + PG_QUERY_INDEX_STMT = "pg_query.IndexStmt" + PG_QUERY_ALTER_TABLE_STMT = "pg_query.AlterTableStmt" + PG_QUERY_POLICY_STMT = "pg_query.CreatePolicyStmt" + PG_QUERY_CREATE_TRIG_STMT = "pg_query.CreateTrigStmt" + PG_QUERY_COMPOSITE_TYPE_STMT = "pg_query.CompositeTypeStmt" + PG_QUERY_ENUM_TYPE_STMT = "pg_query.CreateEnumStmt" + PG_QUERY_FOREIGN_TABLE_STMT = "pg_query.CreateForeignTableStmt" + PG_QUERY_VIEW_STMT = "pg_query.ViewStmt" + PG_QUERY_CREATE_TABLE_AS_STMT = "pg_query.CreateTableAsStmt" + PG_QUERY_CREATE_FUNCTION_STMT = "pg_query.CreateFunctionStmt" + + PG_QUERY_NODE_NODE = "pg_query.Node" + PG_QUERY_STRING_NODE = "pg_query.String" + PG_QUERY_ASTAR_NODE = "pg_query.A_Star" + PG_QUERY_ACONST_NODE = "pg_query.A_Const" + PG_QUERY_TYPECAST_NODE = "pg_query.TypeCast" + PG_QUERY_XMLEXPR_NODE = "pg_query.XmlExpr" + PG_QUERY_FUNCCALL_NODE = "pg_query.FuncCall" + PG_QUERY_COLUMNREF_NODE = "pg_query.ColumnRef" + PG_QUERY_RANGEFUNCTION_NODE = "pg_query.RangeFunction" + PG_QUERY_RANGEVAR_NODE = "pg_query.RangeVar" + PG_QUERY_RANGETABLEFUNC_NODE = "pg_query.RangeTableFunc" + PG_QUERY_PARAMREF_NODE = "pg_query.ParamRef" + PG_QUERY_DEFELEM_NODE = "pg_query.DefElem" + + PG_QUERY_INSERTSTMT_NODE = "pg_query.InsertStmt" + PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" + PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" + PG_QUERY_SELECTSTMT_NODE = "pg_query.SelectStmt" + + PG_QUERY_A_INDIRECTION_NODE = "pg_query.A_Indirection" + PG_QUERY_JSON_OBJECT_AGG_NODE = "pg_query.JsonObjectAgg" + PG_QUERY_JSON_ARRAY_AGG_NODE = "pg_query.JsonArrayAgg" + PG_QUERY_JSON_ARRAY_CONSTRUCTOR_AGG_NODE = "pg_query.JsonArrayConstructor" + PG_QUERY_JSON_FUNC_EXPR_NODE = "pg_query.JsonFuncExpr" + PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" + PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" + PG_QUERY_JSON_IS_PREDICATE_NODE = "pg_query.JsonIsPredicate" + PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" + PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" + PG_QUERY_CONSTRAINT_NODE = "pg_query.Constraint" + PG_QUERY_CTE_NODE = "pg_query.CommonTableExpr" + PG_QUERY_VIEW_STMT_NODE = "pg_query.ViewStmt" + PG_QUERY_COPY_STMT_NODE = "pg_query.CopyStmt" + PG_QUERY_DEFINE_STMT_NODE = "pg_query.DefineStmt" + PG_QUERY_MERGE_STMT_NODE = "pg_query.MergeStmt" + PG_QUERY_INDEX_STMT_NODE = "pg_query.IndexStmt" + + LIMIT_OPTION_WITH_TIES = pg_query.LimitOption_LIMIT_OPTION_WITH_TIES + CTE_MATERIALIZED_DEFAULT = pg_query.CTEMaterialize_CTEMaterializeDefault + ADD_CONSTRAINT = pg_query.AlterTableType_AT_AddConstraint + SET_OPTIONS = pg_query.AlterTableType_AT_SetOptions + DISABLE_RULE = pg_query.AlterTableType_AT_DisableRule + CLUSTER_ON = pg_query.AlterTableType_AT_ClusterOn + EXCLUSION_CONSTR_TYPE = pg_query.ConstrType_CONSTR_EXCLUSION + FOREIGN_CONSTR_TYPE = pg_query.ConstrType_CONSTR_FOREIGN + DEFAULT_SORTING_ORDER = pg_query.SortByDir_SORTBY_DEFAULT + PRIMARY_CONSTR_TYPE = pg_query.ConstrType_CONSTR_PRIMARY + UNIQUE_CONSTR_TYPE = pg_query.ConstrType_CONSTR_UNIQUE + LIST_PARTITION = pg_query.PartitionStrategy_PARTITION_STRATEGY_LIST +) diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index 895d6f518..8e8da06c0 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -1021,6 +1021,7 @@ func (mv *FunctionProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject SchemaName: funcSchemaName, FuncName: funcName, ReturnType: GetReturnTypeOfFunc(parseTree), + HasSqlBody: funcNode.CreateFunctionStmt.GetSqlBody() != nil, } return &function, nil } @@ -1029,6 +1030,7 @@ type Function struct { SchemaName string FuncName string ReturnType string + HasSqlBody bool } func (f *Function) GetObjectName() string { @@ -1097,41 +1099,6 @@ func GetDDLProcessor(parseTree *pg_query.ParseResult) (DDLProcessor, error) { } } -const ( - TABLE_OBJECT_TYPE = "TABLE" - TYPE_OBJECT_TYPE = "TYPE" - VIEW_OBJECT_TYPE = "VIEW" - MVIEW_OBJECT_TYPE = "MVIEW" - FOREIGN_TABLE_OBJECT_TYPE = "FOREIGN TABLE" - FUNCTION_OBJECT_TYPE = "FUNCTION" - PROCEDURE_OBJECT_TYPE = "PROCEDURE" - INDEX_OBJECT_TYPE = "INDEX" - POLICY_OBJECT_TYPE = "POLICY" - TRIGGER_OBJECT_TYPE = "TRIGGER" - COLLATION_OBJECT_TYPE = "COLLATION" - ADD_CONSTRAINT = pg_query.AlterTableType_AT_AddConstraint - SET_OPTIONS = pg_query.AlterTableType_AT_SetOptions - DISABLE_RULE = pg_query.AlterTableType_AT_DisableRule - CLUSTER_ON = pg_query.AlterTableType_AT_ClusterOn - EXCLUSION_CONSTR_TYPE = pg_query.ConstrType_CONSTR_EXCLUSION - FOREIGN_CONSTR_TYPE = pg_query.ConstrType_CONSTR_FOREIGN - DEFAULT_SORTING_ORDER = pg_query.SortByDir_SORTBY_DEFAULT - PRIMARY_CONSTR_TYPE = pg_query.ConstrType_CONSTR_PRIMARY - UNIQUE_CONSTR_TYPE = pg_query.ConstrType_CONSTR_UNIQUE - LIST_PARTITION = pg_query.PartitionStrategy_PARTITION_STRATEGY_LIST - PG_QUERY_CREATE_STMT = "pg_query.CreateStmt" - PG_QUERY_INDEX_STMT = "pg_query.IndexStmt" - PG_QUERY_ALTER_TABLE_STMT = "pg_query.AlterTableStmt" - PG_QUERY_POLICY_STMT = "pg_query.CreatePolicyStmt" - PG_QUERY_CREATE_TRIG_STMT = "pg_query.CreateTrigStmt" - PG_QUERY_COMPOSITE_TYPE_STMT = "pg_query.CompositeTypeStmt" - PG_QUERY_ENUM_TYPE_STMT = "pg_query.CreateEnumStmt" - PG_QUERY_FOREIGN_TABLE_STMT = "pg_query.CreateForeignTableStmt" - PG_QUERY_VIEW_STMT = "pg_query.ViewStmt" - PG_QUERY_CREATE_TABLE_AS_STMT = "pg_query.CreateTableAsStmt" - PG_QUERY_CREATE_FUNCTION_STMT = "pg_query.CreateFunctionStmt" -) - var deferrableConstraintsList = []pg_query.ConstrType{ pg_query.ConstrType_CONSTR_ATTR_DEFERRABLE, pg_query.ConstrType_CONSTR_ATTR_DEFERRED, diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 174270d72..c8155c3b8 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -21,18 +21,9 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" ) -const ( - DOCS_LINK_PREFIX = "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/" - POSTGRESQL_PREFIX = "postgresql/" - ADVISORY_LOCKS_DOC_LINK = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#advisory-locks-is-not-yet-implemented" - SYSTEM_COLUMNS_DOC_LINK = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#system-columns-is-not-yet-supported" - XML_FUNCTIONS_DOC_LINK = DOCS_LINK_PREFIX + POSTGRESQL_PREFIX + "#xml-functions-is-not-yet-supported" -) - func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect.Message { return parseTree.Stmts[0].Stmt.ProtoReflect() @@ -409,39 +400,37 @@ func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { } func ProtoAsSelectStmt(msg protoreflect.Message) (*pg_query.SelectStmt, error) { - protoMsg, ok := msg.Interface().(proto.Message) - if !ok { - return nil, fmt.Errorf("failed to cast msg to proto.Message") - } - selectStmtNode, ok := protoMsg.(*pg_query.SelectStmt) + selectStmtNode, ok := msg.Interface().(*pg_query.SelectStmt) if !ok { return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_SELECTSTMT_NODE) } return selectStmtNode, nil } -func ProtoAsIndexStmt(msg protoreflect.Message) (*pg_query.IndexStmt, error) { - protoMsg, ok := msg.Interface().(proto.Message) +func ProtoAsCTENode(msg protoreflect.Message) (*pg_query.CommonTableExpr, error) { + cteNode, ok := msg.Interface().(*pg_query.CommonTableExpr) if !ok { - return nil, fmt.Errorf("failed to cast msg to proto.Message") + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_CTE_NODE) } - indexStmtNode, ok := protoMsg.(*pg_query.IndexStmt) + return cteNode, nil +} + +func ProtoAsIndexStmt(msg protoreflect.Message) (*pg_query.IndexStmt, error) { + indexStmtNode, ok := msg.Interface().(*pg_query.IndexStmt) if !ok { return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_INDEX_STMT_NODE) } + return indexStmtNode, nil } func ProtoAsTableConstraint(msg protoreflect.Message) (*pg_query.Constraint, error) { - proto, ok := msg.Interface().(proto.Message) - if !ok { - return nil, fmt.Errorf("failed to cast msg to proto.Message") - } - constraintNode, ok := proto.(*pg_query.Constraint) + consNode, ok := msg.Interface().(*pg_query.Constraint) if !ok { return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_CONSTRAINT_NODE) } - return constraintNode, nil + + return consNode, nil } /* diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index 46d02b5ed..bfd70d8c1 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -25,14 +25,10 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) -const ( - LIMIT_OPTION_WITH_TIES = pg_query.LimitOption_LIMIT_OPTION_WITH_TIES -) - func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE - _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) - return isPlPgSQLObject + node, ok := getCreateFuncStmtNode(parseTree) + return ok && (node.CreateFunctionStmt.SqlBody == nil) //TODO fix proper https://github.com/pganalyze/pg_query_go/issues/129 } func IsViewObject(parseTree *pg_query.ParseResult) bool { diff --git a/yb-voyager/src/query/queryparser/traversal_plpgsql.go b/yb-voyager/src/query/queryparser/traversal_plpgsql.go index bc6a3c419..aa5f7db78 100644 --- a/yb-voyager/src/query/queryparser/traversal_plpgsql.go +++ b/yb-voyager/src/query/queryparser/traversal_plpgsql.go @@ -23,19 +23,6 @@ import ( log "github.com/sirupsen/logrus" ) -const ( - PLPGSQL_EXPR = "PLpgSQL_expr" - QUERY = "query" - - ACTION = "action" - DATUMS = "datums" - PLPGSQL_VAR = "PLpgSQL_var" - DATATYPE = "datatype" - TYPENAME = "typname" - PLPGSQL_TYPE = "PLpgSQL_type" - PLPGSQL_FUNCTION = "PLpgSQL_function" -) - /* * This function is not concrete yet because of following limitation from parser - diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 09b2658ac..3acf4f6ec 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -24,40 +24,7 @@ import ( ) const ( - PG_QUERY_NODE_NODE = "pg_query.Node" - PG_QUERY_STRING_NODE = "pg_query.String" - PG_QUERY_ASTAR_NODE = "pg_query.A_Star" - PG_QUERY_ACONST_NODE = "pg_query.A_Const" - PG_QUERY_TYPECAST_NODE = "pg_query.TypeCast" - PG_QUERY_XMLEXPR_NODE = "pg_query.XmlExpr" - PG_QUERY_FUNCCALL_NODE = "pg_query.FuncCall" - PG_QUERY_COLUMNREF_NODE = "pg_query.ColumnRef" - PG_QUERY_RANGEFUNCTION_NODE = "pg_query.RangeFunction" - PG_QUERY_RANGEVAR_NODE = "pg_query.RangeVar" - PG_QUERY_RANGETABLEFUNC_NODE = "pg_query.RangeTableFunc" - PG_QUERY_PARAMREF_NODE = "pg_query.ParamRef" - PG_QUERY_DEFELEM_NODE = "pg_query.DefElem" - - PG_QUERY_INSERTSTMT_NODE = "pg_query.InsertStmt" - PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" - PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" - PG_QUERY_SELECTSTMT_NODE = "pg_query.SelectStmt" - - PG_QUERY_A_INDIRECTION_NODE = "pg_query.A_Indirection" - PG_QUERY_JSON_OBJECT_AGG_NODE = "pg_query.JsonObjectAgg" - PG_QUERY_JSON_ARRAY_AGG_NODE = "pg_query.JsonArrayAgg" - PG_QUERY_JSON_ARRAY_CONSTRUCTOR_AGG_NODE = "pg_query.JsonArrayConstructor" - PG_QUERY_JSON_FUNC_EXPR_NODE = "pg_query.JsonFuncExpr" - PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" - PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" - PG_QUERY_JSON_IS_PREDICATE_NODE = "pg_query.JsonIsPredicate" - PG_QUERY_VIEW_STMT_NODE = "pg_query.ViewStmt" - PG_QUERY_COPY_STMT_NODE = "pg_query.CopyStmt" - - PG_QUERY_DEFINE_STMT_NODE = "pg_query.DefineStmt" - PG_QUERY_MERGE_STMT_NODE = "pg_query.MergeStmt" - PG_QUERY_CONSTRAINT_NODE = "pg_query.Constraint" - PG_QUERY_INDEX_STMT_NODE = "pg_query.IndexStmt" + ) // function type for processing nodes during traversal