Reporting Json Constructor / Query Functions and AnyValue Aggregate f…

…unctions (#2107)

1. Added support to report the JSON constructor functions (`JSON_OBJECT`, `JSON_OBJECTAGG`, `JSON_ARRAYAGG`, `JSON_ARRAY`) and JSON query functions (`JSON_QUERY`, `JSON_EXISTS`, `JSON_VALUE`) with new detectors as these all have a separate to detect.
Currently, the code and unit tests are added for reporting the `JSON_TABLE` in DMLs but it can't be reported in the Query constructs sections as PGSS is giving the queries with placeholders ($1, $2...) and `JSON_TABLE` syntax doesn't allow it so the parser is failing with that.
https://yugabyte.atlassian.net/browse/DB-14542
2. Added support to report the aggregate functions, reporting the `ANY_VALUE` agg function in this PR. https://yugabyte.atlassian.net/browse/DB-14220
3. Fixed the assessment template for adding the Supported Versions in HTML for PL/pgSQL and Migration Caveats sections.

migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql

@@ -32,4 +32,15 @@ CREATE VIEW public.orders_view AS
     pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired,
     orders.ctid AS row_ctid,
     orders.xmin AS transaction_id
-   FROM public.orders;
+   FROM public.orders;
+
+CREATE VIEW public.my_films_view AS 
+SELECT jt.* FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites[*]'
+   COLUMNS (
+    id FOR ORDINALITY,
+    kind text PATH '$.kind',
+    NESTED PATH '$.films[*]' COLUMNS (
+      title text FORMAT JSON PATH '$.title' OMIT QUOTES,
+      director text PATH '$.director' KEEP QUOTES))) AS jt;

migtests/tests/analyze-schema/expected_issues.json

@@ -20,6 +20,36 @@
             "GH": "https://github.com/yugabyte/yb-voyager/issues/1542",
             "MinimumVersionsFixedIn": null
         },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "VIEW",
+            "ObjectName": "public.my_films_view",
+            "Reason": "Json Query Functions",
+            "SqlStatement": "CREATE VIEW public.my_films_view AS\nSELECT jt.* FROM\n my_films,\n JSON_TABLE ( js, '$.favorites[*]'\n   COLUMNS (\n    id FOR ORDINALITY,\n    kind text PATH '$.kind',\n    NESTED PATH '$.films[*]' COLUMNS (\n      title text FORMAT JSON PATH '$.title' OMIT QUOTES,\n      director text PATH '$.director' KEEP QUOTES))) AS jt;",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "VIEW",
+            "ObjectName": "test",
+            "Reason": "Json Constructor Functions",
+            "SqlStatement": "CREATE OR REPLACE view test AS (\n                            select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n                            FROM test1\n                            where t = '1DAY' group by x\n                            );",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "MVIEW",
+            "ObjectName": "test",
+            "Reason": "Json Constructor Functions",
+            "SqlStatement": "CREATE MATERIALIZED VIEW test AS (\n                            select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n                            FROM test1\n                            where t = '1DAY' group by x\n                            );",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
         {
             "IssueType": "unsupported_features",
             "ObjectType": "TABLE",

migtests/tests/analyze-schema/summary.json

@@ -50,9 +50,9 @@
             },
             {
                 "ObjectType": "VIEW",
-                "TotalCount": 5,
-                "InvalidCount": 5,
-                "ObjectNames": "v1, v2, test, public.orders_view, view_name"
+                "TotalCount": 6,
+                "InvalidCount": 6,
+                "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name"
             },
             {
                 "ObjectType": "TRIGGER",

migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql

@@ -41,4 +41,8 @@ INSERT INTO hr.departments (department_name, location) VALUES ('Engineering', 'B
 INSERT INTO hr.departments (department_name, location) VALUES ('Sales', 'Building B');
 INSERT INTO public.employees (name, department_id) VALUES ('Alice', 1), ('Bob', 1), ('Charlie', 2);
 INSERT INTO sales.orders (customer_id, amount) VALUES (101, 500.00), (102, 1200.00);
-INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRate', 0.023), ('ChurnRate', 0.05);
+INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRate', 0.023), ('ChurnRate', 0.05);
+
+create view sales.employ_depart_view AS  SELECT
+        any_value(name) AS any_employee
+    FROM employees;

migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql

@@ -20,4 +20,9 @@ FROM public.employees;
 -- 4) Advisory locks (analytics schema)
 SELECT metric_name, pg_advisory_lock(metric_id)
 FROM analytics.metrics
-WHERE metric_value > 0.02;
+WHERE metric_value > 0.02;
+
+-- Aggregate functions UQC NOT REPORTING as it need PG16 upgarde in pipeline from PG15
+SELECT
+        any_value(name) AS any_employee
+    FROM employees;

yb-voyager/cmd/assessMigrationCommand.go

@@ -1023,6 +1023,9 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, ""))
+	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, ""))
+	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, ""))
+	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, ""))
 
 	return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool {

yb-voyager/cmd/templates/migration_assessment_report.template

@@ -286,7 +286,7 @@
                     {{ if $docsLink }}
                         <a href="{{ $docsLink }}" target="_blank">Docs Link</a>
                     {{ else }}
-                        Not Available
+                        N/A
                     {{ end }}
                 </td>
                 </tr>
@@ -313,6 +313,7 @@
                     {{ $objectsGroupByObjectType := groupByObjectType .Objects }}
                     {{ $numUniqueObjectNamesOfAllTypes := totalUniqueObjectNamesOfAllTypes $objectsGroupByObjectType }}
                     {{ $docsLink := .DocsLink }}
+                    {{ $supportedVerStr := getSupportedVersionString .MinimumVersionsFixedIn }}
                     <td rowspan={{ $numUniqueObjectNamesOfAllTypes }}><strong>{{ .FeatureName }}</strong></td>
                     {{ $isNextRowRequiredForObjectType := false }}
                     {{ range $type, $objectsByType := $objectsGroupByObjectType }}
@@ -338,7 +339,16 @@
                                 </div>
                             </td>
                             {{ if not $isNextRowRequiredForObjectType }}
-                                <td rowspan={{ $numUniqueObjectNamesOfAllTypes }}><a href="{{ $docsLink }}" target="_blank">Link</a></td>
+                                <td rowspan={{ $numUniqueObjectNamesOfAllTypes }}>
+                                    {{ if $supportedVerStr }}
+                                        Supported in Versions: {{ $supportedVerStr }} <br>
+                                    {{ end }}
+                                    {{ if $docsLink }}
+                                        <a href="{{ $docsLink }}" target="_blank">Docs Link</a>
+                                    {{ else }}
+                                        N/A
+                                    {{ end }}
+                                </td>
                             {{ end }}
                             {{ $isNextRowRequiredForObjectName = true }}
                             {{ $isNextRowRequiredForObjectType = true }}
@@ -359,6 +369,10 @@
                     {{ $hasMigrationCaveats = true }}
                     {{if .DisplayDDL }} <!-- for these feature we are displaying the DDLs-->
                         <h4>{{.FeatureName}}</h4>
+                        {{ $supporterVerStr := getSupportedVersionString .MinimumVersionsFixedIn }}
+                        {{ if $supporterVerStr }}
+                            <p>Supported in Versions: {{ $supporterVerStr }} </p>
+                        {{ end }}
                         <p>{{.FeatureDescription}}</p>
                         <div class="scrollable-div">
                             <ul>
@@ -369,6 +383,10 @@
                         </div>
                     {{else}}
                         <h4>{{.FeatureName}}</h4>
+                        {{ $supporterVerStr := getSupportedVersionString .MinimumVersionsFixedIn }}
+                        {{ if $supporterVerStr }}
+                            <p>Supported in Versions: {{ $supporterVerStr }} </p>
+                        {{ end }}
                         <p>{{.FeatureDescription}}</p>
                         <div class="scrollable-div">
                             <ul>

yb-voyager/src/query/queryissue/constants.go

@@ -49,9 +49,15 @@ const (
 	FOREIGN_TABLE                                   = "FOREIGN_TABLE"
 	INHERITANCE                                     = "INHERITANCE"
 
-	LARGE_OBJECT_DATATYPE       = "LARGE_OBJECT_DATATYPE"
-	LARGE_OBJECT_FUNCTIONS      = "LARGE_OBJECT_FUNCTIONS"
-	LARGE_OBJECT_FUNCTIONS_NAME = "Large Object Functions"
+	AGGREGATE_FUNCTION             = "AGGREGATE_FUNCTION"
+	AGGREGATION_FUNCTIONS_NAME     = "Aggregate Functions"
+	JSON_CONSTRUCTOR_FUNCTION      = "JSON_CONSTRUCTOR_FUNCTION"
+	JSON_CONSTRUCTOR_FUNCTION_NAME = "Json Constructor Functions"
+	JSON_QUERY_FUNCTION            = "JSON_QUERY_FUNCTION"
+	JSON_QUERY_FUNCTIONS_NAME      = "Json Query Functions"
+	LARGE_OBJECT_DATATYPE          = "LARGE_OBJECT_DATATYPE"
+	LARGE_OBJECT_FUNCTIONS         = "LARGE_OBJECT_FUNCTIONS"
+	LARGE_OBJECT_FUNCTIONS_NAME    = "Large Object Functions"
 
 	SECURITY_INVOKER_VIEWS      = "SECURITY_INVOKER_VIEWS"
 	SECURITY_INVOKER_VIEWS_NAME = "Security Invoker Views"
@@ -69,6 +75,7 @@ const (
 // Object types
 const (
 	CONSTRAINT_NAME           = "ConstraintName"
+	FUNCTION_NAMES            = "FunctionNames"
 	TABLE_OBJECT_TYPE         = "TABLE"
 	FOREIGN_TABLE_OBJECT_TYPE = "FOREIGN TABLE"
 	FUNCTION_OBJECT_TYPE      = "FUNCTION"

yb-voyager/src/query/queryissue/detectors.go

@@ -35,6 +35,7 @@ type FuncCallDetector struct {
 
 	advisoryLocksFuncsDetected mapset.Set[string]
 	xmlFuncsDetected           mapset.Set[string]
+	aggFuncsDetected           mapset.Set[string]
 	regexFuncsDetected         mapset.Set[string]
 	loFuncsDetected            mapset.Set[string]
 }
@@ -44,6 +45,7 @@ func NewFuncCallDetector(query string) *FuncCallDetector {
 		query:                      query,
 		advisoryLocksFuncsDetected: mapset.NewThreadUnsafeSet[string](),
 		xmlFuncsDetected:           mapset.NewThreadUnsafeSet[string](),
+		aggFuncsDetected:           mapset.NewThreadUnsafeSet[string](),
 		regexFuncsDetected:         mapset.NewThreadUnsafeSet[string](),
 		loFuncsDetected:            mapset.NewThreadUnsafeSet[string](),
 	}
@@ -68,6 +70,9 @@ func (d *FuncCallDetector) Detect(msg protoreflect.Message) error {
 		d.regexFuncsDetected.Add(funcName)
 	}
 
+	if unsupportedAggFunctions.ContainsOne(funcName) {
+		d.aggFuncsDetected.Add(funcName)
+	}
 	if unsupportedLargeObjectFunctions.ContainsOne(funcName) {
 		d.loFuncsDetected.Add(funcName)
 	}
@@ -83,11 +88,14 @@ func (d *FuncCallDetector) GetIssues() []QueryIssue {
 	if d.xmlFuncsDetected.Cardinality() > 0 {
 		issues = append(issues, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query))
 	}
+	if d.aggFuncsDetected.Cardinality() > 0 {
+		issues = append(issues, NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.aggFuncsDetected.ToSlice()))
+	}
 	if d.regexFuncsDetected.Cardinality() > 0 {
 		issues = append(issues, NewRegexFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query))
 	}
 	if d.loFuncsDetected.Cardinality() > 0 {
-		issues = append(issues, NewLOFuntionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query))
+		issues = append(issues, NewLOFuntionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.loFuncsDetected.ToSlice()))
 	}
 	return issues
 }
@@ -195,3 +203,105 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue {
 	}
 	return issues
 }
+
+type JsonConstructorFuncDetector struct {
+	query                                       string
+	unsupportedJsonConstructorFunctionsDetected mapset.Set[string]
+}
+
+func NewJsonConstructorFuncDetector(query string) *JsonConstructorFuncDetector {
+	return &JsonConstructorFuncDetector{
+		query: query,
+		unsupportedJsonConstructorFunctionsDetected: mapset.NewThreadUnsafeSet[string](),
+	}
+}
+
+func (j *JsonConstructorFuncDetector) Detect(msg protoreflect.Message) error {
+	switch queryparser.GetMsgFullName(msg) {
+	case queryparser.PG_QUERY_JSON_ARRAY_AGG_NODE:
+		j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_ARRAYAGG)
+	case queryparser.PG_QUERY_JSON_ARRAY_CONSTRUCTOR_AGG_NODE:
+		j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_ARRAY)
+	case queryparser.PG_QUERY_JSON_OBJECT_AGG_NODE:
+		j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_OBJECTAGG)
+	case queryparser.PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE:
+		j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_OBJECT)
+	}
+	return nil
+}
+
+func (d *JsonConstructorFuncDetector) GetIssues() []QueryIssue {
+	var issues []QueryIssue
+	if d.unsupportedJsonConstructorFunctionsDetected.Cardinality() > 0 {
+		issues = append(issues, NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.unsupportedJsonConstructorFunctionsDetected.ToSlice()))
+	}
+	return issues
+}
+
+type JsonQueryFunctionDetector struct {
+	query                                 string
+	unsupportedJsonQueryFunctionsDetected mapset.Set[string]
+}
+
+func NewJsonQueryFunctionDetector(query string) *JsonQueryFunctionDetector {
+	return &JsonQueryFunctionDetector{
+		query:                                 query,
+		unsupportedJsonQueryFunctionsDetected: mapset.NewThreadUnsafeSet[string](),
+	}
+}
+
+func (j *JsonQueryFunctionDetector) Detect(msg protoreflect.Message) error {
+	if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_TABLE_NODE {
+		/*
+		SELECT * FROM json_table(
+			'[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb,
+			'$[*]'
+			COLUMNS (
+				column_a int4 path '$.a',
+				column_b int4 path '$.b'
+			)
+		);
+		stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}}  location:530}}  location:530}}  
+		from_clause:{json_table:{context_item:{raw_expr:{type_cast:{arg:{a_const:{sval:{sval:"[{\"a\":10,\"b\":20},{\"a\":30,\"b\":40}]"}  
+		location:553}}  type_name:{names:{string:{sval:"jsonb"}}  .....  name_location:-1  location:601}  
+		columns:{json_table_column:{coltype:JTC_REGULAR  name:"column_a"  type_name:{names:{string:{sval:"int4"}}  typemod:-1  location:639}  
+		pathspec:{string:{a_const:{sval:{sval:"$.a"}  location:649}}  name_location:-1  location:649} ...
+		*/
+		j.unsupportedJsonQueryFunctionsDetected.Add(JSON_TABLE)
+		return nil
+	}
+	if queryparser.GetMsgFullName(msg) != queryparser.PG_QUERY_JSON_FUNC_EXPR_NODE {
+		return nil
+	}
+	/*
+		JsonExprOp -
+			enumeration of SQL/JSON query function types
+		typedef enum JsonExprOp
+		{
+			1. JSON_EXISTS_OP,				 JSON_EXISTS()
+			2. JSON_QUERY_OP,				 JSON_QUERY()
+			3. JSON_VALUE_OP,				 JSON_VALUE()
+			4. JSON_TABLE_OP,				JSON_TABLE()
+		} JsonExprOp;
+	*/
+	jsonExprFuncOpNum := queryparser.GetEnumNumField(msg, "op")
+	switch jsonExprFuncOpNum {
+	case 1:
+		j.unsupportedJsonQueryFunctionsDetected.Add(JSON_EXISTS)
+	case 2:
+		j.unsupportedJsonQueryFunctionsDetected.Add(JSON_QUERY)
+	case 3:
+		j.unsupportedJsonQueryFunctionsDetected.Add(JSON_VALUE)
+	case 4:
+		j.unsupportedJsonQueryFunctionsDetected.Add(JSON_TABLE)
+	}
+	return nil
+}
+
+func (d *JsonQueryFunctionDetector) GetIssues() []QueryIssue {
+	var issues []QueryIssue
+	if d.unsupportedJsonQueryFunctionsDetected.Cardinality() > 0 {
+		issues = append(issues, NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.unsupportedJsonQueryFunctionsDetected.ToSlice()))
+	}
+	return issues
+}

yb-voyager/src/query/queryissue/detectors_ddl.go

@@ -551,6 +551,7 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer
 			obj.GetObjectType(),
 			trigger.GetObjectName(),
 			"",
+			[]string{trigger.FuncName},
 		))
 	}