From e5e8f9cf2ad2e4d7f1d99a74346690d1ca29dea4 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Thu, 12 Dec 2024 11:33:25 +0530 Subject: [PATCH] Send target-db-version, MinimumVersionsFixedIn details for assessment/analyze-schema reports to controlplane/callhome (#2049) In assess-migration - yugabyted control plane send target-db-version include MinFixVersion for all issues - callhome send target-db-version In analyze-schema - yugabyted control plane (Already done because we send the entire schema report) send target-db-version include MinFixVersion for all issues - callhome send target-db-version --- yb-voyager/cmd/analyzeSchema.go | 22 ++++---- yb-voyager/cmd/assessMigrationCommand.go | 58 ++++++++++++--------- yb-voyager/cmd/common.go | 18 ++++--- yb-voyager/src/callhome/diagnostics.go | 31 ++++++----- yb-voyager/src/callhome/diagnostics_test.go | 31 ++++++----- 5 files changed, 88 insertions(+), 72 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 33c45caac4..1e30245a2d 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -626,15 +626,16 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil } return utils.Issue{ - ObjectType: issueInstance.ObjectType, - ObjectName: issueInstance.ObjectName, - Reason: issueInstance.TypeName, - SqlStatement: issueInstance.SqlStatement, - DocsLink: issueInstance.DocsLink, - FilePath: fileName, - IssueType: issueType, - Suggestion: issueInstance.Suggestion, - GH: issueInstance.GH, + ObjectType: issueInstance.ObjectType, + ObjectName: issueInstance.ObjectName, + Reason: issueInstance.TypeName, + SqlStatement: issueInstance.SqlStatement, + DocsLink: issueInstance.DocsLink, + FilePath: fileName, + IssueType: issueType, + Suggestion: issueInstance.Suggestion, + GH: issueInstance.GH, + MinimumVersionsFixedIn: issueInstance.MinimumVersionsFixedIn, } } @@ -1217,7 +1218,8 @@ func packAndSendAnalyzeSchemaPayload(status string) { } analyzePayload := callhome.AnalyzePhasePayload{ - Issues: callhome.MarshalledJsonString(callhomeIssues), + TargetDBVersion: schemaAnalysisReport.TargetDBVersion, + Issues: callhome.MarshalledJsonString(callhomeIssues), DatabaseObjects: callhome.MarshalledJsonString(lo.Map(schemaAnalysisReport.SchemaSummary.DBObjects, func(dbObject utils.DBObject, _ int) utils.DBObject { dbObject.ObjectNames = "" return dbObject diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index b158ef8f15..90fa21cff9 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -163,6 +163,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { }) assessPayload := callhome.AssessMigrationPhasePayload{ + TargetDBVersion: assessmentReport.TargetDBVersion, MigrationComplexity: assessmentReport.MigrationComplexity, UnsupportedFeatures: callhome.MarshalledJsonString(lo.Map(assessmentReport.UnsupportedFeatures, func(feature UnsupportedFeature, _ int) callhome.UnsupportedFeature { var objects []string @@ -473,6 +474,7 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE payload := AssessMigrationPayload{ PayloadVersion: ASSESS_MIGRATION_PAYLOAD_VERSION, VoyagerVersion: assessmentReport.VoyagerVersion, + TargetDBVersion: assessmentReport.TargetDBVersion, MigrationComplexity: assessmentReport.MigrationComplexity, SchemaSummary: assessmentReport.SchemaSummary, AssessmentIssues: assessmentIssues, @@ -525,13 +527,14 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, unsupportedFeature := range ar.UnsupportedFeatures { for _, object := range unsupportedFeature.Objects { issues = append(issues, AssessmentIssuePayload{ - Type: FEATURE, - TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION, - Subtype: unsupportedFeature.FeatureName, - SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features - ObjectName: object.ObjectName, - SqlStatement: object.SqlStatement, - DocsLink: unsupportedFeature.DocsLink, + Type: FEATURE, + TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION, + Subtype: unsupportedFeature.FeatureName, + SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features + ObjectName: object.ObjectName, + SqlStatement: object.SqlStatement, + DocsLink: unsupportedFeature.DocsLink, + MinimumVersionsFixedIn: unsupportedFeature.MinimumVersionsFixedIn, }) } } @@ -539,37 +542,40 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, migrationCaveat := range ar.MigrationCaveats { for _, object := range migrationCaveat.Objects { issues = append(issues, AssessmentIssuePayload{ - Type: MIGRATION_CAVEATS, - TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION, - Subtype: migrationCaveat.FeatureName, - SubtypeDescription: migrationCaveat.FeatureDescription, - ObjectName: object.ObjectName, - SqlStatement: object.SqlStatement, - DocsLink: migrationCaveat.DocsLink, + Type: MIGRATION_CAVEATS, + TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION, + Subtype: migrationCaveat.FeatureName, + SubtypeDescription: migrationCaveat.FeatureDescription, + ObjectName: object.ObjectName, + SqlStatement: object.SqlStatement, + DocsLink: migrationCaveat.DocsLink, + MinimumVersionsFixedIn: migrationCaveat.MinimumVersionsFixedIn, }) } } for _, uqc := range ar.UnsupportedQueryConstructs { issues = append(issues, AssessmentIssuePayload{ - Type: QUERY_CONSTRUCT, - TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION, - Subtype: uqc.ConstructTypeName, - SqlStatement: uqc.Query, - DocsLink: uqc.DocsLink, + Type: QUERY_CONSTRUCT, + TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION, + Subtype: uqc.ConstructTypeName, + SqlStatement: uqc.Query, + DocsLink: uqc.DocsLink, + MinimumVersionsFixedIn: uqc.MinimumVersionsFixedIn, }) } for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects { for _, object := range plpgsqlObjects.Objects { issues = append(issues, AssessmentIssuePayload{ - Type: PLPGSQL_OBJECT, - TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION, - Subtype: plpgsqlObjects.FeatureName, - SubtypeDescription: plpgsqlObjects.FeatureDescription, - ObjectName: object.ObjectName, - SqlStatement: object.SqlStatement, - DocsLink: plpgsqlObjects.DocsLink, + Type: PLPGSQL_OBJECT, + TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION, + Subtype: plpgsqlObjects.FeatureName, + SubtypeDescription: plpgsqlObjects.FeatureDescription, + ObjectName: object.ObjectName, + SqlStatement: object.SqlStatement, + DocsLink: plpgsqlObjects.DocsLink, + MinimumVersionsFixedIn: plpgsqlObjects.MinimumVersionsFixedIn, }) } } diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 3e608da7c7..dd6d459da7 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1224,6 +1224,7 @@ type AssessMigrationDBConfig struct { type AssessMigrationPayload struct { PayloadVersion string VoyagerVersion string + TargetDBVersion *ybversion.YBVersion MigrationComplexity string SchemaSummary utils.SchemaSummary AssessmentIssues []AssessmentIssuePayload @@ -1235,13 +1236,14 @@ type AssessMigrationPayload struct { } type AssessmentIssuePayload struct { - Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC - TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type - Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc - SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype - ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC) - SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC) - DocsLink string `json:"DocsLink"` // docs link based on the subtype + Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC + TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type + Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc + SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype + ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC) + SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC) + DocsLink string `json:"DocsLink"` // docs link based on the subtype + MinimumVersionsFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionsFixedIn"` // key: series (2024.1, 2.21, etc) // Store Type-specific details - extensible, can refer any struct Details json.RawMessage `json:"Details,omitempty"` @@ -1268,7 +1270,7 @@ type TargetSizingRecommendations struct { TotalShardedSize int64 } -var ASSESS_MIGRATION_PAYLOAD_VERSION = "1.0" +var ASSESS_MIGRATION_PAYLOAD_VERSION = "1.1" //====== AssesmentReport struct methods ======// diff --git a/yb-voyager/src/callhome/diagnostics.go b/yb-voyager/src/callhome/diagnostics.go index 5b65b609f0..8e53134162 100644 --- a/yb-voyager/src/callhome/diagnostics.go +++ b/yb-voyager/src/callhome/diagnostics.go @@ -30,6 +30,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) // call-home json formats @@ -103,18 +104,19 @@ type UnsupportedFeature struct { } type AssessMigrationPhasePayload struct { - MigrationComplexity string `json:"migration_complexity"` - UnsupportedFeatures string `json:"unsupported_features"` - UnsupportedDatatypes string `json:"unsupported_datatypes"` - UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` - MigrationCaveats string `json:"migration_caveats"` - UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` - Error string `json:"error,omitempty"` // Removed it for now, TODO - TableSizingStats string `json:"table_sizing_stats"` - IndexSizingStats string `json:"index_sizing_stats"` - SchemaSummary string `json:"schema_summary"` - SourceConnectivity bool `json:"source_connectivity"` - IopsInterval int64 `json:"iops_interval"` + TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` + MigrationComplexity string `json:"migration_complexity"` + UnsupportedFeatures string `json:"unsupported_features"` + UnsupportedDatatypes string `json:"unsupported_datatypes"` + UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` + MigrationCaveats string `json:"migration_caveats"` + UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` + Error string `json:"error,omitempty"` // Removed it for now, TODO + TableSizingStats string `json:"table_sizing_stats"` + IndexSizingStats string `json:"index_sizing_stats"` + SchemaSummary string `json:"schema_summary"` + SourceConnectivity bool `json:"source_connectivity"` + IopsInterval int64 `json:"iops_interval"` } type AssessMigrationBulkPhasePayload struct { @@ -139,8 +141,9 @@ type ExportSchemaPhasePayload struct { // SHOULD NOT REMOVE THESE TWO (issues, database_objects) FIELDS of AnalyzePhasePayload as parsing these specifically here // https://github.com/yugabyte/yugabyte-growth/blob/ad5df306c50c05136df77cd6548a1091ae577046/diagnostics_v2/main.py#L563 type AnalyzePhasePayload struct { - Issues string `json:"issues"` - DatabaseObjects string `json:"database_objects"` + TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` + Issues string `json:"issues"` + DatabaseObjects string `json:"database_objects"` } type ExportDataPhasePayload struct { ParallelJobs int64 `json:"parallel_jobs"` diff --git a/yb-voyager/src/callhome/diagnostics_test.go b/yb-voyager/src/callhome/diagnostics_test.go index 2575a8e405..a212004a05 100644 --- a/yb-voyager/src/callhome/diagnostics_test.go +++ b/yb-voyager/src/callhome/diagnostics_test.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) func TestCallhomeStructs(t *testing.T) { @@ -67,18 +68,19 @@ func TestCallhomeStructs(t *testing.T) { name: "Validate AssessMigrationPhasePayload Struct Definition", actualType: reflect.TypeOf(AssessMigrationPhasePayload{}), expectedType: struct { - MigrationComplexity string `json:"migration_complexity"` - UnsupportedFeatures string `json:"unsupported_features"` - UnsupportedDatatypes string `json:"unsupported_datatypes"` - UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` - MigrationCaveats string `json:"migration_caveats"` - UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` - Error string `json:"error,omitempty"` - TableSizingStats string `json:"table_sizing_stats"` - IndexSizingStats string `json:"index_sizing_stats"` - SchemaSummary string `json:"schema_summary"` - SourceConnectivity bool `json:"source_connectivity"` - IopsInterval int64 `json:"iops_interval"` + TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` + MigrationComplexity string `json:"migration_complexity"` + UnsupportedFeatures string `json:"unsupported_features"` + UnsupportedDatatypes string `json:"unsupported_datatypes"` + UnsupportedQueryConstructs string `json:"unsupported_query_constructs"` + MigrationCaveats string `json:"migration_caveats"` + UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"` + Error string `json:"error,omitempty"` + TableSizingStats string `json:"table_sizing_stats"` + IndexSizingStats string `json:"index_sizing_stats"` + SchemaSummary string `json:"schema_summary"` + SourceConnectivity bool `json:"source_connectivity"` + IopsInterval int64 `json:"iops_interval"` }{}, }, { @@ -113,8 +115,9 @@ func TestCallhomeStructs(t *testing.T) { name: "Validate AnalyzePhasePayload Struct Definition", actualType: reflect.TypeOf(AnalyzePhasePayload{}), expectedType: struct { - Issues string `json:"issues"` - DatabaseObjects string `json:"database_objects"` + TargetDBVersion *ybversion.YBVersion `json:"target_db_version"` + Issues string `json:"issues"` + DatabaseObjects string `json:"database_objects"` }{}, }, {