Skip to content

Commit

Permalink
Send target-db-version, MinimumVersionsFixedIn details for assessment…
Browse files Browse the repository at this point in the history 
…/analyze-schema reports to controlplane/callhome (#2049)

In assess-migration
- yugabyted control plane
send target-db-version
include MinFixVersion for all issues
- callhome
send target-db-version

In analyze-schema
- yugabyted control plane (Already done because we send the entire schema report)
send target-db-version
include MinFixVersion for all issues
- callhome
send target-db-version
  • Loading branch information
@makalaaneesh
makalaaneesh authored Dec 12, 2024
1 parent 7990563 commit e5e8f9c
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 72 deletions.
22 changes: 12 additions & 10 deletions yb-voyager/cmd/analyzeSchema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,15 +626,16 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil
}

return utils.Issue{
ObjectType: issueInstance.ObjectType,
ObjectName: issueInstance.ObjectName,
Reason: issueInstance.TypeName,
SqlStatement: issueInstance.SqlStatement,
DocsLink: issueInstance.DocsLink,
FilePath: fileName,
IssueType: issueType,
Suggestion: issueInstance.Suggestion,
GH: issueInstance.GH,
ObjectType: issueInstance.ObjectType,
ObjectName: issueInstance.ObjectName,
Reason: issueInstance.TypeName,
SqlStatement: issueInstance.SqlStatement,
DocsLink: issueInstance.DocsLink,
FilePath: fileName,
IssueType: issueType,
Suggestion: issueInstance.Suggestion,
GH: issueInstance.GH,
MinimumVersionsFixedIn: issueInstance.MinimumVersionsFixedIn,
}
}

Expand Down Expand Up @@ -1217,7 +1218,8 @@ func packAndSendAnalyzeSchemaPayload(status string) {
}

analyzePayload := callhome.AnalyzePhasePayload{
Issues: callhome.MarshalledJsonString(callhomeIssues),
TargetDBVersion: schemaAnalysisReport.TargetDBVersion,
Issues: callhome.MarshalledJsonString(callhomeIssues),
DatabaseObjects: callhome.MarshalledJsonString(lo.Map(schemaAnalysisReport.SchemaSummary.DBObjects, func(dbObject utils.DBObject, _ int) utils.DBObject {
dbObject.ObjectNames = ""
return dbObject
Expand Down
58 changes: 32 additions & 26 deletions yb-voyager/cmd/assessMigrationCommand.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) {
})

assessPayload := callhome.AssessMigrationPhasePayload{
TargetDBVersion: assessmentReport.TargetDBVersion,
MigrationComplexity: assessmentReport.MigrationComplexity,
UnsupportedFeatures: callhome.MarshalledJsonString(lo.Map(assessmentReport.UnsupportedFeatures, func(feature UnsupportedFeature, _ int) callhome.UnsupportedFeature {
var objects []string
Expand Down Expand Up @@ -473,6 +474,7 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE
payload := AssessMigrationPayload{
PayloadVersion: ASSESS_MIGRATION_PAYLOAD_VERSION,
VoyagerVersion: assessmentReport.VoyagerVersion,
TargetDBVersion: assessmentReport.TargetDBVersion,
MigrationComplexity: assessmentReport.MigrationComplexity,
SchemaSummary: assessmentReport.SchemaSummary,
AssessmentIssues: assessmentIssues,
Expand Down Expand Up @@ -525,51 +527,55 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment
for _, unsupportedFeature := range ar.UnsupportedFeatures {
for _, object := range unsupportedFeature.Objects {
issues = append(issues, AssessmentIssuePayload{
Type: FEATURE,
TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION,
Subtype: unsupportedFeature.FeatureName,
SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: unsupportedFeature.DocsLink,
Type: FEATURE,
TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION,
Subtype: unsupportedFeature.FeatureName,
SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: unsupportedFeature.DocsLink,
MinimumVersionsFixedIn: unsupportedFeature.MinimumVersionsFixedIn,
})
}
}

for _, migrationCaveat := range ar.MigrationCaveats {
for _, object := range migrationCaveat.Objects {
issues = append(issues, AssessmentIssuePayload{
Type: MIGRATION_CAVEATS,
TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION,
Subtype: migrationCaveat.FeatureName,
SubtypeDescription: migrationCaveat.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: migrationCaveat.DocsLink,
Type: MIGRATION_CAVEATS,
TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION,
Subtype: migrationCaveat.FeatureName,
SubtypeDescription: migrationCaveat.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: migrationCaveat.DocsLink,
MinimumVersionsFixedIn: migrationCaveat.MinimumVersionsFixedIn,
})
}
}

for _, uqc := range ar.UnsupportedQueryConstructs {
issues = append(issues, AssessmentIssuePayload{
Type: QUERY_CONSTRUCT,
TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION,
Subtype: uqc.ConstructTypeName,
SqlStatement: uqc.Query,
DocsLink: uqc.DocsLink,
Type: QUERY_CONSTRUCT,
TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION,
Subtype: uqc.ConstructTypeName,
SqlStatement: uqc.Query,
DocsLink: uqc.DocsLink,
MinimumVersionsFixedIn: uqc.MinimumVersionsFixedIn,
})
}

for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects {
for _, object := range plpgsqlObjects.Objects {
issues = append(issues, AssessmentIssuePayload{
Type: PLPGSQL_OBJECT,
TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION,
Subtype: plpgsqlObjects.FeatureName,
SubtypeDescription: plpgsqlObjects.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: plpgsqlObjects.DocsLink,
Type: PLPGSQL_OBJECT,
TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION,
Subtype: plpgsqlObjects.FeatureName,
SubtypeDescription: plpgsqlObjects.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: plpgsqlObjects.DocsLink,
MinimumVersionsFixedIn: plpgsqlObjects.MinimumVersionsFixedIn,
})
}
}
Expand Down
18 changes: 10 additions & 8 deletions yb-voyager/cmd/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1224,6 +1224,7 @@ type AssessMigrationDBConfig struct {
type AssessMigrationPayload struct {
PayloadVersion string
VoyagerVersion string
TargetDBVersion *ybversion.YBVersion
MigrationComplexity string
SchemaSummary utils.SchemaSummary
AssessmentIssues []AssessmentIssuePayload
Expand All @@ -1235,13 +1236,14 @@ type AssessMigrationPayload struct {
}

type AssessmentIssuePayload struct {
Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC
TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type
Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc
SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype
ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC)
SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC)
DocsLink string `json:"DocsLink"` // docs link based on the subtype
Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC
TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type
Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc
SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype
ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC)
SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC)
DocsLink string `json:"DocsLink"` // docs link based on the subtype
MinimumVersionsFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionsFixedIn"` // key: series (2024.1, 2.21, etc)

// Store Type-specific details - extensible, can refer any struct
Details json.RawMessage `json:"Details,omitempty"`
Expand All @@ -1268,7 +1270,7 @@ type TargetSizingRecommendations struct {
TotalShardedSize int64
}

var ASSESS_MIGRATION_PAYLOAD_VERSION = "1.0"
var ASSESS_MIGRATION_PAYLOAD_VERSION = "1.1"

//====== AssesmentReport struct methods ======//

Expand Down
31 changes: 17 additions & 14 deletions yb-voyager/src/callhome/diagnostics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
log "github.com/sirupsen/logrus"

"github.com/yugabyte/yb-voyager/yb-voyager/src/utils"
"github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion"
)

// call-home json formats
Expand Down Expand Up @@ -103,18 +104,19 @@ type UnsupportedFeature struct {
}

type AssessMigrationPhasePayload struct {
MigrationComplexity string `json:"migration_complexity"`
UnsupportedFeatures string `json:"unsupported_features"`
UnsupportedDatatypes string `json:"unsupported_datatypes"`
UnsupportedQueryConstructs string `json:"unsupported_query_constructs"`
MigrationCaveats string `json:"migration_caveats"`
UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"`
Error string `json:"error,omitempty"` // Removed it for now, TODO
TableSizingStats string `json:"table_sizing_stats"`
IndexSizingStats string `json:"index_sizing_stats"`
SchemaSummary string `json:"schema_summary"`
SourceConnectivity bool `json:"source_connectivity"`
IopsInterval int64 `json:"iops_interval"`
TargetDBVersion *ybversion.YBVersion `json:"target_db_version"`
MigrationComplexity string `json:"migration_complexity"`
UnsupportedFeatures string `json:"unsupported_features"`
UnsupportedDatatypes string `json:"unsupported_datatypes"`
UnsupportedQueryConstructs string `json:"unsupported_query_constructs"`
MigrationCaveats string `json:"migration_caveats"`
UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"`
Error string `json:"error,omitempty"` // Removed it for now, TODO
TableSizingStats string `json:"table_sizing_stats"`
IndexSizingStats string `json:"index_sizing_stats"`
SchemaSummary string `json:"schema_summary"`
SourceConnectivity bool `json:"source_connectivity"`
IopsInterval int64 `json:"iops_interval"`
}

type AssessMigrationBulkPhasePayload struct {
Expand All @@ -139,8 +141,9 @@ type ExportSchemaPhasePayload struct {
// SHOULD NOT REMOVE THESE TWO (issues, database_objects) FIELDS of AnalyzePhasePayload as parsing these specifically here
// https://github.com/yugabyte/yugabyte-growth/blob/ad5df306c50c05136df77cd6548a1091ae577046/diagnostics_v2/main.py#L563
type AnalyzePhasePayload struct {
Issues string `json:"issues"`
DatabaseObjects string `json:"database_objects"`
TargetDBVersion *ybversion.YBVersion `json:"target_db_version"`
Issues string `json:"issues"`
DatabaseObjects string `json:"database_objects"`
}
type ExportDataPhasePayload struct {
ParallelJobs int64 `json:"parallel_jobs"`
Expand Down
31 changes: 17 additions & 14 deletions yb-voyager/src/callhome/diagnostics_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/google/uuid"
"github.com/yugabyte/yb-voyager/yb-voyager/src/testutils"
"github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion"
)

func TestCallhomeStructs(t *testing.T) {
Expand Down Expand Up @@ -67,18 +68,19 @@ func TestCallhomeStructs(t *testing.T) {
name: "Validate AssessMigrationPhasePayload Struct Definition",
actualType: reflect.TypeOf(AssessMigrationPhasePayload{}),
expectedType: struct {
MigrationComplexity string `json:"migration_complexity"`
UnsupportedFeatures string `json:"unsupported_features"`
UnsupportedDatatypes string `json:"unsupported_datatypes"`
UnsupportedQueryConstructs string `json:"unsupported_query_constructs"`
MigrationCaveats string `json:"migration_caveats"`
UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"`
Error string `json:"error,omitempty"`
TableSizingStats string `json:"table_sizing_stats"`
IndexSizingStats string `json:"index_sizing_stats"`
SchemaSummary string `json:"schema_summary"`
SourceConnectivity bool `json:"source_connectivity"`
IopsInterval int64 `json:"iops_interval"`
TargetDBVersion *ybversion.YBVersion `json:"target_db_version"`
MigrationComplexity string `json:"migration_complexity"`
UnsupportedFeatures string `json:"unsupported_features"`
UnsupportedDatatypes string `json:"unsupported_datatypes"`
UnsupportedQueryConstructs string `json:"unsupported_query_constructs"`
MigrationCaveats string `json:"migration_caveats"`
UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"`
Error string `json:"error,omitempty"`
TableSizingStats string `json:"table_sizing_stats"`
IndexSizingStats string `json:"index_sizing_stats"`
SchemaSummary string `json:"schema_summary"`
SourceConnectivity bool `json:"source_connectivity"`
IopsInterval int64 `json:"iops_interval"`
}{},
},
{
Expand Down Expand Up @@ -113,8 +115,9 @@ func TestCallhomeStructs(t *testing.T) {
name: "Validate AnalyzePhasePayload Struct Definition",
actualType: reflect.TypeOf(AnalyzePhasePayload{}),
expectedType: struct {
Issues string `json:"issues"`
DatabaseObjects string `json:"database_objects"`
TargetDBVersion *ybversion.YBVersion `json:"target_db_version"`
Issues string `json:"issues"`
DatabaseObjects string `json:"database_objects"`
}{},
},
{
Expand Down

0 comments on commit e5e8f9c

Please sign in to comment.