Implement Migration complexity using the assessment issues (#2161)

- now the new algorithm for mig complexity considers all kind of issues including schema, query constructs, plpgsql, datatypes etc...

* Removed migration complexity info from analyze schema reports

* Fixed the expected mig complexity for pg_assessment_report_test_uqc from LOW to MEDIUM
- since the unsupported query constructs are being considered now in mig complexity calculatin

* Fixed expected migration complexity for omnibus and osm schema
- complexity from Medium->High, and Low->Medium, due to new things - unsupported_datatype(level-3) and change in threshold for HIGH

migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json

@@ -1,6 +1,5 @@
 {
     "VoyagerVersion": "IGNORED",
-    "MigrationComplexity": "HIGH",
     "Summary": {
         "Description": "Objects that will be created on the target YugabyteDB.",
         "DbName": "adventureworks",

migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json

@@ -1,7 +1,7 @@
 {
 	"VoyagerVersion": "IGNORED",
 	"TargetDBVersion": "IGNORED",
-	"MigrationComplexity": "LOW",
+	"MigrationComplexity": "MEDIUM",
 	"SchemaSummary": {
 		"Description": "Objects that will be created on the target YugabyteDB.",
 		"DbName": "pg_assessment_report_uqc",

migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json

@@ -1,6 +1,5 @@
 {
     "VoyagerVersion": "IGNORED",
-    "MigrationComplexity": "MEDIUM",
     "Summary": {
         "Description": "Objects that will be created on the target YugabyteDB.",
         "DbName": "test_mgd",

migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json

@@ -1,6 +1,6 @@
 {
 	"VoyagerVersion": "IGNORED",
-	"MigrationComplexity": "MEDIUM",
+	"MigrationComplexity": "HIGH",
 	"SchemaSummary": {
 		"Description": "Objects that will be created on the target YugabyteDB.",
 		"DbName": "test_omnibus",

migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json

@@ -1,6 +1,5 @@
 {
     "VoyagerVersion": "IGNORED",
-    "MigrationComplexity": "MEDIUM",
     "Summary": {
         "Description": "Objects that will be created on the target YugabyteDB.",
         "DbName": "test_omnibus",

migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json

@@ -1,6 +1,6 @@
 {
 	"VoyagerVersion": "IGNORED",
-	"MigrationComplexity": "LOW",
+	"MigrationComplexity": "MEDIUM",
 	"SchemaSummary": {
 		"Description": "Objects that will be created on the target YugabyteDB.",
 		"DbName": "test_osm",

migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json

@@ -1,6 +1,5 @@
 {
     "VoyagerVersion": "IGNORED",
-    "MigrationComplexity": "LOW",
     "Summary": {
         "Description": "Objects that will be created on the target YugabyteDB.",
         "DbName": "test_osm",

migtests/tests/pg/pgtbrus/expected_files/expected_schema_analysis_report.json

@@ -1,6 +1,5 @@
 {
     "VoyagerVersion": "IGNORED",
-    "MigrationComplexity": "LOW",
     "Summary": {
         "Description": "Objects that will be created on the target YugabyteDB.",
         "DbName": "test_pgtbrus",

migtests/tests/pg/rna/expected_files/expected_schema_analysis_report.json

@@ -1,6 +1,5 @@
 {
     "VoyagerVersion": "IGNORED",
-    "MigrationComplexity": "HIGH",
     "Summary": {
         "Description": "Objects that will be created on the target YugabyteDB.",
         "DbName": "test_rna",

migtests/tests/pg/sakila/expected_files/expected_schema_analysis_report.json

@@ -1,6 +1,5 @@
 {
     "VoyagerVersion": "IGNORED",
-    "MigrationComplexity": "HIGH",
     "Summary": {
         "Description": "Objects that will be created on the target YugabyteDB.",
         "DbName": "pg_sakila",

migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json

@@ -1,6 +1,5 @@
 {
     "VoyagerVersion": "IGNORED",
-    "MigrationComplexity": "LOW",
     "Summary": {
         "Description": "Objects that will be created on the target YugabyteDB.",
         "DbName": "test_is",

migtests/tests/pg/stackexchange/expected_files/expected_schema_analysis_report.json

@@ -1,6 +1,5 @@
 {
     "VoyagerVersion": "IGNORED",
-    "MigrationComplexity": "MEDIUM",
     "Summary": {
         "Description": "Objects that will be created on the target YugabyteDB.",
         "DbName": "test_stackex",

yb-voyager/cmd/analyzeSchema.go

@@ -1091,7 +1091,6 @@ func analyzeSchemaInternal(sourceDBConf *srcdb.Source, detectIssues bool) utils.
 	schemaAnalysisReport.SchemaSummary = reportSchemaSummary(sourceDBConf)
 	schemaAnalysisReport.VoyagerVersion = utils.YB_VOYAGER_VERSION
 	schemaAnalysisReport.TargetDBVersion = targetDbVersion
-	schemaAnalysisReport.MigrationComplexity = getMigrationComplexity(sourceDBConf.DBType, schemaDir, schemaAnalysisReport)
 	return schemaAnalysisReport
 }
 

yb-voyager/cmd/assessMigrationCommand.go

@@ -896,6 +896,9 @@ func generateAssessmentReport() (err error) {
 
 	addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB)
 
+	// calculating migration complexity after collecting all assessment issues
+	assessmentReport.MigrationComplexity = calculateMigrationComplexity(source.DBType, schemaDir, assessmentReport)
+
 	assessmentReport.Sizing = migassessment.SizingReport
 	assessmentReport.TableIndexStats, err = assessmentDB.FetchAllStats()
 	if err != nil {
@@ -926,7 +929,6 @@ func getAssessmentReportContentFromAnalyzeSchema() error {
 		But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser).
 	*/
 	schemaAnalysisReport := analyzeSchemaInternal(&source, true)
-	assessmentReport.MigrationComplexity = schemaAnalysisReport.MigrationComplexity
 	assessmentReport.SchemaSummary = schemaAnalysisReport.SchemaSummary
 	assessmentReport.SchemaSummary.Description = lo.Ternary(source.DBType == ORACLE, SCHEMA_SUMMARY_DESCRIPTION_ORACLE, SCHEMA_SUMMARY_DESCRIPTION)
 

yb-voyager/cmd/common.go

@@ -16,7 +16,6 @@ limitations under the License.
 package cmd
 
 import (
-	"encoding/csv"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -1048,130 +1047,6 @@ func storeTableListInMSR(tableList []sqlname.NameTuple) error {
 	return nil
 }
 
-var (
-	UNSUPPORTED_DATATYPE_XML_ISSUE  = fmt.Sprintf("%s - xml", UNSUPPORTED_DATATYPE)
-	UNSUPPORTED_DATATYPE_XID_ISSUE  = fmt.Sprintf("%s - xid", UNSUPPORTED_DATATYPE)
-	APP_CHANGES_HIGH_THRESHOLD      = 5
-	APP_CHANGES_MEDIUM_THRESHOLD    = 1
-	SCHEMA_CHANGES_HIGH_THRESHOLD   = math.MaxInt32
-	SCHEMA_CHANGES_MEDIUM_THRESHOLD = 20
-)
-
-var appChanges = []string{
-	INHERITANCE_ISSUE_REASON,
-	CONVERSION_ISSUE_REASON,
-	DEFERRABLE_CONSTRAINT_ISSUE,
-	UNSUPPORTED_DATATYPE_XML_ISSUE,
-	UNSUPPORTED_DATATYPE_XID_ISSUE,
-	UNSUPPORTED_EXTENSION_ISSUE, // will confirm this
-}
-
-func readEnvForAppOrSchemaCounts() {
-	APP_CHANGES_HIGH_THRESHOLD = utils.GetEnvAsInt("APP_CHANGES_HIGH_THRESHOLD", APP_CHANGES_HIGH_THRESHOLD)
-	APP_CHANGES_MEDIUM_THRESHOLD = utils.GetEnvAsInt("APP_CHANGES_MEDIUM_THRESHOLD", APP_CHANGES_MEDIUM_THRESHOLD)
-	SCHEMA_CHANGES_HIGH_THRESHOLD = utils.GetEnvAsInt("SCHEMA_CHANGES_HIGH_THRESHOLD", SCHEMA_CHANGES_HIGH_THRESHOLD)
-	SCHEMA_CHANGES_MEDIUM_THRESHOLD = utils.GetEnvAsInt("SCHEMA_CHANGES_MEDIUM_THRESHOLD", SCHEMA_CHANGES_MEDIUM_THRESHOLD)
-}
-
-// Migration complexity calculation from the conversion issues
-func getMigrationComplexity(sourceDBType string, schemaDirectory string, analysisReport utils.SchemaReport) string {
-	if analysisReport.MigrationComplexity != "" {
-		return analysisReport.MigrationComplexity
-	}
-
-	if sourceDBType == ORACLE {
-		mc, err := getMigrationComplexityForOracle(schemaDirectory)
-		if err != nil {
-			log.Errorf("failed to get migration complexity for oracle: %v", err)
-			return "NOT AVAILABLE"
-		}
-		return mc
-	} else if sourceDBType != POSTGRESQL {
-		return "NOT AVAILABLE"
-	}
-
-	log.Infof("Calculating migration complexity..")
-	readEnvForAppOrSchemaCounts()
-	appChangesCount := 0
-	for _, issue := range schemaAnalysisReport.Issues {
-		for _, appChange := range appChanges {
-			if strings.Contains(issue.Reason, appChange) {
-				appChangesCount++
-			}
-		}
-	}
-	schemaChangesCount := len(schemaAnalysisReport.Issues) - appChangesCount
-
-	if appChangesCount > APP_CHANGES_HIGH_THRESHOLD || schemaChangesCount > SCHEMA_CHANGES_HIGH_THRESHOLD {
-		return HIGH
-	} else if appChangesCount > APP_CHANGES_MEDIUM_THRESHOLD || schemaChangesCount > SCHEMA_CHANGES_MEDIUM_THRESHOLD {
-		return MEDIUM
-	}
-	//LOW in case appChanges == 0 or schemaChanges [0-20]
-	return LOW
-}
-
-// This is a temporary logic to get migration complexity for oracle based on the migration level from ora2pg report.
-// Ideally, we should ALSO be considering the schema analysis report to get the migration complexity.
-func getMigrationComplexityForOracle(schemaDirectory string) (string, error) {
-	ora2pgReportPath := filepath.Join(schemaDirectory, "ora2pg_report.csv")
-	if !utils.FileOrFolderExists(ora2pgReportPath) {
-		return "", fmt.Errorf("ora2pg report file not found at %s", ora2pgReportPath)
-	}
-	file, err := os.Open(ora2pgReportPath)
-	if err != nil {
-		return "", fmt.Errorf("failed to read file %s: %w", ora2pgReportPath, err)
-	}
-	defer func() {
-		if err := file.Close(); err != nil {
-			log.Errorf("Error while closing file %s: %v", ora2pgReportPath, err)
-		}
-	}()
-	// Sample file contents
-
-	// "dbi:Oracle:(DESCRIPTION = (ADDRESS = (PROTOCOL = TCP)(HOST = xyz)(PORT = 1521))(CONNECT_DATA = (SERVICE_NAME = DMS)))";
-	// "Oracle Database 19c Enterprise Edition Release 19.0.0.0.0";"ASSESS_MIGRATION";"261.62 MB";"1 person-day(s)";"A-2";
-	// "0/0/0.00";"0/0/0";"0/0/0";"25/0/6.50";"0/0/0.00";"0/0/0";"0/0/0";"0/0/0";"0/0/0";"3/0/1.00";"3/0/1.00";
-	// "44/0/4.90";"27/0/2.70";"9/0/1.80";"4/0/16.00";"5/0/3.00";"2/0/2.00";"125/0/58.90"
-	//
-	// X/Y/Z - total/invalid/cost for each type of objects(table,function,etc). Last data element is the sum total.
-	// total cost = 58.90 units (1 unit = 5 minutes). Therefore total cost is approx 1 person-days.
-	// column 6 is Migration level.
-	//  Migration levels:
-	//     A - Migration that might be run automatically
-	//     B - Migration with code rewrite and a human-days cost up to 5 days
-	//     C - Migration with code rewrite and a human-days cost above 5 days
-	// 	Technical levels:
-	//     1 = trivial: no stored functions and no triggers
-	//     2 = easy: no stored functions but with triggers, no manual rewriting
-	//     3 = simple: stored functions and/or triggers, no manual rewriting
-	//     4 = manual: no stored functions but with triggers or views with code rewriting
-	//     5 = difficult: stored functions and/or triggers with code rewriting
-	reader := csv.NewReader(file)
-	reader.Comma = ';'
-	rows, err := reader.ReadAll()
-	if err != nil {
-		log.Errorf("error reading csv file %s: %v", ora2pgReportPath, err)
-		return "", fmt.Errorf("error reading csv file %s: %w", ora2pgReportPath, err)
-	}
-	if len(rows) > 1 {
-		return "", fmt.Errorf("invalid ora2pg report file format. Expected 1 row, found %d. contents = %v", len(rows), rows)
-	}
-	reportData := rows[0]
-	migrationLevel := strings.Split(reportData[5], "-")[0]
-
-	switch migrationLevel {
-	case "A":
-		return LOW, nil
-	case "B":
-		return MEDIUM, nil
-	case "C":
-		return HIGH, nil
-	default:
-		return "", fmt.Errorf("invalid migration level [%s] found in ora2pg report %v", migrationLevel, reportData)
-	}
-}
-
 // =====================================================================
 
 // TODO: consider merging all unsupported field with single AssessmentReport struct member as AssessmentIssue

yb-voyager/cmd/constants.go

@@ -62,9 +62,6 @@ const (
 	ROW_UPDATE_STATUS_IN_PROGRESS   = 1
 	ROW_UPDATE_STATUS_COMPLETED     = 3
 	COLOCATION_CLAUSE               = "colocation"
-	LOW                             = "LOW"
-	MEDIUM                          = "MEDIUM"
-	HIGH                            = "HIGH"
 	//phase names used in call-home payload
 	ANALYZE_PHASE                    = "analyze-schema"
 	EXPORT_SCHEMA_PHASE              = "export-schema"