diff --git a/yb-voyager/src/migassessment/sizing.go b/yb-voyager/src/migassessment/sizing.go index 05383341a1..02729aa4fb 100644 --- a/yb-voyager/src/migassessment/sizing.go +++ b/yb-voyager/src/migassessment/sizing.go @@ -103,6 +103,7 @@ type IntermediateRecommendation struct { EstimatedTimeInMinForImport float64 ParallelVoyagerJobs float64 FailureReasoning string + CoresNeeded float64 } const ( @@ -284,7 +285,7 @@ Returns: - The best IntermediateRecommendation based on the defined criteria. */ func pickBestRecommendation(recommendation map[int]IntermediateRecommendation) IntermediateRecommendation { - // find the one with least number of nodes + // find the one with the least number of nodes var minCores int = math.MaxUint32 var finalRecommendation IntermediateRecommendation var foundRecommendation bool = false @@ -300,11 +301,12 @@ func pickBestRecommendation(recommendation map[int]IntermediateRecommendation) I // Check if the recommendation has no failure reasoning (i.e., it's a valid recommendation) if rec.FailureReasoning == "" { foundRecommendation = true - // Update finalRecommendation if the current recommendation has fewer cores - if minCores > int(rec.NumNodes)*rec.VCPUsPerInstance { + // Update finalRecommendation if the current recommendation has fewer cores. + log.Infof(fmt.Sprintf("vCPU: %v & cores required: %v gives nodes required: %v\n", rec.VCPUsPerInstance, rec.CoresNeeded, rec.NumNodes)) + if minCores > int(rec.CoresNeeded) { finalRecommendation = rec - minCores = int(rec.NumNodes) * rec.VCPUsPerInstance - } else if minCores == int(rec.NumNodes)*rec.VCPUsPerInstance { + minCores = int(rec.CoresNeeded) + } else if minCores == int(rec.CoresNeeded) { // If the number of cores is the same across machines, recommend the machine with higher core count if rec.VCPUsPerInstance > finalRecommendation.VCPUsPerInstance { finalRecommendation = rec @@ -380,6 +382,7 @@ func findNumNodesNeededBasedOnThroughputRequirement(sourceIndexMetadata []Source ShardedSize: previousRecommendation.ShardedSize, EstimatedTimeInMinForImport: previousRecommendation.EstimatedTimeInMinForImport, FailureReasoning: previousRecommendation.FailureReasoning, + CoresNeeded: neededCores, } } // Return updated recommendation map @@ -428,7 +431,9 @@ func findNumNodesNeededBasedOnTabletsRequired(sourceIndexMetadata []SourceDBMeta // update recommendation to use the maximum of the existing recommended nodes and nodes calculated based on tablets // Caveat: if new nodes required is more than the existing recommended nodes, we would need to // re-evaluate tablets required. Although, in this iteration we've skipping re-evaluation. - rec.NumNodes = math.Max(rec.NumNodes, nodesRequired) + currentMaxRequiredNodes := math.Max(nodesRequired, rec.NumNodes) + rec.CoresNeeded = lo.Ternary(nodesRequired <= rec.NumNodes, rec.CoresNeeded, currentMaxRequiredNodes*float64(rec.VCPUsPerInstance)) + rec.NumNodes = currentMaxRequiredNodes recommendation[i] = rec } } @@ -453,7 +458,7 @@ Description: This function calculates which size threshold applies to a table based on its size and determines the number of tablets required. Following details/comments are with assumption that the previous recommended nodes is 3. Similar works for other recommended nodes as well.: - - For sizes up to the low phase limit (1*3 shards of 512 MB each, up to 1.5 GB), the low phase threshold is used. Where 1 is low phase shard count and 3 is the previous recommended nodes. + - For sizes up to the low phase limit (1*3 shards of 512 MB each, up to 1.5 GB), the low phase threshold is used. Where 1 is low phase shard count and 3 is the previous-recommended nodes. - After 1*3 shards, the high phase threshold is used. - Intermediate phase upto 30 GB is calculated based on 3 tablets of 10 GB each. - For sizes up to the high phase limit (72(24*3) shards of 10 GB each, up to 720 GB), the high phase threshold is used. @@ -1213,7 +1218,7 @@ func getMultiplicationFactorForImportTimeBasedOnNumColumns(table SourceDBMetadat var multiplicationFactor float64 // multiplication factor is different for colocated and sharded tables. // multiplication factor would be maximum of the two: - // max of (mf of selected entry from experiment data, mf for table wrt selected entry) + // max of (mf of selected entry from experiment data, mf for table wrt selected entry) if objectType == COLOCATED { multiplicationFactor = math.Max(selectedImpact.multiplicationFactorColocated.Float64, (selectedImpact.multiplicationFactorColocated.Float64/float64(selectedImpact.numColumns.Int64))*float64(numOfColumnsInTable)) diff --git a/yb-voyager/src/migassessment/sizing_test.go b/yb-voyager/src/migassessment/sizing_test.go index f65bfeb5fb..073e5fdbf8 100644 --- a/yb-voyager/src/migassessment/sizing_test.go +++ b/yb-voyager/src/migassessment/sizing_test.go @@ -1,5 +1,3 @@ -//go:build unit - /* Copyright (c) YugabyteDB, Inc. @@ -777,6 +775,7 @@ func TestFindNumNodesNeededBasedOnTabletsRequired_CanSupportTablets(t *testing.T }, VCPUsPerInstance: 4, NumNodes: 3, + CoresNeeded: 10, }, } @@ -786,6 +785,7 @@ func TestFindNumNodesNeededBasedOnTabletsRequired_CanSupportTablets(t *testing.T // check if the num nodes in updated recommendation is same as before(3) meaning no scaling is required assert.Equal(t, float64(3), updatedRecommendation[4].NumNodes) + assert.Equal(t, float64(10), updatedRecommendation[4].CoresNeeded) } // validate that the tablets cannot be supported by existing nodes and scaling is needed @@ -816,6 +816,7 @@ func TestFindNumNodesNeededBasedOnTabletsRequired_NeedMoreNodes(t *testing.T) { }, VCPUsPerInstance: 4, NumNodes: 3, + CoresNeeded: 10, }, } @@ -823,8 +824,9 @@ func TestFindNumNodesNeededBasedOnTabletsRequired_NeedMoreNodes(t *testing.T) { updatedRecommendation := findNumNodesNeededBasedOnTabletsRequired(sourceIndexMetadata, shardedLimits, recommendation) - // check if the num nodes in updated recommendation has increased. Meaning scaling is required. + // check if the num nodes and cores in updated recommendation has increased. Meaning scaling is required. assert.Equal(t, float64(6), updatedRecommendation[4].NumNodes) + assert.Equal(t, float64(24), updatedRecommendation[4].CoresNeeded) } /* @@ -836,11 +838,13 @@ func TestPickBestRecommendation_PickOneWithOptimalNodesAndCores(t *testing.T) { 4: { VCPUsPerInstance: 4, NumNodes: 10, + CoresNeeded: 40, FailureReasoning: "", }, 8: { VCPUsPerInstance: 8, NumNodes: 3, + CoresNeeded: 24, FailureReasoning: "", }, } @@ -855,16 +859,19 @@ func TestPickBestRecommendation_PickOneWithOptimalNodesAndCoresWhenSomeHasFailur 4: { VCPUsPerInstance: 4, NumNodes: 10, + CoresNeeded: 40, FailureReasoning: "has some failure", }, 8: { VCPUsPerInstance: 8, NumNodes: 3, + CoresNeeded: 24, FailureReasoning: "has some failure as well", }, 16: { VCPUsPerInstance: 16, NumNodes: 3, + CoresNeeded: 48, FailureReasoning: "", }, } @@ -897,6 +904,54 @@ func TestPickBestRecommendation_PickLastMaxCoreRecommendationWhenNoneCanSupport( assert.Equal(t, 16, bestPick.VCPUsPerInstance) } +// validate if the recommendation with min required cores is used. +func TestPickBestRecommendation_PickRecommendationWithMinCoresRequired(t *testing.T) { + recommendations := map[int]IntermediateRecommendation{ + 4: { + VCPUsPerInstance: 4, + NumNodes: 83, + CoresNeeded: 329, + }, + 8: { + VCPUsPerInstance: 8, + NumNodes: 37, + CoresNeeded: 296, + }, + 16: { + VCPUsPerInstance: 16, + NumNodes: 19, + CoresNeeded: 290, + }, + } + bestPick := pickBestRecommendation(recommendations) + // validate the best recommendation which is 16 vcpus per instance is picked up + assert.Equal(t, 16, bestPick.VCPUsPerInstance) +} + +// validate if the recommendation with higher vCPU is selected if required cores are same. +func TestPickBestRecommendation_PickHigherVCPURecommendationWhenSameCoresRequired(t *testing.T) { + recommendations := map[int]IntermediateRecommendation{ + 4: { + VCPUsPerInstance: 4, + NumNodes: 12, + CoresNeeded: 48, + }, + 8: { + VCPUsPerInstance: 8, + NumNodes: 6, + CoresNeeded: 48, + }, + 16: { + VCPUsPerInstance: 16, + NumNodes: 3, + CoresNeeded: 48, + }, + } + bestPick := pickBestRecommendation(recommendations) + // validate the best recommendation which is 16 vcpus per instance is picked up + assert.Equal(t, 16, bestPick.VCPUsPerInstance) +} + /* ===== Test functions to test calculateTimeTakenAndParallelJobsForImport function ===== */