Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DB-15464][Sizing Calc] buffer to choose higher vCPU instance type #2348

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[DB-15464][Sizing Calc] buffer to choose higher vCPU instance type #2348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aab9b6e
buffer to choose higher vCPU instance type
shaharuk-yb Feb 20, 2025
c805269
using required number of cores for picking best recommendation
shaharuk-yb Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 13 additions & 8 deletions yb-voyager/src/migassessment/sizing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ type IntermediateRecommendation struct {
EstimatedTimeInMinForImport float64
ParallelVoyagerJobs float64
FailureReasoning string
CoresNeeded float64
}

const (
Expand Down Expand Up @@ -284,7 +285,7 @@ Returns:
- The best IntermediateRecommendation based on the defined criteria.
*/
func pickBestRecommendation(recommendation map[int]IntermediateRecommendation) IntermediateRecommendation {
// find the one with least number of nodes
// find the one with the least number of nodes
var minCores int = math.MaxUint32
var finalRecommendation IntermediateRecommendation
var foundRecommendation bool = false
Expand All @@ -300,11 +301,12 @@ func pickBestRecommendation(recommendation map[int]IntermediateRecommendation) I
// Check if the recommendation has no failure reasoning (i.e., it's a valid recommendation)
if rec.FailureReasoning == "" {
foundRecommendation = true
// Update finalRecommendation if the current recommendation has fewer cores
if minCores > int(rec.NumNodes)*rec.VCPUsPerInstance {
// Update finalRecommendation if the current recommendation has fewer cores.
log.Infof(fmt.Sprintf("vCPU: %v & cores required: %v gives nodes required: %v\n", rec.VCPUsPerInstance, rec.CoresNeeded, rec.NumNodes))
if minCores > int(rec.CoresNeeded) {
finalRecommendation = rec
minCores = int(rec.NumNodes) * rec.VCPUsPerInstance
} else if minCores == int(rec.NumNodes)*rec.VCPUsPerInstance {
minCores = int(rec.CoresNeeded)
} else if minCores == int(rec.CoresNeeded) {
// If the number of cores is the same across machines, recommend the machine with higher core count
if rec.VCPUsPerInstance > finalRecommendation.VCPUsPerInstance {
finalRecommendation = rec
Expand Down Expand Up @@ -380,6 +382,7 @@ func findNumNodesNeededBasedOnThroughputRequirement(sourceIndexMetadata []Source
ShardedSize: previousRecommendation.ShardedSize,
EstimatedTimeInMinForImport: previousRecommendation.EstimatedTimeInMinForImport,
FailureReasoning: previousRecommendation.FailureReasoning,
CoresNeeded: neededCores,
}
}
// Return updated recommendation map
Expand Down Expand Up @@ -428,7 +431,9 @@ func findNumNodesNeededBasedOnTabletsRequired(sourceIndexMetadata []SourceDBMeta
// update recommendation to use the maximum of the existing recommended nodes and nodes calculated based on tablets
// Caveat: if new nodes required is more than the existing recommended nodes, we would need to
// re-evaluate tablets required. Although, in this iteration we've skipping re-evaluation.
rec.NumNodes = math.Max(rec.NumNodes, nodesRequired)
currentMaxRequiredNodes := math.Max(nodesRequired, rec.NumNodes)
rec.CoresNeeded = lo.Ternary(nodesRequired <= rec.NumNodes, rec.CoresNeeded, currentMaxRequiredNodes*float64(rec.VCPUsPerInstance))
rec.NumNodes = currentMaxRequiredNodes
recommendation[i] = rec
}
}
Expand All @@ -453,7 +458,7 @@ Description:
This function calculates which size threshold applies to a table based on its size and determines the number of tablets required.
Following details/comments are with assumption that the previous recommended nodes is 3.
Similar works for other recommended nodes as well.:
- For sizes up to the low phase limit (1*3 shards of 512 MB each, up to 1.5 GB), the low phase threshold is used. Where 1 is low phase shard count and 3 is the previous recommended nodes.
- For sizes up to the low phase limit (1*3 shards of 512 MB each, up to 1.5 GB), the low phase threshold is used. Where 1 is low phase shard count and 3 is the previous-recommended nodes.
- After 1*3 shards, the high phase threshold is used.
- Intermediate phase upto 30 GB is calculated based on 3 tablets of 10 GB each.
- For sizes up to the high phase limit (72(24*3) shards of 10 GB each, up to 720 GB), the high phase threshold is used.
Expand Down Expand Up @@ -1213,7 +1218,7 @@ func getMultiplicationFactorForImportTimeBasedOnNumColumns(table SourceDBMetadat
var multiplicationFactor float64
// multiplication factor is different for colocated and sharded tables.
// multiplication factor would be maximum of the two:
// max of (mf of selected entry from experiment data, mf for table wrt selected entry)
// max of (mf of selected entry from experiment data, mf for table wrt selected entry)
if objectType == COLOCATED {
multiplicationFactor = math.Max(selectedImpact.multiplicationFactorColocated.Float64,
(selectedImpact.multiplicationFactorColocated.Float64/float64(selectedImpact.numColumns.Int64))*float64(numOfColumnsInTable))
Expand Down
61 changes: 58 additions & 3 deletions yb-voyager/src/migassessment/sizing_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
//go:build unit

/*
Copyright (c) YugabyteDB, Inc.

Expand Down Expand Up @@ -777,6 +775,7 @@ func TestFindNumNodesNeededBasedOnTabletsRequired_CanSupportTablets(t *testing.T
},
VCPUsPerInstance: 4,
NumNodes: 3,
CoresNeeded: 10,
},
}

Expand All @@ -786,6 +785,7 @@ func TestFindNumNodesNeededBasedOnTabletsRequired_CanSupportTablets(t *testing.T

// check if the num nodes in updated recommendation is same as before(3) meaning no scaling is required
assert.Equal(t, float64(3), updatedRecommendation[4].NumNodes)
assert.Equal(t, float64(10), updatedRecommendation[4].CoresNeeded)
}

// validate that the tablets cannot be supported by existing nodes and scaling is needed
Expand Down Expand Up @@ -816,15 +816,17 @@ func TestFindNumNodesNeededBasedOnTabletsRequired_NeedMoreNodes(t *testing.T) {
},
VCPUsPerInstance: 4,
NumNodes: 3,
CoresNeeded: 10,
},
}

// Run the function
updatedRecommendation :=
findNumNodesNeededBasedOnTabletsRequired(sourceIndexMetadata, shardedLimits, recommendation)

// check if the num nodes in updated recommendation has increased. Meaning scaling is required.
// check if the num nodes and cores in updated recommendation has increased. Meaning scaling is required.
assert.Equal(t, float64(6), updatedRecommendation[4].NumNodes)
assert.Equal(t, float64(24), updatedRecommendation[4].CoresNeeded)
}

/*
Expand All @@ -836,11 +838,13 @@ func TestPickBestRecommendation_PickOneWithOptimalNodesAndCores(t *testing.T) {
4: {
VCPUsPerInstance: 4,
NumNodes: 10,
CoresNeeded: 40,
FailureReasoning: "",
},
8: {
VCPUsPerInstance: 8,
NumNodes: 3,
CoresNeeded: 24,
FailureReasoning: "",
},
}
Expand All @@ -855,16 +859,19 @@ func TestPickBestRecommendation_PickOneWithOptimalNodesAndCoresWhenSomeHasFailur
4: {
VCPUsPerInstance: 4,
NumNodes: 10,
CoresNeeded: 40,
FailureReasoning: "has some failure",
},
8: {
VCPUsPerInstance: 8,
NumNodes: 3,
CoresNeeded: 24,
FailureReasoning: "has some failure as well",
},
16: {
VCPUsPerInstance: 16,
NumNodes: 3,
CoresNeeded: 48,
FailureReasoning: "",
},
}
Expand Down Expand Up @@ -897,6 +904,54 @@ func TestPickBestRecommendation_PickLastMaxCoreRecommendationWhenNoneCanSupport(
assert.Equal(t, 16, bestPick.VCPUsPerInstance)
}

// validate if the recommendation with min required cores is used.
func TestPickBestRecommendation_PickRecommendationWithMinCoresRequired(t *testing.T) {
recommendations := map[int]IntermediateRecommendation{
4: {
VCPUsPerInstance: 4,
NumNodes: 83,
CoresNeeded: 329,
},
8: {
VCPUsPerInstance: 8,
NumNodes: 37,
CoresNeeded: 296,
},
16: {
VCPUsPerInstance: 16,
NumNodes: 19,
CoresNeeded: 290,
},
}
bestPick := pickBestRecommendation(recommendations)
// validate the best recommendation which is 16 vcpus per instance is picked up
assert.Equal(t, 16, bestPick.VCPUsPerInstance)
}

// validate if the recommendation with higher vCPU is selected if required cores are same.
func TestPickBestRecommendation_PickHigherVCPURecommendationWhenSameCoresRequired(t *testing.T) {
recommendations := map[int]IntermediateRecommendation{
4: {
VCPUsPerInstance: 4,
NumNodes: 12,
CoresNeeded: 48,
},
8: {
VCPUsPerInstance: 8,
NumNodes: 6,
CoresNeeded: 48,
},
16: {
VCPUsPerInstance: 16,
NumNodes: 3,
CoresNeeded: 48,
},
}
bestPick := pickBestRecommendation(recommendations)
// validate the best recommendation which is 16 vcpus per instance is picked up
assert.Equal(t, 16, bestPick.VCPUsPerInstance)
}

/*
===== Test functions to test calculateTimeTakenAndParallelJobsForImport function =====
*/
Expand Down
Loading