Known cfg 2 61 (#12351)

same as #11710 but for e2

Co-authored-by: JkLondon <[email protected]>

erigon-lib/chain/snapcfg/util.go

@@ -3,6 +3,7 @@ package snapcfg
 import (
 	_ "embed"
 	"encoding/json"
+	"errors"
 	"path/filepath"
 	"slices"
 	"sort"
@@ -77,11 +78,22 @@ func (p Preverified) Typed(types []snaptype.Type) Preverified {
 
 		var preferredVersion, minVersion snaptype.Version
 
-		parts := strings.Split(name, "-")
-		if len(parts) < 3 {
+		countSep := 0
+		var lastSep, dot int
+		for i := range name {
+			if name[i] == '-' {
+				countSep++
+				lastSep = i
+			}
+			if name[i] == '.' {
+				dot = i
+			}
+		}
+
+		if countSep < 2 {
 			continue
 		}
-		typeName, _ := strings.CutSuffix(parts[2], filepath.Ext(parts[2]))
+		typeName := name[lastSep+1 : dot]
 		include := false
 
 		for _, typ := range types {
@@ -204,18 +216,13 @@ func (p Preverified) MaxBlock(version snaptype.Version) (uint64, error) {
 		if ext != ".seg" {
 			continue
 		}
-		onlyName := fileName[:len(fileName)-len(ext)]
-		parts := strings.Split(onlyName, "-")
 
-		to, err := strconv.ParseUint(parts[2], 10, 64)
+		to, err := ExtractBlockFromName(fileName[:len(fileName)-len(ext)], version)
 		if err != nil {
-			return 0, err
-		}
-
-		if version != 0 {
-			if v, err := snaptype.ParseVersion(parts[0]); err != nil || v != version {
+			if errors.Is(err, errWrongVersion) {
 				continue
 			}
+			return 0, err
 		}
 
 		if max < to {
@@ -230,6 +237,53 @@ func (p Preverified) MaxBlock(version snaptype.Version) (uint64, error) {
 	return max*1_000 - 1, nil
 }
 
+var errWrongVersion = errors.New("wrong version")
+
+func ExtractBlockFromName(name string, v snaptype.Version) (block uint64, err error) {
+	i := 0
+	for i < len(name) && name[i] != '-' {
+		i++
+	}
+
+	version, err := snaptype.ParseVersion(name[:i])
+	if err != nil {
+		return 0, err
+	}
+
+	if v != 0 && v != version {
+		return 0, errWrongVersion
+	}
+
+	i++
+
+	for i < len(name) && name[i] != '-' { // skipping parts[1]
+		i++
+	}
+
+	i++
+	start := i
+	if start > len(name)-1 {
+		return 0, errors.New("invalid name")
+	}
+
+	for i < len(name) && name[i] != '-' {
+		i++
+	}
+
+	end := i
+
+	if i > len(name) {
+		end = len(name)
+	}
+
+	block, err = strconv.ParseUint(name[start:end], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	return block, nil
+}
+
 func (p Preverified) MarshalJSON() ([]byte, error) {
 	out := map[string]string{}
 
@@ -350,8 +404,8 @@ func Seedable(networkName string, info snaptype.FileInfo) bool {
 	return KnownCfg(networkName).Seedable(info)
 }
 
-func MergeLimit(networkName string, snapType snaptype.Enum, fromBlock uint64) uint64 {
-	return KnownCfg(networkName).MergeLimit(snapType, fromBlock)
+func MergeLimitFromCfg(cfg *Cfg, snapType snaptype.Enum, fromBlock uint64) uint64 {
+	return cfg.MergeLimit(snapType, fromBlock)
 }
 
 func MaxSeedableSegment(chain string, dir string) uint64 {
@@ -370,8 +424,8 @@ func MaxSeedableSegment(chain string, dir string) uint64 {
 
 var oldMergeSteps = append([]uint64{snaptype.Erigon2OldMergeLimit}, snaptype.MergeSteps...)
 
-func MergeSteps(networkName string, snapType snaptype.Enum, fromBlock uint64) []uint64 {
-	mergeLimit := MergeLimit(networkName, snapType, fromBlock)
+func MergeStepsFromCfg(cfg *Cfg, snapType snaptype.Enum, fromBlock uint64) []uint64 {
+	mergeLimit := MergeLimitFromCfg(cfg, snapType, fromBlock)
 
 	if mergeLimit == snaptype.Erigon2OldMergeLimit {
 		return oldMergeSteps

erigon-lib/chain/snapcfg/util_test.go

@@ -0,0 +1,86 @@
+package snapcfg
+
+import (
+	"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
+	"testing"
+)
+
+func TestNameToParts(t *testing.T) {
+	type args struct {
+		name string
+		v    snaptype.Version
+	}
+	tests := []struct {
+		name      string
+		args      args
+		wantBlock uint64
+		wantErr   bool
+	}{
+		{
+			"happy pass",
+			args{
+				name: "v1-asd-12-d",
+				v:    0,
+			},
+			12,
+			false,
+		},
+		{
+			"happy pass with version",
+			args{
+				name: "v2-asd-12-d",
+				v:    2,
+			},
+			12,
+			false,
+		},
+		{
+			"happy pass && block in the end",
+			args{
+				name: "v1-asd-12",
+				v:    0,
+			},
+			12,
+			false,
+		},
+		{
+			"version mismatch",
+			args{
+				name: "v1-asd-12",
+				v:    2,
+			},
+			0,
+			true,
+		},
+		{
+			"block parse error",
+			args{
+				name: "v1-asd-dd12",
+				v:    0,
+			},
+			0,
+			true,
+		},
+		{
+			"bad name",
+			args{
+				name: "v1-dd12",
+				v:    0,
+			},
+			0,
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotBlock, err := ExtractBlockFromName(tt.args.name, tt.args.v)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ExtractBlockFromName() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if gotBlock != tt.wantBlock {
+				t.Errorf("ExtractBlockFromName() gotBlock = %v, want %v", gotBlock, tt.wantBlock)
+			}
+		})
+	}
+}

turbo/snapshotsync/freezeblocks/block_snapshots.go

@@ -1106,7 +1106,7 @@ func chooseSegmentEnd(from, to uint64, snapType snaptype.Enum, chainConfig *chai
 	if chainConfig != nil {
 		chainName = chainConfig.ChainName
 	}
-	blocksPerFile := snapcfg.MergeLimit(chainName, snapType, from)
+	blocksPerFile := snapcfg.MergeLimitFromCfg(snapcfg.KnownCfg(chainName), snapType, from)
 
 	next := (from/blocksPerFile + 1) * blocksPerFile
 	to = cmp.Min(next, to)
@@ -1181,7 +1181,7 @@ func canRetire(from, to uint64, snapType snaptype.Enum, chainConfig *chain.Confi
 		chainName = chainConfig.ChainName
 	}
 
-	mergeLimit := snapcfg.MergeLimit(chainName, snapType, blockFrom)
+	mergeLimit := snapcfg.MergeLimitFromCfg(snapcfg.KnownCfg(chainName), snapType, blockFrom)
 
 	if blockFrom%mergeLimit == 0 {
 		maxJump = mergeLimit
@@ -1862,13 +1862,14 @@ func NewMerger(tmpDir string, compressWorkers int, lvl log.Lvl, chainDB kv.RoDB,
 func (m *Merger) DisableFsync() { m.noFsync = true }
 
 func (m *Merger) FindMergeRanges(currentRanges []Range, maxBlockNum uint64) (toMerge []Range) {
+	cfg := snapcfg.KnownCfg(m.chainConfig.ChainName)
 	for i := len(currentRanges) - 1; i > 0; i-- {
 		r := currentRanges[i]
-		mergeLimit := snapcfg.MergeLimit(m.chainConfig.ChainName, snaptype.Unknown, r.from)
+		mergeLimit := snapcfg.MergeLimitFromCfg(cfg, snaptype.Unknown, r.from)
 		if r.to-r.from >= mergeLimit {
 			continue
 		}
-		for _, span := range snapcfg.MergeSteps(m.chainConfig.ChainName, snaptype.Unknown, r.from) {
+		for _, span := range snapcfg.MergeStepsFromCfg(cfg, snaptype.Unknown, r.from) {
 			if r.to%span != 0 {
 				continue
 			}

turbo/snapshotsync/freezeblocks/block_snapshots_test.go

@@ -61,6 +61,67 @@ func createTestSegmentFile(t *testing.T, from, to uint64, name snaptype.Enum, di
 	}
 }
 
+func BenchmarkFindMergeRange(t *testing.B) {
+	merger := NewMerger("x", 1, log.LvlInfo, nil, params.MainnetChainConfig, nil)
+	merger.DisableFsync()
+	t.Run("big", func(t *testing.B) {
+		for j := 0; j < t.N; j++ {
+			var rangesOld []Range
+			for i := 0; i < 24; i++ {
+				rangesOld = append(rangesOld, Range{from: uint64(i * 100_000), to: uint64((i + 1) * 100_000)})
+			}
+			found := merger.FindMergeRanges(rangesOld, uint64(24*100_000))
+
+			expect := Ranges{{0, 500000}, {500000, 1000000}, {1000000, 1500000}, {1500000, 2000000}}
+			require.Equal(t, expect.String(), Ranges(found).String())
+
+			var rangesNew []Range
+			start := uint64(19_000_000)
+			for i := uint64(0); i < 24; i++ {
+				rangesNew = append(rangesNew, Range{from: start + (i * 100_000), to: start + ((i + 1) * 100_000)})
+			}
+			found = merger.FindMergeRanges(rangesNew, uint64(24*100_000))
+
+			expect = Ranges{}
+			require.Equal(t, expect.String(), Ranges(found).String())
+		}
+	})
+
+	t.Run("small", func(t *testing.B) {
+		for j := 0; j < t.N; j++ {
+			var rangesOld Ranges
+			for i := uint64(0); i < 240; i++ {
+				rangesOld = append(rangesOld, Range{from: i * 10_000, to: (i + 1) * 10_000})
+			}
+			found := merger.FindMergeRanges(rangesOld, uint64(240*10_000))
+			var expect Ranges
+			for i := uint64(0); i < 4; i++ {
+				expect = append(expect, Range{from: i * snaptype.Erigon2OldMergeLimit, to: (i + 1) * snaptype.Erigon2OldMergeLimit})
+			}
+			for i := uint64(0); i < 4; i++ {
+				expect = append(expect, Range{from: 2_000_000 + i*snaptype.Erigon2MergeLimit, to: 2_000_000 + (i+1)*snaptype.Erigon2MergeLimit})
+			}
+
+			require.Equal(t, expect.String(), Ranges(found).String())
+
+			var rangesNew Ranges
+			start := uint64(19_000_000)
+			for i := uint64(0); i < 240; i++ {
+				rangesNew = append(rangesNew, Range{from: start + i*10_000, to: start + (i+1)*10_000})
+			}
+			found = merger.FindMergeRanges(rangesNew, uint64(240*10_000))
+			expect = nil
+			for i := uint64(0); i < 24; i++ {
+				expect = append(expect, Range{from: start + i*snaptype.Erigon2MergeLimit, to: start + (i+1)*snaptype.Erigon2MergeLimit})
+			}
+
+			require.Equal(t, expect.String(), Ranges(found).String())
+		}
+
+	})
+
+}
+
 func TestFindMergeRange(t *testing.T) {
 	merger := NewMerger("x", 1, log.LvlInfo, nil, params.MainnetChainConfig, nil)
 	merger.DisableFsync()

turbo/snapshotsync/freezeblocks/caplin_snapshots.go

@@ -502,9 +502,9 @@ func dumpBlobSidecarsRange(ctx context.Context, db kv.RoDB, storage blob_storage
 }
 
 func DumpBeaconBlocks(ctx context.Context, db kv.RoDB, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
-
+	cfg := snapcfg.KnownCfg("")
 	for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BeaconBlocks, nil) {
-		blocksPerFile := snapcfg.MergeLimit("", snaptype.CaplinEnums.BeaconBlocks, i)
+		blocksPerFile := snapcfg.MergeLimitFromCfg(cfg, snaptype.CaplinEnums.BeaconBlocks, i)
 
 		if toSlot-i < blocksPerFile {
 			break
@@ -519,8 +519,9 @@ func DumpBeaconBlocks(ctx context.Context, db kv.RoDB, fromSlot, toSlot uint64,
 }
 
 func DumpBlobsSidecar(ctx context.Context, blobStorage blob_storage.BlobStorage, db kv.RoDB, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, compressWorkers int, lvl log.Lvl, logger log.Logger) error {
+	cfg := snapcfg.KnownCfg("")
 	for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BlobSidecars, nil) {
-		blocksPerFile := snapcfg.MergeLimit("", snaptype.CaplinEnums.BlobSidecars, i)
+		blocksPerFile := snapcfg.MergeLimitFromCfg(cfg, snaptype.CaplinEnums.BlobSidecars, i)
 
 		if toSlot-i < blocksPerFile {
 			break