[rpc-test] eth_accRange: set upper bound

core/state/dump.go

@@ -22,6 +22,7 @@
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/common/hexutility"
@@ -135,6 +136,10 @@
 	}
 }
 
+var TooMuchIterations = fmt.Errorf("[rpc] Dumper: too much iterations protection triggered")
+
+const DumperIterationsHardLimit = 10_000_000
+
 func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bool, startAddress libcommon.Address, maxResults int) ([]byte, error) {
 	var emptyCodeHash = crypto.Keccak256Hash(nil)
 	var emptyHash = libcommon.Hash{}
@@ -148,6 +153,7 @@
 
 	c.OnRoot(emptyHash) // We do not calculate the root
 
+	fmt.Printf("[dbg] DumpToCollector limits: %t, %t, %d\n", excludeCode, excludeStorage, maxResults)
 	ttx := d.db.(kv.TemporalTx)
 	txNum, err := d.txNumsReader.Min(ttx, d.blockNumber+1)
 	if err != nil {
@@ -158,11 +164,15 @@
 		return nil, err
 	}
 
+	var hardLimit = DumperIterationsHardLimit
+
+	t := time.Now()
 	var nextKey []byte
 	it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nil, txNum, order.Asc, maxResults)
 	if err != nil {
 		return nil, err
 	}
+	fmt.Printf("[dbg] after DomainRange: %s\n", time.Since(t))
 	defer it.Close()
 	for it.HasNext() {
 		k, v, err := it.Next()
@@ -171,11 +181,13 @@
 		}
 		if maxResults > 0 && numberOfResults >= maxResults {
 			nextKey = append(nextKey[:0], k...)
+			fmt.Printf("[dbg] dumper iter acc. break: nextKey %x\n", nextKey)
 			break
 		}
 		if len(v) == 0 {
 			continue
 		}
+		fmt.Printf("[dbg] dumper iter acc: %x\n", k)
 
 		if e := accounts.DeserialiseV3(&acc, v); e != nil {
 			return nil, fmt.Errorf("decoding %x for %x: %w", v, k, e)
@@ -204,7 +216,14 @@
 		addrList = append(addrList, libcommon.BytesToAddress(k))
 
 		numberOfResults++
+
+		if hardLimit--; hardLimit < 0 {
+			return nil, TooMuchIterations
+		}
 	}
+	it.Close()
+
+	fmt.Printf("[dbg] first loop done: %d\n", len(addrList))
 
 	for i, addr := range addrList {
 		account := accountList[i]
@@ -226,10 +245,16 @@
 					continue // Skip deleted entries
 				}
 				loc := k[20:]
+				fmt.Printf("[dbg] dumper iter storage: %x, %x\n", k[:20], k[20:])
 				account.Storage[libcommon.BytesToHash(loc).String()] = common.Bytes2Hex(vs)
 				h, _ := libcommon.HashData(loc)
 				t.Update(h.Bytes(), libcommon.Copy(vs))
+
+				if hardLimit--; hardLimit < 0 {
+					return nil, TooMuchIterations
+				}
 			}
+			r.Close()
 
 			account.Root = t.Hash().Bytes()
 		}

erigon-lib/seg/decompress.go

@@ -29,12 +29,11 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/erigontech/erigon-lib/common/assert"
-	"github.com/erigontech/erigon-lib/log/v3"
-
 	"github.com/c2h5oh/datasize"
 
+	"github.com/erigontech/erigon-lib/common/assert"
 	"github.com/erigontech/erigon-lib/common/dbg"
+	"github.com/erigontech/erigon-lib/log/v3"
 	"github.com/erigontech/erigon-lib/mmap"
 )
 

erigon-lib/state/aggregator_test.go

@@ -32,12 +32,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/erigontech/erigon-lib/commitment"
-
-	"github.com/erigontech/erigon-lib/common/background"
-
 	"github.com/c2h5oh/datasize"
+
+	"github.com/erigontech/erigon-lib/commitment"
 	"github.com/erigontech/erigon-lib/common"
+	"github.com/erigontech/erigon-lib/common/background"
 	"github.com/erigontech/erigon-lib/common/datadir"
 	"github.com/erigontech/erigon-lib/common/length"
 	"github.com/erigontech/erigon-lib/etl"

erigon-lib/state/domain.go

@@ -1962,10 +1962,12 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey
 	//if err != nil {
 	//	return nil, err
 	//}
-	histStateIt, err := dt.ht.WalkAsOf(ctx, ts, fromKey, toKey, tx, limit)
+	fmt.Printf("[dbg] DomainRange 1: %s, %x, %x\n", dt.d.name, fromKey, toKey)
+	histStateIt, err := dt.ht.WalkAsOf(ctx, ts, fromKey, toKey, asc, limit, tx)
 	if err != nil {
 		return nil, err
 	}
+	fmt.Printf("[dbg] DomainRange 2: %s\n", dt.d.name)
 	lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit)
 	if err != nil {
 		return nil, err

erigon-lib/state/domain_shared_test.go

@@ -26,11 +26,10 @@ import (
 	"github.com/holiman/uint256"
 	"github.com/stretchr/testify/require"
 
+	"github.com/erigontech/erigon-lib/common/length"
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon-lib/kv/rawdbv3"
 	"github.com/erigontech/erigon-lib/log/v3"
-
-	"github.com/erigontech/erigon-lib/common/length"
 	"github.com/erigontech/erigon-lib/types"
 )
 

erigon-lib/state/domain_test.go

@@ -1275,6 +1275,7 @@ func filledDomainFixedSize(t *testing.T, keysCount, txCount, aggStep uint64, log
 
 func generateTestDataForDomainCommitment(tb testing.TB, keySize1, keySize2, totalTx, keyTxsLimit, keyLimit uint64) map[string]map[string][]upd {
 	tb.Helper()
+	defer func(t time.Time) { fmt.Printf("domain_test.go:1262: %s\n", time.Since(t)) }(time.Now())
 
 	doms := make(map[string]map[string][]upd)
 	r := newRnd(31)
@@ -1328,7 +1329,6 @@ func generateRandomKey(r *rndGen, size uint64) string {
 func generateRandomKeyBytes(r *rndGen, size uint64) []byte {
 	key := make([]byte, size)
 	r.Read(key)
-
 	return key
 }
 
@@ -1403,8 +1403,8 @@ func TestDomain_GetAfterAggregation(t *testing.T) {
 	defer tx.Rollback()
 
 	d.historyLargeValues = false
-	d.History.compression = seg.CompressKeys | seg.CompressVals
-	d.compression = seg.CompressKeys | seg.CompressVals
+	d.History.compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals
+	d.compression = seg.CompressNone         //seg.CompressKeys | seg.CompressVals
 	d.filenameBase = kv.FileCommitmentDomain
 
 	dc := d.BeginFilesRo()
@@ -1464,6 +1464,67 @@ func TestDomain_GetAfterAggregation(t *testing.T) {
 	}
 }
 
+func TestDomainRange(t *testing.T) {
+	db, d := testDbAndDomainOfStep(t, 25, log.New())
+	require := require.New(t)
+
+	tx, err := db.BeginRw(context.Background())
+	require.NoError(err)
+	defer tx.Rollback()
+
+	d.historyLargeValues = false
+	d.History.compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals
+	d.compression = seg.CompressNone         // seg.CompressKeys | seg.CompressVals
+	d.filenameBase = kv.FileAccountDomain
+
+	dc := d.BeginFilesRo()
+	defer d.Close()
+	writer := dc.NewWriter()
+	defer writer.close()
+
+	keySize1 := uint64(4)
+	keySize2 := uint64(4)
+	totalTx := uint64(300)
+	keyTxsLimit := uint64(2)
+	keyLimit := uint64(10)
+
+	// put some kvs
+	data := generateTestData(t, keySize1, keySize2, totalTx, keyTxsLimit, keyLimit)
+	for key, updates := range data {
+		p := []byte{}
+		for i := 0; i < len(updates); i++ {
+			writer.SetTxNum(updates[i].txNum)
+			err = writer.PutWithPrev([]byte(key), nil, updates[i].value, p, 0)
+			require.NoError(err)
+			p = common.Copy(updates[i].value)
+		}
+	}
+	writer.SetTxNum(totalTx)
+
+	err = writer.Flush(context.Background(), tx)
+	require.NoError(err)
+
+	// aggregate
+	collateAndMerge(t, db, tx, d, totalTx)
+	require.NoError(tx.Commit())
+
+	tx, err = db.BeginRw(context.Background())
+	require.NoError(err)
+	defer tx.Rollback()
+	dc.Close()
+
+	dc = d.BeginFilesRo()
+	defer dc.Close()
+
+	//it, err := dc.DomainRangeLatest(tx, nil, nil, -1)
+	it, err := dc.DomainRange(context.Background(), tx, nil, nil, 190, order.Asc, -1)
+	require.NoError(err)
+	keys, vals, err := stream.ToArrayKV(it)
+	require.NoError(err)
+	fmt.Printf("keys: %x\n", keys)
+	fmt.Printf("vals: %x\n", vals)
+}
+
 func TestDomain_CanPruneAfterAggregation(t *testing.T) {
 	t.Parallel()
 
@@ -1981,10 +2042,10 @@ func TestDomain_Unwind(t *testing.T) {
 			uc := d.BeginFilesRo()
 			defer uc.Close()
 
-			et, err := ectx.ht.WalkAsOf(context.Background(), unwindTo-1, nil, nil, etx, -1)
+			et, err := ectx.ht.WalkAsOf(context.Background(), unwindTo-1, nil, nil, order.Asc, -1, etx)
 			require.NoError(t, err)
 
-			ut, err := uc.ht.WalkAsOf(context.Background(), unwindTo-1, nil, nil, utx, -1)
+			ut, err := uc.ht.WalkAsOf(context.Background(), unwindTo-1, nil, nil, order.Asc, -1, utx)
 			require.NoError(t, err)
 
 			compareIterators(t, et, ut)