diff --git a/accounts/cache.go b/accounts/cache.go index 3b532a495..7cba83596 100644 --- a/accounts/cache.go +++ b/accounts/cache.go @@ -145,7 +145,7 @@ func (ac *addrCache) find(a Account) (Account, error) { return matches[i], nil } } - if (a.Address == common.Address{}) { + if a.Address.IsEmpty() { return Account{}, ErrNoMatch } } @@ -239,7 +239,7 @@ func (ac *addrCache) scan() ([]Account, error) { switch { case err != nil: glog.V(logger.Debug).Infof("can't decode key %s: %v", path, err) - case (keyJSON.Address == common.Address{}): + case keyJSON.Address.IsEmpty(): glog.V(logger.Debug).Infof("can't decode key %s: missing or zero address", path) default: addrs = append(addrs, Account{Address: keyJSON.Address, File: path}) diff --git a/accounts/cachedb.go b/accounts/cachedb.go index 0a4b978be..35910bf67 100644 --- a/accounts/cachedb.go +++ b/accounts/cachedb.go @@ -236,7 +236,7 @@ func (cdb *cacheDB) find(a Account) (Account, error) { return acc, e } // no other possible way - if (a.Address == common.Address{}) { + if a.Address.IsEmpty() { return Account{}, ErrNoMatch } } @@ -498,7 +498,7 @@ func processKeyFile(wg *sync.WaitGroup, path string, fi os.FileInfo, i int, numF case err != nil: glog.V(logger.Debug).Infof("(%v/%v) can't decode key %s: %v", i, numFiles, path, err) errs <- err - case (keyJSON.Address == common.Address{}): + case keyJSON.Address.IsEmpty(): glog.V(logger.Debug).Infof("(%v/%v) can't decode key %s: missing or zero address", i, numFiles, path) errs <- fmt.Errorf("(%v/%v) can't decode key %s: missing or zero address", i, numFiles, path) default: diff --git a/cmd/geth/build_atxi_cmd.go b/cmd/geth/build_atxi_cmd.go new file mode 100644 index 000000000..b9314df58 --- /dev/null +++ b/cmd/geth/build_atxi_cmd.go @@ -0,0 +1,146 @@ +package main + +import ( + "github.com/ethereumproject/go-ethereum/core" + "github.com/ethereumproject/go-ethereum/core/types" + "github.com/ethereumproject/go-ethereum/ethdb" + "github.com/ethereumproject/go-ethereum/logger" + "github.com/ethereumproject/go-ethereum/logger/glog" + "gopkg.in/urfave/cli.v1" + "os" + "time" +) + +var buildAddrTxIndexCommand = cli.Command{ + Action: buildAddrTxIndexCmd, + Name: "atxi-build", + Usage: "Generate index for transactions by address", + Description: ` + Builds an index for transactions by address. + The command is idempotent; it will not hurt to run multiple times on the same range. + If run without --start flag, the command makes use of a persistent placeholder, so you can + run the command on multiple occasions and pick up indexing progress where the last session + left off. + To enable address-transaction indexing during block sync and import, use the '--atxi' flag. + `, + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "start", + Usage: "Block number at which to begin building index", + }, + cli.IntFlag{ + Name: "stop", + Usage: "Block number at which to stop building index", + }, + cli.IntFlag{ + Name: "step", + Usage: "Step increment for batching. Higher number requires more mem, but may be faster", + Value: 10000, + }, + }, +} + +func buildAddrTxIndexCmd(ctx *cli.Context) error { + + // Divide global cache availability equally between chaindata (pre-existing blockdata) and + // address-transaction database. This ratio is arbitrary and could potentially be optimized or delegated to be user configurable. + ethdb.SetCacheRatio("chaindata", 0.5) + ethdb.SetHandleRatio("chaindata", 1) + ethdb.SetCacheRatio("indexes", 0.5) + ethdb.SetHandleRatio("indexes", 1) + + startIndex := uint64(ctx.Int("start")) + var stopIndex uint64 + + indexDb := MakeIndexDatabase(ctx) + if indexDb == nil { + glog.Fatalln("indexes db is nil") + } + defer indexDb.Close() + + // Use persistent placeholder in case start not spec'd + if !ctx.IsSet("start") { + startIndex = core.GetATXIBookmark(indexDb) + } + + bc, chainDB := MakeChain(ctx) + if bc == nil || chainDB == nil { + glog.Fatalln("bc or cdb is nil") + } + defer chainDB.Close() + + stopIndex = uint64(ctx.Int("stop")) + if stopIndex == 0 { + stopIndex = bc.CurrentBlock().NumberU64() + if n := bc.CurrentFastBlock().NumberU64(); n > stopIndex { + stopIndex = n + } + } + + if stopIndex < startIndex { + glog.Fatalln("start must be prior to (smaller than) or equal to stop, got start=", startIndex, "stop=", stopIndex) + } + if startIndex == stopIndex { + glog.D(logger.Error).Infoln("atxi is up to date, exiting") + os.Exit(0) + } + + var block *types.Block + blockIndex := startIndex + block = bc.GetBlockByNumber(blockIndex) + if block == nil { + glog.Fatalln(blockIndex, "block is nil") + } + + var inc = uint64(ctx.Int("step")) + startTime := time.Now() + totalTxCount := uint64(0) + glog.D(logger.Error).Infoln("Address/tx indexing (atxi) start:", startIndex, "stop:", stopIndex, "step:", inc, "| This may take a while.") + breaker := false + for i := startIndex; i <= stopIndex; i = i + inc { + if i+inc > stopIndex { + inc = stopIndex - i + breaker = true + } + + stepStartTime := time.Now() + + // It may seem weird to pass i, i+inc, and inc, but its just a "coincidence" + // The function could accepts a smaller step for batch putting (in this case, inc), + // or a larger stopBlock (i+inc), but this is just how this cmd is using the fn now + // We could mess around a little with exploring batch optimization... + txsCount, err := bc.WriteBlockAddrTxIndexesBatch(indexDb, i, i+inc, inc) + if err != nil { + return err + } + totalTxCount += uint64(txsCount) + + if err := core.SetATXIBookmark(indexDb, i+inc); err != nil { + glog.Fatalln(err) + } + + glog.D(logger.Error).Infof("atxi-build: block %d / %d txs: %d took: %v %.2f bps %.2f txps", i+inc, stopIndex, txsCount, time.Since(stepStartTime).Round(time.Millisecond), float64(inc)/time.Since(stepStartTime).Seconds(), float64(txsCount)/time.Since(stepStartTime).Seconds()) + glog.V(logger.Info).Infof("atxi-build: block %d / %d txs: %d took: %v %.2f bps %.2f txps", i+inc, stopIndex, txsCount, time.Since(stepStartTime).Round(time.Millisecond), float64(inc)/time.Since(stepStartTime).Seconds(), float64(txsCount)/time.Since(stepStartTime).Seconds()) + + if breaker { + break + } + } + + if err := core.SetATXIBookmark(indexDb, stopIndex); err != nil { + glog.Fatalln(err) + } + + // Print summary + totalBlocksF := float64(stopIndex - startIndex) + totalTxsF := float64(totalTxCount) + took := time.Since(startTime) + glog.D(logger.Error).Infof(`Finished atxi-build in %v: %d blocks (~ %.2f blocks/sec), %d txs (~ %.2f txs/sec)`, + took.Round(time.Second), + stopIndex-startIndex, + totalBlocksF/took.Seconds(), + totalTxCount, + totalTxsF/took.Seconds(), + ) + return nil +} diff --git a/cmd/geth/flag.go b/cmd/geth/flag.go index 84e2d8ca9..b00928b20 100644 --- a/cmd/geth/flag.go +++ b/cmd/geth/flag.go @@ -601,6 +601,7 @@ func mustMakeEthConf(ctx *cli.Context, sconf *core.SufficientChainConfig) *eth.C ethConf := ð.Config{ ChainConfig: sconf.ChainConfig, Genesis: sconf.Genesis, + UseAddrTxIndex: ctx.GlobalBool(aliasableName(AddrTxIndexFlag.Name, ctx)), FastSync: ctx.GlobalBool(aliasableName(FastSyncFlag.Name, ctx)), BlockChainVersion: ctx.GlobalInt(aliasableName(BlockchainVersionFlag.Name, ctx)), DatabaseCache: ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx)), @@ -799,18 +800,32 @@ func MustMakeChainConfigFromDefaults(ctx *cli.Context) *core.ChainConfig { // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. func MakeChainDatabase(ctx *cli.Context) ethdb.Database { var ( - datadir = MustMakeChainDataDir(ctx) - cache = ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx)) - handles = MakeDatabaseHandles() + chaindir = MustMakeChainDataDir(ctx) + cache = ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx)) + handles = MakeDatabaseHandles() ) - chainDb, err := ethdb.NewLDBDatabase(filepath.Join(datadir, "chaindata"), cache, handles) + chainDb, err := ethdb.NewLDBDatabase(filepath.Join(chaindir, "chaindata"), cache, handles) if err != nil { glog.Fatal("Could not open database: ", err) } return chainDb } +func MakeIndexDatabase(ctx *cli.Context) ethdb.Database { + var ( + chaindir = MustMakeChainDataDir(ctx) + cache = ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx)) + handles = MakeDatabaseHandles() + ) + + indexesDb, err := ethdb.NewLDBDatabase(filepath.Join(chaindir, "indexes"), cache, handles) + if err != nil { + glog.Fatal("Could not open database: ", err) + } + return indexesDb +} + // MakeChain creates a chain manager from set command line flags. func MakeChain(ctx *cli.Context) (chain *core.BlockChain, chainDb ethdb.Database) { var err error diff --git a/cmd/geth/flags.go b/cmd/geth/flags.go index 1addc6b1b..88559defa 100644 --- a/cmd/geth/flags.go +++ b/cmd/geth/flags.go @@ -98,6 +98,10 @@ var ( Name: "light-kdf,lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", } + AddrTxIndexFlag = cli.BoolFlag{ + Name: "atxi,add-tx-index", + Usage: "Toggle indexes for transactions by address. Pre-existing chaindata can be indexed with command 'atxi-build'", + } // Network Split settings ETFChain = cli.BoolFlag{ Name: "etf", diff --git a/cmd/geth/main.go b/cmd/geth/main.go index b7a5a3027..69866b0bd 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -52,7 +52,7 @@ var makeDagCommand = cli.Command{ Usage: "Generate ethash dag (for testing)", Description: ` The makedag command generates an ethash DAG in /tmp/dag. - + This command exists to support the system testing project. Regular users do not need to execute it. `, @@ -136,6 +136,7 @@ func makeCLIApp() (app *cli.App) { gpuBenchCommand, versionCommand, makeMlogDocCommand, + buildAddrTxIndexCommand, } app.Flags = []cli.Flag{ @@ -153,6 +154,7 @@ func makeCLIApp() (app *cli.App) { ChainIdentityFlag, BlockchainVersionFlag, FastSyncFlag, + AddrTxIndexFlag, CacheFlag, LightKDFFlag, JSpathFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index fc6c915b8..902c170c4 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -25,11 +25,6 @@ import ( ) // AppHelpTemplate is the test template for the default, global app help topic. -//var x = cli.Command{} -//x.Usage -//x.UsageText -//x.ArgsUsage -//x.Subcommands var AppHelpTemplate = `NAME: {{.App.Name}} - {{.App.Usage}} @@ -40,7 +35,7 @@ VERSION: {{.App.Version}}{{if .CommandAndFlagGroups}} COMMANDS AND FLAGS: ------------------------------------------------------------------------- + {{range .CommandAndFlagGroups}}{{.Name}} ------------------------------------------------------------------------ {{if .Commands}}{{range .Commands}} @@ -93,12 +88,14 @@ var AppHelpFlagAndCommandGroups = []flagGroup{ Commands: []cli.Command{ accountCommand, walletCommand, + buildAddrTxIndexCommand, }, Flags: []cli.Flag{ KeyStoreDirFlag, UnlockedAccountFlag, PasswordFileFlag, AccountsIndexFlag, + AddrTxIndexFlag, }, }, { @@ -205,6 +202,10 @@ var AppHelpFlagAndCommandGroups = []flagGroup{ Flags: []cli.Flag{ WhisperEnabledFlag, NatspecEnabledFlag, + DisplayFlag, + DisplayFormatFlag, + NeckbeardFlag, + AddrTxIndexFlag, }, }, { diff --git a/common/registrar/registrar.go b/common/registrar/registrar.go index 84b5ec7be..51c89d84a 100644 --- a/common/registrar/registrar.go +++ b/common/registrar/registrar.go @@ -112,7 +112,7 @@ func (self *Registrar) SetGlobalRegistrar(namereg string, addr common.Address) ( return } if zero.MatchString(GlobalRegistrarAddr) { - if (addr == common.Address{}) { + if addr.IsEmpty() { err = fmt.Errorf("GlobalRegistrar address not found and sender for creation not given") return } else { @@ -142,7 +142,7 @@ func (self *Registrar) SetHashReg(hashreg string, addr common.Address) (txhash s HashRegAddr = "0x" + res[len(res)-40:] } if err != nil || zero.MatchString(HashRegAddr) { - if (addr == common.Address{}) { + if addr.IsEmpty() { err = fmt.Errorf("HashReg address not found and sender for creation not given") return } @@ -177,7 +177,7 @@ func (self *Registrar) SetUrlHint(urlhint string, addr common.Address) (txhash s UrlHintAddr = "0x" + res[len(res)-40:] } if err != nil || zero.MatchString(UrlHintAddr) { - if (addr == common.Address{}) { + if addr.IsEmpty() { err = fmt.Errorf("UrlHint address not found and sender for creation not given") return } diff --git a/common/types.go b/common/types.go index 768146736..b238d89c5 100644 --- a/common/types.go +++ b/common/types.go @@ -125,6 +125,14 @@ func StringToAddress(s string) Address { return BytesToAddress([]byte(s)) } func BigToAddress(b *big.Int) Address { return BytesToAddress(b.Bytes()) } func HexToAddress(s string) Address { return BytesToAddress(FromHex(s)) } +func EmptyAddress(a Address) bool { + return a == Address{} +} + +func (a Address) IsEmpty() bool { + return EmptyAddress(a) +} + // IsHexAddress verifies whether a string can represent a valid hex-encoded // Ethereum address or not. func IsHexAddress(s string) bool { diff --git a/core/blockchain.go b/core/blockchain.go index 4f485cf37..e0f6e4dfc 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -32,6 +32,7 @@ import ( "reflect" "strconv" + "encoding/binary" "github.com/ethereumproject/go-ethereum/common" "github.com/ethereumproject/go-ethereum/core/state" "github.com/ethereumproject/go-ethereum/core/types" @@ -109,6 +110,9 @@ type BlockChain struct { pow pow.PoW processor Processor // block processor interface validator Validator // block and state validator interface + + useAddTxIndex bool + indexesDb ethdb.Database } // NewBlockChain returns a fully initialised block chain using information @@ -214,6 +218,17 @@ func (self *BlockChain) GetEventMux() *event.TypeMux { return self.eventMux } +// SetAddTxIndex sets the db and in-use var for atx indexing. +func (self *BlockChain) SetAddTxIndex(db ethdb.Database, tf bool) { + self.useAddTxIndex = tf + self.indexesDb = db +} + +// GetAddTxIndex return indexes db and if atx index in use. +func (self *BlockChain) GetAddTxIndex() (ethdb.Database, bool) { + return self.indexesDb, self.useAddTxIndex +} + func (self *BlockChain) getProcInterrupt() bool { return atomic.LoadInt32(&self.procInterrupt) == 1 } @@ -773,6 +788,45 @@ func (bc *BlockChain) SetHead(head uint64) error { glog.Fatalf("failed to reset head fast block hash: %v", err) } + if bc.useAddTxIndex { + ldb, ok := bc.indexesDb.(*ethdb.LDBDatabase) + if !ok { + glog.Fatal("could not cast indexes db to level db") + } + + var removals [][]byte + deleteRemovalsFn := func(rs [][]byte) { + for _, r := range rs { + if e := ldb.Delete(r); e != nil { + glog.Fatal(e) + } + } + } + + pre := ethdb.NewBytesPrefix(txAddressIndexPrefix) + it := ldb.NewIteratorRange(pre) + + for it.Next() { + key := it.Key() + _, bn, _, _, _ := resolveAddrTxBytes(key) + n := binary.LittleEndian.Uint64(bn) + if n > head { + removals = append(removals, key) + // Prevent removals from getting too massive in case it's a big rollback + // 100000 is a guess at a big but not-too-big memory allowance + if len(removals) > 100000 { + deleteRemovalsFn(removals) + removals = [][]byte{} + } + } + } + it.Release() + if e := it.Error(); e != nil { + return e + } + deleteRemovalsFn(removals) + } + bc.mu.Unlock() return bc.LoadLastState(false) } @@ -1231,6 +1285,12 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain glog.Fatal(errs[index]) return } + // Store the addr-tx indexes if enabled + if self.useAddTxIndex { + if err := WriteBlockAddTxIndexes(self.indexesDb, block); err != nil { + glog.Fatalf("failed to write block add-tx indexes", err) + } + } atomic.AddInt32(&stats.processed, 1) } } @@ -1276,6 +1336,42 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain return 0, nil } +// WriteBlockAddrTxIndexesBatch builds indexes for a given range of blocks N. It writes batches at increment 'step'. +// If any error occurs during db writing it will be returned immediately. +// It's sole implementation is the command 'atxi-build', since we must use individual block atxi indexing during +// sync and import in order to ensure we're on the canonical chain for each block. +func (self *BlockChain) WriteBlockAddrTxIndexesBatch(indexDb ethdb.Database, startBlockN, stopBlockN, stepN uint64) (txsCount int, err error) { + block := self.GetBlockByNumber(startBlockN) + batch := indexDb.NewBatch() + + blockProcessedCount := uint64(0) + blockProcessedHead := func() uint64 { + return startBlockN + blockProcessedCount + } + + for block != nil && blockProcessedHead() <= stopBlockN { + txP, err := putBlockAddrTxsToBatch(batch, block) + if err != nil { + return txsCount, err + } + txsCount += txP + blockProcessedCount++ + + // Write on stepN mod + if blockProcessedCount%stepN == 0 { + if err := batch.Write(); err != nil { + return txsCount, err + } else { + batch = indexDb.NewBatch() + } + } + block = self.GetBlockByNumber(blockProcessedHead()) + } + + // This will put the last batch + return txsCount, batch.Write() +} + // WriteBlock writes the block to the chain. func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err error) { @@ -1523,6 +1619,12 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (chainIndex int, err err if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil { return i, err } + // Store the addr-tx indexes if enabled + if self.useAddTxIndex { + if err := WriteBlockAddTxIndexes(self.indexesDb, block); err != nil { + glog.Fatalf("failed to write block add-tx indexes", err) + } + } case SideStatTy: if glog.V(logger.Detail) { glog.Infof("inserted forked block #%d (TD=%v) (%d TXs %d UNCs) [%s]. Took %v\n", block.Number(), block.Difficulty(), len(block.Transactions()), len(block.Uncles()), block.Hash().Hex(), time.Since(bstart)) @@ -1655,6 +1757,17 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { ).Send(mlogBlockchain) } + // Remove all atxis from old chain; indexes should only reflect canonical + if self.useAddTxIndex { + for _, block := range oldChain { + for _, tx := range block.Transactions() { + if err := RmAddrTx(self.indexesDb, tx); err != nil { + return err + } + } + } + } + var addedTxs types.Transactions // insert blocks. Order does not matter. Last block will be written in ImportChain itself which creates the new head properly for _, block := range newChain { @@ -1664,6 +1777,12 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { if err := WriteTransactions(self.chainDb, block); err != nil { return err } + // Store the addr-tx indexes if enabled + if self.useAddTxIndex { + if err := WriteBlockAddTxIndexes(self.indexesDb, block); err != nil { + return err + } + } receipts := GetBlockReceipts(self.chainDb, block.Hash()) // write receipts if err := WriteReceipts(self.chainDb, receipts); err != nil { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 0f71dd48a..b0673bd96 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -37,6 +37,8 @@ import ( "github.com/ethereumproject/go-ethereum/logger/glog" "github.com/ethereumproject/go-ethereum/rlp" "github.com/hashicorp/golang-lru" + "io/ioutil" + "strings" ) func init() { @@ -884,6 +886,207 @@ func TestFastVsFullChains(t *testing.T) { } } +func TestFastVsFullChainsATXI(t *testing.T) { + archiveDir, e := ioutil.TempDir("", "archive-") + if e != nil { + t.Fatal(e) + } + fastDir, e := ioutil.TempDir("", "fast-") + if e != nil { + t.Fatal(e) + } + defer os.RemoveAll(archiveDir) + defer os.RemoveAll(fastDir) + + // Create the dbs + // + archiveDb, err := ethdb.NewLDBDatabase(archiveDir, 10, 100) + if err != nil { + t.Fatal(err) + } + fastDb, err := ethdb.NewLDBDatabase(fastDir, 10, 100) + if err != nil { + t.Fatal(err) + } + + MinGasLimit = big.NewInt(125000) + + key1, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + if err != nil { + t.Fatal(err) + } + key2, err := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + if err != nil { + t.Fatal(err) + } + + var ( + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = crypto.PubkeyToAddress(key2.PublicKey) + signer = types.NewChainIdSigner(big.NewInt(63)) + dbs = []ethdb.Database{archiveDb, fastDb} + config = MakeDiehardChainConfig() + ) + + for i, db := range dbs { + t1, err := types.NewTransaction(0, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1) + if err != nil { + t.Fatal(err) + } + t2, err := types.NewTransaction(1, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1) + if err != nil { + t.Fatal(err) + } + t3, err := types.NewTransaction(0, addr1, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2) + if err != nil { + t.Fatal(err) + } + genesis := WriteGenesisBlockForTesting(db, + GenesisAccount{addr1, big.NewInt(1000000)}, + GenesisAccount{addr2, big.NewInt(1000000)}, + ) + blocks, receipts := GenerateChain(config, genesis, db, 3, func(i int, gen *BlockGen) { + if i == 0 { + gen.AddTx(t1) + } + if i == 1 { + gen.AddTx(t2) + } + if i == 2 { + gen.AddTx(t3) + } + }) + + blockchain, err := NewBlockChain(db, config, FakePow{}, new(event.TypeMux)) + if err != nil { + t.Fatal(err) + } + // turn on atxi + blockchain.SetAddTxIndex(db, true) + if i == 0 { + if n, err := blockchain.InsertChain(blocks); err != nil { + t.Fatalf("failed to process block %d: %v", n, err) + } + } else { + headers := make([]*types.Header, len(blocks)) + for i, block := range blocks { + headers[i] = block.Header() + } + if n, err := blockchain.InsertHeaderChain(headers, 1); err != nil { + t.Fatalf("failed to insert header %d: %v", n, err) + } + if n, err := blockchain.InsertReceiptChain(blocks, receipts); err != nil { + t.Fatalf("failed to insert receipt %d: %v", n, err) + } + } + + out := GetAddrTxs(db, addr1, 0, 0, "", "", -1, -1, false) + if len(out) != 3 { + t.Errorf("[%d] got: %v, want: %v", i, len(out), 3) + } + out = GetAddrTxs(db, addr1, 0, 0, "from", "", -1, -1, false) + if len(out) != 2 { + t.Errorf("[%d] got: %v, want: %v", i, len(out), 2) + } + out = GetAddrTxs(db, addr1, 0, 0, "to", "", -1, -1, false) + if len(out) != 1 { + t.Errorf("[%d] got: %v, want: %v", i, len(out), 1) + } + out = GetAddrTxs(db, addr2, 0, 0, "", "", -1, -1, false) + if len(out) != 3 { + t.Errorf("[%d] got: %v, want: %v", i, len(out), 3) + } + out = GetAddrTxs(db, addr2, 3, 3, "", "", -1, -1, false) + if len(out) != 1 { + t.Errorf("[%d] got: %v, want: %v", i, len(out), 1) + } + } +} + +func TestRmAddrTx(t *testing.T) { + archiveDir, e := ioutil.TempDir("", "archive-") + if e != nil { + t.Fatal(e) + } + defer os.RemoveAll(archiveDir) + + // Create the dbs + // + db, err := ethdb.NewLDBDatabase(archiveDir, 10, 100) + if err != nil { + t.Fatal(err) + } + + MinGasLimit = big.NewInt(125000) + + key1, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + if err != nil { + t.Fatal(err) + } + key2, err := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + if err != nil { + t.Fatal(err) + } + + var ( + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = crypto.PubkeyToAddress(key2.PublicKey) + signer = types.NewChainIdSigner(big.NewInt(63)) + config = MakeDiehardChainConfig() + ) + + t1, err := types.NewTransaction(0, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1) + if err != nil { + t.Fatal(err) + } + t2, err := types.NewTransaction(1, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1) + if err != nil { + t.Fatal(err) + } + t3, err := types.NewTransaction(0, addr1, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2) + if err != nil { + t.Fatal(err) + } + genesis := WriteGenesisBlockForTesting(db, + GenesisAccount{addr1, big.NewInt(1000000)}, + GenesisAccount{addr2, big.NewInt(1000000)}, + ) + blocks, _ := GenerateChain(config, genesis, db, 3, func(i int, gen *BlockGen) { + if i == 0 { + gen.AddTx(t1) + } + if i == 1 { + gen.AddTx(t2) + } + if i == 2 { + gen.AddTx(t3) + } + }) + + blockchain, err := NewBlockChain(db, config, FakePow{}, new(event.TypeMux)) + if err != nil { + t.Fatal(err) + } + // turn on atxi + blockchain.SetAddTxIndex(db, true) + + if n, err := blockchain.InsertChain(blocks); err != nil { + t.Fatalf("failed to process block %d: %v", n, err) + } + + out := GetAddrTxs(db, addr1, 0, 0, "", "", -1, -1, false) + if len(out) != 3 { + t.Errorf("got: %v, want: %v", len(out), 3) + } + if err := RmAddrTx(db, t1); err != nil { + t.Fatal(err) + } + out = GetAddrTxs(db, addr1, 0, 0, "", "", -1, -1, false) + if len(out) != 2 { + t.Errorf("got: %v, want: %v", len(out), 2) + } +} + // Tests that various import methods move the chain head pointers to the correct // positions. func TestLightVsFastVsFullChainHeads(t *testing.T) { @@ -986,6 +1189,28 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Tests that chain reorganisations handle transaction removals and reinsertions. func TestChainTxReorgs(t *testing.T) { + db, err := ethdb.NewMemDatabase() + if err != nil { + t.Fatal(err) + } + testChainTxReorgs(t, db, false) +} + +func TestChainTxReorgsAtxi(t *testing.T) { + p, err := ioutil.TempDir("", "test-reorg-atxi-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(p) + + db, err := ethdb.NewLDBDatabase(p, 10, 100) + if err != nil { + t.Fatal(err) + } + testChainTxReorgs(t, db, true) +} + +func testChainTxReorgs(t *testing.T, db ethdb.Database, withATXI bool) { MinGasLimit = big.NewInt(125000) key1, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1000,10 +1225,6 @@ func TestChainTxReorgs(t *testing.T) { if err != nil { t.Fatal(err) } - db, err := ethdb.NewMemDatabase() - if err != nil { - t.Fatal(err) - } var ( addr1 = crypto.PubkeyToAddress(key1.PublicKey) @@ -1017,39 +1238,47 @@ func TestChainTxReorgs(t *testing.T) { GenesisAccount{addr3, big.NewInt(1000000)}, ) // Create two transactions shared between the chains: + // addr1 -> addr2 // - postponed: transaction included at a later block in the forked chain // - swapped: transaction included at the same block number in the forked chain - postponed, err := types.NewTransaction(0, addr1, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1) + postponed, err := types.NewTransaction(0, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1) if err != nil { t.Fatal(err) } - swapped, err := types.NewTransaction(1, addr1, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1) + swapped, err := types.NewTransaction(1, addr2, big.NewInt(1001), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1) if err != nil { t.Fatal(err) } // Create two transactions that will be dropped by the forked chain: + // addr2 -> addr3 // - pastDrop: transaction dropped retroactively from a past block // - freshDrop: transaction dropped exactly at the block where the reorg is detected var pastDrop, freshDrop *types.Transaction // Create three transactions that will be added in the forked chain: + // addr3 -> addr1 // - pastAdd: transaction added before the reorganization is detected // - freshAdd: transaction added at the exact block the reorg is detected // - futureAdd: transaction added after the reorg has already finished var pastAdd, freshAdd, futureAdd *types.Transaction + // ATXI tallies, (means) will be removed + // addr1: 2f+3t + // addr2: 2t+(2f) + // addr3: (2t)+3f + chainConfig := MakeDiehardChainConfig() chain, _ := GenerateChain(chainConfig, genesis, db, 3, func(i int, gen *BlockGen) { switch i { case 0: - pastDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2) + pastDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr3, big.NewInt(1002), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2) gen.AddTx(pastDrop) // This transaction will be dropped in the fork from below the split point gen.AddTx(postponed) // This transaction will be postponed till block #3 in the fork case 2: - freshDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2) + freshDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr3, big.NewInt(1003), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2) gen.AddTx(freshDrop) // This transaction will be dropped in the fork from exactly at the split point gen.AddTx(swapped) // This transaction will be swapped out at the exact height @@ -1064,6 +1293,9 @@ func TestChainTxReorgs(t *testing.T) { if err != nil { t.Fatal(err) } + if withATXI { + blockchain.SetAddTxIndex(db, true) + } if i, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -1072,18 +1304,18 @@ func TestChainTxReorgs(t *testing.T) { chain, _ = GenerateChain(chainConfig, genesis, db, 5, func(i int, gen *BlockGen) { switch i { case 0: - pastAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3) + pastAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr1, big.NewInt(1004), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3) gen.AddTx(pastAdd) // This transaction needs to be injected during reorg case 2: gen.AddTx(postponed) // This transaction was postponed from block #1 in the original chain gen.AddTx(swapped) // This transaction was swapped from the exact current spot in the original chain - freshAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3) + freshAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr1, big.NewInt(1005), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3) gen.AddTx(freshAdd) // This transaction will be added exactly at reorg time case 3: - futureAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3) + futureAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr1, big.NewInt(1006), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3) gen.AddTx(futureAdd) // This transaction will be added after a full reorg } }) @@ -1091,8 +1323,14 @@ func TestChainTxReorgs(t *testing.T) { t.Fatalf("failed to insert forked chain: %v", err) } + // Conveniently grouped + txsRemoved := types.Transactions{pastDrop, freshDrop} + txsAdded := types.Transactions{pastAdd, freshAdd, futureAdd} + txsShared := types.Transactions{postponed, swapped} + txsAll := types.Transactions{pastDrop, freshDrop, pastAdd, freshAdd, futureAdd, postponed, swapped} + // removed tx - for i, tx := range (types.Transactions{pastDrop, freshDrop}) { + for i, tx := range txsRemoved { if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil { t.Errorf("drop %d: tx %v found while shouldn't have been", i, txn) } @@ -1101,7 +1339,7 @@ func TestChainTxReorgs(t *testing.T) { } } // added tx - for i, tx := range (types.Transactions{pastAdd, freshAdd, futureAdd}) { + for i, tx := range txsAdded { if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil { t.Errorf("add %d: expected tx to be found", i) } @@ -1110,7 +1348,7 @@ func TestChainTxReorgs(t *testing.T) { } } // shared tx - for i, tx := range (types.Transactions{postponed, swapped}) { + for i, tx := range txsShared { if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil { t.Errorf("share %d: expected tx to be found", i) } @@ -1118,6 +1356,53 @@ func TestChainTxReorgs(t *testing.T) { t.Errorf("share %d: expected receipt to be found", i) } } + + // ATXI checks + if !withATXI { + return + } + txsh1 := GetAddrTxs(db, addr1, 0, 0, "", "", -1, -1, false) + txsh2 := GetAddrTxs(db, addr2, 0, 0, "", "", -1, -1, false) + txsh3 := GetAddrTxs(db, addr3, 0, 0, "", "", -1, -1, false) + + allAtxis := txsh1 + allAtxis = append(allAtxis, txsh2...) + allAtxis = append(allAtxis, txsh3...) + + // Ensure a transaction exists for each atxi hash + for _, x := range allAtxis { + if tx, _, _, _ := GetTransaction(db, common.HexToHash(x)); tx == nil { + t.Error("atxi not removed") + } + } + + // Ensure no duplicate tx hashes returned +DUPECHECK: + for i, l := range [][]string{txsh1, txsh2, txsh3} { + j := strings.Join(l, "") + for _, h := range l { + if strings.Count(j, h[:8]) > 1 { + // show offending tx + offendingTxN := new(big.Int) + for _, x := range txsAll { + if x.Hash().Hex() == h { + offendingTxN.Set(x.Value()) // use unique value as a way to identify offender + break + } + } + t.Log(strings.Join(l, "\n")) + t.Errorf("[%d] duplicate tx hash (%v)", i, offendingTxN) + break DUPECHECK + } + } + + } + + // Check magnitude; 2 atxis per canonical tx (to & from) + wantMag := (len(txsAdded) + len(txsShared)) * 2 + if len(allAtxis) != wantMag { + t.Errorf("got: %v, want: %v", len(allAtxis), wantMag) + } } func TestLogReorgs(t *testing.T) { diff --git a/core/database_util.go b/core/database_util.go index cbc76b8a2..b72c45cfd 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -28,6 +28,8 @@ import ( "github.com/ethereumproject/go-ethereum/logger" "github.com/ethereumproject/go-ethereum/logger/glog" "github.com/ethereumproject/go-ethereum/rlp" + "sort" + "strings" ) var ( @@ -46,13 +48,16 @@ var ( receiptsPrefix = []byte("receipts-") blockReceiptsPrefix = []byte("receipts-block-") + txAddressIndexPrefix = []byte("atx-") + txAddressBookmarkKey = []byte("ATXIBookmark") + mipmapPre = []byte("mipmap-log-bloom-") MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000} blockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually] - preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage - lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata + preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage + lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata ) // TxLookupEntry is a positional metadata to help looking up the data content of @@ -63,6 +68,21 @@ type TxLookupEntry struct { Index uint64 } +func GetATXIBookmark(db ethdb.Database) uint64 { + v, err := db.Get(txAddressBookmarkKey) + if err != nil || v == nil { + return 0 + } + i := binary.LittleEndian.Uint64(v) + return i +} + +func SetATXIBookmark(db ethdb.Database, i uint64) error { + bn := make([]byte, 8) + binary.LittleEndian.PutUint64(bn, i) + return db.Put(txAddressBookmarkKey, bn) +} + // GetCanonicalHash retrieves a hash assigned to a canonical block number. func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash { data, _ := db.Get(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) @@ -134,6 +154,38 @@ func GetBodyRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { return data } +// formatAddrTxIterator formats the index key prefix iterator, eg. atx-
+func formatAddrTxIterator(address common.Address) (iteratorPrefix []byte) { + iteratorPrefix = append(iteratorPrefix, txAddressIndexPrefix...) + iteratorPrefix = append(iteratorPrefix, address.Bytes()...) + return +} + +// formatAddrTxBytesIndex formats the index key, eg. atx- +// The values for these arguments should be of determinate length and format, see test TestFormatAndResolveAddrTxBytesKey +// for example. +func formatAddrTxBytesIndex(address, blockNumber, direction, kindof, txhash []byte) (key []byte) { + key = make([]byte, 0, 66) // 66 is the total capacity of the key = prefix(4)+addr(20)+blockNumber(8)+dir(1)+kindof(1)+txhash(32) + key = append(key, txAddressIndexPrefix...) + key = append(key, address...) + key = append(key, blockNumber...) + key = append(key, direction...) + key = append(key, kindof...) + key = append(key, txhash...) + return +} + +// resolveAddrTxBytes resolves the index key to individual []byte values +func resolveAddrTxBytes(key []byte) (address, blockNumber, direction, kindof, txhash []byte) { + // prefix = key[:4] + address = key[4:24] // common.AddressLength = 20 + blockNumber = key[24:32] // uint64 via little endian + direction = key[32:33] // == key[32] (1 byte) + kindof = key[33:34] + txhash = key[34:] + return +} + // GetBody retrieves the block body (transactons, uncles) corresponding to the // hash, nil if none found. func GetBody(db ethdb.Database, hash common.Hash) *types.Body { @@ -149,6 +201,233 @@ func GetBody(db ethdb.Database, hash common.Hash) *types.Body { return body } +// WriteBlockAddTxIndexes writes atx-indexes for a given block. +func WriteBlockAddTxIndexes(indexDb ethdb.Database, block *types.Block) error { + batch := indexDb.NewBatch() + if _, err := putBlockAddrTxsToBatch(batch, block); err != nil { + return err + } + return batch.Write() +} + +// putBlockAddrTxsToBatch formats and puts keys for a given block to a db Batch. +// Batch can be written afterward if no errors, ie. batch.Write() +func putBlockAddrTxsToBatch(putBatch ethdb.Batch, block *types.Block) (txsCount int, err error) { + for _, tx := range block.Transactions() { + txsCount++ + + from, err := tx.From() + if err != nil { + return txsCount, err + } + to := tx.To() + // s: standard + // c: contract + txKindOf := []byte("s") + if to == nil || to.IsEmpty() { + to = &common.Address{} + txKindOf = []byte("c") + } + + // Note that len 8 because uint64 guaranteed <= 8 bytes. + bn := make([]byte, 8) + binary.LittleEndian.PutUint64(bn, block.NumberU64()) + + if err := putBatch.Put(formatAddrTxBytesIndex(from.Bytes(), bn, []byte("f"), txKindOf, tx.Hash().Bytes()), nil); err != nil { + return txsCount, err + } + if err := putBatch.Put(formatAddrTxBytesIndex(to.Bytes(), bn, []byte("t"), txKindOf, tx.Hash().Bytes()), nil); err != nil { + return txsCount, err + } + } + return txsCount, nil +} + +type atxi struct { + blockN uint64 + tx string +} +type sortableAtxis []atxi + +// Len implements sort.Sort interface. +func (s sortableAtxis) Len() int { + return len(s) +} + +// Less implements sort.Sort interface. +// By default newer transactions by blockNumber are first. +func (s sortableAtxis) Less(i, j int) bool { + return s[i].blockN > s[j].blockN +} + +// Swap implements sort.Sort interface. +func (s sortableAtxis) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} +func (s sortableAtxis) TxStrings() []string { + var out = make([]string, 0, len(s)) + for _, str := range s { + out = append(out, str.tx) + } + return out +} + +// GetAddrTxs gets the indexed transactions for a given account address. +// 'reverse' means "oldest first" +func GetAddrTxs(db ethdb.Database, address common.Address, blockStartN uint64, blockEndN uint64, direction string, kindof string, paginationStart int, paginationEnd int, reverse bool) []string { + if len(direction) > 0 && !strings.Contains("btf", direction[:1]) { + glog.Fatal("Address transactions list signature requires direction param to be empty string or [b|t|f] prefix (eg. both, to, or from)") + } + if len(kindof) > 0 && !strings.Contains("bsc", kindof[:1]) { + glog.Fatal("Address transactions list signature requires 'kind of' param to be empty string or [s|c] prefix (eg. both, standard, or contract)") + } + + // Have to cast to LevelDB to use iterator. Yuck. + ldb, ok := db.(*ethdb.LDBDatabase) + if !ok { + return nil + } + + // This will be the returnable. + var hashes []string + + // Map direction -> byte + var wantDirectionB byte = 'b' + if len(direction) > 0 { + wantDirectionB = direction[0] + } + var wantKindOf byte = 'b' + if len(kindof) > 0 { + wantKindOf = kindof[0] + } + + // Create address prefix for iteration. + prefix := ethdb.NewBytesPrefix(formatAddrTxIterator(address)) + it := ldb.NewIteratorRange(prefix) + + var atxis sortableAtxis + + for it.Next() { + key := it.Key() + + _, blockNum, torf, k, txh := resolveAddrTxBytes(key) + + bn := binary.LittleEndian.Uint64(blockNum) + + // If atxi is smaller than blockstart, skip + if blockStartN > 0 && bn < blockStartN { + continue + } + // If atxi is greater than blockend, skip + if blockEndN > 0 && bn > blockEndN { + continue + } + // Ensure matching direction if spec'd + if wantDirectionB != 'b' && wantDirectionB != torf[0] { + continue + } + // Ensure filter for/agnostic transaction kind of (contract, standard, both) + if wantKindOf != 'b' && wantKindOf != k[0] { + continue + } + if len(hashes) > 0 { + + } + tx := common.ToHex(txh) + atxis = append(atxis, atxi{blockN: bn, tx: tx}) + } + it.Release() + if e := it.Error(); e != nil { + glog.Fatalln(e) + } + + handleSorting := func(s sortableAtxis) sortableAtxis { + if len(s) <= 1 { + return s + } + sort.Sort(s) // newest txs (by blockNumber) latest + if reverse { + for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { + s[i], s[j] = s[j], s[i] + } + } + if paginationStart < 0 { + paginationStart = 0 + } + if paginationEnd < 0 { + paginationEnd = len(s) + } + return s[paginationStart:paginationEnd] + } + + return handleSorting(atxis).TxStrings() +} + +// RmAddrTx removes all atxi indexes for a given tx in case of a transaction removal, eg. +// in the case of chain reorg. +// It isn't an elegant function, but not a top priority for optimization because of +// expected infrequency of it's being called. +func RmAddrTx(db ethdb.Database, tx *types.Transaction) error { + if tx == nil { + return nil + } + + ldb, ok := db.(*ethdb.LDBDatabase) + if !ok { + return nil + } + + txH := tx.Hash() + from, err := tx.From() + if err != nil { + return err + } + + removals := [][]byte{} + + // TODO: not DRY, could be refactored + pre := ethdb.NewBytesPrefix(formatAddrTxIterator(from)) + it := ldb.NewIteratorRange(pre) + for it.Next() { + key := it.Key() + _, _, _, _, txh := resolveAddrTxBytes(key) + if bytes.Compare(txH.Bytes(), txh) == 0 { + removals = append(removals, key) + break // because there can be only one + } + } + it.Release() + if e := it.Error(); e != nil { + return e + } + + to := tx.To() + if to != nil { + toRef := *to + pre := ethdb.NewBytesPrefix(formatAddrTxIterator(toRef)) + it := ldb.NewIteratorRange(pre) + for it.Next() { + key := it.Key() + _, _, _, _, txh := resolveAddrTxBytes(key) + if bytes.Compare(txH.Bytes(), txh) == 0 { + removals = append(removals, key) + break // because there can be only one + } + } + it.Release() + if e := it.Error(); e != nil { + return e + } + } + + for _, r := range removals { + if err := db.Delete(r); err != nil { + return err + } + } + return nil +} + // GetTd retrieves a block's total difficulty corresponding to the hash, nil if // none found. func GetTd(db ethdb.Database, hash common.Hash) *big.Int { @@ -521,7 +800,6 @@ func WriteTxLookupEntries(db ethdb.Putter, block *types.Block) error { return nil } - // [deprecated by the header/block split, remove eventually] // GetBlockByHashOld returns the old combined block corresponding to the hash // or nil if not found. This method is only used by the upgrade mechanism to diff --git a/core/database_util_test.go b/core/database_util_test.go index a98a81173..b32386741 100644 --- a/core/database_util_test.go +++ b/core/database_util_test.go @@ -26,6 +26,8 @@ import ( "strconv" "testing" + "crypto/ecdsa" + "encoding/binary" "github.com/ethereumproject/go-ethereum/common" "github.com/ethereumproject/go-ethereum/core/types" "github.com/ethereumproject/go-ethereum/core/vm" @@ -33,6 +35,7 @@ import ( "github.com/ethereumproject/go-ethereum/crypto/sha3" "github.com/ethereumproject/go-ethereum/ethdb" "github.com/ethereumproject/go-ethereum/rlp" + "strings" ) type diffTest struct { @@ -298,6 +301,240 @@ func TestTdStorage(t *testing.T) { } } +func TestAddrTxStorage(t *testing.T) { + dbFilepath, err := ioutil.TempDir("", "geth-db-util-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dbFilepath) + db, _ := ethdb.NewLDBDatabase(dbFilepath, 10, 100) + + testKey := func(hex string) (*ecdsa.PrivateKey, common.Address) { + key := crypto.ToECDSA(common.Hex2Bytes(hex)) + addr := crypto.PubkeyToAddress(key.PublicKey) + return key, addr + } + + skey1, from1 := testKey("123915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8") + skey2, from2 := testKey("456915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8") + + from2to := common.BytesToAddress([]byte{0x22}) + + // from1 -> 1 + tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), big.NewInt(1111), big.NewInt(11111), []byte{0x11, 0x11, 0x11}) + + // from2 -> 2,3,txC + tx2 := types.NewTransaction(2, from2to, big.NewInt(222), big.NewInt(2222), big.NewInt(22222), []byte{0x22, 0x22, 0x22}) + tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), big.NewInt(3333), big.NewInt(33333), []byte{0x33, 0x33, 0x33}) + txC := types.NewTransaction(4, common.Address{}, big.NewInt(333), big.NewInt(3333), big.NewInt(33333), []byte{0x33, 0x33, 0x33}) + + txs := []*types.Transaction{tx1, tx2, tx3, txC} + txsSigned := []*types.Transaction{} + + for _, x := range txs { + // Sign em so we get from + key := skey1 + if x.Nonce() != 1 { + key = skey2 + } + x.SetSigner(types.NewChainIdSigner(big.NewInt(1))) + xs, err := x.SignECDSA(key) + if err != nil { + t.Fatal(err) + } + txsSigned = append(txsSigned, xs) + } + + block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txsSigned, nil, nil) + + // Put in the transactions just for fun. + // + // Check that no transactions entries are in a pristine database + for i, tx := range txsSigned { + if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil { + t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn) + } + } + // Insert all the transactions into the database, and verify contents + if err := WriteTransactions(db, block); err != nil { + t.Fatalf("failed to write transactions: %v", err) + } + for i, tx := range txsSigned { + if txn, hash, number, index := GetTransaction(db, tx.Hash()); txn == nil { + t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) + } else { + if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { + t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) + } + if tx.String() != txn.String() { + t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) + } + } + } + + // Write the atx indexes + if err := WriteBlockAddTxIndexes(db, block); err != nil { + t.Fatal(err) + } + + prefix := ethdb.NewBytesPrefix(txAddressIndexPrefix) + it := db.NewIteratorRange(prefix) + count := 0 + for it.Next() { + count++ + //// Debugger -- it's kinda nice to see what the indexes look like + //ad, bn, tf, sc, txh := resolveAddrTxBytes(it.Key()) + //addr, blockn, direc, ko, txhash := common.BytesToAddress(ad), binary.LittleEndian.Uint64(bn), string(tf), string(sc), common.BytesToHash(txh) + //t.Log(addr.Hex(), blockn, direc, ko, txhash.Hex()) + } + it.Release() + if e := it.Error(); e != nil { + t.Fatal(e) + } + if count != 8 { + t.Errorf("want: %v, got: %v", 7, count) + } + + out := GetAddrTxs(db, from2, 0, 0, "", "", -1, -1, false) + if len(out) != 3 { + t.Errorf("want: %v, got: %v", 3, len(out)) + } + + // Test pagination and reverse + outReverse := GetAddrTxs(db, from2, 0, 0, "", "", -1, -1, true) + if len(outReverse) != 3 { + t.Errorf("want: %v, got: %v", 3, len(outReverse)) + } + // reverse + if out[0] != outReverse[2] || out[1] != outReverse[1] || out[2] != outReverse[0] { + t.Errorf("got: %v, want: %v", outReverse, out) + } + // pagination + outPag := GetAddrTxs(db, from2, 0, 0, "", "", 1, -1, false) + if len(outPag) != 2 { + t.Errorf("got: %v, want: %v", len(outPag), 2) + } + + out = GetAddrTxs(db, from2, 0, 0, "", "c", -1, -1, false) + if len(out) != 1 { + t.Errorf("got: %v, want: %v", len(out), 1) + } + out = GetAddrTxs(db, common.Address{}, 0, 0, "", "", -1, -1, false) + if len(out) != 1 { + t.Errorf("got: %v, want: %v", len(out), 1) + } + + out = GetAddrTxs(db, from1, 314, 314, "", "", -1, -1, false) + if len(out) != 1 { + t.Errorf("want: %v, got: %v", 1, len(out)) + } else { + h := out[0] + if !strings.HasPrefix(h, "0x") { + t.Errorf("want: 0x-prefix, got: %v", h) + } + if !common.IsHex(h) { + t.Errorf("want: hex, got: %v", h) + } + txh := common.HexToHash(h) + + if txh != txsSigned[0].Hash() { + t.Errorf("got: %x, want: %x", txh, txsSigned[0].Hash()) + } + + gx, _, _, _ := GetTransaction(db, txh) + if gx == nil { + t.Errorf("missing tx: %x", txh) + } + } + + out = GetAddrTxs(db, from2to, 314, 314, "to", "", -1, -1, false) + if len(out) != 1 { + t.Errorf("want: %v, got: %v", 1, len(out)) + } else { + h := out[0] + if !strings.HasPrefix(h, "0x") { + t.Errorf("want: 0x-prefix, got: %v", h) + } + if !common.IsHex(h) { + t.Errorf("want: hex, got: %v", h) + } + txh := common.HexToHash(h) + if txh != txsSigned[1].Hash() { + t.Errorf("got: %x, want: %x", txh, txsSigned[0].Hash()) + } + gx, _, _, _ := GetTransaction(db, txh) + if gx == nil { + t.Errorf("missing tx: %x", txh) + } + f, e := gx.From() + if e != nil { + t.Error(e) + } + if f != from2 { + t.Errorf("got: %v, want: %v", f, from2) + } + } + out = GetAddrTxs(db, from2to, 314, 314, "from", "", -1, -1, false) + if len(out) != 0 { + t.Errorf("want: %v, got: %v", 0, len(out)) + } +} + +func TestFormatAndResolveAddrTxBytesKey(t *testing.T) { + testAddr := common.Address{} + testBN := uint64(42) + testTorf := "f" + testKindOf := "s" + testTxH := common.Hash{} + + testBNBytes := make([]byte, 8) + binary.LittleEndian.PutUint64(testBNBytes, testBN) + + key := formatAddrTxBytesIndex(testAddr.Bytes(), testBNBytes, []byte(testTorf), []byte(testKindOf), testTxH.Bytes()) + + // Test key/prefix iterator-ability. + itPrefix := formatAddrTxIterator(testAddr) + if !bytes.HasPrefix(key, itPrefix) { + t.Fatalf("key/prefix mismatch: prefix=%s key=%s", itPrefix, key) + } + + // Reverse engineer key and ensure expected. + outAddr, outBNBytes, outTorf, outKindOf, outTxH := resolveAddrTxBytes(key) + + if gotAddr := common.BytesToAddress(outAddr); gotAddr != testAddr { + t.Errorf("got: %v, want: %v", gotAddr.Hex(), testAddr.Hex()) + } + if gotBN := binary.LittleEndian.Uint64(outBNBytes); gotBN != testBN { + t.Errorf("got: %v, want: %v", gotBN, testBN) + } + if gotTorf := string(outTorf); gotTorf != testTorf { + t.Errorf("got: %v, want: %v", gotTorf, testTorf) + } + if gotKindOf := string(outKindOf); gotKindOf != testKindOf { + t.Errorf("got: %v, want: %v", gotKindOf, testKindOf) + } + if gotTxH := common.BytesToHash(outTxH); gotTxH != testTxH { + t.Errorf("got: %v, want: %v", gotTxH, testTxH) + } + + // Ensure proper key part sizing. + sizes := []struct { + b []byte + expectedLen int + }{ + {outAddr, common.AddressLength}, + {outBNBytes, 8}, + {outTorf, 1}, + {outKindOf, 1}, + {outTxH, common.HashLength}, + } + for _, s := range sizes { + if l := len(s.b); l != s.expectedLen { + t.Errorf("want: %v, got: %v", s.expectedLen, l) + } + } +} + // Tests that canonical numbers can be mapped to hashes and retrieved. func TestCanonicalMappingStorage(t *testing.T) { db, _ := ethdb.NewMemDatabase() diff --git a/core/state/database.go b/core/state/database.go index 1b10b2d93..30784e19d 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -152,4 +152,3 @@ func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) { } return root, err } - diff --git a/core/state/dump.go b/core/state/dump.go index ba19418d1..0f3214618 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -25,9 +25,9 @@ import ( "sort" "sync" + "fmt" "github.com/ethereumproject/go-ethereum/common" "github.com/ethereumproject/go-ethereum/rlp" - "fmt" "github.com/ethereumproject/go-ethereum/trie" ) diff --git a/core/state/iterator.go b/core/state/iterator.go index d2a17ba4a..91ff589f3 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -151,4 +151,3 @@ func (it *NodeIterator) retrieve() bool { } return true } - diff --git a/core/state/journal.go b/core/state/journal.go index dc15861b4..d164a6360 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -138,4 +138,3 @@ func (ch addLogChange) undo(s *StateDB) { func (ch addPreimageChange) undo(s *StateDB) { delete(s.preimages, ch.hash) } - diff --git a/core/state/state_object.go b/core/state/state_object.go index 13d8f0c3b..20b541460 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -24,10 +24,10 @@ import ( "github.com/ethereumproject/go-ethereum/common" "github.com/ethereumproject/go-ethereum/crypto" - "github.com/ethereumproject/go-ethereum/rlp" - "github.com/ethereumproject/go-ethereum/trie" "github.com/ethereumproject/go-ethereum/logger" "github.com/ethereumproject/go-ethereum/logger/glog" + "github.com/ethereumproject/go-ethereum/rlp" + "github.com/ethereumproject/go-ethereum/trie" ) var emptyCodeHash = crypto.Keccak256(nil) @@ -425,4 +425,4 @@ func (self *StateObject) ForEachStorage(cb func(key, value common.Hash) bool) { cb(key, common.BytesToHash(it.Value)) } } -} \ No newline at end of file +} diff --git a/core/state/state_test.go b/core/state/state_test.go index 6db8cdd15..cd0dcc317 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -231,4 +231,3 @@ func compareStateObjects(so0, so1 *StateObject, t *testing.T) { t.Fatalf("Deleted mismatch: have %v, want %v", so0.deleted, so1.deleted) } } - diff --git a/core/state/statedb.go b/core/state/statedb.go index ef8a34a1a..bc46ea010 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -24,12 +24,12 @@ import ( "sync" "github.com/ethereumproject/go-ethereum/common" + "github.com/ethereumproject/go-ethereum/core/vm" "github.com/ethereumproject/go-ethereum/crypto" "github.com/ethereumproject/go-ethereum/logger" "github.com/ethereumproject/go-ethereum/logger/glog" "github.com/ethereumproject/go-ethereum/rlp" "github.com/ethereumproject/go-ethereum/trie" - "github.com/ethereumproject/go-ethereum/core/vm" ) // The starting nonce determines the default nonce when new accounts are being @@ -62,9 +62,9 @@ type revision struct { // * Contracts // * Accounts type StateDB struct { - db Database - trie Trie - pastTries []*trie.SecureTrie + db Database + trie Trie + pastTries []*trie.SecureTrie // DB error. // State objects are used by the consensus core and VM which are @@ -616,4 +616,3 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common } } } - diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index c6bd6512c..4a8bee569 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -29,9 +29,9 @@ import ( "testing/quick" "github.com/ethereumproject/go-ethereum/common" + "github.com/ethereumproject/go-ethereum/core/vm" "github.com/ethereumproject/go-ethereum/ethdb" "gopkg.in/check.v1" - "github.com/ethereumproject/go-ethereum/core/vm" ) // Tests that updating a state trie does not leak any database writes prior to @@ -421,4 +421,4 @@ func (s *StateSuite) TestTouchDelete(c *check.C) { if len(s.state.stateObjectsDirty) != 0 { c.Fatal("expected no dirty state object") } -} \ No newline at end of file +} diff --git a/core/types/log.go b/core/types/log.go index 8d112fbe2..b90fd9b29 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -134,4 +134,3 @@ func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error { } return err } - diff --git a/core/types/transaction.go b/core/types/transaction.go index e660eb608..b7c96b587 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -274,8 +274,8 @@ func (s Transactions) GetRlp(i int) []byte { } // Returns a new set t which is the difference between a to b -func TxDifference(a, b Transactions) (keep Transactions) { - keep = make(Transactions, 0, len(a)) +func TxDifference(a, b Transactions) (diff Transactions) { + diff = make(Transactions, 0, len(a)) remove := make(map[common.Hash]struct{}) for _, tx := range b { @@ -284,11 +284,11 @@ func TxDifference(a, b Transactions) (keep Transactions) { for _, tx := range a { if _, ok := remove[tx.Hash()]; !ok { - keep = append(keep, tx) + diff = append(diff, tx) } } - return keep + return diff } // TxByNonce implements the sort interface to allow sorting a list of transactions diff --git a/eth/api.go b/eth/api.go index 8cbb8b59a..1cdcf5fa6 100644 --- a/eth/api.go +++ b/eth/api.go @@ -543,6 +543,7 @@ type PublicBlockChainAPI struct { config *core.ChainConfig bc *core.BlockChain chainDb ethdb.Database + indexesDb ethdb.Database eventMux *event.TypeMux muNewBlockSubscriptions sync.Mutex // protects newBlocksSubscriptions newBlockSubscriptions map[string]func(core.ChainEvent) error // callbacks for new block subscriptions @@ -1644,6 +1645,49 @@ func (api *PrivateAdminAPI) ImportChain(file string) (bool, error) { return true, nil } +// PublicDebugAPI is the collection of Etheruem APIs exposed over the public +// debugging endpoint. +type PublicGethAPI struct { + eth *Ethereum +} + +// NewPublicDebugAPI creates a new API definition for the public debug methods +// of the Ethereum service. +func NewPublicGethAPI(eth *Ethereum) *PublicGethAPI { + return &PublicGethAPI{eth: eth} +} + +// AddressTransactions gets transactions for a given address. +// Optional values include start and stop block numbers, and to/from/both value for tx/address relation. +// Returns a slice of strings of transactions hashes. +func (api *PublicGethAPI) GetAddressTransactions(address common.Address, blockStartN uint64, blockEndN uint64, toOrFrom string, txKindOf string, pagStart, pagEnd int, reverse bool) (list []string, err error) { + glog.V(logger.Debug).Infoln("RPC call: debug_getAddressTransactions %s %d %d %s %s", address, blockStartN, blockEndN, toOrFrom, txKindOf) + + db, inUse := api.eth.BlockChain().GetAddTxIndex() + if !inUse { + return nil, errors.New("addr-tx indexing not enabled") + } + // Use human-friendly abbreviations, per https://github.com/ethereumproject/go-ethereum/pull/475#issuecomment-366065122 + // so 't' => to, 'f' => from, 'tf|ft' => either/both. Same pattern for txKindOf. + // _t_o OR _f_rom + if toOrFrom == "tf" || toOrFrom == "ft" { + toOrFrom = "b" + } + // _s_tandard OR _c_ontract + if txKindOf == "sc" || txKindOf == "cs" { + txKindOf = "b" + } + + list = core.GetAddrTxs(db, address, blockStartN, blockEndN, toOrFrom, txKindOf, pagStart, pagEnd, reverse) + + // Since list is a slice, it can be nil, which returns 'null'. + // Should return empty 'array' if no txs found. + if list == nil { + list = []string{} + } + return list, nil +} + // PublicDebugAPI is the collection of Etheruem APIs exposed over the public // debugging endpoint. type PublicDebugAPI struct { diff --git a/eth/backend.go b/eth/backend.go index d098f87eb..a5904cead 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -81,6 +81,8 @@ type Config struct { MinerThreads int SolcPath string + UseAddrTxIndex bool + GpoMinGasPrice *big.Int GpoMaxGasPrice *big.Int GpoFullBlockRatio int @@ -98,8 +100,9 @@ type Ethereum struct { shutdownChan chan bool // DB interfaces - chainDb ethdb.Database // Block chain database - dappDb ethdb.Database // Dapp database + chainDb ethdb.Database // Block chain database + dappDb ethdb.Database // Dapp database + indexesDb ethdb.Database // Indexes database (optional -- eg. add-tx indexes) // Handlers txPool *core.TxPool @@ -226,6 +229,24 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.pow = ethash.New() } + // Initialize indexes db if enabled + // Blockchain will be assigned the db and atx enabled after blockchain is initialized below. + var indexesDb ethdb.Database + if config.UseAddrTxIndex { + // TODO: these are arbitrary numbers I just made up. Optimize? + // The reason these numbers are different than the atxi-build command is because for "appending" (vs. building) + // the atxi database should require far fewer resources since application performance is limited primarily by block import (chaindata db). + ethdb.SetCacheRatio("chaindata", 0.95) + ethdb.SetHandleRatio("chaindata", 0.95) + ethdb.SetCacheRatio("indexes", 0.05) + ethdb.SetHandleRatio("indexes", 0.05) + indexesDb, err = ctx.OpenDatabase("indexes", config.DatabaseCache, config.DatabaseCache) + if err != nil { + return nil, err + } + eth.indexesDb = indexesDb + } + // load the genesis block or write a new one if no genesis // block is present in the database. genesis := core.GetBlock(chainDb, core.GetCanonicalHash(chainDb, 0)) @@ -263,6 +284,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { } return nil, err } + // Configure enabled atxi for blockchain + if config.UseAddrTxIndex { + eth.blockchain.SetAddTxIndex(eth.indexesDb, true) + } + eth.gpo = NewGasPriceOracle(eth) newPool := core.NewTxPool(eth.chainConfig, eth.EventMux(), eth.blockchain.State, eth.blockchain.GasLimit) @@ -351,6 +377,11 @@ func (s *Ethereum) APIs() []rpc.API { Namespace: "admin", Version: "1.0", Service: ethreg.NewPrivateRegistarAPI(s.chainConfig, s.blockchain, s.chainDb, s.txPool, s.accountManager), + }, { + Namespace: "geth", + Version: "1.0", + Service: NewPublicGethAPI(s), + Public: true, }, } } @@ -361,7 +392,7 @@ func (s *Ethereum) ResetWithGenesisBlock(gb *types.Block) { func (s *Ethereum) Etherbase() (eb common.Address, err error) { eb = s.etherbase - if (eb == common.Address{}) { + if eb.IsEmpty() { firstAccount, err := s.AccountManager().AccountByIndex(0) eb = firstAccount.Address if err != nil { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 996f12a79..f61dfa5d9 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -32,9 +32,9 @@ import ( "github.com/ethereumproject/go-ethereum/core/types" "github.com/ethereumproject/go-ethereum/ethdb" "github.com/ethereumproject/go-ethereum/event" - "github.com/ethereumproject/go-ethereum/metrics" "github.com/ethereumproject/go-ethereum/logger" "github.com/ethereumproject/go-ethereum/logger/glog" + "github.com/ethereumproject/go-ethereum/metrics" ) const ( @@ -65,12 +65,12 @@ var ( maxHeadersProcess = 2048 // Number of header download results to import at once into the chain maxResultsProcess = 2048 // Number of content download results to import at once into the chain - fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync - fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected - fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it - fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point - fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync - fsCriticalTrials uint32 = 10 // Number of times to retry in the cricical section before bailing + fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync + fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected + fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it + fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point + fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync + fsCriticalTrials uint32 = 10 // Number of times to retry in the cricical section before bailing ) var ( @@ -129,7 +129,7 @@ type Downloader struct { stateDB ethdb.Database fsPivotLock *types.Header // Pivot header on critical section entry (cannot change between retries) - fsPivotFails uint32 // Number of fast sync failures in the critical section + fsPivotFails uint32 // Number of fast sync failures in the critical section rttEstimate uint64 // Round trip time to target for download requests rttConfidence uint64 // Confidence in the estimated RTT (unit: millionths to allow atomic ops) diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 70b165578..298756bd3 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -345,8 +345,8 @@ func (p *peer) String() string { // peerSet represents the collection of active peer participating in the chain // download procedure. type peerSet struct { - peers map[string]*peer - lock sync.RWMutex + peers map[string]*peer + lock sync.RWMutex } // newPeerSet creates a new peer set top track the active download sources. diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index ffcc26ca1..da963440d 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -27,10 +27,10 @@ import ( "github.com/ethereumproject/go-ethereum/common" "github.com/ethereumproject/go-ethereum/core/types" + "github.com/ethereumproject/go-ethereum/logger" "github.com/ethereumproject/go-ethereum/logger/glog" "github.com/ethereumproject/go-ethereum/metrics" "gopkg.in/karalabe/cookiejar.v2/collections/prque" - "github.com/ethereumproject/go-ethereum/logger" ) var blockCacheLimit = 8192 // Maximum number of blocks to cache before throttling the download diff --git a/ethdb/database.go b/ethdb/database.go index 6b66a07fc..5857bce14 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -28,12 +28,13 @@ import ( "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" + ldbutil "github.com/syndtr/goleveldb/leveldb/util" "sync" ) var OpenFileLimit = 64 -// cacheRatio specifies how the total alloted cache is distributed between the +// cacheRatio specifies how the total allotted cache is distributed between the // various system databases. var cacheRatio = map[string]float64{ "dapp": 0.0, @@ -47,6 +48,14 @@ var handleRatio = map[string]float64{ "chaindata": 1.0, } +func SetCacheRatio(db string, ratio float64) { + cacheRatio[db] = ratio +} + +func SetHandleRatio(db string, ratio float64) { + handleRatio[db] = ratio +} + type LDBDatabase struct { file string db *leveldb.DB @@ -89,7 +98,6 @@ func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { }, nil } - // Path returns the path to the database directory. func (db *LDBDatabase) Path() string { return db.file @@ -124,6 +132,14 @@ func (self *LDBDatabase) NewIterator() iterator.Iterator { return self.db.NewIterator(nil, nil) } +func (self *LDBDatabase) NewIteratorRange(slice *ldbutil.Range) iterator.Iterator { + return self.db.NewIterator(slice, nil) +} + +func NewBytesPrefix(prefix []byte) *ldbutil.Range { + return ldbutil.BytesPrefix(prefix) +} + func (self *LDBDatabase) Close() { if err := self.db.Close(); err != nil { glog.Errorf("eth: DB %s: %s", self.file, err) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 3e5b62361..ce9ea34ed 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -27,6 +27,7 @@ var Modules = map[string]string{ "rpc": RPC_JS, "shh": Shh_JS, "txpool": TxPool_JS, + "geth": Geth_JS, } const Admin_JS = ` @@ -149,6 +150,22 @@ web3._extend({ }); ` +const Geth_JS = ` +web3._extend({ + property: 'geth', + methods: + [ + new web3._extend.Method({ + name: 'getAddressTransactions', + call: 'geth_getAddressTransactions', + params: 8, + inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, null, null, null, null, null, null] + }) + ], + properties: [] +}); +` + const Debug_JS = ` web3._extend({ property: 'debug', @@ -206,6 +223,12 @@ web3._extend({ name: 'accountExist', call: 'debug_accountExist', params: 2 + }), + new web3._extend.Method({ + name: 'getAddressTransactions', + call: 'debug_getAddressTransactions', + params: 8, + inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, null, null, null, null, null, null] }) ], properties: [] diff --git a/rpc/server.go b/rpc/server.go index 7bece588d..9f4af3b9a 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -36,7 +36,7 @@ const ( notificationBufferSize = 10000 // max buffered notifications before codec is closed MetadataApi = "rpc" - DefaultIPCApis = "admin,debug,eth,miner,net,personal,shh,txpool,web3" + DefaultIPCApis = "admin,debug,eth,miner,net,personal,shh,txpool,web3,geth" DefaultHTTPApis = "eth,net,web3" ) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 9b7986279..e0a435917 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -243,7 +243,7 @@ func RunState(ruleSet RuleSet, db ethdb.Database, statedb *state.StateDB, env, t if core.IsNonceErr(err) || core.IsInvalidTxErr(err) || core.IsGasLimitErr(err) { statedb.RevertToSnapshot(snapshot) } - statedb.CommitTo(db,false) + statedb.CommitTo(db, false) return ret, vmenv.state.Logs(), vmenv.Gas, err } diff --git a/trie/encoding.go b/trie/encoding.go index 6db8dd00d..e96a786e4 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -15,6 +15,7 @@ // along with the go-ethereum library. If not, see . package trie + // Trie keys are dealt with in three distinct encodings: // // KEYBYTES encoding contains the actual key and nothing else. This encoding is the @@ -110,4 +111,4 @@ func prefixLen(a, b []byte) int { // hasTerm returns whether a hex key has the terminator flag. func hasTerm(s []byte) bool { return len(s) > 0 && s[len(s)-1] == 16 -} \ No newline at end of file +} diff --git a/trie/encoding_test.go b/trie/encoding_test.go index 75e886272..97d8da136 100644 --- a/trie/encoding_test.go +++ b/trie/encoding_test.go @@ -17,8 +17,8 @@ package trie import ( - "testing" "bytes" + "testing" ) func TestHexCompact(t *testing.T) { diff --git a/trie/errors.go b/trie/errors.go index 65c6d46c3..5d16d8b30 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -52,4 +52,3 @@ type MissingNodeError struct { func (err *MissingNodeError) Error() string { return fmt.Sprintf("missing trie node %x (path %x)", err.NodeHash, err.Path) } - diff --git a/trie/hasher.go b/trie/hasher.go index 1b9fc813c..cd2d7b5d6 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -227,4 +227,4 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) { return hash, err } return hash, nil -} \ No newline at end of file +} diff --git a/trie/iterator.go b/trie/iterator.go index 31a01b5bc..af30efb11 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -24,7 +24,6 @@ import ( "github.com/ethereumproject/go-ethereum/common" ) - // Iterator is a key-value trie iterator that traverses a Trie. type Iterator struct { nodeIt NodeIterator diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 5ecb2feef..ab811d35e 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -19,10 +19,10 @@ package trie import ( "testing" + "bytes" + "fmt" "github.com/ethereumproject/go-ethereum/common" "github.com/ethereumproject/go-ethereum/ethdb" - "fmt" - "bytes" "math/rand" ) diff --git a/trie/node.go b/trie/node.go index adfc3be6e..2b559c36d 100644 --- a/trie/node.go +++ b/trie/node.go @@ -201,7 +201,6 @@ func decodeRef(buf []byte, cachegen uint16) (node, []byte, error) { } } - // wraps a decoding error with information about the path to the // invalid child node (for debugging encoding issues). type decodeError struct { diff --git a/trie/proof.go b/trie/proof.go index 2fc0be62b..eea29e11c 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -20,11 +20,12 @@ import ( "bytes" "fmt" - "github.com/ethereumproject/go-ethereum/rlp" - "github.com/ethereumproject/go-ethereum/crypto" "github.com/ethereumproject/go-ethereum/common" + "github.com/ethereumproject/go-ethereum/crypto" "github.com/ethereumproject/go-ethereum/logger/glog" + "github.com/ethereumproject/go-ethereum/rlp" ) + // Prove constructs a merkle proof for key. The result contains all // encoded nodes on the path to the value at key. The value itself is // also included in the last node and can be retrieved by verifying diff --git a/trie/proof_test.go b/trie/proof_test.go index f83581f95..882156676 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -24,8 +24,8 @@ import ( "time" "github.com/ethereumproject/go-ethereum/common" - "github.com/ethereumproject/go-ethereum/ethdb" "github.com/ethereumproject/go-ethereum/crypto" + "github.com/ethereumproject/go-ethereum/ethdb" ) func init() { @@ -161,4 +161,4 @@ func randBytes(n int) []byte { r := make([]byte, n) crand.Read(r) return r -} \ No newline at end of file +} diff --git a/trie/secure_trie.go b/trie/secure_trie.go index a8c438145..813883f3c 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -17,8 +17,8 @@ package trie import ( - "github.com/ethereumproject/go-ethereum/common" "fmt" + "github.com/ethereumproject/go-ethereum/common" "github.com/ethereumproject/go-ethereum/logger/glog" ) @@ -214,4 +214,4 @@ func (t *SecureTrie) getSecKeyCache() map[string][]byte { t.secKeyCache = make(map[string][]byte) } return t.secKeyCache -} \ No newline at end of file +} diff --git a/trie/sync.go b/trie/sync.go index cee208e63..b89906c78 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -38,7 +38,7 @@ type request struct { hash common.Hash // Hash of the node data content to retrieve data []byte // Data content of the node, cached until all subtrees complete object *node // Target node to populate with retrieved data (hashnode originally) - raw bool // Whether this is a raw entry (code) or a trie node + raw bool // Whether this is a raw entry (code) or a trie node parents []*request // Parent state nodes referencing this entry (notify all upon completion) depth int // Depth level within the trie the node is located to prioritise DFS diff --git a/trie/trie.go b/trie/trie.go index 763648660..9ac48cccf 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -23,8 +23,8 @@ import ( "github.com/ethereumproject/go-ethereum/common" "github.com/ethereumproject/go-ethereum/crypto/sha3" - "github.com/rcrowley/go-metrics" "github.com/ethereumproject/go-ethereum/logger/glog" + "github.com/rcrowley/go-metrics" ) var ( diff --git a/trie/trie_test.go b/trie/trie_test.go index 3253dfa82..e175bb71d 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -18,21 +18,21 @@ package trie import ( "bytes" - "testing" + "encoding/binary" + "errors" "fmt" "github.com/davecgh/go-spew/spew" "github.com/ethereumproject/go-ethereum/common" + "github.com/ethereumproject/go-ethereum/crypto" "github.com/ethereumproject/go-ethereum/ethdb" - "encoding/binary" - "reflect" - "testing/quick" - "os" + "github.com/ethereumproject/go-ethereum/rlp" + "io/ioutil" "math/big" "math/rand" - "errors" - "io/ioutil" - "github.com/ethereumproject/go-ethereum/crypto" - "github.com/ethereumproject/go-ethereum/rlp" + "os" + "reflect" + "testing" + "testing/quick" ) func init() { @@ -606,4 +606,4 @@ func updateString(trie *Trie, k, v string) { func deleteString(trie *Trie, k string) { trie.Delete([]byte(k)) -} \ No newline at end of file +}