diff --git a/accounts/cache.go b/accounts/cache.go
index 3b532a495..7cba83596 100644
--- a/accounts/cache.go
+++ b/accounts/cache.go
@@ -145,7 +145,7 @@ func (ac *addrCache) find(a Account) (Account, error) {
return matches[i], nil
}
}
- if (a.Address == common.Address{}) {
+ if a.Address.IsEmpty() {
return Account{}, ErrNoMatch
}
}
@@ -239,7 +239,7 @@ func (ac *addrCache) scan() ([]Account, error) {
switch {
case err != nil:
glog.V(logger.Debug).Infof("can't decode key %s: %v", path, err)
- case (keyJSON.Address == common.Address{}):
+ case keyJSON.Address.IsEmpty():
glog.V(logger.Debug).Infof("can't decode key %s: missing or zero address", path)
default:
addrs = append(addrs, Account{Address: keyJSON.Address, File: path})
diff --git a/accounts/cachedb.go b/accounts/cachedb.go
index 0a4b978be..35910bf67 100644
--- a/accounts/cachedb.go
+++ b/accounts/cachedb.go
@@ -236,7 +236,7 @@ func (cdb *cacheDB) find(a Account) (Account, error) {
return acc, e
}
// no other possible way
- if (a.Address == common.Address{}) {
+ if a.Address.IsEmpty() {
return Account{}, ErrNoMatch
}
}
@@ -498,7 +498,7 @@ func processKeyFile(wg *sync.WaitGroup, path string, fi os.FileInfo, i int, numF
case err != nil:
glog.V(logger.Debug).Infof("(%v/%v) can't decode key %s: %v", i, numFiles, path, err)
errs <- err
- case (keyJSON.Address == common.Address{}):
+ case keyJSON.Address.IsEmpty():
glog.V(logger.Debug).Infof("(%v/%v) can't decode key %s: missing or zero address", i, numFiles, path)
errs <- fmt.Errorf("(%v/%v) can't decode key %s: missing or zero address", i, numFiles, path)
default:
diff --git a/cmd/geth/build_atxi_cmd.go b/cmd/geth/build_atxi_cmd.go
new file mode 100644
index 000000000..b9314df58
--- /dev/null
+++ b/cmd/geth/build_atxi_cmd.go
@@ -0,0 +1,146 @@
+package main
+
+import (
+ "github.com/ethereumproject/go-ethereum/core"
+ "github.com/ethereumproject/go-ethereum/core/types"
+ "github.com/ethereumproject/go-ethereum/ethdb"
+ "github.com/ethereumproject/go-ethereum/logger"
+ "github.com/ethereumproject/go-ethereum/logger/glog"
+ "gopkg.in/urfave/cli.v1"
+ "os"
+ "time"
+)
+
+var buildAddrTxIndexCommand = cli.Command{
+ Action: buildAddrTxIndexCmd,
+ Name: "atxi-build",
+ Usage: "Generate index for transactions by address",
+ Description: `
+ Builds an index for transactions by address.
+ The command is idempotent; it will not hurt to run multiple times on the same range.
+ If run without --start flag, the command makes use of a persistent placeholder, so you can
+ run the command on multiple occasions and pick up indexing progress where the last session
+ left off.
+ To enable address-transaction indexing during block sync and import, use the '--atxi' flag.
+ `,
+ Flags: []cli.Flag{
+ cli.IntFlag{
+ Name: "start",
+ Usage: "Block number at which to begin building index",
+ },
+ cli.IntFlag{
+ Name: "stop",
+ Usage: "Block number at which to stop building index",
+ },
+ cli.IntFlag{
+ Name: "step",
+ Usage: "Step increment for batching. Higher number requires more mem, but may be faster",
+ Value: 10000,
+ },
+ },
+}
+
+func buildAddrTxIndexCmd(ctx *cli.Context) error {
+
+ // Divide global cache availability equally between chaindata (pre-existing blockdata) and
+ // address-transaction database. This ratio is arbitrary and could potentially be optimized or delegated to be user configurable.
+ ethdb.SetCacheRatio("chaindata", 0.5)
+ ethdb.SetHandleRatio("chaindata", 1)
+ ethdb.SetCacheRatio("indexes", 0.5)
+ ethdb.SetHandleRatio("indexes", 1)
+
+ startIndex := uint64(ctx.Int("start"))
+ var stopIndex uint64
+
+ indexDb := MakeIndexDatabase(ctx)
+ if indexDb == nil {
+ glog.Fatalln("indexes db is nil")
+ }
+ defer indexDb.Close()
+
+ // Use persistent placeholder in case start not spec'd
+ if !ctx.IsSet("start") {
+ startIndex = core.GetATXIBookmark(indexDb)
+ }
+
+ bc, chainDB := MakeChain(ctx)
+ if bc == nil || chainDB == nil {
+ glog.Fatalln("bc or cdb is nil")
+ }
+ defer chainDB.Close()
+
+ stopIndex = uint64(ctx.Int("stop"))
+ if stopIndex == 0 {
+ stopIndex = bc.CurrentBlock().NumberU64()
+ if n := bc.CurrentFastBlock().NumberU64(); n > stopIndex {
+ stopIndex = n
+ }
+ }
+
+ if stopIndex < startIndex {
+ glog.Fatalln("start must be prior to (smaller than) or equal to stop, got start=", startIndex, "stop=", stopIndex)
+ }
+ if startIndex == stopIndex {
+ glog.D(logger.Error).Infoln("atxi is up to date, exiting")
+ os.Exit(0)
+ }
+
+ var block *types.Block
+ blockIndex := startIndex
+ block = bc.GetBlockByNumber(blockIndex)
+ if block == nil {
+ glog.Fatalln(blockIndex, "block is nil")
+ }
+
+ var inc = uint64(ctx.Int("step"))
+ startTime := time.Now()
+ totalTxCount := uint64(0)
+ glog.D(logger.Error).Infoln("Address/tx indexing (atxi) start:", startIndex, "stop:", stopIndex, "step:", inc, "| This may take a while.")
+ breaker := false
+ for i := startIndex; i <= stopIndex; i = i + inc {
+ if i+inc > stopIndex {
+ inc = stopIndex - i
+ breaker = true
+ }
+
+ stepStartTime := time.Now()
+
+ // It may seem weird to pass i, i+inc, and inc, but its just a "coincidence"
+ // The function could accepts a smaller step for batch putting (in this case, inc),
+ // or a larger stopBlock (i+inc), but this is just how this cmd is using the fn now
+ // We could mess around a little with exploring batch optimization...
+ txsCount, err := bc.WriteBlockAddrTxIndexesBatch(indexDb, i, i+inc, inc)
+ if err != nil {
+ return err
+ }
+ totalTxCount += uint64(txsCount)
+
+ if err := core.SetATXIBookmark(indexDb, i+inc); err != nil {
+ glog.Fatalln(err)
+ }
+
+ glog.D(logger.Error).Infof("atxi-build: block %d / %d txs: %d took: %v %.2f bps %.2f txps", i+inc, stopIndex, txsCount, time.Since(stepStartTime).Round(time.Millisecond), float64(inc)/time.Since(stepStartTime).Seconds(), float64(txsCount)/time.Since(stepStartTime).Seconds())
+ glog.V(logger.Info).Infof("atxi-build: block %d / %d txs: %d took: %v %.2f bps %.2f txps", i+inc, stopIndex, txsCount, time.Since(stepStartTime).Round(time.Millisecond), float64(inc)/time.Since(stepStartTime).Seconds(), float64(txsCount)/time.Since(stepStartTime).Seconds())
+
+ if breaker {
+ break
+ }
+ }
+
+ if err := core.SetATXIBookmark(indexDb, stopIndex); err != nil {
+ glog.Fatalln(err)
+ }
+
+ // Print summary
+ totalBlocksF := float64(stopIndex - startIndex)
+ totalTxsF := float64(totalTxCount)
+ took := time.Since(startTime)
+ glog.D(logger.Error).Infof(`Finished atxi-build in %v: %d blocks (~ %.2f blocks/sec), %d txs (~ %.2f txs/sec)`,
+ took.Round(time.Second),
+ stopIndex-startIndex,
+ totalBlocksF/took.Seconds(),
+ totalTxCount,
+ totalTxsF/took.Seconds(),
+ )
+ return nil
+}
diff --git a/cmd/geth/flag.go b/cmd/geth/flag.go
index 84e2d8ca9..b00928b20 100644
--- a/cmd/geth/flag.go
+++ b/cmd/geth/flag.go
@@ -601,6 +601,7 @@ func mustMakeEthConf(ctx *cli.Context, sconf *core.SufficientChainConfig) *eth.C
ethConf := ð.Config{
ChainConfig: sconf.ChainConfig,
Genesis: sconf.Genesis,
+ UseAddrTxIndex: ctx.GlobalBool(aliasableName(AddrTxIndexFlag.Name, ctx)),
FastSync: ctx.GlobalBool(aliasableName(FastSyncFlag.Name, ctx)),
BlockChainVersion: ctx.GlobalInt(aliasableName(BlockchainVersionFlag.Name, ctx)),
DatabaseCache: ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx)),
@@ -799,18 +800,32 @@ func MustMakeChainConfigFromDefaults(ctx *cli.Context) *core.ChainConfig {
// MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails.
func MakeChainDatabase(ctx *cli.Context) ethdb.Database {
var (
- datadir = MustMakeChainDataDir(ctx)
- cache = ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx))
- handles = MakeDatabaseHandles()
+ chaindir = MustMakeChainDataDir(ctx)
+ cache = ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx))
+ handles = MakeDatabaseHandles()
)
- chainDb, err := ethdb.NewLDBDatabase(filepath.Join(datadir, "chaindata"), cache, handles)
+ chainDb, err := ethdb.NewLDBDatabase(filepath.Join(chaindir, "chaindata"), cache, handles)
if err != nil {
glog.Fatal("Could not open database: ", err)
}
return chainDb
}
+func MakeIndexDatabase(ctx *cli.Context) ethdb.Database {
+ var (
+ chaindir = MustMakeChainDataDir(ctx)
+ cache = ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx))
+ handles = MakeDatabaseHandles()
+ )
+
+ indexesDb, err := ethdb.NewLDBDatabase(filepath.Join(chaindir, "indexes"), cache, handles)
+ if err != nil {
+ glog.Fatal("Could not open database: ", err)
+ }
+ return indexesDb
+}
+
// MakeChain creates a chain manager from set command line flags.
func MakeChain(ctx *cli.Context) (chain *core.BlockChain, chainDb ethdb.Database) {
var err error
diff --git a/cmd/geth/flags.go b/cmd/geth/flags.go
index 1addc6b1b..88559defa 100644
--- a/cmd/geth/flags.go
+++ b/cmd/geth/flags.go
@@ -98,6 +98,10 @@ var (
Name: "light-kdf,lightkdf",
Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
}
+ AddrTxIndexFlag = cli.BoolFlag{
+ Name: "atxi,add-tx-index",
+ Usage: "Toggle indexes for transactions by address. Pre-existing chaindata can be indexed with command 'atxi-build'",
+ }
// Network Split settings
ETFChain = cli.BoolFlag{
Name: "etf",
diff --git a/cmd/geth/main.go b/cmd/geth/main.go
index b7a5a3027..69866b0bd 100644
--- a/cmd/geth/main.go
+++ b/cmd/geth/main.go
@@ -52,7 +52,7 @@ var makeDagCommand = cli.Command{
Usage: "Generate ethash dag (for testing)",
Description: `
The makedag command generates an ethash DAG in /tmp/dag.
-
+
This command exists to support the system testing project.
Regular users do not need to execute it.
`,
@@ -136,6 +136,7 @@ func makeCLIApp() (app *cli.App) {
gpuBenchCommand,
versionCommand,
makeMlogDocCommand,
+ buildAddrTxIndexCommand,
}
app.Flags = []cli.Flag{
@@ -153,6 +154,7 @@ func makeCLIApp() (app *cli.App) {
ChainIdentityFlag,
BlockchainVersionFlag,
FastSyncFlag,
+ AddrTxIndexFlag,
CacheFlag,
LightKDFFlag,
JSpathFlag,
diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go
index fc6c915b8..902c170c4 100644
--- a/cmd/geth/usage.go
+++ b/cmd/geth/usage.go
@@ -25,11 +25,6 @@ import (
)
// AppHelpTemplate is the test template for the default, global app help topic.
-//var x = cli.Command{}
-//x.Usage
-//x.UsageText
-//x.ArgsUsage
-//x.Subcommands
var AppHelpTemplate = `NAME:
{{.App.Name}} - {{.App.Usage}}
@@ -40,7 +35,7 @@ VERSION:
{{.App.Version}}{{if .CommandAndFlagGroups}}
COMMANDS AND FLAGS:
-------------------------------------------------------------------------
+
{{range .CommandAndFlagGroups}}{{.Name}}
------------------------------------------------------------------------
{{if .Commands}}{{range .Commands}}
@@ -93,12 +88,14 @@ var AppHelpFlagAndCommandGroups = []flagGroup{
Commands: []cli.Command{
accountCommand,
walletCommand,
+ buildAddrTxIndexCommand,
},
Flags: []cli.Flag{
KeyStoreDirFlag,
UnlockedAccountFlag,
PasswordFileFlag,
AccountsIndexFlag,
+ AddrTxIndexFlag,
},
},
{
@@ -205,6 +202,10 @@ var AppHelpFlagAndCommandGroups = []flagGroup{
Flags: []cli.Flag{
WhisperEnabledFlag,
NatspecEnabledFlag,
+ DisplayFlag,
+ DisplayFormatFlag,
+ NeckbeardFlag,
+ AddrTxIndexFlag,
},
},
{
diff --git a/common/registrar/registrar.go b/common/registrar/registrar.go
index 84b5ec7be..51c89d84a 100644
--- a/common/registrar/registrar.go
+++ b/common/registrar/registrar.go
@@ -112,7 +112,7 @@ func (self *Registrar) SetGlobalRegistrar(namereg string, addr common.Address) (
return
}
if zero.MatchString(GlobalRegistrarAddr) {
- if (addr == common.Address{}) {
+ if addr.IsEmpty() {
err = fmt.Errorf("GlobalRegistrar address not found and sender for creation not given")
return
} else {
@@ -142,7 +142,7 @@ func (self *Registrar) SetHashReg(hashreg string, addr common.Address) (txhash s
HashRegAddr = "0x" + res[len(res)-40:]
}
if err != nil || zero.MatchString(HashRegAddr) {
- if (addr == common.Address{}) {
+ if addr.IsEmpty() {
err = fmt.Errorf("HashReg address not found and sender for creation not given")
return
}
@@ -177,7 +177,7 @@ func (self *Registrar) SetUrlHint(urlhint string, addr common.Address) (txhash s
UrlHintAddr = "0x" + res[len(res)-40:]
}
if err != nil || zero.MatchString(UrlHintAddr) {
- if (addr == common.Address{}) {
+ if addr.IsEmpty() {
err = fmt.Errorf("UrlHint address not found and sender for creation not given")
return
}
diff --git a/common/types.go b/common/types.go
index 768146736..b238d89c5 100644
--- a/common/types.go
+++ b/common/types.go
@@ -125,6 +125,14 @@ func StringToAddress(s string) Address { return BytesToAddress([]byte(s)) }
func BigToAddress(b *big.Int) Address { return BytesToAddress(b.Bytes()) }
func HexToAddress(s string) Address { return BytesToAddress(FromHex(s)) }
+func EmptyAddress(a Address) bool {
+ return a == Address{}
+}
+
+func (a Address) IsEmpty() bool {
+ return EmptyAddress(a)
+}
+
// IsHexAddress verifies whether a string can represent a valid hex-encoded
// Ethereum address or not.
func IsHexAddress(s string) bool {
diff --git a/core/blockchain.go b/core/blockchain.go
index 4f485cf37..e0f6e4dfc 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -32,6 +32,7 @@ import (
"reflect"
"strconv"
+ "encoding/binary"
"github.com/ethereumproject/go-ethereum/common"
"github.com/ethereumproject/go-ethereum/core/state"
"github.com/ethereumproject/go-ethereum/core/types"
@@ -109,6 +110,9 @@ type BlockChain struct {
pow pow.PoW
processor Processor // block processor interface
validator Validator // block and state validator interface
+
+ useAddTxIndex bool
+ indexesDb ethdb.Database
}
// NewBlockChain returns a fully initialised block chain using information
@@ -214,6 +218,17 @@ func (self *BlockChain) GetEventMux() *event.TypeMux {
return self.eventMux
}
+// SetAddTxIndex sets the db and in-use var for atx indexing.
+func (self *BlockChain) SetAddTxIndex(db ethdb.Database, tf bool) {
+ self.useAddTxIndex = tf
+ self.indexesDb = db
+}
+
+// GetAddTxIndex return indexes db and if atx index in use.
+func (self *BlockChain) GetAddTxIndex() (ethdb.Database, bool) {
+ return self.indexesDb, self.useAddTxIndex
+}
+
func (self *BlockChain) getProcInterrupt() bool {
return atomic.LoadInt32(&self.procInterrupt) == 1
}
@@ -773,6 +788,45 @@ func (bc *BlockChain) SetHead(head uint64) error {
glog.Fatalf("failed to reset head fast block hash: %v", err)
}
+ if bc.useAddTxIndex {
+ ldb, ok := bc.indexesDb.(*ethdb.LDBDatabase)
+ if !ok {
+ glog.Fatal("could not cast indexes db to level db")
+ }
+
+ var removals [][]byte
+ deleteRemovalsFn := func(rs [][]byte) {
+ for _, r := range rs {
+ if e := ldb.Delete(r); e != nil {
+ glog.Fatal(e)
+ }
+ }
+ }
+
+ pre := ethdb.NewBytesPrefix(txAddressIndexPrefix)
+ it := ldb.NewIteratorRange(pre)
+
+ for it.Next() {
+ key := it.Key()
+ _, bn, _, _, _ := resolveAddrTxBytes(key)
+ n := binary.LittleEndian.Uint64(bn)
+ if n > head {
+ removals = append(removals, key)
+ // Prevent removals from getting too massive in case it's a big rollback
+ // 100000 is a guess at a big but not-too-big memory allowance
+ if len(removals) > 100000 {
+ deleteRemovalsFn(removals)
+ removals = [][]byte{}
+ }
+ }
+ }
+ it.Release()
+ if e := it.Error(); e != nil {
+ return e
+ }
+ deleteRemovalsFn(removals)
+ }
+
bc.mu.Unlock()
return bc.LoadLastState(false)
}
@@ -1231,6 +1285,12 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain
glog.Fatal(errs[index])
return
}
+ // Store the addr-tx indexes if enabled
+ if self.useAddTxIndex {
+ if err := WriteBlockAddTxIndexes(self.indexesDb, block); err != nil {
+ glog.Fatalf("failed to write block add-tx indexes", err)
+ }
+ }
atomic.AddInt32(&stats.processed, 1)
}
}
@@ -1276,6 +1336,42 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain
return 0, nil
}
+// WriteBlockAddrTxIndexesBatch builds indexes for a given range of blocks N. It writes batches at increment 'step'.
+// If any error occurs during db writing it will be returned immediately.
+// It's sole implementation is the command 'atxi-build', since we must use individual block atxi indexing during
+// sync and import in order to ensure we're on the canonical chain for each block.
+func (self *BlockChain) WriteBlockAddrTxIndexesBatch(indexDb ethdb.Database, startBlockN, stopBlockN, stepN uint64) (txsCount int, err error) {
+ block := self.GetBlockByNumber(startBlockN)
+ batch := indexDb.NewBatch()
+
+ blockProcessedCount := uint64(0)
+ blockProcessedHead := func() uint64 {
+ return startBlockN + blockProcessedCount
+ }
+
+ for block != nil && blockProcessedHead() <= stopBlockN {
+ txP, err := putBlockAddrTxsToBatch(batch, block)
+ if err != nil {
+ return txsCount, err
+ }
+ txsCount += txP
+ blockProcessedCount++
+
+ // Write on stepN mod
+ if blockProcessedCount%stepN == 0 {
+ if err := batch.Write(); err != nil {
+ return txsCount, err
+ } else {
+ batch = indexDb.NewBatch()
+ }
+ }
+ block = self.GetBlockByNumber(blockProcessedHead())
+ }
+
+ // This will put the last batch
+ return txsCount, batch.Write()
+}
+
// WriteBlock writes the block to the chain.
func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err error) {
@@ -1523,6 +1619,12 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (chainIndex int, err err
if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
return i, err
}
+ // Store the addr-tx indexes if enabled
+ if self.useAddTxIndex {
+ if err := WriteBlockAddTxIndexes(self.indexesDb, block); err != nil {
+ glog.Fatalf("failed to write block add-tx indexes", err)
+ }
+ }
case SideStatTy:
if glog.V(logger.Detail) {
glog.Infof("inserted forked block #%d (TD=%v) (%d TXs %d UNCs) [%s]. Took %v\n", block.Number(), block.Difficulty(), len(block.Transactions()), len(block.Uncles()), block.Hash().Hex(), time.Since(bstart))
@@ -1655,6 +1757,17 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
).Send(mlogBlockchain)
}
+ // Remove all atxis from old chain; indexes should only reflect canonical
+ if self.useAddTxIndex {
+ for _, block := range oldChain {
+ for _, tx := range block.Transactions() {
+ if err := RmAddrTx(self.indexesDb, tx); err != nil {
+ return err
+ }
+ }
+ }
+ }
+
var addedTxs types.Transactions
// insert blocks. Order does not matter. Last block will be written in ImportChain itself which creates the new head properly
for _, block := range newChain {
@@ -1664,6 +1777,12 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
if err := WriteTransactions(self.chainDb, block); err != nil {
return err
}
+ // Store the addr-tx indexes if enabled
+ if self.useAddTxIndex {
+ if err := WriteBlockAddTxIndexes(self.indexesDb, block); err != nil {
+ return err
+ }
+ }
receipts := GetBlockReceipts(self.chainDb, block.Hash())
// write receipts
if err := WriteReceipts(self.chainDb, receipts); err != nil {
diff --git a/core/blockchain_test.go b/core/blockchain_test.go
index 0f71dd48a..b0673bd96 100644
--- a/core/blockchain_test.go
+++ b/core/blockchain_test.go
@@ -37,6 +37,8 @@ import (
"github.com/ethereumproject/go-ethereum/logger/glog"
"github.com/ethereumproject/go-ethereum/rlp"
"github.com/hashicorp/golang-lru"
+ "io/ioutil"
+ "strings"
)
func init() {
@@ -884,6 +886,207 @@ func TestFastVsFullChains(t *testing.T) {
}
}
+func TestFastVsFullChainsATXI(t *testing.T) {
+ archiveDir, e := ioutil.TempDir("", "archive-")
+ if e != nil {
+ t.Fatal(e)
+ }
+ fastDir, e := ioutil.TempDir("", "fast-")
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer os.RemoveAll(archiveDir)
+ defer os.RemoveAll(fastDir)
+
+ // Create the dbs
+ //
+ archiveDb, err := ethdb.NewLDBDatabase(archiveDir, 10, 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fastDb, err := ethdb.NewLDBDatabase(fastDir, 10, 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ MinGasLimit = big.NewInt(125000)
+
+ key1, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+ if err != nil {
+ t.Fatal(err)
+ }
+ key2, err := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var (
+ addr1 = crypto.PubkeyToAddress(key1.PublicKey)
+ addr2 = crypto.PubkeyToAddress(key2.PublicKey)
+ signer = types.NewChainIdSigner(big.NewInt(63))
+ dbs = []ethdb.Database{archiveDb, fastDb}
+ config = MakeDiehardChainConfig()
+ )
+
+ for i, db := range dbs {
+ t1, err := types.NewTransaction(0, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t2, err := types.NewTransaction(1, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t3, err := types.NewTransaction(0, addr1, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2)
+ if err != nil {
+ t.Fatal(err)
+ }
+ genesis := WriteGenesisBlockForTesting(db,
+ GenesisAccount{addr1, big.NewInt(1000000)},
+ GenesisAccount{addr2, big.NewInt(1000000)},
+ )
+ blocks, receipts := GenerateChain(config, genesis, db, 3, func(i int, gen *BlockGen) {
+ if i == 0 {
+ gen.AddTx(t1)
+ }
+ if i == 1 {
+ gen.AddTx(t2)
+ }
+ if i == 2 {
+ gen.AddTx(t3)
+ }
+ })
+
+ blockchain, err := NewBlockChain(db, config, FakePow{}, new(event.TypeMux))
+ if err != nil {
+ t.Fatal(err)
+ }
+ // turn on atxi
+ blockchain.SetAddTxIndex(db, true)
+ if i == 0 {
+ if n, err := blockchain.InsertChain(blocks); err != nil {
+ t.Fatalf("failed to process block %d: %v", n, err)
+ }
+ } else {
+ headers := make([]*types.Header, len(blocks))
+ for i, block := range blocks {
+ headers[i] = block.Header()
+ }
+ if n, err := blockchain.InsertHeaderChain(headers, 1); err != nil {
+ t.Fatalf("failed to insert header %d: %v", n, err)
+ }
+ if n, err := blockchain.InsertReceiptChain(blocks, receipts); err != nil {
+ t.Fatalf("failed to insert receipt %d: %v", n, err)
+ }
+ }
+
+ out := GetAddrTxs(db, addr1, 0, 0, "", "", -1, -1, false)
+ if len(out) != 3 {
+ t.Errorf("[%d] got: %v, want: %v", i, len(out), 3)
+ }
+ out = GetAddrTxs(db, addr1, 0, 0, "from", "", -1, -1, false)
+ if len(out) != 2 {
+ t.Errorf("[%d] got: %v, want: %v", i, len(out), 2)
+ }
+ out = GetAddrTxs(db, addr1, 0, 0, "to", "", -1, -1, false)
+ if len(out) != 1 {
+ t.Errorf("[%d] got: %v, want: %v", i, len(out), 1)
+ }
+ out = GetAddrTxs(db, addr2, 0, 0, "", "", -1, -1, false)
+ if len(out) != 3 {
+ t.Errorf("[%d] got: %v, want: %v", i, len(out), 3)
+ }
+ out = GetAddrTxs(db, addr2, 3, 3, "", "", -1, -1, false)
+ if len(out) != 1 {
+ t.Errorf("[%d] got: %v, want: %v", i, len(out), 1)
+ }
+ }
+}
+
+func TestRmAddrTx(t *testing.T) {
+ archiveDir, e := ioutil.TempDir("", "archive-")
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer os.RemoveAll(archiveDir)
+
+ // Create the dbs
+ //
+ db, err := ethdb.NewLDBDatabase(archiveDir, 10, 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ MinGasLimit = big.NewInt(125000)
+
+ key1, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+ if err != nil {
+ t.Fatal(err)
+ }
+ key2, err := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var (
+ addr1 = crypto.PubkeyToAddress(key1.PublicKey)
+ addr2 = crypto.PubkeyToAddress(key2.PublicKey)
+ signer = types.NewChainIdSigner(big.NewInt(63))
+ config = MakeDiehardChainConfig()
+ )
+
+ t1, err := types.NewTransaction(0, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t2, err := types.NewTransaction(1, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t3, err := types.NewTransaction(0, addr1, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2)
+ if err != nil {
+ t.Fatal(err)
+ }
+ genesis := WriteGenesisBlockForTesting(db,
+ GenesisAccount{addr1, big.NewInt(1000000)},
+ GenesisAccount{addr2, big.NewInt(1000000)},
+ )
+ blocks, _ := GenerateChain(config, genesis, db, 3, func(i int, gen *BlockGen) {
+ if i == 0 {
+ gen.AddTx(t1)
+ }
+ if i == 1 {
+ gen.AddTx(t2)
+ }
+ if i == 2 {
+ gen.AddTx(t3)
+ }
+ })
+
+ blockchain, err := NewBlockChain(db, config, FakePow{}, new(event.TypeMux))
+ if err != nil {
+ t.Fatal(err)
+ }
+ // turn on atxi
+ blockchain.SetAddTxIndex(db, true)
+
+ if n, err := blockchain.InsertChain(blocks); err != nil {
+ t.Fatalf("failed to process block %d: %v", n, err)
+ }
+
+ out := GetAddrTxs(db, addr1, 0, 0, "", "", -1, -1, false)
+ if len(out) != 3 {
+ t.Errorf("got: %v, want: %v", len(out), 3)
+ }
+ if err := RmAddrTx(db, t1); err != nil {
+ t.Fatal(err)
+ }
+ out = GetAddrTxs(db, addr1, 0, 0, "", "", -1, -1, false)
+ if len(out) != 2 {
+ t.Errorf("got: %v, want: %v", len(out), 2)
+ }
+}
+
// Tests that various import methods move the chain head pointers to the correct
// positions.
func TestLightVsFastVsFullChainHeads(t *testing.T) {
@@ -986,6 +1189,28 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
// Tests that chain reorganisations handle transaction removals and reinsertions.
func TestChainTxReorgs(t *testing.T) {
+ db, err := ethdb.NewMemDatabase()
+ if err != nil {
+ t.Fatal(err)
+ }
+ testChainTxReorgs(t, db, false)
+}
+
+func TestChainTxReorgsAtxi(t *testing.T) {
+ p, err := ioutil.TempDir("", "test-reorg-atxi-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(p)
+
+ db, err := ethdb.NewLDBDatabase(p, 10, 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+ testChainTxReorgs(t, db, true)
+}
+
+func testChainTxReorgs(t *testing.T, db ethdb.Database, withATXI bool) {
MinGasLimit = big.NewInt(125000)
key1, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
@@ -1000,10 +1225,6 @@ func TestChainTxReorgs(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- db, err := ethdb.NewMemDatabase()
- if err != nil {
- t.Fatal(err)
- }
var (
addr1 = crypto.PubkeyToAddress(key1.PublicKey)
@@ -1017,39 +1238,47 @@ func TestChainTxReorgs(t *testing.T) {
GenesisAccount{addr3, big.NewInt(1000000)},
)
// Create two transactions shared between the chains:
+ // addr1 -> addr2
// - postponed: transaction included at a later block in the forked chain
// - swapped: transaction included at the same block number in the forked chain
- postponed, err := types.NewTransaction(0, addr1, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1)
+ postponed, err := types.NewTransaction(0, addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1)
if err != nil {
t.Fatal(err)
}
- swapped, err := types.NewTransaction(1, addr1, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1)
+ swapped, err := types.NewTransaction(1, addr2, big.NewInt(1001), TxGas, nil, nil).WithSigner(signer).SignECDSA(key1)
if err != nil {
t.Fatal(err)
}
// Create two transactions that will be dropped by the forked chain:
+ // addr2 -> addr3
// - pastDrop: transaction dropped retroactively from a past block
// - freshDrop: transaction dropped exactly at the block where the reorg is detected
var pastDrop, freshDrop *types.Transaction
// Create three transactions that will be added in the forked chain:
+ // addr3 -> addr1
// - pastAdd: transaction added before the reorganization is detected
// - freshAdd: transaction added at the exact block the reorg is detected
// - futureAdd: transaction added after the reorg has already finished
var pastAdd, freshAdd, futureAdd *types.Transaction
+ // ATXI tallies, (means) will be removed
+ // addr1: 2f+3t
+ // addr2: 2t+(2f)
+ // addr3: (2t)+3f
+
chainConfig := MakeDiehardChainConfig()
chain, _ := GenerateChain(chainConfig, genesis, db, 3, func(i int, gen *BlockGen) {
switch i {
case 0:
- pastDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2)
+ pastDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr3, big.NewInt(1002), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2)
gen.AddTx(pastDrop) // This transaction will be dropped in the fork from below the split point
gen.AddTx(postponed) // This transaction will be postponed till block #3 in the fork
case 2:
- freshDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2)
+ freshDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr3, big.NewInt(1003), TxGas, nil, nil).WithSigner(signer).SignECDSA(key2)
gen.AddTx(freshDrop) // This transaction will be dropped in the fork from exactly at the split point
gen.AddTx(swapped) // This transaction will be swapped out at the exact height
@@ -1064,6 +1293,9 @@ func TestChainTxReorgs(t *testing.T) {
if err != nil {
t.Fatal(err)
}
+ if withATXI {
+ blockchain.SetAddTxIndex(db, true)
+ }
if i, err := blockchain.InsertChain(chain); err != nil {
t.Fatalf("failed to insert original chain[%d]: %v", i, err)
}
@@ -1072,18 +1304,18 @@ func TestChainTxReorgs(t *testing.T) {
chain, _ = GenerateChain(chainConfig, genesis, db, 5, func(i int, gen *BlockGen) {
switch i {
case 0:
- pastAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3)
+ pastAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr1, big.NewInt(1004), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3)
gen.AddTx(pastAdd) // This transaction needs to be injected during reorg
case 2:
gen.AddTx(postponed) // This transaction was postponed from block #1 in the original chain
gen.AddTx(swapped) // This transaction was swapped from the exact current spot in the original chain
- freshAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3)
+ freshAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr1, big.NewInt(1005), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3)
gen.AddTx(freshAdd) // This transaction will be added exactly at reorg time
case 3:
- futureAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3)
+ futureAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr1, big.NewInt(1006), TxGas, nil, nil).WithSigner(signer).SignECDSA(key3)
gen.AddTx(futureAdd) // This transaction will be added after a full reorg
}
})
@@ -1091,8 +1323,14 @@ func TestChainTxReorgs(t *testing.T) {
t.Fatalf("failed to insert forked chain: %v", err)
}
+ // Conveniently grouped
+ txsRemoved := types.Transactions{pastDrop, freshDrop}
+ txsAdded := types.Transactions{pastAdd, freshAdd, futureAdd}
+ txsShared := types.Transactions{postponed, swapped}
+ txsAll := types.Transactions{pastDrop, freshDrop, pastAdd, freshAdd, futureAdd, postponed, swapped}
+
// removed tx
- for i, tx := range (types.Transactions{pastDrop, freshDrop}) {
+ for i, tx := range txsRemoved {
if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil {
t.Errorf("drop %d: tx %v found while shouldn't have been", i, txn)
}
@@ -1101,7 +1339,7 @@ func TestChainTxReorgs(t *testing.T) {
}
}
// added tx
- for i, tx := range (types.Transactions{pastAdd, freshAdd, futureAdd}) {
+ for i, tx := range txsAdded {
if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil {
t.Errorf("add %d: expected tx to be found", i)
}
@@ -1110,7 +1348,7 @@ func TestChainTxReorgs(t *testing.T) {
}
}
// shared tx
- for i, tx := range (types.Transactions{postponed, swapped}) {
+ for i, tx := range txsShared {
if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil {
t.Errorf("share %d: expected tx to be found", i)
}
@@ -1118,6 +1356,53 @@ func TestChainTxReorgs(t *testing.T) {
t.Errorf("share %d: expected receipt to be found", i)
}
}
+
+ // ATXI checks
+ if !withATXI {
+ return
+ }
+ txsh1 := GetAddrTxs(db, addr1, 0, 0, "", "", -1, -1, false)
+ txsh2 := GetAddrTxs(db, addr2, 0, 0, "", "", -1, -1, false)
+ txsh3 := GetAddrTxs(db, addr3, 0, 0, "", "", -1, -1, false)
+
+ allAtxis := txsh1
+ allAtxis = append(allAtxis, txsh2...)
+ allAtxis = append(allAtxis, txsh3...)
+
+ // Ensure a transaction exists for each atxi hash
+ for _, x := range allAtxis {
+ if tx, _, _, _ := GetTransaction(db, common.HexToHash(x)); tx == nil {
+ t.Error("atxi not removed")
+ }
+ }
+
+ // Ensure no duplicate tx hashes returned
+DUPECHECK:
+ for i, l := range [][]string{txsh1, txsh2, txsh3} {
+ j := strings.Join(l, "")
+ for _, h := range l {
+ if strings.Count(j, h[:8]) > 1 {
+ // show offending tx
+ offendingTxN := new(big.Int)
+ for _, x := range txsAll {
+ if x.Hash().Hex() == h {
+ offendingTxN.Set(x.Value()) // use unique value as a way to identify offender
+ break
+ }
+ }
+ t.Log(strings.Join(l, "\n"))
+ t.Errorf("[%d] duplicate tx hash (%v)", i, offendingTxN)
+ break DUPECHECK
+ }
+ }
+
+ }
+
+ // Check magnitude; 2 atxis per canonical tx (to & from)
+ wantMag := (len(txsAdded) + len(txsShared)) * 2
+ if len(allAtxis) != wantMag {
+ t.Errorf("got: %v, want: %v", len(allAtxis), wantMag)
+ }
}
func TestLogReorgs(t *testing.T) {
diff --git a/core/database_util.go b/core/database_util.go
index cbc76b8a2..b72c45cfd 100644
--- a/core/database_util.go
+++ b/core/database_util.go
@@ -28,6 +28,8 @@ import (
"github.com/ethereumproject/go-ethereum/logger"
"github.com/ethereumproject/go-ethereum/logger/glog"
"github.com/ethereumproject/go-ethereum/rlp"
+ "sort"
+ "strings"
)
var (
@@ -46,13 +48,16 @@ var (
receiptsPrefix = []byte("receipts-")
blockReceiptsPrefix = []byte("receipts-block-")
+ txAddressIndexPrefix = []byte("atx-")
+ txAddressBookmarkKey = []byte("ATXIBookmark")
+
mipmapPre = []byte("mipmap-log-bloom-")
MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000}
blockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually]
- preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage
- lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata
+ preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage
+ lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata
)
// TxLookupEntry is a positional metadata to help looking up the data content of
@@ -63,6 +68,21 @@ type TxLookupEntry struct {
Index uint64
}
+func GetATXIBookmark(db ethdb.Database) uint64 {
+ v, err := db.Get(txAddressBookmarkKey)
+ if err != nil || v == nil {
+ return 0
+ }
+ i := binary.LittleEndian.Uint64(v)
+ return i
+}
+
+func SetATXIBookmark(db ethdb.Database, i uint64) error {
+ bn := make([]byte, 8)
+ binary.LittleEndian.PutUint64(bn, i)
+ return db.Put(txAddressBookmarkKey, bn)
+}
+
// GetCanonicalHash retrieves a hash assigned to a canonical block number.
func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash {
data, _ := db.Get(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...))
@@ -134,6 +154,38 @@ func GetBodyRLP(db ethdb.Database, hash common.Hash) rlp.RawValue {
return data
}
+// formatAddrTxIterator formats the index key prefix iterator, eg. atx-
+func formatAddrTxIterator(address common.Address) (iteratorPrefix []byte) {
+ iteratorPrefix = append(iteratorPrefix, txAddressIndexPrefix...)
+ iteratorPrefix = append(iteratorPrefix, address.Bytes()...)
+ return
+}
+
+// formatAddrTxBytesIndex formats the index key, eg. atx-
+// The values for these arguments should be of determinate length and format, see test TestFormatAndResolveAddrTxBytesKey
+// for example.
+func formatAddrTxBytesIndex(address, blockNumber, direction, kindof, txhash []byte) (key []byte) {
+ key = make([]byte, 0, 66) // 66 is the total capacity of the key = prefix(4)+addr(20)+blockNumber(8)+dir(1)+kindof(1)+txhash(32)
+ key = append(key, txAddressIndexPrefix...)
+ key = append(key, address...)
+ key = append(key, blockNumber...)
+ key = append(key, direction...)
+ key = append(key, kindof...)
+ key = append(key, txhash...)
+ return
+}
+
+// resolveAddrTxBytes resolves the index key to individual []byte values
+func resolveAddrTxBytes(key []byte) (address, blockNumber, direction, kindof, txhash []byte) {
+ // prefix = key[:4]
+ address = key[4:24] // common.AddressLength = 20
+ blockNumber = key[24:32] // uint64 via little endian
+ direction = key[32:33] // == key[32] (1 byte)
+ kindof = key[33:34]
+ txhash = key[34:]
+ return
+}
+
// GetBody retrieves the block body (transactons, uncles) corresponding to the
// hash, nil if none found.
func GetBody(db ethdb.Database, hash common.Hash) *types.Body {
@@ -149,6 +201,233 @@ func GetBody(db ethdb.Database, hash common.Hash) *types.Body {
return body
}
+// WriteBlockAddTxIndexes writes atx-indexes for a given block.
+func WriteBlockAddTxIndexes(indexDb ethdb.Database, block *types.Block) error {
+ batch := indexDb.NewBatch()
+ if _, err := putBlockAddrTxsToBatch(batch, block); err != nil {
+ return err
+ }
+ return batch.Write()
+}
+
+// putBlockAddrTxsToBatch formats and puts keys for a given block to a db Batch.
+// Batch can be written afterward if no errors, ie. batch.Write()
+func putBlockAddrTxsToBatch(putBatch ethdb.Batch, block *types.Block) (txsCount int, err error) {
+ for _, tx := range block.Transactions() {
+ txsCount++
+
+ from, err := tx.From()
+ if err != nil {
+ return txsCount, err
+ }
+ to := tx.To()
+ // s: standard
+ // c: contract
+ txKindOf := []byte("s")
+ if to == nil || to.IsEmpty() {
+ to = &common.Address{}
+ txKindOf = []byte("c")
+ }
+
+ // Note that len 8 because uint64 guaranteed <= 8 bytes.
+ bn := make([]byte, 8)
+ binary.LittleEndian.PutUint64(bn, block.NumberU64())
+
+ if err := putBatch.Put(formatAddrTxBytesIndex(from.Bytes(), bn, []byte("f"), txKindOf, tx.Hash().Bytes()), nil); err != nil {
+ return txsCount, err
+ }
+ if err := putBatch.Put(formatAddrTxBytesIndex(to.Bytes(), bn, []byte("t"), txKindOf, tx.Hash().Bytes()), nil); err != nil {
+ return txsCount, err
+ }
+ }
+ return txsCount, nil
+}
+
+type atxi struct {
+ blockN uint64
+ tx string
+}
+type sortableAtxis []atxi
+
+// Len implements sort.Sort interface.
+func (s sortableAtxis) Len() int {
+ return len(s)
+}
+
+// Less implements sort.Sort interface.
+// By default newer transactions by blockNumber are first.
+func (s sortableAtxis) Less(i, j int) bool {
+ return s[i].blockN > s[j].blockN
+}
+
+// Swap implements sort.Sort interface.
+func (s sortableAtxis) Swap(i, j int) {
+ s[i], s[j] = s[j], s[i]
+}
+func (s sortableAtxis) TxStrings() []string {
+ var out = make([]string, 0, len(s))
+ for _, str := range s {
+ out = append(out, str.tx)
+ }
+ return out
+}
+
+// GetAddrTxs gets the indexed transactions for a given account address.
+// 'reverse' means "oldest first"
+func GetAddrTxs(db ethdb.Database, address common.Address, blockStartN uint64, blockEndN uint64, direction string, kindof string, paginationStart int, paginationEnd int, reverse bool) []string {
+ if len(direction) > 0 && !strings.Contains("btf", direction[:1]) {
+ glog.Fatal("Address transactions list signature requires direction param to be empty string or [b|t|f] prefix (eg. both, to, or from)")
+ }
+ if len(kindof) > 0 && !strings.Contains("bsc", kindof[:1]) {
+ glog.Fatal("Address transactions list signature requires 'kind of' param to be empty string or [s|c] prefix (eg. both, standard, or contract)")
+ }
+
+ // Have to cast to LevelDB to use iterator. Yuck.
+ ldb, ok := db.(*ethdb.LDBDatabase)
+ if !ok {
+ return nil
+ }
+
+ // This will be the returnable.
+ var hashes []string
+
+ // Map direction -> byte
+ var wantDirectionB byte = 'b'
+ if len(direction) > 0 {
+ wantDirectionB = direction[0]
+ }
+ var wantKindOf byte = 'b'
+ if len(kindof) > 0 {
+ wantKindOf = kindof[0]
+ }
+
+ // Create address prefix for iteration.
+ prefix := ethdb.NewBytesPrefix(formatAddrTxIterator(address))
+ it := ldb.NewIteratorRange(prefix)
+
+ var atxis sortableAtxis
+
+ for it.Next() {
+ key := it.Key()
+
+ _, blockNum, torf, k, txh := resolveAddrTxBytes(key)
+
+ bn := binary.LittleEndian.Uint64(blockNum)
+
+ // If atxi is smaller than blockstart, skip
+ if blockStartN > 0 && bn < blockStartN {
+ continue
+ }
+ // If atxi is greater than blockend, skip
+ if blockEndN > 0 && bn > blockEndN {
+ continue
+ }
+ // Ensure matching direction if spec'd
+ if wantDirectionB != 'b' && wantDirectionB != torf[0] {
+ continue
+ }
+ // Ensure filter for/agnostic transaction kind of (contract, standard, both)
+ if wantKindOf != 'b' && wantKindOf != k[0] {
+ continue
+ }
+ if len(hashes) > 0 {
+
+ }
+ tx := common.ToHex(txh)
+ atxis = append(atxis, atxi{blockN: bn, tx: tx})
+ }
+ it.Release()
+ if e := it.Error(); e != nil {
+ glog.Fatalln(e)
+ }
+
+ handleSorting := func(s sortableAtxis) sortableAtxis {
+ if len(s) <= 1 {
+ return s
+ }
+ sort.Sort(s) // newest txs (by blockNumber) latest
+ if reverse {
+ for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+ s[i], s[j] = s[j], s[i]
+ }
+ }
+ if paginationStart < 0 {
+ paginationStart = 0
+ }
+ if paginationEnd < 0 {
+ paginationEnd = len(s)
+ }
+ return s[paginationStart:paginationEnd]
+ }
+
+ return handleSorting(atxis).TxStrings()
+}
+
+// RmAddrTx removes all atxi indexes for a given tx in case of a transaction removal, eg.
+// in the case of chain reorg.
+// It isn't an elegant function, but not a top priority for optimization because of
+// expected infrequency of it's being called.
+func RmAddrTx(db ethdb.Database, tx *types.Transaction) error {
+ if tx == nil {
+ return nil
+ }
+
+ ldb, ok := db.(*ethdb.LDBDatabase)
+ if !ok {
+ return nil
+ }
+
+ txH := tx.Hash()
+ from, err := tx.From()
+ if err != nil {
+ return err
+ }
+
+ removals := [][]byte{}
+
+ // TODO: not DRY, could be refactored
+ pre := ethdb.NewBytesPrefix(formatAddrTxIterator(from))
+ it := ldb.NewIteratorRange(pre)
+ for it.Next() {
+ key := it.Key()
+ _, _, _, _, txh := resolveAddrTxBytes(key)
+ if bytes.Compare(txH.Bytes(), txh) == 0 {
+ removals = append(removals, key)
+ break // because there can be only one
+ }
+ }
+ it.Release()
+ if e := it.Error(); e != nil {
+ return e
+ }
+
+ to := tx.To()
+ if to != nil {
+ toRef := *to
+ pre := ethdb.NewBytesPrefix(formatAddrTxIterator(toRef))
+ it := ldb.NewIteratorRange(pre)
+ for it.Next() {
+ key := it.Key()
+ _, _, _, _, txh := resolveAddrTxBytes(key)
+ if bytes.Compare(txH.Bytes(), txh) == 0 {
+ removals = append(removals, key)
+ break // because there can be only one
+ }
+ }
+ it.Release()
+ if e := it.Error(); e != nil {
+ return e
+ }
+ }
+
+ for _, r := range removals {
+ if err := db.Delete(r); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
// GetTd retrieves a block's total difficulty corresponding to the hash, nil if
// none found.
func GetTd(db ethdb.Database, hash common.Hash) *big.Int {
@@ -521,7 +800,6 @@ func WriteTxLookupEntries(db ethdb.Putter, block *types.Block) error {
return nil
}
-
// [deprecated by the header/block split, remove eventually]
// GetBlockByHashOld returns the old combined block corresponding to the hash
// or nil if not found. This method is only used by the upgrade mechanism to
diff --git a/core/database_util_test.go b/core/database_util_test.go
index a98a81173..b32386741 100644
--- a/core/database_util_test.go
+++ b/core/database_util_test.go
@@ -26,6 +26,8 @@ import (
"strconv"
"testing"
+ "crypto/ecdsa"
+ "encoding/binary"
"github.com/ethereumproject/go-ethereum/common"
"github.com/ethereumproject/go-ethereum/core/types"
"github.com/ethereumproject/go-ethereum/core/vm"
@@ -33,6 +35,7 @@ import (
"github.com/ethereumproject/go-ethereum/crypto/sha3"
"github.com/ethereumproject/go-ethereum/ethdb"
"github.com/ethereumproject/go-ethereum/rlp"
+ "strings"
)
type diffTest struct {
@@ -298,6 +301,240 @@ func TestTdStorage(t *testing.T) {
}
}
+func TestAddrTxStorage(t *testing.T) {
+ dbFilepath, err := ioutil.TempDir("", "geth-db-util-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(dbFilepath)
+ db, _ := ethdb.NewLDBDatabase(dbFilepath, 10, 100)
+
+ testKey := func(hex string) (*ecdsa.PrivateKey, common.Address) {
+ key := crypto.ToECDSA(common.Hex2Bytes(hex))
+ addr := crypto.PubkeyToAddress(key.PublicKey)
+ return key, addr
+ }
+
+ skey1, from1 := testKey("123915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8")
+ skey2, from2 := testKey("456915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8")
+
+ from2to := common.BytesToAddress([]byte{0x22})
+
+ // from1 -> 1
+ tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), big.NewInt(1111), big.NewInt(11111), []byte{0x11, 0x11, 0x11})
+
+ // from2 -> 2,3,txC
+ tx2 := types.NewTransaction(2, from2to, big.NewInt(222), big.NewInt(2222), big.NewInt(22222), []byte{0x22, 0x22, 0x22})
+ tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), big.NewInt(3333), big.NewInt(33333), []byte{0x33, 0x33, 0x33})
+ txC := types.NewTransaction(4, common.Address{}, big.NewInt(333), big.NewInt(3333), big.NewInt(33333), []byte{0x33, 0x33, 0x33})
+
+ txs := []*types.Transaction{tx1, tx2, tx3, txC}
+ txsSigned := []*types.Transaction{}
+
+ for _, x := range txs {
+ // Sign em so we get from
+ key := skey1
+ if x.Nonce() != 1 {
+ key = skey2
+ }
+ x.SetSigner(types.NewChainIdSigner(big.NewInt(1)))
+ xs, err := x.SignECDSA(key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ txsSigned = append(txsSigned, xs)
+ }
+
+ block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txsSigned, nil, nil)
+
+ // Put in the transactions just for fun.
+ //
+ // Check that no transactions entries are in a pristine database
+ for i, tx := range txsSigned {
+ if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil {
+ t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn)
+ }
+ }
+ // Insert all the transactions into the database, and verify contents
+ if err := WriteTransactions(db, block); err != nil {
+ t.Fatalf("failed to write transactions: %v", err)
+ }
+ for i, tx := range txsSigned {
+ if txn, hash, number, index := GetTransaction(db, tx.Hash()); txn == nil {
+ t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash())
+ } else {
+ if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
+ t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
+ }
+ if tx.String() != txn.String() {
+ t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
+ }
+ }
+ }
+
+ // Write the atx indexes
+ if err := WriteBlockAddTxIndexes(db, block); err != nil {
+ t.Fatal(err)
+ }
+
+ prefix := ethdb.NewBytesPrefix(txAddressIndexPrefix)
+ it := db.NewIteratorRange(prefix)
+ count := 0
+ for it.Next() {
+ count++
+ //// Debugger -- it's kinda nice to see what the indexes look like
+ //ad, bn, tf, sc, txh := resolveAddrTxBytes(it.Key())
+ //addr, blockn, direc, ko, txhash := common.BytesToAddress(ad), binary.LittleEndian.Uint64(bn), string(tf), string(sc), common.BytesToHash(txh)
+ //t.Log(addr.Hex(), blockn, direc, ko, txhash.Hex())
+ }
+ it.Release()
+ if e := it.Error(); e != nil {
+ t.Fatal(e)
+ }
+ if count != 8 {
+ t.Errorf("want: %v, got: %v", 7, count)
+ }
+
+ out := GetAddrTxs(db, from2, 0, 0, "", "", -1, -1, false)
+ if len(out) != 3 {
+ t.Errorf("want: %v, got: %v", 3, len(out))
+ }
+
+ // Test pagination and reverse
+ outReverse := GetAddrTxs(db, from2, 0, 0, "", "", -1, -1, true)
+ if len(outReverse) != 3 {
+ t.Errorf("want: %v, got: %v", 3, len(outReverse))
+ }
+ // reverse
+ if out[0] != outReverse[2] || out[1] != outReverse[1] || out[2] != outReverse[0] {
+ t.Errorf("got: %v, want: %v", outReverse, out)
+ }
+ // pagination
+ outPag := GetAddrTxs(db, from2, 0, 0, "", "", 1, -1, false)
+ if len(outPag) != 2 {
+ t.Errorf("got: %v, want: %v", len(outPag), 2)
+ }
+
+ out = GetAddrTxs(db, from2, 0, 0, "", "c", -1, -1, false)
+ if len(out) != 1 {
+ t.Errorf("got: %v, want: %v", len(out), 1)
+ }
+ out = GetAddrTxs(db, common.Address{}, 0, 0, "", "", -1, -1, false)
+ if len(out) != 1 {
+ t.Errorf("got: %v, want: %v", len(out), 1)
+ }
+
+ out = GetAddrTxs(db, from1, 314, 314, "", "", -1, -1, false)
+ if len(out) != 1 {
+ t.Errorf("want: %v, got: %v", 1, len(out))
+ } else {
+ h := out[0]
+ if !strings.HasPrefix(h, "0x") {
+ t.Errorf("want: 0x-prefix, got: %v", h)
+ }
+ if !common.IsHex(h) {
+ t.Errorf("want: hex, got: %v", h)
+ }
+ txh := common.HexToHash(h)
+
+ if txh != txsSigned[0].Hash() {
+ t.Errorf("got: %x, want: %x", txh, txsSigned[0].Hash())
+ }
+
+ gx, _, _, _ := GetTransaction(db, txh)
+ if gx == nil {
+ t.Errorf("missing tx: %x", txh)
+ }
+ }
+
+ out = GetAddrTxs(db, from2to, 314, 314, "to", "", -1, -1, false)
+ if len(out) != 1 {
+ t.Errorf("want: %v, got: %v", 1, len(out))
+ } else {
+ h := out[0]
+ if !strings.HasPrefix(h, "0x") {
+ t.Errorf("want: 0x-prefix, got: %v", h)
+ }
+ if !common.IsHex(h) {
+ t.Errorf("want: hex, got: %v", h)
+ }
+ txh := common.HexToHash(h)
+ if txh != txsSigned[1].Hash() {
+ t.Errorf("got: %x, want: %x", txh, txsSigned[0].Hash())
+ }
+ gx, _, _, _ := GetTransaction(db, txh)
+ if gx == nil {
+ t.Errorf("missing tx: %x", txh)
+ }
+ f, e := gx.From()
+ if e != nil {
+ t.Error(e)
+ }
+ if f != from2 {
+ t.Errorf("got: %v, want: %v", f, from2)
+ }
+ }
+ out = GetAddrTxs(db, from2to, 314, 314, "from", "", -1, -1, false)
+ if len(out) != 0 {
+ t.Errorf("want: %v, got: %v", 0, len(out))
+ }
+}
+
+func TestFormatAndResolveAddrTxBytesKey(t *testing.T) {
+ testAddr := common.Address{}
+ testBN := uint64(42)
+ testTorf := "f"
+ testKindOf := "s"
+ testTxH := common.Hash{}
+
+ testBNBytes := make([]byte, 8)
+ binary.LittleEndian.PutUint64(testBNBytes, testBN)
+
+ key := formatAddrTxBytesIndex(testAddr.Bytes(), testBNBytes, []byte(testTorf), []byte(testKindOf), testTxH.Bytes())
+
+ // Test key/prefix iterator-ability.
+ itPrefix := formatAddrTxIterator(testAddr)
+ if !bytes.HasPrefix(key, itPrefix) {
+ t.Fatalf("key/prefix mismatch: prefix=%s key=%s", itPrefix, key)
+ }
+
+ // Reverse engineer key and ensure expected.
+ outAddr, outBNBytes, outTorf, outKindOf, outTxH := resolveAddrTxBytes(key)
+
+ if gotAddr := common.BytesToAddress(outAddr); gotAddr != testAddr {
+ t.Errorf("got: %v, want: %v", gotAddr.Hex(), testAddr.Hex())
+ }
+ if gotBN := binary.LittleEndian.Uint64(outBNBytes); gotBN != testBN {
+ t.Errorf("got: %v, want: %v", gotBN, testBN)
+ }
+ if gotTorf := string(outTorf); gotTorf != testTorf {
+ t.Errorf("got: %v, want: %v", gotTorf, testTorf)
+ }
+ if gotKindOf := string(outKindOf); gotKindOf != testKindOf {
+ t.Errorf("got: %v, want: %v", gotKindOf, testKindOf)
+ }
+ if gotTxH := common.BytesToHash(outTxH); gotTxH != testTxH {
+ t.Errorf("got: %v, want: %v", gotTxH, testTxH)
+ }
+
+ // Ensure proper key part sizing.
+ sizes := []struct {
+ b []byte
+ expectedLen int
+ }{
+ {outAddr, common.AddressLength},
+ {outBNBytes, 8},
+ {outTorf, 1},
+ {outKindOf, 1},
+ {outTxH, common.HashLength},
+ }
+ for _, s := range sizes {
+ if l := len(s.b); l != s.expectedLen {
+ t.Errorf("want: %v, got: %v", s.expectedLen, l)
+ }
+ }
+}
+
// Tests that canonical numbers can be mapped to hashes and retrieved.
func TestCanonicalMappingStorage(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
diff --git a/core/state/database.go b/core/state/database.go
index 1b10b2d93..30784e19d 100644
--- a/core/state/database.go
+++ b/core/state/database.go
@@ -152,4 +152,3 @@ func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) {
}
return root, err
}
-
diff --git a/core/state/dump.go b/core/state/dump.go
index ba19418d1..0f3214618 100644
--- a/core/state/dump.go
+++ b/core/state/dump.go
@@ -25,9 +25,9 @@ import (
"sort"
"sync"
+ "fmt"
"github.com/ethereumproject/go-ethereum/common"
"github.com/ethereumproject/go-ethereum/rlp"
- "fmt"
"github.com/ethereumproject/go-ethereum/trie"
)
diff --git a/core/state/iterator.go b/core/state/iterator.go
index d2a17ba4a..91ff589f3 100644
--- a/core/state/iterator.go
+++ b/core/state/iterator.go
@@ -151,4 +151,3 @@ func (it *NodeIterator) retrieve() bool {
}
return true
}
-
diff --git a/core/state/journal.go b/core/state/journal.go
index dc15861b4..d164a6360 100644
--- a/core/state/journal.go
+++ b/core/state/journal.go
@@ -138,4 +138,3 @@ func (ch addLogChange) undo(s *StateDB) {
func (ch addPreimageChange) undo(s *StateDB) {
delete(s.preimages, ch.hash)
}
-
diff --git a/core/state/state_object.go b/core/state/state_object.go
index 13d8f0c3b..20b541460 100644
--- a/core/state/state_object.go
+++ b/core/state/state_object.go
@@ -24,10 +24,10 @@ import (
"github.com/ethereumproject/go-ethereum/common"
"github.com/ethereumproject/go-ethereum/crypto"
- "github.com/ethereumproject/go-ethereum/rlp"
- "github.com/ethereumproject/go-ethereum/trie"
"github.com/ethereumproject/go-ethereum/logger"
"github.com/ethereumproject/go-ethereum/logger/glog"
+ "github.com/ethereumproject/go-ethereum/rlp"
+ "github.com/ethereumproject/go-ethereum/trie"
)
var emptyCodeHash = crypto.Keccak256(nil)
@@ -425,4 +425,4 @@ func (self *StateObject) ForEachStorage(cb func(key, value common.Hash) bool) {
cb(key, common.BytesToHash(it.Value))
}
}
-}
\ No newline at end of file
+}
diff --git a/core/state/state_test.go b/core/state/state_test.go
index 6db8cdd15..cd0dcc317 100644
--- a/core/state/state_test.go
+++ b/core/state/state_test.go
@@ -231,4 +231,3 @@ func compareStateObjects(so0, so1 *StateObject, t *testing.T) {
t.Fatalf("Deleted mismatch: have %v, want %v", so0.deleted, so1.deleted)
}
}
-
diff --git a/core/state/statedb.go b/core/state/statedb.go
index ef8a34a1a..bc46ea010 100644
--- a/core/state/statedb.go
+++ b/core/state/statedb.go
@@ -24,12 +24,12 @@ import (
"sync"
"github.com/ethereumproject/go-ethereum/common"
+ "github.com/ethereumproject/go-ethereum/core/vm"
"github.com/ethereumproject/go-ethereum/crypto"
"github.com/ethereumproject/go-ethereum/logger"
"github.com/ethereumproject/go-ethereum/logger/glog"
"github.com/ethereumproject/go-ethereum/rlp"
"github.com/ethereumproject/go-ethereum/trie"
- "github.com/ethereumproject/go-ethereum/core/vm"
)
// The starting nonce determines the default nonce when new accounts are being
@@ -62,9 +62,9 @@ type revision struct {
// * Contracts
// * Accounts
type StateDB struct {
- db Database
- trie Trie
- pastTries []*trie.SecureTrie
+ db Database
+ trie Trie
+ pastTries []*trie.SecureTrie
// DB error.
// State objects are used by the consensus core and VM which are
@@ -616,4 +616,3 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common
}
}
}
-
diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go
index c6bd6512c..4a8bee569 100644
--- a/core/state/statedb_test.go
+++ b/core/state/statedb_test.go
@@ -29,9 +29,9 @@ import (
"testing/quick"
"github.com/ethereumproject/go-ethereum/common"
+ "github.com/ethereumproject/go-ethereum/core/vm"
"github.com/ethereumproject/go-ethereum/ethdb"
"gopkg.in/check.v1"
- "github.com/ethereumproject/go-ethereum/core/vm"
)
// Tests that updating a state trie does not leak any database writes prior to
@@ -421,4 +421,4 @@ func (s *StateSuite) TestTouchDelete(c *check.C) {
if len(s.state.stateObjectsDirty) != 0 {
c.Fatal("expected no dirty state object")
}
-}
\ No newline at end of file
+}
diff --git a/core/types/log.go b/core/types/log.go
index 8d112fbe2..b90fd9b29 100644
--- a/core/types/log.go
+++ b/core/types/log.go
@@ -134,4 +134,3 @@ func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error {
}
return err
}
-
diff --git a/core/types/transaction.go b/core/types/transaction.go
index e660eb608..b7c96b587 100644
--- a/core/types/transaction.go
+++ b/core/types/transaction.go
@@ -274,8 +274,8 @@ func (s Transactions) GetRlp(i int) []byte {
}
// Returns a new set t which is the difference between a to b
-func TxDifference(a, b Transactions) (keep Transactions) {
- keep = make(Transactions, 0, len(a))
+func TxDifference(a, b Transactions) (diff Transactions) {
+ diff = make(Transactions, 0, len(a))
remove := make(map[common.Hash]struct{})
for _, tx := range b {
@@ -284,11 +284,11 @@ func TxDifference(a, b Transactions) (keep Transactions) {
for _, tx := range a {
if _, ok := remove[tx.Hash()]; !ok {
- keep = append(keep, tx)
+ diff = append(diff, tx)
}
}
- return keep
+ return diff
}
// TxByNonce implements the sort interface to allow sorting a list of transactions
diff --git a/eth/api.go b/eth/api.go
index 8cbb8b59a..1cdcf5fa6 100644
--- a/eth/api.go
+++ b/eth/api.go
@@ -543,6 +543,7 @@ type PublicBlockChainAPI struct {
config *core.ChainConfig
bc *core.BlockChain
chainDb ethdb.Database
+ indexesDb ethdb.Database
eventMux *event.TypeMux
muNewBlockSubscriptions sync.Mutex // protects newBlocksSubscriptions
newBlockSubscriptions map[string]func(core.ChainEvent) error // callbacks for new block subscriptions
@@ -1644,6 +1645,49 @@ func (api *PrivateAdminAPI) ImportChain(file string) (bool, error) {
return true, nil
}
+// PublicDebugAPI is the collection of Etheruem APIs exposed over the public
+// debugging endpoint.
+type PublicGethAPI struct {
+ eth *Ethereum
+}
+
+// NewPublicDebugAPI creates a new API definition for the public debug methods
+// of the Ethereum service.
+func NewPublicGethAPI(eth *Ethereum) *PublicGethAPI {
+ return &PublicGethAPI{eth: eth}
+}
+
+// AddressTransactions gets transactions for a given address.
+// Optional values include start and stop block numbers, and to/from/both value for tx/address relation.
+// Returns a slice of strings of transactions hashes.
+func (api *PublicGethAPI) GetAddressTransactions(address common.Address, blockStartN uint64, blockEndN uint64, toOrFrom string, txKindOf string, pagStart, pagEnd int, reverse bool) (list []string, err error) {
+ glog.V(logger.Debug).Infoln("RPC call: debug_getAddressTransactions %s %d %d %s %s", address, blockStartN, blockEndN, toOrFrom, txKindOf)
+
+ db, inUse := api.eth.BlockChain().GetAddTxIndex()
+ if !inUse {
+ return nil, errors.New("addr-tx indexing not enabled")
+ }
+ // Use human-friendly abbreviations, per https://github.com/ethereumproject/go-ethereum/pull/475#issuecomment-366065122
+ // so 't' => to, 'f' => from, 'tf|ft' => either/both. Same pattern for txKindOf.
+ // _t_o OR _f_rom
+ if toOrFrom == "tf" || toOrFrom == "ft" {
+ toOrFrom = "b"
+ }
+ // _s_tandard OR _c_ontract
+ if txKindOf == "sc" || txKindOf == "cs" {
+ txKindOf = "b"
+ }
+
+ list = core.GetAddrTxs(db, address, blockStartN, blockEndN, toOrFrom, txKindOf, pagStart, pagEnd, reverse)
+
+ // Since list is a slice, it can be nil, which returns 'null'.
+ // Should return empty 'array' if no txs found.
+ if list == nil {
+ list = []string{}
+ }
+ return list, nil
+}
+
// PublicDebugAPI is the collection of Etheruem APIs exposed over the public
// debugging endpoint.
type PublicDebugAPI struct {
diff --git a/eth/backend.go b/eth/backend.go
index d098f87eb..a5904cead 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -81,6 +81,8 @@ type Config struct {
MinerThreads int
SolcPath string
+ UseAddrTxIndex bool
+
GpoMinGasPrice *big.Int
GpoMaxGasPrice *big.Int
GpoFullBlockRatio int
@@ -98,8 +100,9 @@ type Ethereum struct {
shutdownChan chan bool
// DB interfaces
- chainDb ethdb.Database // Block chain database
- dappDb ethdb.Database // Dapp database
+ chainDb ethdb.Database // Block chain database
+ dappDb ethdb.Database // Dapp database
+ indexesDb ethdb.Database // Indexes database (optional -- eg. add-tx indexes)
// Handlers
txPool *core.TxPool
@@ -226,6 +229,24 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
eth.pow = ethash.New()
}
+ // Initialize indexes db if enabled
+ // Blockchain will be assigned the db and atx enabled after blockchain is initialized below.
+ var indexesDb ethdb.Database
+ if config.UseAddrTxIndex {
+ // TODO: these are arbitrary numbers I just made up. Optimize?
+ // The reason these numbers are different than the atxi-build command is because for "appending" (vs. building)
+ // the atxi database should require far fewer resources since application performance is limited primarily by block import (chaindata db).
+ ethdb.SetCacheRatio("chaindata", 0.95)
+ ethdb.SetHandleRatio("chaindata", 0.95)
+ ethdb.SetCacheRatio("indexes", 0.05)
+ ethdb.SetHandleRatio("indexes", 0.05)
+ indexesDb, err = ctx.OpenDatabase("indexes", config.DatabaseCache, config.DatabaseCache)
+ if err != nil {
+ return nil, err
+ }
+ eth.indexesDb = indexesDb
+ }
+
// load the genesis block or write a new one if no genesis
// block is present in the database.
genesis := core.GetBlock(chainDb, core.GetCanonicalHash(chainDb, 0))
@@ -263,6 +284,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
}
return nil, err
}
+ // Configure enabled atxi for blockchain
+ if config.UseAddrTxIndex {
+ eth.blockchain.SetAddTxIndex(eth.indexesDb, true)
+ }
+
eth.gpo = NewGasPriceOracle(eth)
newPool := core.NewTxPool(eth.chainConfig, eth.EventMux(), eth.blockchain.State, eth.blockchain.GasLimit)
@@ -351,6 +377,11 @@ func (s *Ethereum) APIs() []rpc.API {
Namespace: "admin",
Version: "1.0",
Service: ethreg.NewPrivateRegistarAPI(s.chainConfig, s.blockchain, s.chainDb, s.txPool, s.accountManager),
+ }, {
+ Namespace: "geth",
+ Version: "1.0",
+ Service: NewPublicGethAPI(s),
+ Public: true,
},
}
}
@@ -361,7 +392,7 @@ func (s *Ethereum) ResetWithGenesisBlock(gb *types.Block) {
func (s *Ethereum) Etherbase() (eb common.Address, err error) {
eb = s.etherbase
- if (eb == common.Address{}) {
+ if eb.IsEmpty() {
firstAccount, err := s.AccountManager().AccountByIndex(0)
eb = firstAccount.Address
if err != nil {
diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go
index 996f12a79..f61dfa5d9 100644
--- a/eth/downloader/downloader.go
+++ b/eth/downloader/downloader.go
@@ -32,9 +32,9 @@ import (
"github.com/ethereumproject/go-ethereum/core/types"
"github.com/ethereumproject/go-ethereum/ethdb"
"github.com/ethereumproject/go-ethereum/event"
- "github.com/ethereumproject/go-ethereum/metrics"
"github.com/ethereumproject/go-ethereum/logger"
"github.com/ethereumproject/go-ethereum/logger/glog"
+ "github.com/ethereumproject/go-ethereum/metrics"
)
const (
@@ -65,12 +65,12 @@ var (
maxHeadersProcess = 2048 // Number of header download results to import at once into the chain
maxResultsProcess = 2048 // Number of content download results to import at once into the chain
- fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync
- fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected
- fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it
- fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point
- fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync
- fsCriticalTrials uint32 = 10 // Number of times to retry in the cricical section before bailing
+ fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync
+ fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected
+ fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it
+ fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point
+ fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync
+ fsCriticalTrials uint32 = 10 // Number of times to retry in the cricical section before bailing
)
var (
@@ -129,7 +129,7 @@ type Downloader struct {
stateDB ethdb.Database
fsPivotLock *types.Header // Pivot header on critical section entry (cannot change between retries)
- fsPivotFails uint32 // Number of fast sync failures in the critical section
+ fsPivotFails uint32 // Number of fast sync failures in the critical section
rttEstimate uint64 // Round trip time to target for download requests
rttConfidence uint64 // Confidence in the estimated RTT (unit: millionths to allow atomic ops)
diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go
index 70b165578..298756bd3 100644
--- a/eth/downloader/peer.go
+++ b/eth/downloader/peer.go
@@ -345,8 +345,8 @@ func (p *peer) String() string {
// peerSet represents the collection of active peer participating in the chain
// download procedure.
type peerSet struct {
- peers map[string]*peer
- lock sync.RWMutex
+ peers map[string]*peer
+ lock sync.RWMutex
}
// newPeerSet creates a new peer set top track the active download sources.
diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go
index ffcc26ca1..da963440d 100644
--- a/eth/downloader/queue.go
+++ b/eth/downloader/queue.go
@@ -27,10 +27,10 @@ import (
"github.com/ethereumproject/go-ethereum/common"
"github.com/ethereumproject/go-ethereum/core/types"
+ "github.com/ethereumproject/go-ethereum/logger"
"github.com/ethereumproject/go-ethereum/logger/glog"
"github.com/ethereumproject/go-ethereum/metrics"
"gopkg.in/karalabe/cookiejar.v2/collections/prque"
- "github.com/ethereumproject/go-ethereum/logger"
)
var blockCacheLimit = 8192 // Maximum number of blocks to cache before throttling the download
diff --git a/ethdb/database.go b/ethdb/database.go
index 6b66a07fc..5857bce14 100644
--- a/ethdb/database.go
+++ b/ethdb/database.go
@@ -28,12 +28,13 @@ import (
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
+ ldbutil "github.com/syndtr/goleveldb/leveldb/util"
"sync"
)
var OpenFileLimit = 64
-// cacheRatio specifies how the total alloted cache is distributed between the
+// cacheRatio specifies how the total allotted cache is distributed between the
// various system databases.
var cacheRatio = map[string]float64{
"dapp": 0.0,
@@ -47,6 +48,14 @@ var handleRatio = map[string]float64{
"chaindata": 1.0,
}
+func SetCacheRatio(db string, ratio float64) {
+ cacheRatio[db] = ratio
+}
+
+func SetHandleRatio(db string, ratio float64) {
+ handleRatio[db] = ratio
+}
+
type LDBDatabase struct {
file string
db *leveldb.DB
@@ -89,7 +98,6 @@ func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) {
}, nil
}
-
// Path returns the path to the database directory.
func (db *LDBDatabase) Path() string {
return db.file
@@ -124,6 +132,14 @@ func (self *LDBDatabase) NewIterator() iterator.Iterator {
return self.db.NewIterator(nil, nil)
}
+func (self *LDBDatabase) NewIteratorRange(slice *ldbutil.Range) iterator.Iterator {
+ return self.db.NewIterator(slice, nil)
+}
+
+func NewBytesPrefix(prefix []byte) *ldbutil.Range {
+ return ldbutil.BytesPrefix(prefix)
+}
+
func (self *LDBDatabase) Close() {
if err := self.db.Close(); err != nil {
glog.Errorf("eth: DB %s: %s", self.file, err)
diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go
index 3e5b62361..ce9ea34ed 100644
--- a/internal/web3ext/web3ext.go
+++ b/internal/web3ext/web3ext.go
@@ -27,6 +27,7 @@ var Modules = map[string]string{
"rpc": RPC_JS,
"shh": Shh_JS,
"txpool": TxPool_JS,
+ "geth": Geth_JS,
}
const Admin_JS = `
@@ -149,6 +150,22 @@ web3._extend({
});
`
+const Geth_JS = `
+web3._extend({
+ property: 'geth',
+ methods:
+ [
+ new web3._extend.Method({
+ name: 'getAddressTransactions',
+ call: 'geth_getAddressTransactions',
+ params: 8,
+ inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, null, null, null, null, null, null]
+ })
+ ],
+ properties: []
+});
+`
+
const Debug_JS = `
web3._extend({
property: 'debug',
@@ -206,6 +223,12 @@ web3._extend({
name: 'accountExist',
call: 'debug_accountExist',
params: 2
+ }),
+ new web3._extend.Method({
+ name: 'getAddressTransactions',
+ call: 'debug_getAddressTransactions',
+ params: 8,
+ inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, null, null, null, null, null, null]
})
],
properties: []
diff --git a/rpc/server.go b/rpc/server.go
index 7bece588d..9f4af3b9a 100644
--- a/rpc/server.go
+++ b/rpc/server.go
@@ -36,7 +36,7 @@ const (
notificationBufferSize = 10000 // max buffered notifications before codec is closed
MetadataApi = "rpc"
- DefaultIPCApis = "admin,debug,eth,miner,net,personal,shh,txpool,web3"
+ DefaultIPCApis = "admin,debug,eth,miner,net,personal,shh,txpool,web3,geth"
DefaultHTTPApis = "eth,net,web3"
)
diff --git a/tests/state_test_util.go b/tests/state_test_util.go
index 9b7986279..e0a435917 100644
--- a/tests/state_test_util.go
+++ b/tests/state_test_util.go
@@ -243,7 +243,7 @@ func RunState(ruleSet RuleSet, db ethdb.Database, statedb *state.StateDB, env, t
if core.IsNonceErr(err) || core.IsInvalidTxErr(err) || core.IsGasLimitErr(err) {
statedb.RevertToSnapshot(snapshot)
}
- statedb.CommitTo(db,false)
+ statedb.CommitTo(db, false)
return ret, vmenv.state.Logs(), vmenv.Gas, err
}
diff --git a/trie/encoding.go b/trie/encoding.go
index 6db8dd00d..e96a786e4 100644
--- a/trie/encoding.go
+++ b/trie/encoding.go
@@ -15,6 +15,7 @@
// along with the go-ethereum library. If not, see .
package trie
+
// Trie keys are dealt with in three distinct encodings:
//
// KEYBYTES encoding contains the actual key and nothing else. This encoding is the
@@ -110,4 +111,4 @@ func prefixLen(a, b []byte) int {
// hasTerm returns whether a hex key has the terminator flag.
func hasTerm(s []byte) bool {
return len(s) > 0 && s[len(s)-1] == 16
-}
\ No newline at end of file
+}
diff --git a/trie/encoding_test.go b/trie/encoding_test.go
index 75e886272..97d8da136 100644
--- a/trie/encoding_test.go
+++ b/trie/encoding_test.go
@@ -17,8 +17,8 @@
package trie
import (
- "testing"
"bytes"
+ "testing"
)
func TestHexCompact(t *testing.T) {
diff --git a/trie/errors.go b/trie/errors.go
index 65c6d46c3..5d16d8b30 100644
--- a/trie/errors.go
+++ b/trie/errors.go
@@ -52,4 +52,3 @@ type MissingNodeError struct {
func (err *MissingNodeError) Error() string {
return fmt.Sprintf("missing trie node %x (path %x)", err.NodeHash, err.Path)
}
-
diff --git a/trie/hasher.go b/trie/hasher.go
index 1b9fc813c..cd2d7b5d6 100644
--- a/trie/hasher.go
+++ b/trie/hasher.go
@@ -227,4 +227,4 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
return hash, err
}
return hash, nil
-}
\ No newline at end of file
+}
diff --git a/trie/iterator.go b/trie/iterator.go
index 31a01b5bc..af30efb11 100644
--- a/trie/iterator.go
+++ b/trie/iterator.go
@@ -24,7 +24,6 @@ import (
"github.com/ethereumproject/go-ethereum/common"
)
-
// Iterator is a key-value trie iterator that traverses a Trie.
type Iterator struct {
nodeIt NodeIterator
diff --git a/trie/iterator_test.go b/trie/iterator_test.go
index 5ecb2feef..ab811d35e 100644
--- a/trie/iterator_test.go
+++ b/trie/iterator_test.go
@@ -19,10 +19,10 @@ package trie
import (
"testing"
+ "bytes"
+ "fmt"
"github.com/ethereumproject/go-ethereum/common"
"github.com/ethereumproject/go-ethereum/ethdb"
- "fmt"
- "bytes"
"math/rand"
)
diff --git a/trie/node.go b/trie/node.go
index adfc3be6e..2b559c36d 100644
--- a/trie/node.go
+++ b/trie/node.go
@@ -201,7 +201,6 @@ func decodeRef(buf []byte, cachegen uint16) (node, []byte, error) {
}
}
-
// wraps a decoding error with information about the path to the
// invalid child node (for debugging encoding issues).
type decodeError struct {
diff --git a/trie/proof.go b/trie/proof.go
index 2fc0be62b..eea29e11c 100644
--- a/trie/proof.go
+++ b/trie/proof.go
@@ -20,11 +20,12 @@ import (
"bytes"
"fmt"
- "github.com/ethereumproject/go-ethereum/rlp"
- "github.com/ethereumproject/go-ethereum/crypto"
"github.com/ethereumproject/go-ethereum/common"
+ "github.com/ethereumproject/go-ethereum/crypto"
"github.com/ethereumproject/go-ethereum/logger/glog"
+ "github.com/ethereumproject/go-ethereum/rlp"
)
+
// Prove constructs a merkle proof for key. The result contains all
// encoded nodes on the path to the value at key. The value itself is
// also included in the last node and can be retrieved by verifying
diff --git a/trie/proof_test.go b/trie/proof_test.go
index f83581f95..882156676 100644
--- a/trie/proof_test.go
+++ b/trie/proof_test.go
@@ -24,8 +24,8 @@ import (
"time"
"github.com/ethereumproject/go-ethereum/common"
- "github.com/ethereumproject/go-ethereum/ethdb"
"github.com/ethereumproject/go-ethereum/crypto"
+ "github.com/ethereumproject/go-ethereum/ethdb"
)
func init() {
@@ -161,4 +161,4 @@ func randBytes(n int) []byte {
r := make([]byte, n)
crand.Read(r)
return r
-}
\ No newline at end of file
+}
diff --git a/trie/secure_trie.go b/trie/secure_trie.go
index a8c438145..813883f3c 100644
--- a/trie/secure_trie.go
+++ b/trie/secure_trie.go
@@ -17,8 +17,8 @@
package trie
import (
- "github.com/ethereumproject/go-ethereum/common"
"fmt"
+ "github.com/ethereumproject/go-ethereum/common"
"github.com/ethereumproject/go-ethereum/logger/glog"
)
@@ -214,4 +214,4 @@ func (t *SecureTrie) getSecKeyCache() map[string][]byte {
t.secKeyCache = make(map[string][]byte)
}
return t.secKeyCache
-}
\ No newline at end of file
+}
diff --git a/trie/sync.go b/trie/sync.go
index cee208e63..b89906c78 100644
--- a/trie/sync.go
+++ b/trie/sync.go
@@ -38,7 +38,7 @@ type request struct {
hash common.Hash // Hash of the node data content to retrieve
data []byte // Data content of the node, cached until all subtrees complete
object *node // Target node to populate with retrieved data (hashnode originally)
- raw bool // Whether this is a raw entry (code) or a trie node
+ raw bool // Whether this is a raw entry (code) or a trie node
parents []*request // Parent state nodes referencing this entry (notify all upon completion)
depth int // Depth level within the trie the node is located to prioritise DFS
diff --git a/trie/trie.go b/trie/trie.go
index 763648660..9ac48cccf 100644
--- a/trie/trie.go
+++ b/trie/trie.go
@@ -23,8 +23,8 @@ import (
"github.com/ethereumproject/go-ethereum/common"
"github.com/ethereumproject/go-ethereum/crypto/sha3"
- "github.com/rcrowley/go-metrics"
"github.com/ethereumproject/go-ethereum/logger/glog"
+ "github.com/rcrowley/go-metrics"
)
var (
diff --git a/trie/trie_test.go b/trie/trie_test.go
index 3253dfa82..e175bb71d 100644
--- a/trie/trie_test.go
+++ b/trie/trie_test.go
@@ -18,21 +18,21 @@ package trie
import (
"bytes"
- "testing"
+ "encoding/binary"
+ "errors"
"fmt"
"github.com/davecgh/go-spew/spew"
"github.com/ethereumproject/go-ethereum/common"
+ "github.com/ethereumproject/go-ethereum/crypto"
"github.com/ethereumproject/go-ethereum/ethdb"
- "encoding/binary"
- "reflect"
- "testing/quick"
- "os"
+ "github.com/ethereumproject/go-ethereum/rlp"
+ "io/ioutil"
"math/big"
"math/rand"
- "errors"
- "io/ioutil"
- "github.com/ethereumproject/go-ethereum/crypto"
- "github.com/ethereumproject/go-ethereum/rlp"
+ "os"
+ "reflect"
+ "testing"
+ "testing/quick"
)
func init() {
@@ -606,4 +606,4 @@ func updateString(trie *Trie, k, v string) {
func deleteString(trie *Trie, k string) {
trie.Delete([]byte(k))
-}
\ No newline at end of file
+}