From 83f625076be14341cbb7047abce4bb1646a8bd92 Mon Sep 17 00:00:00 2001 From: noot Date: Wed, 24 Mar 2021 14:38:52 -0400 Subject: [PATCH 01/10] maybe fix sync --- dot/network/sync.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dot/network/sync.go b/dot/network/sync.go index af0b02c3ab..0acc42ec37 100644 --- a/dot/network/sync.go +++ b/dot/network/sync.go @@ -192,10 +192,11 @@ func (q *syncQueue) syncAtHead() { prev = curr start := uint64(curr.Number.Int64()) + 1 logger.Debug("haven't received new blocks since last check, pushing request", "start", start) - q.requestData.Store(start, requestData{ - sent: true, - received: false, - }) + // q.requestData.Store(start, requestData{ + // sent: true, + // received: false, + // }) + q.requestData.Delete(start) q.pushRequest(start, 1, "") } } From e76a6060cc0a075cd63c6a3b0f68975cd59e1d11 Mon Sep 17 00:00:00 2001 From: noot Date: Wed, 24 Mar 2021 17:55:00 -0400 Subject: [PATCH 02/10] fix prune peers goroutine --- dot/network/sync.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dot/network/sync.go b/dot/network/sync.go index 0acc42ec37..d30638d548 100644 --- a/dot/network/sync.go +++ b/dot/network/sync.go @@ -249,7 +249,12 @@ func (q *syncQueue) handleResponseQueue() { // prune peers with low score and connect to new peers func (q *syncQueue) prunePeers() { for { - time.Sleep(time.Second * 30) + select { + case <-time.After(time.Second * 30): + case <-q.ctx.Done(): + return + } + logger.Debug("✂️ pruning peers w/ low score...") peers := q.getSortedPeers() From bd1b1b743999942ce67cb08c3ea3a92625d03db6 Mon Sep 17 00:00:00 2001 From: noot Date: Wed, 24 Mar 2021 18:02:49 -0400 Subject: [PATCH 03/10] force garbage collection --- dot/state/storage.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dot/state/storage.go b/dot/state/storage.go index 5c9a45be54..4a8d6f1e01 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -20,6 +20,7 @@ import ( "encoding/binary" "errors" "fmt" + "runtime/debug" "sync" "github.com/ChainSafe/chaindb" @@ -86,6 +87,7 @@ func (s *StorageState) pruneKey(keyHeader *types.Header) { } delete(s.tries, keyHeader.StateRoot) + debug.FreeOSMemory() // TODO: database pruning needs to be refactored since the trie is now stored by nodes } From 99a349ddc16636cea7e8376ad045e2fdc7372f63 Mon Sep 17 00:00:00 2001 From: noot Date: Wed, 24 Mar 2021 18:33:30 -0400 Subject: [PATCH 04/10] gc in StoreTrie --- dot/state/storage.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dot/state/storage.go b/dot/state/storage.go index 4a8d6f1e01..89e2b243b7 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -110,6 +110,8 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState) error { logger.Warn("failed to notify storage subscriptions", "error", err) } }() + + debug.FreeOSMemory() return nil } From c5041f8e5f3788b93cf2b8395e87d3b78e67da6c Mon Sep 17 00:00:00 2001 From: noot Date: Wed, 24 Mar 2021 18:53:47 -0400 Subject: [PATCH 05/10] add syncing mode to StorageState --- dot/core/test_helpers.go | 2 ++ dot/network/state.go | 2 ++ dot/network/sync.go | 5 +++++ dot/state/storage.go | 21 +++++++++++++++++++-- dot/sync/interface.go | 1 + dot/sync/syncer.go | 5 +++++ 6 files changed, 34 insertions(+), 2 deletions(-) diff --git a/dot/core/test_helpers.go b/dot/core/test_helpers.go index 3da5758d64..e5657c6a28 100644 --- a/dot/core/test_helpers.go +++ b/dot/core/test_helpers.go @@ -261,6 +261,8 @@ func (s *mockSyncer) IsSynced() bool { return false } +func (s *mockSyncer) SetSyncing(bool) {} + type mockDigestItem struct { //nolint i int } diff --git a/dot/network/state.go b/dot/network/state.go index 2077ada3b3..77725bd7de 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -46,6 +46,8 @@ type Syncer interface { // IsSynced exposes the internal synced state // TODO: use syncQueue for this IsSynced() bool + + SetSyncing(bool) } // TransactionHandler is the interface used by the transactions sub-protocol diff --git a/dot/network/sync.go b/dot/network/sync.go index d30638d548..ac41d9de8e 100644 --- a/dot/network/sync.go +++ b/dot/network/sync.go @@ -164,6 +164,8 @@ func (q *syncQueue) syncAtHead() { return } + q.s.syncer.SetSyncing(true) + for { select { // sleep for average block time TODO: make this configurable from slot duration @@ -183,6 +185,9 @@ func (q *syncQueue) syncAtHead() { continue } + logger.Info("set syncing to false") + q.s.syncer.SetSyncing(false) + // we have received new blocks since the last check, sleep if prev.Number.Int64() < curr.Number.Int64() { prev = curr diff --git a/dot/state/storage.go b/dot/state/storage.go index 89e2b243b7..65a3ab5d01 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -53,6 +53,8 @@ type StorageState struct { // change notifiers changedLock sync.RWMutex subscriptions map[byte]*StorageSubscription + + syncing bool } // NewStorageState creates a new StorageState backed by the given trie and database located at basePath. @@ -77,6 +79,10 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie) }, nil } +func (s *StorageState) SetSyncing(syncing bool) { + s.syncing = syncing +} + func (s *StorageState) pruneKey(keyHeader *types.Header) { s.lock.Lock() defer s.lock.Unlock() @@ -87,7 +93,9 @@ func (s *StorageState) pruneKey(keyHeader *types.Header) { } delete(s.tries, keyHeader.StateRoot) - debug.FreeOSMemory() + if !s.syncing { + debug.FreeOSMemory() + } // TODO: database pruning needs to be refactored since the trie is now stored by nodes } @@ -95,6 +103,12 @@ func (s *StorageState) pruneKey(keyHeader *types.Header) { func (s *StorageState) StoreTrie(ts *rtstorage.TrieState) error { s.lock.Lock() root := ts.MustRoot() + if s.syncing { + // keep only the trie at the head of the chain when syncing + for key := range s.tries { + delete(s.tries, key) + } + } s.tries[root] = ts.Trie() s.lock.Unlock() @@ -111,7 +125,10 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState) error { } }() - debug.FreeOSMemory() + // this is turned on when near the head, since it makes syncing significantly slower + if !s.syncing { + debug.FreeOSMemory() + } return nil } diff --git a/dot/sync/interface.go b/dot/sync/interface.go index 2500d0df5d..1a457091e2 100644 --- a/dot/sync/interface.go +++ b/dot/sync/interface.go @@ -52,6 +52,7 @@ type StorageState interface { TrieState(root *common.Hash) (*rtstorage.TrieState, error) StoreTrie(ts *rtstorage.TrieState) error LoadCodeHash(*common.Hash) (common.Hash, error) + SetSyncing(bool) } // TransactionState is the interface for transaction queue methods diff --git a/dot/sync/syncer.go b/dot/sync/syncer.go index fb92a91b37..5a7ea0769d 100644 --- a/dot/sync/syncer.go +++ b/dot/sync/syncer.go @@ -412,3 +412,8 @@ func (s *Service) handleDigests(header *types.Header) { func (s *Service) IsSynced() bool { return s.synced } + +func (s *Service) SetSyncing(syncing bool) { + s.synced = !syncing + s.storageState.SetSyncing(syncing) +} From 21e6e92fbfbedf28a13ac93834876cf3994899b4 Mon Sep 17 00:00:00 2001 From: noot Date: Wed, 24 Mar 2021 19:49:47 -0400 Subject: [PATCH 06/10] lint --- dot/network/service_test.go | 2 +- dot/network/sync.go | 4 ---- dot/network/test_helpers.go | 4 ++-- dot/rpc/modules/system_test.go | 2 ++ dot/state/storage.go | 1 + dot/sync/syncer.go | 1 + 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dot/network/service_test.go b/dot/network/service_test.go index 54f4c7b413..dab4ccb26c 100644 --- a/dot/network/service_test.go +++ b/dot/network/service_test.go @@ -187,7 +187,7 @@ func TestService_Health(t *testing.T) { require.Equal(t, s.Health().IsSyncing, true) mockSync := s.syncer.(*mockSyncer) - mockSync.setSyncedState(true) + mockSync.SetSyncing(false) require.Equal(t, s.Health().IsSyncing, false) } diff --git a/dot/network/sync.go b/dot/network/sync.go index ac41d9de8e..a456e22df8 100644 --- a/dot/network/sync.go +++ b/dot/network/sync.go @@ -197,10 +197,6 @@ func (q *syncQueue) syncAtHead() { prev = curr start := uint64(curr.Number.Int64()) + 1 logger.Debug("haven't received new blocks since last check, pushing request", "start", start) - // q.requestData.Store(start, requestData{ - // sent: true, - // received: false, - // }) q.requestData.Delete(start) q.pushRequest(start, 1, "") } diff --git a/dot/network/test_helpers.go b/dot/network/test_helpers.go index 17df3c7a10..03851ac325 100644 --- a/dot/network/test_helpers.go +++ b/dot/network/test_helpers.go @@ -65,8 +65,8 @@ func (s *mockSyncer) IsSynced() bool { return s.synced } -func (s *mockSyncer) setSyncedState(newState bool) { - s.synced = newState +func (s *mockSyncer) SetSyncing(syncing bool) { + s.synced = !syncing } type testStreamHandler struct { diff --git a/dot/rpc/modules/system_test.go b/dot/rpc/modules/system_test.go index a9d6c207af..08e3512faf 100644 --- a/dot/rpc/modules/system_test.go +++ b/dot/rpc/modules/system_test.go @@ -63,6 +63,8 @@ func (s *mockSyncer) IsSynced() bool { return false } +func (s *mockSyncer) SetSyncing(_ bool) {} + type mockBlockState struct{} func (s *mockBlockState) BestBlockHeader() (*types.Header, error) { diff --git a/dot/state/storage.go b/dot/state/storage.go index 65a3ab5d01..6cc9b131a0 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -79,6 +79,7 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie) }, nil } +// SetSyncing sets whether the node is currently syncing or not func (s *StorageState) SetSyncing(syncing bool) { s.syncing = syncing } diff --git a/dot/sync/syncer.go b/dot/sync/syncer.go index 5a7ea0769d..e1ed6dc95a 100644 --- a/dot/sync/syncer.go +++ b/dot/sync/syncer.go @@ -413,6 +413,7 @@ func (s *Service) IsSynced() bool { return s.synced } +// SetSyncing sets whether the node is currently syncing or not func (s *Service) SetSyncing(syncing bool) { s.synced = !syncing s.storageState.SetSyncing(syncing) From 27b399e7f2f2d31ff9f0b78717a71b0f4b17faac Mon Sep 17 00:00:00 2001 From: noot Date: Thu, 25 Mar 2021 12:07:43 -0400 Subject: [PATCH 07/10] get GC percentage on startup --- dot/node.go | 4 ++++ dot/state/storage.go | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/dot/node.go b/dot/node.go index 53ef7103c4..224557e5e6 100644 --- a/dot/node.go +++ b/dot/node.go @@ -22,6 +22,7 @@ import ( "os" "os/signal" "path" + "runtime/debug" "sync" "syscall" @@ -158,6 +159,9 @@ func NodeInitialized(basepath string, expected bool) bool { // NewNode creates a new dot node from a dot node configuration func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, error) { + // set garbage collection percent to 10% + debug.SetGCPercent(10) + setupLogger(cfg) // if authority node, should have at least 1 key in keystore diff --git a/dot/state/storage.go b/dot/state/storage.go index 6cc9b131a0..7cb3f84929 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -20,7 +20,7 @@ import ( "encoding/binary" "errors" "fmt" - "runtime/debug" + //"runtime/debug" "sync" "github.com/ChainSafe/chaindb" @@ -94,9 +94,9 @@ func (s *StorageState) pruneKey(keyHeader *types.Header) { } delete(s.tries, keyHeader.StateRoot) - if !s.syncing { - debug.FreeOSMemory() - } + // if !s.syncing { + // debug.FreeOSMemory() + // } // TODO: database pruning needs to be refactored since the trie is now stored by nodes } @@ -104,12 +104,12 @@ func (s *StorageState) pruneKey(keyHeader *types.Header) { func (s *StorageState) StoreTrie(ts *rtstorage.TrieState) error { s.lock.Lock() root := ts.MustRoot() - if s.syncing { - // keep only the trie at the head of the chain when syncing - for key := range s.tries { - delete(s.tries, key) - } - } + // if s.syncing { + // // keep only the trie at the head of the chain when syncing + // for key := range s.tries { + // delete(s.tries, key) + // } + // } s.tries[root] = ts.Trie() s.lock.Unlock() @@ -127,9 +127,9 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState) error { }() // this is turned on when near the head, since it makes syncing significantly slower - if !s.syncing { - debug.FreeOSMemory() - } + // if !s.syncing { + // debug.FreeOSMemory() + // } return nil } From 9892130925f73795c407ae00d6c1db6bab4e402b Mon Sep 17 00:00:00 2001 From: noot Date: Thu, 25 Mar 2021 12:27:55 -0400 Subject: [PATCH 08/10] re-add only keeping trie at head when syncing --- dot/node.go | 6 +++++- dot/state/storage.go | 19 ++++++------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/dot/node.go b/dot/node.go index 224557e5e6..34d0abcdcb 100644 --- a/dot/node.go +++ b/dot/node.go @@ -160,7 +160,11 @@ func NodeInitialized(basepath string, expected bool) bool { // NewNode creates a new dot node from a dot node configuration func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, error) { // set garbage collection percent to 10% - debug.SetGCPercent(10) + // can be overwritten by setting the GOGC env veriable, which defaults to 100 + prev := debug.SetGCPercent(10) + if prev != 100 { + debug.SetGCPercent(prev) + } setupLogger(cfg) diff --git a/dot/state/storage.go b/dot/state/storage.go index 7cb3f84929..10e2ffe56f 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -94,9 +94,6 @@ func (s *StorageState) pruneKey(keyHeader *types.Header) { } delete(s.tries, keyHeader.StateRoot) - // if !s.syncing { - // debug.FreeOSMemory() - // } // TODO: database pruning needs to be refactored since the trie is now stored by nodes } @@ -104,12 +101,12 @@ func (s *StorageState) pruneKey(keyHeader *types.Header) { func (s *StorageState) StoreTrie(ts *rtstorage.TrieState) error { s.lock.Lock() root := ts.MustRoot() - // if s.syncing { - // // keep only the trie at the head of the chain when syncing - // for key := range s.tries { - // delete(s.tries, key) - // } - // } + if s.syncing { + // keep only the trie at the head of the chain when syncing + for key := range s.tries { + delete(s.tries, key) + } + } s.tries[root] = ts.Trie() s.lock.Unlock() @@ -126,10 +123,6 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState) error { } }() - // this is turned on when near the head, since it makes syncing significantly slower - // if !s.syncing { - // debug.FreeOSMemory() - // } return nil } From cc33293a32a5e15569c1ba739951a04ffc7b0157 Mon Sep 17 00:00:00 2001 From: noot Date: Thu, 25 Mar 2021 13:48:20 -0400 Subject: [PATCH 09/10] lint, add tests --- dot/state/storage.go | 1 - dot/state/storage_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/dot/state/storage.go b/dot/state/storage.go index 10e2ffe56f..10b21ccd30 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -20,7 +20,6 @@ import ( "encoding/binary" "errors" "fmt" - //"runtime/debug" "sync" "github.com/ChainSafe/chaindb" diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 5c55c4e5c8..43f4b9ba4f 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -120,3 +120,33 @@ func TestStorage_LoadFromDB(t *testing.T) { require.NoError(t, err) require.Equal(t, 3, len(entries)) } + +func TestStorage_StoreTrie_Syncing(t *testing.T) { + storage := newTestStorageState(t) + ts, err := storage.TrieState(&trie.EmptyHash) + require.NoError(t, err) + + key := []byte("testkey") + value := []byte("testvalue") + ts.Set(key, value) + + storage.SetSyncing(true) + err = storage.StoreTrie(ts) + require.NoError(t, err) + require.Equal(t, 1, len(storage.tries)) +} + +func TestStorage_StoreTrie_NotSyncing(t *testing.T) { + storage := newTestStorageState(t) + ts, err := storage.TrieState(&trie.EmptyHash) + require.NoError(t, err) + + key := []byte("testkey") + value := []byte("testvalue") + ts.Set(key, value) + + storage.SetSyncing(false) + err = storage.StoreTrie(ts) + require.NoError(t, err) + require.Equal(t, 2, len(storage.tries)) +} From cc43689fa5932d4dd545851b51cdd080c5a46dc0 Mon Sep 17 00:00:00 2001 From: noot Date: Thu, 25 Mar 2021 15:30:56 -0400 Subject: [PATCH 10/10] fix rpc test --- tests/rpc/rpc_01-system_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rpc/rpc_01-system_test.go b/tests/rpc/rpc_01-system_test.go index 48c4d2ab1d..b64641b1a0 100644 --- a/tests/rpc/rpc_01-system_test.go +++ b/tests/rpc/rpc_01-system_test.go @@ -60,7 +60,7 @@ func TestSystemRPC(t *testing.T) { expected: modules.SystemHealthResponse{ Peers: 2, - IsSyncing: false, + IsSyncing: true, ShouldHavePeers: true, }, params: "{}",