From 18b87986fcef43981c0e2cb6dbde71da0ac16c73 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Mon, 17 May 2021 20:23:39 +0530 Subject: [PATCH 01/18] Implement state trie online pruning. --- go.mod | 2 +- go.sum | 35 ++--------- lib/trie/database.go | 38 ++++++++++++ lib/trie/node.go | 14 ++++- lib/trie/trie.go | 131 +++++++++++++++++++++--------------------- lib/trie/trie_test.go | 80 ++++++++++++++++++++++++++ 6 files changed, 202 insertions(+), 98 deletions(-) diff --git a/go.mod b/go.mod index 30a8b0fb26..66882b536a 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cosmos/go-bip39 v1.0.0 github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f // indirect github.com/dgraph-io/badger/v2 v2.2007.2 - github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d + github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de github.com/disiqueira/gotree v1.0.0 github.com/docker/docker v1.13.1 github.com/elastic/gosigar v0.14.0 // indirect diff --git a/go.sum b/go.sum index 13ce535da6..3ebd2acf63 100644 --- a/go.sum +++ b/go.sum @@ -51,13 +51,11 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -77,9 +75,8 @@ github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLI github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d h1:eQYOG6A4td1tht0NdJB9Ls6DsXRGb2Ft6X9REU/MbbE= -github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d/go.mod h1:tv2ec8nA7vRpSYX7/MbP52ihrUMXIHit54CQMq8npXQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -106,7 +103,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-delve/delve v1.5.0/go.mod h1:c6b3a1Gry6x8a4LGCe/CWzrocrfaHvkUxCj3k4bvSUQ= github.com/go-interpreter/wagon v0.6.0 h1:BBxDxjiJiHgw9EdkYXAWs8NHhwnazZ5P2EWBW5hFNWw= github.com/go-interpreter/wagon v0.6.0/go.mod h1:5+b/MBYkclRZngKF5s6qrgWxSLgE9F5dFdO1hAueZLc= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= @@ -152,7 +148,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-dap v0.2.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= @@ -472,13 +467,11 @@ github.com/libp2p/go-yamux v1.4.0 h1:7nqe0T95T2CWh40IdJ/tp8RMor4ubc9/wYZpB2a/Hx0 github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -501,7 +494,6 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mmcloughlin/avo v0.0.0-20201105074841-5d2f697d268f/go.mod h1:6aKT4zZIrpGqB3RpFU14ByCSSyKY6LfJz4J/JJChHfI= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -589,7 +581,6 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/perlin-network/life v0.0.0-20191203030451-05c0e0f7eaea h1:okKoivlkNRRLqXraEtatHfEhW+D71QTwkaj+4n4M2Xc= github.com/perlin-network/life v0.0.0-20191203030451-05c0e0f7eaea/go.mod h1:3KEU5Dm8MAYWZqity880wOFJ9PhQjyKVZGwAEfc5Q4E= -github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/phuslu/iploc v1.0.20200807 h1:LIBm2Y9l5zmUvnJhQgMcLZ0iVwuG+5/L6AgbMwSOpE4= github.com/phuslu/iploc v1.0.20200807/go.mod h1:Q/0VX0txvbxekt4NhWIi3Q3eyZ139lHhnlzvDxyXhuc= github.com/pierrec/xxHash v0.1.5 h1:n/jBpwTHiER4xYvK3/CdPVnLDPchj8eTJFFLUb4QHBo= @@ -619,10 +610,8 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= @@ -640,9 +629,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D6 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= +github.com/twitchyliquid64/golang-asm v0.0.0-20190126203739-365674df15fc h1:RTUQlKzoZZVG3umWNzOYeFecQLIh+dbxXvJp1zPQJTI= github.com/twitchyliquid64/golang-asm v0.0.0-20190126203739-365674df15fc/go.mod h1:NoCfSFWosfqMqmmD7hApkirIK9ozpHjxRnRxs1l413A= -github.com/twitchyliquid64/golang-asm v0.15.0 h1:WYZ15YKpC5xM8PwpBTDsAgemoLB/lyhRkzJSEw9eAew= -github.com/twitchyliquid64/golang-asm v0.15.0/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -662,7 +650,6 @@ github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1: github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -670,7 +657,6 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.starlark.net v0.0.0-20190702223751-32f345186213/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -686,8 +672,6 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= -golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -717,8 +701,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNT golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -731,17 +713,15 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -769,7 +749,6 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= @@ -790,12 +769,9 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174 h1:0rx0F4EjJNbxTuzWe0KjKcIzs+3VEb/Mrs/d1ciNz1c= -golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -854,4 +830,3 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/lib/trie/database.go b/lib/trie/database.go index 447c53171e..90f310eac3 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -287,3 +287,41 @@ func (t *Trie) writeDirty(db chaindb.Batch, curr node) error { curr.setDirty(false) return nil } + +func (t *Trie) getInsertedNodeHashes(curr node) ([]common.Hash, error) { + var nodeHashes []common.Hash + if curr == nil || !curr.isDirty() { + return nil, nil + } + + enc, hash, err := curr.encodeAndHash() + if err != nil { + return nil, err + } + + if curr == t.root { + h, err := common.Blake2bHash(enc) //nolint + if err != nil { + return nil, err + } + + hash = h[:] + } + + nodeHashes = append(nodeHashes, common.BytesToHash(hash)) + + if c, ok := curr.(*branch); ok { + for _, child := range c.children { + if child == nil { + continue + } + nodes, err := t.getInsertedNodeHashes(child) + if err != nil { + return nil, err + } + nodeHashes = append(nodeHashes, nodes...) + } + } + + return nodeHashes, nil +} diff --git a/lib/trie/node.go b/lib/trie/node.go index f011cdf6cc..d3ef64954b 100644 --- a/lib/trie/node.go +++ b/lib/trie/node.go @@ -62,6 +62,8 @@ type node interface { setEncodingAndHash([]byte, []byte) getHash() []byte getGeneration() uint64 + setGeneration(uint64) + copy() node } type ( @@ -86,7 +88,15 @@ type ( } ) -func (b *branch) copy() *branch { +func (b *branch) setGeneration(generation uint64) { + b.generation = generation +} + +func (l *leaf) setGeneration(generation uint64) { + l.generation = generation +} + +func (b *branch) copy() node { cpy := &branch{ key: make([]byte, len(b.key)), children: b.children, @@ -109,7 +119,7 @@ func (b *branch) copy() *branch { return cpy } -func (l *leaf) copy() *leaf { +func (l *leaf) copy() node { cpy := &leaf{ key: make([]byte, len(l.key)), value: make([]byte, len(l.value)), diff --git a/lib/trie/trie.go b/lib/trie/trie.go index d1db892612..ee779dda41 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -29,9 +29,10 @@ var EmptyHash, _ = NewEmptyTrie().Hash() // The zero value is an empty trie with no database. // Use NewTrie to create a trie that sits on top of a database. type Trie struct { - generation uint64 - root node - childTries map[common.Hash]*Trie // Used to store the child tries. + generation uint64 + root node + childTries map[common.Hash]*Trie // Used to store the child tries. + deletedKeys []common.Hash } // NewEmptyTrie creates a trie with a nil root @@ -42,42 +43,42 @@ func NewEmptyTrie() *Trie { // NewTrie creates a trie with an existing root node func NewTrie(root node) *Trie { return &Trie{ - root: root, - childTries: make(map[common.Hash]*Trie), - generation: 0, // Initially zero but increases after every snapshot. + root: root, + childTries: make(map[common.Hash]*Trie), + generation: 0, // Initially zero but increases after every snapshot. + deletedKeys: make([]common.Hash, 0), } } // Snapshot created a copy of the trie. func (t *Trie) Snapshot() *Trie { oldTrie := &Trie{ - generation: t.generation, - root: t.root, - childTries: t.childTries, + generation: t.generation, + root: t.root, + childTries: t.childTries, + deletedKeys: t.deletedKeys, } + t.generation++ + t.deletedKeys = make([]common.Hash, 0) return oldTrie } -func (t *Trie) maybeUpdateLeafGeneration(n *leaf) *leaf { - // Make a copy if the generation is updated. - if n.getGeneration() < t.generation { - // Insert a new leaf node in the current generation. - newLeaf := n.copy() - newLeaf.generation = t.generation - return newLeaf +func (t *Trie) maybeUpdateGeneration(n node) node { + if n == nil { + return nil } - return n -} -func (t *Trie) maybeUpdateBranchGeneration(n *branch) *branch { // Make a copy if the generation is updated. if n.getGeneration() < t.generation { - // Insert a new branch node in the current generation. - newBranch := n.copy() - newBranch.generation = t.generation - return newBranch + // Insert a new node in the current generation. + newNode := n.copy() + newNode.setGeneration(t.generation) + + t.deletedKeys = append(t.deletedKeys, common.BytesToHash(n.getHash())) + return newNode } + return n } @@ -246,14 +247,15 @@ func (t *Trie) tryPut(key, value []byte) { // TryPut attempts to insert a key with value into the trie func (t *Trie) insert(parent node, key []byte, value node) node { + parent = t.maybeUpdateGeneration(parent) + switch p := parent.(type) { case *branch: - nn := t.maybeUpdateBranchGeneration(p) - n := t.updateBranch(nn, key, value) + n := t.updateBranch(p, key, value) - if nn != nil && n != nil && n.isDirty() { + if p != nil && n != nil && n.isDirty() { // TODO: set all `Copy` nodes as dirty? - nn.setDirty(true) + p.setDirty(true) } return n case nil: @@ -267,22 +269,20 @@ func (t *Trie) insert(parent node, key []byte, value node) node { return v } case *leaf: - nn := t.maybeUpdateLeafGeneration(p) - // if a value already exists in the trie at this key, overwrite it with the new value // if the values are the same, don't mark node dirty - if nn.value != nil && bytes.Equal(nn.key, key) { - if !bytes.Equal(value.(*leaf).value, nn.value) { - nn.value = value.(*leaf).value - nn.dirty = true + if p.value != nil && bytes.Equal(p.key, key) { + if !bytes.Equal(value.(*leaf).value, p.value) { + p.value = value.(*leaf).value + p.dirty = true } - return nn + return p } - length := lenCommonPrefix(key, nn.key) + length := lenCommonPrefix(key, p.key) // need to convert this leaf into a branch br := &branch{key: key[:length], dirty: true, generation: t.generation} - parentKey := nn.key + parentKey := p.key // value goes at this branch if len(key) == length { @@ -291,9 +291,9 @@ func (t *Trie) insert(parent node, key []byte, value node) node { // if we are not replacing previous leaf, then add it as a child to the new branch if len(parentKey) > len(key) { - nn.key = nn.key[length+1:] - br.children[parentKey[length]] = nn - nn.setDirty(true) + p.key = p.key[length+1:] + br.children[parentKey[length]] = p + p.setDirty(true) } return br @@ -301,16 +301,16 @@ func (t *Trie) insert(parent node, key []byte, value node) node { value.setKey(key[length+1:]) - if length == len(nn.key) { + if length == len(p.key) { // if leaf's key is covered by this branch, then make the leaf's // value the value at this branch - br.value = nn.value + br.value = p.value br.children[key[length]] = value } else { // otherwise, make the leaf a child of the branch and update its partial key - nn.key = nn.key[length+1:] - nn.setDirty(true) - br.children[parentKey[length]] = nn + p.key = p.key[length+1:] + p.setDirty(true) + br.children[parentKey[length]] = p br.children[key[length]] = value } @@ -516,6 +516,7 @@ func (t *Trie) ClearPrefix(prefix []byte) { } func (t *Trie) clearPrefix(curr node, prefix []byte) (node, bool) { + curr = t.maybeUpdateGeneration(curr) switch c := curr.(type) { case *branch: length := lenCommonPrefix(c.key, prefix) @@ -526,14 +527,13 @@ func (t *Trie) clearPrefix(curr node, prefix []byte) (node, bool) { } // Store the current node and return it, if the trie is not updated. - nn := t.maybeUpdateBranchGeneration(c) - if len(prefix) == len(nn.key)+1 && length == len(prefix)-1 { + if len(prefix) == len(c.key)+1 && length == len(prefix)-1 { // found prefix at child index, delete child - i := prefix[len(nn.key)] - nn.children[i] = nil - nn.setDirty(true) - curr = handleDeletion(nn, prefix) + i := prefix[len(c.key)] + c.children[i] = nil + c.setDirty(true) + curr = handleDeletion(c, prefix) return curr, true } @@ -543,12 +543,12 @@ func (t *Trie) clearPrefix(curr node, prefix []byte) (node, bool) { } var wasUpdated bool - i := prefix[len(nn.key)] + i := prefix[len(c.key)] - nn.children[i], wasUpdated = t.clearPrefix(nn.children[i], prefix[len(nn.key)+1:]) + c.children[i], wasUpdated = t.clearPrefix(c.children[i], prefix[len(c.key)+1:]) if wasUpdated { - nn.setDirty(true) - curr = handleDeletion(nn, prefix) + c.setDirty(true) + curr = handleDeletion(c, prefix) } return curr, curr.isDirty() @@ -572,28 +572,28 @@ func (t *Trie) Delete(key []byte) { } func (t *Trie) delete(parent node, key []byte) (node, bool) { + // Store the current node and return it, if the trie is not updated. + parent = t.maybeUpdateGeneration(parent) switch p := parent.(type) { case *branch: - // Store the current node and return it, if the trie is not updated. - nn := t.maybeUpdateBranchGeneration(p) - length := lenCommonPrefix(nn.key, key) - if bytes.Equal(nn.key, key) || len(key) == 0 { + length := lenCommonPrefix(p.key, key) + if bytes.Equal(p.key, key) || len(key) == 0 { // found the value at this node - nn.value = nil - nn.setDirty(true) - return handleDeletion(nn, key), true + p.value = nil + p.setDirty(true) + return handleDeletion(p, key), true } - n, del := t.delete(nn.children[key[length]], key[length+1:]) + n, del := t.delete(p.children[key[length]], key[length+1:]) if !del { // If nothing was deleted then don't copy the path. return p, false } - nn.children[key[length]] = n - nn.setDirty(true) - n = handleDeletion(nn, key) + p.children[key[length]] = n + p.setDirty(true) + n = handleDeletion(p, key) return n, true case *leaf: if bytes.Equal(key, p.key) || len(key) == 0 { @@ -652,6 +652,7 @@ func handleDeletion(p *branch, key []byte) node { // do nothing } n.setDirty(true) + } return n } diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 795ead0d62..ba3739fc48 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -29,6 +29,7 @@ import ( "strings" "testing" + "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" "github.com/stretchr/testify/require" @@ -469,6 +470,85 @@ func TestDeleteOddKeyLengths(t *testing.T) { runTests(t, trie, tests) } +func TestTrieDiff(t *testing.T) { + testDataDirPath, _ := ioutil.TempDir(t.TempDir(), "test-badger-datadir") + defer os.RemoveAll(testDataDirPath) + + cfg := &chaindb.Config{ + DataDir: testDataDirPath, + InMemory: false, + } + + db, err := chaindb.NewBadgerDB(cfg) + require.NoError(t, err) + + defer db.Close() + trie := NewEmptyTrie() + + var testKey = []byte{0x01, 0x78} + var testValue = []byte("test") + + tests := []Test{ + {key: testKey, value: testValue}, + {key: []byte{0x01, 0x78, 0x32}, value: []byte("nootagain")}, + {key: []byte{0x06, 0xaf, 0xb1}, value: []byte("odd")}, + } + + for _, test := range tests { + trie.Put(test.key, test.value) + } + + trie.Delete([]byte{0x01, 0x78, 0x32}) + + deleteKeys := trie.deletedKeys + oldTrie := trie.Snapshot() + + err = oldTrie.Store(db) + require.NoError(t, err) + + // testKey is in both trie have same value for testKey + require.Equal(t, oldTrie.Get(testKey), trie.Get(testKey)) + + var newTestValue = []byte("test1") + tests = []Test{ + {key: testKey, value: newTestValue}, + {key: []byte{0x01, 0x78, 0x99}, value: []byte("test2")}, + {key: []byte{0x01, 0x78, 0x88}, value: []byte("test3")}, + {key: []byte{0x01, 0x78, 0x88, 0x66}, value: []byte("test4")}, + {key: []byte{0x06, 0xaf, 0xb1}, value: []byte("even")}, + } + + for _, test := range tests { + trie.Put(test.key, test.value) + } + + // testKey value changed in trie + require.NotEqual(t, oldTrie.Get(testKey), trie.Get(testKey)) + + trie.Delete([]byte{0x01, 0x78, 0x88, 0x66}) + for _, key := range deleteKeys { + trie.Delete(key.ToBytes()) + } + + expected := []Test{ + {key: testKey, value: newTestValue}, + {key: []byte{0x01, 0x78, 0x99}, value: []byte("test2")}, + {key: []byte{0x01, 0x78, 0x88}, value: []byte("test3")}, + {key: []byte{0x06, 0xaf, 0xb1}, value: []byte("even")}, + } + + nodes, err := trie.getInsertedNodeHashes(trie.root) + require.NoError(t, err) + + // total inserted keys + require.Equal(t, len(tests), len(nodes)) + + for _, exp := range expected { + val := trie.Get(exp.key) + require.Equal(t, val, exp.value) + } +} + func TestDelete(t *testing.T) { trie := NewEmptyTrie() From 4b49d262a601681197208b202e3b685899254b0b Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Wed, 19 May 2021 14:39:43 +0530 Subject: [PATCH 02/18] Test state trie diff --- dot/core/interface.go | 1 + dot/state/storage.go | 93 +++++++++++++++++- dot/sync/interface.go | 5 + dot/sync/pruner.go | 181 ++++++++++++++++++++++++++++++++++++ dot/sync/syncer.go | 16 ++++ lib/runtime/storage/trie.go | 12 +++ lib/trie/database.go | 13 ++- lib/trie/trie.go | 9 +- lib/trie/trie_test.go | 62 +++++------- 9 files changed, 345 insertions(+), 47 deletions(-) create mode 100644 dot/sync/pruner.go diff --git a/dot/core/interface.go b/dot/core/interface.go index bab25e3099..1faeef524f 100644 --- a/dot/core/interface.go +++ b/dot/core/interface.go @@ -57,6 +57,7 @@ type StorageState interface { LoadCodeHash(root *common.Hash) (common.Hash, error) TrieState(root *common.Hash) (*rtstorage.TrieState, error) GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error) + //StoreJournal(record *JournalRecord)error } // TransactionState is the interface for transaction state methods diff --git a/dot/state/storage.go b/dot/state/storage.go index ea9d997808..0bfe6f6b88 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -26,11 +26,14 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" + "github.com/ChainSafe/gossamer/lib/scale" "github.com/ChainSafe/gossamer/lib/trie" ) // storagePrefix storage key prefix. var storagePrefix = "storage" +var journalPrefix = "journal" +var lastPruned = "last_pruned" var codeKey = common.CodeKey // ErrTrieDoesNotExist is returned when attempting to interact with a trie that is not stored in the StorageState @@ -45,8 +48,9 @@ type StorageState struct { blockState *BlockState tries map[common.Hash]*trie.Trie // map of root -> trie - db chaindb.Database - lock sync.RWMutex + db chaindb.Database + journalDB chaindb.Database + lock sync.RWMutex // change notifiers changedLock sync.RWMutex @@ -72,6 +76,8 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie) blockState: blockState, tries: tries, db: chaindb.NewTable(db, storagePrefix), + journalDB: chaindb.NewTable(db, journalPrefix), + //lastPrunedDB: chaindb.NewTable(db, lastPruned), observerList: []Observer{}, }, nil } @@ -383,3 +389,86 @@ func (s *StorageState) pruneStorage(closeCh chan interface{}) { } } } + +func (s *StorageState) StoreJournal(num uint64, record []byte) error { + s.lock.Lock() + defer s.lock.Unlock() + + encNum,err := scale.Encode(num) + if err != nil { + return err + } + + err = s.journalDB.Put(encNum, record) + if err != nil { + return err + } + + return nil +} + +func (s *StorageState) GetJournalRecord(num uint64) ([]byte,error) { + s.lock.Lock() + defer s.lock.Unlock() + + encNum,err := scale.Encode(num) + if err != nil { + return nil,err + } + + val,err := s.journalDB.Get(encNum) + if err != nil { + return nil,err + } + + return val,nil +} + +func (s *StorageState) StoreLastPrunedIndex(blockNum uint64)error{ + s.lock.Lock() + defer s.lock.Unlock() + + encNum,err := scale.Encode(blockNum) + if err != nil{ + return err + } + + err = s.journalDB.Put([]byte("lastPruned"),encNum) + if err != nil { + return err + } + + return nil +} + +func (s *StorageState) GetLastPrunedIndex()(uint64, error){ + s.lock.RLock() + defer s.lock.RUnlock() + + val, err := s.journalDB.Get([]byte("lastPruned")) + if err != nil { + return 0, err + } + + var blockNum uint64 + _,err = scale.Decode(val,blockNum) + if err != nil { + return 0,err + } + + return blockNum,err +} + +func (s *StorageState) DeleteKeys(nodesHash map[*common.Hash]uint64) error { + s.lock.RLock() + defer s.lock.RUnlock() + + for k := range nodesHash { + err := s.db.Del(k.ToBytes()) + if err != nil { + return err + } + } + + return nil +} \ No newline at end of file diff --git a/dot/sync/interface.go b/dot/sync/interface.go index 2836b9e384..704eb18c7b 100644 --- a/dot/sync/interface.go +++ b/dot/sync/interface.go @@ -54,6 +54,11 @@ type StorageState interface { StoreTrie(ts *rtstorage.TrieState) error LoadCodeHash(*common.Hash) (common.Hash, error) SetSyncing(bool) + StoreJournal(uint64, []byte) error + GetJournalRecord(uint64) ([]byte, error) + StoreLastPrunedIndex(uint64) error + GetLastPrunedIndex()(uint64,error) + DeleteKeys(keys map[*common.Hash]uint64)error } // TransactionState is the interface for transaction queue methods diff --git a/dot/sync/pruner.go b/dot/sync/pruner.go new file mode 100644 index 0000000000..04a89d600c --- /dev/null +++ b/dot/sync/pruner.go @@ -0,0 +1,181 @@ +package sync + +import ( + "fmt" + "math/big" + "sync" + + "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/runtime/storage" + "github.com/ChainSafe/gossamer/lib/scale" +) + +type JournalRecord struct { + blockHash *common.Hash + // Hash of keys that are inserted into state trie of the block + insertedKeys []*common.Hash + // Hash of keys that are deleted from state trie of the block + deletedKeys []*common.Hash + + lock sync.RWMutex +} + +type DeathRow struct { + blockHash *common.Hash + deletedKeys map[*common.Hash]uint64 // keys hash that will be deleted from DB + lock sync.RWMutex +} + +type pruner struct { + deathList []*DeathRow + deathIndex map[*common.Hash]uint64 + pendingNumber uint64 + sync.Mutex +} + +func createJournalRecord(hash *common.Hash, insertedKeys, deletedKeys []*common.Hash) *JournalRecord { + return &JournalRecord{ + blockHash: hash, + insertedKeys: insertedKeys, + deletedKeys: deletedKeys, + } +} + +func newPruner(s StorageState) (*pruner, error) { + blockNum, err := s.GetLastPrunedIndex() + if err != nil { + if err == chaindb.ErrKeyNotFound { + blockNum = 0 + } + return nil, err + } + + blockNum += 1 + + p := &pruner{ + deathList: make([]*DeathRow, 0), + deathIndex: make(map[*common.Hash]uint64, 0), + pendingNumber: blockNum, + } + + // load deathList and deathIndex from JournalRecord + for { + record, err := s.GetJournalRecord(blockNum) + if err != nil { + if err == chaindb.ErrKeyNotFound { + break + } + return nil, err + } + + jr := &JournalRecord{} + _, err = scale.Decode(record, jr) + if err != nil { + return nil, err + } + + + err = p.addDeathRow(jr, blockNum) + if err != nil { + return nil, err + } + + blockNum += 1 + } + + return p, nil +} + +func (p *pruner) storeJournalRecord(ts *storage.TrieState, s StorageState, blockHash *common.Hash, blockNum *big.Int) error { + insKeys, err := ts.GetInsertedNodeHashes() + if err != nil { + return fmt.Errorf("failed to get inserted keys for %d: %w", blockNum, err) + } + + delKeys := ts.GetDeletedNodeHashes() + + jr := createJournalRecord(blockHash, insKeys, delKeys) + + encRecord, err := scale.Encode(jr) + if err != nil { + return fmt.Errorf("failed to encode journal record %d: %w", blockNum, err) + } + + err = s.StoreJournal(blockNum.Uint64(), encRecord) + if err != nil { + return fmt.Errorf("failed to store journal record for %d: %w", blockNum, err) + } + + err = p.addDeathRow(jr, blockNum.Uint64()) + if err != nil { + return err + } + + return nil +} + +func (p *pruner) addDeathRow(jr *JournalRecord, blockNum uint64) error { + p.Lock() + defer p.Unlock() + + // remove re-inserted keys + for _, k := range jr.insertedKeys { + if num, ok := p.deathIndex[k]; ok { + delete(p.deathIndex, k) + delete(p.deathList[num-p.pendingNumber].deletedKeys, k) + } + } + + // add deleted keys from journal to death index + for _, k := range jr.deletedKeys { + p.deathIndex[k] = blockNum + } + + deletedKeys := make(map[*common.Hash]uint64) + for _, data := range jr.deletedKeys { + deletedKeys[data] = blockNum + } + + dr := &DeathRow{ + blockHash: jr.blockHash, + deletedKeys: deletedKeys, + } + + // add DeathRow to deathList + p.deathList = append(p.deathList, dr) + return nil +} + +func (p *pruner) pruneOne(s StorageState) { + p.Lock() + defer p.Unlock() + + for { + if len(p.deathList) < 1 { + logger.Error("%s", "trying to prune when there's nothing to prune") + } + + // pop first element from death list + dr := p.deathList[0] + err := s.DeleteKeys(dr.deletedKeys) + if err != nil { + logger.Error("pruner failed to delete keys") + continue + } + + for k := range dr.deletedKeys { + delete(p.deathIndex, k) + } + + err = s.StoreLastPrunedIndex(p.pendingNumber) //TODO: change lastPrunedIndex to lastPrunedIndex + 1 + if err != nil { + logger.Error("pruner failed to store last pruned index") + continue + } + + p.deathList = p.deathList[1:] + //p.pendingPruning += 1 + p.pendingNumber += 1 + } +} diff --git a/dot/sync/syncer.go b/dot/sync/syncer.go index 88497b49a4..52fd2c1b7c 100644 --- a/dot/sync/syncer.go +++ b/dot/sync/syncer.go @@ -56,6 +56,7 @@ type Service struct { // Consensus digest handling digestHandler DigestHandler + pruner *pruner } // Config is the configuration for the sync Service. @@ -102,6 +103,11 @@ func NewService(cfg *Config) (*Service, error) { return nil, err } + p, err := newPruner(cfg.StorageState) + if err != nil { + return nil, err + } + return &Service{ codeHash: codeHash, blockState: cfg.BlockState, @@ -114,6 +120,7 @@ func NewService(cfg *Config) (*Service, error) { runtime: cfg.Runtime, verifier: cfg.Verifier, digestHandler: cfg.DigestHandler, + pruner: p, }, nil } @@ -346,6 +353,13 @@ func (s *Service) handleBlock(block *types.Block) error { return fmt.Errorf("failed to execute block %d: %w", block.Header.Number, err) } + blockHash := block.Header.Hash() + + err = s.pruner.storeJournalRecord(ts, s.storageState, &blockHash, block.Header.Number) + if err != nil { + return err + } + err = s.storageState.StoreTrie(ts) if err != nil { return err @@ -372,6 +386,8 @@ func (s *Service) handleBlock(block *types.Block) error { s.handleDigests(block.Header) } + go s.pruner.pruneOne(s.storageState) + return s.handleRuntimeChanges(ts) } diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index 4ea5a018a7..64d556b43a 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -268,3 +268,15 @@ func (s *TrieState) LoadCodeHash() (common.Hash, error) { code := s.LoadCode() return common.Blake2bHash(code) } + +func (s *TrieState) GetInsertedNodeHashes() ([]*common.Hash, error) { + s.lock.RLock() + defer s.lock.RUnlock() + return s.t.GetInsertedNodeHashes(s.t.RootNode()) +} + +func (s *TrieState) GetDeletedNodeHashes() []*common.Hash { + s.lock.RLock() + defer s.lock.RUnlock() + return s.t.GetDeletedNodeHash() +} \ No newline at end of file diff --git a/lib/trie/database.go b/lib/trie/database.go index 90f310eac3..ead3869609 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -288,8 +288,8 @@ func (t *Trie) writeDirty(db chaindb.Batch, curr node) error { return nil } -func (t *Trie) getInsertedNodeHashes(curr node) ([]common.Hash, error) { - var nodeHashes []common.Hash +func (t *Trie) GetInsertedNodeHashes(curr node) ([]*common.Hash, error) { + var nodeHashes []*common.Hash if curr == nil || !curr.isDirty() { return nil, nil } @@ -308,14 +308,15 @@ func (t *Trie) getInsertedNodeHashes(curr node) ([]common.Hash, error) { hash = h[:] } - nodeHashes = append(nodeHashes, common.BytesToHash(hash)) + nodeHash := common.BytesToHash(hash) + nodeHashes = append(nodeHashes, &nodeHash) if c, ok := curr.(*branch); ok { for _, child := range c.children { if child == nil { continue } - nodes, err := t.getInsertedNodeHashes(child) + nodes, err := t.GetInsertedNodeHashes(child) if err != nil { return nil, err } @@ -325,3 +326,7 @@ func (t *Trie) getInsertedNodeHashes(curr node) ([]common.Hash, error) { return nodeHashes, nil } + +func (t *Trie) GetDeletedNodeHash() []*common.Hash { + return t.deletedKeys +} \ No newline at end of file diff --git a/lib/trie/trie.go b/lib/trie/trie.go index ee779dda41..3fcbada62b 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -32,7 +32,7 @@ type Trie struct { generation uint64 root node childTries map[common.Hash]*Trie // Used to store the child tries. - deletedKeys []common.Hash + deletedKeys []*common.Hash } // NewEmptyTrie creates a trie with a nil root @@ -46,7 +46,7 @@ func NewTrie(root node) *Trie { root: root, childTries: make(map[common.Hash]*Trie), generation: 0, // Initially zero but increases after every snapshot. - deletedKeys: make([]common.Hash, 0), + deletedKeys: make([]*common.Hash, 0), } } @@ -60,7 +60,7 @@ func (t *Trie) Snapshot() *Trie { } t.generation++ - t.deletedKeys = make([]common.Hash, 0) + t.deletedKeys = make([]*common.Hash, 0) return oldTrie } @@ -75,7 +75,8 @@ func (t *Trie) maybeUpdateGeneration(n node) node { newNode := n.copy() newNode.setGeneration(t.generation) - t.deletedKeys = append(t.deletedKeys, common.BytesToHash(n.getHash())) + hash := common.BytesToHash(n.getHash()) + t.deletedKeys = append(t.deletedKeys, &hash) return newNode } diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index ba3739fc48..896202d06c 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -482,71 +482,59 @@ func TestTrieDiff(t *testing.T) { db, err := chaindb.NewBadgerDB(cfg) require.NoError(t, err) + storageDB := chaindb.NewTable(db, "storage") + defer db.Close() trie := NewEmptyTrie() - var testKey = []byte{0x01, 0x78} - var testValue = []byte("test") + var testKey = []byte("testKey") tests := []Test{ - {key: testKey, value: testValue}, - {key: []byte{0x01, 0x78, 0x32}, value: []byte("nootagain")}, - {key: []byte{0x06, 0xaf, 0xb1}, value: []byte("odd")}, + {key: testKey, value: testKey}, + {key: []byte("testKey1"), value: []byte("testKey1")}, + {key: []byte("testKey2"), value: []byte("testKey2")}, } for _, test := range tests { trie.Put(test.key, test.value) } - trie.Delete([]byte{0x01, 0x78, 0x32}) - - deleteKeys := trie.deletedKeys oldTrie := trie.Snapshot() - - err = oldTrie.Store(db) + err = oldTrie.Store(storageDB) require.NoError(t, err) - // testKey is in both trie have same value for testKey - require.Equal(t, oldTrie.Get(testKey), trie.Get(testKey)) - - var newTestValue = []byte("test1") tests = []Test{ - {key: testKey, value: newTestValue}, - {key: []byte{0x01, 0x78, 0x99}, value: []byte("test2")}, - {key: []byte{0x01, 0x78, 0x88}, value: []byte("test3")}, - {key: []byte{0x01, 0x78, 0x88, 0x66}, value: []byte("test4")}, - {key: []byte{0x06, 0xaf, 0xb1}, value: []byte("even")}, + {key: testKey, value: []byte("newTestKey2")}, + {key: []byte("testKey2"), value: []byte("newKey")}, + {key: []byte("testKey3"), value: []byte("testKey3")}, + {key: []byte("testKey4"), value: []byte("testKey2")}, + {key: []byte("testKey5"), value: []byte("testKey5")}, } for _, test := range tests { trie.Put(test.key, test.value) } + deletedKeys := trie.deletedKeys - // testKey value changed in trie - require.NotEqual(t, oldTrie.Get(testKey), trie.Get(testKey)) + err = trie.WriteDirty(storageDB) + require.NoError(t, err) - trie.Delete([]byte{0x01, 0x78, 0x88, 0x66}) - for _, key := range deleteKeys { - trie.Delete(key.ToBytes()) + for _, key := range deletedKeys { + err = storageDB.Del(key.ToBytes()) + require.NoError(t, err) } - expected := []Test{ - {key: testKey, value: newTestValue}, - {key: []byte{0x01, 0x78, 0x99}, value: []byte("test2")}, - {key: []byte{0x01, 0x78, 0x88}, value: []byte("test3")}, - {key: []byte{0x06, 0xaf, 0xb1}, value: []byte("even")}, - } + newTrie := NewEmptyTrie() + err = newTrie.Load(storageDB, common.BytesToHash(trie.root.getHash())) + require.NoError(t, err) - nodes, err := trie.getInsertedNodeHashes(trie.root) + enc, err := trie.Encode() require.NoError(t, err) - // total inserted keys - require.Equal(t, len(tests), len(nodes)) + newEnc, err := newTrie.Encode() + require.NoError(t, err) - for _, exp := range expected { - val := trie.Get(exp.key) - require.Equal(t, val, exp.value) - } + require.Equal(t, enc, newEnc) } func TestDelete(t *testing.T) { From 2325f98531a8bc72d76000cf985a44b75cdd32c6 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Sat, 22 May 2021 17:21:54 +0530 Subject: [PATCH 03/18] Implement state trie online pruning --- cmd/gossamer/config.go | 4 + cmd/gossamer/export.go | 11 +- cmd/gossamer/main.go | 6 +- dot/config.go | 1 + dot/config/toml/config.go | 11 +- dot/core/interface.go | 1 - dot/node.go | 9 +- dot/services.go | 6 +- dot/services_test.go | 4 +- dot/state/{prune.go => offline_pruner.go} | 16 +- dot/state/pruner.go | 258 ++++++++++++++++++++++ dot/state/service.go | 4 + dot/state/storage.go | 93 +------- dot/sync/interface.go | 5 - dot/sync/pruner.go | 181 --------------- dot/sync/syncer.go | 22 +- dot/sync/syncer_test.go | 1 + lib/babe/babe.go | 20 +- lib/runtime/storage/trie.go | 4 +- lib/trie/database.go | 4 +- tests/utils/gossamer_utils.go | 9 +- 21 files changed, 351 insertions(+), 319 deletions(-) rename dot/state/{prune.go => offline_pruner.go} (92%) create mode 100644 dot/state/pruner.go delete mode 100644 dot/sync/pruner.go diff --git a/cmd/gossamer/config.go b/cmd/gossamer/config.go index f055a1db44..d172802d57 100644 --- a/cmd/gossamer/config.go +++ b/cmd/gossamer/config.go @@ -443,6 +443,7 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) { } cfg.MetricsPort = tomlCfg.Global.MetricsPort + cfg.RetainBlocks = tomlCfg.Global.RetainBlocks } } @@ -472,6 +473,9 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) { cfg.MetricsPort = uint32(metricsPort) } + // check --retain-blocks flag + cfg.RetainBlocks = ctx.GlobalInt64(RetainBlockNumberFlag.Name) + cfg.NoTelemetry = ctx.Bool("no-telemetry") } diff --git a/cmd/gossamer/export.go b/cmd/gossamer/export.go index fc6b5b06f1..90e7b8ea6a 100644 --- a/cmd/gossamer/export.go +++ b/cmd/gossamer/export.go @@ -79,11 +79,12 @@ func dotConfigToToml(dcfg *dot.Config) *ctoml.Config { cfg := &ctoml.Config{} cfg.Global = ctoml.GlobalConfig{ - Name: dcfg.Global.Name, - ID: dcfg.Global.ID, - BasePath: dcfg.Global.BasePath, - LogLvl: dcfg.Global.LogLvl.String(), - MetricsPort: dcfg.Global.MetricsPort, + Name: dcfg.Global.Name, + ID: dcfg.Global.ID, + BasePath: dcfg.Global.BasePath, + LogLvl: dcfg.Global.LogLvl.String(), + MetricsPort: dcfg.Global.MetricsPort, + RetainBlocks: dcfg.Global.RetainBlocks, } cfg.Log = ctoml.LogConfig{ diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index 46fe0f7409..623bbb6c63 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -234,6 +234,10 @@ func gossamerAction(ctx *cli.Context) error { return err } + if cfg.Global.RetainBlocks < 256 { + return fmt.Errorf("retain blocks cannot be less than 256") + } + cfg.Global.LogLvl = lvl // expand data directory and update node configuration (performed separately @@ -448,7 +452,7 @@ func pruneState(ctx *cli.Context) error { return err } - logger.Info("Pruner initialised") + logger.Info("Offline pruner initialised") err = pruner.SetBloomFilter() if err != nil { diff --git a/dot/config.go b/dot/config.go index 8741d943ae..07554245f5 100644 --- a/dot/config.go +++ b/dot/config.go @@ -52,6 +52,7 @@ type GlobalConfig struct { PublishMetrics bool MetricsPort uint32 NoTelemetry bool + RetainBlocks int64 } // LogConfig represents the log levels for individual packages diff --git a/dot/config/toml/config.go b/dot/config/toml/config.go index 2097b65801..ec84c537a2 100644 --- a/dot/config/toml/config.go +++ b/dot/config/toml/config.go @@ -29,11 +29,12 @@ type Config struct { // GlobalConfig is to marshal/unmarshal toml global config vars type GlobalConfig struct { - Name string `toml:"name,omitempty"` - ID string `toml:"id,omitempty"` - BasePath string `toml:"basepath,omitempty"` - LogLvl string `toml:"log,omitempty"` - MetricsPort uint32 `toml:"metrics-port,omitempty"` + Name string `toml:"name,omitempty"` + ID string `toml:"id,omitempty"` + BasePath string `toml:"basepath,omitempty"` + LogLvl string `toml:"log,omitempty"` + MetricsPort uint32 `toml:"metrics-port,omitempty"` + RetainBlocks int64 `toml:"retain-blocks,omitempty"` } // LogConfig represents the log levels for individual packages diff --git a/dot/core/interface.go b/dot/core/interface.go index 1faeef524f..bab25e3099 100644 --- a/dot/core/interface.go +++ b/dot/core/interface.go @@ -57,7 +57,6 @@ type StorageState interface { LoadCodeHash(root *common.Hash) (common.Hash, error) TrieState(root *common.Hash) (*rtstorage.TrieState, error) GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error) - //StoreJournal(record *JournalRecord)error } // TransactionState is the interface for transaction state methods diff --git a/dot/node.go b/dot/node.go index 59444d47e5..2ab8dfe0a9 100644 --- a/dot/node.go +++ b/dot/node.go @@ -237,6 +237,11 @@ func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, // Network Service var networkSrvc *network.Service + pruner, err := state.CreatePruner(stateSrvc.DB(), cfg.Global.RetainBlocks) + if err != nil { + logger.Error("failed to create pruner:", err) + } + // check if network service is enabled if enabled := networkServiceEnabled(cfg); enabled { // create network service and append network service to node services @@ -262,7 +267,7 @@ func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, } // create BABE service - bp, err := createBABEService(cfg, rt, stateSrvc, ks.Babe) + bp, err := createBABEService(cfg, rt, stateSrvc, ks.Babe, pruner) if err != nil { return nil, err } @@ -283,7 +288,7 @@ func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, nodeSrvcs = append(nodeSrvcs, fg) // Syncer - syncer, err := createSyncService(cfg, stateSrvc, bp, fg, dh, ver, rt) + syncer, err := createSyncService(cfg, stateSrvc, bp, fg, dh, ver, rt, pruner) if err != nil { return nil, err } diff --git a/dot/services.go b/dot/services.go index 8f8092720c..b4241f07c4 100644 --- a/dot/services.go +++ b/dot/services.go @@ -165,7 +165,7 @@ func createRuntime(cfg *Config, st *state.Service, ks *keystore.GlobalKeystore, return rt, nil } -func createBABEService(cfg *Config, rt runtime.Instance, st *state.Service, ks keystore.Keystore) (*babe.Service, error) { +func createBABEService(cfg *Config, rt runtime.Instance, st *state.Service, ks keystore.Keystore, pruner *state.Pruner) (*babe.Service, error) { logger.Info( "creating BABE service...", "authority", cfg.Core.BabeAuthority, @@ -192,6 +192,7 @@ func createBABEService(cfg *Config, rt runtime.Instance, st *state.Service, ks k SlotDuration: cfg.Core.SlotDuration, // TODO: remove this, should only be modified via runtime constant Authority: cfg.Core.BabeAuthority, IsDev: cfg.Global.ID == "dev", + Pruner: pruner, } if cfg.Core.BabeAuthority { @@ -376,7 +377,7 @@ func createBlockVerifier(st *state.Service) (*babe.VerificationManager, error) { return ver, nil } -func createSyncService(cfg *Config, st *state.Service, bp sync.BlockProducer, fg sync.FinalityGadget, dh *core.DigestHandler, verifier *babe.VerificationManager, rt runtime.Instance) (*sync.Service, error) { +func createSyncService(cfg *Config, st *state.Service, bp sync.BlockProducer, fg sync.FinalityGadget, dh *core.DigestHandler, verifier *babe.VerificationManager, rt runtime.Instance, pruner *state.Pruner) (*sync.Service, error) { syncCfg := &sync.Config{ LogLvl: cfg.Log.SyncLvl, BlockState: st.Block, @@ -387,6 +388,7 @@ func createSyncService(cfg *Config, st *state.Service, bp sync.BlockProducer, fg Verifier: verifier, Runtime: rt, DigestHandler: dh, + Pruner: pruner, } return sync.NewService(syncCfg) diff --git a/dot/services_test.go b/dot/services_test.go index 2a6746d186..a727c34f29 100644 --- a/dot/services_test.go +++ b/dot/services_test.go @@ -136,7 +136,7 @@ func TestCreateSyncService(t *testing.T) { ver, err := createBlockVerifier(stateSrvc) require.NoError(t, err) - _, err = createSyncService(cfg, stateSrvc, nil, nil, nil, ver, rt) + _, err = createSyncService(cfg, stateSrvc, nil, nil, nil, ver, rt, nil) require.NoError(t, err) } @@ -233,7 +233,7 @@ func TestCreateBABEService(t *testing.T) { rt, err := createRuntime(cfg, stateSrvc, ks, &network.Service{}) require.NoError(t, err) - bs, err := createBABEService(cfg, rt, stateSrvc, ks.Babe) + bs, err := createBABEService(cfg, rt, stateSrvc, ks.Babe, nil) require.NoError(t, err) require.NotNil(t, bs) } diff --git a/dot/state/prune.go b/dot/state/offline_pruner.go similarity index 92% rename from dot/state/prune.go rename to dot/state/offline_pruner.go index 2df24d848a..878626f383 100644 --- a/dot/state/prune.go +++ b/dot/state/offline_pruner.go @@ -14,11 +14,11 @@ import ( "github.com/dgraph-io/badger/v2/pb" ) -// Pruner is an offline tool to prune the stale state with the help of -// bloom filter, The workflow of pruner is very simple: +// OfflinePruner is a tool to prune the stale state with the help of +// bloom filter, The workflow of Pruner is very simple: // - iterate the storage state, reconstruct the relevant state tries // - iterate the database, stream all the targeted keys to new DB -type Pruner struct { +type OfflinePruner struct { inputDB *chaindb.BadgerDB storageState *StorageState blockState *BlockState @@ -30,8 +30,8 @@ type Pruner struct { prunedDBPath string } -// NewPruner creates an instance of Pruner. -func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNum int64) (*Pruner, error) { +// NewPruner creates an instance of OfflinePruner. +func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNum int64) (*OfflinePruner, error) { db, err := utils.LoadChainDB(inputDBPath) if err != nil { return nil, fmt.Errorf("failed to load DB %w", err) @@ -67,7 +67,7 @@ func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNu return nil, fmt.Errorf("failed to create new storage state %w", err) } - return &Pruner{ + return &OfflinePruner{ inputDB: db, storageState: storageState, blockState: blockState, @@ -80,7 +80,7 @@ func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNu } // SetBloomFilter loads keys with storage prefix of last `retainBlockNum` blocks into the bloom filter -func (p *Pruner) SetBloomFilter() error { +func (p *OfflinePruner) SetBloomFilter() error { defer p.inputDB.Close() // nolint: errcheck finalisedHash, err := p.blockState.GetFinalizedHash(0, 0) if err != nil { @@ -134,7 +134,7 @@ func (p *Pruner) SetBloomFilter() error { } // Prune starts streaming the data from input db to the pruned db. -func (p *Pruner) Prune() error { +func (p *OfflinePruner) Prune() error { inputDB, err := utils.LoadBadgerDB(p.inputDBPath) if err != nil { return fmt.Errorf("failed to load DB %w", err) diff --git a/dot/state/pruner.go b/dot/state/pruner.go new file mode 100644 index 0000000000..40ba42ff92 --- /dev/null +++ b/dot/state/pruner.go @@ -0,0 +1,258 @@ +package state + +import ( + "fmt" + "math/big" + "sync" + "time" + + "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/scale" +) + +const ( + journalPrefix = "journal" + lastPruned = "last_pruned" +) + +type journalRecord struct { + blockHash *common.Hash + // Hash of keys that are inserted into state trie of the block + insertedKeys []*common.Hash + // Hash of keys that are deleted from state trie of the block + deletedKeys []*common.Hash +} + +type deathRow struct { + blockHash *common.Hash + deletedKeys map[*common.Hash]int64 // keys hash that will be deleted from DB +} + +// Pruner stores state trie diff and allows online state trie pruning +type Pruner struct { + deathList []*deathRow + storageDB chaindb.Database + journalDB chaindb.Database + deathIndex map[*common.Hash]int64 + pendingNumber int64 + retainBlocks int64 + sync.RWMutex +} + +func newJournalRecord(hash *common.Hash, insertedKeys, deletedKeys []*common.Hash) *journalRecord { + return &journalRecord{ + blockHash: hash, + insertedKeys: insertedKeys, + deletedKeys: deletedKeys, + } +} + +// CreatePruner creates a pruner +func CreatePruner(db chaindb.Database, retainBlocks int64) (*Pruner, error) { + p := &Pruner{ + deathList: make([]*deathRow, 0), + deathIndex: make(map[*common.Hash]int64), + storageDB: chaindb.NewTable(db, storagePrefix), + journalDB: chaindb.NewTable(db, journalPrefix), + retainBlocks: retainBlocks, + } + + blockNum, err := p.getLastPrunedIndex() + if err != nil { + if err == chaindb.ErrKeyNotFound { + blockNum = 0 + } else { + logger.Error("pruner", "failed to get last pruned key", err) + return nil, err + } + } + + logger.Info("pruner", "last pruned block", blockNum) + blockNum++ + + p.pendingNumber = blockNum + + // load deathList and deathIndex from journalRecord + for { + record, err := p.getJournalRecord(blockNum) + if err != nil { + if err == chaindb.ErrKeyNotFound { + break + } + return nil, err + } + + jr, err := scale.Decode(record, new(journalRecord)) + if err != nil { + return nil, err + } + + j := jr.(journalRecord) + err = p.addDeathRow(&j, blockNum) + if err != nil { + return nil, err + } + + blockNum++ + } + + return p, nil +} + +// StoreJournalRecord stores journal record into DB and add deathRow into deathList +func (p *Pruner) StoreJournalRecord(deleted, inserted []*common.Hash, blockHash *common.Hash, blockNum *big.Int) error { + jr := newJournalRecord(blockHash, inserted, deleted) + encRecord, err := scale.Encode(jr) + if err != nil { + return fmt.Errorf("failed to encode journal record %d: %w", blockNum, err) + } + + err = p.storeJournal(blockNum.Int64(), encRecord) + if err != nil { + return fmt.Errorf("failed to store journal record for %d: %w", blockNum, err) + } + + logger.Info("journal record stored") + err = p.addDeathRow(jr, blockNum.Int64()) + if err != nil { + return err + } + + return nil +} + +func (p *Pruner) addDeathRow(jr *journalRecord, blockNum int64) error { + p.Lock() + defer p.Unlock() + + // remove re-inserted keys + for _, k := range jr.insertedKeys { + if num, ok := p.deathIndex[k]; ok { + delete(p.deathIndex, k) + delete(p.deathList[num-p.pendingNumber].deletedKeys, k) + } + } + + // add deleted keys from journal to death index + for _, k := range jr.deletedKeys { + p.deathIndex[k] = blockNum + } + + deletedKeys := make(map[*common.Hash]int64) + for _, data := range jr.deletedKeys { + deletedKeys[data] = blockNum + } + + dr := &deathRow{ + blockHash: jr.blockHash, + deletedKeys: deletedKeys, + } + + // add deathRow to deathList + p.deathList = append(p.deathList, dr) + return nil +} + +// PruneOne starts online pruning process +func (p *Pruner) PruneOne() { + logger.Info("pruning started") + + for { + if int64(len(p.deathList)) <= p.retainBlocks { + time.Sleep(2 * time.Second) + continue + } + + logger.Info("pruner", "pruning block ", p.pendingNumber) + p.Lock() + + // pop first element from death list + dr := p.deathList[0] + err := p.deleteKeys(dr.deletedKeys) + if err != nil { + logger.Error("pruner", "failed to delete keys for block", p.pendingNumber) + continue + } + + for k := range dr.deletedKeys { + delete(p.deathIndex, k) + } + + err = p.storeLastPrunedIndex(p.pendingNumber) + if err != nil { + logger.Error("pruner", "failed to store last pruned index") + } + + p.deathList = p.deathList[1:] + p.pendingNumber++ + p.Unlock() + } +} + +func (p *Pruner) storeJournal(num int64, record []byte) error { + encNum, err := scale.Encode(num) + if err != nil { + return err + } + + err = p.journalDB.Put(encNum, record) + if err != nil { + return err + } + + return nil +} + +func (p *Pruner) getJournalRecord(num int64) ([]byte, error) { + encNum, err := scale.Encode(num) + if err != nil { + return nil, err + } + + val, err := p.journalDB.Get(encNum) + if err != nil { + return nil, err + } + + return val, nil +} + +func (p *Pruner) storeLastPrunedIndex(blockNum int64) error { + encNum, err := scale.Encode(blockNum) + if err != nil { + return err + } + + err = p.journalDB.Put([]byte(lastPruned), encNum) + if err != nil { + return err + } + + return nil +} + +func (p *Pruner) getLastPrunedIndex() (int64, error) { + val, err := p.journalDB.Get([]byte(lastPruned)) + if err != nil { + return 0, err + } + + blockNum, err := scale.Decode(val, int64(0)) + if err != nil { + return 0, err + } + + return blockNum.(int64), err +} + +func (p *Pruner) deleteKeys(nodesHash map[*common.Hash]int64) error { + for k := range nodesHash { + err := p.storageDB.Del(k.ToBytes()) + if err != nil { + return err + } + } + + return nil +} diff --git a/dot/state/service.go b/dot/state/service.go index dc3788d9bd..1160fdb405 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -51,6 +51,8 @@ type Service struct { // Below are for testing only. BabeThresholdNumerator uint64 BabeThresholdDenominator uint64 + + pruner *Pruner } // NewService create a new instance of Service @@ -67,6 +69,7 @@ func NewService(path string, lvl log.Lvl) *Service { Storage: nil, Block: nil, closeCh: make(chan interface{}), + pruner: nil, } } @@ -177,6 +180,7 @@ func (s *Service) Start() error { logger.Info("created state service", "head", s.Block.BestBlockHash(), "highest number", num) // Start background goroutine to GC pruned keys. go s.Storage.pruneStorage(s.closeCh) + return nil } diff --git a/dot/state/storage.go b/dot/state/storage.go index 0bfe6f6b88..ea9d997808 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -26,14 +26,11 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/scale" "github.com/ChainSafe/gossamer/lib/trie" ) // storagePrefix storage key prefix. var storagePrefix = "storage" -var journalPrefix = "journal" -var lastPruned = "last_pruned" var codeKey = common.CodeKey // ErrTrieDoesNotExist is returned when attempting to interact with a trie that is not stored in the StorageState @@ -48,9 +45,8 @@ type StorageState struct { blockState *BlockState tries map[common.Hash]*trie.Trie // map of root -> trie - db chaindb.Database - journalDB chaindb.Database - lock sync.RWMutex + db chaindb.Database + lock sync.RWMutex // change notifiers changedLock sync.RWMutex @@ -76,8 +72,6 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie) blockState: blockState, tries: tries, db: chaindb.NewTable(db, storagePrefix), - journalDB: chaindb.NewTable(db, journalPrefix), - //lastPrunedDB: chaindb.NewTable(db, lastPruned), observerList: []Observer{}, }, nil } @@ -389,86 +383,3 @@ func (s *StorageState) pruneStorage(closeCh chan interface{}) { } } } - -func (s *StorageState) StoreJournal(num uint64, record []byte) error { - s.lock.Lock() - defer s.lock.Unlock() - - encNum,err := scale.Encode(num) - if err != nil { - return err - } - - err = s.journalDB.Put(encNum, record) - if err != nil { - return err - } - - return nil -} - -func (s *StorageState) GetJournalRecord(num uint64) ([]byte,error) { - s.lock.Lock() - defer s.lock.Unlock() - - encNum,err := scale.Encode(num) - if err != nil { - return nil,err - } - - val,err := s.journalDB.Get(encNum) - if err != nil { - return nil,err - } - - return val,nil -} - -func (s *StorageState) StoreLastPrunedIndex(blockNum uint64)error{ - s.lock.Lock() - defer s.lock.Unlock() - - encNum,err := scale.Encode(blockNum) - if err != nil{ - return err - } - - err = s.journalDB.Put([]byte("lastPruned"),encNum) - if err != nil { - return err - } - - return nil -} - -func (s *StorageState) GetLastPrunedIndex()(uint64, error){ - s.lock.RLock() - defer s.lock.RUnlock() - - val, err := s.journalDB.Get([]byte("lastPruned")) - if err != nil { - return 0, err - } - - var blockNum uint64 - _,err = scale.Decode(val,blockNum) - if err != nil { - return 0,err - } - - return blockNum,err -} - -func (s *StorageState) DeleteKeys(nodesHash map[*common.Hash]uint64) error { - s.lock.RLock() - defer s.lock.RUnlock() - - for k := range nodesHash { - err := s.db.Del(k.ToBytes()) - if err != nil { - return err - } - } - - return nil -} \ No newline at end of file diff --git a/dot/sync/interface.go b/dot/sync/interface.go index 704eb18c7b..2836b9e384 100644 --- a/dot/sync/interface.go +++ b/dot/sync/interface.go @@ -54,11 +54,6 @@ type StorageState interface { StoreTrie(ts *rtstorage.TrieState) error LoadCodeHash(*common.Hash) (common.Hash, error) SetSyncing(bool) - StoreJournal(uint64, []byte) error - GetJournalRecord(uint64) ([]byte, error) - StoreLastPrunedIndex(uint64) error - GetLastPrunedIndex()(uint64,error) - DeleteKeys(keys map[*common.Hash]uint64)error } // TransactionState is the interface for transaction queue methods diff --git a/dot/sync/pruner.go b/dot/sync/pruner.go deleted file mode 100644 index 04a89d600c..0000000000 --- a/dot/sync/pruner.go +++ /dev/null @@ -1,181 +0,0 @@ -package sync - -import ( - "fmt" - "math/big" - "sync" - - "github.com/ChainSafe/chaindb" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/scale" -) - -type JournalRecord struct { - blockHash *common.Hash - // Hash of keys that are inserted into state trie of the block - insertedKeys []*common.Hash - // Hash of keys that are deleted from state trie of the block - deletedKeys []*common.Hash - - lock sync.RWMutex -} - -type DeathRow struct { - blockHash *common.Hash - deletedKeys map[*common.Hash]uint64 // keys hash that will be deleted from DB - lock sync.RWMutex -} - -type pruner struct { - deathList []*DeathRow - deathIndex map[*common.Hash]uint64 - pendingNumber uint64 - sync.Mutex -} - -func createJournalRecord(hash *common.Hash, insertedKeys, deletedKeys []*common.Hash) *JournalRecord { - return &JournalRecord{ - blockHash: hash, - insertedKeys: insertedKeys, - deletedKeys: deletedKeys, - } -} - -func newPruner(s StorageState) (*pruner, error) { - blockNum, err := s.GetLastPrunedIndex() - if err != nil { - if err == chaindb.ErrKeyNotFound { - blockNum = 0 - } - return nil, err - } - - blockNum += 1 - - p := &pruner{ - deathList: make([]*DeathRow, 0), - deathIndex: make(map[*common.Hash]uint64, 0), - pendingNumber: blockNum, - } - - // load deathList and deathIndex from JournalRecord - for { - record, err := s.GetJournalRecord(blockNum) - if err != nil { - if err == chaindb.ErrKeyNotFound { - break - } - return nil, err - } - - jr := &JournalRecord{} - _, err = scale.Decode(record, jr) - if err != nil { - return nil, err - } - - - err = p.addDeathRow(jr, blockNum) - if err != nil { - return nil, err - } - - blockNum += 1 - } - - return p, nil -} - -func (p *pruner) storeJournalRecord(ts *storage.TrieState, s StorageState, blockHash *common.Hash, blockNum *big.Int) error { - insKeys, err := ts.GetInsertedNodeHashes() - if err != nil { - return fmt.Errorf("failed to get inserted keys for %d: %w", blockNum, err) - } - - delKeys := ts.GetDeletedNodeHashes() - - jr := createJournalRecord(blockHash, insKeys, delKeys) - - encRecord, err := scale.Encode(jr) - if err != nil { - return fmt.Errorf("failed to encode journal record %d: %w", blockNum, err) - } - - err = s.StoreJournal(blockNum.Uint64(), encRecord) - if err != nil { - return fmt.Errorf("failed to store journal record for %d: %w", blockNum, err) - } - - err = p.addDeathRow(jr, blockNum.Uint64()) - if err != nil { - return err - } - - return nil -} - -func (p *pruner) addDeathRow(jr *JournalRecord, blockNum uint64) error { - p.Lock() - defer p.Unlock() - - // remove re-inserted keys - for _, k := range jr.insertedKeys { - if num, ok := p.deathIndex[k]; ok { - delete(p.deathIndex, k) - delete(p.deathList[num-p.pendingNumber].deletedKeys, k) - } - } - - // add deleted keys from journal to death index - for _, k := range jr.deletedKeys { - p.deathIndex[k] = blockNum - } - - deletedKeys := make(map[*common.Hash]uint64) - for _, data := range jr.deletedKeys { - deletedKeys[data] = blockNum - } - - dr := &DeathRow{ - blockHash: jr.blockHash, - deletedKeys: deletedKeys, - } - - // add DeathRow to deathList - p.deathList = append(p.deathList, dr) - return nil -} - -func (p *pruner) pruneOne(s StorageState) { - p.Lock() - defer p.Unlock() - - for { - if len(p.deathList) < 1 { - logger.Error("%s", "trying to prune when there's nothing to prune") - } - - // pop first element from death list - dr := p.deathList[0] - err := s.DeleteKeys(dr.deletedKeys) - if err != nil { - logger.Error("pruner failed to delete keys") - continue - } - - for k := range dr.deletedKeys { - delete(p.deathIndex, k) - } - - err = s.StoreLastPrunedIndex(p.pendingNumber) //TODO: change lastPrunedIndex to lastPrunedIndex + 1 - if err != nil { - logger.Error("pruner failed to store last pruned index") - continue - } - - p.deathList = p.deathList[1:] - //p.pendingPruning += 1 - p.pendingNumber += 1 - } -} diff --git a/dot/sync/syncer.go b/dot/sync/syncer.go index 52fd2c1b7c..e48299811d 100644 --- a/dot/sync/syncer.go +++ b/dot/sync/syncer.go @@ -24,6 +24,7 @@ import ( "os" "github.com/ChainSafe/gossamer/dot/network" + "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/blocktree" @@ -56,7 +57,7 @@ type Service struct { // Consensus digest handling digestHandler DigestHandler - pruner *pruner + pruner *state.Pruner } // Config is the configuration for the sync Service. @@ -70,6 +71,8 @@ type Config struct { Runtime runtime.Instance Verifier Verifier DigestHandler DigestHandler + + Pruner *state.Pruner } // NewService returns a new *sync.Service @@ -103,11 +106,6 @@ func NewService(cfg *Config) (*Service, error) { return nil, err } - p, err := newPruner(cfg.StorageState) - if err != nil { - return nil, err - } - return &Service{ codeHash: codeHash, blockState: cfg.BlockState, @@ -120,7 +118,7 @@ func NewService(cfg *Config) (*Service, error) { runtime: cfg.Runtime, verifier: cfg.Verifier, digestHandler: cfg.DigestHandler, - pruner: p, + pruner: cfg.Pruner, }, nil } @@ -354,8 +352,14 @@ func (s *Service) handleBlock(block *types.Block) error { } blockHash := block.Header.Hash() + insKeys, err := ts.GetInsertedNodeHashes() + if err != nil { + logger.Error("failed to get state trie inserted keys: block ", block.Header.Number, err) + } + + delKeys := ts.GetDeletedNodeHashes() - err = s.pruner.storeJournalRecord(ts, s.storageState, &blockHash, block.Header.Number) + err = s.pruner.StoreJournalRecord(delKeys, insKeys, &blockHash, block.Header.Number) if err != nil { return err } @@ -386,7 +390,7 @@ func (s *Service) handleBlock(block *types.Block) error { s.handleDigests(block.Header) } - go s.pruner.pruneOne(s.storageState) + go s.pruner.PruneOne() return s.handleRuntimeChanges(ts) } diff --git a/dot/sync/syncer_test.go b/dot/sync/syncer_test.go index 7fae639cc8..7400446cfa 100644 --- a/dot/sync/syncer_test.go +++ b/dot/sync/syncer_test.go @@ -65,6 +65,7 @@ func newTestSyncer(t *testing.T) *Service { cfg := &Config{} testDatadirPath, _ := ioutil.TempDir("/tmp", "test-datadir-*") + stateSrvc := state.NewService(testDatadirPath, log.LvlInfo) stateSrvc.UseMemDB() diff --git a/lib/babe/babe.go b/lib/babe/babe.go index 072d22ada0..6675d9d0a7 100644 --- a/lib/babe/babe.go +++ b/lib/babe/babe.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/crypto/sr25519" "github.com/ChainSafe/gossamer/lib/runtime" @@ -49,6 +50,7 @@ type Service struct { transactionState TransactionState epochState EpochState epochLength uint64 + pruner *state.Pruner // BABE authority keypair keypair *sr25519.Keypair // TODO: change to BABE keystore @@ -86,6 +88,7 @@ type ServiceConfig struct { SlotDuration uint64 // for development purposes; in milliseconds EpochLength uint64 // for development purposes; in slots Authority bool + Pruner *state.Pruner } // NewService returns a new Babe Service using the provided VRF keys and runtime @@ -128,9 +131,9 @@ func NewService(cfg *ServiceConfig) (*Service, error) { pause: make(chan struct{}), authority: cfg.Authority, dev: cfg.IsDev, + pruner: cfg.Pruner, } - var err error genCfg, err := babeService.rt.BabeConfiguration() if err != nil { return nil, err @@ -411,6 +414,8 @@ func (b *Service) invokeBlockAuthoring(epoch uint64) { slotDone[i] = time.After(b.getSlotDuration() * time.Duration(i)) } + go b.pruner.PruneOne() + for i := 0; i < int(b.epochLength-intoEpoch); i++ { select { case <-b.ctx.Done(): @@ -488,6 +493,19 @@ func (b *Service) handleSlot(slotNum uint64) error { return nil } + blockHash := block.Header.Hash() + insKeys, err := ts.GetInsertedNodeHashes() + if err != nil { + logger.Error("failed to get inserted keys for ", block.Header.Number, err) + } + + delKeys := ts.GetDeletedNodeHashes() + + err = b.pruner.StoreJournalRecord(delKeys, insKeys, &blockHash, block.Header.Number) + if err != nil { + return err + } + old := ts.Snapshot() // block built successfully, store resulting trie in storage state diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index 64d556b43a..c1b4f373f4 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -269,14 +269,16 @@ func (s *TrieState) LoadCodeHash() (common.Hash, error) { return common.Blake2bHash(code) } +// GetInsertedNodeHashes returns the hash of nodes inserted into state trie since last block produced func (s *TrieState) GetInsertedNodeHashes() ([]*common.Hash, error) { s.lock.RLock() defer s.lock.RUnlock() return s.t.GetInsertedNodeHashes(s.t.RootNode()) } +// GetDeletedNodeHashes returns the hash of nodes that are deleted from state trie since last block produced func (s *TrieState) GetDeletedNodeHashes() []*common.Hash { s.lock.RLock() defer s.lock.RUnlock() return s.t.GetDeletedNodeHash() -} \ No newline at end of file +} diff --git a/lib/trie/database.go b/lib/trie/database.go index ead3869609..c66edb9a18 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -288,6 +288,7 @@ func (t *Trie) writeDirty(db chaindb.Batch, curr node) error { return nil } +// GetInsertedNodeHashes returns the hash of nodes that are inserted into state trie since last snapshot is called func (t *Trie) GetInsertedNodeHashes(curr node) ([]*common.Hash, error) { var nodeHashes []*common.Hash if curr == nil || !curr.isDirty() { @@ -327,6 +328,7 @@ func (t *Trie) GetInsertedNodeHashes(curr node) ([]*common.Hash, error) { return nodeHashes, nil } +// GetDeletedNodeHash returns the hash of nodes that are deleted from state trie since last snapshot is called func (t *Trie) GetDeletedNodeHash() []*common.Hash { return t.deletedKeys -} \ No newline at end of file +} diff --git a/tests/utils/gossamer_utils.go b/tests/utils/gossamer_utils.go index e8c170b774..b1ca3dcfd8 100644 --- a/tests/utils/gossamer_utils.go +++ b/tests/utils/gossamer_utils.go @@ -417,10 +417,11 @@ func GenerateGenesisSixAuth() { func generateDefaultConfig() *ctoml.Config { return &ctoml.Config{ Global: ctoml.GlobalConfig{ - Name: "Gossamer", - ID: "gssmr", - LogLvl: "crit", - MetricsPort: 9876, + Name: "Gossamer", + ID: "gssmr", + LogLvl: "crit", + MetricsPort: 9876, + RetainBlocks: 256, }, Log: ctoml.LogConfig{ CoreLvl: "info", From 238222dcb4dad40ef10dbaf42561bcd6b6818cfb Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 27 May 2021 11:34:02 +0530 Subject: [PATCH 04/18] Self review. --- cmd/gossamer/config.go | 5 +- cmd/gossamer/export.go | 1 + cmd/gossamer/flags.go | 8 +++ cmd/gossamer/main.go | 8 ++- dot/config.go | 1 + dot/config/toml/config.go | 1 + dot/node.go | 11 ++- dot/services.go | 4 +- dot/services_test.go | 5 +- dot/state/pruner.go | 122 +++++++++++++++++++--------------- dot/state/service.go | 4 -- dot/sync/syncer.go | 11 ++- dot/sync/syncer_test.go | 1 - lib/babe/babe.go | 10 ++- lib/trie/database.go | 1 + lib/trie/trie.go | 19 +++--- tests/utils/gossamer_utils.go | 1 + 17 files changed, 122 insertions(+), 91 deletions(-) diff --git a/cmd/gossamer/config.go b/cmd/gossamer/config.go index d172802d57..15eb95ce5b 100644 --- a/cmd/gossamer/config.go +++ b/cmd/gossamer/config.go @@ -443,7 +443,9 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) { } cfg.MetricsPort = tomlCfg.Global.MetricsPort + cfg.RetainBlocks = tomlCfg.Global.RetainBlocks + cfg.GCMode = tomlCfg.Global.GCMode } } @@ -473,9 +475,8 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) { cfg.MetricsPort = uint32(metricsPort) } - // check --retain-blocks flag cfg.RetainBlocks = ctx.GlobalInt64(RetainBlockNumberFlag.Name) - + cfg.GCMode = ctx.GlobalString(GCModeFlag.Name) cfg.NoTelemetry = ctx.Bool("no-telemetry") } diff --git a/cmd/gossamer/export.go b/cmd/gossamer/export.go index 90e7b8ea6a..670fdbf0ab 100644 --- a/cmd/gossamer/export.go +++ b/cmd/gossamer/export.go @@ -85,6 +85,7 @@ func dotConfigToToml(dcfg *dot.Config) *ctoml.Config { LogLvl: dcfg.Global.LogLvl.String(), MetricsPort: dcfg.Global.MetricsPort, RetainBlocks: dcfg.Global.RetainBlocks, + GCMode: dcfg.Global.GCMode, } cfg.Log = ctoml.LogConfig{ diff --git a/cmd/gossamer/flags.go b/cmd/gossamer/flags.go index ba3559234b..6d59f41f22 100644 --- a/cmd/gossamer/flags.go +++ b/cmd/gossamer/flags.go @@ -285,6 +285,13 @@ var ( Usage: "Retain number of block from latest block while pruning", Value: 256, } + + // GCModeFlag sets the blockchain GC mode. It's either full or archive. + GCModeFlag = cli.StringFlag{ + Name: "gcmode", + Usage: `Blockchain garbage collection mode ("full", "archive")`, + Value: "full", + } ) // flag sets that are shared by multiple commands @@ -302,6 +309,7 @@ var ( DBPathFlag, BloomFilterSizeFlag, RetainBlockNumberFlag, + GCModeFlag, } // StartupFlags are flags that are valid for use with the root command and the export subcommand diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index 623bbb6c63..58d486a851 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -234,8 +234,12 @@ func gossamerAction(ctx *cli.Context) error { return err } - if cfg.Global.RetainBlocks < 256 { - return fmt.Errorf("retain blocks cannot be less than 256") + if cfg.Global.RetainBlocks < 10 { + return fmt.Errorf("--%s cannot be less than 256", RetainBlockNumberFlag.Name) + } + + if cfg.Global.GCMode != "full" && cfg.Global.GCMode != "archive" { + return fmt.Errorf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } cfg.Global.LogLvl = lvl diff --git a/dot/config.go b/dot/config.go index 07554245f5..884a63b5fe 100644 --- a/dot/config.go +++ b/dot/config.go @@ -53,6 +53,7 @@ type GlobalConfig struct { MetricsPort uint32 NoTelemetry bool RetainBlocks int64 + GCMode string } // LogConfig represents the log levels for individual packages diff --git a/dot/config/toml/config.go b/dot/config/toml/config.go index ec84c537a2..7ca982f45f 100644 --- a/dot/config/toml/config.go +++ b/dot/config/toml/config.go @@ -35,6 +35,7 @@ type GlobalConfig struct { LogLvl string `toml:"log,omitempty"` MetricsPort uint32 `toml:"metrics-port,omitempty"` RetainBlocks int64 `toml:"retain-blocks,omitempty"` + GCMode string `toml:"gc-mode,omitempty"` } // LogConfig represents the log levels for individual packages diff --git a/dot/node.go b/dot/node.go index 2ab8dfe0a9..fbe5848ae9 100644 --- a/dot/node.go +++ b/dot/node.go @@ -237,9 +237,14 @@ func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, // Network Service var networkSrvc *network.Service - pruner, err := state.CreatePruner(stateSrvc.DB(), cfg.Global.RetainBlocks) - if err != nil { - logger.Error("failed to create pruner:", err) + var pruner state.Pruner + if cfg.Global.GCMode == "full" { + pruner, err = state.CreatePruner(stateSrvc.DB(), cfg.Global.RetainBlocks) + if err != nil { + return nil, err + } + } else { + pruner = &state.ArchivalNodePruner{} } // check if network service is enabled diff --git a/dot/services.go b/dot/services.go index b4241f07c4..2b6eac58fe 100644 --- a/dot/services.go +++ b/dot/services.go @@ -165,7 +165,7 @@ func createRuntime(cfg *Config, st *state.Service, ks *keystore.GlobalKeystore, return rt, nil } -func createBABEService(cfg *Config, rt runtime.Instance, st *state.Service, ks keystore.Keystore, pruner *state.Pruner) (*babe.Service, error) { +func createBABEService(cfg *Config, rt runtime.Instance, st *state.Service, ks keystore.Keystore, pruner state.Pruner) (*babe.Service, error) { logger.Info( "creating BABE service...", "authority", cfg.Core.BabeAuthority, @@ -377,7 +377,7 @@ func createBlockVerifier(st *state.Service) (*babe.VerificationManager, error) { return ver, nil } -func createSyncService(cfg *Config, st *state.Service, bp sync.BlockProducer, fg sync.FinalityGadget, dh *core.DigestHandler, verifier *babe.VerificationManager, rt runtime.Instance, pruner *state.Pruner) (*sync.Service, error) { +func createSyncService(cfg *Config, st *state.Service, bp sync.BlockProducer, fg sync.FinalityGadget, dh *core.DigestHandler, verifier *babe.VerificationManager, rt runtime.Instance, pruner state.Pruner) (*sync.Service, error) { syncCfg := &sync.Config{ LogLvl: cfg.Log.SyncLvl, BlockState: st.Block, diff --git a/dot/services_test.go b/dot/services_test.go index a727c34f29..6dfbc9efd2 100644 --- a/dot/services_test.go +++ b/dot/services_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/ChainSafe/gossamer/dot/network" + "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/utils" @@ -136,7 +137,7 @@ func TestCreateSyncService(t *testing.T) { ver, err := createBlockVerifier(stateSrvc) require.NoError(t, err) - _, err = createSyncService(cfg, stateSrvc, nil, nil, nil, ver, rt, nil) + _, err = createSyncService(cfg, stateSrvc, nil, nil, nil, ver, rt, &state.ArchivalNodePruner{}) require.NoError(t, err) } @@ -233,7 +234,7 @@ func TestCreateBABEService(t *testing.T) { rt, err := createRuntime(cfg, stateSrvc, ks, &network.Service{}) require.NoError(t, err) - bs, err := createBABEService(cfg, rt, stateSrvc, ks.Babe, nil) + bs, err := createBABEService(cfg, rt, stateSrvc, ks.Babe, &state.ArchivalNodePruner{}) require.NoError(t, err) require.NotNil(t, bs) } diff --git a/dot/state/pruner.go b/dot/state/pruner.go index 40ba42ff92..f3ceaa813f 100644 --- a/dot/state/pruner.go +++ b/dot/state/pruner.go @@ -13,24 +13,24 @@ import ( const ( journalPrefix = "journal" - lastPruned = "last_pruned" + lastPrunedKey = "last_pruned" ) -type journalRecord struct { - blockHash *common.Hash - // Hash of keys that are inserted into state trie of the block - insertedKeys []*common.Hash - // Hash of keys that are deleted from state trie of the block - deletedKeys []*common.Hash +// Pruner is implemented by FullNodePruner and ArchivalNodePruner. +type Pruner interface { + StoreJournalRecord(deleted, inserted []*common.Hash, blockHash *common.Hash, blockNum *big.Int) error } -type deathRow struct { - blockHash *common.Hash - deletedKeys map[*common.Hash]int64 // keys hash that will be deleted from DB +// ArchivalNodePruner is a no-op since we don't prune nodes in archive mode. +type ArchivalNodePruner struct{} + +// StoreJournalRecord for archive node doesn't do anything. +func (a *ArchivalNodePruner) StoreJournalRecord(deleted, inserted []*common.Hash, blockHash *common.Hash, blockNum *big.Int) error { + return nil } -// Pruner stores state trie diff and allows online state trie pruning -type Pruner struct { +// FullNodePruner stores state trie diff and allows online state trie pruning +type FullNodePruner struct { deathList []*deathRow storageDB chaindb.Database journalDB chaindb.Database @@ -40,6 +40,15 @@ type Pruner struct { sync.RWMutex } +type journalRecord struct { + // blockHash of the block corresponding to journal record + blockHash *common.Hash + // Hash of keys that are inserted into state trie of the block + insertedKeys []*common.Hash + // Hash of keys that are deleted from state trie of the block + deletedKeys []*common.Hash +} + func newJournalRecord(hash *common.Hash, insertedKeys, deletedKeys []*common.Hash) *journalRecord { return &journalRecord{ blockHash: hash, @@ -48,9 +57,14 @@ func newJournalRecord(hash *common.Hash, insertedKeys, deletedKeys []*common.Has } } +type deathRow struct { + blockHash *common.Hash + deletedKeys map[*common.Hash]int64 // keys hash that will be deleted from DB +} + // CreatePruner creates a pruner -func CreatePruner(db chaindb.Database, retainBlocks int64) (*Pruner, error) { - p := &Pruner{ +func CreatePruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { + p := &FullNodePruner{ deathList: make([]*deathRow, 0), deathIndex: make(map[*common.Hash]int64), storageDB: chaindb.NewTable(db, storagePrefix), @@ -60,12 +74,7 @@ func CreatePruner(db chaindb.Database, retainBlocks int64) (*Pruner, error) { blockNum, err := p.getLastPrunedIndex() if err != nil { - if err == chaindb.ErrKeyNotFound { - blockNum = 0 - } else { - logger.Error("pruner", "failed to get last pruned key", err) - return nil, err - } + return nil, err } logger.Info("pruner", "last pruned block", blockNum) @@ -76,20 +85,15 @@ func CreatePruner(db chaindb.Database, retainBlocks int64) (*Pruner, error) { // load deathList and deathIndex from journalRecord for { record, err := p.getJournalRecord(blockNum) - if err != nil { - if err == chaindb.ErrKeyNotFound { - break - } - return nil, err + if err == chaindb.ErrKeyNotFound { + break } - jr, err := scale.Decode(record, new(journalRecord)) if err != nil { return nil, err } - j := jr.(journalRecord) - err = p.addDeathRow(&j, blockNum) + err = p.addDeathRow(record, blockNum) if err != nil { return nil, err } @@ -97,23 +101,21 @@ func CreatePruner(db chaindb.Database, retainBlocks int64) (*Pruner, error) { blockNum++ } + go p.start() + return p, nil } // StoreJournalRecord stores journal record into DB and add deathRow into deathList -func (p *Pruner) StoreJournalRecord(deleted, inserted []*common.Hash, blockHash *common.Hash, blockNum *big.Int) error { +func (p *FullNodePruner) StoreJournalRecord(deleted, inserted []*common.Hash, blockHash *common.Hash, blockNum *big.Int) error { jr := newJournalRecord(blockHash, inserted, deleted) - encRecord, err := scale.Encode(jr) - if err != nil { - return fmt.Errorf("failed to encode journal record %d: %w", blockNum, err) - } - err = p.storeJournal(blockNum.Int64(), encRecord) + err := p.storeJournal(blockNum.Int64(), jr) if err != nil { return fmt.Errorf("failed to store journal record for %d: %w", blockNum, err) } - logger.Info("journal record stored") + logger.Info("journal record stored", "block", blockNum.Int64()) err = p.addDeathRow(jr, blockNum.Int64()) if err != nil { return err @@ -122,15 +124,15 @@ func (p *Pruner) StoreJournalRecord(deleted, inserted []*common.Hash, blockHash return nil } -func (p *Pruner) addDeathRow(jr *journalRecord, blockNum int64) error { +func (p *FullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) error { p.Lock() defer p.Unlock() // remove re-inserted keys for _, k := range jr.insertedKeys { if num, ok := p.deathIndex[k]; ok { - delete(p.deathIndex, k) delete(p.deathList[num-p.pendingNumber].deletedKeys, k) + delete(p.deathIndex, k) } } @@ -154,28 +156,28 @@ func (p *Pruner) addDeathRow(jr *journalRecord, blockNum int64) error { return nil } -// PruneOne starts online pruning process -func (p *Pruner) PruneOne() { +func (p *FullNodePruner) start() { logger.Info("pruning started") for { + p.Lock() if int64(len(p.deathList)) <= p.retainBlocks { + p.Unlock() time.Sleep(2 * time.Second) continue } logger.Info("pruner", "pruning block ", p.pendingNumber) - p.Lock() // pop first element from death list - dr := p.deathList[0] - err := p.deleteKeys(dr.deletedKeys) + row := p.deathList[0] + err := p.deleteKeys(row.deletedKeys) if err != nil { logger.Error("pruner", "failed to delete keys for block", p.pendingNumber) continue } - for k := range dr.deletedKeys { + for k := range row.deletedKeys { delete(p.deathIndex, k) } @@ -190,13 +192,18 @@ func (p *Pruner) PruneOne() { } } -func (p *Pruner) storeJournal(num int64, record []byte) error { - encNum, err := scale.Encode(num) +func (p *FullNodePruner) storeJournal(blockNum int64, jr *journalRecord) error { + encRecord, err := scale.Encode(jr) + if err != nil { + return fmt.Errorf("failed to encode journal record %d: %w", blockNum, err) + } + + encNum, err := scale.Encode(blockNum) if err != nil { return err } - err = p.journalDB.Put(encNum, record) + err = p.journalDB.Put(encNum, encRecord) if err != nil { return err } @@ -204,7 +211,7 @@ func (p *Pruner) storeJournal(num int64, record []byte) error { return nil } -func (p *Pruner) getJournalRecord(num int64) ([]byte, error) { +func (p *FullNodePruner) getJournalRecord(num int64) (*journalRecord, error) { encNum, err := scale.Encode(num) if err != nil { return nil, err @@ -215,16 +222,21 @@ func (p *Pruner) getJournalRecord(num int64) ([]byte, error) { return nil, err } - return val, nil + decJR, err := scale.Decode(val, new(journalRecord)) + if err != nil { + return nil, err + } + + return decJR.(*journalRecord), nil } -func (p *Pruner) storeLastPrunedIndex(blockNum int64) error { +func (p *FullNodePruner) storeLastPrunedIndex(blockNum int64) error { encNum, err := scale.Encode(blockNum) if err != nil { return err } - err = p.journalDB.Put([]byte(lastPruned), encNum) + err = p.journalDB.Put([]byte(lastPrunedKey), encNum) if err != nil { return err } @@ -232,8 +244,12 @@ func (p *Pruner) storeLastPrunedIndex(blockNum int64) error { return nil } -func (p *Pruner) getLastPrunedIndex() (int64, error) { - val, err := p.journalDB.Get([]byte(lastPruned)) +func (p *FullNodePruner) getLastPrunedIndex() (int64, error) { + val, err := p.journalDB.Get([]byte(lastPrunedKey)) + if err == chaindb.ErrKeyNotFound { + return 0, nil + } + if err != nil { return 0, err } @@ -246,7 +262,7 @@ func (p *Pruner) getLastPrunedIndex() (int64, error) { return blockNum.(int64), err } -func (p *Pruner) deleteKeys(nodesHash map[*common.Hash]int64) error { +func (p *FullNodePruner) deleteKeys(nodesHash map[*common.Hash]int64) error { for k := range nodesHash { err := p.storageDB.Del(k.ToBytes()) if err != nil { diff --git a/dot/state/service.go b/dot/state/service.go index 1160fdb405..dc3788d9bd 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -51,8 +51,6 @@ type Service struct { // Below are for testing only. BabeThresholdNumerator uint64 BabeThresholdDenominator uint64 - - pruner *Pruner } // NewService create a new instance of Service @@ -69,7 +67,6 @@ func NewService(path string, lvl log.Lvl) *Service { Storage: nil, Block: nil, closeCh: make(chan interface{}), - pruner: nil, } } @@ -180,7 +177,6 @@ func (s *Service) Start() error { logger.Info("created state service", "head", s.Block.BestBlockHash(), "highest number", num) // Start background goroutine to GC pruned keys. go s.Storage.pruneStorage(s.closeCh) - return nil } diff --git a/dot/sync/syncer.go b/dot/sync/syncer.go index e48299811d..8e69d2eaeb 100644 --- a/dot/sync/syncer.go +++ b/dot/sync/syncer.go @@ -57,7 +57,7 @@ type Service struct { // Consensus digest handling digestHandler DigestHandler - pruner *state.Pruner + pruner state.Pruner } // Config is the configuration for the sync Service. @@ -71,8 +71,7 @@ type Config struct { Runtime runtime.Instance Verifier Verifier DigestHandler DigestHandler - - Pruner *state.Pruner + Pruner state.Pruner } // NewService returns a new *sync.Service @@ -351,14 +350,14 @@ func (s *Service) handleBlock(block *types.Block) error { return fmt.Errorf("failed to execute block %d: %w", block.Header.Number, err) } - blockHash := block.Header.Hash() insKeys, err := ts.GetInsertedNodeHashes() if err != nil { - logger.Error("failed to get state trie inserted keys: block ", block.Header.Number, err) + return fmt.Errorf("failed to get state trie inserted keys: block %s %w", block.Header.Number, err) } delKeys := ts.GetDeletedNodeHashes() + blockHash := block.Header.Hash() err = s.pruner.StoreJournalRecord(delKeys, insKeys, &blockHash, block.Header.Number) if err != nil { return err @@ -390,8 +389,6 @@ func (s *Service) handleBlock(block *types.Block) error { s.handleDigests(block.Header) } - go s.pruner.PruneOne() - return s.handleRuntimeChanges(ts) } diff --git a/dot/sync/syncer_test.go b/dot/sync/syncer_test.go index 7400446cfa..7fae639cc8 100644 --- a/dot/sync/syncer_test.go +++ b/dot/sync/syncer_test.go @@ -65,7 +65,6 @@ func newTestSyncer(t *testing.T) *Service { cfg := &Config{} testDatadirPath, _ := ioutil.TempDir("/tmp", "test-datadir-*") - stateSrvc := state.NewService(testDatadirPath, log.LvlInfo) stateSrvc.UseMemDB() diff --git a/lib/babe/babe.go b/lib/babe/babe.go index 6675d9d0a7..e861ed2317 100644 --- a/lib/babe/babe.go +++ b/lib/babe/babe.go @@ -50,7 +50,7 @@ type Service struct { transactionState TransactionState epochState EpochState epochLength uint64 - pruner *state.Pruner + pruner state.Pruner // BABE authority keypair keypair *sr25519.Keypair // TODO: change to BABE keystore @@ -88,7 +88,7 @@ type ServiceConfig struct { SlotDuration uint64 // for development purposes; in milliseconds EpochLength uint64 // for development purposes; in slots Authority bool - Pruner *state.Pruner + Pruner state.Pruner } // NewService returns a new Babe Service using the provided VRF keys and runtime @@ -320,7 +320,7 @@ func (b *Service) safeSend(msg types.Block) error { defer b.lock.Unlock() if b.IsStopped() { - return errors.New("Service has been stopped") + return errors.New("service has been stopped") } b.blockChan <- msg @@ -414,8 +414,6 @@ func (b *Service) invokeBlockAuthoring(epoch uint64) { slotDone[i] = time.After(b.getSlotDuration() * time.Duration(i)) } - go b.pruner.PruneOne() - for i := 0; i < int(b.epochLength-intoEpoch); i++ { select { case <-b.ctx.Done(): @@ -493,7 +491,6 @@ func (b *Service) handleSlot(slotNum uint64) error { return nil } - blockHash := block.Header.Hash() insKeys, err := ts.GetInsertedNodeHashes() if err != nil { logger.Error("failed to get inserted keys for ", block.Header.Number, err) @@ -501,6 +498,7 @@ func (b *Service) handleSlot(slotNum uint64) error { delKeys := ts.GetDeletedNodeHashes() + blockHash := block.Header.Hash() err = b.pruner.StoreJournalRecord(delKeys, insKeys, &blockHash, block.Header.Number) if err != nil { return err diff --git a/lib/trie/database.go b/lib/trie/database.go index c66edb9a18..53787a8084 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -289,6 +289,7 @@ func (t *Trie) writeDirty(db chaindb.Batch, curr node) error { } // GetInsertedNodeHashes returns the hash of nodes that are inserted into state trie since last snapshot is called +// Since inserted nodesare newly created we need to compute their hash values. func (t *Trie) GetInsertedNodeHashes(curr node) ([]*common.Hash, error) { var nodeHashes []*common.Hash if curr == nil || !curr.isDirty() { diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 3fcbada62b..61bfeee002 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -75,8 +75,12 @@ func (t *Trie) maybeUpdateGeneration(n node) node { newNode := n.copy() newNode.setGeneration(t.generation) - hash := common.BytesToHash(n.getHash()) - t.deletedKeys = append(t.deletedKeys, &hash) + // Hash of old nodes should already be computed since it belongs to older generation. + oldNodeHash := n.getHash() + if len(oldNodeHash) > 0 { + hash := common.BytesToHash(oldNodeHash) + t.deletedKeys = append(t.deletedKeys, &hash) + } return newNode } @@ -248,9 +252,7 @@ func (t *Trie) tryPut(key, value []byte) { // TryPut attempts to insert a key with value into the trie func (t *Trie) insert(parent node, key []byte, value node) node { - parent = t.maybeUpdateGeneration(parent) - - switch p := parent.(type) { + switch p := t.maybeUpdateGeneration(parent).(type) { case *branch: n := t.updateBranch(p, key, value) @@ -516,8 +518,8 @@ func (t *Trie) ClearPrefix(prefix []byte) { t.root, _ = t.clearPrefix(t.root, p) } -func (t *Trie) clearPrefix(curr node, prefix []byte) (node, bool) { - curr = t.maybeUpdateGeneration(curr) +func (t *Trie) clearPrefix(cn node, prefix []byte) (node, bool) { + curr := t.maybeUpdateGeneration(cn) switch c := curr.(type) { case *branch: length := lenCommonPrefix(c.key, prefix) @@ -574,8 +576,7 @@ func (t *Trie) Delete(key []byte) { func (t *Trie) delete(parent node, key []byte) (node, bool) { // Store the current node and return it, if the trie is not updated. - parent = t.maybeUpdateGeneration(parent) - switch p := parent.(type) { + switch p := t.maybeUpdateGeneration(parent).(type) { case *branch: length := lenCommonPrefix(p.key, key) diff --git a/tests/utils/gossamer_utils.go b/tests/utils/gossamer_utils.go index b1ca3dcfd8..ebfe658e32 100644 --- a/tests/utils/gossamer_utils.go +++ b/tests/utils/gossamer_utils.go @@ -422,6 +422,7 @@ func generateDefaultConfig() *ctoml.Config { LogLvl: "crit", MetricsPort: 9876, RetainBlocks: 256, + GCMode: "archive", }, Log: ctoml.LogConfig{ CoreLvl: "info", From 52705d5624e48e6bceaeaee1497cc1f80119abf6 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 27 May 2021 18:04:01 +0530 Subject: [PATCH 05/18] Minor fix. --- cmd/gossamer/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index 58d486a851..1a2ed07019 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -234,7 +234,7 @@ func gossamerAction(ctx *cli.Context) error { return err } - if cfg.Global.RetainBlocks < 10 { + if cfg.Global.RetainBlocks < 256 { return fmt.Errorf("--%s cannot be less than 256", RetainBlockNumberFlag.Name) } From 00f54e63623b111c3c9100c68ec20329734b9c2f Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Mon, 31 May 2021 19:47:13 +0530 Subject: [PATCH 06/18] Fix pruning for non canoncial chain. --- dot/state/pruner.go | 180 ++++++++++++++++++++++++++++---------------- 1 file changed, 114 insertions(+), 66 deletions(-) diff --git a/dot/state/pruner.go b/dot/state/pruner.go index f3ceaa813f..39812353be 100644 --- a/dot/state/pruner.go +++ b/dot/state/pruner.go @@ -29,9 +29,16 @@ func (a *ArchivalNodePruner) StoreJournalRecord(deleted, inserted []*common.Hash return nil } +type deathRecord struct { + blockHash *common.Hash + deletedKeys map[*common.Hash]int64 // key hash that will be deleted from DB +} + +type deathRow []*deathRecord + // FullNodePruner stores state trie diff and allows online state trie pruning type FullNodePruner struct { - deathList []*deathRow + deathList []deathRow storageDB chaindb.Database journalDB chaindb.Database deathIndex map[*common.Hash]int64 @@ -57,15 +64,10 @@ func newJournalRecord(hash *common.Hash, insertedKeys, deletedKeys []*common.Has } } -type deathRow struct { - blockHash *common.Hash - deletedKeys map[*common.Hash]int64 // keys hash that will be deleted from DB -} - // CreatePruner creates a pruner func CreatePruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { p := &FullNodePruner{ - deathList: make([]*deathRow, 0), + deathList: make([]deathRow, 0), deathIndex: make(map[*common.Hash]int64), storageDB: chaindb.NewTable(db, storagePrefix), journalDB: chaindb.NewTable(db, journalPrefix), @@ -77,28 +79,14 @@ func CreatePruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { return nil, err } - logger.Info("pruner", "last pruned block", blockNum) + logger.Info("last pruned block", "block num", blockNum) blockNum++ p.pendingNumber = blockNum - // load deathList and deathIndex from journalRecord - for { - record, err := p.getJournalRecord(blockNum) - if err == chaindb.ErrKeyNotFound { - break - } - - if err != nil { - return nil, err - } - - err = p.addDeathRow(record, blockNum) - if err != nil { - return nil, err - } - - blockNum++ + err = p.loadDeathList() + if err != nil { + return nil, err } go p.start() @@ -110,32 +98,31 @@ func CreatePruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { func (p *FullNodePruner) StoreJournalRecord(deleted, inserted []*common.Hash, blockHash *common.Hash, blockNum *big.Int) error { jr := newJournalRecord(blockHash, inserted, deleted) - err := p.storeJournal(blockNum.Int64(), jr) + key := &journalKey{blockNum.Int64(), blockHash} + err := p.storeJournal(key, jr) if err != nil { return fmt.Errorf("failed to store journal record for %d: %w", blockNum, err) } - logger.Info("journal record stored", "block", blockNum.Int64()) - err = p.addDeathRow(jr, blockNum.Int64()) - if err != nil { - return err - } - + logger.Info("journal record stored", "block num", blockNum.Int64()) + p.addDeathRow(jr, blockNum.Int64()) return nil } -func (p *FullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) error { +func (p *FullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) { + if blockNum == 0 { + return + } + p.Lock() defer p.Unlock() - // remove re-inserted keys - for _, k := range jr.insertedKeys { - if num, ok := p.deathIndex[k]; ok { - delete(p.deathList[num-p.pendingNumber].deletedKeys, k) - delete(p.deathIndex, k) - } + if blockNum < p.pendingNumber { + return } + p.processInsertedKeys(jr.insertedKeys, jr.blockHash) + // add deleted keys from journal to death index for _, k := range jr.deletedKeys { p.deathIndex[k] = blockNum @@ -146,14 +133,33 @@ func (p *FullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) error { deletedKeys[data] = blockNum } - dr := &deathRow{ + blockIndex := blockNum - p.pendingNumber + for idx := blockIndex - int64(len(p.deathList)); idx >= 0; idx-- { + p.deathList = append(p.deathList, deathRow{}) + } + + record := &deathRecord{ blockHash: jr.blockHash, deletedKeys: deletedKeys, } // add deathRow to deathList - p.deathList = append(p.deathList, dr) - return nil + p.deathList[blockIndex] = append(p.deathList[blockIndex], record) +} + +// Remove re-inserted keys +func (p *FullNodePruner) processInsertedKeys(insKeys []*common.Hash, blockHash *common.Hash) { + for _, k := range insKeys { + if num, ok := p.deathIndex[k]; ok { + records := p.deathList[num-p.pendingNumber] + for _, v := range records { + if v.blockHash == blockHash { + delete(v.deletedKeys, k) + } + } + delete(p.deathIndex, k) + } + } } func (p *FullNodePruner) start() { @@ -167,43 +173,66 @@ func (p *FullNodePruner) start() { continue } - logger.Info("pruner", "pruning block ", p.pendingNumber) - // pop first element from death list row := p.deathList[0] - err := p.deleteKeys(row.deletedKeys) - if err != nil { - logger.Error("pruner", "failed to delete keys for block", p.pendingNumber) + blockNum := p.pendingNumber + if len(row) == 0 { + logger.Info("death row is empty", "block num", p.pendingNumber) continue } - for k := range row.deletedKeys { - delete(p.deathIndex, k) + blockHash := row[0].blockHash + logger.Info("pruning block", "block num", blockNum) + + for _, record := range row { + err := p.deleteKeys(record.deletedKeys) + if err != nil { + logger.Error("failed to prune keys", "block num", blockNum, "error", err) + continue + } + + for k := range record.deletedKeys { + delete(p.deathIndex, k) + } } - err = p.storeLastPrunedIndex(p.pendingNumber) + err := p.storeLastPrunedIndex(blockNum) if err != nil { - logger.Error("pruner", "failed to store last pruned index") + logger.Error("failed to store last pruned index", "block num", blockNum, "error", err) + continue } p.deathList = p.deathList[1:] + + jk := &journalKey{blockNum, blockHash} + err = p.deleteJournalRecord(jk) + if err != nil { + logger.Error("failed to delete journal record", "block num", blockNum, "error", err) + continue + } + p.pendingNumber++ p.Unlock() } } -func (p *FullNodePruner) storeJournal(blockNum int64, jr *journalRecord) error { - encRecord, err := scale.Encode(jr) +type journalKey struct { + blockNum int64 + blockHash *common.Hash +} + +func (p *FullNodePruner) storeJournal(key *journalKey, jr *journalRecord) error { + encKey, err := scale.Encode(key) if err != nil { - return fmt.Errorf("failed to encode journal record %d: %w", blockNum, err) + return fmt.Errorf("failed to encode journal key block num %d: %w", key.blockNum, err) } - encNum, err := scale.Encode(blockNum) + encRecord, err := scale.Encode(jr) if err != nil { - return err + return fmt.Errorf("failed to encode journal record block num %d: %w", key.blockNum, err) } - err = p.journalDB.Put(encNum, encRecord) + err = p.journalDB.Put(encKey, encRecord) if err != nil { return err } @@ -211,23 +240,42 @@ func (p *FullNodePruner) storeJournal(blockNum int64, jr *journalRecord) error { return nil } -func (p *FullNodePruner) getJournalRecord(num int64) (*journalRecord, error) { - encNum, err := scale.Encode(num) - if err != nil { - return nil, err +// loadDeathList loads deathList and deathIndex from journalRecord. +func (p *FullNodePruner) loadDeathList() error { + itr := p.journalDB.NewIterator() + defer itr.Release() + + for itr.Next() { + jk, err := scale.Decode(itr.Key(), new(journalKey)) + if err != nil { + return fmt.Errorf("failed to decode journal key %w", err) + } + + key := jk.(*journalKey) + val := itr.Value() + + jr, err := scale.Decode(val, new(journalRecord)) + if err != nil { + return fmt.Errorf("failed to decode journal record block num %d : %w", key.blockNum, err) + } + + p.addDeathRow(jr.(*journalRecord), key.blockNum) } + return nil +} - val, err := p.journalDB.Get(encNum) +func (p *FullNodePruner) deleteJournalRecord(key *journalKey) error { + encKey, err := scale.Encode(key) if err != nil { - return nil, err + return err } - decJR, err := scale.Decode(val, new(journalRecord)) + err = p.journalDB.Del(encKey) if err != nil { - return nil, err + return err } - return decJR.(*journalRecord), nil + return nil } func (p *FullNodePruner) storeLastPrunedIndex(blockNum int64) error { From 2b725f17a6625245e4c6ef194346a1ca8d348b64 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 1 Jun 2021 13:08:40 +0530 Subject: [PATCH 07/18] Move online pruner to storage state --- dot/build_spec.go | 2 +- dot/core/digest_test.go | 2 +- dot/core/test_helpers.go | 2 +- dot/import.go | 2 +- dot/node.go | 16 +++---------- dot/node_test.go | 2 +- dot/rpc/modules/chain_test.go | 2 +- dot/rpc/modules/state_test.go | 2 +- dot/rpc/modules/system_test.go | 2 +- dot/services.go | 8 +++---- dot/services_test.go | 5 ++-- dot/state/initialize.go | 2 +- dot/state/offline_pruner.go | 2 +- dot/state/pruner.go | 37 +++++++++++++++--------------- dot/state/service.go | 24 ++++++++++++-------- dot/state/service_test.go | 18 +++++++-------- dot/state/storage.go | 39 ++++++++++++++++++++++++++++---- dot/state/storage_notify_test.go | 6 ++--- dot/state/storage_test.go | 14 ++++++------ dot/sync/interface.go | 2 +- dot/sync/syncer.go | 15 +----------- dot/sync/syncer_test.go | 2 +- lib/babe/babe.go | 19 +--------------- lib/babe/babe_test.go | 2 +- lib/babe/state.go | 2 +- lib/babe/verify_test.go | 2 +- lib/runtime/storage/trie.go | 4 ++-- lib/trie/database.go | 8 +++---- lib/trie/trie.go | 8 +++---- 29 files changed, 122 insertions(+), 129 deletions(-) diff --git a/dot/build_spec.go b/dot/build_spec.go index 53e150f1e6..ee6d01ca1a 100644 --- a/dot/build_spec.go +++ b/dot/build_spec.go @@ -107,7 +107,7 @@ func BuildFromDB(path string) (*BuildSpec, error) { tmpGen.Genesis.Raw = make(map[string]map[string]string) tmpGen.Genesis.Runtime = make(map[string]map[string]interface{}) - stateSrvc := state.NewService(path, log.LvlCrit) + stateSrvc := state.NewService(path, log.LvlCrit, "", 0) // start state service (initialise state database) err := stateSrvc.Start() diff --git a/dot/core/digest_test.go b/dot/core/digest_test.go index 51f78e11ff..8d16d19183 100644 --- a/dot/core/digest_test.go +++ b/dot/core/digest_test.go @@ -35,7 +35,7 @@ import ( func newTestDigestHandler(t *testing.T, withBABE, withGrandpa bool) *DigestHandler { //nolint testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") require.NoError(t, err) - stateSrvc := state.NewService(testDatadirPath, log.LvlInfo) + stateSrvc := state.NewService(testDatadirPath, log.LvlInfo, "", 0) stateSrvc.UseMemDB() gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t) diff --git a/dot/core/test_helpers.go b/dot/core/test_helpers.go index 5366d4cf8f..5fc1975dba 100644 --- a/dot/core/test_helpers.go +++ b/dot/core/test_helpers.go @@ -130,7 +130,7 @@ func NewTestService(t *testing.T, cfg *Config) *Service { gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t) if cfg.BlockState == nil || cfg.StorageState == nil || cfg.TransactionState == nil || cfg.EpochState == nil { - stateSrvc = state.NewService(testDatadirPath, log.LvlInfo) + stateSrvc = state.NewService(testDatadirPath, log.LvlInfo, "", 0) stateSrvc.UseMemDB() err = stateSrvc.Initialise(gen, genHeader, genTrie) diff --git a/dot/import.go b/dot/import.go index 0a78503f82..e888c8680a 100644 --- a/dot/import.go +++ b/dot/import.go @@ -45,7 +45,7 @@ func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error { log.Info("ImportState", "header", header) - srv := state.NewService(basepath, log.LvlInfo) + srv := state.NewService(basepath, log.LvlInfo, "", 0) return srv.Import(header, tr, firstSlot) } diff --git a/dot/node.go b/dot/node.go index fbe5848ae9..79ad26218a 100644 --- a/dot/node.go +++ b/dot/node.go @@ -91,7 +91,7 @@ func InitNode(cfg *Config) error { } // create new state service - stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Global.LogLvl) + stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Global.LogLvl, "", 0) // initialise state service with genesis data, block, and trie err = stateSrvc.Initialise(gen, header, t) @@ -237,16 +237,6 @@ func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, // Network Service var networkSrvc *network.Service - var pruner state.Pruner - if cfg.Global.GCMode == "full" { - pruner, err = state.CreatePruner(stateSrvc.DB(), cfg.Global.RetainBlocks) - if err != nil { - return nil, err - } - } else { - pruner = &state.ArchivalNodePruner{} - } - // check if network service is enabled if enabled := networkServiceEnabled(cfg); enabled { // create network service and append network service to node services @@ -272,7 +262,7 @@ func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, } // create BABE service - bp, err := createBABEService(cfg, rt, stateSrvc, ks.Babe, pruner) + bp, err := createBABEService(cfg, rt, stateSrvc, ks.Babe) if err != nil { return nil, err } @@ -293,7 +283,7 @@ func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, nodeSrvcs = append(nodeSrvcs, fg) // Syncer - syncer, err := createSyncService(cfg, stateSrvc, bp, fg, dh, ver, rt, pruner) + syncer, err := createSyncService(cfg, stateSrvc, bp, fg, dh, ver, rt) if err != nil { return nil, err } diff --git a/dot/node_test.go b/dot/node_test.go index 29d331470b..61f9d78617 100644 --- a/dot/node_test.go +++ b/dot/node_test.go @@ -212,7 +212,7 @@ func TestInitNode_LoadGenesisData(t *testing.T) { err := InitNode(cfg) require.NoError(t, err) - stateSrvc := state.NewService(cfg.Global.BasePath, log.LvlTrace) + stateSrvc := state.NewService(cfg.Global.BasePath, log.LvlTrace, "", 0) gen, err := genesis.NewGenesisFromJSONRaw(genPath) require.NoError(t, err) diff --git a/dot/rpc/modules/chain_test.go b/dot/rpc/modules/chain_test.go index cb00fcf714..f008e33357 100644 --- a/dot/rpc/modules/chain_test.go +++ b/dot/rpc/modules/chain_test.go @@ -280,7 +280,7 @@ var gen, genTrie, genesisHeader = newTestGenesisWithTrieAndHeader() func newTestStateService(t *testing.T) *state.Service { testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") require.NoError(t, err) - stateSrvc := state.NewService(testDatadirPath, log.LvlInfo) + stateSrvc := state.NewService(testDatadirPath, log.LvlInfo, "", 0) stateSrvc.UseMemDB() err = stateSrvc.Initialise(gen, genesisHeader, genTrie) diff --git a/dot/rpc/modules/state_test.go b/dot/rpc/modules/state_test.go index f08dfc56a4..b5bed60398 100644 --- a/dot/rpc/modules/state_test.go +++ b/dot/rpc/modules/state_test.go @@ -429,7 +429,7 @@ func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) { sr1, err := ts.Root() require.NoError(t, err) - err = chain.Storage.StoreTrie(ts) + err = chain.Storage.StoreTrie(ts, nil) require.NoError(t, err) err = chain.Block.AddBlock(&types.Block{ diff --git a/dot/rpc/modules/system_test.go b/dot/rpc/modules/system_test.go index a632029709..ee2cd1a0dd 100644 --- a/dot/rpc/modules/system_test.go +++ b/dot/rpc/modules/system_test.go @@ -359,7 +359,7 @@ func setupSystemModule(t *testing.T) *SystemModule { require.NoError(t, err) ts.Set(aliceAcctStoKey, aliceAcctEncoded) - err = chain.Storage.StoreTrie(ts) + err = chain.Storage.StoreTrie(ts, nil) require.NoError(t, err) err = chain.Block.AddBlock(&types.Block{ Header: &types.Header{ diff --git a/dot/services.go b/dot/services.go index 2b6eac58fe..037f71d7d6 100644 --- a/dot/services.go +++ b/dot/services.go @@ -53,7 +53,7 @@ func newInMemoryDB(path string) (chaindb.Database, error) { // createStateService creates the state service and initialise state database func createStateService(cfg *Config) (*state.Service, error) { logger.Debug("creating state service...") - stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Log.StateLvl) + stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Log.StateLvl, cfg.Global.GCMode, cfg.Global.RetainBlocks) // start state service (initialise state database) err := stateSrvc.Start() @@ -165,7 +165,7 @@ func createRuntime(cfg *Config, st *state.Service, ks *keystore.GlobalKeystore, return rt, nil } -func createBABEService(cfg *Config, rt runtime.Instance, st *state.Service, ks keystore.Keystore, pruner state.Pruner) (*babe.Service, error) { +func createBABEService(cfg *Config, rt runtime.Instance, st *state.Service, ks keystore.Keystore) (*babe.Service, error) { logger.Info( "creating BABE service...", "authority", cfg.Core.BabeAuthority, @@ -192,7 +192,6 @@ func createBABEService(cfg *Config, rt runtime.Instance, st *state.Service, ks k SlotDuration: cfg.Core.SlotDuration, // TODO: remove this, should only be modified via runtime constant Authority: cfg.Core.BabeAuthority, IsDev: cfg.Global.ID == "dev", - Pruner: pruner, } if cfg.Core.BabeAuthority { @@ -377,7 +376,7 @@ func createBlockVerifier(st *state.Service) (*babe.VerificationManager, error) { return ver, nil } -func createSyncService(cfg *Config, st *state.Service, bp sync.BlockProducer, fg sync.FinalityGadget, dh *core.DigestHandler, verifier *babe.VerificationManager, rt runtime.Instance, pruner state.Pruner) (*sync.Service, error) { +func createSyncService(cfg *Config, st *state.Service, bp sync.BlockProducer, fg sync.FinalityGadget, dh *core.DigestHandler, verifier *babe.VerificationManager, rt runtime.Instance) (*sync.Service, error) { syncCfg := &sync.Config{ LogLvl: cfg.Log.SyncLvl, BlockState: st.Block, @@ -388,7 +387,6 @@ func createSyncService(cfg *Config, st *state.Service, bp sync.BlockProducer, fg Verifier: verifier, Runtime: rt, DigestHandler: dh, - Pruner: pruner, } return sync.NewService(syncCfg) diff --git a/dot/services_test.go b/dot/services_test.go index 6dfbc9efd2..2a6746d186 100644 --- a/dot/services_test.go +++ b/dot/services_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/utils" @@ -137,7 +136,7 @@ func TestCreateSyncService(t *testing.T) { ver, err := createBlockVerifier(stateSrvc) require.NoError(t, err) - _, err = createSyncService(cfg, stateSrvc, nil, nil, nil, ver, rt, &state.ArchivalNodePruner{}) + _, err = createSyncService(cfg, stateSrvc, nil, nil, nil, ver, rt) require.NoError(t, err) } @@ -234,7 +233,7 @@ func TestCreateBABEService(t *testing.T) { rt, err := createRuntime(cfg, stateSrvc, ks, &network.Service{}) require.NoError(t, err) - bs, err := createBABEService(cfg, rt, stateSrvc, ks.Babe, &state.ArchivalNodePruner{}) + bs, err := createBABEService(cfg, rt, stateSrvc, ks.Babe) require.NoError(t, err) require.NotNil(t, bs) } diff --git a/dot/state/initialize.go b/dot/state/initialize.go index 11185a7862..f5d1533718 100644 --- a/dot/state/initialize.go +++ b/dot/state/initialize.go @@ -89,7 +89,7 @@ func (s *Service) Initialise(gen *genesis.Genesis, header *types.Header, t *trie } // create storage state from genesis trie - storageState, err := NewStorageState(db, blockState, t) + storageState, err := NewStorageState(db, blockState, t, "", 0) if err != nil { return fmt.Errorf("failed to create storage state from trie: %s", err) } diff --git a/dot/state/offline_pruner.go b/dot/state/offline_pruner.go index 878626f383..9b36729fcf 100644 --- a/dot/state/offline_pruner.go +++ b/dot/state/offline_pruner.go @@ -62,7 +62,7 @@ func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNu } // load storage state - storageState, err := NewStorageState(db, blockState, trie.NewEmptyTrie()) + storageState, err := NewStorageState(db, blockState, trie.NewEmptyTrie(), "", 0) if err != nil { return nil, fmt.Errorf("failed to create new storage state %w", err) } diff --git a/dot/state/pruner.go b/dot/state/pruner.go index 39812353be..59e633f7dc 100644 --- a/dot/state/pruner.go +++ b/dot/state/pruner.go @@ -2,7 +2,6 @@ package state import ( "fmt" - "math/big" "sync" "time" @@ -18,20 +17,20 @@ const ( // Pruner is implemented by FullNodePruner and ArchivalNodePruner. type Pruner interface { - StoreJournalRecord(deleted, inserted []*common.Hash, blockHash *common.Hash, blockNum *big.Int) error + StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error } // ArchivalNodePruner is a no-op since we don't prune nodes in archive mode. type ArchivalNodePruner struct{} // StoreJournalRecord for archive node doesn't do anything. -func (a *ArchivalNodePruner) StoreJournalRecord(deleted, inserted []*common.Hash, blockHash *common.Hash, blockNum *big.Int) error { +func (a *ArchivalNodePruner) StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { return nil } type deathRecord struct { - blockHash *common.Hash - deletedKeys map[*common.Hash]int64 // key hash that will be deleted from DB + blockHash common.Hash + deletedKeys map[common.Hash]int64 // key hash that will be deleted from DB } type deathRow []*deathRecord @@ -41,7 +40,7 @@ type FullNodePruner struct { deathList []deathRow storageDB chaindb.Database journalDB chaindb.Database - deathIndex map[*common.Hash]int64 + deathIndex map[common.Hash]int64 pendingNumber int64 retainBlocks int64 sync.RWMutex @@ -49,14 +48,14 @@ type FullNodePruner struct { type journalRecord struct { // blockHash of the block corresponding to journal record - blockHash *common.Hash + blockHash common.Hash // Hash of keys that are inserted into state trie of the block - insertedKeys []*common.Hash + insertedKeys []common.Hash // Hash of keys that are deleted from state trie of the block - deletedKeys []*common.Hash + deletedKeys []common.Hash } -func newJournalRecord(hash *common.Hash, insertedKeys, deletedKeys []*common.Hash) *journalRecord { +func newJournalRecord(hash common.Hash, insertedKeys, deletedKeys []common.Hash) *journalRecord { return &journalRecord{ blockHash: hash, insertedKeys: insertedKeys, @@ -68,7 +67,7 @@ func newJournalRecord(hash *common.Hash, insertedKeys, deletedKeys []*common.Has func CreatePruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { p := &FullNodePruner{ deathList: make([]deathRow, 0), - deathIndex: make(map[*common.Hash]int64), + deathIndex: make(map[common.Hash]int64), storageDB: chaindb.NewTable(db, storagePrefix), journalDB: chaindb.NewTable(db, journalPrefix), retainBlocks: retainBlocks, @@ -95,17 +94,17 @@ func CreatePruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { } // StoreJournalRecord stores journal record into DB and add deathRow into deathList -func (p *FullNodePruner) StoreJournalRecord(deleted, inserted []*common.Hash, blockHash *common.Hash, blockNum *big.Int) error { +func (p *FullNodePruner) StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { jr := newJournalRecord(blockHash, inserted, deleted) - key := &journalKey{blockNum.Int64(), blockHash} + key := &journalKey{blockNum, blockHash} err := p.storeJournal(key, jr) if err != nil { return fmt.Errorf("failed to store journal record for %d: %w", blockNum, err) } - logger.Info("journal record stored", "block num", blockNum.Int64()) - p.addDeathRow(jr, blockNum.Int64()) + logger.Info("journal record stored", "block num", blockNum) + p.addDeathRow(jr, blockNum) return nil } @@ -128,7 +127,7 @@ func (p *FullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) { p.deathIndex[k] = blockNum } - deletedKeys := make(map[*common.Hash]int64) + deletedKeys := make(map[common.Hash]int64) for _, data := range jr.deletedKeys { deletedKeys[data] = blockNum } @@ -148,7 +147,7 @@ func (p *FullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) { } // Remove re-inserted keys -func (p *FullNodePruner) processInsertedKeys(insKeys []*common.Hash, blockHash *common.Hash) { +func (p *FullNodePruner) processInsertedKeys(insKeys []common.Hash, blockHash common.Hash) { for _, k := range insKeys { if num, ok := p.deathIndex[k]; ok { records := p.deathList[num-p.pendingNumber] @@ -218,7 +217,7 @@ func (p *FullNodePruner) start() { type journalKey struct { blockNum int64 - blockHash *common.Hash + blockHash common.Hash } func (p *FullNodePruner) storeJournal(key *journalKey, jr *journalRecord) error { @@ -310,7 +309,7 @@ func (p *FullNodePruner) getLastPrunedIndex() (int64, error) { return blockNum.(int64), err } -func (p *FullNodePruner) deleteKeys(nodesHash map[*common.Hash]int64) error { +func (p *FullNodePruner) deleteKeys(nodesHash map[common.Hash]int64) error { for k := range nodesHash { err := p.storageDB.Del(k.ToBytes()) if err != nil { diff --git a/dot/state/service.go b/dot/state/service.go index dc3788d9bd..7b7a3d75a4 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -51,22 +51,28 @@ type Service struct { // Below are for testing only. BabeThresholdNumerator uint64 BabeThresholdDenominator uint64 + + // Below are for state trie online pruner + retainBlocks int64 + gcMode string } // NewService create a new instance of Service -func NewService(path string, lvl log.Lvl) *Service { +func NewService(path string, lvl log.Lvl, gcMode string, retainBlocks int64) *Service { handler := log.StreamHandler(os.Stdout, log.TerminalFormat()) handler = log.CallerFileHandler(handler) logger.SetHandler(log.LvlFilterHandler(lvl, handler)) return &Service{ - dbPath: path, - logLvl: lvl, - db: nil, - isMemDB: false, - Storage: nil, - Block: nil, - closeCh: make(chan interface{}), + dbPath: path, + logLvl: lvl, + db: nil, + isMemDB: false, + Storage: nil, + Block: nil, + closeCh: make(chan interface{}), + gcMode: gcMode, + retainBlocks: retainBlocks, } } @@ -141,7 +147,7 @@ func (s *Service) Start() error { } // create storage state - s.Storage, err = NewStorageState(db, s.Block, trie.NewEmptyTrie()) + s.Storage, err = NewStorageState(db, s.Block, trie.NewEmptyTrie(), s.gcMode, s.retainBlocks) if err != nil { return fmt.Errorf("failed to create storage state: %w", err) } diff --git a/dot/state/service_test.go b/dot/state/service_test.go index c16df057da..f0ded75537 100644 --- a/dot/state/service_test.go +++ b/dot/state/service_test.go @@ -48,13 +48,13 @@ func newTestGenesisWithTrieAndHeader(t *testing.T) (*genesis.Genesis, *trie.Trie // helper method to create and start test state service func newTestService(t *testing.T) (state *Service) { testDir := utils.NewTestDir(t) - state = NewService(testDir, log.LvlTrace) + state = NewService(testDir, log.LvlTrace, "", 0) return state } func newTestMemDBService() *Service { testDatadirPath, _ := ioutil.TempDir("/tmp", "test-datadir-*") - state := NewService(testDatadirPath, log.LvlTrace) + state := NewService(testDatadirPath, log.LvlTrace, "", 0) state.UseMemDB() return state } @@ -116,7 +116,7 @@ func TestService_BlockTree(t *testing.T) { // removes all data directories created within test directory defer utils.RemoveTestDir(t) - stateA := NewService(testDir, log.LvlTrace) + stateA := NewService(testDir, log.LvlTrace, "", 0) genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) err := stateA.Initialise(genData, genesisHeader, genTrie) @@ -131,7 +131,7 @@ func TestService_BlockTree(t *testing.T) { err = stateA.Stop() require.NoError(t, err) - stateB := NewService(testDir, log.LvlTrace) + stateB := NewService(testDir, log.LvlTrace, "", 0) err = stateB.Start() require.NoError(t, err) @@ -145,7 +145,7 @@ func TestService_PruneStorage(t *testing.T) { testDir := utils.NewTestDir(t) defer utils.RemoveTestDir(t) - serv := NewService(testDir, log.LvlTrace) + serv := NewService(testDir, log.LvlTrace, "", 0) serv.UseMemDB() genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) @@ -169,7 +169,7 @@ func TestService_PruneStorage(t *testing.T) { err = serv.Storage.blockState.AddBlock(block) require.NoError(t, err) - err = serv.Storage.StoreTrie(trieState) + err = serv.Storage.StoreTrie(trieState, nil) require.NoError(t, err) // Only finalise a block at height 3 @@ -187,7 +187,7 @@ func TestService_PruneStorage(t *testing.T) { err = serv.Storage.blockState.AddBlock(block) require.NoError(t, err) - err = serv.Storage.StoreTrie(trieState) + err = serv.Storage.StoreTrie(trieState, nil) require.NoError(t, err) // Store the other blocks that will be pruned. @@ -220,7 +220,7 @@ func TestService_Rewind(t *testing.T) { testDir := utils.NewTestDir(t) defer utils.RemoveTestDir(t) - serv := NewService(testDir, log.LvlTrace) + serv := NewService(testDir, log.LvlTrace, "", 0) serv.UseMemDB() genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) @@ -268,7 +268,7 @@ func TestService_Import(t *testing.T) { testDir := utils.NewTestDir(t) defer utils.RemoveTestDir(t) - serv := NewService(testDir, log.LvlTrace) + serv := NewService(testDir, log.LvlTrace, "", 0) serv.UseMemDB() genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) diff --git a/dot/state/storage.go b/dot/state/storage.go index ea9d997808..dba7769f10 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -51,12 +51,12 @@ type StorageState struct { // change notifiers changedLock sync.RWMutex observerList []Observer - - syncing bool + pruner Pruner + syncing bool } // NewStorageState creates a new StorageState backed by the given trie and database located at basePath. -func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie) (*StorageState, error) { +func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, gcMode string, retainBlocks int64) (*StorageState, error) { if db == nil { return nil, fmt.Errorf("cannot have nil database") } @@ -68,11 +68,23 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie) tries := make(map[common.Hash]*trie.Trie) tries[t.MustHash()] = t + var pruner Pruner + var err error + if gcMode == "full" { + pruner, err = CreatePruner(db, retainBlocks) + if err != nil { + return nil, err + } + } else { + pruner = &ArchivalNodePruner{} + } + return &StorageState{ blockState: blockState, tries: tries, db: chaindb.NewTable(db, storagePrefix), observerList: []Observer{}, + pruner: pruner, }, nil } @@ -95,7 +107,7 @@ func (s *StorageState) pruneKey(keyHeader *types.Header) { } // StoreTrie stores the given trie in the StorageState and writes it to the database -func (s *StorageState) StoreTrie(ts *rtstorage.TrieState) error { +func (s *StorageState) StoreTrie(ts *rtstorage.TrieState, header *types.Header) error { s.lock.Lock() root := ts.MustRoot() if s.syncing { @@ -107,6 +119,25 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState) error { s.tries[root] = ts.Trie() s.lock.Unlock() + _, ok := s.pruner.(*FullNodePruner) + if header == nil && ok { + return fmt.Errorf("block cannot be empty for Full node pruner") + } + + if header != nil { + insKeys, err := ts.GetInsertedNodeHashes() + if err != nil { + return fmt.Errorf("failed to get state trie inserted keys: block %s %w", header.Hash(), err) + } + + delKeys := ts.GetDeletedNodeHashes() + + err = s.pruner.StoreJournalRecord(delKeys, insKeys, header.Hash(), header.Number.Int64()) + if err != nil { + return err + } + } + logger.Trace("cached trie in storage state", "root", root) if err := ts.Trie().WriteDirty(s.db); err != nil { diff --git a/dot/state/storage_notify_test.go b/dot/state/storage_notify_test.go index 20be087aa8..0f96f8dd34 100644 --- a/dot/state/storage_notify_test.go +++ b/dot/state/storage_notify_test.go @@ -62,7 +62,7 @@ func TestStorageState_RegisterStorageObserver(t *testing.T) { defer ss.UnregisterStorageObserver(observer) ts.Set([]byte("mackcom"), []byte("wuz here")) - err = ss.StoreTrie(ts) + err = ss.StoreTrie(ts, nil) require.NoError(t, err) expectedResult := &SubscriptionResult{ @@ -101,7 +101,7 @@ func TestStorageState_RegisterStorageObserver_Multi(t *testing.T) { ts.Set(key1, value1) - err = ss.StoreTrie(ts) + err = ss.StoreTrie(ts, nil) require.NoError(t, err) time.Sleep(time.Millisecond * 10) @@ -143,7 +143,7 @@ func TestStorageState_RegisterStorageObserver_Multi_Filter(t *testing.T) { } ts.Set(key1, value1) - err = ss.StoreTrie(ts) + err = ss.StoreTrie(ts, nil) require.NoError(t, err) time.Sleep(time.Millisecond * 10) diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index f6601eebdf..f49534bfb4 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -16,7 +16,7 @@ func newTestStorageState(t *testing.T) *StorageState { db := NewInMemoryDB(t) bs := newTestBlockState(t, testGenesisHeader) - s, err := NewStorageState(db, bs, trie.NewEmptyTrie()) + s, err := NewStorageState(db, bs, trie.NewEmptyTrie(), "", 0) require.NoError(t, err) return s } @@ -28,7 +28,7 @@ func TestStorage_StoreAndLoadTrie(t *testing.T) { root, err := ts.Root() require.NoError(t, err) - err = storage.StoreTrie(ts) + err = storage.StoreTrie(ts, nil) require.NoError(t, err) time.Sleep(time.Millisecond * 100) @@ -52,7 +52,7 @@ func TestStorage_GetStorageByBlockHash(t *testing.T) { root, err := ts.Root() require.NoError(t, err) - err = storage.StoreTrie(ts) + err = storage.StoreTrie(ts, nil) require.NoError(t, err) block := &types.Block{ @@ -79,7 +79,7 @@ func TestStorage_TrieState(t *testing.T) { root, err := ts.Root() require.NoError(t, err) - err = storage.StoreTrie(ts) + err = storage.StoreTrie(ts, nil) require.NoError(t, err) time.Sleep(time.Millisecond * 100) @@ -115,7 +115,7 @@ func TestStorage_LoadFromDB(t *testing.T) { require.NoError(t, err) // Write trie to disk. - err = storage.StoreTrie(ts) + err = storage.StoreTrie(ts, nil) require.NoError(t, err) // Clear trie from cache and fetch data from disk. @@ -154,7 +154,7 @@ func TestStorage_StoreTrie_Syncing(t *testing.T) { ts.Set(key, value) storage.SetSyncing(true) - err = storage.StoreTrie(ts) + err = storage.StoreTrie(ts, nil) require.NoError(t, err) require.Equal(t, 1, len(storage.tries)) } @@ -169,7 +169,7 @@ func TestStorage_StoreTrie_NotSyncing(t *testing.T) { ts.Set(key, value) storage.SetSyncing(false) - err = storage.StoreTrie(ts) + err = storage.StoreTrie(ts, nil) require.NoError(t, err) require.Equal(t, 2, len(storage.tries)) } diff --git a/dot/sync/interface.go b/dot/sync/interface.go index 2836b9e384..cb8abb27f5 100644 --- a/dot/sync/interface.go +++ b/dot/sync/interface.go @@ -51,7 +51,7 @@ type BlockState interface { // StorageState is the interface for the storage state type StorageState interface { TrieState(root *common.Hash) (*rtstorage.TrieState, error) - StoreTrie(ts *rtstorage.TrieState) error + StoreTrie(ts *rtstorage.TrieState, header *types.Header) error LoadCodeHash(*common.Hash) (common.Hash, error) SetSyncing(bool) } diff --git a/dot/sync/syncer.go b/dot/sync/syncer.go index 8e69d2eaeb..743f57a609 100644 --- a/dot/sync/syncer.go +++ b/dot/sync/syncer.go @@ -350,20 +350,7 @@ func (s *Service) handleBlock(block *types.Block) error { return fmt.Errorf("failed to execute block %d: %w", block.Header.Number, err) } - insKeys, err := ts.GetInsertedNodeHashes() - if err != nil { - return fmt.Errorf("failed to get state trie inserted keys: block %s %w", block.Header.Number, err) - } - - delKeys := ts.GetDeletedNodeHashes() - - blockHash := block.Header.Hash() - err = s.pruner.StoreJournalRecord(delKeys, insKeys, &blockHash, block.Header.Number) - if err != nil { - return err - } - - err = s.storageState.StoreTrie(ts) + err = s.storageState.StoreTrie(ts, block.Header) if err != nil { return err } diff --git a/dot/sync/syncer_test.go b/dot/sync/syncer_test.go index 7fae639cc8..ba986ef45a 100644 --- a/dot/sync/syncer_test.go +++ b/dot/sync/syncer_test.go @@ -65,7 +65,7 @@ func newTestSyncer(t *testing.T) *Service { cfg := &Config{} testDatadirPath, _ := ioutil.TempDir("/tmp", "test-datadir-*") - stateSrvc := state.NewService(testDatadirPath, log.LvlInfo) + stateSrvc := state.NewService(testDatadirPath, log.LvlInfo, "", 0) stateSrvc.UseMemDB() gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t) diff --git a/lib/babe/babe.go b/lib/babe/babe.go index e861ed2317..f118ea5349 100644 --- a/lib/babe/babe.go +++ b/lib/babe/babe.go @@ -26,7 +26,6 @@ import ( "sync" "time" - "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/crypto/sr25519" "github.com/ChainSafe/gossamer/lib/runtime" @@ -50,7 +49,6 @@ type Service struct { transactionState TransactionState epochState EpochState epochLength uint64 - pruner state.Pruner // BABE authority keypair keypair *sr25519.Keypair // TODO: change to BABE keystore @@ -88,7 +86,6 @@ type ServiceConfig struct { SlotDuration uint64 // for development purposes; in milliseconds EpochLength uint64 // for development purposes; in slots Authority bool - Pruner state.Pruner } // NewService returns a new Babe Service using the provided VRF keys and runtime @@ -131,7 +128,6 @@ func NewService(cfg *ServiceConfig) (*Service, error) { pause: make(chan struct{}), authority: cfg.Authority, dev: cfg.IsDev, - pruner: cfg.Pruner, } genCfg, err := babeService.rt.BabeConfiguration() @@ -491,19 +487,6 @@ func (b *Service) handleSlot(slotNum uint64) error { return nil } - insKeys, err := ts.GetInsertedNodeHashes() - if err != nil { - logger.Error("failed to get inserted keys for ", block.Header.Number, err) - } - - delKeys := ts.GetDeletedNodeHashes() - - blockHash := block.Header.Hash() - err = b.pruner.StoreJournalRecord(delKeys, insKeys, &blockHash, block.Header.Number) - if err != nil { - return err - } - old := ts.Snapshot() // block built successfully, store resulting trie in storage state @@ -512,7 +495,7 @@ func (b *Service) handleSlot(slotNum uint64) error { return err } - err = b.storageState.StoreTrie(oldTs) + err = b.storageState.StoreTrie(oldTs, block.Header) if err != nil { logger.Error("failed to store trie in storage state", "error", err) } diff --git a/lib/babe/babe_test.go b/lib/babe/babe_test.go index a1a7058f3e..9bd989dab9 100644 --- a/lib/babe/babe_test.go +++ b/lib/babe/babe_test.go @@ -109,7 +109,7 @@ func createTestService(t *testing.T, cfg *ServiceConfig) *Service { if cfg.BlockState == nil || cfg.StorageState == nil || cfg.EpochState == nil { testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") //nolint require.NoError(t, err) - dbSrv := state.NewService(testDatadirPath, log.LvlInfo) + dbSrv := state.NewService(testDatadirPath, log.LvlInfo, "", 0) dbSrv.UseMemDB() if cfg.EpochLength > 0 { diff --git a/lib/babe/state.go b/lib/babe/state.go index c34b4cb13f..373ec91ccc 100644 --- a/lib/babe/state.go +++ b/lib/babe/state.go @@ -48,7 +48,7 @@ type BlockState interface { // StorageState interface for storage state methods type StorageState interface { TrieState(hash *common.Hash) (*rtstorage.TrieState, error) - StoreTrie(ts *rtstorage.TrieState) error + StoreTrie(ts *rtstorage.TrieState, header *types.Header) error } // TransactionState is the interface for transaction queue methods diff --git a/lib/babe/verify_test.go b/lib/babe/verify_test.go index 335208159b..48fb838fef 100644 --- a/lib/babe/verify_test.go +++ b/lib/babe/verify_test.go @@ -36,7 +36,7 @@ func newTestVerificationManager(t *testing.T, genCfg *types.BabeConfiguration) * testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") require.NoError(t, err) - dbSrv := state.NewService(testDatadirPath, log.LvlInfo) + dbSrv := state.NewService(testDatadirPath, log.LvlInfo, "", 0) dbSrv.UseMemDB() if genCfg == nil { diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index c1b4f373f4..bf2d299a69 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -270,14 +270,14 @@ func (s *TrieState) LoadCodeHash() (common.Hash, error) { } // GetInsertedNodeHashes returns the hash of nodes inserted into state trie since last block produced -func (s *TrieState) GetInsertedNodeHashes() ([]*common.Hash, error) { +func (s *TrieState) GetInsertedNodeHashes() ([]common.Hash, error) { s.lock.RLock() defer s.lock.RUnlock() return s.t.GetInsertedNodeHashes(s.t.RootNode()) } // GetDeletedNodeHashes returns the hash of nodes that are deleted from state trie since last block produced -func (s *TrieState) GetDeletedNodeHashes() []*common.Hash { +func (s *TrieState) GetDeletedNodeHashes() []common.Hash { s.lock.RLock() defer s.lock.RUnlock() return s.t.GetDeletedNodeHash() diff --git a/lib/trie/database.go b/lib/trie/database.go index 53787a8084..98f88c2908 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -290,8 +290,8 @@ func (t *Trie) writeDirty(db chaindb.Batch, curr node) error { // GetInsertedNodeHashes returns the hash of nodes that are inserted into state trie since last snapshot is called // Since inserted nodesare newly created we need to compute their hash values. -func (t *Trie) GetInsertedNodeHashes(curr node) ([]*common.Hash, error) { - var nodeHashes []*common.Hash +func (t *Trie) GetInsertedNodeHashes(curr node) ([]common.Hash, error) { + var nodeHashes []common.Hash if curr == nil || !curr.isDirty() { return nil, nil } @@ -311,7 +311,7 @@ func (t *Trie) GetInsertedNodeHashes(curr node) ([]*common.Hash, error) { } nodeHash := common.BytesToHash(hash) - nodeHashes = append(nodeHashes, &nodeHash) + nodeHashes = append(nodeHashes, nodeHash) if c, ok := curr.(*branch); ok { for _, child := range c.children { @@ -330,6 +330,6 @@ func (t *Trie) GetInsertedNodeHashes(curr node) ([]*common.Hash, error) { } // GetDeletedNodeHash returns the hash of nodes that are deleted from state trie since last snapshot is called -func (t *Trie) GetDeletedNodeHash() []*common.Hash { +func (t *Trie) GetDeletedNodeHash() []common.Hash { return t.deletedKeys } diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 61bfeee002..e11fe5ee1a 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -32,7 +32,7 @@ type Trie struct { generation uint64 root node childTries map[common.Hash]*Trie // Used to store the child tries. - deletedKeys []*common.Hash + deletedKeys []common.Hash } // NewEmptyTrie creates a trie with a nil root @@ -46,7 +46,7 @@ func NewTrie(root node) *Trie { root: root, childTries: make(map[common.Hash]*Trie), generation: 0, // Initially zero but increases after every snapshot. - deletedKeys: make([]*common.Hash, 0), + deletedKeys: make([]common.Hash, 0), } } @@ -60,7 +60,7 @@ func (t *Trie) Snapshot() *Trie { } t.generation++ - t.deletedKeys = make([]*common.Hash, 0) + t.deletedKeys = make([]common.Hash, 0) return oldTrie } @@ -79,7 +79,7 @@ func (t *Trie) maybeUpdateGeneration(n node) node { oldNodeHash := n.getHash() if len(oldNodeHash) > 0 { hash := common.BytesToHash(oldNodeHash) - t.deletedKeys = append(t.deletedKeys, &hash) + t.deletedKeys = append(t.deletedKeys, hash) } return newNode } From 97ea1a1555c7047cf1ced5cde523bbbefe3554a3 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 1 Jun 2021 15:34:25 +0530 Subject: [PATCH 08/18] Address comments. --- cmd/gossamer/config.go | 4 +- cmd/gossamer/export.go | 2 +- cmd/gossamer/flags.go | 11 ++-- cmd/gossamer/main.go | 13 ++-- dot/config.go | 2 +- dot/config/toml/config.go | 2 +- dot/services.go | 2 +- dot/state/pruner.go | 119 +++++++++++++++++----------------- dot/state/storage.go | 13 ++-- lib/runtime/storage/trie.go | 2 +- lib/trie/database.go | 12 ++-- lib/trie/trie_test.go | 1 + tests/utils/gossamer_utils.go | 2 +- 13 files changed, 100 insertions(+), 85 deletions(-) diff --git a/cmd/gossamer/config.go b/cmd/gossamer/config.go index 15eb95ce5b..8d8b5ad2d6 100644 --- a/cmd/gossamer/config.go +++ b/cmd/gossamer/config.go @@ -445,7 +445,7 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) { cfg.MetricsPort = tomlCfg.Global.MetricsPort cfg.RetainBlocks = tomlCfg.Global.RetainBlocks - cfg.GCMode = tomlCfg.Global.GCMode + cfg.Pruning = tomlCfg.Global.Pruning } } @@ -476,7 +476,7 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) { } cfg.RetainBlocks = ctx.GlobalInt64(RetainBlockNumberFlag.Name) - cfg.GCMode = ctx.GlobalString(GCModeFlag.Name) + cfg.Pruning = ctx.GlobalString(Pruning.Name) cfg.NoTelemetry = ctx.Bool("no-telemetry") } diff --git a/cmd/gossamer/export.go b/cmd/gossamer/export.go index 670fdbf0ab..5e45ec1654 100644 --- a/cmd/gossamer/export.go +++ b/cmd/gossamer/export.go @@ -85,7 +85,7 @@ func dotConfigToToml(dcfg *dot.Config) *ctoml.Config { LogLvl: dcfg.Global.LogLvl.String(), MetricsPort: dcfg.Global.MetricsPort, RetainBlocks: dcfg.Global.RetainBlocks, - GCMode: dcfg.Global.GCMode, + Pruning: dcfg.Global.Pruning, } cfg.Log = ctoml.LogConfig{ diff --git a/cmd/gossamer/flags.go b/cmd/gossamer/flags.go index 6d59f41f22..049216fdec 100644 --- a/cmd/gossamer/flags.go +++ b/cmd/gossamer/flags.go @@ -286,10 +286,11 @@ var ( Value: 256, } - // GCModeFlag sets the blockchain GC mode. It's either full or archive. - GCModeFlag = cli.StringFlag{ - Name: "gcmode", - Usage: `Blockchain garbage collection mode ("full", "archive")`, + // Pruning triggers the online pruning of state trie. It's either full or archive. To enable pruning the value + // should be set to `full`. + Pruning = cli.StringFlag{ + Name: "pruning", + Usage: `State trie online pruning ("full", "archive")`, Value: "full", } ) @@ -309,7 +310,7 @@ var ( DBPathFlag, BloomFilterSizeFlag, RetainBlockNumberFlag, - GCModeFlag, + Pruning, } // StartupFlags are flags that are valid for use with the root command and the export subcommand diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index 1a2ed07019..d6bc3ceb0b 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -37,6 +37,7 @@ const ( importRuntimeCommandName = "import-runtime" importStateCommandName = "import-state" pruningStateCommandName = "prune-state" + defaultRetainBlocks = 256 ) // app is the cli application @@ -234,12 +235,12 @@ func gossamerAction(ctx *cli.Context) error { return err } - if cfg.Global.RetainBlocks < 256 { - return fmt.Errorf("--%s cannot be less than 256", RetainBlockNumberFlag.Name) + if cfg.Global.RetainBlocks < defaultRetainBlocks { + return fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, defaultRetainBlocks) } - if cfg.Global.GCMode != "full" && cfg.Global.GCMode != "archive" { - return fmt.Errorf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) + if cfg.Global.Pruning != "full" && cfg.Global.Pruning != "archive" { + return fmt.Errorf("--%s must be either 'full' or 'archive'", Pruning.Name) } cfg.Global.LogLvl = lvl @@ -451,6 +452,10 @@ func pruneState(ctx *cli.Context) error { bloomSize := ctx.GlobalUint64(BloomFilterSizeFlag.Name) retainBlocks := ctx.GlobalInt64(RetainBlockNumberFlag.Name) + if retainBlocks < defaultRetainBlocks { + return fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, defaultRetainBlocks) + } + pruner, err := state.NewPruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks) if err != nil { return err diff --git a/dot/config.go b/dot/config.go index 884a63b5fe..48a6ccb35e 100644 --- a/dot/config.go +++ b/dot/config.go @@ -53,7 +53,7 @@ type GlobalConfig struct { MetricsPort uint32 NoTelemetry bool RetainBlocks int64 - GCMode string + Pruning string } // LogConfig represents the log levels for individual packages diff --git a/dot/config/toml/config.go b/dot/config/toml/config.go index 7ca982f45f..cf6555f3b5 100644 --- a/dot/config/toml/config.go +++ b/dot/config/toml/config.go @@ -35,7 +35,7 @@ type GlobalConfig struct { LogLvl string `toml:"log,omitempty"` MetricsPort uint32 `toml:"metrics-port,omitempty"` RetainBlocks int64 `toml:"retain-blocks,omitempty"` - GCMode string `toml:"gc-mode,omitempty"` + Pruning string `toml:"pruning,omitempty"` } // LogConfig represents the log levels for individual packages diff --git a/dot/services.go b/dot/services.go index 037f71d7d6..35b3858ad4 100644 --- a/dot/services.go +++ b/dot/services.go @@ -53,7 +53,7 @@ func newInMemoryDB(path string) (chaindb.Database, error) { // createStateService creates the state service and initialise state database func createStateService(cfg *Config) (*state.Service, error) { logger.Debug("creating state service...") - stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Log.StateLvl, cfg.Global.GCMode, cfg.Global.RetainBlocks) + stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Log.StateLvl, cfg.Global.Pruning, cfg.Global.RetainBlocks) // start state service (initialise state database) err := stateSrvc.Start() diff --git a/dot/state/pruner.go b/dot/state/pruner.go index 59e633f7dc..03cf1913b1 100644 --- a/dot/state/pruner.go +++ b/dot/state/pruner.go @@ -13,35 +13,36 @@ import ( const ( journalPrefix = "journal" lastPrunedKey = "last_pruned" + pruneInterval = time.Second ) -// Pruner is implemented by FullNodePruner and ArchivalNodePruner. +// Pruner is implemented by fullNodePruner and archivalNodePruner. type Pruner interface { - StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error + storeJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error } -// ArchivalNodePruner is a no-op since we don't prune nodes in archive mode. -type ArchivalNodePruner struct{} +// archivalNodePruner is a no-op since we don't prune nodes in archive mode. +type archivalNodePruner struct{} -// StoreJournalRecord for archive node doesn't do anything. -func (a *ArchivalNodePruner) StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { +// storeJournalRecord for archive node doesn't do anything. +func (a *archivalNodePruner) storeJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { return nil } type deathRecord struct { blockHash common.Hash - deletedKeys map[common.Hash]int64 // key hash that will be deleted from DB + deletedKeys map[common.Hash]int64 // Mapping from deleted key hash to block number. } type deathRow []*deathRecord -// FullNodePruner stores state trie diff and allows online state trie pruning -type FullNodePruner struct { +// fullNodePruner stores state trie diff and allows online state trie pruning +type fullNodePruner struct { deathList []deathRow storageDB chaindb.Database journalDB chaindb.Database - deathIndex map[common.Hash]int64 - pendingNumber int64 + deathIndex map[common.Hash]int64 // Mapping from deleted key hash to block number. + pendingNumber int64 // block number to be pruned. Initial value is set to 1 and is incremented after every block pruning. retainBlocks int64 sync.RWMutex } @@ -55,6 +56,11 @@ type journalRecord struct { deletedKeys []common.Hash } +type journalKey struct { + blockNum int64 + blockHash common.Hash +} + func newJournalRecord(hash common.Hash, insertedKeys, deletedKeys []common.Hash) *journalRecord { return &journalRecord{ blockHash: hash, @@ -63,9 +69,9 @@ func newJournalRecord(hash common.Hash, insertedKeys, deletedKeys []common.Hash) } } -// CreatePruner creates a pruner -func CreatePruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { - p := &FullNodePruner{ +// createPruner creates a pruner +func createPruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { + p := &fullNodePruner{ deathList: make([]deathRow, 0), deathIndex: make(map[common.Hash]int64), storageDB: chaindb.NewTable(db, storagePrefix), @@ -94,7 +100,7 @@ func CreatePruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { } // StoreJournalRecord stores journal record into DB and add deathRow into deathList -func (p *FullNodePruner) StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { +func (p *fullNodePruner) storeJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { jr := newJournalRecord(blockHash, inserted, deleted) key := &journalKey{blockNum, blockHash} @@ -103,12 +109,12 @@ func (p *FullNodePruner) StoreJournalRecord(deleted, inserted []common.Hash, blo return fmt.Errorf("failed to store journal record for %d: %w", blockNum, err) } - logger.Info("journal record stored", "block num", blockNum) + logger.Debug("journal record stored", "block num", blockNum) p.addDeathRow(jr, blockNum) return nil } -func (p *FullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) { +func (p *fullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) { if blockNum == 0 { return } @@ -116,6 +122,7 @@ func (p *FullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) { p.Lock() defer p.Unlock() + // The block is already pruned. if blockNum < p.pendingNumber { return } @@ -147,47 +154,43 @@ func (p *FullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) { } // Remove re-inserted keys -func (p *FullNodePruner) processInsertedKeys(insKeys []common.Hash, blockHash common.Hash) { +func (p *fullNodePruner) processInsertedKeys(insKeys []common.Hash, blockHash common.Hash) { for _, k := range insKeys { - if num, ok := p.deathIndex[k]; ok { - records := p.deathList[num-p.pendingNumber] - for _, v := range records { - if v.blockHash == blockHash { - delete(v.deletedKeys, k) - } + num, ok := p.deathIndex[k] + if !ok { + continue + } + records := p.deathList[num-p.pendingNumber] + for _, v := range records { + if v.blockHash == blockHash { + delete(v.deletedKeys, k) } - delete(p.deathIndex, k) } + delete(p.deathIndex, k) } } -func (p *FullNodePruner) start() { +func (p *fullNodePruner) start() { logger.Info("pruning started") - for { + checkPruning := func() { p.Lock() + defer p.Unlock() if int64(len(p.deathList)) <= p.retainBlocks { - p.Unlock() - time.Sleep(2 * time.Second) - continue + return } // pop first element from death list row := p.deathList[0] blockNum := p.pendingNumber - if len(row) == 0 { - logger.Info("death row is empty", "block num", p.pendingNumber) - continue - } - blockHash := row[0].blockHash - logger.Info("pruning block", "block num", blockNum) + logger.Debug("pruning block", "block num", blockNum) for _, record := range row { err := p.deleteKeys(record.deletedKeys) if err != nil { - logger.Error("failed to prune keys", "block num", blockNum, "error", err) - continue + logger.Warn("failed to prune keys", "block num", blockNum, "error", err) + return } for k := range record.deletedKeys { @@ -198,29 +201,29 @@ func (p *FullNodePruner) start() { err := p.storeLastPrunedIndex(blockNum) if err != nil { logger.Error("failed to store last pruned index", "block num", blockNum, "error", err) - continue + return } p.deathList = p.deathList[1:] + p.pendingNumber++ - jk := &journalKey{blockNum, blockHash} - err = p.deleteJournalRecord(jk) - if err != nil { - logger.Error("failed to delete journal record", "block num", blockNum, "error", err) - continue + for _, record := range row { + jk := &journalKey{blockNum, record.blockHash} + err = p.deleteJournalRecord(jk) + if err != nil { + logger.Error("failed to delete journal record", "block num", blockNum, "error", err) + return + } } - - p.pendingNumber++ - p.Unlock() } -} -type journalKey struct { - blockNum int64 - blockHash common.Hash + for { + checkPruning() + time.Sleep(pruneInterval) + } } -func (p *FullNodePruner) storeJournal(key *journalKey, jr *journalRecord) error { +func (p *fullNodePruner) storeJournal(key *journalKey, jr *journalRecord) error { encKey, err := scale.Encode(key) if err != nil { return fmt.Errorf("failed to encode journal key block num %d: %w", key.blockNum, err) @@ -240,7 +243,7 @@ func (p *FullNodePruner) storeJournal(key *journalKey, jr *journalRecord) error } // loadDeathList loads deathList and deathIndex from journalRecord. -func (p *FullNodePruner) loadDeathList() error { +func (p *fullNodePruner) loadDeathList() error { itr := p.journalDB.NewIterator() defer itr.Release() @@ -263,7 +266,7 @@ func (p *FullNodePruner) loadDeathList() error { return nil } -func (p *FullNodePruner) deleteJournalRecord(key *journalKey) error { +func (p *fullNodePruner) deleteJournalRecord(key *journalKey) error { encKey, err := scale.Encode(key) if err != nil { return err @@ -277,7 +280,7 @@ func (p *FullNodePruner) deleteJournalRecord(key *journalKey) error { return nil } -func (p *FullNodePruner) storeLastPrunedIndex(blockNum int64) error { +func (p *fullNodePruner) storeLastPrunedIndex(blockNum int64) error { encNum, err := scale.Encode(blockNum) if err != nil { return err @@ -291,7 +294,7 @@ func (p *FullNodePruner) storeLastPrunedIndex(blockNum int64) error { return nil } -func (p *FullNodePruner) getLastPrunedIndex() (int64, error) { +func (p *fullNodePruner) getLastPrunedIndex() (int64, error) { val, err := p.journalDB.Get([]byte(lastPrunedKey)) if err == chaindb.ErrKeyNotFound { return 0, nil @@ -306,10 +309,10 @@ func (p *FullNodePruner) getLastPrunedIndex() (int64, error) { return 0, err } - return blockNum.(int64), err + return blockNum.(int64), nil } -func (p *FullNodePruner) deleteKeys(nodesHash map[common.Hash]int64) error { +func (p *fullNodePruner) deleteKeys(nodesHash map[common.Hash]int64) error { for k := range nodesHash { err := p.storageDB.Del(k.ToBytes()) if err != nil { diff --git a/dot/state/storage.go b/dot/state/storage.go index dba7769f10..5b6a464413 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -69,14 +69,14 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, tries[t.MustHash()] = t var pruner Pruner - var err error if gcMode == "full" { - pruner, err = CreatePruner(db, retainBlocks) + var err error + pruner, err = createPruner(db, retainBlocks) if err != nil { return nil, err } } else { - pruner = &ArchivalNodePruner{} + pruner = &archivalNodePruner{} } return &StorageState{ @@ -119,8 +119,7 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState, header *types.Header) s.tries[root] = ts.Trie() s.lock.Unlock() - _, ok := s.pruner.(*FullNodePruner) - if header == nil && ok { + if _, ok := s.pruner.(*fullNodePruner); header == nil && ok { return fmt.Errorf("block cannot be empty for Full node pruner") } @@ -129,10 +128,12 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState, header *types.Header) if err != nil { return fmt.Errorf("failed to get state trie inserted keys: block %s %w", header.Hash(), err) } + logger.Debug("inserted keys", len(insKeys), "for block", header.Number) delKeys := ts.GetDeletedNodeHashes() - err = s.pruner.StoreJournalRecord(delKeys, insKeys, header.Hash(), header.Number.Int64()) + logger.Debug("deleted keys", len(delKeys), "for block", header.Number) + err = s.pruner.storeJournalRecord(delKeys, insKeys, header.Hash(), header.Number.Int64()) if err != nil { return err } diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index bf2d299a69..331af85988 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -273,7 +273,7 @@ func (s *TrieState) LoadCodeHash() (common.Hash, error) { func (s *TrieState) GetInsertedNodeHashes() ([]common.Hash, error) { s.lock.RLock() defer s.lock.RUnlock() - return s.t.GetInsertedNodeHashes(s.t.RootNode()) + return s.t.GetInsertedNodeHashes() } // GetDeletedNodeHashes returns the hash of nodes that are deleted from state trie since last block produced diff --git a/lib/trie/database.go b/lib/trie/database.go index 98f88c2908..8218bd1b38 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -289,8 +289,12 @@ func (t *Trie) writeDirty(db chaindb.Batch, curr node) error { } // GetInsertedNodeHashes returns the hash of nodes that are inserted into state trie since last snapshot is called -// Since inserted nodesare newly created we need to compute their hash values. -func (t *Trie) GetInsertedNodeHashes(curr node) ([]common.Hash, error) { +// Since inserted nodes are newly created we need to compute their hash values. +func (t *Trie) GetInsertedNodeHashes() ([]common.Hash, error) { + return t.getInsertedNodeHashes(t.root) +} + +func (t *Trie) getInsertedNodeHashes(curr node) ([]common.Hash, error) { var nodeHashes []common.Hash if curr == nil || !curr.isDirty() { return nil, nil @@ -301,7 +305,7 @@ func (t *Trie) GetInsertedNodeHashes(curr node) ([]common.Hash, error) { return nil, err } - if curr == t.root { + if curr == t.root && len(enc) < 32 { h, err := common.Blake2bHash(enc) //nolint if err != nil { return nil, err @@ -318,7 +322,7 @@ func (t *Trie) GetInsertedNodeHashes(curr node) ([]common.Hash, error) { if child == nil { continue } - nodes, err := t.GetInsertedNodeHashes(child) + nodes, err := t.getInsertedNodeHashes(child) if err != nil { return nil, err } diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 896202d06c..d9f6818f64 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -515,6 +515,7 @@ func TestTrieDiff(t *testing.T) { trie.Put(test.key, test.value) } deletedKeys := trie.deletedKeys + require.Len(t, deletedKeys, 3) err = trie.WriteDirty(storageDB) require.NoError(t, err) diff --git a/tests/utils/gossamer_utils.go b/tests/utils/gossamer_utils.go index ebfe658e32..69149681b3 100644 --- a/tests/utils/gossamer_utils.go +++ b/tests/utils/gossamer_utils.go @@ -422,7 +422,7 @@ func generateDefaultConfig() *ctoml.Config { LogLvl: "crit", MetricsPort: 9876, RetainBlocks: 256, - GCMode: "archive", + Pruning: "archive", }, Log: ctoml.LogConfig{ CoreLvl: "info", From 224dd60571b223a53b28f1f690787c2f8f524000 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 1 Jun 2021 19:32:33 +0530 Subject: [PATCH 09/18] Remove prune check from offline pruning. --- cmd/gossamer/main.go | 4 ---- go.sum | 3 --- 2 files changed, 7 deletions(-) diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index d6bc3ceb0b..9eef8f14ed 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -452,10 +452,6 @@ func pruneState(ctx *cli.Context) error { bloomSize := ctx.GlobalUint64(BloomFilterSizeFlag.Name) retainBlocks := ctx.GlobalInt64(RetainBlockNumberFlag.Name) - if retainBlocks < defaultRetainBlocks { - return fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, defaultRetainBlocks) - } - pruner, err := state.NewPruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks) if err != nil { return err diff --git a/go.sum b/go.sum index 69a999b40f..fb63c9d20f 100644 --- a/go.sum +++ b/go.sum @@ -264,7 +264,6 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -598,7 +597,6 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= @@ -732,7 +730,6 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190306220234-b354f8bf4d9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From ff357b0cafafad367ebd00482106a692f623901f Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 3 Jun 2021 16:48:14 +0530 Subject: [PATCH 10/18] Add test for online pruning. --- dot/state/pruner.go | 8 +++++- dot/state/service_test.go | 58 +++++++++++++++++++++++++++++++++++++-- dot/state/storage.go | 4 +-- dot/state/test_helpers.go | 7 ++--- 4 files changed, 67 insertions(+), 10 deletions(-) diff --git a/dot/state/pruner.go b/dot/state/pruner.go index 03cf1913b1..1d211da062 100644 --- a/dot/state/pruner.go +++ b/dot/state/pruner.go @@ -173,12 +173,15 @@ func (p *fullNodePruner) processInsertedKeys(insKeys []common.Hash, blockHash co func (p *fullNodePruner) start() { logger.Info("pruning started") + var canPrune bool checkPruning := func() { p.Lock() defer p.Unlock() if int64(len(p.deathList)) <= p.retainBlocks { + canPrune = false return } + canPrune = true // pop first element from death list row := p.deathList[0] @@ -219,7 +222,10 @@ func (p *fullNodePruner) start() { for { checkPruning() - time.Sleep(pruneInterval) + // Don't sleep if we have data to prune. + if !canPrune { + time.Sleep(pruneInterval) + } } } diff --git a/dot/state/service_test.go b/dot/state/service_test.go index f0ded75537..47ec7501a6 100644 --- a/dot/state/service_test.go +++ b/dot/state/service_test.go @@ -17,6 +17,7 @@ package state import ( + "fmt" "io/ioutil" "math/big" "testing" @@ -25,6 +26,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" + rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" @@ -141,6 +143,56 @@ func TestService_BlockTree(t *testing.T) { require.Equal(t, stateA.Block.BestBlockHash(), stateB.Block.BestBlockHash()) } +func TestService_StorageTriePruning(t *testing.T) { + testDir := utils.NewTestDir(t) + defer utils.RemoveTestDir(t) + + retainBlocks := 2 + serv := NewService(testDir, log.LvlTrace, "full", int64(retainBlocks)) + serv.UseMemDB() + + genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) + + err := serv.Initialise(genData, genesisHeader, genTrie) + require.NoError(t, err) + + err = serv.Start() + require.NoError(t, err) + + var blocks []*types.Block + parentHash := serv.Block.GenesisHash() + + totalBlock := 10 + for i := 1; i < totalBlock; i++ { + block, trieState := generateBlockWithRandomTrie(t, serv, &parentHash, int64(i)) + + err = serv.Storage.blockState.AddBlock(block) + require.NoError(t, err) + + old := trieState.Snapshot() + + oldTs, err := rtstorage.NewTrieState(old) + require.NoError(t, err) + + err = serv.Storage.StoreTrie(oldTs, block.Header) + require.NoError(t, err) + + blocks = append(blocks, block) + parentHash = block.Header.Hash() + } + + time.Sleep(2 * time.Second) + + for _, b := range blocks { + _, err := serv.Storage.LoadFromDB(b.Header.StateRoot) + if b.Header.Number.Int64() >= int64(totalBlock-retainBlocks-1) { + require.NoError(t, err, fmt.Sprintf("Got error for block %d", b.Header.Number.Int64())) + continue + } + require.Error(t, err, fmt.Sprintf("Expected error for block %d", b.Header.Number.Int64())) + } +} + func TestService_PruneStorage(t *testing.T) { testDir := utils.NewTestDir(t) defer utils.RemoveTestDir(t) @@ -164,7 +216,7 @@ func TestService_PruneStorage(t *testing.T) { var toFinalize common.Hash for i := 0; i < 3; i++ { - block, trieState := generateBlockWithRandomTrie(t, serv, nil) + block, trieState := generateBlockWithRandomTrie(t, serv, nil, int64(i+1)) err = serv.Storage.blockState.AddBlock(block) require.NoError(t, err) @@ -179,10 +231,10 @@ func TestService_PruneStorage(t *testing.T) { } // add some blocks to prune, on a different chain from the finalised block - prunedArr := []prunedBlock{} + var prunedArr []prunedBlock parentHash := serv.Block.GenesisHash() for i := 0; i < 3; i++ { - block, trieState := generateBlockWithRandomTrie(t, serv, &parentHash) + block, trieState := generateBlockWithRandomTrie(t, serv, &parentHash, int64(i+1)) err = serv.Storage.blockState.AddBlock(block) require.NoError(t, err) diff --git a/dot/state/storage.go b/dot/state/storage.go index 5b6a464413..82903768ef 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -128,11 +128,11 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState, header *types.Header) if err != nil { return fmt.Errorf("failed to get state trie inserted keys: block %s %w", header.Hash(), err) } - logger.Debug("inserted keys", len(insKeys), "for block", header.Number) + logger.Debug("storage", "inserted keys", len(insKeys), "for block", header.Number) delKeys := ts.GetDeletedNodeHashes() - logger.Debug("deleted keys", len(delKeys), "for block", header.Number) + logger.Debug("storage ", "deleted keys", len(delKeys), "for block", header.Number) err = s.pruner.storeJournalRecord(delKeys, insKeys, header.Hash(), header.Number.Int64()) if err != nil { return err diff --git a/dot/state/test_helpers.go b/dot/state/test_helpers.go index f75faa3e0a..02b2e70065 100644 --- a/dot/state/test_helpers.go +++ b/dot/state/test_helpers.go @@ -206,8 +206,8 @@ func AddBlocksToStateWithFixedBranches(t *testing.T, blockState *BlockState, dep } } -func generateBlockWithRandomTrie(t *testing.T, serv *Service, parent *common.Hash) (*types.Block, *runtime.TrieState) { - trieState, err := serv.Storage.TrieState(&trie.EmptyHash) +func generateBlockWithRandomTrie(t *testing.T, serv *Service, parent *common.Hash, bNum int64) (*types.Block, *runtime.TrieState) { + trieState, err := serv.Storage.TrieState(nil) require.NoError(t, err) // Generate random data for trie state. @@ -224,11 +224,10 @@ func generateBlockWithRandomTrie(t *testing.T, serv *Service, parent *common.Has parent = &bb } - // Generate a block with the above StateRoot. block := &types.Block{ Header: &types.Header{ ParentHash: *parent, - Number: big.NewInt(rand), + Number: big.NewInt(bNum), StateRoot: trieStateRoot, }, Body: types.NewBody([]byte{}), From 862d706d9b01cb171675ea6a227c6c156085d48f Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 3 Jun 2021 17:59:56 +0530 Subject: [PATCH 11/18] Address comments. --- cmd/gossamer/config.go | 4 ++-- cmd/gossamer/export.go | 2 +- cmd/gossamer/flags.go | 2 +- cmd/gossamer/main.go | 4 ++-- dot/config.go | 3 ++- dot/state/pruner.go | 33 +++++++++++++++++++++++++++++++++ dot/state/service.go | 8 ++++---- dot/state/service_test.go | 2 +- dot/state/storage.go | 7 ++----- dot/sync/syncer.go | 4 ---- 10 files changed, 48 insertions(+), 21 deletions(-) diff --git a/cmd/gossamer/config.go b/cmd/gossamer/config.go index 8d8b5ad2d6..83a57318aa 100644 --- a/cmd/gossamer/config.go +++ b/cmd/gossamer/config.go @@ -445,7 +445,7 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) { cfg.MetricsPort = tomlCfg.Global.MetricsPort cfg.RetainBlocks = tomlCfg.Global.RetainBlocks - cfg.Pruning = tomlCfg.Global.Pruning + cfg.Pruning = state.Pruning(tomlCfg.Global.Pruning) } } @@ -476,7 +476,7 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) { } cfg.RetainBlocks = ctx.GlobalInt64(RetainBlockNumberFlag.Name) - cfg.Pruning = ctx.GlobalString(Pruning.Name) + cfg.Pruning = state.Pruning(ctx.GlobalString(Pruning.Name)) cfg.NoTelemetry = ctx.Bool("no-telemetry") } diff --git a/cmd/gossamer/export.go b/cmd/gossamer/export.go index 5e45ec1654..b9cb63cc32 100644 --- a/cmd/gossamer/export.go +++ b/cmd/gossamer/export.go @@ -85,7 +85,7 @@ func dotConfigToToml(dcfg *dot.Config) *ctoml.Config { LogLvl: dcfg.Global.LogLvl.String(), MetricsPort: dcfg.Global.MetricsPort, RetainBlocks: dcfg.Global.RetainBlocks, - Pruning: dcfg.Global.Pruning, + Pruning: dcfg.Global.Pruning.String(), } cfg.Log = ctoml.LogConfig{ diff --git a/cmd/gossamer/flags.go b/cmd/gossamer/flags.go index 049216fdec..1c464db955 100644 --- a/cmd/gossamer/flags.go +++ b/cmd/gossamer/flags.go @@ -286,7 +286,7 @@ var ( Value: 256, } - // Pruning triggers the online pruning of state trie. It's either full or archive. To enable pruning the value + // Pruning triggers the online pruning of historical state tries. It's either full or archive. To enable pruning the value // should be set to `full`. Pruning = cli.StringFlag{ Name: "pruning", diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index 9eef8f14ed..d399bd2ee3 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -239,8 +239,8 @@ func gossamerAction(ctx *cli.Context) error { return fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, defaultRetainBlocks) } - if cfg.Global.Pruning != "full" && cfg.Global.Pruning != "archive" { - return fmt.Errorf("--%s must be either 'full' or 'archive'", Pruning.Name) + if !cfg.Global.Pruning.IsValid() { + return fmt.Errorf("--%s must be either %s or %s", state.FullNode, state.ArchiveNode, Pruning.Name) } cfg.Global.LogLvl = lvl diff --git a/dot/config.go b/dot/config.go index 48a6ccb35e..7a500a8418 100644 --- a/dot/config.go +++ b/dot/config.go @@ -23,6 +23,7 @@ import ( "github.com/ChainSafe/gossamer/chain/gssmr" "github.com/ChainSafe/gossamer/chain/kusama" "github.com/ChainSafe/gossamer/chain/polkadot" + "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/types" log "github.com/ChainSafe/log15" ) @@ -53,7 +54,7 @@ type GlobalConfig struct { MetricsPort uint32 NoTelemetry bool RetainBlocks int64 - Pruning string + Pruning state.Pruning } // LogConfig represents the log levels for individual packages diff --git a/dot/state/pruner.go b/dot/state/pruner.go index 1d211da062..7aed8a9d56 100644 --- a/dot/state/pruner.go +++ b/dot/state/pruner.go @@ -16,6 +16,39 @@ const ( pruneInterval = time.Second ) +// nolint +const ( + FullNode = "full" + ArchiveNode = "archive" +) + +// Pruning online pruning mode of historical state tries +type Pruning string + +// IsValid checks whether the pruning mode is valid +func (p Pruning) IsValid() bool { + switch p { + case FullNode: + return true + case ArchiveNode: + return true + default: + return false + } +} + +// String returns format of Pruning +func (p Pruning) String() string { + switch p { + case FullNode: + return "full" + case ArchiveNode: + return "archive" + default: + return "" + } +} + // Pruner is implemented by fullNodePruner and archivalNodePruner. type Pruner interface { storeJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error diff --git a/dot/state/service.go b/dot/state/service.go index 7b7a3d75a4..4947990bc7 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -54,11 +54,11 @@ type Service struct { // Below are for state trie online pruner retainBlocks int64 - gcMode string + pruningMode Pruning } // NewService create a new instance of Service -func NewService(path string, lvl log.Lvl, gcMode string, retainBlocks int64) *Service { +func NewService(path string, lvl log.Lvl, pruningMode Pruning, retainBlocks int64) *Service { handler := log.StreamHandler(os.Stdout, log.TerminalFormat()) handler = log.CallerFileHandler(handler) logger.SetHandler(log.LvlFilterHandler(lvl, handler)) @@ -71,7 +71,7 @@ func NewService(path string, lvl log.Lvl, gcMode string, retainBlocks int64) *Se Storage: nil, Block: nil, closeCh: make(chan interface{}), - gcMode: gcMode, + pruningMode: pruningMode, retainBlocks: retainBlocks, } } @@ -147,7 +147,7 @@ func (s *Service) Start() error { } // create storage state - s.Storage, err = NewStorageState(db, s.Block, trie.NewEmptyTrie(), s.gcMode, s.retainBlocks) + s.Storage, err = NewStorageState(db, s.Block, trie.NewEmptyTrie(), s.pruningMode, s.retainBlocks) if err != nil { return fmt.Errorf("failed to create storage state: %w", err) } diff --git a/dot/state/service_test.go b/dot/state/service_test.go index 47ec7501a6..d88f6d4688 100644 --- a/dot/state/service_test.go +++ b/dot/state/service_test.go @@ -148,7 +148,7 @@ func TestService_StorageTriePruning(t *testing.T) { defer utils.RemoveTestDir(t) retainBlocks := 2 - serv := NewService(testDir, log.LvlTrace, "full", int64(retainBlocks)) + serv := NewService(testDir, log.LvlInfo, FullNode, int64(retainBlocks)) serv.UseMemDB() genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) diff --git a/dot/state/storage.go b/dot/state/storage.go index 82903768ef..9613d2f452 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -56,7 +56,7 @@ type StorageState struct { } // NewStorageState creates a new StorageState backed by the given trie and database located at basePath. -func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, gcMode string, retainBlocks int64) (*StorageState, error) { +func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, pruningMode Pruning, retainBlocks int64) (*StorageState, error) { if db == nil { return nil, fmt.Errorf("cannot have nil database") } @@ -69,7 +69,7 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, tries[t.MustHash()] = t var pruner Pruner - if gcMode == "full" { + if pruningMode == FullNode { var err error pruner, err = createPruner(db, retainBlocks) if err != nil { @@ -128,11 +128,8 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState, header *types.Header) if err != nil { return fmt.Errorf("failed to get state trie inserted keys: block %s %w", header.Hash(), err) } - logger.Debug("storage", "inserted keys", len(insKeys), "for block", header.Number) delKeys := ts.GetDeletedNodeHashes() - - logger.Debug("storage ", "deleted keys", len(delKeys), "for block", header.Number) err = s.pruner.storeJournalRecord(delKeys, insKeys, header.Hash(), header.Number.Int64()) if err != nil { return err diff --git a/dot/sync/syncer.go b/dot/sync/syncer.go index 08d2b053fa..2760252aa1 100644 --- a/dot/sync/syncer.go +++ b/dot/sync/syncer.go @@ -24,7 +24,6 @@ import ( "os" "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/blocktree" @@ -57,7 +56,6 @@ type Service struct { // Consensus digest handling digestHandler DigestHandler - pruner state.Pruner } // Config is the configuration for the sync Service. @@ -71,7 +69,6 @@ type Config struct { Runtime runtime.Instance Verifier Verifier DigestHandler DigestHandler - Pruner state.Pruner } // NewService returns a new *sync.Service @@ -117,7 +114,6 @@ func NewService(cfg *Config) (*Service, error) { runtime: cfg.Runtime, verifier: cfg.Verifier, digestHandler: cfg.DigestHandler, - pruner: cfg.Pruner, }, nil } From 402a64276cc78ffe06080d63cee5763d60ac1ad9 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Fri, 4 Jun 2021 17:43:20 +0530 Subject: [PATCH 12/18] Address comments. --- cmd/gossamer/config.go | 5 +- cmd/gossamer/export.go | 2 +- cmd/gossamer/main.go | 3 +- dot/build_spec.go | 6 ++- dot/config.go | 4 +- dot/core/digest_test.go | 7 ++- dot/core/test_helpers.go | 6 ++- dot/import.go | 6 ++- dot/node.go | 7 ++- dot/node_test.go | 6 ++- dot/rpc/modules/chain_test.go | 7 ++- dot/services.go | 16 +++++- dot/state/{ => pruner}/pruner.go | 89 ++++++++++++++------------------ dot/state/service.go | 25 ++++++--- dot/state/service_test.go | 52 ++++++++++++++++--- dot/state/storage.go | 23 +++++---- dot/sync/syncer_test.go | 7 ++- lib/babe/babe_test.go | 7 ++- lib/babe/verify_test.go | 6 ++- 19 files changed, 193 insertions(+), 91 deletions(-) rename dot/state/{ => pruner}/pruner.go (71%) diff --git a/cmd/gossamer/config.go b/cmd/gossamer/config.go index 83a57318aa..ff75abf867 100644 --- a/cmd/gossamer/config.go +++ b/cmd/gossamer/config.go @@ -25,6 +25,7 @@ import ( "github.com/ChainSafe/gossamer/dot" ctoml "github.com/ChainSafe/gossamer/dot/config/toml" "github.com/ChainSafe/gossamer/dot/state" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" @@ -445,7 +446,7 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) { cfg.MetricsPort = tomlCfg.Global.MetricsPort cfg.RetainBlocks = tomlCfg.Global.RetainBlocks - cfg.Pruning = state.Pruning(tomlCfg.Global.Pruning) + cfg.Pruning = pruner.Mode(tomlCfg.Global.Pruning) } } @@ -476,7 +477,7 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) { } cfg.RetainBlocks = ctx.GlobalInt64(RetainBlockNumberFlag.Name) - cfg.Pruning = state.Pruning(ctx.GlobalString(Pruning.Name)) + cfg.Pruning = pruner.Mode(ctx.GlobalString(Pruning.Name)) cfg.NoTelemetry = ctx.Bool("no-telemetry") } diff --git a/cmd/gossamer/export.go b/cmd/gossamer/export.go index b9cb63cc32..b088137d2d 100644 --- a/cmd/gossamer/export.go +++ b/cmd/gossamer/export.go @@ -85,7 +85,7 @@ func dotConfigToToml(dcfg *dot.Config) *ctoml.Config { LogLvl: dcfg.Global.LogLvl.String(), MetricsPort: dcfg.Global.MetricsPort, RetainBlocks: dcfg.Global.RetainBlocks, - Pruning: dcfg.Global.Pruning.String(), + Pruning: string(dcfg.Global.Pruning), } cfg.Log = ctoml.LogConfig{ diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index d399bd2ee3..1d8413816d 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -23,6 +23,7 @@ import ( "github.com/ChainSafe/gossamer/dot" "github.com/ChainSafe/gossamer/dot/state" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/utils" log "github.com/ChainSafe/log15" @@ -240,7 +241,7 @@ func gossamerAction(ctx *cli.Context) error { } if !cfg.Global.Pruning.IsValid() { - return fmt.Errorf("--%s must be either %s or %s", state.FullNode, state.ArchiveNode, Pruning.Name) + return fmt.Errorf("--%s must be either %s or %s", pruner.Full, pruner.Archive, Pruning.Name) } cfg.Global.LogLvl = lvl diff --git a/dot/build_spec.go b/dot/build_spec.go index ee6d01ca1a..7abe4a2727 100644 --- a/dot/build_spec.go +++ b/dot/build_spec.go @@ -107,7 +107,11 @@ func BuildFromDB(path string) (*BuildSpec, error) { tmpGen.Genesis.Raw = make(map[string]map[string]string) tmpGen.Genesis.Runtime = make(map[string]map[string]interface{}) - stateSrvc := state.NewService(path, log.LvlCrit, "", 0) + config := state.Config{ + Path: path, + LogLevel: log.LvlInfo, + } + stateSrvc := state.NewService(config) // start state service (initialise state database) err := stateSrvc.Start() diff --git a/dot/config.go b/dot/config.go index 7a500a8418..5dd2949edc 100644 --- a/dot/config.go +++ b/dot/config.go @@ -23,7 +23,7 @@ import ( "github.com/ChainSafe/gossamer/chain/gssmr" "github.com/ChainSafe/gossamer/chain/kusama" "github.com/ChainSafe/gossamer/chain/polkadot" - "github.com/ChainSafe/gossamer/dot/state" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/types" log "github.com/ChainSafe/log15" ) @@ -54,7 +54,7 @@ type GlobalConfig struct { MetricsPort uint32 NoTelemetry bool RetainBlocks int64 - Pruning state.Pruning + Pruning pruner.Mode } // LogConfig represents the log levels for individual packages diff --git a/dot/core/digest_test.go b/dot/core/digest_test.go index 8d16d19183..122e4070c5 100644 --- a/dot/core/digest_test.go +++ b/dot/core/digest_test.go @@ -35,7 +35,12 @@ import ( func newTestDigestHandler(t *testing.T, withBABE, withGrandpa bool) *DigestHandler { //nolint testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") require.NoError(t, err) - stateSrvc := state.NewService(testDatadirPath, log.LvlInfo, "", 0) + + config := state.Config{ + Path: testDatadirPath, + LogLevel: log.LvlInfo, + } + stateSrvc := state.NewService(config) stateSrvc.UseMemDB() gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t) diff --git a/dot/core/test_helpers.go b/dot/core/test_helpers.go index 5fc1975dba..5fd2b8629a 100644 --- a/dot/core/test_helpers.go +++ b/dot/core/test_helpers.go @@ -130,7 +130,11 @@ func NewTestService(t *testing.T, cfg *Config) *Service { gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t) if cfg.BlockState == nil || cfg.StorageState == nil || cfg.TransactionState == nil || cfg.EpochState == nil { - stateSrvc = state.NewService(testDatadirPath, log.LvlInfo, "", 0) + config := state.Config{ + Path: testDatadirPath, + LogLevel: log.LvlInfo, + } + stateSrvc = state.NewService(config) stateSrvc.UseMemDB() err = stateSrvc.Initialise(gen, genHeader, genTrie) diff --git a/dot/import.go b/dot/import.go index e888c8680a..6546871b42 100644 --- a/dot/import.go +++ b/dot/import.go @@ -45,7 +45,11 @@ func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error { log.Info("ImportState", "header", header) - srv := state.NewService(basepath, log.LvlInfo, "", 0) + config := state.Config{ + Path: basepath, + LogLevel: log.LvlInfo, + } + srv := state.NewService(config) return srv.Import(header, tr, firstSlot) } diff --git a/dot/node.go b/dot/node.go index 9699d7e2c9..9a02cd079b 100644 --- a/dot/node.go +++ b/dot/node.go @@ -90,8 +90,13 @@ func InitNode(cfg *Config) error { return fmt.Errorf("failed to create genesis block from trie: %w", err) } + config := state.Config{ + Path: cfg.Global.BasePath, + LogLevel: cfg.Global.LogLvl, + } + // create new state service - stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Global.LogLvl, "", 0) + stateSrvc := state.NewService(config) // initialise state service with genesis data, block, and trie err = stateSrvc.Initialise(gen, header, t) diff --git a/dot/node_test.go b/dot/node_test.go index 61f9d78617..455bebb1d4 100644 --- a/dot/node_test.go +++ b/dot/node_test.go @@ -212,7 +212,11 @@ func TestInitNode_LoadGenesisData(t *testing.T) { err := InitNode(cfg) require.NoError(t, err) - stateSrvc := state.NewService(cfg.Global.BasePath, log.LvlTrace, "", 0) + config := state.Config{ + Path: cfg.Global.BasePath, + LogLevel: log.LvlInfo, + } + stateSrvc := state.NewService(config) gen, err := genesis.NewGenesisFromJSONRaw(genPath) require.NoError(t, err) diff --git a/dot/rpc/modules/chain_test.go b/dot/rpc/modules/chain_test.go index f008e33357..595f723e47 100644 --- a/dot/rpc/modules/chain_test.go +++ b/dot/rpc/modules/chain_test.go @@ -280,7 +280,12 @@ var gen, genTrie, genesisHeader = newTestGenesisWithTrieAndHeader() func newTestStateService(t *testing.T) *state.Service { testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") require.NoError(t, err) - stateSrvc := state.NewService(testDatadirPath, log.LvlInfo, "", 0) + + config := state.Config{ + Path: testDatadirPath, + LogLevel: log.LvlInfo, + } + stateSrvc := state.NewService(config) stateSrvc.UseMemDB() err = stateSrvc.Initialise(gen, genesisHeader, genTrie) diff --git a/dot/services.go b/dot/services.go index 35b3858ad4..0f11be7e99 100644 --- a/dot/services.go +++ b/dot/services.go @@ -22,6 +22,7 @@ import ( "path/filepath" "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/core" "github.com/ChainSafe/gossamer/dot/network" @@ -53,7 +54,20 @@ func newInMemoryDB(path string) (chaindb.Database, error) { // createStateService creates the state service and initialise state database func createStateService(cfg *Config) (*state.Service, error) { logger.Debug("creating state service...") - stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Log.StateLvl, cfg.Global.Pruning, cfg.Global.RetainBlocks) + + config := state.Config{ + Path: cfg.Global.BasePath, + LogLevel: cfg.Log.StateLvl, + Pruning: struct { + Mode pruner.Mode + NumRetainedBlocks int64 + }{ + Mode: cfg.Global.Pruning, + NumRetainedBlocks: cfg.Global.RetainBlocks, + }, + } + + stateSrvc := state.NewService(config) // start state service (initialise state database) err := stateSrvc.Start() diff --git a/dot/state/pruner.go b/dot/state/pruner/pruner.go similarity index 71% rename from dot/state/pruner.go rename to dot/state/pruner/pruner.go index 7aed8a9d56..4b8e7e4319 100644 --- a/dot/state/pruner.go +++ b/dot/state/pruner/pruner.go @@ -1,4 +1,4 @@ -package state +package pruner import ( "fmt" @@ -8,6 +8,7 @@ import ( "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/scale" + log "github.com/ChainSafe/log15" ) const ( @@ -18,47 +19,35 @@ const ( // nolint const ( - FullNode = "full" - ArchiveNode = "archive" + Full = Mode("full") + Archive = Mode("archive") ) -// Pruning online pruning mode of historical state tries -type Pruning string +// Mode online pruning mode of historical state tries +type Mode string // IsValid checks whether the pruning mode is valid -func (p Pruning) IsValid() bool { +func (p Mode) IsValid() bool { switch p { - case FullNode: + case Full: return true - case ArchiveNode: + case Archive: return true default: return false } } -// String returns format of Pruning -func (p Pruning) String() string { - switch p { - case FullNode: - return "full" - case ArchiveNode: - return "archive" - default: - return "" - } -} - -// Pruner is implemented by fullNodePruner and archivalNodePruner. +// Pruner is implemented by FullNode and ArchiveNode. type Pruner interface { - storeJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error + StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error } -// archivalNodePruner is a no-op since we don't prune nodes in archive mode. -type archivalNodePruner struct{} +// ArchiveNode is a no-op since we don't prune nodes in archive mode. +type ArchiveNode struct{} -// storeJournalRecord for archive node doesn't do anything. -func (a *archivalNodePruner) storeJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { +// StoreJournalRecord for archive node doesn't do anything. +func (a *ArchiveNode) StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { return nil } @@ -69,8 +58,9 @@ type deathRecord struct { type deathRow []*deathRecord -// fullNodePruner stores state trie diff and allows online state trie pruning -type fullNodePruner struct { +// FullNode stores state trie diff and allows online state trie pruning +type FullNode struct { + logger log.Logger deathList []deathRow storageDB chaindb.Database journalDB chaindb.Database @@ -102,14 +92,15 @@ func newJournalRecord(hash common.Hash, insertedKeys, deletedKeys []common.Hash) } } -// createPruner creates a pruner -func createPruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { - p := &fullNodePruner{ +// NewFullNode creates a Pruner for full node. +func NewFullNode(db, storageDB chaindb.Database, retainBlocks int64, l log.Logger) (Pruner, error) { + p := &FullNode{ deathList: make([]deathRow, 0), deathIndex: make(map[common.Hash]int64), - storageDB: chaindb.NewTable(db, storagePrefix), + storageDB: storageDB, journalDB: chaindb.NewTable(db, journalPrefix), retainBlocks: retainBlocks, + logger: l, } blockNum, err := p.getLastPrunedIndex() @@ -117,7 +108,7 @@ func createPruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { return nil, err } - logger.Info("last pruned block", "block num", blockNum) + p.logger.Info("last pruned block", "block num", blockNum) blockNum++ p.pendingNumber = blockNum @@ -133,7 +124,7 @@ func createPruner(db chaindb.Database, retainBlocks int64) (Pruner, error) { } // StoreJournalRecord stores journal record into DB and add deathRow into deathList -func (p *fullNodePruner) storeJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { +func (p *FullNode) StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error { jr := newJournalRecord(blockHash, inserted, deleted) key := &journalKey{blockNum, blockHash} @@ -142,12 +133,12 @@ func (p *fullNodePruner) storeJournalRecord(deleted, inserted []common.Hash, blo return fmt.Errorf("failed to store journal record for %d: %w", blockNum, err) } - logger.Debug("journal record stored", "block num", blockNum) + p.logger.Debug("journal record stored", "block num", blockNum) p.addDeathRow(jr, blockNum) return nil } -func (p *fullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) { +func (p *FullNode) addDeathRow(jr *journalRecord, blockNum int64) { if blockNum == 0 { return } @@ -187,7 +178,7 @@ func (p *fullNodePruner) addDeathRow(jr *journalRecord, blockNum int64) { } // Remove re-inserted keys -func (p *fullNodePruner) processInsertedKeys(insKeys []common.Hash, blockHash common.Hash) { +func (p *FullNode) processInsertedKeys(insKeys []common.Hash, blockHash common.Hash) { for _, k := range insKeys { num, ok := p.deathIndex[k] if !ok { @@ -203,8 +194,8 @@ func (p *fullNodePruner) processInsertedKeys(insKeys []common.Hash, blockHash co } } -func (p *fullNodePruner) start() { - logger.Info("pruning started") +func (p *FullNode) start() { + p.logger.Info("pruning started") var canPrune bool checkPruning := func() { @@ -220,12 +211,12 @@ func (p *fullNodePruner) start() { row := p.deathList[0] blockNum := p.pendingNumber - logger.Debug("pruning block", "block num", blockNum) + p.logger.Debug("pruning block", "block num", blockNum) for _, record := range row { err := p.deleteKeys(record.deletedKeys) if err != nil { - logger.Warn("failed to prune keys", "block num", blockNum, "error", err) + p.logger.Warn("failed to prune keys", "block num", blockNum, "error", err) return } @@ -236,7 +227,7 @@ func (p *fullNodePruner) start() { err := p.storeLastPrunedIndex(blockNum) if err != nil { - logger.Error("failed to store last pruned index", "block num", blockNum, "error", err) + p.logger.Error("failed to store last pruned index", "block num", blockNum, "error", err) return } @@ -247,7 +238,7 @@ func (p *fullNodePruner) start() { jk := &journalKey{blockNum, record.blockHash} err = p.deleteJournalRecord(jk) if err != nil { - logger.Error("failed to delete journal record", "block num", blockNum, "error", err) + p.logger.Error("failed to delete journal record", "block num", blockNum, "error", err) return } } @@ -262,7 +253,7 @@ func (p *fullNodePruner) start() { } } -func (p *fullNodePruner) storeJournal(key *journalKey, jr *journalRecord) error { +func (p *FullNode) storeJournal(key *journalKey, jr *journalRecord) error { encKey, err := scale.Encode(key) if err != nil { return fmt.Errorf("failed to encode journal key block num %d: %w", key.blockNum, err) @@ -282,7 +273,7 @@ func (p *fullNodePruner) storeJournal(key *journalKey, jr *journalRecord) error } // loadDeathList loads deathList and deathIndex from journalRecord. -func (p *fullNodePruner) loadDeathList() error { +func (p *FullNode) loadDeathList() error { itr := p.journalDB.NewIterator() defer itr.Release() @@ -305,7 +296,7 @@ func (p *fullNodePruner) loadDeathList() error { return nil } -func (p *fullNodePruner) deleteJournalRecord(key *journalKey) error { +func (p *FullNode) deleteJournalRecord(key *journalKey) error { encKey, err := scale.Encode(key) if err != nil { return err @@ -319,7 +310,7 @@ func (p *fullNodePruner) deleteJournalRecord(key *journalKey) error { return nil } -func (p *fullNodePruner) storeLastPrunedIndex(blockNum int64) error { +func (p *FullNode) storeLastPrunedIndex(blockNum int64) error { encNum, err := scale.Encode(blockNum) if err != nil { return err @@ -333,7 +324,7 @@ func (p *fullNodePruner) storeLastPrunedIndex(blockNum int64) error { return nil } -func (p *fullNodePruner) getLastPrunedIndex() (int64, error) { +func (p *FullNode) getLastPrunedIndex() (int64, error) { val, err := p.journalDB.Get([]byte(lastPrunedKey)) if err == chaindb.ErrKeyNotFound { return 0, nil @@ -351,7 +342,7 @@ func (p *fullNodePruner) getLastPrunedIndex() (int64, error) { return blockNum.(int64), nil } -func (p *fullNodePruner) deleteKeys(nodesHash map[common.Hash]int64) error { +func (p *FullNode) deleteKeys(nodesHash map[common.Hash]int64) error { for k := range nodesHash { err := p.storageDB.Del(k.ToBytes()) if err != nil { diff --git a/dot/state/service.go b/dot/state/service.go index 4947990bc7..bbd8d3c09a 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/blocktree" "github.com/ChainSafe/gossamer/lib/trie" @@ -54,25 +55,35 @@ type Service struct { // Below are for state trie online pruner retainBlocks int64 - pruningMode Pruning + pruningMode pruner.Mode +} + +// Config is the default configuration used by state service. +type Config struct { + Path string + LogLevel log.Lvl + Pruning struct { + Mode pruner.Mode + NumRetainedBlocks int64 + } } // NewService create a new instance of Service -func NewService(path string, lvl log.Lvl, pruningMode Pruning, retainBlocks int64) *Service { +func NewService(config Config) *Service { handler := log.StreamHandler(os.Stdout, log.TerminalFormat()) handler = log.CallerFileHandler(handler) - logger.SetHandler(log.LvlFilterHandler(lvl, handler)) + logger.SetHandler(log.LvlFilterHandler(config.LogLevel, handler)) return &Service{ - dbPath: path, - logLvl: lvl, + dbPath: config.Path, + logLvl: config.LogLevel, db: nil, isMemDB: false, Storage: nil, Block: nil, closeCh: make(chan interface{}), - pruningMode: pruningMode, - retainBlocks: retainBlocks, + pruningMode: config.Pruning.Mode, + retainBlocks: config.Pruning.NumRetainedBlocks, } } diff --git a/dot/state/service_test.go b/dot/state/service_test.go index d88f6d4688..ae2ad7f034 100644 --- a/dot/state/service_test.go +++ b/dot/state/service_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" @@ -50,13 +51,21 @@ func newTestGenesisWithTrieAndHeader(t *testing.T) (*genesis.Genesis, *trie.Trie // helper method to create and start test state service func newTestService(t *testing.T) (state *Service) { testDir := utils.NewTestDir(t) - state = NewService(testDir, log.LvlTrace, "", 0) + config := Config{ + Path: testDir, + LogLevel: log.LvlInfo, + } + state = NewService(config) return state } func newTestMemDBService() *Service { testDatadirPath, _ := ioutil.TempDir("/tmp", "test-datadir-*") - state := NewService(testDatadirPath, log.LvlTrace, "", 0) + config := Config{ + Path: testDatadirPath, + LogLevel: log.LvlInfo, + } + state := NewService(config) state.UseMemDB() return state } @@ -118,7 +127,11 @@ func TestService_BlockTree(t *testing.T) { // removes all data directories created within test directory defer utils.RemoveTestDir(t) - stateA := NewService(testDir, log.LvlTrace, "", 0) + config := Config{ + Path: testDir, + LogLevel: log.LvlInfo, + } + stateA := NewService(config) genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) err := stateA.Initialise(genData, genesisHeader, genTrie) @@ -133,7 +146,7 @@ func TestService_BlockTree(t *testing.T) { err = stateA.Stop() require.NoError(t, err) - stateB := NewService(testDir, log.LvlTrace, "", 0) + stateB := NewService(config) err = stateB.Start() require.NoError(t, err) @@ -148,7 +161,18 @@ func TestService_StorageTriePruning(t *testing.T) { defer utils.RemoveTestDir(t) retainBlocks := 2 - serv := NewService(testDir, log.LvlInfo, FullNode, int64(retainBlocks)) + config := Config{ + Path: testDir, + LogLevel: log.LvlInfo, + Pruning: struct { + Mode pruner.Mode + NumRetainedBlocks int64 + }{ + Mode: pruner.Full, + NumRetainedBlocks: int64(retainBlocks), + }, + } + serv := NewService(config) serv.UseMemDB() genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) @@ -197,7 +221,11 @@ func TestService_PruneStorage(t *testing.T) { testDir := utils.NewTestDir(t) defer utils.RemoveTestDir(t) - serv := NewService(testDir, log.LvlTrace, "", 0) + config := Config{ + Path: testDir, + LogLevel: log.LvlInfo, + } + serv := NewService(config) serv.UseMemDB() genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) @@ -272,7 +300,11 @@ func TestService_Rewind(t *testing.T) { testDir := utils.NewTestDir(t) defer utils.RemoveTestDir(t) - serv := NewService(testDir, log.LvlTrace, "", 0) + config := Config{ + Path: testDir, + LogLevel: log.LvlInfo, + } + serv := NewService(config) serv.UseMemDB() genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) @@ -320,7 +352,11 @@ func TestService_Import(t *testing.T) { testDir := utils.NewTestDir(t) defer utils.RemoveTestDir(t) - serv := NewService(testDir, log.LvlTrace, "", 0) + config := Config{ + Path: testDir, + LogLevel: log.LvlInfo, + } + serv := NewService(config) serv.UseMemDB() genData, genTrie, genesisHeader := newTestGenesisWithTrieAndHeader(t) diff --git a/dot/state/storage.go b/dot/state/storage.go index 9613d2f452..48f7059b1b 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" @@ -51,12 +52,12 @@ type StorageState struct { // change notifiers changedLock sync.RWMutex observerList []Observer - pruner Pruner + pruner pruner.Pruner syncing bool } // NewStorageState creates a new StorageState backed by the given trie and database located at basePath. -func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, pruningMode Pruning, retainBlocks int64) (*StorageState, error) { +func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, pruningMode pruner.Mode, retainBlocks int64) (*StorageState, error) { if db == nil { return nil, fmt.Errorf("cannot have nil database") } @@ -68,23 +69,25 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, tries := make(map[common.Hash]*trie.Trie) tries[t.MustHash()] = t - var pruner Pruner - if pruningMode == FullNode { + storageTable := chaindb.NewTable(db, storagePrefix) + + var p pruner.Pruner + if pruningMode == pruner.Full { var err error - pruner, err = createPruner(db, retainBlocks) + p, err = pruner.NewFullNode(db, storageTable, retainBlocks, logger) if err != nil { return nil, err } } else { - pruner = &archivalNodePruner{} + p = &pruner.ArchiveNode{} } return &StorageState{ blockState: blockState, tries: tries, - db: chaindb.NewTable(db, storagePrefix), + db: storageTable, observerList: []Observer{}, - pruner: pruner, + pruner: p, }, nil } @@ -119,7 +122,7 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState, header *types.Header) s.tries[root] = ts.Trie() s.lock.Unlock() - if _, ok := s.pruner.(*fullNodePruner); header == nil && ok { + if _, ok := s.pruner.(*pruner.FullNode); header == nil && ok { return fmt.Errorf("block cannot be empty for Full node pruner") } @@ -130,7 +133,7 @@ func (s *StorageState) StoreTrie(ts *rtstorage.TrieState, header *types.Header) } delKeys := ts.GetDeletedNodeHashes() - err = s.pruner.storeJournalRecord(delKeys, insKeys, header.Hash(), header.Number.Int64()) + err = s.pruner.StoreJournalRecord(delKeys, insKeys, header.Hash(), header.Number.Int64()) if err != nil { return err } diff --git a/dot/sync/syncer_test.go b/dot/sync/syncer_test.go index ba986ef45a..8f8cb4eaf3 100644 --- a/dot/sync/syncer_test.go +++ b/dot/sync/syncer_test.go @@ -65,7 +65,12 @@ func newTestSyncer(t *testing.T) *Service { cfg := &Config{} testDatadirPath, _ := ioutil.TempDir("/tmp", "test-datadir-*") - stateSrvc := state.NewService(testDatadirPath, log.LvlInfo, "", 0) + + config := state.Config{ + Path: testDatadirPath, + LogLevel: log.LvlInfo, + } + stateSrvc := state.NewService(config) stateSrvc.UseMemDB() gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t) diff --git a/lib/babe/babe_test.go b/lib/babe/babe_test.go index 9bd989dab9..d80ca4b6b7 100644 --- a/lib/babe/babe_test.go +++ b/lib/babe/babe_test.go @@ -109,7 +109,12 @@ func createTestService(t *testing.T, cfg *ServiceConfig) *Service { if cfg.BlockState == nil || cfg.StorageState == nil || cfg.EpochState == nil { testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") //nolint require.NoError(t, err) - dbSrv := state.NewService(testDatadirPath, log.LvlInfo, "", 0) + + config := state.Config{ + Path: testDatadirPath, + LogLevel: log.LvlInfo, + } + dbSrv := state.NewService(config) dbSrv.UseMemDB() if cfg.EpochLength > 0 { diff --git a/lib/babe/verify_test.go b/lib/babe/verify_test.go index 48fb838fef..50a03dddbe 100644 --- a/lib/babe/verify_test.go +++ b/lib/babe/verify_test.go @@ -36,7 +36,11 @@ func newTestVerificationManager(t *testing.T, genCfg *types.BabeConfiguration) * testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") require.NoError(t, err) - dbSrv := state.NewService(testDatadirPath, log.LvlInfo, "", 0) + config := state.Config{ + Path: testDatadirPath, + LogLevel: log.LvlInfo, + } + dbSrv := state.NewService(config) dbSrv.UseMemDB() if genCfg == nil { From e2844a3484bdb848e374a2f58c185041299df0bf Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 8 Jun 2021 19:05:01 +0530 Subject: [PATCH 13/18] Temp --- cmd/gossamer/main.go | 2 +- dot/state/initialize.go | 3 +-- dot/state/offline_pruner.go | 4 ++-- dot/state/pruner/pruner.go | 31 +++++++++++++++++++++++-------- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index 1d8413816d..246256e549 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -453,7 +453,7 @@ func pruneState(ctx *cli.Context) error { bloomSize := ctx.GlobalUint64(BloomFilterSizeFlag.Name) retainBlocks := ctx.GlobalInt64(RetainBlockNumberFlag.Name) - pruner, err := state.NewPruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks) + pruner, err := state.NewOfflinePruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks) if err != nil { return err } diff --git a/dot/state/initialize.go b/dot/state/initialize.go index f5d1533718..456bd8dfd7 100644 --- a/dot/state/initialize.go +++ b/dot/state/initialize.go @@ -21,6 +21,7 @@ import ( "fmt" "path/filepath" + "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/blocktree" "github.com/ChainSafe/gossamer/lib/genesis" @@ -29,8 +30,6 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime/wasmer" "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" - - "github.com/ChainSafe/chaindb" ) // Initialise initialises the genesis state of the DB using the given storage trie. The trie should be loaded with the genesis storage state. diff --git a/dot/state/offline_pruner.go b/dot/state/offline_pruner.go index 9b36729fcf..b152263f88 100644 --- a/dot/state/offline_pruner.go +++ b/dot/state/offline_pruner.go @@ -30,8 +30,8 @@ type OfflinePruner struct { prunedDBPath string } -// NewPruner creates an instance of OfflinePruner. -func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNum int64) (*OfflinePruner, error) { +// NewOfflinePruner creates an instance of OfflinePruner. +func NewOfflinePruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNum int64) (*OfflinePruner, error) { db, err := utils.LoadChainDB(inputDBPath) if err != nil { return nil, fmt.Errorf("failed to load DB %w", err) diff --git a/dot/state/pruner/pruner.go b/dot/state/pruner/pruner.go index 4b8e7e4319..c26d92f868 100644 --- a/dot/state/pruner/pruner.go +++ b/dot/state/pruner/pruner.go @@ -213,10 +213,12 @@ func (p *FullNode) start() { p.logger.Debug("pruning block", "block num", blockNum) + sdbBatch := p.storageDB.NewBatch() for _, record := range row { - err := p.deleteKeys(record.deletedKeys) + err := p.deleteKeys(sdbBatch, record.deletedKeys) if err != nil { p.logger.Warn("failed to prune keys", "block num", blockNum, "error", err) + sdbBatch.Reset() return } @@ -225,23 +227,36 @@ func (p *FullNode) start() { } } + if err := sdbBatch.Flush(); err != nil { + p.logger.Warn("failed to prune keys", "block num", blockNum, "error", err) + return + } + err := p.storeLastPrunedIndex(blockNum) if err != nil { - p.logger.Error("failed to store last pruned index", "block num", blockNum, "error", err) + p.logger.Warn("failed to store last pruned index", "block num", blockNum, "error", err) return } p.deathList = p.deathList[1:] p.pendingNumber++ + jdbBatch := p.journalDB.NewBatch() for _, record := range row { jk := &journalKey{blockNum, record.blockHash} - err = p.deleteJournalRecord(jk) + err = p.deleteJournalRecord(jdbBatch, jk) if err != nil { - p.logger.Error("failed to delete journal record", "block num", blockNum, "error", err) + p.logger.Warn("failed to delete journal record", "block num", blockNum, "error", err) + jdbBatch.Reset() return } } + + if err = jdbBatch.Flush(); err != nil { + p.logger.Warn("failed to delete journal record", "block num", blockNum, "error", err) + return + } + p.logger.Debug("pruned block", "block num", blockNum) } for { @@ -296,13 +311,13 @@ func (p *FullNode) loadDeathList() error { return nil } -func (p *FullNode) deleteJournalRecord(key *journalKey) error { +func (p *FullNode) deleteJournalRecord(b chaindb.Batch, key *journalKey) error { encKey, err := scale.Encode(key) if err != nil { return err } - err = p.journalDB.Del(encKey) + err = b.Del(encKey) if err != nil { return err } @@ -342,9 +357,9 @@ func (p *FullNode) getLastPrunedIndex() (int64, error) { return blockNum.(int64), nil } -func (p *FullNode) deleteKeys(nodesHash map[common.Hash]int64) error { +func (p *FullNode) deleteKeys(b chaindb.Batch, nodesHash map[common.Hash]int64) error { for k := range nodesHash { - err := p.storageDB.Del(k.ToBytes()) + err := b.Del(k.ToBytes()) if err != nil { return err } diff --git a/go.mod b/go.mod index 21adbad85a..d2685b92e1 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/ChainSafe/gossamer require ( - github.com/ChainSafe/chaindb v0.1.5-0.20210117220933-15e75f27268f + github.com/ChainSafe/chaindb v0.1.5-0.20210608140454-9606fe8c3985 github.com/ChainSafe/go-schnorrkel v0.0.0-20210222182958-bd440c890782 github.com/ChainSafe/log15 v1.0.0 github.com/OneOfOne/xxhash v1.2.5 diff --git a/go.sum b/go.sum index fb63c9d20f..361a842c8c 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/ChainSafe/chaindb v0.1.5-0.20210117220933-15e75f27268f h1:qDmWdUIE1cgG19K/eVB9nkQMkldaGwcjU9U5OyUV11k= -github.com/ChainSafe/chaindb v0.1.5-0.20210117220933-15e75f27268f/go.mod h1:WBsCSLGM7+DvSYU6cFVUltahwU7Sw4cN3e8kiLdNFJM= +github.com/ChainSafe/chaindb v0.1.5-0.20210608140454-9606fe8c3985 h1:jyFsOjzTMoRwNvmW/OORZpffmItkoLvsbxB8koHX4ns= +github.com/ChainSafe/chaindb v0.1.5-0.20210608140454-9606fe8c3985/go.mod h1:P01m9E6xj6Mps1rtf7SurEX9oOcy1jYEyccZQAEw9+4= github.com/ChainSafe/go-schnorrkel v0.0.0-20210222182958-bd440c890782 h1:lwmjzta2Xu+3rPVY/VeNQj2xfNkyih4CwyRxYg3cpRQ= github.com/ChainSafe/go-schnorrkel v0.0.0-20210222182958-bd440c890782/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= github.com/ChainSafe/log15 v1.0.0 h1:vRDVtWtVwIH5uSCBvgTTZh6FA58UBJ6+QiiypaZfBf8= From a7cb2f12227ef06cf6c459e911f34da2f58c57bc Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Mon, 14 Jun 2021 17:57:46 +0530 Subject: [PATCH 14/18] Address comments. --- cmd/gossamer/config.go | 4 ++-- cmd/gossamer/flags.go | 12 ++++++------ cmd/gossamer/main.go | 24 ++++++++++++++---------- dot/node.go | 8 ++++++++ dot/services.go | 9 --------- dot/state/base.go | 27 +++++++++++++++++++++++++-- dot/state/initialize.go | 7 ++++++- dot/state/offline_pruner.go | 3 ++- dot/state/pruner/pruner.go | 6 ++++++ dot/state/service.go | 36 ++++++++++++++++++------------------ dot/state/service_test.go | 9 +++------ dot/state/storage.go | 6 +++--- dot/state/storage_test.go | 3 ++- lib/common/db_keys.go | 2 ++ 14 files changed, 97 insertions(+), 59 deletions(-) diff --git a/cmd/gossamer/config.go b/cmd/gossamer/config.go index ff75abf867..be2cc83fab 100644 --- a/cmd/gossamer/config.go +++ b/cmd/gossamer/config.go @@ -476,8 +476,8 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) { cfg.MetricsPort = uint32(metricsPort) } - cfg.RetainBlocks = ctx.GlobalInt64(RetainBlockNumberFlag.Name) - cfg.Pruning = pruner.Mode(ctx.GlobalString(Pruning.Name)) + cfg.RetainBlocks = ctx.Int64(RetainBlockNumberFlag.Name) + cfg.Pruning = pruner.Mode(ctx.String(PruningFlag.Name)) cfg.NoTelemetry = ctx.Bool("no-telemetry") } diff --git a/cmd/gossamer/flags.go b/cmd/gossamer/flags.go index 1c464db955..e9efdf04be 100644 --- a/cmd/gossamer/flags.go +++ b/cmd/gossamer/flags.go @@ -283,15 +283,15 @@ var ( RetainBlockNumberFlag = cli.IntFlag{ Name: "retain-blocks", Usage: "Retain number of block from latest block while pruning", - Value: 256, + Value: 512, } - // Pruning triggers the online pruning of historical state tries. It's either full or archive. To enable pruning the value + // PruningFlag triggers the online pruning of historical state tries. It's either full or archive. To enable pruning the value // should be set to `full`. - Pruning = cli.StringFlag{ + PruningFlag = cli.StringFlag{ Name: "pruning", Usage: `State trie online pruning ("full", "archive")`, - Value: "full", + Value: "archive", } ) @@ -309,8 +309,6 @@ var ( RewindFlag, DBPathFlag, BloomFilterSizeFlag, - RetainBlockNumberFlag, - Pruning, } // StartupFlags are flags that are valid for use with the root command and the export subcommand @@ -355,6 +353,8 @@ var ( InitFlags = append([]cli.Flag{ ForceFlag, GenesisFlag, + PruningFlag, + RetainBlockNumberFlag, }, GlobalFlags...) BuildSpecFlags = append([]cli.Flag{ diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index 246256e549..74b02b2365 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -38,7 +38,7 @@ const ( importRuntimeCommandName = "import-runtime" importStateCommandName = "import-state" pruningStateCommandName = "prune-state" - defaultRetainBlocks = 256 + defaultRetainBlocks = 512 ) // app is the cli application @@ -236,14 +236,6 @@ func gossamerAction(ctx *cli.Context) error { return err } - if cfg.Global.RetainBlocks < defaultRetainBlocks { - return fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, defaultRetainBlocks) - } - - if !cfg.Global.Pruning.IsValid() { - return fmt.Errorf("--%s must be either %s or %s", pruner.Full, pruner.Archive, Pruning.Name) - } - cfg.Global.LogLvl = lvl // expand data directory and update node configuration (performed separately @@ -344,6 +336,18 @@ func initAction(ctx *cli.Context) error { // from createDotConfig because dot config should not include expanded path) cfg.Global.BasePath = utils.ExpandDir(cfg.Global.BasePath) + mode := pruner.Mode(ctx.String(PruningFlag.Name)) + if !mode.IsValid() { + return fmt.Errorf("--%s must be either %s or %s", PruningFlag.Name, pruner.Full, pruner.Archive) + } + cfg.Global.Pruning = mode + + rb := ctx.Int64(RetainBlockNumberFlag.Name) + if rb < defaultRetainBlocks { + return fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, defaultRetainBlocks) + } + cfg.Global.RetainBlocks = rb + // check if node has been initialised (expected false - no warning log) if dot.NodeInitialized(cfg.Global.BasePath, false) { @@ -451,7 +455,7 @@ func pruneState(ctx *cli.Context) error { } bloomSize := ctx.GlobalUint64(BloomFilterSizeFlag.Name) - retainBlocks := ctx.GlobalInt64(RetainBlockNumberFlag.Name) + retainBlocks := ctx.Int64(RetainBlockNumberFlag.Name) pruner, err := state.NewOfflinePruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks) if err != nil { diff --git a/dot/node.go b/dot/node.go index 9a02cd079b..3f128ad525 100644 --- a/dot/node.go +++ b/dot/node.go @@ -31,6 +31,7 @@ import ( gssmrmetrics "github.com/ChainSafe/gossamer/dot/metrics" "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/state" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/genesis" @@ -93,6 +94,13 @@ func InitNode(cfg *Config) error { config := state.Config{ Path: cfg.Global.BasePath, LogLevel: cfg.Global.LogLvl, + PrunerCfg: struct { + Mode pruner.Mode + RetainedBlocks int64 + }{ + Mode: cfg.Global.Pruning, + RetainedBlocks: cfg.Global.RetainBlocks, + }, } // create new state service diff --git a/dot/services.go b/dot/services.go index 0f11be7e99..e0910abf17 100644 --- a/dot/services.go +++ b/dot/services.go @@ -22,8 +22,6 @@ import ( "path/filepath" "github.com/ChainSafe/chaindb" - "github.com/ChainSafe/gossamer/dot/state/pruner" - "github.com/ChainSafe/gossamer/dot/core" "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/rpc" @@ -58,13 +56,6 @@ func createStateService(cfg *Config) (*state.Service, error) { config := state.Config{ Path: cfg.Global.BasePath, LogLevel: cfg.Log.StateLvl, - Pruning: struct { - Mode pruner.Mode - NumRetainedBlocks int64 - }{ - Mode: cfg.Global.Pruning, - NumRetainedBlocks: cfg.Global.RetainBlocks, - }, } stateSrvc := state.NewService(config) diff --git a/dot/state/base.go b/dot/state/base.go index f66d462096..20e12ce7c0 100644 --- a/dot/state/base.go +++ b/dot/state/base.go @@ -21,10 +21,10 @@ import ( "encoding/json" "fmt" + "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" - - "github.com/ChainSafe/chaindb" ) // BaseState is a wrapper for the chaindb.Database, without any prefixes @@ -139,3 +139,26 @@ func (s *BaseState) loadFirstSlot() (uint64, error) { return binary.LittleEndian.Uint64(data), nil } + +// storePruningData stores the pruner configuration. +func (s *BaseState) storePruningData(mode pruner.Config) error { + encMode, err := json.Marshal(mode) + if err != nil { + return fmt.Errorf("cannot scale encode pruning mode: %s", err) + } + + return s.db.Put(common.PruningKey, encMode) +} + +// loadPruningData retrieves pruner configuration from db. +func (s *BaseState) loadPruningData() (pruner.Config, error) { + data, err := s.db.Get(common.PruningKey) + if err != nil { + return pruner.Config{}, err + } + + mode := &pruner.Config{} + err = json.Unmarshal(data, mode) + + return *mode, err +} diff --git a/dot/state/initialize.go b/dot/state/initialize.go index 456bd8dfd7..e4a6afdac1 100644 --- a/dot/state/initialize.go +++ b/dot/state/initialize.go @@ -22,6 +22,7 @@ import ( "path/filepath" "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/blocktree" "github.com/ChainSafe/gossamer/lib/genesis" @@ -88,7 +89,7 @@ func (s *Service) Initialise(gen *genesis.Genesis, header *types.Header, t *trie } // create storage state from genesis trie - storageState, err := NewStorageState(db, blockState, t, "", 0) + storageState, err := NewStorageState(db, blockState, t, pruner.Config{}) if err != nil { return fmt.Errorf("failed to create storage state from trie: %s", err) } @@ -173,6 +174,10 @@ func (s *Service) storeInitialValues(data *genesis.Data, header *types.Header, t return fmt.Errorf("failed to write genesis data to database: %s", err) } + if err := s.Base.storePruningData(s.PrunerCfg); err != nil { + return fmt.Errorf("failed to write pruning data to database: %s", err) + } + return nil } diff --git a/dot/state/offline_pruner.go b/dot/state/offline_pruner.go index b152263f88..8ebe28f36e 100644 --- a/dot/state/offline_pruner.go +++ b/dot/state/offline_pruner.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/lib/blocktree" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/trie" @@ -62,7 +63,7 @@ func NewOfflinePruner(inputDBPath, prunedDBPath string, bloomSize uint64, retain } // load storage state - storageState, err := NewStorageState(db, blockState, trie.NewEmptyTrie(), "", 0) + storageState, err := NewStorageState(db, blockState, trie.NewEmptyTrie(), pruner.Config{}) if err != nil { return nil, fmt.Errorf("failed to create new storage state %w", err) } diff --git a/dot/state/pruner/pruner.go b/dot/state/pruner/pruner.go index c26d92f868..1bf8da2f80 100644 --- a/dot/state/pruner/pruner.go +++ b/dot/state/pruner/pruner.go @@ -38,6 +38,12 @@ func (p Mode) IsValid() bool { } } +// Config holds state trie pruning mode and retained blocks +type Config struct { + Mode Mode + RetainedBlocks int64 +} + // Pruner is implemented by FullNode and ArchiveNode. type Pruner interface { StoreJournalRecord(deleted, inserted []common.Hash, blockHash common.Hash, blockNum int64) error diff --git a/dot/state/service.go b/dot/state/service.go index bbd8d3c09a..5871595edf 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -54,18 +54,14 @@ type Service struct { BabeThresholdDenominator uint64 // Below are for state trie online pruner - retainBlocks int64 - pruningMode pruner.Mode + PrunerCfg pruner.Config } // Config is the default configuration used by state service. type Config struct { - Path string - LogLevel log.Lvl - Pruning struct { - Mode pruner.Mode - NumRetainedBlocks int64 - } + Path string + LogLevel log.Lvl + PrunerCfg pruner.Config } // NewService create a new instance of Service @@ -75,15 +71,14 @@ func NewService(config Config) *Service { logger.SetHandler(log.LvlFilterHandler(config.LogLevel, handler)) return &Service{ - dbPath: config.Path, - logLvl: config.LogLevel, - db: nil, - isMemDB: false, - Storage: nil, - Block: nil, - closeCh: make(chan interface{}), - pruningMode: config.Pruning.Mode, - retainBlocks: config.Pruning.NumRetainedBlocks, + dbPath: config.Path, + logLvl: config.LogLevel, + db: nil, + isMemDB: false, + Storage: nil, + Block: nil, + closeCh: make(chan interface{}), + PrunerCfg: config.PrunerCfg, } } @@ -157,8 +152,13 @@ func (s *Service) Start() error { s.Block.bt = blocktree.NewBlockTreeFromRoot(lastFinalised, db) } + pruner, err := s.Base.loadPruningData() + if err != nil { + return err + } + // create storage state - s.Storage, err = NewStorageState(db, s.Block, trie.NewEmptyTrie(), s.pruningMode, s.retainBlocks) + s.Storage, err = NewStorageState(db, s.Block, trie.NewEmptyTrie(), pruner) if err != nil { return fmt.Errorf("failed to create storage state: %w", err) } diff --git a/dot/state/service_test.go b/dot/state/service_test.go index ae2ad7f034..bc987865bf 100644 --- a/dot/state/service_test.go +++ b/dot/state/service_test.go @@ -164,12 +164,9 @@ func TestService_StorageTriePruning(t *testing.T) { config := Config{ Path: testDir, LogLevel: log.LvlInfo, - Pruning: struct { - Mode pruner.Mode - NumRetainedBlocks int64 - }{ - Mode: pruner.Full, - NumRetainedBlocks: int64(retainBlocks), + PrunerCfg: pruner.Config{ + Mode: pruner.Full, + RetainedBlocks: int64(retainBlocks), }, } serv := NewService(config) diff --git a/dot/state/storage.go b/dot/state/storage.go index 48f7059b1b..a014bf535b 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -57,7 +57,7 @@ type StorageState struct { } // NewStorageState creates a new StorageState backed by the given trie and database located at basePath. -func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, pruningMode pruner.Mode, retainBlocks int64) (*StorageState, error) { +func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, onlinePruner pruner.Config) (*StorageState, error) { if db == nil { return nil, fmt.Errorf("cannot have nil database") } @@ -72,9 +72,9 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, storageTable := chaindb.NewTable(db, storagePrefix) var p pruner.Pruner - if pruningMode == pruner.Full { + if onlinePruner.Mode == pruner.Full { var err error - p, err = pruner.NewFullNode(db, storageTable, retainBlocks, logger) + p, err = pruner.NewFullNode(db, storageTable, onlinePruner.RetainedBlocks, logger) if err != nil { return nil, err } diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index f49534bfb4..edbd2d7fcc 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/types" runtime "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/trie" @@ -16,7 +17,7 @@ func newTestStorageState(t *testing.T) *StorageState { db := NewInMemoryDB(t) bs := newTestBlockState(t, testGenesisHeader) - s, err := NewStorageState(db, bs, trie.NewEmptyTrie(), "", 0) + s, err := NewStorageState(db, bs, trie.NewEmptyTrie(), pruner.Config{}) require.NoError(t, err) return s } diff --git a/lib/common/db_keys.go b/lib/common/db_keys.go index 2de8019943..c7e3fff783 100644 --- a/lib/common/db_keys.go +++ b/lib/common/db_keys.go @@ -33,4 +33,6 @@ var ( WorkingStorageHashKey = []byte("working_storage_hash") //NodeNameKey is the storage key to store de current node name and avoid create a new name every initialization NodeNameKey = []byte("node_name") + // PruningKey is the storage key to store the current pruning mode. + PruningKey = []byte("prune") ) From 82b11d6c5b36c3fe56c639fd37c8fab015bc37c2 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Mon, 14 Jun 2021 17:59:10 +0530 Subject: [PATCH 15/18] Minor change. --- lib/common/db_keys.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/db_keys.go b/lib/common/db_keys.go index c7e3fff783..781394a864 100644 --- a/lib/common/db_keys.go +++ b/lib/common/db_keys.go @@ -33,6 +33,6 @@ var ( WorkingStorageHashKey = []byte("working_storage_hash") //NodeNameKey is the storage key to store de current node name and avoid create a new name every initialization NodeNameKey = []byte("node_name") - // PruningKey is the storage key to store the current pruning mode. + // PruningKey is the storage key to store the current pruning configuration. PruningKey = []byte("prune") ) From b07af753c297a3db75bea92b1ce1c37824bcf6cf Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Mon, 14 Jun 2021 18:00:14 +0530 Subject: [PATCH 16/18] Minor fix. --- dot/state/base.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dot/state/base.go b/dot/state/base.go index 20e12ce7c0..c657f36e97 100644 --- a/dot/state/base.go +++ b/dot/state/base.go @@ -157,8 +157,8 @@ func (s *BaseState) loadPruningData() (pruner.Config, error) { return pruner.Config{}, err } - mode := &pruner.Config{} - err = json.Unmarshal(data, mode) + var mode pruner.Config + err = json.Unmarshal(data, &mode) - return *mode, err + return mode, err } From 8faf10c10af80223bc57f99eb8904266e8b4ad00 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Mon, 14 Jun 2021 23:00:35 +0530 Subject: [PATCH 17/18] Fix tests. --- cmd/gossamer/flags.go | 2 +- dot/state/service_test.go | 10 ++-------- dot/state/storage.go | 6 ------ lib/trie/trie_test.go | 16 ++++++++-------- 4 files changed, 11 insertions(+), 23 deletions(-) diff --git a/cmd/gossamer/flags.go b/cmd/gossamer/flags.go index e9efdf04be..a31fd64a4c 100644 --- a/cmd/gossamer/flags.go +++ b/cmd/gossamer/flags.go @@ -283,7 +283,7 @@ var ( RetainBlockNumberFlag = cli.IntFlag{ Name: "retain-blocks", Usage: "Retain number of block from latest block while pruning", - Value: 512, + Value: defaultRetainBlocks, } // PruningFlag triggers the online pruning of historical state tries. It's either full or archive. To enable pruning the value diff --git a/dot/state/service_test.go b/dot/state/service_test.go index bc987865bf..70d0a0d48d 100644 --- a/dot/state/service_test.go +++ b/dot/state/service_test.go @@ -27,7 +27,6 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" - rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" @@ -190,12 +189,7 @@ func TestService_StorageTriePruning(t *testing.T) { err = serv.Storage.blockState.AddBlock(block) require.NoError(t, err) - old := trieState.Snapshot() - - oldTs, err := rtstorage.NewTrieState(old) - require.NoError(t, err) - - err = serv.Storage.StoreTrie(oldTs, block.Header) + err = serv.Storage.StoreTrie(trieState, block.Header) require.NoError(t, err) blocks = append(blocks, block) @@ -210,7 +204,7 @@ func TestService_StorageTriePruning(t *testing.T) { require.NoError(t, err, fmt.Sprintf("Got error for block %d", b.Header.Number.Int64())) continue } - require.Error(t, err, fmt.Sprintf("Expected error for block %d", b.Header.Number.Int64())) + require.ErrorIs(t, err, chaindb.ErrKeyNotFound, fmt.Sprintf("Expected error for block %d", b.Header.Number.Int64())) } } diff --git a/dot/state/storage.go b/dot/state/storage.go index 4d50a32b6f..104a8a0f67 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -100,13 +100,7 @@ func (s *StorageState) pruneKey(keyHeader *types.Header) { s.lock.Lock() defer s.lock.Unlock() - _, ok := s.tries[keyHeader.StateRoot] - if !ok { - return - } - delete(s.tries, keyHeader.StateRoot) - // TODO: database pruning needs to be refactored since the trie is now stored by nodes } // StoreTrie stores the given trie in the StorageState and writes it to the database diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 215ea822c3..17838f9094 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -499,8 +499,8 @@ func TestTrieDiff(t *testing.T) { trie.Put(test.key, test.value) } - oldTrie := trie.Snapshot() - err = oldTrie.Store(storageDB) + newTrie := trie.Snapshot() + err = trie.Store(storageDB) require.NoError(t, err) tests = []Test{ @@ -512,12 +512,12 @@ func TestTrieDiff(t *testing.T) { } for _, test := range tests { - trie.Put(test.key, test.value) + newTrie.Put(test.key, test.value) } - deletedKeys := trie.deletedKeys + deletedKeys := newTrie.deletedKeys require.Len(t, deletedKeys, 3) - err = trie.WriteDirty(storageDB) + err = newTrie.WriteDirty(storageDB) require.NoError(t, err) for _, key := range deletedKeys { @@ -525,11 +525,11 @@ func TestTrieDiff(t *testing.T) { require.NoError(t, err) } - newTrie := NewEmptyTrie() - err = newTrie.Load(storageDB, common.BytesToHash(trie.root.getHash())) + dbTrie := NewEmptyTrie() + err = dbTrie.Load(storageDB, common.BytesToHash(newTrie.root.getHash())) require.NoError(t, err) - enc, err := trie.Encode() + enc, err := dbTrie.Encode() require.NoError(t, err) newEnc, err := newTrie.Encode() From 30289d2d23b9a8aa724707b469f2ce8cf56a0840 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Wed, 16 Jun 2021 12:08:37 +0530 Subject: [PATCH 18/18] Fix failing test. --- chain/dev/defaults.go | 5 ++++ chain/gssmr/defaults.go | 5 ++++ chain/kusama/defaults.go | 5 ++++ chain/polkadot/defaults.go | 5 ++++ cmd/gossamer/config.go | 9 ++++++++ cmd/gossamer/config_test.go | 21 +++++++++-------- cmd/gossamer/export_test.go | 9 ++++---- cmd/gossamer/flags.go | 7 +++--- cmd/gossamer/flags_test.go | 25 ++++++++++---------- cmd/gossamer/main.go | 17 +------------- cmd/gossamer/prune_test.go | 2 +- cmd/gossamer/utils.go | 2 ++ cmd/gossamer/utils_test.go | 7 ++++++ dot/config.go | 46 ++++++++++++++++++++++--------------- dot/state/service.go | 4 ++-- 15 files changed, 102 insertions(+), 67 deletions(-) diff --git a/chain/dev/defaults.go b/chain/dev/defaults.go index 1636fc0d1b..68a9bc4fc2 100644 --- a/chain/dev/defaults.go +++ b/chain/dev/defaults.go @@ -39,6 +39,11 @@ var ( // DefaultLvl is the default log level DefaultLvl = log.LvlInfo + // DefaultPruningMode is the default pruning mode + DefaultPruningMode = "archive" + // DefaultRetainBlocks is the default retained blocks + DefaultRetainBlocks = int64(512) + // InitConfig // DefaultGenesis is the default genesis configuration path diff --git a/chain/gssmr/defaults.go b/chain/gssmr/defaults.go index a1d16aade5..5d7ebfc7a0 100644 --- a/chain/gssmr/defaults.go +++ b/chain/gssmr/defaults.go @@ -39,6 +39,11 @@ var ( // DefaultLvl is the default log level DefaultLvl = log.LvlInfo + // DefaultPruningMode is the default pruning mode + DefaultPruningMode = "archive" + // DefaultRetainBlocks is the default retained blocks + DefaultRetainBlocks = int64(512) + // InitConfig // DefaultGenesis is the default genesis configuration path diff --git a/chain/kusama/defaults.go b/chain/kusama/defaults.go index 948568acb4..e4c2e2657a 100644 --- a/chain/kusama/defaults.go +++ b/chain/kusama/defaults.go @@ -39,6 +39,11 @@ var ( // DefaultLvl is the default log level DefaultLvl = log.LvlInfo + // DefaultPruningMode is the default pruning mode + DefaultPruningMode = "archive" + // DefaultRetainBlocks is the default retained blocks + DefaultRetainBlocks = int64(512) + // InitConfig // DefaultGenesis is the default genesis configuration path diff --git a/chain/polkadot/defaults.go b/chain/polkadot/defaults.go index 46a5dc7989..4af96caf0e 100644 --- a/chain/polkadot/defaults.go +++ b/chain/polkadot/defaults.go @@ -36,6 +36,11 @@ var ( // DefaultLvl is the default log level DefaultLvl = log.LvlInfo + // DefaultPruningMode is the default pruning mode + DefaultPruningMode = "archive" + // DefaultRetainBlocks is the default pruning mode + DefaultRetainBlocks = int64(512) + // InitConfig // DefaultGenesis is the default genesis configuration path diff --git a/cmd/gossamer/config.go b/cmd/gossamer/config.go index be2cc83fab..8ddd67902d 100644 --- a/cmd/gossamer/config.go +++ b/cmd/gossamer/config.go @@ -21,6 +21,7 @@ import ( "strconv" "strings" + "github.com/ChainSafe/gossamer/chain/dev" "github.com/ChainSafe/gossamer/chain/gssmr" "github.com/ChainSafe/gossamer/dot" ctoml "github.com/ChainSafe/gossamer/dot/config/toml" @@ -175,6 +176,14 @@ func createInitConfig(ctx *cli.Context) (*dot.Config, error) { return nil, err } + if !cfg.Global.Pruning.IsValid() { + return nil, fmt.Errorf("--%s must be either %s or %s", PruningFlag.Name, pruner.Full, pruner.Archive) + } + + if cfg.Global.RetainBlocks < dev.DefaultRetainBlocks { + return nil, fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, dev.DefaultRetainBlocks) + } + // set log config err = setLogConfig(ctx, tomlCfg, &cfg.Global, &cfg.Log) if err != nil { diff --git a/cmd/gossamer/config_test.go b/cmd/gossamer/config_test.go index 2f2a3368de..36430ec0df 100644 --- a/cmd/gossamer/config_test.go +++ b/cmd/gossamer/config_test.go @@ -20,6 +20,7 @@ import ( "io/ioutil" "testing" + "github.com/ChainSafe/gossamer/chain/dev" "github.com/ChainSafe/gossamer/chain/gssmr" "github.com/ChainSafe/gossamer/dot" "github.com/ChainSafe/gossamer/dot/state" @@ -45,26 +46,26 @@ func TestConfigFromChainFlag(t *testing.T) { }{ { "Test gossamer --chain gssmr", - []string{"chain", "name"}, - []interface{}{"gssmr", dot.GssmrConfig().Global.Name}, + []string{"chain", "name", "pruning", "retain-blocks"}, + []interface{}{"gssmr", dot.GssmrConfig().Global.Name, gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks}, dot.GssmrConfig(), }, { "Test gossamer --chain kusama", - []string{"chain", "name"}, - []interface{}{"kusama", dot.KusamaConfig().Global.Name}, + []string{"chain", "name", "pruning", "retain-blocks"}, + []interface{}{"kusama", dot.KusamaConfig().Global.Name, gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks}, dot.KusamaConfig(), }, { "Test gossamer --chain polkadot", - []string{"chain", "name"}, - []interface{}{"polkadot", dot.PolkadotConfig().Global.Name}, + []string{"chain", "name", "pruning", "retain-blocks"}, + []interface{}{"polkadot", dot.PolkadotConfig().Global.Name, gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks}, dot.PolkadotConfig(), }, { "Test gossamer --chain dev", - []string{"chain", "name"}, - []interface{}{"dev", dot.DevConfig().Global.Name}, + []string{"chain", "name", "pruning", "retain-blocks"}, + []interface{}{"dev", dot.DevConfig().Global.Name, dev.DefaultPruningMode, dev.DefaultRetainBlocks}, dot.DevConfig(), }, } @@ -100,8 +101,8 @@ func TestInitConfigFromFlags(t *testing.T) { }{ { "Test gossamer --genesis", - []string{"config", "genesis"}, - []interface{}{testCfgFile.Name(), "test_genesis"}, + []string{"config", "genesis", "pruning", "retain-blocks"}, + []interface{}{testCfgFile.Name(), "test_genesis", dev.DefaultPruningMode, dev.DefaultRetainBlocks}, dot.InitConfig{ Genesis: "test_genesis", }, diff --git a/cmd/gossamer/export_test.go b/cmd/gossamer/export_test.go index b89e664471..ae247aabb4 100644 --- a/cmd/gossamer/export_test.go +++ b/cmd/gossamer/export_test.go @@ -20,6 +20,7 @@ import ( "io/ioutil" "testing" + "github.com/ChainSafe/gossamer/chain/gssmr" "github.com/ChainSafe/gossamer/dot" "github.com/ChainSafe/gossamer/lib/utils" @@ -91,8 +92,8 @@ func TestExportCommand(t *testing.T) { }, { "Test gossamer export --config --genesis --bootnodes --log --force", - []string{"config", "genesis", "bootnodes", "name", "force"}, - []interface{}{testConfig, genFile.Name(), testBootnode, "Gossamer", "true"}, + []string{"config", "genesis", "bootnodes", "name", "force", "pruning", "retain-blocks"}, + []interface{}{testConfig, genFile.Name(), testBootnode, "Gossamer", "true", gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks}, &dot.Config{ Global: testCfg.Global, Init: dot.InitConfig{ @@ -122,8 +123,8 @@ func TestExportCommand(t *testing.T) { }, { "Test gossamer export --config --genesis --protocol --log --force", - []string{"config", "genesis", "protocol", "force", "name"}, - []interface{}{testConfig, genFile.Name(), testProtocol, "true", "Gossamer"}, + []string{"config", "genesis", "protocol", "force", "name", "pruning", "retain-blocks"}, + []interface{}{testConfig, genFile.Name(), testProtocol, "true", "Gossamer", gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks}, &dot.Config{ Global: testCfg.Global, Init: dot.InitConfig{ diff --git a/cmd/gossamer/flags.go b/cmd/gossamer/flags.go index a31fd64a4c..284a8a3e25 100644 --- a/cmd/gossamer/flags.go +++ b/cmd/gossamer/flags.go @@ -17,6 +17,7 @@ package main import ( + "github.com/ChainSafe/gossamer/chain/dev" log "github.com/ChainSafe/log15" "github.com/urfave/cli" ) @@ -280,10 +281,10 @@ var ( } // RetainBlockNumberFlag retain number of block from latest block while pruning, valid for the use with prune-state subcommand - RetainBlockNumberFlag = cli.IntFlag{ + RetainBlockNumberFlag = cli.Int64Flag{ Name: "retain-blocks", Usage: "Retain number of block from latest block while pruning", - Value: defaultRetainBlocks, + Value: dev.DefaultRetainBlocks, } // PruningFlag triggers the online pruning of historical state tries. It's either full or archive. To enable pruning the value @@ -291,7 +292,7 @@ var ( PruningFlag = cli.StringFlag{ Name: "pruning", Usage: `State trie online pruning ("full", "archive")`, - Value: "archive", + Value: dev.DefaultPruningMode, } ) diff --git a/cmd/gossamer/flags_test.go b/cmd/gossamer/flags_test.go index 6b459f338d..c095f4be9a 100644 --- a/cmd/gossamer/flags_test.go +++ b/cmd/gossamer/flags_test.go @@ -20,6 +20,7 @@ import ( "io/ioutil" "testing" + "github.com/ChainSafe/gossamer/chain/dev" "github.com/ChainSafe/gossamer/dot" "github.com/ChainSafe/gossamer/lib/utils" @@ -43,24 +44,24 @@ func TestFixFlagOrder(t *testing.T) { values []interface{} }{ { - "Test gossamer --config --genesis --log --force", - []string{"config", "genesis", "log", "force"}, - []interface{}{testConfig.Name(), genFile.Name(), "trace", true}, + "Test gossamer --config --genesis --log --force --pruning --retain-blocks", + []string{"config", "genesis", "log", "force", "pruning", "retain-blocks"}, + []interface{}{testConfig.Name(), genFile.Name(), "trace", true, dev.DefaultPruningMode, dev.DefaultRetainBlocks}, }, { - "Test gossamer --config --genesis --force --log", - []string{"config", "genesis", "force", "log"}, - []interface{}{testConfig.Name(), genFile.Name(), true, "trace"}, + "Test gossamer --config --genesis --force --log --pruning --retain-blocks", + []string{"config", "genesis", "force", "log", "pruning", "retain-blocks"}, + []interface{}{testConfig.Name(), genFile.Name(), true, "trace", dev.DefaultPruningMode, dev.DefaultRetainBlocks}, }, { - "Test gossamer --config --force --genesis --log", - []string{"config", "force", "genesis", "log"}, - []interface{}{testConfig.Name(), true, genFile.Name(), "trace"}, + "Test gossamer --config --force --genesis --log ---pruning --retain-blocks", + []string{"config", "force", "genesis", "log", "pruning", "retain-blocks"}, + []interface{}{testConfig.Name(), true, genFile.Name(), "trace", dev.DefaultPruningMode, dev.DefaultRetainBlocks}, }, { - "Test gossamer --force --config --genesis --log", - []string{"force", "config", "genesis", "log"}, - []interface{}{true, testConfig.Name(), genFile.Name(), "trace"}, + "Test gossamer --force --config --genesis --log --pruning --retain-blocks", + []string{"force", "config", "genesis", "log", "pruning", "retain-blocks"}, + []interface{}{true, testConfig.Name(), genFile.Name(), "trace", dev.DefaultPruningMode, dev.DefaultRetainBlocks}, }, } diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index 74b02b2365..98faf82db3 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -23,7 +23,6 @@ import ( "github.com/ChainSafe/gossamer/dot" "github.com/ChainSafe/gossamer/dot/state" - "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/utils" log "github.com/ChainSafe/log15" @@ -38,7 +37,6 @@ const ( importRuntimeCommandName = "import-runtime" importStateCommandName = "import-state" pruningStateCommandName = "prune-state" - defaultRetainBlocks = 512 ) // app is the cli application @@ -335,19 +333,6 @@ func initAction(ctx *cli.Context) error { // expand data directory and update node configuration (performed separately // from createDotConfig because dot config should not include expanded path) cfg.Global.BasePath = utils.ExpandDir(cfg.Global.BasePath) - - mode := pruner.Mode(ctx.String(PruningFlag.Name)) - if !mode.IsValid() { - return fmt.Errorf("--%s must be either %s or %s", PruningFlag.Name, pruner.Full, pruner.Archive) - } - cfg.Global.Pruning = mode - - rb := ctx.Int64(RetainBlockNumberFlag.Name) - if rb < defaultRetainBlocks { - return fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, defaultRetainBlocks) - } - cfg.Global.RetainBlocks = rb - // check if node has been initialised (expected false - no warning log) if dot.NodeInitialized(cfg.Global.BasePath, false) { @@ -455,7 +440,7 @@ func pruneState(ctx *cli.Context) error { } bloomSize := ctx.GlobalUint64(BloomFilterSizeFlag.Name) - retainBlocks := ctx.Int64(RetainBlockNumberFlag.Name) + retainBlocks := ctx.GlobalInt64(RetainBlockNumberFlag.Name) pruner, err := state.NewOfflinePruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks) if err != nil { diff --git a/cmd/gossamer/prune_test.go b/cmd/gossamer/prune_test.go index e84e6c7c9e..14c8fba6a9 100644 --- a/cmd/gossamer/prune_test.go +++ b/cmd/gossamer/prune_test.go @@ -22,7 +22,7 @@ func runPruneCmd(t *testing.T, configFile, prunedDBPath string) { ctx, err := newTestContext( "Test state trie offline pruning --prune-state", []string{"config", "pruned-db-path", "bloom-size", "retain-blocks"}, - []interface{}{configFile, prunedDBPath, "256", "5"}, + []interface{}{configFile, prunedDBPath, "256", int64(5)}, ) if err != nil { t.Fatal(err) diff --git a/cmd/gossamer/utils.go b/cmd/gossamer/utils.go index edd4f36a5b..d83a4045d2 100644 --- a/cmd/gossamer/utils.go +++ b/cmd/gossamer/utils.go @@ -95,6 +95,8 @@ func newTestConfig(t *testing.T) *dot.Config { LogLvl: log.LvlInfo, PublishMetrics: dot.GssmrConfig().Global.PublishMetrics, MetricsPort: dot.GssmrConfig().Global.MetricsPort, + RetainBlocks: dot.GssmrConfig().Global.RetainBlocks, + Pruning: dot.GssmrConfig().Global.Pruning, }, Log: dot.LogConfig{ CoreLvl: log.LvlInfo, diff --git a/cmd/gossamer/utils_test.go b/cmd/gossamer/utils_test.go index 699f55e1c1..dd825f394d 100644 --- a/cmd/gossamer/utils_test.go +++ b/cmd/gossamer/utils_test.go @@ -38,6 +38,8 @@ func newTestContext(description string, flags []string, values []interface{}) (* set.String(flags[i], v, "") case uint: set.Uint(flags[i], v, "") + case int64: + set.Int64(flags[i], v, "") default: return nil, fmt.Errorf("unexpected cli value type: %T", values[i]) } @@ -69,6 +71,11 @@ func newTestContext(description string, flags []string, values []interface{}) (* if err != nil { return nil, fmt.Errorf("failed to set cli flag: %T", flags[i]) } + case int64: + err := ctx.Set(flags[i], strconv.Itoa(int(values[i].(int64)))) + if err != nil { + return nil, fmt.Errorf("failed to set cli flag: %T", flags[i]) + } default: return nil, fmt.Errorf("unexpected cli value type: %T", values[i]) } diff --git a/dot/config.go b/dot/config.go index 5dd2949edc..e5572e10ab 100644 --- a/dot/config.go +++ b/dot/config.go @@ -134,11 +134,13 @@ func networkServiceEnabled(cfg *Config) bool { func GssmrConfig() *Config { return &Config{ Global: GlobalConfig{ - Name: gssmr.DefaultName, - ID: gssmr.DefaultID, - BasePath: gssmr.DefaultBasePath, - LogLvl: gssmr.DefaultLvl, - MetricsPort: gssmr.DefaultMetricsPort, + Name: gssmr.DefaultName, + ID: gssmr.DefaultID, + BasePath: gssmr.DefaultBasePath, + LogLvl: gssmr.DefaultLvl, + MetricsPort: gssmr.DefaultMetricsPort, + RetainBlocks: gssmr.DefaultRetainBlocks, + Pruning: pruner.Mode(gssmr.DefaultPruningMode), }, Log: LogConfig{ CoreLvl: gssmr.DefaultLvl, @@ -182,11 +184,13 @@ func GssmrConfig() *Config { func KusamaConfig() *Config { return &Config{ Global: GlobalConfig{ - Name: kusama.DefaultName, - ID: kusama.DefaultID, - BasePath: kusama.DefaultBasePath, - LogLvl: kusama.DefaultLvl, - MetricsPort: kusama.DefaultMetricsPort, + Name: kusama.DefaultName, + ID: kusama.DefaultID, + BasePath: kusama.DefaultBasePath, + LogLvl: kusama.DefaultLvl, + MetricsPort: kusama.DefaultMetricsPort, + RetainBlocks: gssmr.DefaultRetainBlocks, + Pruning: pruner.Mode(gssmr.DefaultPruningMode), }, Log: LogConfig{ CoreLvl: kusama.DefaultLvl, @@ -228,10 +232,12 @@ func KusamaConfig() *Config { func PolkadotConfig() *Config { return &Config{ Global: GlobalConfig{ - Name: polkadot.DefaultName, - ID: polkadot.DefaultID, - BasePath: polkadot.DefaultBasePath, - LogLvl: polkadot.DefaultLvl, + Name: polkadot.DefaultName, + ID: polkadot.DefaultID, + BasePath: polkadot.DefaultBasePath, + LogLvl: polkadot.DefaultLvl, + RetainBlocks: gssmr.DefaultRetainBlocks, + Pruning: pruner.Mode(gssmr.DefaultPruningMode), }, Log: LogConfig{ CoreLvl: polkadot.DefaultLvl, @@ -273,11 +279,13 @@ func PolkadotConfig() *Config { func DevConfig() *Config { return &Config{ Global: GlobalConfig{ - Name: dev.DefaultName, - ID: dev.DefaultID, - BasePath: dev.DefaultBasePath, - LogLvl: dev.DefaultLvl, - MetricsPort: dev.DefaultMetricsPort, + Name: dev.DefaultName, + ID: dev.DefaultID, + BasePath: dev.DefaultBasePath, + LogLvl: dev.DefaultLvl, + MetricsPort: dev.DefaultMetricsPort, + RetainBlocks: dev.DefaultRetainBlocks, + Pruning: pruner.Mode(dev.DefaultPruningMode), }, Log: LogConfig{ CoreLvl: dev.DefaultLvl, diff --git a/dot/state/service.go b/dot/state/service.go index 5871595edf..462ea5c0ac 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -152,13 +152,13 @@ func (s *Service) Start() error { s.Block.bt = blocktree.NewBlockTreeFromRoot(lastFinalised, db) } - pruner, err := s.Base.loadPruningData() + pr, err := s.Base.loadPruningData() if err != nil { return err } // create storage state - s.Storage, err = NewStorageState(db, s.Block, trie.NewEmptyTrie(), pruner) + s.Storage, err = NewStorageState(db, s.Block, trie.NewEmptyTrie(), pr) if err != nil { return fmt.Errorf("failed to create storage state: %w", err) }