From 57164e7c6240677863394afc8fc5db58fb7c5cc4 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 13 Feb 2024 20:31:14 +0700 Subject: [PATCH 1/4] save --- erigon-lib/downloader/downloader.go | 2 +- erigon-lib/downloader/webseed.go | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 106aafea7ad..a9548394f97 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -156,7 +156,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, dirs datadir.Dirs, logger if !discover { return } - d.webseeds.Discover(d.ctx, d.cfg.WebSeedUrls, d.cfg.WebSeedFiles, d.cfg.Dirs.Snap, lock.Downloads) + d.webseeds.Discover(d.ctx, d.cfg.WebSeedUrls, d.cfg.WebSeedFiles, d.cfg.Dirs.Snap) // webseeds.Discover may create new .torrent files on disk if err := d.addTorrentFilesFromDisk(true); err != nil && !errors.Is(err, context.Canceled) { d.logger.Warn("[snapshots] addTorrentFilesFromDisk", "err", err) diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index 125f34343e7..224800ae8ac 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -40,9 +40,9 @@ type WebSeeds struct { torrentFiles *TorrentFiles } -func (d *WebSeeds) Discover(ctx context.Context, urls []*url.URL, files []string, rootDir string, ignore snapcfg.Preverified) { +func (d *WebSeeds) Discover(ctx context.Context, urls []*url.URL, files []string, rootDir string) { d.downloadWebseedTomlFromProviders(ctx, urls, files) - d.downloadTorrentFilesFromProviders(ctx, rootDir, ignore) + d.downloadTorrentFilesFromProviders(ctx, rootDir) } func (d *WebSeeds) downloadWebseedTomlFromProviders(ctx context.Context, httpProviders []*url.URL, diskProviders []string) { @@ -164,7 +164,7 @@ func (d *WebSeeds) readWebSeedsFile(webSeedProviderPath string) (snaptype.WebSee } // downloadTorrentFilesFromProviders - if they are not exist on file-system -func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDir string, ignore snapcfg.Preverified) { +func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDir string) { // TODO: need more tests, need handle more forward-compatibility and backward-compatibility case // - now, if add new type of .torrent files to S3 bucket - existing nodes will start downloading it. maybe need whitelist of file types // - maybe need download new files if --snap.stop=true @@ -174,9 +174,6 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi if len(d.TorrentUrls()) == 0 { return } - if d.torrentFiles.newDownloadsAreProhibited() { - return - } var addedNew int e, ctx := errgroup.WithContext(ctx) e.SetLimit(1024) From d7afc1e7c5ab3f9d639535636a8a61490cb21336 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 13 Feb 2024 20:58:44 +0700 Subject: [PATCH 2/4] save --- erigon-lib/downloader/webseed.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index 224800ae8ac..20dbb7e2aed 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -181,16 +181,7 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi //TODO: // - what to do if node already synced? - fileName := func(name string) string { - name, _ = strings.CutSuffix(name, filepath.Ext(name)) - return name - } - for name, tUrls := range urlsByName { - if ignore.Contains(fileName(name)) { - continue - } - tPath := filepath.Join(rootDir, name) if dir.FileExist(tPath) { continue From 8c179c2d0b207780d5b613063173bfbfc054351e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 14 Feb 2024 10:45:48 +0700 Subject: [PATCH 3/4] mumbai: fix sn hash --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index e021e87f568..ce2d02b10b0 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/erigontech/mdbx-go v0.27.21 - github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 + github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec github.com/ledgerwatch/interfaces v0.0.0-20240203142514-1cf37a5264cc github.com/ledgerwatch/log/v3 v3.9.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 24ceab0399f..47cbe546a27 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -254,8 +254,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 h1:A+ofEWJjD3HVFF8wLdPxgNv3AtOz3iqjPYJA2VeTWVU= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec h1:64YluBxCSivzxlQ5o1cCZs+3oSg7rfAFf6A2YJRydrs= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/interfaces v0.0.0-20240203142514-1cf37a5264cc h1:lZ+Qg1oL8mlIjACPfeYKkD89LFdwIITtBt985wKwyjA= github.com/ledgerwatch/interfaces v0.0.0-20240203142514-1cf37a5264cc/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= diff --git a/go.mod b/go.mod index 24df3647fe5..0d8d807b913 100644 --- a/go.mod +++ b/go.mod @@ -175,7 +175,7 @@ require ( github.com/koron/go-ssdp v0.0.4 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 // indirect + github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect diff --git a/go.sum b/go.sum index f73fa7f51a1..371c0b75695 100644 --- a/go.sum +++ b/go.sum @@ -520,8 +520,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 h1:A+ofEWJjD3HVFF8wLdPxgNv3AtOz3iqjPYJA2VeTWVU= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec h1:64YluBxCSivzxlQ5o1cCZs+3oSg7rfAFf6A2YJRydrs= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= From 5248fbfc39e4546eb089b65f43b547cbb133b5ff Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 14 Feb 2024 10:50:57 +0700 Subject: [PATCH 4/4] mumbai: fix sn hash --- erigon-lib/downloader/webseed.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index 20dbb7e2aed..daa54fcf561 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -198,12 +198,11 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi for _, url := range tUrls { res, err := d.callTorrentHttpProvider(ctx, url, name) if err != nil { - d.logger.Log(d.verbosity, "[snapshots] got from webseed", "name", name, "err", err) + d.logger.Log(d.verbosity, "[snapshots] got from webseed", "name", name, "err", err, "url", url) continue } - d.logger.Log(d.verbosity, "[snapshots] got from webseed", "name", name) if err := d.torrentFiles.Create(tPath, res); err != nil { - d.logger.Debug("[snapshots] saveTorrent", "err", err) + d.logger.Log(d.verbosity, "[snapshots] .torrent from webseed rejected", "name", name, "err", err, "url", url) continue } return nil