[release] fix to get .torrent from snaps (#9452)

Cherry pick PR #9435 --------- Co-authored-by: Alex Sharov <[email protected]>
erigontech · Feb 15, 2024 · 9731f71 · 9731f71
1 parent ef9e04f
commit 9731f71
Show file tree

Hide file tree

Showing 7 changed files with 16 additions and 29 deletions.
diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
@@ -156,7 +156,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, dirs datadir.Dirs, logger
 		if !discover {
 			return
 		}
-		d.webseeds.Discover(d.ctx, d.cfg.WebSeedUrls, d.cfg.WebSeedFiles, d.cfg.Dirs.Snap, lock.Downloads)
+		d.webseeds.Discover(d.ctx, d.cfg.WebSeedUrls, d.cfg.WebSeedFiles, d.cfg.Dirs.Snap)
 		// webseeds.Discover may create new .torrent files on disk
 		if err := d.addTorrentFilesFromDisk(true); err != nil && !errors.Is(err, context.Canceled) {
 			d.logger.Warn("[snapshots] addTorrentFilesFromDisk", "err", err)

diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go
@@ -40,9 +40,9 @@ type WebSeeds struct {
 	torrentFiles *TorrentFiles
 }
 
-func (d *WebSeeds) Discover(ctx context.Context, urls []*url.URL, files []string, rootDir string, ignore snapcfg.Preverified) {
+func (d *WebSeeds) Discover(ctx context.Context, urls []*url.URL, files []string, rootDir string) {
 	d.downloadWebseedTomlFromProviders(ctx, urls, files)
-	d.downloadTorrentFilesFromProviders(ctx, rootDir, ignore)
+	d.downloadTorrentFilesFromProviders(ctx, rootDir)
 }
 
 func (d *WebSeeds) downloadWebseedTomlFromProviders(ctx context.Context, httpProviders []*url.URL, diskProviders []string) {
@@ -164,7 +164,7 @@ func (d *WebSeeds) readWebSeedsFile(webSeedProviderPath string) (snaptype.WebSee
 }
 
 // downloadTorrentFilesFromProviders - if they are not exist on file-system
-func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDir string, ignore snapcfg.Preverified) {
+func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDir string) {
 	// TODO: need more tests, need handle more forward-compatibility and backward-compatibility case
 	//  - now, if add new type of .torrent files to S3 bucket - existing nodes will start downloading it. maybe need whitelist of file types
 	//  - maybe need download new files if --snap.stop=true
@@ -174,26 +174,14 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi
 	if len(d.TorrentUrls()) == 0 {
 		return
 	}
-	if d.torrentFiles.newDownloadsAreProhibited() {
-		return
-	}
 	var addedNew int
 	e, ctx := errgroup.WithContext(ctx)
 	e.SetLimit(1024)
 	urlsByName := d.TorrentUrls()
 	//TODO:
 	// - what to do if node already synced?
 
-	fileName := func(name string) string {
-		name, _ = strings.CutSuffix(name, filepath.Ext(name))
-		return name
-	}
-
 	for name, tUrls := range urlsByName {
-		if ignore.Contains(fileName(name)) {
-			continue
-		}
-
 		tPath := filepath.Join(rootDir, name)
 		if dir.FileExist(tPath) {
 			continue
@@ -210,12 +198,11 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi
 			for _, url := range tUrls {
 				res, err := d.callTorrentHttpProvider(ctx, url, name)
 				if err != nil {
-					d.logger.Log(d.verbosity, "[snapshots] got from webseed", "name", name, "err", err)
+					d.logger.Log(d.verbosity, "[snapshots] got from webseed", "name", name, "err", err, "url", url)
 					continue
 				}
-				d.logger.Log(d.verbosity, "[snapshots] got from webseed", "name", name)
 				if err := d.torrentFiles.Create(tPath, res); err != nil {
-					d.logger.Debug("[snapshots] saveTorrent", "err", err)
+					d.logger.Log(d.verbosity, "[snapshots] .torrent from webseed rejected", "name", name, "err", err, "url", url)
 					continue
 				}
 				return nil

diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod
@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/erigontech/mdbx-go v0.27.21
-	github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585
+	github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec
 	github.com/ledgerwatch/interfaces v0.0.0-20240203142514-1cf37a5264cc
 	github.com/ledgerwatch/log/v3 v3.9.0
 	github.com/ledgerwatch/secp256k1 v1.0.0

diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum
@@ -254,8 +254,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
-github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 h1:A+ofEWJjD3HVFF8wLdPxgNv3AtOz3iqjPYJA2VeTWVU=
-github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
+github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec h1:64YluBxCSivzxlQ5o1cCZs+3oSg7rfAFf6A2YJRydrs=
+github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
 github.com/ledgerwatch/interfaces v0.0.0-20240203142514-1cf37a5264cc h1:lZ+Qg1oL8mlIjACPfeYKkD89LFdwIITtBt985wKwyjA=
 github.com/ledgerwatch/interfaces v0.0.0-20240203142514-1cf37a5264cc/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
 github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=

diff --git a/go.mod b/go.mod
@@ -175,7 +175,7 @@ require (
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 // indirect
+	github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
 	github.com/libp2p/go-flow-metrics v0.1.0 // indirect

diff --git a/go.sum b/go.sum
@@ -520,8 +520,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 h1:A+ofEWJjD3HVFF8wLdPxgNv3AtOz3iqjPYJA2VeTWVU=
-github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
+github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec h1:64YluBxCSivzxlQ5o1cCZs+3oSg7rfAFf6A2YJRydrs=
+github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
 github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
 github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE=
 github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=

diff --git a/params/version.go b/params/version.go
@@ -31,10 +31,10 @@ var (
 
 // see https://calver.org
 const (
-	VersionMajor       = 2     // Major version component of the current release
-	VersionMinor       = 58    // Minor version component of the current release
-	VersionMicro       = 0     // Patch version component of the current release
-	VersionModifier    = "dev" // Modifier component of the current release
+	VersionMajor       = 2  // Major version component of the current release
+	VersionMinor       = 58 // Minor version component of the current release
+	VersionMicro       = 0  // Patch version component of the current release
+	VersionModifier    = "" // Modifier component of the current release
 	VersionKeyCreated  = "ErigonVersionCreated"
 	VersionKeyFinished = "ErigonVersionFinished"
 )