Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[release] fix to get .torrent from snaps #9435

Merged
merged 4 commits into from
Feb 15, 2024
Merged

[release] fix to get .torrent from snaps #9435

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
57164e7
save
AskAlexSharov Feb 13, 2024
d7afc1e
save
AskAlexSharov Feb 13, 2024
8c179c2
mumbai: fix sn hash
AskAlexSharov Feb 14, 2024
5248fbf
mumbai: fix sn hash
AskAlexSharov Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion erigon-lib/downloader/downloader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, dirs datadir.Dirs, logger
if !discover {
return
}
d.webseeds.Discover(d.ctx, d.cfg.WebSeedUrls, d.cfg.WebSeedFiles, d.cfg.Dirs.Snap, lock.Downloads)
d.webseeds.Discover(d.ctx, d.cfg.WebSeedUrls, d.cfg.WebSeedFiles, d.cfg.Dirs.Snap)
// webseeds.Discover may create new .torrent files on disk
if err := d.addTorrentFilesFromDisk(true); err != nil && !errors.Is(err, context.Canceled) {
d.logger.Warn("[snapshots] addTorrentFilesFromDisk", "err", err)
Expand Down
23 changes: 5 additions & 18 deletions erigon-lib/downloader/webseed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ type WebSeeds struct {
torrentFiles *TorrentFiles
}

func (d *WebSeeds) Discover(ctx context.Context, urls []*url.URL, files []string, rootDir string, ignore snapcfg.Preverified) {
func (d *WebSeeds) Discover(ctx context.Context, urls []*url.URL, files []string, rootDir string) {
d.downloadWebseedTomlFromProviders(ctx, urls, files)
d.downloadTorrentFilesFromProviders(ctx, rootDir, ignore)
d.downloadTorrentFilesFromProviders(ctx, rootDir)
}

func (d *WebSeeds) downloadWebseedTomlFromProviders(ctx context.Context, httpProviders []*url.URL, diskProviders []string) {
Expand Down Expand Up @@ -164,7 +164,7 @@ func (d *WebSeeds) readWebSeedsFile(webSeedProviderPath string) (snaptype.WebSee
}

// downloadTorrentFilesFromProviders - if they are not exist on file-system
func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDir string, ignore snapcfg.Preverified) {
func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDir string) {
// TODO: need more tests, need handle more forward-compatibility and backward-compatibility case
// - now, if add new type of .torrent files to S3 bucket - existing nodes will start downloading it. maybe need whitelist of file types
// - maybe need download new files if --snap.stop=true
Expand All @@ -174,26 +174,14 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi
if len(d.TorrentUrls()) == 0 {
return
}
if d.torrentFiles.newDownloadsAreProhibited() {
return
}
var addedNew int
e, ctx := errgroup.WithContext(ctx)
e.SetLimit(1024)
urlsByName := d.TorrentUrls()
//TODO:
// - what to do if node already synced?

fileName := func(name string) string {
name, _ = strings.CutSuffix(name, filepath.Ext(name))
return name
}

for name, tUrls := range urlsByName {
if ignore.Contains(fileName(name)) {
continue
}

tPath := filepath.Join(rootDir, name)
if dir.FileExist(tPath) {
continue
Expand All @@ -210,12 +198,11 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi
for _, url := range tUrls {
res, err := d.callTorrentHttpProvider(ctx, url, name)
if err != nil {
d.logger.Log(d.verbosity, "[snapshots] got from webseed", "name", name, "err", err)
d.logger.Log(d.verbosity, "[snapshots] got from webseed", "name", name, "err", err, "url", url)
continue
}
d.logger.Log(d.verbosity, "[snapshots] got from webseed", "name", name)
if err := d.torrentFiles.Create(tPath, res); err != nil {
d.logger.Debug("[snapshots] saveTorrent", "err", err)
d.logger.Log(d.verbosity, "[snapshots] .torrent from webseed rejected", "name", name, "err", err, "url", url)
continue
}
return nil
Expand Down
2 changes: 1 addition & 1 deletion erigon-lib/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.20

require (
github.com/erigontech/mdbx-go v0.27.21
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec
github.com/ledgerwatch/interfaces v0.0.0-20240203142514-1cf37a5264cc
github.com/ledgerwatch/log/v3 v3.9.0
github.com/ledgerwatch/secp256k1 v1.0.0
Expand Down
4 changes: 2 additions & 2 deletions erigon-lib/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,8 +254,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 h1:A+ofEWJjD3HVFF8wLdPxgNv3AtOz3iqjPYJA2VeTWVU=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec h1:64YluBxCSivzxlQ5o1cCZs+3oSg7rfAFf6A2YJRydrs=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/interfaces v0.0.0-20240203142514-1cf37a5264cc h1:lZ+Qg1oL8mlIjACPfeYKkD89LFdwIITtBt985wKwyjA=
github.com/ledgerwatch/interfaces v0.0.0-20240203142514-1cf37a5264cc/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ require (
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 // indirect
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -520,8 +520,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585 h1:A+ofEWJjD3HVFF8wLdPxgNv3AtOz3iqjPYJA2VeTWVU=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240209065041-d3281a89c585/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec h1:64YluBxCSivzxlQ5o1cCZs+3oSg7rfAFf6A2YJRydrs=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240214034330-a436079c93ec/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=
Expand Down
Loading