From 38d3c3632a084298e6cc8e57439a10ba8c7cf04a Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 3 May 2021 16:08:28 -0700 Subject: [PATCH] Attempt to read Bootstrap and Peering from config - Peer info is not provided by Migration condif - Do not fail, but use defaults, if Bootstrap or Peering config is not readable --- cmd/ipfs/daemon.go | 46 ++-- cmd/ipfs/migration.go | 88 +++++-- cmd/ipfs/migration_test.go | 230 +++++++++++------- go.mod | 8 +- go.sum | 20 +- .../migrations/ipfsfetcher/ipfsfetcher.go | 50 ++-- .../ipfsfetcher/ipfsfetcher_test.go | 6 +- 7 files changed, 266 insertions(+), 182 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index b20849ebc1e..eb78f87ec28 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -297,27 +297,16 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment return err } - keep := migrationCfg.Keep - if keep == "" { - keep = config.DefaultMigrationKeep + fetcher, err = getMigrationFetcher(migrationCfg, &cctx.ConfigRoot) + if err != nil { + return err } - switch keep { - case "discard": - case "cache": + defer fetcher.Close() + + if migrationCfg.Keep == "cache" { cacheMigrations = true - case "pin": + } else if migrationCfg.Keep == "pin" { pinMigrations = true - default: - return errors.New("unrecognized value for %s, must be 'cache', 'pin', or 'discard'") - } - - dlSources := migrationCfg.DownloadSources - if len(dlSources) == 0 { - dlSources = config.DefaultMigrationDownloadSources - } - fetcher, err = getMigrationFetcher(dlSources, migrationCfg.Peers) - if err != nil { - return err } if cacheMigrations || pinMigrations { @@ -339,21 +328,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment fmt.Printf(" %s\n", err) fmt.Println("If you think this is a bug, please file an issue and include this whole log output.") fmt.Println(" https://github.com/ipfs/fs-repo-migrations") - fetcher.Close() return err } - if cacheMigrations || pinMigrations { - defer fetcher.Close() - } else { - // If there is an error closing the IpfsFetcher, then print error, but - // do not fail because of it. - err = fetcher.Close() - if err != nil { - log.Errorf("error closing IPFS fetcher: %s", err) - } - } - repo, err = fsrepo.Open(cctx.ConfigRoot) if err != nil { return err @@ -481,10 +458,17 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment if err != nil { fmt.Fprintln(os.Stderr, "Could not add migragion to IPFS:", err) } - fetcher.Close() os.RemoveAll(migrations.DownloadDirectory) migrations.DownloadDirectory = "" } + if fetcher != nil { + // If there is an error closing the IpfsFetcher, then print error, but + // do not fail because of it. + err = fetcher.Close() + if err != nil { + log.Errorf("error closing IPFS fetcher: %s", err) + } + } // construct http gateway gwErrc, err := serveHTTPGateway(req, cctx) diff --git a/cmd/ipfs/migration.go b/cmd/ipfs/migration.go index b9d84aef18f..2b497231d44 100644 --- a/cmd/ipfs/migration.go +++ b/cmd/ipfs/migration.go @@ -24,9 +24,9 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -// readMigrationConfig reads the migration config out of the config, avoiding reading anything other -// than the migration section. That way, we're free to make arbitrary changes to all _other_ -// sections in migrations. +// readMigrationConfig reads the migration config out of the config, avoiding +// reading anything other than the migration section. That way, we're free to +// make arbitrary changes to all _other_ sections in migrations. func readMigrationConfig(repoRoot string) (*config.Migration, error) { var cfg struct { Migration config.Migration @@ -34,6 +34,7 @@ func readMigrationConfig(repoRoot string) (*config.Migration, error) { cfgPath, err := config.Filename(repoRoot) if err != nil { + fmt.Fprintln(os.Stderr, err) return nil, err } @@ -41,7 +42,6 @@ func readMigrationConfig(repoRoot string) (*config.Migration, error) { if err != nil { return nil, err } - defer cfgFile.Close() err = json.NewDecoder(cfgFile).Decode(&cfg) @@ -49,29 +49,85 @@ func readMigrationConfig(repoRoot string) (*config.Migration, error) { return nil, err } + switch cfg.Migration.Keep { + case "": + cfg.Migration.Keep = config.DefaultMigrationKeep + case "discard", "cache", "keep": + default: + return nil, errors.New("unknown config value, Migrations.Keep must be 'cache', 'pin', or 'discard'") + } + + if len(cfg.Migration.DownloadSources) == 0 { + cfg.Migration.DownloadSources = config.DefaultMigrationDownloadSources + } + return &cfg.Migration, nil } +func readIpfsConfig(repoRoot *string) (bootstrap []string, peers []peer.AddrInfo) { + if repoRoot == nil { + return nil, nil + } + + cfgPath, err := config.Filename(*repoRoot) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + + cfgFile, err := os.Open(cfgPath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + defer cfgFile.Close() + + // Attempt to read bootstrap addresses + var bootstrapCfg struct { + Bootstrap []string + } + err = json.NewDecoder(cfgFile).Decode(&bootstrapCfg) + if err != nil { + fmt.Fprintln(os.Stderr, "cannot read bootstrap peers from config") + } else { + bootstrap = bootstrapCfg.Bootstrap + } + + if _, err = cfgFile.Seek(0, 0); err != nil { + fmt.Fprintln(os.Stderr, err) + } + + // Attempt to read peers + var peeringCfg struct { + Peering config.Peering + } + err = json.NewDecoder(cfgFile).Decode(&peeringCfg) + if err != nil { + fmt.Fprintln(os.Stderr, "cannot read peering from config") + } else { + peers = peeringCfg.Peering.Peers + } + + return +} + // getMigrationFetcher creates one or more fetchers according to -// downloadPolicy. -// -// The downloadPolicy parameter is a comma-separated string. It may contain -// "ipfs" to indicate using the IpfsFetcher and "http" to indicate using the -// HttpFetcher. Any other string is treated as a gateway URL to use with -// another HttpFetcher. If downloadPolicy is is an empty string, then the -// default policy ("http,ipfs")is used. -func getMigrationFetcher(downloadSources []string, peers []peer.AddrInfo) (migrations.Fetcher, error) { +// config.Migration.DownloadSources. If an IpfsFetcher is required, then +// bootstrap and peer information in read from the config file in repoRoot, +// unless repoRoot is nil. +func getMigrationFetcher(cfg *config.Migration, repoRoot *string) (migrations.Fetcher, error) { const httpUserAgent = "go-ipfs" // Fetch migrations from current distribution, or location from environ fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist) var fetchers []migrations.Fetcher - for _, src := range downloadSources { + for _, src := range cfg.DownloadSources { src := strings.TrimSpace(src) switch src { case "IPFS", "ipfs": - fetchers = append(fetchers, ipfsfetcher.NewIpfsFetcher(fetchDistPath, 0, peers)) + bootstrap, peers := readIpfsConfig(repoRoot) + fetchers = append(fetchers, ipfsfetcher.NewIpfsFetcher(fetchDistPath, 0, bootstrap, peers)) case "HTTPS", "https", "HTTP", "http": fetchers = append(fetchers, migrations.NewHttpFetcher(fetchDistPath, "", httpUserAgent, 0)) default: @@ -92,7 +148,7 @@ func getMigrationFetcher(downloadSources []string, peers []peer.AddrInfo) (migra } } if len(fetchers) == 0 { - return nil, errors.New("no fetchers specified") + return nil, errors.New("no sources specified") } if len(fetchers) == 1 { @@ -230,7 +286,7 @@ func ipfsGet(ctx context.Context, ufs coreiface.UnixfsAPI, ipfsPath ipath.Path) } _, err = io.Copy(ioutil.Discard, fnd) if err != nil { - return fmt.Errorf("could not read migration: %w", err) + return fmt.Errorf("cannot read migration: %w", err) } fmt.Printf("Added migration file: %q\n", ipfsPath) return nil diff --git a/cmd/ipfs/migration_test.go b/cmd/ipfs/migration_test.go index 48ec0a35443..c34b108ef82 100644 --- a/cmd/ipfs/migration_test.go +++ b/cmd/ipfs/migration_test.go @@ -6,60 +6,68 @@ import ( "path/filepath" "testing" + config "github.com/ipfs/go-ipfs-config" "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" "github.com/ipfs/go-ipfs/repo/fsrepo/migrations/ipfsfetcher" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/multiformats/go-multiaddr" ) -var testPeerStrs = []string{ - "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5", - "/ip4/127.0.0.1/udp/4001/quic/p2p/12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ7", +var configData = ` +{ + "Bootstrap": [ + "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", + "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" + ], + "Migration": { + "DownloadSources": ["IPFS", "HTTP", "127.0.0.1"], + "Keep": "cache" + }, + "Peering": { + "Peers": [ + { + "ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5", + "Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"] + } + ] + } } - -var testPeers []peer.AddrInfo - -func init() { - var err error - testPeers, err = parsePeers(testPeerStrs) - if err != nil { - panic(err) +` + +var configDataBadPeers = ` +{ + "Bootstrap": [ + "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", + "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" + ], + "Migration": { + "DownloadSources": ["IPFS", "HTTP", "127.0.0.1"], + "Keep": "cache" + }, + "Peering": "Unreadable-data" +} +` + +var configDataBadBootstrap = ` +{ + "Bootstrap": "unreadable", + "Migration": { + "DownloadSources": ["IPFS", "HTTP", "127.0.0.1"], + "Keep": "cache" + }, + "Peering": { + "Peers": [ + { + "ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5", + "Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"] + } + ] } } +` func TestReadMigrationConfig(t *testing.T) { - str := ` - { - "Migration": { - "DownloadSources": ["IPFS", "HTTP", "127.0.0.1"], - "Keep": "cache", - "Peers": [ - { - "ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5", - "Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"] - } - ] - } - } - ` - - tmpDir, err := ioutil.TempDir("", "migration_test") - if err != nil { - panic(err) - } + tmpDir := makeConfig(configData) defer os.RemoveAll(tmpDir) - cfgFile, err := os.Create(filepath.Join(tmpDir, "config")) - if err != nil { - panic(err) - } - if _, err = cfgFile.Write([]byte(str)); err != nil { - panic(err) - } - if err = cfgFile.Close(); err != nil { - panic(err) - } - cfg, err := readMigrationConfig(tmpDir) if err != nil { t.Fatal(err) @@ -78,12 +86,30 @@ func TestReadMigrationConfig(t *testing.T) { if cfg.Keep != "cache" { t.Error("wrong value for Keep") } +} - if len(cfg.Peers) != 1 { +func TestReadIpfsConfig(t *testing.T) { + tmpDir := makeConfig(configData) + defer os.RemoveAll(tmpDir) + + bootstrap, peers := readIpfsConfig(nil) + if bootstrap != nil || peers != nil { + t.Fatal("expected nil ipfs config items") + } + + bootstrap, peers = readIpfsConfig(&tmpDir) + if len(bootstrap) != 2 { + t.Fatal("wrong number of bootstrap addresses") + } + if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" { + t.Fatal("wrong bootstrap address") + } + + if len(peers) != 1 { t.Fatal("wrong number of peers") } - peer := cfg.Peers[0] + peer := peers[0] if peer.ID.String() != "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5" { t.Errorf("wrong ID for first peer") } @@ -92,15 +118,70 @@ func TestReadMigrationConfig(t *testing.T) { } } +func TestReadPartialIpfsConfig(t *testing.T) { + tmpDir := makeConfig(configDataBadBootstrap) + defer os.RemoveAll(tmpDir) + + bootstrap, peers := readIpfsConfig(&tmpDir) + if bootstrap != nil { + t.Fatal("expected nil bootstrap") + } + if len(peers) != 1 { + t.Fatal("wrong number of peers") + } + if len(peers[0].Addrs) != 2 { + t.Error("wrong number of addrs for first peer") + } + os.RemoveAll(tmpDir) + + tmpDir = makeConfig(configDataBadPeers) + defer os.RemoveAll(tmpDir) + + bootstrap, peers = readIpfsConfig(&tmpDir) + if peers != nil { + t.Fatal("expected nil peers") + } + if len(bootstrap) != 2 { + t.Fatal("wrong number of bootstrap addresses") + } + if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" { + t.Fatal("wrong bootstrap address") + } +} + +func makeConfig(configData string) string { + tmpDir, err := ioutil.TempDir("", "migration_test") + if err != nil { + panic(err) + } + + cfgFile, err := os.Create(filepath.Join(tmpDir, "config")) + if err != nil { + panic(err) + } + if _, err = cfgFile.Write([]byte(configData)); err != nil { + panic(err) + } + if err = cfgFile.Close(); err != nil { + panic(err) + } + return tmpDir +} + func TestGetMigrationFetcher(t *testing.T) { var f migrations.Fetcher var err error - _, err = getMigrationFetcher([]string{"ftp://bad.gateway.io"}, nil) + + cfg := &config.Migration{} + + cfg.DownloadSources = []string{"ftp://bad.gateway.io"} + _, err = getMigrationFetcher(cfg, nil) if err == nil { t.Fatal("Expected bad URL scheme error") } - f, err = getMigrationFetcher([]string{"ipfs"}, nil) + cfg.DownloadSources = []string{"ipfs"} + f, err = getMigrationFetcher(cfg, nil) if err != nil { t.Fatal(err) } @@ -108,7 +189,8 @@ func TestGetMigrationFetcher(t *testing.T) { t.Fatal("expected IpfsFetcher") } - f, err = getMigrationFetcher([]string{"http"}, nil) + cfg.DownloadSources = []string{"http"} + f, err = getMigrationFetcher(cfg, nil) if err != nil { t.Fatal(err) } @@ -116,7 +198,8 @@ func TestGetMigrationFetcher(t *testing.T) { t.Fatal("expected HttpFetcher") } - f, err = getMigrationFetcher([]string{"IPFS", "HTTPS"}, nil) + cfg.DownloadSources = []string{"IPFS", "HTTPS"} + f, err = getMigrationFetcher(cfg, nil) if err != nil { t.Fatal(err) } @@ -128,7 +211,8 @@ func TestGetMigrationFetcher(t *testing.T) { t.Fatal("expected 2 fetchers in MultiFetcher") } - f, err = getMigrationFetcher([]string{"ipfs", "https", "some.domain.io"}, testPeers) + cfg.DownloadSources = []string{"ipfs", "https", "some.domain.io"} + f, err = getMigrationFetcher(cfg, nil) if err != nil { t.Fatal(err) } @@ -140,51 +224,15 @@ func TestGetMigrationFetcher(t *testing.T) { t.Fatal("expected 3 fetchers in MultiFetcher") } - _, err = getMigrationFetcher(nil, nil) + cfg.DownloadSources = nil + _, err = getMigrationFetcher(cfg, nil) if err == nil { - t.Fatal("expected error when no fetchers specified") + t.Fatal("expected error when no sources specified") } - _, err = getMigrationFetcher([]string{"", ""}, nil) + cfg.DownloadSources = []string{"", ""} + _, err = getMigrationFetcher(cfg, nil) if err == nil { t.Fatal("expected error when empty string fetchers specified") } } - -// parsePeers parses multiaddr strings in the form: -// /////p2p/ -func parsePeers(peers []string) ([]peer.AddrInfo, error) { - if len(peers) == 0 { - return nil, nil - } - - // Parse the peer addresses - pinfos := make(map[peer.ID]*peer.AddrInfo, len(peers)) - for _, addrStr := range peers { - addr, err := multiaddr.NewMultiaddr(addrStr) - if err != nil { - return nil, err - } - pii, err := peer.AddrInfoFromP2pAddr(addr) - if err != nil { - return nil, err - } - pi, ok := pinfos[pii.ID] - if !ok { - pi = &peer.AddrInfo{ID: pii.ID} - pinfos[pi.ID] = pi - } - pi.Addrs = append(pi.Addrs, pii.Addrs...) - } - peerAddrs := make([]peer.AddrInfo, len(pinfos)) - var i int - for _, pi := range pinfos { - peerAddrs[i] = peer.AddrInfo{ - ID: pi.ID, - Addrs: pi.Addrs, - } - i++ - } - - return peerAddrs, nil -} diff --git a/go.mod b/go.mod index 8635ed9a114..de2f8362402 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.6.0 - github.com/ipfs/go-ipfs-config v0.12.1-0.20210420072938-852f7f8bf182 + github.com/ipfs/go-ipfs-config v0.12.1-0.20210503175038-9a8ef0f5c436 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.8 @@ -88,7 +88,7 @@ require ( github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multibase v0.0.3 - github.com/multiformats/go-multihash v0.0.14 + github.com/multiformats/go-multihash v0.0.15 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.10.0 @@ -100,10 +100,10 @@ require ( go.opencensus.io v0.23.0 go.uber.org/fx v1.13.1 go.uber.org/zap v1.16.0 - golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 + golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 + golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 ) go 1.14 diff --git a/go.sum b/go.sum index 7f592bd1019..8a86d9dfa34 100644 --- a/go.sum +++ b/go.sum @@ -402,8 +402,8 @@ github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7Na github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-cmds v0.6.0 h1:yAxdowQZzoFKjcLI08sXVNnqVj3jnABbf9smrPQmBsw= github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= -github.com/ipfs/go-ipfs-config v0.12.1-0.20210420072938-852f7f8bf182 h1:ohpRgjGT2tzPRt9jhqxqwi3UKNp1YFVrvid6DDh87+k= -github.com/ipfs/go-ipfs-config v0.12.1-0.20210420072938-852f7f8bf182/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0= +github.com/ipfs/go-ipfs-config v0.12.1-0.20210503175038-9a8ef0f5c436 h1:SO4be9Pwrt8cSmNAZj8Qb9Y6MKgMaZwA+D0vLWoQENk= +github.com/ipfs/go-ipfs-config v0.12.1-0.20210503175038-9a8ef0f5c436/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -544,6 +544,8 @@ github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHz github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -882,8 +884,9 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -943,8 +946,9 @@ github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM= +github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= @@ -1253,8 +1257,10 @@ golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1399,8 +1405,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 h1:cdsMqa2nXzqlgs183pHxtvoVwU7CyzaCTAUOg94af4c= +golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go index a1917257235..e4e118f980d 100644 --- a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go +++ b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go @@ -34,9 +34,10 @@ const ( ) type IpfsFetcher struct { - distPath string - limit int64 - peers []peer.AddrInfo + distPath string + limit int64 + bootstrap []string + peers []peer.AddrInfo openOnce sync.Once openErr error @@ -57,11 +58,12 @@ type IpfsFetcher struct { // // Specifying "" for distPath sets the default IPNS path. // Specifying 0 for fetchLimit sets the default, -1 means no limit. -func NewIpfsFetcher(distPath string, fetchLimit int64, peers []peer.AddrInfo) *IpfsFetcher { +func NewIpfsFetcher(distPath string, fetchLimit int64, bootstrap []string, peers []peer.AddrInfo) *IpfsFetcher { f := &IpfsFetcher{ - limit: defaultFetchLimit, - distPath: migrations.LatestIpfsDist, - peers: peers, + limit: defaultFetchLimit, + distPath: migrations.LatestIpfsDist, + bootstrap: bootstrap, + peers: peers, } if distPath != "" { @@ -88,7 +90,7 @@ func (f *IpfsFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser // Initialize and start IPFS node on first call to Fetch, since the fetcher // may be created by not used. f.openOnce.Do(func() { - f.ipfsTmpDir, f.openErr = initTempNode(ctx) + f.ipfsTmpDir, f.openErr = initTempNode(ctx, f.bootstrap, f.peers) if f.openErr != nil { return } @@ -154,7 +156,7 @@ func (f *IpfsFetcher) recordFetched(fetchedPath ipath.Path) { f.fetched = append(f.fetched, fetchedPath) } -func initTempNode(ctx context.Context) (string, error) { +func initTempNode(ctx context.Context, bootstrap []string, peers []peer.AddrInfo) (string, error) { err := setupPlugins() if err != nil { return "", err @@ -185,6 +187,14 @@ func initTempNode(ctx context.Context) (string, error) { cfg.Addresses.API = []string{} cfg.Addresses.Swarm = []string{tempNodeTcpAddr} + if len(bootstrap) != 0 { + cfg.Bootstrap = bootstrap + } + + if len(peers) != 0 { + cfg.Peering.Peers = peers + } + err = fsrepo.Init(dir, cfg) if err != nil { os.RemoveAll(dir) @@ -231,14 +241,6 @@ func (f *IpfsFetcher) startTempNode(ctx context.Context) error { fmt.Println("migration peer", node.Identity, "shutdown") } - // Asynchronously connect to peers. The peer may be a close-by node that - // helps get migrations faster, and not connecting is not considered a - // failure. Migrations may still be available through defaults or other - // peers. - if len(f.peers) != 0 { - go connectPeers(ctxIpfsLife, ipfs, f.peers) - } - addrs, err := ipfs.Swarm().LocalAddrs(ctx) if err != nil { // Failure to get the local swarm address only means that the @@ -310,17 +312,3 @@ func setupPlugins() error { return nil } - -func connectPeers(ctx context.Context, ipfs iface.CoreAPI, peers []peer.AddrInfo) { - // Do not return an error if there is a failure to connect to a peer, since - // node may still be able to operate. Only write the errors to stderr. - for i := range peers { - go func(pi peer.AddrInfo) { - if err := ipfs.Swarm().Connect(ctx, pi); err != nil { - fmt.Fprintf(os.Stderr, "cound not connec to %q: %s\n", pi.ID, err) - } else { - fmt.Fprintf(os.Stderr, "conneced to peer %q\n", pi.ID) - } - }(peers[i]) - } -} diff --git a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go index 6e871ac9df5..d4fca0d7981 100644 --- a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go +++ b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go @@ -17,7 +17,7 @@ func TestIpfsFetcher(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fetcher := NewIpfsFetcher("", 0, nil) + fetcher := NewIpfsFetcher("", 0, nil, nil) defer fetcher.Close() rc, err := fetcher.Fetch(ctx, "go-ipfs/versions") @@ -55,11 +55,11 @@ func TestInitIpfsFetcher(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - f := NewIpfsFetcher("", 0, nil) + f := NewIpfsFetcher("", 0, nil, nil) defer f.Close() // Init ipfs repo - f.ipfsTmpDir, f.openErr = initTempNode(ctx) + f.ipfsTmpDir, f.openErr = initTempNode(ctx, f.bootstrap, f.peers) if f.openErr != nil { t.Fatalf("failed to init ipfs node: %s", f.openErr) }