From 298e15398b8c597c060b3b733d0c465216c3c725 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Mon, 29 Apr 2024 15:35:32 +0200 Subject: [PATCH] Revert "backward compatibility of .lock" and Backward compatibility by Giulio (#10077) Reverts ledgerwatch/erigon#10006 and add a proper migration routine --- erigon-lib/common/dir/rw_dir.go | 11 +- erigon-lib/direct/downloader_client.go | 4 +- erigon-lib/direct/sentry_client_mock.go | 16 +- .../downloader/downloader_grpc_server.go | 5 +- erigon-lib/downloader/torrent_files.go | 106 ++++---- erigon-lib/downloader/torrent_files_test.go | 59 ----- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 +- .../gointerfaces/downloader/downloader.pb.go | 241 ++++++------------ .../downloader/downloader_grpc.pb.go | 50 ++-- eth/backend.go | 7 +- migrations/migrations.go | 16 +- migrations/prohibit_new_downloads2.go | 62 +++++ turbo/snapshotsync/snapshotsync.go | 34 +-- 14 files changed, 274 insertions(+), 343 deletions(-) delete mode 100644 erigon-lib/downloader/torrent_files_test.go create mode 100644 migrations/prohibit_new_downloads2.go diff --git a/erigon-lib/common/dir/rw_dir.go b/erigon-lib/common/dir/rw_dir.go index a4e4d69bc31..769ba0f3cd2 100644 --- a/erigon-lib/common/dir/rw_dir.go +++ b/erigon-lib/common/dir/rw_dir.go @@ -72,16 +72,15 @@ func WriteFileWithFsync(name string, data []byte, perm os.FileMode) error { return err } defer f.Close() - if _, err = f.Write(data); err != nil { - return err - } - if err = f.Sync(); err != nil { + _, err = f.Write(data) + if err != nil { return err } - if err = f.Close(); err != nil { + err = f.Sync() + if err != nil { return err } - return nil + return err } func Recreate(dir string) { diff --git a/erigon-lib/direct/downloader_client.go b/erigon-lib/direct/downloader_client.go index 63ba7cb477b..319e3bcd1d2 100644 --- a/erigon-lib/direct/downloader_client.go +++ b/erigon-lib/direct/downloader_client.go @@ -36,8 +36,8 @@ func (c *DownloaderClient) Add(ctx context.Context, in *proto_downloader.AddRequ return c.server.Add(ctx, in) } -func (c *DownloaderClient) Prohibit(ctx context.Context, in *proto_downloader.ProhibitRequest, opts ...grpc.CallOption) (*proto_downloader.ProhibitReply, error) { - return c.server.Prohibit(ctx, in) +func (c *DownloaderClient) ProhibitNewDownloads(ctx context.Context, in *proto_downloader.ProhibitNewDownloadsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + return c.server.ProhibitNewDownloads(ctx, in) } func (c *DownloaderClient) Delete(ctx context.Context, in *proto_downloader.DeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { return c.server.Delete(ctx, in) diff --git a/erigon-lib/direct/sentry_client_mock.go b/erigon-lib/direct/sentry_client_mock.go index 3cf18f11298..48074023d41 100644 --- a/erigon-lib/direct/sentry_client_mock.go +++ b/erigon-lib/direct/sentry_client_mock.go @@ -10,14 +10,14 @@ package direct import ( - context "context" - reflect "reflect" - - sentry "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" - types "github.com/ledgerwatch/erigon-lib/gointerfaces/types" - gomock "go.uber.org/mock/gomock" - grpc "google.golang.org/grpc" - emptypb "google.golang.org/protobuf/types/known/emptypb" + "context" + "reflect" + + "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" + "github.com/ledgerwatch/erigon-lib/gointerfaces/types" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/emptypb" ) // MockSentryClient is a mock of SentryClient interface. diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index dea7e0dc769..4e0aa0edd34 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -45,9 +45,8 @@ type GrpcServer struct { d *Downloader } -func (s *GrpcServer) Prohibit(ctx context.Context, req *proto_downloader.ProhibitRequest) (*proto_downloader.ProhibitReply, error) { - whitelist, err := s.d.torrentFS.ProhibitNewDownloads(req.WhitelistAdd, req.WhitelistRemove) - return &proto_downloader.ProhibitReply{Whitelist: whitelist}, err +func (s *GrpcServer) ProhibitNewDownloads(ctx context.Context, req *proto_downloader.ProhibitNewDownloadsRequest) (*emptypb.Empty, error) { + return &emptypb.Empty{}, s.d.torrentFS.ProhibitNewDownloads(req.Type) } // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index 9df668f3cf5..4007713c5f9 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -3,6 +3,7 @@ package downloader import ( "encoding/json" "fmt" + "io" "os" "path/filepath" "slices" @@ -168,90 +169,79 @@ const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock" // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) // After "download once" - Erigon will produce and seed new files -// After `Prohibit` call - downloader stil will able: -// - seed new (generated by Erigon) files -// - seed existing on Disk files -// - download uncomplete parts of existing on Disk files (if Verify found some bad parts) -// -// `Prohibit` has `whitelist` feature - based on file-type -func (tf *AtomicTorrentFS) ProhibitNewDownloads(whitelistAdd, whitelistRemove []string) (whitelist []string, err error) { +// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) +func (tf *AtomicTorrentFS) ProhibitNewDownloads(t string) error { tf.lock.Lock() defer tf.lock.Unlock() - return tf.prohibitNewDownloads(whitelistAdd, whitelistRemove) + return tf.prohibitNewDownloads(t) } -func (tf *AtomicTorrentFS) prohibitNewDownloads(whitelistAdd, whitelistRemove []string) (whitelist []string, err error) { - fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName) - exist := dir.FileExist(fPath) - - var _currentWhiteList []string - if exist { - torrentListJsonBytes, err := os.ReadFile(fPath) - if err != nil { - return nil, fmt.Errorf("read file: %w", err) - } - if len(torrentListJsonBytes) > 0 { - if err := json.Unmarshal(torrentListJsonBytes, &_currentWhiteList); err != nil { - return nil, fmt.Errorf("unmarshal: %w", err) - } - } +func (tf *AtomicTorrentFS) prohibitNewDownloads(t string) error { + // open or create file ProhibitNewDownloadsFileName + f, err := os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_CREATE|os.O_RDONLY, 0644) + if err != nil { + return fmt.Errorf("open file: %w", err) } - - whiteList := make([]string, 0, len(_currentWhiteList)) - // copy all item except delted - for _, it := range _currentWhiteList { - if slices.Contains(whitelistRemove, it) { - continue - } - whiteList = append(whiteList, it) + defer f.Close() + var prohibitedList []string + torrentListJsonBytes, err := io.ReadAll(f) + if err != nil { + return fmt.Errorf("read file: %w", err) } - - // add all new whitelisted items - for _, it := range whitelistAdd { - if !slices.Contains(whiteList, it) { - whiteList = append(whiteList, it) + if len(torrentListJsonBytes) > 0 { + if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil { + return fmt.Errorf("unmarshal: %w", err) } } - slices.Sort(whiteList) + if slices.Contains(prohibitedList, t) { + return nil + } + prohibitedList = append(prohibitedList, t) + f.Close() - whiteListBytes, err := json.Marshal(whiteList) + // write new prohibited list by opening the file in truncate mode + f, err = os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_TRUNC|os.O_WRONLY, 0644) if err != nil { - return _currentWhiteList, fmt.Errorf("marshal: %w", err) + return fmt.Errorf("open file for writing: %w", err) } - if err := dir.WriteFileWithFsync(fPath, whiteListBytes, 0644); err != nil { - return _currentWhiteList, fmt.Errorf("write: %w", err) + defer f.Close() + prohibitedListJsonBytes, err := json.Marshal(prohibitedList) + if err != nil { + return fmt.Errorf("marshal: %w", err) + } + if _, err := f.Write(prohibitedListJsonBytes); err != nil { + return fmt.Errorf("write: %w", err) } - return whiteList, nil + + return f.Sync() } -func (tf *AtomicTorrentFS) NewDownloadsAreProhibited(name string) (prohibited bool, err error) { +func (tf *AtomicTorrentFS) NewDownloadsAreProhibited(name string) (bool, error) { tf.lock.Lock() defer tf.lock.Unlock() return tf.newDownloadsAreProhibited(name) } -func (tf *AtomicTorrentFS) newDownloadsAreProhibited(name string) (prohibited bool, err error) { - fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName) - exists := dir.FileExist(fPath) - if !exists { // no .lock - means all allowed - return false, nil +func (tf *AtomicTorrentFS) newDownloadsAreProhibited(name string) (bool, error) { + f, err := os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_CREATE|os.O_RDONLY, 0644) + if err != nil { + return false, err } - - var whiteList []string - whiteListBytes, err := os.ReadFile(fPath) + defer f.Close() + var prohibitedList []string + torrentListJsonBytes, err := io.ReadAll(f) if err != nil { return false, fmt.Errorf("NewDownloadsAreProhibited: read file: %w", err) } - if len(whiteListBytes) > 0 { - if err := json.Unmarshal(whiteListBytes, &whiteList); err != nil { + if len(torrentListJsonBytes) > 0 { + if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil { return false, fmt.Errorf("NewDownloadsAreProhibited: unmarshal: %w", err) } } - - for _, whiteListedItem := range whiteList { - if strings.Contains(name, whiteListedItem) { - return false, nil + for _, p := range prohibitedList { + if strings.Contains(name, p) { + return true, nil } } - return true, nil + return false, nil } diff --git a/erigon-lib/downloader/torrent_files_test.go b/erigon-lib/downloader/torrent_files_test.go deleted file mode 100644 index 6be124b6ce0..00000000000 --- a/erigon-lib/downloader/torrent_files_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package downloader - -import ( - "os" - "path/filepath" - "testing" - - "github.com/ledgerwatch/erigon-lib/common/datadir" - "github.com/stretchr/testify/require" -) - -func TestFSProhibitBackwardCompat(t *testing.T) { - require := require.New(t) - dirs := datadir.New(t.TempDir()) - - //prev version of .lock - is empty .lock file which exitence prohibiting everything - t.Run("no prev version .lock", func(t *testing.T) { - tf := NewAtomicTorrentFS(dirs.Snap) - prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") - require.NoError(err) - require.False(prohibited) - prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent") - require.NoError(err) - require.False(prohibited) - }) - t.Run("prev version .lock support", func(t *testing.T) { - err := os.WriteFile(filepath.Join(dirs.Snap, ProhibitNewDownloadsFileName), nil, 0644) - require.NoError(err) - - tf := NewAtomicTorrentFS(dirs.Snap) - prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") - require.NoError(err) - require.True(prohibited) - prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent") - require.NoError(err) - require.True(prohibited) - }) - t.Run("prev version .lock upgrade", func(t *testing.T) { - //old lock - err := os.WriteFile(filepath.Join(dirs.Snap, ProhibitNewDownloadsFileName), nil, 0644) - require.NoError(err) - - tf := NewAtomicTorrentFS(dirs.Snap) - wl, err := tf.prohibitNewDownloads([]string{"transactions"}, nil) //upgrade - require.NoError(err) - require.Equal([]string{"transactions"}, wl) - - prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") - require.NoError(err) - require.True(prohibited) - prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent") - require.NoError(err) - require.True(prohibited) - - prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-transactions.seg") - require.NoError(err) - require.False(prohibited) - }) -} diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 7c7a97c2850..fed8d85b34d 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/erigontech/mdbx-go v0.27.24 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901 - github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08 + github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087 github.com/ledgerwatch/log/v3 v3.9.0 github.com/ledgerwatch/secp256k1 v1.0.0 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 6fba8f15897..2d1e47500c5 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -269,8 +269,8 @@ github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901 h1:gAcI47OHnt/1e/APIV0093NVdviIfAnBUzFyybmKL1Q= github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= -github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08 h1:NQRyMIGIapAFnr7hAY0xXQZPMBjtYCUAQ0UF1/saBaE= -github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= +github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087 h1:Y59HUAT/+02Qbm6g7MuY7i8E0kUihPe7+ftDnR8oQzQ= +github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= diff --git a/erigon-lib/gointerfaces/downloader/downloader.pb.go b/erigon-lib/gointerfaces/downloader/downloader.pb.go index 3761a45a6b1..dec9c5cc3e7 100644 --- a/erigon-lib/gointerfaces/downloader/downloader.pb.go +++ b/erigon-lib/gointerfaces/downloader/downloader.pb.go @@ -251,17 +251,16 @@ func (*StatsRequest) Descriptor() ([]byte, []int) { return file_downloader_downloader_proto_rawDescGZIP(), []int{4} } -type ProhibitRequest struct { +type ProhibitNewDownloadsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - WhitelistAdd []string `protobuf:"bytes,1,rep,name=whitelistAdd,proto3" json:"whitelistAdd,omitempty"` // nil - means "don't modify". non-nil - means "merge with current whitelist". - WhitelistRemove []string `protobuf:"bytes,2,rep,name=whitelistRemove,proto3" json:"whitelistRemove,omitempty"` // nil - means "don't modify" + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` } -func (x *ProhibitRequest) Reset() { - *x = ProhibitRequest{} +func (x *ProhibitNewDownloadsRequest) Reset() { + *x = ProhibitNewDownloadsRequest{} if protoimpl.UnsafeEnabled { mi := &file_downloader_downloader_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -269,13 +268,13 @@ func (x *ProhibitRequest) Reset() { } } -func (x *ProhibitRequest) String() string { +func (x *ProhibitNewDownloadsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ProhibitRequest) ProtoMessage() {} +func (*ProhibitNewDownloadsRequest) ProtoMessage() {} -func (x *ProhibitRequest) ProtoReflect() protoreflect.Message { +func (x *ProhibitNewDownloadsRequest) ProtoReflect() protoreflect.Message { mi := &file_downloader_downloader_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -287,70 +286,16 @@ func (x *ProhibitRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ProhibitRequest.ProtoReflect.Descriptor instead. -func (*ProhibitRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use ProhibitNewDownloadsRequest.ProtoReflect.Descriptor instead. +func (*ProhibitNewDownloadsRequest) Descriptor() ([]byte, []int) { return file_downloader_downloader_proto_rawDescGZIP(), []int{5} } -func (x *ProhibitRequest) GetWhitelistAdd() []string { +func (x *ProhibitNewDownloadsRequest) GetType() string { if x != nil { - return x.WhitelistAdd + return x.Type } - return nil -} - -func (x *ProhibitRequest) GetWhitelistRemove() []string { - if x != nil { - return x.WhitelistRemove - } - return nil -} - -type ProhibitReply struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Whitelist []string `protobuf:"bytes,1,rep,name=whitelist,proto3" json:"whitelist,omitempty"` // current whitelist -} - -func (x *ProhibitReply) Reset() { - *x = ProhibitReply{} - if protoimpl.UnsafeEnabled { - mi := &file_downloader_downloader_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ProhibitReply) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ProhibitReply) ProtoMessage() {} - -func (x *ProhibitReply) ProtoReflect() protoreflect.Message { - mi := &file_downloader_downloader_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ProhibitReply.ProtoReflect.Descriptor instead. -func (*ProhibitReply) Descriptor() ([]byte, []int) { - return file_downloader_downloader_proto_rawDescGZIP(), []int{6} -} - -func (x *ProhibitReply) GetWhitelist() []string { - if x != nil { - return x.Whitelist - } - return nil + return "" } type StatsReply struct { @@ -378,7 +323,7 @@ type StatsReply struct { func (x *StatsReply) Reset() { *x = StatsReply{} if protoimpl.UnsafeEnabled { - mi := &file_downloader_downloader_proto_msgTypes[7] + mi := &file_downloader_downloader_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -391,7 +336,7 @@ func (x *StatsReply) String() string { func (*StatsReply) ProtoMessage() {} func (x *StatsReply) ProtoReflect() protoreflect.Message { - mi := &file_downloader_downloader_proto_msgTypes[7] + mi := &file_downloader_downloader_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -404,7 +349,7 @@ func (x *StatsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use StatsReply.ProtoReflect.Descriptor instead. func (*StatsReply) Descriptor() ([]byte, []int) { - return file_downloader_downloader_proto_rawDescGZIP(), []int{7} + return file_downloader_downloader_proto_rawDescGZIP(), []int{6} } func (x *StatsReply) GetMetadataReady() int32 { @@ -498,62 +443,57 @@ var file_downloader_downloader_proto_rawDesc = []byte{ 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0x0f, 0x0a, 0x0d, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5f, 0x0a, 0x0f, 0x50, 0x72, 0x6f, - 0x68, 0x69, 0x62, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, - 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x0c, 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x41, 0x64, 0x64, - 0x12, 0x28, 0x0a, 0x0f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x68, 0x69, 0x74, 0x65, - 0x6c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x22, 0x2d, 0x0a, 0x0d, 0x50, 0x72, - 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x77, - 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, - 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x22, 0xee, 0x02, 0x0a, 0x0a, 0x53, 0x74, - 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0d, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, - 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, - 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x65, 0x65, 0x72, 0x73, 0x5f, 0x75, 0x6e, 0x69, 0x71, 0x75, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x70, 0x65, 0x65, 0x72, 0x73, 0x55, 0x6e, 0x69, - 0x71, 0x75, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, - 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x1a, - 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x02, - 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x62, 0x79, - 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, - 0x74, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, - 0x61, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, - 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x72, - 0x61, 0x74, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x75, 0x70, 0x6c, 0x6f, 0x61, - 0x64, 0x52, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, - 0x64, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x64, 0x6f, - 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x61, 0x74, 0x65, 0x32, 0xc6, 0x02, 0x0a, 0x0a, 0x44, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x12, 0x44, 0x0a, 0x08, 0x50, 0x72, 0x6f, - 0x68, 0x69, 0x62, 0x69, 0x74, 0x12, 0x1b, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, - 0x65, 0x72, 0x2e, 0x50, 0x72, 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, - 0x50, 0x72, 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, - 0x37, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x12, 0x16, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, - 0x64, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x12, 0x19, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x1b, 0x50, 0x72, 0x6f, + 0x68, 0x69, 0x62, 0x69, 0x74, 0x4e, 0x65, 0x77, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xee, 0x02, 0x0a, + 0x0a, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x61, + 0x64, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x54, 0x6f, + 0x74, 0x61, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x65, 0x65, 0x72, 0x73, 0x5f, 0x75, 0x6e, 0x69, + 0x71, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x70, 0x65, 0x65, 0x72, 0x73, + 0x55, 0x6e, 0x69, 0x71, 0x75, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x54, 0x6f, + 0x74, 0x61, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, + 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x02, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x27, 0x0a, + 0x0f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, + 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, + 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x62, 0x79, 0x74, + 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x70, 0x6c, 0x6f, 0x61, + 0x64, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x75, 0x70, + 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x6f, 0x77, 0x6e, + 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0c, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x61, 0x74, 0x65, 0x32, 0xdb, 0x02, + 0x0a, 0x0a, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x12, 0x59, 0x0a, 0x14, + 0x50, 0x72, 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x4e, 0x65, 0x77, 0x44, 0x6f, 0x77, 0x6e, 0x6c, + 0x6f, 0x61, 0x64, 0x73, 0x12, 0x27, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, + 0x72, 0x2e, 0x50, 0x72, 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x4e, 0x65, 0x77, 0x44, 0x6f, 0x77, + 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x56, 0x65, 0x72, 0x69, 0x66, - 0x79, 0x12, 0x19, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x56, - 0x65, 0x72, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, - 0x18, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x6f, 0x77, 0x6e, - 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, - 0x79, 0x22, 0x00, 0x42, 0x19, 0x5a, 0x17, 0x2e, 0x2f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, - 0x64, 0x65, 0x72, 0x3b, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x12, 0x16, + 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, + 0x12, 0x3d, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x19, 0x2e, 0x64, 0x6f, 0x77, + 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, + 0x3d, 0x0a, 0x06, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x12, 0x19, 0x2e, 0x64, 0x6f, 0x77, 0x6e, + 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, + 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x18, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, + 0x61, 0x64, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x19, 0x5a, 0x17, 0x2e, + 0x2f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x3b, 0x64, 0x6f, 0x77, 0x6e, + 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -568,32 +508,31 @@ func file_downloader_downloader_proto_rawDescGZIP() []byte { return file_downloader_downloader_proto_rawDescData } -var file_downloader_downloader_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_downloader_downloader_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_downloader_downloader_proto_goTypes = []interface{}{ - (*AddItem)(nil), // 0: downloader.AddItem - (*AddRequest)(nil), // 1: downloader.AddRequest - (*DeleteRequest)(nil), // 2: downloader.DeleteRequest - (*VerifyRequest)(nil), // 3: downloader.VerifyRequest - (*StatsRequest)(nil), // 4: downloader.StatsRequest - (*ProhibitRequest)(nil), // 5: downloader.ProhibitRequest - (*ProhibitReply)(nil), // 6: downloader.ProhibitReply - (*StatsReply)(nil), // 7: downloader.StatsReply - (*types.H160)(nil), // 8: types.H160 - (*emptypb.Empty)(nil), // 9: google.protobuf.Empty + (*AddItem)(nil), // 0: downloader.AddItem + (*AddRequest)(nil), // 1: downloader.AddRequest + (*DeleteRequest)(nil), // 2: downloader.DeleteRequest + (*VerifyRequest)(nil), // 3: downloader.VerifyRequest + (*StatsRequest)(nil), // 4: downloader.StatsRequest + (*ProhibitNewDownloadsRequest)(nil), // 5: downloader.ProhibitNewDownloadsRequest + (*StatsReply)(nil), // 6: downloader.StatsReply + (*types.H160)(nil), // 7: types.H160 + (*emptypb.Empty)(nil), // 8: google.protobuf.Empty } var file_downloader_downloader_proto_depIdxs = []int32{ - 8, // 0: downloader.AddItem.torrent_hash:type_name -> types.H160 + 7, // 0: downloader.AddItem.torrent_hash:type_name -> types.H160 0, // 1: downloader.AddRequest.items:type_name -> downloader.AddItem - 5, // 2: downloader.Downloader.Prohibit:input_type -> downloader.ProhibitRequest + 5, // 2: downloader.Downloader.ProhibitNewDownloads:input_type -> downloader.ProhibitNewDownloadsRequest 1, // 3: downloader.Downloader.Add:input_type -> downloader.AddRequest 2, // 4: downloader.Downloader.Delete:input_type -> downloader.DeleteRequest 3, // 5: downloader.Downloader.Verify:input_type -> downloader.VerifyRequest 4, // 6: downloader.Downloader.Stats:input_type -> downloader.StatsRequest - 6, // 7: downloader.Downloader.Prohibit:output_type -> downloader.ProhibitReply - 9, // 8: downloader.Downloader.Add:output_type -> google.protobuf.Empty - 9, // 9: downloader.Downloader.Delete:output_type -> google.protobuf.Empty - 9, // 10: downloader.Downloader.Verify:output_type -> google.protobuf.Empty - 7, // 11: downloader.Downloader.Stats:output_type -> downloader.StatsReply + 8, // 7: downloader.Downloader.ProhibitNewDownloads:output_type -> google.protobuf.Empty + 8, // 8: downloader.Downloader.Add:output_type -> google.protobuf.Empty + 8, // 9: downloader.Downloader.Delete:output_type -> google.protobuf.Empty + 8, // 10: downloader.Downloader.Verify:output_type -> google.protobuf.Empty + 6, // 11: downloader.Downloader.Stats:output_type -> downloader.StatsReply 7, // [7:12] is the sub-list for method output_type 2, // [2:7] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name @@ -668,7 +607,7 @@ func file_downloader_downloader_proto_init() { } } file_downloader_downloader_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProhibitRequest); i { + switch v := v.(*ProhibitNewDownloadsRequest); i { case 0: return &v.state case 1: @@ -680,18 +619,6 @@ func file_downloader_downloader_proto_init() { } } file_downloader_downloader_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProhibitReply); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_downloader_downloader_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StatsReply); i { case 0: return &v.state @@ -710,7 +637,7 @@ func file_downloader_downloader_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_downloader_downloader_proto_rawDesc, NumEnums: 0, - NumMessages: 8, + NumMessages: 7, NumExtensions: 0, NumServices: 1, }, diff --git a/erigon-lib/gointerfaces/downloader/downloader_grpc.pb.go b/erigon-lib/gointerfaces/downloader/downloader_grpc.pb.go index 5a70ffb6656..369c9b494c4 100644 --- a/erigon-lib/gointerfaces/downloader/downloader_grpc.pb.go +++ b/erigon-lib/gointerfaces/downloader/downloader_grpc.pb.go @@ -20,11 +20,11 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( - Downloader_Prohibit_FullMethodName = "/downloader.Downloader/Prohibit" - Downloader_Add_FullMethodName = "/downloader.Downloader/Add" - Downloader_Delete_FullMethodName = "/downloader.Downloader/Delete" - Downloader_Verify_FullMethodName = "/downloader.Downloader/Verify" - Downloader_Stats_FullMethodName = "/downloader.Downloader/Stats" + Downloader_ProhibitNewDownloads_FullMethodName = "/downloader.Downloader/ProhibitNewDownloads" + Downloader_Add_FullMethodName = "/downloader.Downloader/Add" + Downloader_Delete_FullMethodName = "/downloader.Downloader/Delete" + Downloader_Verify_FullMethodName = "/downloader.Downloader/Verify" + Downloader_Stats_FullMethodName = "/downloader.Downloader/Stats" ) // DownloaderClient is the client API for Downloader service. @@ -33,12 +33,8 @@ const ( type DownloaderClient interface { // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) // After "download once" - Erigon will produce and seed new files - // After `ProhibitNew` call - downloader stil will able: - // - seed new files (already existing on FS) - // - download uncomplete parts of existing files (if Verify found some bad parts) - // - // `ProhibitNew` what whitelist based on file-type - can add remove items there. - Prohibit(ctx context.Context, in *ProhibitRequest, opts ...grpc.CallOption) (*ProhibitReply, error) + // Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) + ProhibitNewDownloads(ctx context.Context, in *ProhibitNewDownloadsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) // Adding new file to downloader: non-existing files it will download, existing - seed Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) @@ -56,9 +52,9 @@ func NewDownloaderClient(cc grpc.ClientConnInterface) DownloaderClient { return &downloaderClient{cc} } -func (c *downloaderClient) Prohibit(ctx context.Context, in *ProhibitRequest, opts ...grpc.CallOption) (*ProhibitReply, error) { - out := new(ProhibitReply) - err := c.cc.Invoke(ctx, Downloader_Prohibit_FullMethodName, in, out, opts...) +func (c *downloaderClient) ProhibitNewDownloads(ctx context.Context, in *ProhibitNewDownloadsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, Downloader_ProhibitNewDownloads_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -107,12 +103,8 @@ func (c *downloaderClient) Stats(ctx context.Context, in *StatsRequest, opts ... type DownloaderServer interface { // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) // After "download once" - Erigon will produce and seed new files - // After `ProhibitNew` call - downloader stil will able: - // - seed new files (already existing on FS) - // - download uncomplete parts of existing files (if Verify found some bad parts) - // - // `ProhibitNew` what whitelist based on file-type - can add remove items there. - Prohibit(context.Context, *ProhibitRequest) (*ProhibitReply, error) + // Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) + ProhibitNewDownloads(context.Context, *ProhibitNewDownloadsRequest) (*emptypb.Empty, error) // Adding new file to downloader: non-existing files it will download, existing - seed Add(context.Context, *AddRequest) (*emptypb.Empty, error) Delete(context.Context, *DeleteRequest) (*emptypb.Empty, error) @@ -127,8 +119,8 @@ type DownloaderServer interface { type UnimplementedDownloaderServer struct { } -func (UnimplementedDownloaderServer) Prohibit(context.Context, *ProhibitRequest) (*ProhibitReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method Prohibit not implemented") +func (UnimplementedDownloaderServer) ProhibitNewDownloads(context.Context, *ProhibitNewDownloadsRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ProhibitNewDownloads not implemented") } func (UnimplementedDownloaderServer) Add(context.Context, *AddRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Add not implemented") @@ -155,20 +147,20 @@ func RegisterDownloaderServer(s grpc.ServiceRegistrar, srv DownloaderServer) { s.RegisterService(&Downloader_ServiceDesc, srv) } -func _Downloader_Prohibit_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ProhibitRequest) +func _Downloader_ProhibitNewDownloads_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProhibitNewDownloadsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(DownloaderServer).Prohibit(ctx, in) + return srv.(DownloaderServer).ProhibitNewDownloads(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: Downloader_Prohibit_FullMethodName, + FullMethod: Downloader_ProhibitNewDownloads_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(DownloaderServer).Prohibit(ctx, req.(*ProhibitRequest)) + return srv.(DownloaderServer).ProhibitNewDownloads(ctx, req.(*ProhibitNewDownloadsRequest)) } return interceptor(ctx, in, info, handler) } @@ -253,8 +245,8 @@ var Downloader_ServiceDesc = grpc.ServiceDesc{ HandlerType: (*DownloaderServer)(nil), Methods: []grpc.MethodDesc{ { - MethodName: "Prohibit", - Handler: _Downloader_Prohibit_Handler, + MethodName: "ProhibitNewDownloads", + Handler: _Downloader_ProhibitNewDownloads_Handler, }, { MethodName: "Add", diff --git a/eth/backend.go b/eth/backend.go index 20419c524c8..4a9712d4540 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -55,6 +55,7 @@ import ( "github.com/ledgerwatch/erigon-lib/downloader" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/downloadergrpc" + "github.com/ledgerwatch/erigon-lib/downloader/snaptype" protodownloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader" "github.com/ledgerwatch/erigon-lib/gointerfaces/grpcutil" "github.com/ledgerwatch/erigon-lib/gointerfaces/remote" @@ -789,7 +790,11 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger hook := stages2.NewHook(backend.sentryCtx, backend.chainDB, backend.notifications, backend.stagedSync, backend.blockReader, backend.chainConfig, backend.logger, backend.sentriesClient.SetStatus) if !config.Sync.UseSnapshots && backend.downloaderClient != nil { - _, _ = backend.downloaderClient.Prohibit(ctx, &protodownloader.ProhibitRequest{}) + for _, p := range snaptype.AllTypes { + backend.downloaderClient.ProhibitNewDownloads(ctx, &protodownloader.ProhibitNewDownloadsRequest{ + Type: p.String(), + }) + } } checkStateRoot := true diff --git a/migrations/migrations.go b/migrations/migrations.go index 7bcd4824d5e..a6e9de92f77 100644 --- a/migrations/migrations.go +++ b/migrations/migrations.go @@ -37,6 +37,7 @@ var migrations = map[kv.Label][]Migration{ TxsBeginEnd, TxsV3, ProhibitNewDownloadsLock, + ProhibitNewDownloadsLock2, }, kv.TxPoolDB: {}, kv.SentryDB: {}, @@ -51,7 +52,9 @@ type Migration struct { var ( ErrMigrationNonUniqueName = fmt.Errorf("please provide unique migration name") ErrMigrationCommitNotCalled = fmt.Errorf("migration before-commit function was not called") - ErrMigrationETLFilesDeleted = fmt.Errorf("db migration progress was interrupted after extraction step and ETL files was deleted, please contact development team for help or re-sync from scratch") + ErrMigrationETLFilesDeleted = fmt.Errorf( + "db migration progress was interrupted after extraction step and ETL files was deleted, please contact development team for help or re-sync from scratch", + ) ) func NewMigrator(label kv.Label) *Migrator { @@ -238,7 +241,16 @@ func (m *Migrator) Apply(db kv.RwDB, dataDir string, logger log.Logger) error { }); err != nil { return fmt.Errorf("migrator.Apply: %w", err) } - logger.Info("Updated DB schema to", "version", fmt.Sprintf("%d.%d.%d", kv.DBSchemaVersion.Major, kv.DBSchemaVersion.Minor, kv.DBSchemaVersion.Patch)) + logger.Info( + "Updated DB schema to", + "version", + fmt.Sprintf( + "%d.%d.%d", + kv.DBSchemaVersion.Major, + kv.DBSchemaVersion.Minor, + kv.DBSchemaVersion.Patch, + ), + ) return nil } diff --git a/migrations/prohibit_new_downloads2.go b/migrations/prohibit_new_downloads2.go new file mode 100644 index 00000000000..9f890fc6de3 --- /dev/null +++ b/migrations/prohibit_new_downloads2.go @@ -0,0 +1,62 @@ +package migrations + +import ( + "context" + "encoding/json" + "io/fs" + "os" + "path/filepath" + + "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/ledgerwatch/erigon-lib/common/dir" + "github.com/ledgerwatch/erigon-lib/downloader" + "github.com/ledgerwatch/erigon-lib/downloader/snaptype" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/log/v3" +) + +// Switch to the second version of download.lock. +var ProhibitNewDownloadsLock2 = Migration{ + Name: "prohibit_new_downloads_lock2", + Up: func(db kv.RwDB, dirs datadir.Dirs, progress []byte, BeforeCommit Callback, logger log.Logger) (err error) { + tx, err := db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + fPath := filepath.Join(dirs.Snap, downloader.ProhibitNewDownloadsFileName) + if !dir.FileExist(fPath) { + if err := BeforeCommit(tx, nil, true); err != nil { + return err + } + return tx.Commit() + + } + content, err := os.ReadFile(fPath) + if err != nil { + return err + } + if len(content) == 0 { // old format, need to change to all snaptypes except blob sidecars + locked := []string{} + ts := snaptype.AllTypes + for _, t := range ts { + if t.String() != snaptype.BlobSidecars.String() { + locked = append(locked, t.String()) + } + } + newContent, err := json.Marshal(locked) + if err != nil { + return err + } + if err := os.WriteFile(fPath, newContent, fs.FileMode(os.O_TRUNC|os.O_WRONLY)); err != nil { + return err + } + } + + // This migration is no-op, but it forces the migration mechanism to apply it and thus write the DB schema version info + if err := BeforeCommit(tx, nil, true); err != nil { + return err + } + return tx.Commit() + }, +} diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index a1406fab152..334fe592d0d 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -196,24 +196,28 @@ func WaitForDownloader(ctx context.Context, logPrefix string, histV3, blobs bool return err } - // Erigon has "download once" invariant - means restart/upgrade/downgrade will not download new files (and will be fast) + // ProhibitNewDownloads implies - so only make the download request once, + // + // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) // After "download once" - Erigon will produce and seed new files - // After `Prohibit` call - downloader stil will able: - // - seed new (generated by Erigon) files - // - seed existing on Disk files - // - download uncomplete parts of existing on Disk files (if Verify found some bad parts) + // Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) + // + // after the initial call the downloader or snapshot-lock.file will prevent this download from running // - // Caplin's code is ready for backgrond 1-time download of it's files. - // So, we allow users of Caplin download and index new type of files - in background. - var whitelist []string - if caplin != NoCaplin { - whitelist = append(whitelist, snaptype.BeaconBlocks.Enum().String()) - if blobs { - whitelist = append(whitelist, snaptype.BlobSidecars.Enum().String()) + + // prohibits further downloads, except some exceptions + for _, p := range snaptype.AllTypes { + if (p.Enum() == snaptype.BeaconBlocks.Enum() || p.Enum() == snaptype.BlobSidecars.Enum()) && caplin == NoCaplin { + continue + } + if p.Enum() == snaptype.BlobSidecars.Enum() && !blobs { + continue + } + if _, err := snapshotDownloader.ProhibitNewDownloads(ctx, &proto_downloader.ProhibitNewDownloadsRequest{ + Type: p.String(), + }); err != nil { + return err } - } - if _, err := snapshotDownloader.Prohibit(ctx, &proto_downloader.ProhibitRequest{WhitelistAdd: whitelist}); err != nil { - return err } if err := rawdb.WriteSnapshots(tx, blockReader.FrozenFiles(), agg.Files()); err != nil {