services: add new service for fetching blocks from NeoFS

Close #3496

Signed-off-by: Ekaterina Pavlova <[email protected]>

cli/server/dump_bin.go

@@ -0,0 +1,88 @@
+package server
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/nspcc-dev/neo-go/cli/cmdargs"
+	"github.com/nspcc-dev/neo-go/cli/options"
+	"github.com/nspcc-dev/neo-go/pkg/core/block"
+	"github.com/nspcc-dev/neo-go/pkg/io"
+	"github.com/urfave/cli/v2"
+)
+
+func dumpBin(ctx *cli.Context) error {
+	if err := cmdargs.EnsureNone(ctx); err != nil {
+		return err
+	}
+	cfg, err := options.GetConfigFromContext(ctx)
+	if err != nil {
+		return cli.Exit(err, 1)
+	}
+	log, _, logCloser, err := options.HandleLoggingParams(ctx.Bool("debug"), cfg.ApplicationConfiguration)
+	if err != nil {
+		return cli.Exit(err, 1)
+	}
+	if logCloser != nil {
+		defer func() { _ = logCloser() }()
+	}
+	count := uint32(ctx.Uint("count"))
+	start := uint32(ctx.Uint("start"))
+
+	chain, prometheus, pprof, err := initBCWithMetrics(cfg, log)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		pprof.ShutDown()
+		prometheus.ShutDown()
+		chain.Close()
+	}()
+
+	blocksCount := chain.BlockHeight() + 1
+	if start+count > blocksCount {
+		return cli.Exit(fmt.Errorf("chain is not that high (%d) to dump %d blocks starting from %d", blocksCount-1, count, start), 1)
+	}
+	if count == 0 {
+		count = blocksCount - start
+	}
+
+	out := ctx.String("out")
+	if out == "" {
+		return cli.Exit("output directory is not specified", 1)
+	}
+	if _, err = os.Stat(out); os.IsNotExist(err) {
+		if err = os.MkdirAll(out, os.ModePerm); err != nil {
+			return cli.Exit(fmt.Sprintf("failed to create directory %s: %s", out, err), 1)
+		}
+	}
+
+	for i := start; i < start+count; i++ {
+		bh := chain.GetHeaderHash(i)
+		blk, err := chain.GetBlock(bh)
+		if err != nil {
+			return cli.Exit(fmt.Sprintf("failed to get block %d: %s", i, err), 1)
+		}
+		filePath := filepath.Join(out, fmt.Sprintf("block-%d.bin", i))
+		if err = saveBlockToFile(blk, filePath); err != nil {
+			return cli.Exit(fmt.Sprintf("failed to save block %d to file %s: %s", i, filePath, err), 1)
+		}
+	}
+	return nil
+}
+
+func saveBlockToFile(blk *block.Block, filePath string) error {
+	file, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	writer := io.NewBinWriterFromIO(file)
+	blk.EncodeBinary(writer)
+	if writer.Err != nil {
+		return writer.Err
+	}
+	return nil
+}

cli/server/dump_bin_test.go

@@ -0,0 +1,101 @@
+package server_test
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	"github.com/nspcc-dev/neo-go/internal/testcli"
+	"github.com/nspcc-dev/neo-go/pkg/config"
+	"github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestDumpBin(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	loadConfig := func(t *testing.T) config.Config {
+		chainPath := filepath.Join(tmpDir, "neogotestchain")
+		cfg, err := config.LoadFile(filepath.Join("..", "..", "config", "protocol.unit_testnet.yml"))
+		require.NoError(t, err, "could not load config")
+		cfg.ApplicationConfiguration.DBConfiguration.Type = dbconfig.LevelDB
+		cfg.ApplicationConfiguration.DBConfiguration.LevelDBOptions.DataDirectoryPath = chainPath
+		return cfg
+	}
+
+	cfg := loadConfig(t)
+	out, err := yaml.Marshal(cfg)
+	require.NoError(t, err)
+
+	cfgPath := filepath.Join(tmpDir, "protocol.unit_testnet.yml")
+	require.NoError(t, os.WriteFile(cfgPath, out, os.ModePerm))
+
+	e := testcli.NewExecutor(t, true)
+
+	restoreArgs := []string{"neo-go", "db", "restore",
+		"--config-file", cfgPath, "--in", inDump}
+	e.Run(t, restoreArgs...)
+
+	t.Run("missing output directory", func(t *testing.T) {
+		args := []string{"neo-go", "db", "dump-bin", "--unittest",
+			"--config-path", tmpDir, "--out", ""}
+		e.RunWithErrorCheck(t, "output directory is not specified", args...)
+	})
+
+	t.Run("successful dump", func(t *testing.T) {
+		outDir := filepath.Join(tmpDir, "blocks")
+		args := []string{"neo-go", "db", "dump-bin", "--unittest",
+			"--config-path", tmpDir, "--out", outDir, "--count", "5", "--start", "0"}
+
+		e.Run(t, args...)
+
+		require.DirExists(t, outDir)
+
+		for i := 0; i < 5; i++ {
+			blockFile := filepath.Join(outDir, "block-"+strconv.Itoa(i)+".bin")
+			require.FileExists(t, blockFile)
+		}
+	})
+
+	t.Run("invalid block range", func(t *testing.T) {
+		outDir := filepath.Join(tmpDir, "invalid-blocks")
+		args := []string{"neo-go", "db", "dump-bin", "--unittest",
+			"--config-path", tmpDir, "--out", outDir, "--count", "1000", "--start", "0"}
+
+		e.RunWithError(t, args...)
+	})
+
+	t.Run("output directory with no write permission", func(t *testing.T) {
+		outDir := filepath.Join(tmpDir, "no-write-permission")
+		require.NoError(t, os.Mkdir(outDir, 0400))
+
+		args := []string{"neo-go", "db", "dump-bin", "--unittest",
+			"--config-path", tmpDir, "--out", outDir, "--count", "5", "--start", "0"}
+
+		e.RunWithError(t, args...)
+	})
+
+	t.Run("zero blocks (full chain dump)", func(t *testing.T) {
+		outDir := filepath.Join(tmpDir, "full-dump")
+		args := []string{"neo-go", "db", "dump-bin", "--unittest",
+			"--config-path", tmpDir, "--out", outDir}
+
+		e.Run(t, args...)
+
+		require.DirExists(t, outDir)
+		for i := 0; i < 50; i++ {
+			blockFile := filepath.Join(outDir, "block-"+strconv.Itoa(i)+".bin")
+			require.FileExists(t, blockFile)
+		}
+	})
+
+	t.Run("invalid config file", func(t *testing.T) {
+		outDir := filepath.Join(tmpDir, "blocks")
+		args := []string{"neo-go", "db", "dump-bin", "--unittest",
+			"--config-path", "invalid-config-path", "--out", outDir}
+
+		e.RunWithError(t, args...)
+	})
+}

cli/server/server.go

@@ -105,6 +105,13 @@ func NewCommands() []*cli.Command {
 					Action:    dumpDB,
 					Flags:     cfgCountOutFlags,
 				},
+				{
+					Name:      "dump-bin",
+					Usage:     "Dump blocks (starting with the genesis or specified block) to the directory in binary format",
+					UsageText: "neo-go db dump-bin -o directory [-s start] [-c count] [--config-path path] [-p/-m/-t] [--config-file file]",
+					Action:    dumpBin,
+					Flags:     cfgCountOutFlags,
+				},
 				{
 					Name:      "restore",
 					Usage:     "Restore blocks from the file",

config/protocol.testnet.yml

@@ -100,3 +100,19 @@ ApplicationConfiguration:
     Enabled: false
     Addresses:
       - ":2113"
+#  NeoFSBlockFetcher:
+#    Enabled: true
+#    UnlockWallet:
+#      Path: "./testnet_wallet.json"
+#      Password: "111"
+#    Addresses:
+#      - st1.t5.fs.neo.org:8080
+#    Timeout: 100s
+#    ContainerID: "9iVfUg8aDHKjPC4LhQXEkVUM4HDkR7UCXYLs8NQwYfSG"
+#    Mode: "indexSearch"
+#    BatchSize: 100
+#    BlockAttribute: "blocks_index_1"
+#    OidAttribute: "block_oids"
+#    HeaderAttribute: "index_header"
+
+

go.mod

@@ -32,6 +32,7 @@ require (
 	golang.org/x/term v0.18.0
 	golang.org/x/text v0.14.0
 	golang.org/x/tools v0.19.0
+	google.golang.org/grpc v1.62.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -67,7 +68,6 @@ require (
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect
-	google.golang.org/grpc v1.62.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )

pkg/config/application_config.go

@@ -23,12 +23,13 @@ type ApplicationConfiguration struct {
 	Pprof      BasicService `yaml:"Pprof"`
 	Prometheus BasicService `yaml:"Prometheus"`
 
-	Relay     bool                `yaml:"Relay"`
-	Consensus Consensus           `yaml:"Consensus"`
-	RPC       RPC                 `yaml:"RPC"`
-	Oracle    OracleConfiguration `yaml:"Oracle"`
-	P2PNotary P2PNotary           `yaml:"P2PNotary"`
-	StateRoot StateRoot           `yaml:"StateRoot"`
+	Relay             bool                `yaml:"Relay"`
+	Consensus         Consensus           `yaml:"Consensus"`
+	RPC               RPC                 `yaml:"RPC"`
+	Oracle            OracleConfiguration `yaml:"Oracle"`
+	P2PNotary         P2PNotary           `yaml:"P2PNotary"`
+	StateRoot         StateRoot           `yaml:"StateRoot"`
+	NeoFSBlockFetcher NeoFSBlockFetcher   `yaml:"NeoFSBlockFetcher"`
 }
 
 // EqualsButServices returns true when the o is the same as a except for services
@@ -145,3 +146,13 @@ func (a *ApplicationConfiguration) GetAddresses() ([]AnnounceableAddress, error)
 	}
 	return addrs, nil
 }
+
+// Validate checks ApplicationConfiguration for internal consistency and returns
+// an error if any invalid settings are found. This ensures that the application
+// configuration is valid and safe to use for further operations.
+func (a *ApplicationConfiguration) Validate() error {
+	if err := a.NeoFSBlockFetcher.Validate(); err != nil {
+		return fmt.Errorf("failed to validate NeoFSBlockFetcher section: %w", err)
+	}
+	return nil
+}

pkg/config/blockfetcher_config.go

@@ -0,0 +1,49 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
+)
+
+// NeoFSBlockFetcher represents the configuration for the NeoFS block fetcher service.
+type NeoFSBlockFetcher struct {
+	InternalService `yaml:",inline"`
+	Timeout         time.Duration `yaml:"Timeout"`
+	ContainerID     string        `yaml:"ContainerID"`
+	Mode            string        `yaml:"Mode"`
+	Addresses       []string      `yaml:"Addresses"`
+	BatchSize       int           `yaml:"BatchSize"`
+	BlockAttribute  string        `yaml:"BlockAttribute"`
+	OidAttribute    string        `yaml:"OidAttribute"`
+	HeaderAttribute string        `yaml:"HeaderAttribute"`
+}
+
+// Validate checks NeoFSBlockFetcher for internal consistency and ensures
+// that all required fields are properly set. It returns an error if the
+// configuration is invalid or if the ContainerID cannot be properly decoded.
+func (cfg *NeoFSBlockFetcher) Validate() error {
+	if !cfg.Enabled {
+		return nil
+	}
+	if cfg.ContainerID == "" {
+		return errors.New("container ID is not set")
+	}
+	var containerID cid.ID
+	err := containerID.DecodeString(cfg.ContainerID)
+	if err != nil {
+		return fmt.Errorf("invalid container ID: %w", err)
+	}
+	if cfg.Timeout == 0 {
+		cfg.Timeout = 10 * time.Second
+	}
+	if cfg.BatchSize == 0 {
+		cfg.BatchSize = 50
+	}
+	if cfg.Mode == "" {
+		cfg.Mode = "indexSearch"
+	}
+	return nil
+}

pkg/config/config.go

@@ -116,6 +116,10 @@ func LoadFile(configPath string, relativePath ...string) (Config, error) {
 	if err != nil {
 		return Config{}, err
 	}
+	err = config.ApplicationConfiguration.Validate()
+	if err != nil {
+		return Config{}, err
+	}
 
 	return config, nil
 }