Merge pull request #451 from ipfs-force-community/opt/offline-commp

opt: support offline compute commp

Author: LinZexiao

client/client.go

@@ -45,7 +45,6 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	cborutil "github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-commp-utils/ffiwrapper"
 	"github.com/filecoin-project/go-commp-utils/writer"
 	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
 	"github.com/filecoin-project/go-fil-markets/discovery"
@@ -1332,47 +1331,7 @@ func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Addre
 }
 
 func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*types.CommPRet, error) {
-	// Hard-code the sector type to 32GiBV1_1, because:
-	// - ffiwrapper.GeneratePieceCIDFromFile requires a RegisteredSealProof
-	// - commP itself is sector-size independent, with rather low probability of that changing
-	//   ( note how the final rust call is identical for every RegSP type )
-	//   https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/v5.0.0/src/seal.rs#L1040-L1050
-	//
-	// IF/WHEN this changes in the future we will have to be able to calculate
-	// "old style" commP, and thus will need to introduce a version switch or similar
-	arbitraryProofType := abi.RegisteredSealProof_StackedDrg32GiBV1_1
-
-	rdr, err := os.Open(inpath)
-	if err != nil {
-		return nil, err
-	}
-	defer rdr.Close() //nolint:errcheck
-
-	stat, err := rdr.Stat()
-	if err != nil {
-		return nil, err
-	}
-
-	// check that the data is a car file; if it's not, retrieval won't work
-	_, err = car.ReadHeader(bufio.NewReader(rdr))
-	if err != nil {
-		return nil, fmt.Errorf("not a car file: %w", err)
-	}
-
-	if _, err := rdr.Seek(0, io.SeekStart); err != nil {
-		return nil, fmt.Errorf("seek to start: %w", err)
-	}
-
-	pieceReader, pieceSize := padreader.New(rdr, uint64(stat.Size()))
-	commP, err := ffiwrapper.GeneratePieceCIDFromFile(arbitraryProofType, pieceReader, pieceSize)
-	if err != nil {
-		return nil, fmt.Errorf("computing commP failed: %w", err)
-	}
-
-	return &types.CommPRet{
-		Root: commP,
-		Size: pieceSize,
-	}, nil
+	return utils.CalcCommP(ctx, inpath)
 }
 
 type lenWriter int64

cmd/droplet-client/data.go

@@ -13,6 +13,7 @@ import (
 	"github.com/filecoin-project/venus/venus-shared/types"
 	"github.com/filecoin-project/venus/venus-shared/types/market/client"
 	cli2 "github.com/ipfs-force-community/droplet/v2/cli"
+	"github.com/ipfs-force-community/droplet/v2/utils"
 )
 
 var dataCmd = &cli.Command{
@@ -205,18 +206,13 @@ var dataCommPCmd = &cli.Command{
 		&cli2.CidBaseFlag,
 	},
 	Action: func(cctx *cli.Context) error {
-		api, closer, err := cli2.NewMarketClientNode(cctx)
-		if err != nil {
-			return err
-		}
-		defer closer()
 		ctx := cli2.ReqContext(cctx)
 
 		if cctx.Args().Len() != 1 {
 			return fmt.Errorf("usage: commP <inputPath>")
 		}
 
-		ret, err := api.ClientCalcCommP(ctx, cctx.Args().Get(0))
+		ret, err := utils.CalcCommP(ctx, cctx.Args().Get(0))
 		if err != nil {
 			return err
 		}

utils/commp.go

@@ -0,0 +1,59 @@
+package utils
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/filecoin-project/go-commp-utils/ffiwrapper"
+	"github.com/filecoin-project/go-padreader"
+	"github.com/filecoin-project/go-state-types/abi"
+	types "github.com/filecoin-project/venus/venus-shared/types/market/client"
+	"github.com/ipld/go-car"
+)
+
+func CalcCommP(ctx context.Context, inpath string) (*types.CommPRet, error) {
+	// Hard-code the sector type to 32GiBV1_1, because:
+	// - ffiwrapper.GeneratePieceCIDFromFile requires a RegisteredSealProof
+	// - commP itself is sector-size independent, with rather low probability of that changing
+	//   ( note how the final rust call is identical for every RegSP type )
+	//   https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/v5.0.0/src/seal.rs#L1040-L1050
+	//
+	// IF/WHEN this changes in the future we will have to be able to calculate
+	// "old style" commP, and thus will need to introduce a version switch or similar
+	arbitraryProofType := abi.RegisteredSealProof_StackedDrg32GiBV1_1
+
+	rdr, err := os.Open(inpath)
+	if err != nil {
+		return nil, err
+	}
+	defer rdr.Close() //nolint:errcheck
+
+	stat, err := rdr.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	// check that the data is a car file; if it's not, retrieval won't work
+	_, err = car.ReadHeader(bufio.NewReader(rdr))
+	if err != nil {
+		return nil, fmt.Errorf("not a car file: %w", err)
+	}
+
+	if _, err := rdr.Seek(0, io.SeekStart); err != nil {
+		return nil, fmt.Errorf("seek to start: %w", err)
+	}
+
+	pieceReader, pieceSize := padreader.New(rdr, uint64(stat.Size()))
+	commP, err := ffiwrapper.GeneratePieceCIDFromFile(arbitraryProofType, pieceReader, pieceSize)
+	if err != nil {
+		return nil, fmt.Errorf("computing commP failed: %w", err)
+	}
+
+	return &types.CommPRet{
+		Root: commP,
+		Size: pieceSize,
+	}, nil
+}