Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

lib: extract unixfs filestore into lib #8354

Merged
merged 1 commit into from
Mar 21, 2022
+21 −23
Merged

lib: extract unixfs filestore into lib #8354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions node/impl/client/import.go → lib/unixfs/filestore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package client
package unixfs

import (
"context"
Expand All @@ -19,12 +19,15 @@ import (
"github.com/ipfs/go-merkledag"
"github.com/ipfs/go-unixfs/importer/balanced"
ihelper "github.com/ipfs/go-unixfs/importer/helpers"
mh "github.com/multiformats/go-multihash"
"golang.org/x/xerrors"

"github.com/filecoin-project/lotus/build"
)

func unixFSCidBuilder() (cid.Builder, error) {
var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31)

func CidBuilder() (cid.Builder, error) {
prefix, err := merkledag.PrefixForCidVersion(1)
if err != nil {
return nil, fmt.Errorf("failed to initialize UnixFS CID Builder: %w", err)
Expand All @@ -37,9 +40,9 @@ func unixFSCidBuilder() (cid.Builder, error) {
return b, nil
}

// createUnixFSFilestore takes a standard file whose path is src, forms a UnixFS DAG, and
// CreateFilestore takes a standard file whose path is src, forms a UnixFS DAG, and
// writes a CARv2 file with positional mapping (backed by the go-filestore library).
func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath string) (cid.Cid, error) {
func CreateFilestore(ctx context.Context, srcPath string, dstPath string) (cid.Cid, error) {
// This method uses a two-phase approach with a staging CAR blockstore and
// a final CAR blockstore.
//
Expand Down Expand Up @@ -80,7 +83,7 @@ func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath
return cid.Undef, xerrors.Errorf("failed to create temporary filestore: %w", err)
}

finalRoot1, err := buildUnixFS(ctx, file, fstore, true)
finalRoot1, err := Build(ctx, file, fstore, true)
if err != nil {
_ = fstore.Close()
return cid.Undef, xerrors.Errorf("failed to import file to store to compute root: %w", err)
Expand All @@ -102,7 +105,7 @@ func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath
return cid.Undef, xerrors.Errorf("failed to rewind file: %w", err)
}

finalRoot2, err := buildUnixFS(ctx, file, bs, true)
finalRoot2, err := Build(ctx, file, bs, true)
if err != nil {
_ = bs.Close()
return cid.Undef, xerrors.Errorf("failed to create UnixFS DAG with carv2 blockstore: %w", err)
Expand All @@ -119,10 +122,10 @@ func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath
return finalRoot1, nil
}

// buildUnixFS builds a UnixFS DAG out of the supplied reader,
// Build builds a UnixFS DAG out of the supplied reader,
// and imports the DAG into the supplied service.
func buildUnixFS(ctx context.Context, reader io.Reader, into bstore.Blockstore, filestore bool) (cid.Cid, error) {
b, err := unixFSCidBuilder()
func Build(ctx context.Context, reader io.Reader, into bstore.Blockstore, filestore bool) (cid.Cid, error) {
b, err := CidBuilder()
if err != nil {
return cid.Undef, err
}
Expand Down
11 changes: 3 additions & 8 deletions node/impl/client/import_test.go → lib/unixfs/filestore_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//stm: #unit
package client
package unixfs

import (
"bytes"
Expand All @@ -21,8 +21,6 @@ import (
"github.com/stretchr/testify/require"

"github.com/filecoin-project/go-fil-markets/stores"

"github.com/filecoin-project/lotus/node/repo/imports"
)

// This test uses a full "dense" CARv2, and not a filestore (positional mapping).
Expand All @@ -42,7 +40,7 @@ func TestRoundtripUnixFS_Dense(t *testing.T) {
blockstore.UseWholeCIDs(true))
require.NoError(t, err)

root, err := buildUnixFS(ctx, bytes.NewBuffer(inputContents), bs, false)
root, err := Build(ctx, bytes.NewBuffer(inputContents), bs, false)
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)
require.NoError(t, bs.Finalize())
Expand Down Expand Up @@ -78,17 +76,14 @@ func TestRoundtripUnixFS_Dense(t *testing.T) {
func TestRoundtripUnixFS_Filestore(t *testing.T) {
//stm: @CLIENT_DATA_IMPORT_001
ctx := context.Background()
a := &API{
Imports: &imports.Manager{},
}

inputPath, inputContents := genInputFile(t)
defer os.Remove(inputPath) //nolint:errcheck

dst := newTmpFile(t)
defer os.Remove(dst) //nolint:errcheck

root, err := a.createUnixFSFilestore(ctx, inputPath, dst)
root, err := CreateFilestore(ctx, inputPath, dst)
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)

Expand Down
12 changes: 6 additions & 6 deletions node/impl/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ import (
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/multiformats/go-multibase"
mh "github.com/multiformats/go-multihash"
"go.uber.org/fx"

"github.com/filecoin-project/go-address"
Expand All @@ -56,6 +55,7 @@ import (
"github.com/filecoin-project/go-fil-markets/storagemarket/network"
"github.com/filecoin-project/go-fil-markets/stores"

"github.com/filecoin-project/lotus/lib/unixfs"
"github.com/filecoin-project/lotus/markets/retrievaladapter"
"github.com/filecoin-project/lotus/markets/storageadapter"

Expand All @@ -79,7 +79,7 @@ import (

var log = logging.Logger("client")

var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31)
var DefaultHashFunction = unixfs.DefaultHashFunction

// 8 days ~= SealDuration + PreCommit + MaxProveCommitDuration + 8 hour buffer
const dealStartBufferHours uint64 = 8 * 24
Expand Down Expand Up @@ -548,7 +548,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
}()

// perform the unixfs chunking.
root, err = a.createUnixFSFilestore(ctx, ref.Path, carPath)
root, err = unixfs.CreateFilestore(ctx, ref.Path, carPath)
if err != nil {
return nil, xerrors.Errorf("failed to import file using unixfs: %w", err)
}
Expand Down Expand Up @@ -618,7 +618,7 @@ func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, erro
// once the DAG is formed and the root is calculated, we overwrite the
// inner carv1 header with the final root.

b, err := unixFSCidBuilder()
b, err := unixfs.CidBuilder()
if err != nil {
return cid.Undef, err
}
Expand All @@ -635,7 +635,7 @@ func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, erro
return cid.Undef, xerrors.Errorf("failed to create carv2 read/write blockstore: %w", err)
}

root, err := buildUnixFS(ctx, file, bs, false)
root, err := unixfs.Build(ctx, file, bs, false)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to build unixfs dag: %w", err)
}
Expand Down Expand Up @@ -1364,7 +1364,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri
defer os.Remove(tmp) //nolint:errcheck

// generate and import the UnixFS DAG into a filestore (positional reference) CAR.
root, err := a.createUnixFSFilestore(ctx, ref.Path, tmp)
root, err := unixfs.CreateFilestore(ctx, ref.Path, tmp)
if err != nil {
return xerrors.Errorf("failed to import file using unixfs: %w", err)
}
Expand Down