diff --git a/core/corerepo/gc.go b/core/corerepo/gc.go index 87afc87439d..af678eca2d7 100644 --- a/core/corerepo/gc.go +++ b/core/corerepo/gc.go @@ -86,7 +86,7 @@ func GarbageCollect(n *core.IpfsNode, ctx context.Context) error { if err != nil { return err } - rmed := gc.GC(ctx, n.Blockstore, n.Pinning, roots) + rmed := gc.GC(ctx, n.Blockstore, n.Repo.Datastore(), n.Pinning, roots) return CollectResult(ctx, rmed, nil) } @@ -154,7 +154,7 @@ func GarbageCollectAsync(n *core.IpfsNode, ctx context.Context) <-chan gc.Result return out } - return gc.GC(ctx, n.Blockstore, n.Pinning, roots) + return gc.GC(ctx, n.Blockstore, n.Repo.Datastore(), n.Pinning, roots) } func PeriodicGC(ctx context.Context, node *core.IpfsNode) error { diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index b41dee22d96..5a12d3c15b8 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -104,7 +104,7 @@ func TestAddGCLive(t *testing.T) { gcstarted := make(chan struct{}) go func() { defer close(gcstarted) - gcout = gc.GC(context.Background(), node.Blockstore, node.Pinning, nil) + gcout = gc.GC(context.Background(), node.Blockstore, node.Repo.Datastore(), node.Pinning, nil) }() // gc shouldnt start until we let the add finish its current file. diff --git a/pin/gc/gc.go b/pin/gc/gc.go index 20ed77b4e18..92c8cb52a21 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -11,6 +11,7 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" pin "github.com/ipfs/go-ipfs/pin" + dstore "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" @@ -35,7 +36,7 @@ type Result struct { // The routine then iterates over every block in the blockstore and // deletes any block that is not found in the marked set. // -func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner, bestEffortRoots []*cid.Cid) <-chan Result { +func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn pin.Pinner, bestEffortRoots []*cid.Cid) <-chan Result { elock := log.EventBegin(ctx, "GC.lockWait") unlocker := bs.GCLock() @@ -107,6 +108,18 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner, bestEffortRo if errors { output <- Result{Error: ErrCannotDeleteSomeBlocks} } + + defer log.EventBegin(ctx, "GC.datastore").Done() + gds, ok := dstor.(dstore.GCDatastore) + if !ok { + return + } + + err = gds.CollectGarbage() + if err != nil { + output <- Result{Error: err} + return + } }() return output diff --git a/repo/fsrepo/datastores.go b/repo/fsrepo/datastores.go index 193801027c2..27254426ed1 100644 --- a/repo/fsrepo/datastores.go +++ b/repo/fsrepo/datastores.go @@ -16,6 +16,7 @@ import ( ds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore" mount "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore/syncmount" + humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt" badgerds "gx/ipfs/Qmbjb3c2KRPVNZWSvQED8zAf12Brdbp3ksSnGdsJiytqUs/go-ds-badger" levelds "gx/ipfs/Qmbkc8BMfEixGCeKRuGGbf34mAjTb9xPmJ8Pm5gHU7ohZ4/go-ds-leveldb" @@ -342,6 +343,8 @@ func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) { type badgerdsDatastoreConfig struct { path string syncWrites bool + + vlogFileSize int64 } // BadgerdsDatastoreConfig returns a configuration stub for a badger datastore @@ -366,6 +369,22 @@ func BadgerdsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, er } } + vls, ok := params["vlogFileSize"] + if !ok { + // default to 1GiB + c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize + } else { + if vlogSize, ok := vls.(string); ok { + s, err := humanize.ParseBytes(vlogSize) + if err != nil { + return nil, err + } + c.vlogFileSize = int64(s) + } else { + return nil, fmt.Errorf("'vlogFileSize' field was not a string") + } + } + return &c, nil } @@ -389,6 +408,7 @@ func (c *badgerdsDatastoreConfig) Create(path string) (repo.Datastore, error) { defopts := badgerds.DefaultOptions defopts.SyncWrites = c.syncWrites + defopts.ValueLogFileSize = c.vlogFileSize return badgerds.NewDatastore(p, &defopts) }