chore: optimize compute piece size function (#528)

Signed-off-by: Jim Ma <[email protected]>
dragonflyoss · Aug 11, 2021 · e464523 · e464523
1 parent e04408f
commit e464523
Show file tree

Hide file tree

Showing 8 changed files with 37 additions and 49 deletions.
diff --git a/cdnsystem/cdnutil/cdn_util.go b/cdnsystem/cdnutil/cdn_util.go
@@ -19,9 +19,27 @@ package cdnutil
 import (
 	"fmt"
 
+	"d7y.io/dragonfly/v2/cdnsystem/config"
 	"d7y.io/dragonfly/v2/pkg/util/net/iputils"
 )
 
 func GenCDNPeerID(taskID string) string {
 	return fmt.Sprintf("%s-%s_%s", iputils.HostName, taskID, "CDN")
 }
+
+// ComputePieceSize computes the piece size with specified fileLength.
+//
+// If the fileLength<=0, which means failed to get fileLength
+// and then use the DefaultPieceSize.
+func ComputePieceSize(length int64) int32 {
+	if length <= 0 || length <= 200*1024*1024 {
+		return config.DefaultPieceSize
+	}
+
+	gapCount := length / int64(100*1024*1024)
+	mpSize := (gapCount-2)*1024*1024 + config.DefaultPieceSize
+	if mpSize > config.DefaultPieceSizeLimit {
+		return config.DefaultPieceSizeLimit
+	}
+	return int32(mpSize)
+}
diff --git a/cdnsystem/daemon/task/manager_util.go b/cdnsystem/daemon/task/manager_util.go
@@ -22,15 +22,16 @@ import (
 	"reflect"
 	"time"
 
-	"d7y.io/dragonfly/v2/cdnsystem/config"
+	"github.com/pkg/errors"
+
+	"d7y.io/dragonfly/v2/cdnsystem/cdnutil"
 	cdnerrors "d7y.io/dragonfly/v2/cdnsystem/errors"
 	"d7y.io/dragonfly/v2/cdnsystem/types"
 	logger "d7y.io/dragonfly/v2/internal/dflog"
 	"d7y.io/dragonfly/v2/pkg/source"
 	"d7y.io/dragonfly/v2/pkg/synclock"
 	"d7y.io/dragonfly/v2/pkg/util/net/urlutils"
 	"d7y.io/dragonfly/v2/pkg/util/stringutils"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -108,7 +109,7 @@ func (tm *Manager) addOrUpdateTask(ctx context.Context, request *types.TaskRegis
 
 	// calculate piece size and update the PieceSize and PieceTotal
 	if task.PieceSize <= 0 {
-		pieceSize := computePieceSize(task.SourceFileLength)
+		pieceSize := cdnutil.ComputePieceSize(task.SourceFileLength)
 		task.PieceSize = pieceSize
 	}
 	tm.taskStore.Add(task.TaskID, task)
@@ -193,20 +194,3 @@ func isSameTask(task1, task2 *types.SeedTask) bool {
 
 	return true
 }
-
-// computePieceSize computes the piece size with specified fileLength.
-//
-// If the fileLength<=0, which means failed to get fileLength
-// and then use the DefaultPieceSize.
-func computePieceSize(length int64) int32 {
-	if length <= 0 || length <= 200*1024*1024 {
-		return config.DefaultPieceSize
-	}
-
-	gapCount := length / int64(100*1024*1024)
-	mpSize := (gapCount-2)*1024*1024 + config.DefaultPieceSize
-	if mpSize > config.DefaultPieceSizeLimit {
-		return config.DefaultPieceSizeLimit
-	}
-	return int32(mpSize)
-}
diff --git a/client/daemon/peer/peertask_file_test.go b/client/daemon/peer/peertask_file_test.go
@@ -26,17 +26,19 @@ import (
 	"testing"
 	"time"
 
+	"d7y.io/dragonfly/v2/cdnsystem/cdnutil"
 	"d7y.io/dragonfly/v2/pkg/rpc/base"
 	rangers "d7y.io/dragonfly/v2/pkg/util/rangeutils"
 
+	"github.com/golang/mock/gomock"
+	testifyassert "github.com/stretchr/testify/assert"
+
 	"d7y.io/dragonfly/v2/client/clientutil"
 	"d7y.io/dragonfly/v2/client/config"
 	"d7y.io/dragonfly/v2/client/daemon/test"
 	"d7y.io/dragonfly/v2/pkg/rpc/scheduler"
 	"d7y.io/dragonfly/v2/pkg/source"
 	sourceMock "d7y.io/dragonfly/v2/pkg/source/mock"
-	"github.com/golang/mock/gomock"
-	testifyassert "github.com/stretchr/testify/assert"
 )
 
 func TestFilePeerTask_BackSource_WithContentLength(t *testing.T) {
@@ -100,7 +102,7 @@ func TestFilePeerTask_BackSource_WithContentLength(t *testing.T) {
 		pieceManager: &pieceManager{
 			storageManager:   storageManager,
 			pieceDownloader:  downloader,
-			computePieceSize: computePieceSize,
+			computePieceSize: cdnutil.ComputePieceSize,
 		},
 		storageManager:  storageManager,
 		schedulerClient: schedulerClient,
@@ -219,7 +221,7 @@ func TestFilePeerTask_BackSource_WithoutContentLength(t *testing.T) {
 		pieceManager: &pieceManager{
 			storageManager:   storageManager,
 			pieceDownloader:  downloader,
-			computePieceSize: computePieceSize,
+			computePieceSize: cdnutil.ComputePieceSize,
 		},
 		storageManager:  storageManager,
 		schedulerClient: schedulerClient,

diff --git a/client/daemon/peer/peertask_manager_test.go b/client/daemon/peer/peertask_manager_test.go
@@ -35,6 +35,7 @@ import (
 	testifyassert "github.com/stretchr/testify/assert"
 	"google.golang.org/grpc"
 
+	"d7y.io/dragonfly/v2/cdnsystem/cdnutil"
 	"d7y.io/dragonfly/v2/client/clientutil"
 	"d7y.io/dragonfly/v2/client/config"
 	"d7y.io/dragonfly/v2/client/daemon/storage"
@@ -362,7 +363,7 @@ func TestPeerTaskManager_StartStreamPeerTask_BackSource(t *testing.T) {
 		pieceManager: &pieceManager{
 			storageManager:   storageManager,
 			pieceDownloader:  NewMockPieceDownloader(ctrl),
-			computePieceSize: computePieceSize,
+			computePieceSize: cdnutil.ComputePieceSize,
 		},
 		storageManager:  storageManager,
 		schedulerClient: sched,

diff --git a/client/daemon/peer/peertask_stream_test.go b/client/daemon/peer/peertask_stream_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/golang/mock/gomock"
 	testifyassert "github.com/stretchr/testify/assert"
 
+	"d7y.io/dragonfly/v2/cdnsystem/cdnutil"
 	"d7y.io/dragonfly/v2/client/clientutil"
 	"d7y.io/dragonfly/v2/client/config"
 	"d7y.io/dragonfly/v2/client/daemon/test"
@@ -97,7 +98,7 @@ func TestStreamPeerTask_BackSource_WithContentLength(t *testing.T) {
 		pieceManager: &pieceManager{
 			storageManager:   storageManager,
 			pieceDownloader:  downloader,
-			computePieceSize: computePieceSize,
+			computePieceSize: cdnutil.ComputePieceSize,
 		},
 		storageManager:  storageManager,
 		schedulerClient: schedulerClient,
@@ -204,7 +205,7 @@ func TestStreamPeerTask_BackSource_WithoutContentLength(t *testing.T) {
 		pieceManager: &pieceManager{
 			storageManager:   storageManager,
 			pieceDownloader:  downloader,
-			computePieceSize: computePieceSize,
+			computePieceSize: cdnutil.ComputePieceSize,
 		},
 		storageManager:  storageManager,
 		schedulerClient: schedulerClient,

diff --git a/client/daemon/peer/piece_manager.go b/client/daemon/peer/piece_manager.go
@@ -24,7 +24,7 @@ import (
 
 	"golang.org/x/time/rate"
 
-	cdnconfig "d7y.io/dragonfly/v2/cdnsystem/config"
+	"d7y.io/dragonfly/v2/cdnsystem/cdnutil"
 	"d7y.io/dragonfly/v2/client/clientutil"
 	"d7y.io/dragonfly/v2/client/config"
 	"d7y.io/dragonfly/v2/client/daemon/storage"
@@ -56,7 +56,7 @@ var _ PieceManager = (*pieceManager)(nil)
 func NewPieceManager(s storage.TaskStorageDriver, opts ...func(*pieceManager)) (PieceManager, error) {
 	pm := &pieceManager{
 		storageManager:   s,
-		computePieceSize: computePieceSize,
+		computePieceSize: cdnutil.ComputePieceSize,
 		calculateDigest:  true,
 	}
 	for _, opt := range opts {
@@ -386,21 +386,3 @@ func (pm *pieceManager) DownloadSource(ctx context.Context, pt Task, request *sc
 	log.Infof("download from source ok")
 	return nil
 }
-
-// TODO copy from cdnsystem/daemon/mgr/task/manager_util.go
-// computePieceSize computes the piece size with specified fileLength.
-//
-// If the fileLength<=0, which means failed to get fileLength
-// and then use the DefaultPieceSize.
-func computePieceSize(length int64) int32 {
-	if length <= 0 || length <= 200*1024*1024 {
-		return cdnconfig.DefaultPieceSize
-	}
-
-	gapCount := length / int64(100*1024*1024)
-	mpSize := (gapCount-2)*1024*1024 + cdnconfig.DefaultPieceSize
-	if mpSize > cdnconfig.DefaultPieceSizeLimit {
-		return cdnconfig.DefaultPieceSizeLimit
-	}
-	return int32(mpSize)
-}
diff --git a/test/stress/README.md → test/tools/stress/README.md b/test/stress/README.md → test/tools/stress/README.md
diff --git a/test/stress/main.go → test/tools/stress/main.go b/test/stress/main.go → test/tools/stress/main.go
@@ -33,13 +33,13 @@ import (
 	"syscall"
 	"time"
 
-	"d7y.io/dragonfly/v2/pkg/unit"
-	"d7y.io/dragonfly/v2/pkg/util/net/iputils"
 	"github.com/go-echarts/statsview"
 	"github.com/go-echarts/statsview/viewer"
 	"github.com/montanaflynn/stats"
 
 	"d7y.io/dragonfly/v2/client/config"
+	"d7y.io/dragonfly/v2/pkg/unit"
+	"d7y.io/dragonfly/v2/pkg/util/net/iputils"
 )
 
 var (