From d063ef24f4a8c4735adbf5c332d0cdfb12c3a8a3 Mon Sep 17 00:00:00 2001 From: sunwp <244372610@qq.com> Date: Thu, 2 Dec 2021 19:47:22 +0800 Subject: [PATCH] unify piece size (#854) * unify piece size Signed-off-by: sunwp <244372610@qq.com> --- cdn/config/constants.go | 8 --- cdn/supervisor/task/manager_util.go | 4 +- client/daemon/peer/peertask_file_test.go | 6 +- client/daemon/peer/peertask_manager_test.go | 4 +- client/daemon/peer/peertask_stream_test.go | 6 +- client/daemon/peer/piece_manager.go | 4 +- .../dfutils/dfutils.go | 20 +++--- internal/dfutils/dfutils_test.go | 71 +++++++++++++++++++ 8 files changed, 95 insertions(+), 28 deletions(-) rename cdn/cdnutil/cdn_util.go => internal/dfutils/dfutils.go (70%) create mode 100644 internal/dfutils/dfutils_test.go diff --git a/cdn/config/constants.go b/cdn/config/constants.go index 0106c77cafc..3829fb2b00a 100644 --- a/cdn/config/constants.go +++ b/cdn/config/constants.go @@ -37,14 +37,6 @@ const ( DefaultStorageMode = "disk" ) -const ( - // DefaultPieceSize 4M - DefaultPieceSize = 4 * 1024 * 1024 - - // DefaultPieceSizeLimit 15M - DefaultPieceSizeLimit = 15 * 1024 * 1024 -) - const ( // DefaultFailAccessInterval is the interval time after failed to access the URL. DefaultFailAccessInterval = 3 * time.Minute diff --git a/cdn/supervisor/task/manager_util.go b/cdn/supervisor/task/manager_util.go index 5458c1c236c..f8bdf86e9bc 100644 --- a/cdn/supervisor/task/manager_util.go +++ b/cdn/supervisor/task/manager_util.go @@ -25,11 +25,11 @@ import ( "github.com/pkg/errors" "go.opentelemetry.io/otel/trace" - "d7y.io/dragonfly/v2/cdn/cdnutil" "d7y.io/dragonfly/v2/cdn/config" cdnerrors "d7y.io/dragonfly/v2/cdn/errors" "d7y.io/dragonfly/v2/cdn/types" logger "d7y.io/dragonfly/v2/internal/dflog" + "d7y.io/dragonfly/v2/internal/dfutils" "d7y.io/dragonfly/v2/pkg/source" "d7y.io/dragonfly/v2/pkg/synclock" "d7y.io/dragonfly/v2/pkg/util/net/urlutils" @@ -118,7 +118,7 @@ func (tm *Manager) addOrUpdateTask(ctx context.Context, request *types.TaskRegis // calculate piece size and update the PieceSize and PieceTotal if task.PieceSize <= 0 { - pieceSize := cdnutil.ComputePieceSize(task.SourceFileLength) + pieceSize := dfutils.ComputePieceSize(task.SourceFileLength) task.PieceSize = int32(pieceSize) } if err := tm.taskStore.Add(task.TaskID, task); err != nil { diff --git a/client/daemon/peer/peertask_file_test.go b/client/daemon/peer/peertask_file_test.go index e748e8ab617..893a6e0739b 100644 --- a/client/daemon/peer/peertask_file_test.go +++ b/client/daemon/peer/peertask_file_test.go @@ -29,10 +29,10 @@ import ( "github.com/golang/mock/gomock" testifyassert "github.com/stretchr/testify/assert" - "d7y.io/dragonfly/v2/cdn/cdnutil" "d7y.io/dragonfly/v2/client/clientutil" "d7y.io/dragonfly/v2/client/config" "d7y.io/dragonfly/v2/client/daemon/test" + "d7y.io/dragonfly/v2/internal/dfutils" "d7y.io/dragonfly/v2/pkg/rpc/base" "d7y.io/dragonfly/v2/pkg/rpc/scheduler" "d7y.io/dragonfly/v2/pkg/source" @@ -101,7 +101,7 @@ func TestFilePeerTask_BackSource_WithContentLength(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: downloader, - computePieceSize: cdnutil.ComputePieceSize, + computePieceSize: dfutils.ComputePieceSize, }, storageManager: storageManager, schedulerClient: schedulerClient, @@ -219,7 +219,7 @@ func TestFilePeerTask_BackSource_WithoutContentLength(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: downloader, - computePieceSize: cdnutil.ComputePieceSize, + computePieceSize: dfutils.ComputePieceSize, }, storageManager: storageManager, schedulerClient: schedulerClient, diff --git a/client/daemon/peer/peertask_manager_test.go b/client/daemon/peer/peertask_manager_test.go index ca18b53c9d1..1593a64eee3 100644 --- a/client/daemon/peer/peertask_manager_test.go +++ b/client/daemon/peer/peertask_manager_test.go @@ -35,13 +35,13 @@ import ( testifyassert "github.com/stretchr/testify/assert" "google.golang.org/grpc" - "d7y.io/dragonfly/v2/cdn/cdnutil" "d7y.io/dragonfly/v2/client/clientutil" "d7y.io/dragonfly/v2/client/config" "d7y.io/dragonfly/v2/client/daemon/storage" "d7y.io/dragonfly/v2/client/daemon/test" mock_daemon "d7y.io/dragonfly/v2/client/daemon/test/mock/daemon" mock_scheduler "d7y.io/dragonfly/v2/client/daemon/test/mock/scheduler" + "d7y.io/dragonfly/v2/internal/dfutils" "d7y.io/dragonfly/v2/pkg/basic/dfnet" "d7y.io/dragonfly/v2/pkg/rpc" "d7y.io/dragonfly/v2/pkg/rpc/base" @@ -367,7 +367,7 @@ func TestPeerTaskManager_StartStreamPeerTask_BackSource(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: NewMockPieceDownloader(ctrl), - computePieceSize: cdnutil.ComputePieceSize, + computePieceSize: dfutils.ComputePieceSize, }, storageManager: storageManager, schedulerClient: sched, diff --git a/client/daemon/peer/peertask_stream_test.go b/client/daemon/peer/peertask_stream_test.go index f8b98a22b16..6b50c84feca 100644 --- a/client/daemon/peer/peertask_stream_test.go +++ b/client/daemon/peer/peertask_stream_test.go @@ -28,10 +28,10 @@ import ( "github.com/golang/mock/gomock" testifyassert "github.com/stretchr/testify/assert" - "d7y.io/dragonfly/v2/cdn/cdnutil" "d7y.io/dragonfly/v2/client/clientutil" "d7y.io/dragonfly/v2/client/config" "d7y.io/dragonfly/v2/client/daemon/test" + "d7y.io/dragonfly/v2/internal/dfutils" "d7y.io/dragonfly/v2/pkg/rpc/base" "d7y.io/dragonfly/v2/pkg/rpc/scheduler" "d7y.io/dragonfly/v2/pkg/source" @@ -98,7 +98,7 @@ func TestStreamPeerTask_BackSource_WithContentLength(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: downloader, - computePieceSize: cdnutil.ComputePieceSize, + computePieceSize: dfutils.ComputePieceSize, }, storageManager: storageManager, schedulerClient: schedulerClient, @@ -204,7 +204,7 @@ func TestStreamPeerTask_BackSource_WithoutContentLength(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: downloader, - computePieceSize: cdnutil.ComputePieceSize, + computePieceSize: dfutils.ComputePieceSize, }, storageManager: storageManager, schedulerClient: schedulerClient, diff --git a/client/daemon/peer/piece_manager.go b/client/daemon/peer/piece_manager.go index fa42b0e4276..7ccfd1e90e5 100644 --- a/client/daemon/peer/piece_manager.go +++ b/client/daemon/peer/piece_manager.go @@ -24,11 +24,11 @@ import ( "golang.org/x/time/rate" - "d7y.io/dragonfly/v2/cdn/cdnutil" "d7y.io/dragonfly/v2/client/clientutil" "d7y.io/dragonfly/v2/client/config" "d7y.io/dragonfly/v2/client/daemon/storage" logger "d7y.io/dragonfly/v2/internal/dflog" + "d7y.io/dragonfly/v2/internal/dfutils" "d7y.io/dragonfly/v2/pkg/rpc/base" "d7y.io/dragonfly/v2/pkg/rpc/scheduler" "d7y.io/dragonfly/v2/pkg/source" @@ -55,7 +55,7 @@ var _ PieceManager = (*pieceManager)(nil) func NewPieceManager(s storage.TaskStorageDriver, pieceDownloadTimeout time.Duration, opts ...func(*pieceManager)) (PieceManager, error) { pm := &pieceManager{ storageManager: s, - computePieceSize: cdnutil.ComputePieceSize, + computePieceSize: dfutils.ComputePieceSize, calculateDigest: true, } for _, opt := range opts { diff --git a/cdn/cdnutil/cdn_util.go b/internal/dfutils/dfutils.go similarity index 70% rename from cdn/cdnutil/cdn_util.go rename to internal/dfutils/dfutils.go index a2a628f07a3..236c773edac 100644 --- a/cdn/cdnutil/cdn_util.go +++ b/internal/dfutils/dfutils.go @@ -14,25 +14,29 @@ * limitations under the License. */ -package cdnutil +package dfutils -import ( - "d7y.io/dragonfly/v2/cdn/config" +const ( + // DefaultPieceSize 4M + DefaultPieceSize = 4 * 1024 * 1024 + + // DefaultPieceSizeLimit 15M + DefaultPieceSizeLimit = 15 * 1024 * 1024 ) // ComputePieceSize computes the piece size with specified fileLength. // -// If the fileLength<=0, which means failed to get fileLength +// If the fileLength<0, which means failed to get fileLength // and then use the DefaultPieceSize. func ComputePieceSize(length int64) uint32 { if length <= 0 || length <= 200*1024*1024 { - return config.DefaultPieceSize + return DefaultPieceSize } gapCount := length / int64(100*1024*1024) - mpSize := (gapCount-2)*1024*1024 + config.DefaultPieceSize - if mpSize > config.DefaultPieceSizeLimit { - return config.DefaultPieceSizeLimit + mpSize := (gapCount-2)*1024*1024 + DefaultPieceSize + if mpSize > DefaultPieceSizeLimit { + return DefaultPieceSizeLimit } return uint32(mpSize) } diff --git a/internal/dfutils/dfutils_test.go b/internal/dfutils/dfutils_test.go new file mode 100644 index 00000000000..ca2401b8ee5 --- /dev/null +++ b/internal/dfutils/dfutils_test.go @@ -0,0 +1,71 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dfutils + +import ( + "testing" +) + +func TestComputePieceSize(t *testing.T) { + type args struct { + length int64 + } + tests := []struct { + name string + args args + want uint32 + }{ + { + name: "length equal 200M and get default piece size", + args: args{ + length: 200 * 1024 * 1024, + }, + want: DefaultPieceSize, + }, { + name: "length smaller than 200M and get default piece size", + args: args{ + length: 100 * 1024 * 1024, + }, + want: DefaultPieceSize, + }, { + name: "length greater than 200M", + args: args{ + length: 205 * 1024 * 1024, + }, + want: DefaultPieceSize, + }, { + name: "length greater than 300M", + args: args{ + length: 310 * 1024 * 1024, + }, + want: DefaultPieceSize + 1*1024*1024, + }, { + name: "length reach piece size limit", + args: args{ + length: 3100 * 1024 * 1024, + }, + want: DefaultPieceSizeLimit, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ComputePieceSize(tt.args.length); got != tt.want { + t.Errorf("ComputePieceSize() = %v, want %v", got, tt.want) + } + }) + } +}