Skip to content

Commit

Permalink
unify piece size (#854)
Browse files Browse the repository at this point in the history
* unify piece size

Signed-off-by: sunwp <[email protected]>
  • Loading branch information
244372610 authored Dec 2, 2021
1 parent 046afdc commit d063ef2
Show file tree
Hide file tree
Showing 8 changed files with 95 additions and 28 deletions.
8 changes: 0 additions & 8 deletions cdn/config/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,6 @@ const (
DefaultStorageMode = "disk"
)

const (
// DefaultPieceSize 4M
DefaultPieceSize = 4 * 1024 * 1024

// DefaultPieceSizeLimit 15M
DefaultPieceSizeLimit = 15 * 1024 * 1024
)

const (
// DefaultFailAccessInterval is the interval time after failed to access the URL.
DefaultFailAccessInterval = 3 * time.Minute
Expand Down
4 changes: 2 additions & 2 deletions cdn/supervisor/task/manager_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ import (
"github.com/pkg/errors"
"go.opentelemetry.io/otel/trace"

"d7y.io/dragonfly/v2/cdn/cdnutil"
"d7y.io/dragonfly/v2/cdn/config"
cdnerrors "d7y.io/dragonfly/v2/cdn/errors"
"d7y.io/dragonfly/v2/cdn/types"
logger "d7y.io/dragonfly/v2/internal/dflog"
"d7y.io/dragonfly/v2/internal/dfutils"
"d7y.io/dragonfly/v2/pkg/source"
"d7y.io/dragonfly/v2/pkg/synclock"
"d7y.io/dragonfly/v2/pkg/util/net/urlutils"
Expand Down Expand Up @@ -118,7 +118,7 @@ func (tm *Manager) addOrUpdateTask(ctx context.Context, request *types.TaskRegis

// calculate piece size and update the PieceSize and PieceTotal
if task.PieceSize <= 0 {
pieceSize := cdnutil.ComputePieceSize(task.SourceFileLength)
pieceSize := dfutils.ComputePieceSize(task.SourceFileLength)
task.PieceSize = int32(pieceSize)
}
if err := tm.taskStore.Add(task.TaskID, task); err != nil {
Expand Down
6 changes: 3 additions & 3 deletions client/daemon/peer/peertask_file_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ import (
"github.com/golang/mock/gomock"
testifyassert "github.com/stretchr/testify/assert"

"d7y.io/dragonfly/v2/cdn/cdnutil"
"d7y.io/dragonfly/v2/client/clientutil"
"d7y.io/dragonfly/v2/client/config"
"d7y.io/dragonfly/v2/client/daemon/test"
"d7y.io/dragonfly/v2/internal/dfutils"
"d7y.io/dragonfly/v2/pkg/rpc/base"
"d7y.io/dragonfly/v2/pkg/rpc/scheduler"
"d7y.io/dragonfly/v2/pkg/source"
Expand Down Expand Up @@ -101,7 +101,7 @@ func TestFilePeerTask_BackSource_WithContentLength(t *testing.T) {
pieceManager: &pieceManager{
storageManager: storageManager,
pieceDownloader: downloader,
computePieceSize: cdnutil.ComputePieceSize,
computePieceSize: dfutils.ComputePieceSize,
},
storageManager: storageManager,
schedulerClient: schedulerClient,
Expand Down Expand Up @@ -219,7 +219,7 @@ func TestFilePeerTask_BackSource_WithoutContentLength(t *testing.T) {
pieceManager: &pieceManager{
storageManager: storageManager,
pieceDownloader: downloader,
computePieceSize: cdnutil.ComputePieceSize,
computePieceSize: dfutils.ComputePieceSize,
},
storageManager: storageManager,
schedulerClient: schedulerClient,
Expand Down
4 changes: 2 additions & 2 deletions client/daemon/peer/peertask_manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,13 @@ import (
testifyassert "github.com/stretchr/testify/assert"
"google.golang.org/grpc"

"d7y.io/dragonfly/v2/cdn/cdnutil"
"d7y.io/dragonfly/v2/client/clientutil"
"d7y.io/dragonfly/v2/client/config"
"d7y.io/dragonfly/v2/client/daemon/storage"
"d7y.io/dragonfly/v2/client/daemon/test"
mock_daemon "d7y.io/dragonfly/v2/client/daemon/test/mock/daemon"
mock_scheduler "d7y.io/dragonfly/v2/client/daemon/test/mock/scheduler"
"d7y.io/dragonfly/v2/internal/dfutils"
"d7y.io/dragonfly/v2/pkg/basic/dfnet"
"d7y.io/dragonfly/v2/pkg/rpc"
"d7y.io/dragonfly/v2/pkg/rpc/base"
Expand Down Expand Up @@ -367,7 +367,7 @@ func TestPeerTaskManager_StartStreamPeerTask_BackSource(t *testing.T) {
pieceManager: &pieceManager{
storageManager: storageManager,
pieceDownloader: NewMockPieceDownloader(ctrl),
computePieceSize: cdnutil.ComputePieceSize,
computePieceSize: dfutils.ComputePieceSize,
},
storageManager: storageManager,
schedulerClient: sched,
Expand Down
6 changes: 3 additions & 3 deletions client/daemon/peer/peertask_stream_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ import (
"github.com/golang/mock/gomock"
testifyassert "github.com/stretchr/testify/assert"

"d7y.io/dragonfly/v2/cdn/cdnutil"
"d7y.io/dragonfly/v2/client/clientutil"
"d7y.io/dragonfly/v2/client/config"
"d7y.io/dragonfly/v2/client/daemon/test"
"d7y.io/dragonfly/v2/internal/dfutils"
"d7y.io/dragonfly/v2/pkg/rpc/base"
"d7y.io/dragonfly/v2/pkg/rpc/scheduler"
"d7y.io/dragonfly/v2/pkg/source"
Expand Down Expand Up @@ -98,7 +98,7 @@ func TestStreamPeerTask_BackSource_WithContentLength(t *testing.T) {
pieceManager: &pieceManager{
storageManager: storageManager,
pieceDownloader: downloader,
computePieceSize: cdnutil.ComputePieceSize,
computePieceSize: dfutils.ComputePieceSize,
},
storageManager: storageManager,
schedulerClient: schedulerClient,
Expand Down Expand Up @@ -204,7 +204,7 @@ func TestStreamPeerTask_BackSource_WithoutContentLength(t *testing.T) {
pieceManager: &pieceManager{
storageManager: storageManager,
pieceDownloader: downloader,
computePieceSize: cdnutil.ComputePieceSize,
computePieceSize: dfutils.ComputePieceSize,
},
storageManager: storageManager,
schedulerClient: schedulerClient,
Expand Down
4 changes: 2 additions & 2 deletions client/daemon/peer/piece_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ import (

"golang.org/x/time/rate"

"d7y.io/dragonfly/v2/cdn/cdnutil"
"d7y.io/dragonfly/v2/client/clientutil"
"d7y.io/dragonfly/v2/client/config"
"d7y.io/dragonfly/v2/client/daemon/storage"
logger "d7y.io/dragonfly/v2/internal/dflog"
"d7y.io/dragonfly/v2/internal/dfutils"
"d7y.io/dragonfly/v2/pkg/rpc/base"
"d7y.io/dragonfly/v2/pkg/rpc/scheduler"
"d7y.io/dragonfly/v2/pkg/source"
Expand All @@ -55,7 +55,7 @@ var _ PieceManager = (*pieceManager)(nil)
func NewPieceManager(s storage.TaskStorageDriver, pieceDownloadTimeout time.Duration, opts ...func(*pieceManager)) (PieceManager, error) {
pm := &pieceManager{
storageManager: s,
computePieceSize: cdnutil.ComputePieceSize,
computePieceSize: dfutils.ComputePieceSize,
calculateDigest: true,
}
for _, opt := range opts {
Expand Down
20 changes: 12 additions & 8 deletions cdn/cdnutil/cdn_util.go → internal/dfutils/dfutils.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,25 +14,29 @@
* limitations under the License.
*/

package cdnutil
package dfutils

import (
"d7y.io/dragonfly/v2/cdn/config"
const (
// DefaultPieceSize 4M
DefaultPieceSize = 4 * 1024 * 1024

// DefaultPieceSizeLimit 15M
DefaultPieceSizeLimit = 15 * 1024 * 1024
)

// ComputePieceSize computes the piece size with specified fileLength.
//
// If the fileLength<=0, which means failed to get fileLength
// If the fileLength<0, which means failed to get fileLength
// and then use the DefaultPieceSize.
func ComputePieceSize(length int64) uint32 {
if length <= 0 || length <= 200*1024*1024 {
return config.DefaultPieceSize
return DefaultPieceSize
}

gapCount := length / int64(100*1024*1024)
mpSize := (gapCount-2)*1024*1024 + config.DefaultPieceSize
if mpSize > config.DefaultPieceSizeLimit {
return config.DefaultPieceSizeLimit
mpSize := (gapCount-2)*1024*1024 + DefaultPieceSize
if mpSize > DefaultPieceSizeLimit {
return DefaultPieceSizeLimit
}
return uint32(mpSize)
}
71 changes: 71 additions & 0 deletions internal/dfutils/dfutils_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
/*
* Copyright 2020 The Dragonfly Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package dfutils

import (
"testing"
)

func TestComputePieceSize(t *testing.T) {
type args struct {
length int64
}
tests := []struct {
name string
args args
want uint32
}{
{
name: "length equal 200M and get default piece size",
args: args{
length: 200 * 1024 * 1024,
},
want: DefaultPieceSize,
}, {
name: "length smaller than 200M and get default piece size",
args: args{
length: 100 * 1024 * 1024,
},
want: DefaultPieceSize,
}, {
name: "length greater than 200M",
args: args{
length: 205 * 1024 * 1024,
},
want: DefaultPieceSize,
}, {
name: "length greater than 300M",
args: args{
length: 310 * 1024 * 1024,
},
want: DefaultPieceSize + 1*1024*1024,
}, {
name: "length reach piece size limit",
args: args{
length: 3100 * 1024 * 1024,
},
want: DefaultPieceSizeLimit,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ComputePieceSize(tt.args.length); got != tt.want {
t.Errorf("ComputePieceSize() = %v, want %v", got, tt.want)
}
})
}
}

0 comments on commit d063ef2

Please sign in to comment.