diff --git a/api/manager/docs.go b/api/manager/docs.go index 0505eb3cd6a..50d21d177e6 100644 --- a/api/manager/docs.go +++ b/api/manager/docs.go @@ -4208,10 +4208,6 @@ const docTemplate = `{ "bio": { "type": "string" }, - "result": { - "type": "object", - "additionalProperties": {} - }, "scheduler_cluster_ids": { "type": "array", "items": { diff --git a/api/manager/swagger.json b/api/manager/swagger.json index 1354704b777..63997b5a11f 100644 --- a/api/manager/swagger.json +++ b/api/manager/swagger.json @@ -4202,10 +4202,6 @@ "bio": { "type": "string" }, - "result": { - "type": "object", - "additionalProperties": {} - }, "scheduler_cluster_ids": { "type": "array", "items": { diff --git a/api/manager/swagger.yaml b/api/manager/swagger.yaml index 897ed91f424..3d281176b0f 100644 --- a/api/manager/swagger.yaml +++ b/api/manager/swagger.yaml @@ -464,9 +464,6 @@ definitions: type: object bio: type: string - result: - additionalProperties: {} - type: object scheduler_cluster_ids: items: type: integer diff --git a/manager/service/job.go b/manager/service/job.go index 05dbc891d7e..f68a445601b 100644 --- a/manager/service/job.go +++ b/manager/service/job.go @@ -28,6 +28,7 @@ import ( "d7y.io/dragonfly/v2/manager/metrics" "d7y.io/dragonfly/v2/manager/models" "d7y.io/dragonfly/v2/manager/types" + "d7y.io/dragonfly/v2/pkg/net/http" "d7y.io/dragonfly/v2/pkg/retry" "d7y.io/dragonfly/v2/pkg/slices" "d7y.io/dragonfly/v2/pkg/structure" @@ -46,6 +47,10 @@ func (s *service) CreatePreheatJob(ctx context.Context, json types.CreatePreheat json.Args.Timeout = types.DefaultJobTimeout } + if json.Args.FilteredQueryParams == "" { + json.Args.FilteredQueryParams = http.RawDefaultFilteredQueryParams + } + args, err := structure.StructToMap(json.Args) if err != nil { return nil, err @@ -85,6 +90,10 @@ func (s *service) CreatePreheatJob(ctx context.Context, json types.CreatePreheat } func (s *service) CreateGetTaskJob(ctx context.Context, json types.CreateGetTaskJobRequest) (*models.Job, error) { + if json.Args.FilteredQueryParams == "" { + json.Args.FilteredQueryParams = http.RawDefaultFilteredQueryParams + } + args, err := structure.StructToMap(json.Args) if err != nil { return nil, err @@ -128,6 +137,10 @@ func (s *service) CreateDeleteTaskJob(ctx context.Context, json types.CreateDele json.Args.Timeout = types.DefaultJobTimeout } + if json.Args.FilteredQueryParams == "" { + json.Args.FilteredQueryParams = http.RawDefaultFilteredQueryParams + } + args, err := structure.StructToMap(json.Args) if err != nil { return nil, err diff --git a/manager/types/model.go b/manager/types/model.go deleted file mode 100644 index c18621315f2..00000000000 --- a/manager/types/model.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2023 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package types - -import ( - "fmt" -) - -const ( - // ModelFileName is model file name. - ModelFileName = "model.graphdef" - - // ModelConfigFileName is model config file name. - ModelConfigFileName = "config.pbtxt" - - // GNNModelNameSuffix is suffix of GNN model name. - GNNModelNameSuffix = "gnn" - - // MLPModelNameSuffix is suffix of MLP model name. - MLPModelNameSuffix = "mlp" - - // DefaultTritonPlatform is default triton backend configuration. - DefaultTritonPlatform = "tensorrt_plan" -) - -type ModelParams struct { - ID uint `uri:"id" binding:"required"` -} - -type UpdateModelRequest struct { - BIO string `json:"BIO" binding:"omitempty"` - State string `json:"state" binding:"omitempty,oneof=active inactive"` -} - -type GetModelsQuery struct { - Name string `json:"name" binding:"omitempty"` - Type string `json:"type" binding:"omitempty"` - Version string `json:"version" binding:"omitempty"` - SchedulerID uint `json:"scheduler_id" binding:"omitempty"` - Page int `form:"page" binding:"omitempty,gte=1"` - PerPage int `form:"per_page" binding:"omitempty,gte=1,lte=10000000"` -} - -type ModelEvaluation struct { - Recall float64 `json:"recall" binding:"omitempty,gte=0,lte=1"` - Precision float64 `json:"precision" binding:"omitempty,gte=0,lte=1"` - F1Score float64 `json:"f1_score" binding:"omitempty,gte=0,lte=1"` - MSE float64 `json:"mse" binding:"omitempty,gte=0"` - MAE float64 `json:"mae" binding:"omitempty,gte=0"` -} - -// MakeObjectKeyOfModelFile returns object key of model file. -func MakeObjectKeyOfModelFile(id string, version int) string { - return fmt.Sprintf("%s/%s/%s", id, fmt.Sprint(version), ModelFileName) -} - -// MakeObjectKeyOfModelConfigFile returns object key of model config file. -func MakeObjectKeyOfModelConfigFile(id string) string { - return fmt.Sprintf("%s/%s", id, ModelConfigFileName) -} diff --git a/manager/types/model_test.go b/manager/types/model_test.go deleted file mode 100644 index 30385acfaf0..00000000000 --- a/manager/types/model_test.go +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2023 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package types - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_MakeObjectKeyOfModelFile(t *testing.T) { - tests := []struct { - name string - modelName string - version int - expect func(t *testing.T, s string) - }{ - { - name: "make objectKey of model file", - modelName: "foo", - version: 1, - expect: func(t *testing.T, s string) { - assert := assert.New(t) - assert.Equal(s, "foo/1/model.graphdef") - }, - }, - { - name: "modelName is empty", - modelName: "", - version: 1, - expect: func(t *testing.T, s string) { - assert := assert.New(t) - assert.Equal(s, "/1/model.graphdef") - }, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - tc.expect(t, MakeObjectKeyOfModelFile(tc.modelName, tc.version)) - }) - } -} - -func Test_MakeObjectKeyOfModelConfigFile(t *testing.T) { - tests := []struct { - name string - modelName string - version int - expect func(t *testing.T, s string) - }{ - { - name: "make objectKey of model file", - modelName: "foo", - expect: func(t *testing.T, s string) { - assert := assert.New(t) - assert.Equal(s, "foo/config.pbtxt") - }, - }, - { - name: "modelName is empty", - modelName: "", - expect: func(t *testing.T, s string) { - assert := assert.New(t) - assert.Equal(s, "/config.pbtxt") - }, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - tc.expect(t, MakeObjectKeyOfModelConfigFile(tc.modelName)) - }) - } -} diff --git a/pkg/idgen/model_id.go b/pkg/idgen/model_id.go deleted file mode 100644 index 8e2efcbd4cf..00000000000 --- a/pkg/idgen/model_id.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package idgen - -import ( - "d7y.io/dragonfly/v2/pkg/digest" -) - -const ( - // GNNModelNameSuffix is suffix of GNN model id. - GNNModelNameSuffix = "gnn" - - // MLPModelNameSuffix is suffix of MLP model id. - MLPModelNameSuffix = "mlp" -) - -// GNNModelIDV1 generates v1 version of gnn model id. -func GNNModelIDV1(ip, hostname string) string { - return digest.SHA256FromStrings(ip, hostname, GNNModelNameSuffix) -} - -// MLPModelIDV1 generates v1 version of mlp model id. -func MLPModelIDV1(ip, hostname string) string { - return digest.SHA256FromStrings(ip, hostname, MLPModelNameSuffix) -} diff --git a/pkg/idgen/model_id_test.go b/pkg/idgen/model_id_test.go deleted file mode 100644 index b5643497b91..00000000000 --- a/pkg/idgen/model_id_test.go +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2023 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package idgen - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestGNNModelIDV1(t *testing.T) { - tests := []struct { - name string - ip string - hostname string - expect func(t *testing.T, d string) - }{ - { - name: "generate GNNModelID", - ip: "127.0.0.1", - hostname: "foo", - expect: func(t *testing.T, d string) { - assert := assert.New(t) - assert.Equal(d, "0c1cfa1cf4b2f58b0e632dca66537cae6596453ec793c38bb14b0de4fa232474") - }, - }, - { - name: "generate GNNModelID with empty ip", - ip: "", - hostname: "foo", - expect: func(t *testing.T, d string) { - assert := assert.New(t) - assert.Equal(d, "10ad70f3d95e523e4d9f6d830ea92b96bb9a8c91da76c135bc66208fb744454c") - }, - }, - { - name: "generate GNNModelID with empty host", - ip: "127.0.0.1", - hostname: "", - expect: func(t *testing.T, d string) { - assert := assert.New(t) - assert.Equal(d, "562a69955f8592589d5ed747888c8c3e9d81420657b7bd33847b5bb2d1d3db4c") - }, - }, - { - name: "generate GNNModelID with zero clusterID", - ip: "127.0.0.1", - hostname: "127.0.0.1", - expect: func(t *testing.T, d string) { - assert := assert.New(t) - assert.Equal(d, "b057d986d82d071f356e13e6f3042b14fe182d57b801a211fa9f21c76ba5290b") - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - tc.expect(t, GNNModelIDV1(tc.ip, tc.hostname)) - }) - } -} - -func TestMLPModelIDV1(t *testing.T) { - tests := []struct { - name string - ip string - hostname string - expect func(t *testing.T, d string) - }{ - { - name: "generate MLPModelID", - ip: "127.0.0.1", - hostname: "foo", - expect: func(t *testing.T, d string) { - assert := assert.New(t) - assert.Equal(d, "2ba6ab2e9d9eec939b98890c095891aef9864d88558b7b3727fb05ae87d6e037") - }, - }, - { - name: "generate MLPModelID with empty ip", - ip: "", - hostname: "foo", - expect: func(t *testing.T, d string) { - assert := assert.New(t) - assert.Equal(d, "6639d7f1cfa7842016ba5b0a19bf03930ff85d406e6f7763bd4ff88774400298") - }, - }, - { - name: "generate MLPModelID with empty host", - ip: "127.0.0.1", - hostname: "", - expect: func(t *testing.T, d string) { - assert := assert.New(t) - assert.Equal(d, "3b40fd716824d6fc0d5a0f2eff2eb051c526b75a29d4c82a1b2d1174f6db4e7f") - }, - }, - { - name: "generate MLPModelID with zero clusterID", - ip: "127.0.0.1", - hostname: "127.0.0.1", - expect: func(t *testing.T, d string) { - assert := assert.New(t) - assert.Equal(d, "16e2fe757406d847974f711ebe8285df132e5f4f99c297b1bd16b952fe7eee2a") - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - tc.expect(t, MLPModelIDV1(tc.ip, tc.hostname)) - }) - } -} diff --git a/pkg/net/http/header.go b/pkg/net/http/header.go new file mode 100644 index 00000000000..079d4d83253 --- /dev/null +++ b/pkg/net/http/header.go @@ -0,0 +1,81 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package http + +import ( + "strings" + + "d7y.io/dragonfly/v2/pkg/idgen" + pkgstrings "d7y.io/dragonfly/v2/pkg/strings" +) + +// S3FilteredQueryParams is the default filtered query params with s3 protocol to generate the task id. +var S3FilteredQueryParams = []string{ + "X-Amz-Algorithm", + "X-Amz-Credential", + "X-Amz-Date", + "X-Amz-Expires", + "X-Amz-SignedHeaders", + "X-Amz-Signature", + "X-Amz-Security-Token", + "X-Amz-User-Agent", +} + +// GCSFilteredQueryParams is the default filtered query params with gcs protocol to generate the task id. +var GCSFilteredQueryParams = []string{ + "X-Goog-Algorithm", + "X-Goog-Credential", + "X-Goog-Date", + "X-Goog-Expires", + "X-Goog-SignedHeaders", + "X-Goog-Signature", +} + +// OSSFilteredQueryParams is the default filtered query params with oss protocol to generate the task id. +var OSSFilteredQueryParams = []string{ + "OSSAccessKeyId", + "Expires", + "Signature", + "SecurityToken", +} + +// OBSFilteredQueryParams is the default filtered query params with obs protocol to generate the task id. +var OBSFilteredQueryParams = []string{ + "AccessKeyId", + "Signature", + "Expires", + "X-Obs-Date", + "X-Obs-Security-Token", +} + +// COSFilteredQueryParams is the default filtered query params with cos protocol to generate the task id. +var COSFilteredQueryParams = []string{ + "q-sign-algorithm", + "q-ak", + "q-sign-time", + "q-key-time", + "q-header-list", + "q-url-param-list", + "q-signature", + "x-cos-security-token", +} + +// DefaultFilteredQueryParams is the default filtered query params to generate the task id. +var DefaultFilteredQueryParams = pkgstrings.Concat(S3FilteredQueryParams, GCSFilteredQueryParams, OSSFilteredQueryParams, OBSFilteredQueryParams, COSFilteredQueryParams) + +// RawDefaultFilteredQueryParams is the raw default filtered query params to generate the task id. +var RawDefaultFilteredQueryParams = strings.Join(DefaultFilteredQueryParams, idgen.FilteredQueryParamsSeparator) diff --git a/pkg/reachable/mocks/reachable_mock.go b/pkg/reachable/mocks/reachable_mock.go deleted file mode 100644 index 2728fc4e2bb..00000000000 --- a/pkg/reachable/mocks/reachable_mock.go +++ /dev/null @@ -1,53 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: reachable.go -// -// Generated by this command: -// -// mockgen -destination mocks/reachable_mock.go -source reachable.go -package mocks -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - reflect "reflect" - - gomock "go.uber.org/mock/gomock" -) - -// MockReachable is a mock of Reachable interface. -type MockReachable struct { - ctrl *gomock.Controller - recorder *MockReachableMockRecorder -} - -// MockReachableMockRecorder is the mock recorder for MockReachable. -type MockReachableMockRecorder struct { - mock *MockReachable -} - -// NewMockReachable creates a new mock instance. -func NewMockReachable(ctrl *gomock.Controller) *MockReachable { - mock := &MockReachable{ctrl: ctrl} - mock.recorder = &MockReachableMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockReachable) EXPECT() *MockReachableMockRecorder { - return m.recorder -} - -// Check mocks base method. -func (m *MockReachable) Check() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Check") - ret0, _ := ret[0].(error) - return ret0 -} - -// Check indicates an expected call of Check. -func (mr *MockReachableMockRecorder) Check() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockReachable)(nil).Check)) -} diff --git a/pkg/reachable/reachable.go b/pkg/reachable/reachable.go deleted file mode 100644 index f5f368f7c71..00000000000 --- a/pkg/reachable/reachable.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2020 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//go:generate mockgen -destination mocks/reachable_mock.go -source reachable.go -package mocks - -package reachable - -import ( - "fmt" - "net" - "strings" - "time" -) - -const ( - // DefaultPort is the default tcp port. - DefaultPort = "80" - // DefaultNetwork is the default network type. - DefaultNetwork = "tcp" - // DefaultTimeout is the default dial timeout. - DefaultTimeout = 1 * time.Second -) - -type Reachable interface { - // Check that the address can be accessed. - Check() error -} - -type reachable struct { - address string - network string - timeout time.Duration -} - -type Config struct { - Address string - Network string - Timeout time.Duration -} - -// New returns a new ReachableInterface interface. -func New(r *Config) Reachable { - network := DefaultNetwork - if r.Network != "" { - network = r.Network - } - - timeout := DefaultTimeout - if r.Timeout != 0 { - timeout = r.Timeout - } - - return &reachable{ - address: r.Address, - network: network, - timeout: timeout, - } -} - -// Check that the address can be accessed. -func (r *reachable) Check() error { - if !strings.Contains(r.address, ":") { - r.address = fmt.Sprintf("%s:%s", r.address, DefaultPort) - } - - conn, err := net.DialTimeout(r.network, r.address, r.timeout) - if err != nil { - return err - } - conn.Close() - - return nil -} diff --git a/pkg/reachable/reachable_test.go b/pkg/reachable/reachable_test.go deleted file mode 100644 index 74336752394..00000000000 --- a/pkg/reachable/reachable_test.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2020 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package reachable - -import ( - "net" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestReachableCheck(t *testing.T) { - l, err := net.Listen("tcp", ":3000") - if err != nil { - t.Fatal(err) - } - defer l.Close() - - tests := []struct { - name string - address string - network string - timeout time.Duration - expect func(t *testing.T, err error) - }{ - { - name: "check address", - address: ":3000", - network: "tcp", - timeout: 1 * time.Second, - expect: func(t *testing.T, err error) { - assert := assert.New(t) - assert.NoError(err) - }, - }, - { - name: "check address without network", - address: ":3000", - network: "", - timeout: 1 * time.Second, - expect: func(t *testing.T, err error) { - assert := assert.New(t) - assert.NoError(err) - }, - }, - { - name: "check address without address", - address: "", - network: "tcp", - timeout: 1 * time.Second, - expect: func(t *testing.T, err error) { - assert := assert.New(t) - assert.Error(err) - }, - }, - { - name: "check invalid address", - address: "example", - network: "tcp", - timeout: 1 * time.Second, - expect: func(t *testing.T, err error) { - assert := assert.New(t) - assert.Error(err) - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - r := New(&Config{ - Address: tc.address, - Network: tc.network, - Timeout: tc.timeout, - }) - tc.expect(t, r.Check()) - }) - } -} diff --git a/pkg/strings/strings.go b/pkg/strings/strings.go index 68e627eeebe..c5f8f76be33 100644 --- a/pkg/strings/strings.go +++ b/pkg/strings/strings.go @@ -49,3 +49,12 @@ func Unique(slice []string) []string { return result } + +// Concat concatenates multiple string slices. +func Concat(slices ...[]string) []string { + var result []string + for _, slice := range slices { + result = append(result, slice...) + } + return result +} diff --git a/pkg/strings/strings_test.go b/pkg/strings/strings_test.go index 6b59036fa92..560f1643419 100644 --- a/pkg/strings/strings_test.go +++ b/pkg/strings/strings_test.go @@ -40,3 +40,10 @@ func TestUnique(t *testing.T) { assert.EqualValues(t, Unique([]string{}), []string{}) assert.EqualValues(t, Unique([]string{}), []string{}) } + +func TestConcat(t *testing.T) { + assert.EqualValues(t, Concat([]string{"a", "B"}, []string{"c", "D"}), []string{"a", "B", "c", "D"}) + assert.EqualValues(t, Concat([]string{"a", "B"}, []string{}), []string{"a", "B"}) + assert.EqualValues(t, Concat([]string{}, []string{"c", "D"}), []string{"c", "D"}) + assert.EqualValues(t, Concat([]string{}, []string{}), []string(nil)) +}