Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Swift: Add HTTP_Config #99

Merged
merged 1 commit into from
Feb 12, 2024
Merged

Swift: Add HTTP_Config #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
- [#94](https://github.com/thanos-io/objstore/pull/94) Allow timingReadCloser to be seeker
- [#96](https://github.com/thanos-io/objstore/pull/96) Allow nopCloserWithObjectSize to be seeker
- [#86](https://github.com/thanos-io/objstore/pull/86) GCS: Add HTTP Config to GCS
- [#99](https://github.com/thanos-io/objstore/pull/99) Swift: Add HTTP_Config

### Changed
- [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`.
Expand Down
72 changes: 46 additions & 26 deletions providers/swift/swift.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
"fmt"
"io"
"net/http"
"os"
"strconv"
"strings"
Expand All @@ -21,6 +22,7 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/common/model"
"github.com/thanos-io/objstore"
"github.com/thanos-io/objstore/exthttp"
"gopkg.in/yaml.v2"
)

Expand All @@ -37,33 +39,35 @@ var DefaultConfig = Config{
Retries: 3,
ConnectTimeout: model.Duration(10 * time.Second),
Timeout: model.Duration(5 * time.Minute),
HTTPConfig: exthttp.DefaultHTTPConfig,
}

type Config struct {
AuthVersion int `yaml:"auth_version"`
AuthUrl string `yaml:"auth_url"`
Username string `yaml:"username"`
UserDomainName string `yaml:"user_domain_name"`
UserDomainID string `yaml:"user_domain_id"`
UserId string `yaml:"user_id"`
Password string `yaml:"password"`
DomainId string `yaml:"domain_id"`
DomainName string `yaml:"domain_name"`
ApplicationCredentialID string `yaml:"application_credential_id"`
ApplicationCredentialName string `yaml:"application_credential_name"`
ApplicationCredentialSecret string `yaml:"application_credential_secret"`
ProjectID string `yaml:"project_id"`
ProjectName string `yaml:"project_name"`
ProjectDomainID string `yaml:"project_domain_id"`
ProjectDomainName string `yaml:"project_domain_name"`
RegionName string `yaml:"region_name"`
ContainerName string `yaml:"container_name"`
ChunkSize int64 `yaml:"large_object_chunk_size"`
SegmentContainerName string `yaml:"large_object_segments_container_name"`
Retries int `yaml:"retries"`
ConnectTimeout model.Duration `yaml:"connect_timeout"`
Timeout model.Duration `yaml:"timeout"`
UseDynamicLargeObjects bool `yaml:"use_dynamic_large_objects"`
AuthVersion int `yaml:"auth_version"`
AuthUrl string `yaml:"auth_url"`
Username string `yaml:"username"`
UserDomainName string `yaml:"user_domain_name"`
UserDomainID string `yaml:"user_domain_id"`
UserId string `yaml:"user_id"`
Password string `yaml:"password"`
DomainId string `yaml:"domain_id"`
DomainName string `yaml:"domain_name"`
ApplicationCredentialID string `yaml:"application_credential_id"`
ApplicationCredentialName string `yaml:"application_credential_name"`
ApplicationCredentialSecret string `yaml:"application_credential_secret"`
ProjectID string `yaml:"project_id"`
ProjectName string `yaml:"project_name"`
ProjectDomainID string `yaml:"project_domain_id"`
ProjectDomainName string `yaml:"project_domain_name"`
RegionName string `yaml:"region_name"`
ContainerName string `yaml:"container_name"`
ChunkSize int64 `yaml:"large_object_chunk_size"`
SegmentContainerName string `yaml:"large_object_segments_container_name"`
Retries int `yaml:"retries"`
ConnectTimeout model.Duration `yaml:"connect_timeout"`
Timeout model.Duration `yaml:"timeout"`
UseDynamicLargeObjects bool `yaml:"use_dynamic_large_objects"`
HTTPConfig exthttp.HTTPConfig `yaml:"http_config"`
}

func parseConfig(conf []byte) (*Config, error) {
Expand Down Expand Up @@ -101,6 +105,7 @@ func configFromEnv() (*Config, error) {
ConnectTimeout: model.Duration(c.ConnectTimeout),
Timeout: model.Duration(c.Timeout),
UseDynamicLargeObjects: false,
HTTPConfig: DefaultConfig.HTTPConfig,
}
if os.Getenv("SWIFT_CHUNK_SIZE") != "" {
var err error
Expand All @@ -115,7 +120,7 @@ func configFromEnv() (*Config, error) {
return &config, nil
}

func connectionFromConfig(sc *Config) *swift.Connection {
func connectionFromConfig(sc *Config, rt http.RoundTripper) *swift.Connection {
connection := swift.Connection{
AuthVersion: sc.AuthVersion,
AuthUrl: sc.AuthUrl,
Expand All @@ -135,6 +140,7 @@ func connectionFromConfig(sc *Config) *swift.Connection {
Retries: sc.Retries,
ConnectTimeout: time.Duration(sc.ConnectTimeout),
Timeout: time.Duration(sc.Timeout),
Transport: rt,
}
return &connection
}
Expand Down Expand Up @@ -173,7 +179,21 @@ func ensureContainer(connection *swift.Connection, name string, createIfNotExist
}

func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool) (*Container, error) {
connection := connectionFromConfig(sc)

// Check if a roundtripper has been set in the config
// otherwise build the default transport.
var rt http.RoundTripper
if sc.HTTPConfig.Transport != nil {
rt = sc.HTTPConfig.Transport
} else {
var err error
rt, err = exthttp.DefaultTransport(sc.HTTPConfig)
if err != nil {
return nil, err
}
}

connection := connectionFromConfig(sc, rt)
if err := connection.Authenticate(); err != nil {
return nil, errors.Wrap(err, "authentication")
}
Expand Down
30 changes: 30 additions & 0 deletions providers/swift/swift_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ package swift

import (
"testing"
"time"

"github.com/efficientgo/core/testutil"
"github.com/prometheus/common/model"
)

func TestParseConfig(t *testing.T) {
Expand Down Expand Up @@ -34,3 +36,31 @@ tenant_name: something`)
// Must result in unmarshal error as there's no `tenant_name` in SwiftConfig.
testutil.NotOk(t, err)
}

func TestParseConfig_HTTPConfig(t *testing.T) {
input := []byte(`auth_url: http://identity.something.com/v3
username: thanos
user_domain_name: userDomain
project_name: thanosProject
project_domain_name: projectDomain
http_config:
tls_config:
ca_file: /certs/ca.crt
cert_file: /certs/cert.crt
key_file: /certs/key.key
server_name: server
insecure_skip_verify: false`)
cfg, err := parseConfig([]byte(input))

testutil.Ok(t, err)

testutil.Equals(t, "http://identity.something.com/v3", cfg.AuthUrl)
testutil.Equals(t, "thanos", cfg.Username)
testutil.Equals(t, "userDomain", cfg.UserDomainName)
testutil.Equals(t, "thanosProject", cfg.ProjectName)
testutil.Equals(t, "projectDomain", cfg.ProjectDomainName)
testutil.Equals(t, model.Duration(90*time.Second), cfg.HTTPConfig.IdleConnTimeout)
testutil.Equals(t, model.Duration(2*time.Minute), cfg.HTTPConfig.ResponseHeaderTimeout)
testutil.Equals(t, false, cfg.HTTPConfig.InsecureSkipVerify)

}
Loading