Skip to content

Commit 8d266b9

Browse files
fpetkovskifpetkovski
authored
Merge pull request #155 from fpetkovski/bucket-provider
Add Provider() method to objstore.Client
2 parents d1dd89d + 01c67f7 commit 8d266b9

File tree

21 files changed

+95
-54
lines changed

21 files changed

+95
-54
lines changed

‎CHANGELOG.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
2525
- [#79](https://github.com/thanos-io/objstore/pull/79) Metrics: Fix `objstore_bucket_operation_duration_seconds` for `iter` operations.
2626

2727
### Added
28-
- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client.
2928
- [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support.
3029
- [#25](https://github.com/thanos-io/objstore/pull/25) S3: Support specifying S3 storage class.
3130
- [#32](https://github.com/thanos-io/objstore/pull/32) Swift: Support authentication using application credentials.
@@ -56,6 +55,9 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
5655
- [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket
5756
- [#147](https://github.com/thanos-io/objstore/pull/147) feat: Add MaxRetries config to cos, gcs and obs.
5857
- [#150](https://github.com/thanos-io/objstore/pull/150) Add support for roundtripper wrapper.
58+
- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client.
59+
- [#155](https://github.com/thanos-io/objstore/pull/155) Add a `Provider` method on `objstore.Client`.
60+
5961

6062
### Changed
6163
- [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`.

‎README.md

+4-2
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,15 @@ See [MAINTAINERS.md](https://github.com/thanos-io/thanos/blob/main/MAINTAINERS.m
4848

4949
The core this module is the [`Bucket` interface](objstore.go):
5050

51-
```go mdox-exec="sed -n '39,55p' objstore.go"
51+
```go mdox-exec="sed -n '55,73p' objstore.go"
5252
// Bucket provides read and write access to an object storage bucket.
5353
// NOTE: We assume strong consistency for write-read flow.
5454
type Bucket interface {
5555
io.Closer
5656
BucketReader
5757

58+
Provider() ObjProvider
59+
5860
// Upload the contents of the reader as an object into the bucket.
5961
// Upload should be idempotent.
6062
Upload(ctx context.Context, name string, r io.Reader) error
@@ -70,7 +72,7 @@ type Bucket interface {
7072

7173
All [provider implementations](providers) have to implement `Bucket` interface that allows common read and write operations that all supported by all object providers. If you want to limit the code that will do bucket operation to only read access (smart idea, allowing to limit access permissions), you can use the [`BucketReader` interface](objstore.go):
7274

73-
```go mdox-exec="sed -n '71,106p' objstore.go"
75+
```go mdox-exec="sed -n '89,124p' objstore.go"
7476
// BucketReader provides read access to an object storage bucket.
7577
type BucketReader interface {
7678
// Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full

‎client/factory.go

+13-28
Original file line numberDiff line numberDiff line change
@@ -27,25 +27,10 @@ import (
2727
"gopkg.in/yaml.v2"
2828
)
2929

30-
type ObjProvider string
31-
32-
const (
33-
FILESYSTEM ObjProvider = "FILESYSTEM"
34-
GCS ObjProvider = "GCS"
35-
S3 ObjProvider = "S3"
36-
AZURE ObjProvider = "AZURE"
37-
SWIFT ObjProvider = "SWIFT"
38-
COS ObjProvider = "COS"
39-
ALIYUNOSS ObjProvider = "ALIYUNOSS"
40-
BOS ObjProvider = "BOS"
41-
OCI ObjProvider = "OCI"
42-
OBS ObjProvider = "OBS"
43-
)
44-
4530
type BucketConfig struct {
46-
Type ObjProvider `yaml:"type"`
47-
Config interface{} `yaml:"config"`
48-
Prefix string `yaml:"prefix" default:""`
31+
Type objstore.ObjProvider `yaml:"type"`
32+
Config interface{} `yaml:"config"`
33+
Prefix string `yaml:"prefix" default:""`
4934
}
5035

5136
// NewBucket initializes and returns new object storage clients.
@@ -64,25 +49,25 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, wrap
6449

6550
var bucket objstore.Bucket
6651
switch strings.ToUpper(string(bucketConf.Type)) {
67-
case string(GCS):
52+
case string(objstore.GCS):
6853
bucket, err = gcs.NewBucket(context.Background(), logger, config, component, wrapRoundtripper)
69-
case string(S3):
54+
case string(objstore.S3):
7055
bucket, err = s3.NewBucket(logger, config, component, wrapRoundtripper)
71-
case string(AZURE):
56+
case string(objstore.AZURE):
7257
bucket, err = azure.NewBucket(logger, config, component, wrapRoundtripper)
73-
case string(SWIFT):
58+
case string(objstore.SWIFT):
7459
bucket, err = swift.NewContainer(logger, config, wrapRoundtripper)
75-
case string(COS):
60+
case string(objstore.COS):
7661
bucket, err = cos.NewBucket(logger, config, component, wrapRoundtripper)
77-
case string(ALIYUNOSS):
62+
case string(objstore.ALIYUNOSS):
7863
bucket, err = oss.NewBucket(logger, config, component, wrapRoundtripper)
79-
case string(FILESYSTEM):
64+
case string(objstore.FILESYSTEM):
8065
bucket, err = filesystem.NewBucketFromConfig(config)
81-
case string(BOS):
66+
case string(objstore.BOS):
8267
bucket, err = bos.NewBucket(logger, config, component)
83-
case string(OCI):
68+
case string(objstore.OCI):
8469
bucket, err = oci.NewBucket(logger, config, wrapRoundtripper)
85-
case string(OBS):
70+
case string(objstore.OBS):
8671
bucket, err = obs.NewBucket(logger, config)
8772
default:
8873
return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type)

‎inmem.go

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ func NewInMemBucket() *InMemBucket {
3434
}
3535
}
3636

37+
func (b *InMemBucket) Provider() ObjProvider { return MEMORY }
38+
3739
// Objects returns a copy of the internally stored objects.
3840
// NOTE: For assert purposes.
3941
func (b *InMemBucket) Objects() map[string][]byte {

‎objstore.go

+22
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,22 @@ import (
2626
"golang.org/x/sync/errgroup"
2727
)
2828

29+
type ObjProvider string
30+
31+
const (
32+
MEMORY ObjProvider = "MEMORY"
33+
FILESYSTEM ObjProvider = "FILESYSTEM"
34+
GCS ObjProvider = "GCS"
35+
S3 ObjProvider = "S3"
36+
AZURE ObjProvider = "AZURE"
37+
SWIFT ObjProvider = "SWIFT"
38+
COS ObjProvider = "COS"
39+
ALIYUNOSS ObjProvider = "ALIYUNOSS"
40+
BOS ObjProvider = "BOS"
41+
OCI ObjProvider = "OCI"
42+
OBS ObjProvider = "OBS"
43+
)
44+
2945
const (
3046
OpIter = "iter"
3147
OpGet = "get"
@@ -42,6 +58,8 @@ type Bucket interface {
4258
io.Closer
4359
BucketReader
4460

61+
Provider() ObjProvider
62+
4563
// Upload the contents of the reader as an object into the bucket.
4664
// Upload should be idempotent.
4765
Upload(ctx context.Context, name string, r io.Reader) error
@@ -583,6 +601,10 @@ type metricBucket struct {
583601
metrics *Metrics
584602
}
585603

604+
func (b *metricBucket) Provider() ObjProvider {
605+
return b.bkt.Provider()
606+
}
607+
586608
func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket {
587609
return &metricBucket{
588610
bkt: b.bkt,

‎objtesting/foreach.go

+10-11
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"testing"
1010

1111
"github.com/thanos-io/objstore"
12-
"github.com/thanos-io/objstore/client"
1312
"github.com/thanos-io/objstore/providers/azure"
1413
"github.com/thanos-io/objstore/providers/bos"
1514
"github.com/thanos-io/objstore/providers/cos"
@@ -26,7 +25,7 @@ import (
2625

2726
// IsObjStoreSkipped returns true if given provider ID is found in THANOS_TEST_OBJSTORE_SKIP array delimited by comma e.g:
2827
// THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI.
29-
func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool {
28+
func IsObjStoreSkipped(t *testing.T, provider objstore.ObjProvider) bool {
3029
if e, ok := os.LookupEnv("THANOS_TEST_OBJSTORE_SKIP"); ok {
3130
obstores := strings.Split(e, ",")
3231
for _, objstore := range obstores {
@@ -69,7 +68,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
6968
})
7069

7170
// Optional GCS.
72-
if !IsObjStoreSkipped(t, client.GCS) {
71+
if !IsObjStoreSkipped(t, objstore.GCS) {
7372
t.Run("gcs", func(t *testing.T) {
7473
bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT"))
7574
testutil.Ok(t, err)
@@ -84,7 +83,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
8483
}
8584

8685
// Optional S3.
87-
if !IsObjStoreSkipped(t, client.S3) {
86+
if !IsObjStoreSkipped(t, objstore.S3) {
8887
t.Run("aws s3", func(t *testing.T) {
8988
// TODO(bwplotka): Allow taking location from envvar.
9089
bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2")
@@ -103,7 +102,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
103102
}
104103

105104
// Optional Azure.
106-
if !IsObjStoreSkipped(t, client.AZURE) {
105+
if !IsObjStoreSkipped(t, objstore.AZURE) {
107106
t.Run("azure", func(t *testing.T) {
108107
bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests")
109108
testutil.Ok(t, err)
@@ -117,7 +116,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
117116
}
118117

119118
// Optional SWIFT.
120-
if !IsObjStoreSkipped(t, client.SWIFT) {
119+
if !IsObjStoreSkipped(t, objstore.SWIFT) {
121120
t.Run("swift", func(t *testing.T) {
122121
container, closeFn, err := swift.NewTestContainer(t)
123122
testutil.Ok(t, err)
@@ -131,7 +130,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
131130
}
132131

133132
// Optional COS.
134-
if !IsObjStoreSkipped(t, client.COS) {
133+
if !IsObjStoreSkipped(t, objstore.COS) {
135134
t.Run("Tencent cos", func(t *testing.T) {
136135
bkt, closeFn, err := cos.NewTestBucket(t)
137136
testutil.Ok(t, err)
@@ -145,7 +144,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
145144
}
146145

147146
// Optional OSS.
148-
if !IsObjStoreSkipped(t, client.ALIYUNOSS) {
147+
if !IsObjStoreSkipped(t, objstore.ALIYUNOSS) {
149148
t.Run("AliYun oss", func(t *testing.T) {
150149
bkt, closeFn, err := oss.NewTestBucket(t)
151150
testutil.Ok(t, err)
@@ -159,7 +158,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
159158
}
160159

161160
// Optional BOS.
162-
if !IsObjStoreSkipped(t, client.BOS) {
161+
if !IsObjStoreSkipped(t, objstore.BOS) {
163162
t.Run("Baidu BOS", func(t *testing.T) {
164163
bkt, closeFn, err := bos.NewTestBucket(t)
165164
testutil.Ok(t, err)
@@ -173,7 +172,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
173172
}
174173

175174
// Optional OCI.
176-
if !IsObjStoreSkipped(t, client.OCI) {
175+
if !IsObjStoreSkipped(t, objstore.OCI) {
177176
t.Run("oci", func(t *testing.T) {
178177
bkt, closeFn, err := oci.NewTestBucket(t)
179178
testutil.Ok(t, err)
@@ -186,7 +185,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
186185
}
187186

188187
// Optional OBS.
189-
if !IsObjStoreSkipped(t, client.OBS) {
188+
if !IsObjStoreSkipped(t, objstore.OBS) {
190189
t.Run("obs", func(t *testing.T) {
191190
bkt, closeFn, err := obs.NewTestBucket(t, "cn-south-1")
192191
testutil.Ok(t, err)

‎prefixed_bucket.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ func withPrefix(prefix, name string) string {
3939
return prefix + DirDelim + name
4040
}
4141

42+
func (p *PrefixedBucket) Provider() ObjProvider { return p.bkt.Provider() }
43+
4244
func (p *PrefixedBucket) Close() error {
4345
return p.bkt.Close()
4446
}
@@ -93,7 +95,7 @@ func (p *PrefixedBucket) IsAccessDeniedErr(err error) bool {
9395
}
9496

9597
// Attributes returns information about the specified object.
96-
func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) {
98+
func (p *PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) {
9799
return p.bkt.Attributes(ctx, conditionalPrefix(p.prefix, name))
98100
}
99101

‎providers/azure/azure.go

+2
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,8 @@ func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapR
193193
return bkt, nil
194194
}
195195

196+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.AZURE }
197+
196198
func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType {
197199
return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt}
198200
}

‎providers/bos/bos.go

+2
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B
100100
return bkt, nil
101101
}
102102

103+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.BOS }
104+
103105
// Name returns the bucket name for the provider.
104106
func (b *Bucket) Name() string {
105107
return b.name

‎providers/cos/cos.go

+2
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra
159159
return bkt, nil
160160
}
161161

162+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.COS }
163+
162164
// Name returns the bucket name for COS.
163165
func (b *Bucket) Name() string {
164166
return b.name

‎providers/filesystem/filesystem.go

+2
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ func NewBucket(rootDir string) (*Bucket, error) {
5050
return &Bucket{rootDir: absDir}, nil
5151
}
5252

53+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.FILESYSTEM }
54+
5355
func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType {
5456
return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt}
5557
}

‎providers/gcs/gcs.go

+2
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option.
186186
return bkt, nil
187187
}
188188

189+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.GCS }
190+
189191
// Name returns the bucket name for gcs.
190192
func (b *Bucket) Name() string {
191193
return b.name

‎providers/obs/obs.go

+2
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) {
122122
return bkt, nil
123123
}
124124

125+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OBS }
126+
125127
// Name returns the bucket name for the provider.
126128
func (b *Bucket) Name() string {
127129
return b.name

‎providers/oci/oci.go

+2
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ type Bucket struct {
9696
requestMetadata common.RequestMetadata
9797
}
9898

99+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OCI }
100+
99101
// Name returns the bucket name for the provider.
100102
func (b *Bucket) Name() string {
101103
return b.name

‎providers/oss/oss.go

+2
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
6868
return NewTestBucketFromConfig(t, c, false)
6969
}
7070

71+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.ALIYUNOSS }
72+
7173
// Upload the contents of the reader as an object into the bucket.
7274
func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error {
7375
// TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this.

‎providers/s3/s3.go

+2
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra
345345
return bkt, nil
346346
}
347347

348+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.S3 }
349+
348350
// Name returns the bucket name for s3.
349351
func (b *Bucket) Name() string {
350352
return b.name

‎providers/swift/swift.go

+2
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,8 @@ func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool,
218218
}, nil
219219
}
220220

221+
func (c *Container) Provider() objstore.ObjProvider { return objstore.SWIFT }
222+
221223
// Name returns the container name for swift.
222224
func (c *Container) Name() string {
223225
return c.name

0 commit comments

Comments
 (0)