Skip to content

Commit 49d8d1c

Browse files
fpetkovskifpetkovski
committed
Add Provider() method to objstore.Client
This commit adds a Provider() method to the objstore.Client interface. The method is useful when the client was created from a config file using the client factory. Signed-off-by: Filip Petkovski <[email protected]>
1 parent 2c4ff97 commit 49d8d1c

File tree

20 files changed

+92
-53
lines changed

20 files changed

+92
-53
lines changed

‎README.md

+4-2
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,15 @@ See [MAINTAINERS.md](https://github.com/thanos-io/thanos/blob/main/MAINTAINERS.m
4848

4949
The core this module is the [`Bucket` interface](objstore.go):
5050

51-
```go mdox-exec="sed -n '39,55p' objstore.go"
51+
```go mdox-exec="sed -n '55,73p' objstore.go"
5252
// Bucket provides read and write access to an object storage bucket.
5353
// NOTE: We assume strong consistency for write-read flow.
5454
type Bucket interface {
5555
io.Closer
5656
BucketReader
5757

58+
Provider() ObjProvider
59+
5860
// Upload the contents of the reader as an object into the bucket.
5961
// Upload should be idempotent.
6062
Upload(ctx context.Context, name string, r io.Reader) error
@@ -70,7 +72,7 @@ type Bucket interface {
7072

7173
All [provider implementations](providers) have to implement `Bucket` interface that allows common read and write operations that all supported by all object providers. If you want to limit the code that will do bucket operation to only read access (smart idea, allowing to limit access permissions), you can use the [`BucketReader` interface](objstore.go):
7274

73-
```go mdox-exec="sed -n '71,106p' objstore.go"
75+
```go mdox-exec="sed -n '89,124p' objstore.go"
7476
// BucketReader provides read access to an object storage bucket.
7577
type BucketReader interface {
7678
// Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full

‎client/factory.go

+13-28
Original file line numberDiff line numberDiff line change
@@ -27,25 +27,10 @@ import (
2727
"gopkg.in/yaml.v2"
2828
)
2929

30-
type ObjProvider string
31-
32-
const (
33-
FILESYSTEM ObjProvider = "FILESYSTEM"
34-
GCS ObjProvider = "GCS"
35-
S3 ObjProvider = "S3"
36-
AZURE ObjProvider = "AZURE"
37-
SWIFT ObjProvider = "SWIFT"
38-
COS ObjProvider = "COS"
39-
ALIYUNOSS ObjProvider = "ALIYUNOSS"
40-
BOS ObjProvider = "BOS"
41-
OCI ObjProvider = "OCI"
42-
OBS ObjProvider = "OBS"
43-
)
44-
4530
type BucketConfig struct {
46-
Type ObjProvider `yaml:"type"`
47-
Config interface{} `yaml:"config"`
48-
Prefix string `yaml:"prefix" default:""`
31+
Type objstore.ObjProvider `yaml:"type"`
32+
Config interface{} `yaml:"config"`
33+
Prefix string `yaml:"prefix" default:""`
4934
}
5035

5136
// NewBucket initializes and returns new object storage clients.
@@ -64,25 +49,25 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, wrap
6449

6550
var bucket objstore.Bucket
6651
switch strings.ToUpper(string(bucketConf.Type)) {
67-
case string(GCS):
52+
case string(objstore.GCS):
6853
bucket, err = gcs.NewBucket(context.Background(), logger, config, component, wrapRoundtripper)
69-
case string(S3):
54+
case string(objstore.S3):
7055
bucket, err = s3.NewBucket(logger, config, component, wrapRoundtripper)
71-
case string(AZURE):
56+
case string(objstore.AZURE):
7257
bucket, err = azure.NewBucket(logger, config, component, wrapRoundtripper)
73-
case string(SWIFT):
58+
case string(objstore.SWIFT):
7459
bucket, err = swift.NewContainer(logger, config, wrapRoundtripper)
75-
case string(COS):
60+
case string(objstore.COS):
7661
bucket, err = cos.NewBucket(logger, config, component, wrapRoundtripper)
77-
case string(ALIYUNOSS):
62+
case string(objstore.ALIYUNOSS):
7863
bucket, err = oss.NewBucket(logger, config, component, wrapRoundtripper)
79-
case string(FILESYSTEM):
64+
case string(objstore.FILESYSTEM):
8065
bucket, err = filesystem.NewBucketFromConfig(config)
81-
case string(BOS):
66+
case string(objstore.BOS):
8267
bucket, err = bos.NewBucket(logger, config, component)
83-
case string(OCI):
68+
case string(objstore.OCI):
8469
bucket, err = oci.NewBucket(logger, config, wrapRoundtripper)
85-
case string(OBS):
70+
case string(objstore.OBS):
8671
bucket, err = obs.NewBucket(logger, config)
8772
default:
8873
return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type)

‎inmem.go

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ func NewInMemBucket() *InMemBucket {
3434
}
3535
}
3636

37+
func (b *InMemBucket) Provider() ObjProvider { return MEMORY }
38+
3739
// Objects returns a copy of the internally stored objects.
3840
// NOTE: For assert purposes.
3941
func (b *InMemBucket) Objects() map[string][]byte {

‎objstore.go

+22
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,22 @@ import (
2626
"golang.org/x/sync/errgroup"
2727
)
2828

29+
type ObjProvider string
30+
31+
const (
32+
MEMORY ObjProvider = "MEMORY"
33+
FILESYSTEM ObjProvider = "FILESYSTEM"
34+
GCS ObjProvider = "GCS"
35+
S3 ObjProvider = "S3"
36+
AZURE ObjProvider = "AZURE"
37+
SWIFT ObjProvider = "SWIFT"
38+
COS ObjProvider = "COS"
39+
ALIYUNOSS ObjProvider = "ALIYUNOSS"
40+
BOS ObjProvider = "BOS"
41+
OCI ObjProvider = "OCI"
42+
OBS ObjProvider = "OBS"
43+
)
44+
2945
const (
3046
OpIter = "iter"
3147
OpGet = "get"
@@ -42,6 +58,8 @@ type Bucket interface {
4258
io.Closer
4359
BucketReader
4460

61+
Provider() ObjProvider
62+
4563
// Upload the contents of the reader as an object into the bucket.
4664
// Upload should be idempotent.
4765
Upload(ctx context.Context, name string, r io.Reader) error
@@ -583,6 +601,10 @@ type metricBucket struct {
583601
metrics *Metrics
584602
}
585603

604+
func (b *metricBucket) Provider() ObjProvider {
605+
return b.bkt.Provider()
606+
}
607+
586608
func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket {
587609
return &metricBucket{
588610
bkt: b.bkt,

‎objtesting/foreach.go

+10-11
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"testing"
1010

1111
"github.com/thanos-io/objstore"
12-
"github.com/thanos-io/objstore/client"
1312
"github.com/thanos-io/objstore/providers/azure"
1413
"github.com/thanos-io/objstore/providers/bos"
1514
"github.com/thanos-io/objstore/providers/cos"
@@ -26,7 +25,7 @@ import (
2625

2726
// IsObjStoreSkipped returns true if given provider ID is found in THANOS_TEST_OBJSTORE_SKIP array delimited by comma e.g:
2827
// THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI.
29-
func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool {
28+
func IsObjStoreSkipped(t *testing.T, provider objstore.ObjProvider) bool {
3029
if e, ok := os.LookupEnv("THANOS_TEST_OBJSTORE_SKIP"); ok {
3130
obstores := strings.Split(e, ",")
3231
for _, objstore := range obstores {
@@ -69,7 +68,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
6968
})
7069

7170
// Optional GCS.
72-
if !IsObjStoreSkipped(t, client.GCS) {
71+
if !IsObjStoreSkipped(t, objstore.GCS) {
7372
t.Run("gcs", func(t *testing.T) {
7473
bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT"))
7574
testutil.Ok(t, err)
@@ -84,7 +83,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
8483
}
8584

8685
// Optional S3.
87-
if !IsObjStoreSkipped(t, client.S3) {
86+
if !IsObjStoreSkipped(t, objstore.S3) {
8887
t.Run("aws s3", func(t *testing.T) {
8988
// TODO(bwplotka): Allow taking location from envvar.
9089
bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2")
@@ -103,7 +102,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
103102
}
104103

105104
// Optional Azure.
106-
if !IsObjStoreSkipped(t, client.AZURE) {
105+
if !IsObjStoreSkipped(t, objstore.AZURE) {
107106
t.Run("azure", func(t *testing.T) {
108107
bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests")
109108
testutil.Ok(t, err)
@@ -117,7 +116,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
117116
}
118117

119118
// Optional SWIFT.
120-
if !IsObjStoreSkipped(t, client.SWIFT) {
119+
if !IsObjStoreSkipped(t, objstore.SWIFT) {
121120
t.Run("swift", func(t *testing.T) {
122121
container, closeFn, err := swift.NewTestContainer(t)
123122
testutil.Ok(t, err)
@@ -131,7 +130,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
131130
}
132131

133132
// Optional COS.
134-
if !IsObjStoreSkipped(t, client.COS) {
133+
if !IsObjStoreSkipped(t, objstore.COS) {
135134
t.Run("Tencent cos", func(t *testing.T) {
136135
bkt, closeFn, err := cos.NewTestBucket(t)
137136
testutil.Ok(t, err)
@@ -145,7 +144,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
145144
}
146145

147146
// Optional OSS.
148-
if !IsObjStoreSkipped(t, client.ALIYUNOSS) {
147+
if !IsObjStoreSkipped(t, objstore.ALIYUNOSS) {
149148
t.Run("AliYun oss", func(t *testing.T) {
150149
bkt, closeFn, err := oss.NewTestBucket(t)
151150
testutil.Ok(t, err)
@@ -159,7 +158,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
159158
}
160159

161160
// Optional BOS.
162-
if !IsObjStoreSkipped(t, client.BOS) {
161+
if !IsObjStoreSkipped(t, objstore.BOS) {
163162
t.Run("Baidu BOS", func(t *testing.T) {
164163
bkt, closeFn, err := bos.NewTestBucket(t)
165164
testutil.Ok(t, err)
@@ -173,7 +172,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
173172
}
174173

175174
// Optional OCI.
176-
if !IsObjStoreSkipped(t, client.OCI) {
175+
if !IsObjStoreSkipped(t, objstore.OCI) {
177176
t.Run("oci", func(t *testing.T) {
178177
bkt, closeFn, err := oci.NewTestBucket(t)
179178
testutil.Ok(t, err)
@@ -186,7 +185,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
186185
}
187186

188187
// Optional OBS.
189-
if !IsObjStoreSkipped(t, client.OBS) {
188+
if !IsObjStoreSkipped(t, objstore.OBS) {
190189
t.Run("obs", func(t *testing.T) {
191190
bkt, closeFn, err := obs.NewTestBucket(t, "cn-south-1")
192191
testutil.Ok(t, err)

‎prefixed_bucket.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ func withPrefix(prefix, name string) string {
3939
return prefix + DirDelim + name
4040
}
4141

42+
func (p *PrefixedBucket) Provider() ObjProvider { return p.bkt.Provider() }
43+
4244
func (p *PrefixedBucket) Close() error {
4345
return p.bkt.Close()
4446
}
@@ -93,7 +95,7 @@ func (p *PrefixedBucket) IsAccessDeniedErr(err error) bool {
9395
}
9496

9597
// Attributes returns information about the specified object.
96-
func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) {
98+
func (p *PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) {
9799
return p.bkt.Attributes(ctx, conditionalPrefix(p.prefix, name))
98100
}
99101

‎providers/azure/azure.go

+2
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,8 @@ func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapR
193193
return bkt, nil
194194
}
195195

196+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.AZURE }
197+
196198
func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType {
197199
return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt}
198200
}

‎providers/bos/bos.go

+2
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B
100100
return bkt, nil
101101
}
102102

103+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.BOS }
104+
103105
// Name returns the bucket name for the provider.
104106
func (b *Bucket) Name() string {
105107
return b.name

‎providers/cos/cos.go

+2
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra
154154
return bkt, nil
155155
}
156156

157+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.COS }
158+
157159
// Name returns the bucket name for COS.
158160
func (b *Bucket) Name() string {
159161
return b.name

‎providers/filesystem/filesystem.go

+2
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ func NewBucket(rootDir string) (*Bucket, error) {
5050
return &Bucket{rootDir: absDir}, nil
5151
}
5252

53+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.FILESYSTEM }
54+
5355
func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType {
5456
return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt}
5557
}

‎providers/gcs/gcs.go

+2
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,8 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option.
176176
return bkt, nil
177177
}
178178

179+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.GCS }
180+
179181
// Name returns the bucket name for gcs.
180182
func (b *Bucket) Name() string {
181183
return b.name

‎providers/obs/obs.go

+2
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) {
115115
return bkt, nil
116116
}
117117

118+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OBS }
119+
118120
// Name returns the bucket name for the provider.
119121
func (b *Bucket) Name() string {
120122
return b.name

‎providers/oci/oci.go

+2
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ type Bucket struct {
9696
requestMetadata common.RequestMetadata
9797
}
9898

99+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OCI }
100+
99101
// Name returns the bucket name for the provider.
100102
func (b *Bucket) Name() string {
101103
return b.name

‎providers/oss/oss.go

+2
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
6868
return NewTestBucketFromConfig(t, c, false)
6969
}
7070

71+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.ALIYUNOSS }
72+
7173
// Upload the contents of the reader as an object into the bucket.
7274
func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error {
7375
// TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this.

‎providers/s3/s3.go

+2
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra
343343
return bkt, nil
344344
}
345345

346+
func (b *Bucket) Provider() objstore.ObjProvider { return objstore.S3 }
347+
346348
// Name returns the bucket name for s3.
347349
func (b *Bucket) Name() string {
348350
return b.name

‎providers/swift/swift.go

+2
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,8 @@ func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool,
218218
}, nil
219219
}
220220

221+
func (c *Container) Provider() objstore.ObjProvider { return objstore.SWIFT }
222+
221223
// Name returns the container name for swift.
222224
func (c *Container) Name() string {
223225
return c.name

‎scripts/cfggen/main.go

+12-11
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package main
55

66
import (
77
"fmt"
8+
"github.com/thanos-io/objstore"
89
"io"
910
"os"
1011
"path/filepath"
@@ -35,17 +36,17 @@ var (
3536
configs map[string]interface{}
3637
possibleValues []string
3738

38-
bucketConfigs = map[client.ObjProvider]interface{}{
39-
client.AZURE: azure.Config{},
40-
client.GCS: gcs.Config{},
41-
client.S3: s3.DefaultConfig,
42-
client.SWIFT: swift.DefaultConfig,
43-
client.COS: cos.DefaultConfig,
44-
client.ALIYUNOSS: oss.Config{},
45-
client.FILESYSTEM: filesystem.Config{},
46-
client.BOS: bos.Config{},
47-
client.OCI: oci.Config{},
48-
client.OBS: obs.DefaultConfig,
39+
bucketConfigs = map[objstore.ObjProvider]interface{}{
40+
objstore.AZURE: azure.Config{},
41+
objstore.GCS: gcs.Config{},
42+
objstore.S3: s3.DefaultConfig,
43+
objstore.SWIFT: swift.DefaultConfig,
44+
objstore.COS: cos.DefaultConfig,
45+
objstore.ALIYUNOSS: oss.Config{},
46+
objstore.FILESYSTEM: filesystem.Config{},
47+
objstore.BOS: bos.Config{},
48+
objstore.OCI: oci.Config{},
49+
objstore.OBS: obs.DefaultConfig,
4950
}
5051
)
5152

‎testing.go

+2
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,8 @@ func WithDelay(bkt Bucket, delay time.Duration) Bucket {
280280
return &delayingBucket{bkt: bkt, delay: delay}
281281
}
282282

283+
func (d *delayingBucket) Provider() ObjProvider { return d.bkt.Provider() }
284+
283285
func (d *delayingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) {
284286
time.Sleep(d.delay)
285287
return d.bkt.Get(ctx, name)

0 commit comments

Comments
 (0)