Skip to content

Commit 5f04b8b

Browse files
fpetkovskifpetkovski
authored
Merge pull request #145 from ashwanthgoli/get-include-size
Include content length in the response of Get and GetRange
2 parents f90c89a + dfed39a commit 5f04b8b

File tree

14 files changed

+174
-33
lines changed

14 files changed

+174
-33
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
1010

1111
## Unreleased
1212
- [#38](https://github.com/thanos-io/objstore/pull/38) GCS: Upgrade cloud.google.com/go/storage version to `v1.43.0`.
13+
- [#145](https://github.com/thanos-io/objstore/pull/145) Include content length in the response of Get and GetRange.
1314

1415
### Fixed
1516
- [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents.

‎inmem.go

+27-5
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,12 @@ func (b *InMemBucket) Get(_ context.Context, name string) (io.ReadCloser, error)
119119
return nil, errNotFound
120120
}
121121

122-
return io.NopCloser(bytes.NewReader(file)), nil
122+
return ObjectSizerReadCloser{
123+
ReadCloser: io.NopCloser(bytes.NewReader(file)),
124+
Size: func() (int64, error) {
125+
return int64(len(file)), nil
126+
},
127+
}, nil
123128
}
124129

125130
// GetRange returns a new range reader for the given object name and range.
@@ -136,23 +141,40 @@ func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64
136141
}
137142

138143
if int64(len(file)) < off {
139-
return io.NopCloser(bytes.NewReader(nil)), nil
144+
return ObjectSizerReadCloser{
145+
ReadCloser: io.NopCloser(bytes.NewReader(nil)),
146+
Size: func() (int64, error) { return 0, nil },
147+
}, nil
140148
}
141149

142150
if length == -1 {
143-
return io.NopCloser(bytes.NewReader(file[off:])), nil
151+
return ObjectSizerReadCloser{
152+
ReadCloser: io.NopCloser(bytes.NewReader(file[off:])),
153+
Size: func() (int64, error) {
154+
return int64(len(file[off:])), nil
155+
},
156+
}, nil
144157
}
145158

146159
if length <= 0 {
147-
return io.NopCloser(bytes.NewReader(nil)), errors.New("length cannot be smaller or equal 0")
160+
// wrap with ObjectSizerReadCloser to return 0 size.
161+
return ObjectSizerReadCloser{
162+
ReadCloser: io.NopCloser(bytes.NewReader(nil)),
163+
Size: func() (int64, error) { return 0, nil },
164+
}, errors.New("length cannot be smaller or equal 0")
148165
}
149166

150167
if int64(len(file)) <= off+length {
151168
// Just return maximum of what we have.
152169
length = int64(len(file)) - off
153170
}
154171

155-
return io.NopCloser(bytes.NewReader(file[off : off+length])), nil
172+
return ObjectSizerReadCloser{
173+
ReadCloser: io.NopCloser(bytes.NewReader(file[off : off+length])),
174+
Size: func() (int64, error) {
175+
return length, nil
176+
},
177+
}, nil
156178
}
157179

158180
// Exists checks if the given directory exists in memory.

‎objstore.go

+14
Original file line numberDiff line numberDiff line change
@@ -829,3 +829,17 @@ func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) {
829829
t.timingReader.updateMetrics(int(n), err)
830830
return n, err
831831
}
832+
833+
type ObjectSizerReadCloser struct {
834+
io.ReadCloser
835+
Size func() (int64, error)
836+
}
837+
838+
// ObjectSize implement ObjectSizer.
839+
func (o ObjectSizerReadCloser) ObjectSize() (int64, error) {
840+
if o.Size == nil {
841+
return 0, errors.New("unknown size")
842+
}
843+
844+
return o.Size()
845+
}

‎providers/azure/azure.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,13 @@ func (b *Bucket) getBlobReader(ctx context.Context, name string, httpRange blob.
273273
return nil, errors.Wrapf(err, "cannot download blob, address: %s", blobClient.URL())
274274
}
275275
retryOpts := azblob.RetryReaderOptions{MaxRetries: int32(b.readerMaxRetries)}
276-
return resp.NewRetryReader(ctx, &retryOpts), nil
276+
277+
return objstore.ObjectSizerReadCloser{
278+
ReadCloser: resp.NewRetryReader(ctx, &retryOpts),
279+
Size: func() (int64, error) {
280+
return *resp.ContentLength, nil
281+
},
282+
}, nil
277283
}
278284

279285
// Get returns a reader for the given object name.

‎providers/bos/bos.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,12 @@ func (b *Bucket) getRange(_ context.Context, bucketName, objectKey string, off,
308308
return nil, err
309309
}
310310

311-
return obj.Body, nil
311+
return objstore.ObjectSizerReadCloser{
312+
ReadCloser: obj.Body,
313+
Size: func() (int64, error) {
314+
return obj.ContentLength, nil
315+
},
316+
}, err
312317
}
313318

314319
func configFromEnv() Config {

‎providers/cos/cos.go

+6-12
Original file line numberDiff line numberDiff line change
@@ -320,18 +320,12 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (
320320
return nil, err
321321
}
322322
// Add size info into reader to pass it to Upload function.
323-
r := objectSizerReadCloser{ReadCloser: resp.Body, size: resp.ContentLength}
324-
return r, nil
325-
}
326-
327-
type objectSizerReadCloser struct {
328-
io.ReadCloser
329-
size int64
330-
}
331-
332-
// ObjectSize implement objstore.ObjectSizer.
333-
func (o objectSizerReadCloser) ObjectSize() (int64, error) {
334-
return o.size, nil
323+
return objstore.ObjectSizerReadCloser{
324+
ReadCloser: resp.Body,
325+
Size: func() (int64, error) {
326+
return resp.ContentLength, nil
327+
},
328+
}, nil
335329
}
336330

337331
// Get returns a reader for the given object name.

‎providers/filesystem/filesystem.go

+24-5
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,12 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (
150150
return nil, errors.New("object name is empty")
151151
}
152152

153-
file := filepath.Join(b.rootDir, name)
154-
if _, err := os.Stat(file); err != nil {
153+
var (
154+
file = filepath.Join(b.rootDir, name)
155+
stat os.FileInfo
156+
err error
157+
)
158+
if stat, err = os.Stat(file); err != nil {
155159
return nil, errors.Wrapf(err, "stat %s", file)
156160
}
157161

@@ -160,18 +164,33 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (
160164
return nil, err
161165
}
162166

167+
var newOffset int64
163168
if off > 0 {
164-
_, err := f.Seek(off, 0)
169+
newOffset, err = f.Seek(off, 0)
165170
if err != nil {
166171
return nil, errors.Wrapf(err, "seek %v", off)
167172
}
168173
}
169174

175+
size := stat.Size() - newOffset
170176
if length == -1 {
171-
return f, nil
177+
return objstore.ObjectSizerReadCloser{
178+
ReadCloser: f,
179+
Size: func() (int64, error) {
180+
return size, nil
181+
},
182+
}, nil
172183
}
173184

174-
return &rangeReaderCloser{Reader: io.LimitReader(f, length), f: f}, nil
185+
return objstore.ObjectSizerReadCloser{
186+
ReadCloser: &rangeReaderCloser{
187+
Reader: io.LimitReader(f, length),
188+
f: f,
189+
},
190+
Size: func() (int64, error) {
191+
return min(length, size), nil
192+
},
193+
}, nil
175194
}
176195

177196
// Exists checks if the given directory exists in memory.

‎providers/gcs/gcs.go

+23-2
Original file line numberDiff line numberDiff line change
@@ -226,12 +226,33 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt
226226

227227
// Get returns a reader for the given object name.
228228
func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) {
229-
return b.bkt.Object(name).NewReader(ctx)
229+
r, err := b.bkt.Object(name).NewReader(ctx)
230+
if err != nil {
231+
return r, err
232+
}
233+
234+
return objstore.ObjectSizerReadCloser{
235+
ReadCloser: r,
236+
Size: func() (int64, error) {
237+
return r.Attrs.Size, nil
238+
},
239+
}, nil
230240
}
231241

232242
// GetRange returns a new range reader for the given object name and range.
233243
func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) {
234-
return b.bkt.Object(name).NewRangeReader(ctx, off, length)
244+
r, err := b.bkt.Object(name).NewRangeReader(ctx, off, length)
245+
if err != nil {
246+
return r, err
247+
}
248+
249+
sz := r.Remain()
250+
return objstore.ObjectSizerReadCloser{
251+
ReadCloser: r,
252+
Size: func() (int64, error) {
253+
return sz, nil
254+
},
255+
}, nil
235256
}
236257

237258
// Attributes returns information about the specified object.

‎providers/obs/obs.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,12 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io
299299
if err != nil {
300300
return nil, errors.Wrap(err, "failed to get object")
301301
}
302-
return output.Body, nil
302+
return objstore.ObjectSizerReadCloser{
303+
ReadCloser: output.Body,
304+
Size: func() (int64, error) {
305+
return output.ContentLength, nil
306+
},
307+
}, nil
303308
}
304309

305310
// Exists checks if the given object exists in the bucket.

‎providers/oci/oci.go

+11-2
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,12 @@ func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) {
134134
if err != nil {
135135
return nil, err
136136
}
137-
return response.Content, nil
137+
return objstore.ObjectSizerReadCloser{
138+
ReadCloser: response.Content,
139+
Size: func() (int64, error) {
140+
return *response.ContentLength, nil
141+
},
142+
}, nil
138143
}
139144

140145
// GetRange returns a new range reader for the given object name and range.
@@ -164,7 +169,11 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64
164169
if err != nil {
165170
return nil, err
166171
}
167-
return response.Content, nil
172+
return objstore.ObjectSizerReadCloser{ReadCloser: response.Content,
173+
Size: func() (int64, error) {
174+
return *response.ContentLength, nil
175+
},
176+
}, nil
168177
}
169178

170179
// Upload the contents of the reader as an object into the bucket.

‎providers/oss/oss.go

+13-2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"testing"
1717
"time"
1818

19+
"github.com/aliyun/aliyun-oss-go-sdk/oss"
1920
alioss "github.com/aliyun/aliyun-oss-go-sdk/oss"
2021
"github.com/go-kit/log"
2122
"github.com/pkg/errors"
@@ -342,12 +343,22 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io
342343
opts = append(opts, opt)
343344
}
344345

345-
resp, err := b.bucket.GetObject(name, opts...)
346+
resp, err := b.bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: name}, opts)
346347
if err != nil {
347348
return nil, err
348349
}
349350

350-
return resp, nil
351+
size, err := clientutil.ParseContentLength(resp.Response.Headers)
352+
if err == nil {
353+
return objstore.ObjectSizerReadCloser{
354+
ReadCloser: resp.Response,
355+
Size: func() (int64, error) {
356+
return size, nil
357+
},
358+
}, nil
359+
}
360+
361+
return resp.Response, nil
351362
}
352363

353364
// Get returns a reader for the given object name.

‎providers/s3/s3.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,17 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (
452452
return nil, err
453453
}
454454

455-
return r, nil
455+
return objstore.ObjectSizerReadCloser{
456+
ReadCloser: r,
457+
Size: func() (int64, error) {
458+
stat, err := r.Stat()
459+
if err != nil {
460+
return 0, err
461+
}
462+
463+
return stat.Size, nil
464+
},
465+
}, nil
456466
}
457467

458468
// Get returns a reader for the given object name.

‎providers/swift/swift.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,11 @@ func (c *Container) get(name string, headers swift.Headers, checkHash bool) (io.
262262
if err != nil {
263263
return nil, errors.Wrap(err, "open object")
264264
}
265-
return file, err
265+
266+
return objstore.ObjectSizerReadCloser{
267+
ReadCloser: file,
268+
Size: file.Length,
269+
}, nil
266270
}
267271

268272
// Get returns a reader for the given object name.

‎testing.go

+20
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) {
106106
rc1, err := bkt.Get(ctx, "id1/obj_1.some")
107107
testutil.Ok(t, err)
108108
defer func() { testutil.Ok(t, rc1.Close()) }()
109+
110+
sz, err := TryToGetSize(rc1)
111+
testutil.Ok(t, err)
112+
testutil.Equals(t, int64(11), sz, "expected size to be equal to 11")
113+
109114
content, err := io.ReadAll(rc1)
110115
testutil.Ok(t, err)
111116
testutil.Equals(t, "@test-data@", string(content))
@@ -118,6 +123,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) {
118123
rc2, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, 3)
119124
testutil.Ok(t, err)
120125
defer func() { testutil.Ok(t, rc2.Close()) }()
126+
127+
sz, err = TryToGetSize(rc2)
128+
testutil.Ok(t, err)
129+
testutil.Equals(t, int64(3), sz, "expected size to be equal to 3")
130+
121131
content, err = io.ReadAll(rc2)
122132
testutil.Ok(t, err)
123133
testutil.Equals(t, "tes", string(content))
@@ -126,6 +136,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) {
126136
rcUnspecifiedLen, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, -1)
127137
testutil.Ok(t, err)
128138
defer func() { testutil.Ok(t, rcUnspecifiedLen.Close()) }()
139+
140+
sz, err = TryToGetSize(rcUnspecifiedLen)
141+
testutil.Ok(t, err)
142+
testutil.Equals(t, int64(10), sz, "expected size to be equal to 10")
143+
129144
content, err = io.ReadAll(rcUnspecifiedLen)
130145
testutil.Ok(t, err)
131146
testutil.Equals(t, "test-data@", string(content))
@@ -141,6 +156,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) {
141156
rcLength, err := bkt.GetRange(ctx, "id1/obj_1.some", 3, 9999)
142157
testutil.Ok(t, err)
143158
defer func() { testutil.Ok(t, rcLength.Close()) }()
159+
160+
sz, err = TryToGetSize(rcLength)
161+
testutil.Ok(t, err)
162+
testutil.Equals(t, int64(8), sz, "expected size to be equal to 8")
163+
144164
content, err = io.ReadAll(rcLength)
145165
testutil.Ok(t, err)
146166
testutil.Equals(t, "st-data@", string(content))

0 commit comments

Comments
 (0)