From 9388c3f44769240b1399fa33723ad6009c5ca510 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:16:56 -0800 Subject: [PATCH 1/3] Updates busybox SHA (#6897) Signed-off-by: GitHub Co-authored-by: fpetkovski --- .busybox-versions | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.busybox-versions b/.busybox-versions index 23656d81d36..b925f4994f9 100644 --- a/.busybox-versions +++ b/.busybox-versions @@ -1,7 +1,7 @@ # Auto generated by busybox-updater.sh. DO NOT EDIT -amd64=9990834ed59b823ba7ea1374a4c4233f692926312ac3a1ae6b13b47be07d6934 -arm64=e96ea17e7b6a5fe22366931fe90dd5307bd6f5a3612ae14490d1b5f68b291476 -arm=a84fbdcadfd8715f11877dd3d963342e427a8718b1aa2d194115c85982cb33c5 -ppc64le=32a0ce3a915ec8b7b2ed4fb4e19a85d85af68384312fc9721b8f872ca03274a8 -riscv64=8bd68bf643b76d1b55788aee51399f6f2b910c5975dbad4ca9e624f88dd2e892 -s390x=b6863ec884fcce31cd1c7c3bb3bf6db1190ccc20845cee6dc034d82687e1b36c +amd64=393d14abb68b8b2d88304c72ac25b5ce130aa3a1d57ba7363e2c4d07d294513d +arm64=9fe410fe5b8f283d057939a5b0a6f464ecb4bfe4a07d132d2846cfbe82cf43ea +arm=a237b18458d6bcc8964e59ced627ea46eb9aae68875ea833c61d5050a742e624 +ppc64le=cbb9892625fd0d4c625afe8255fe35699a163bc4d74925dfcca74ee7cc43d4ba +riscv64=fa1350d80e4481d3671d808fbe239e4075205f69c940e7e85711bdc39bf8e181 +s390x=1e3e5a05847ad67da2b148d952931cf6f716a334ab06ea00742560a2ff985c7d From 3b87641ca0d778e189f6de446950d2d171b5676e Mon Sep 17 00:00:00 2001 From: Rikhil Shah <60136852+rikhil-s@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:40:24 +0000 Subject: [PATCH 2/3] Objstore: Bump Objstore for Azure Workload Identity support (#6891) * Bump objstore for Azure Workload Identity support Signed-off-by: Rikhil Shah * make check-docs Signed-off-by: Rikhil Shah * Add changelog entry Signed-off-by: Rikhil Shah * Update Azure client docs Signed-off-by: Rikhil Shah * make check-docs Signed-off-by: Rikhil Shah * Move changelog entry to 0.33.0 release Signed-off-by: Rikhil Shah * Move changelog entry Signed-off-by: Rikhil Shah --------- Signed-off-by: Rikhil Shah --- CHANGELOG.md | 2 ++ docs/storage.md | 13 +++++++++++-- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3140315ed89..ab281bdf4ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Added +- [#6891](https://github.com/thanos-io/thanos/pull/6891) Objstore: Bump `objstore` which adds support for Azure Workload Identity. + ### Changed ### Removed diff --git a/docs/storage.md b/docs/storage.md index f1f331cb407..25cc1e7dfd4 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -93,6 +93,7 @@ config: enable: false list_objects_version: "" bucket_lookup_type: auto + send_content_md5: true part_size: 67108864 sse_config: type: "" @@ -389,9 +390,17 @@ config: prefix: "" ``` -If `msi_resource` is used, authentication is done via system-assigned managed identity. The value for Azure should be `https://.blob.core.windows.net`. +If `storage_account_key` is used, authentication is done via storage account key. -If `user_assigned_id` is used, authentication is done via user-assigned managed identity. When using `user_assigned_id` the `msi_resource` defaults to `https://.` +If `user_assigned_id` is used, authentication is done via user-assigned managed identity. + +If `user_assigned_id` or `storage_account_key` is not passed, authentication is attempted with each of these credential types, in the following order, stopping when one provides a token: +- EnvironmentCredential +- WorkloadIdentityCredential +- ManagedIdentityCredential +- AzureCLICredential + +For the first three authentication types, the correct environment variables must be set for authentication to be successful. More information about the required environment variables for each authentication type can be found in the [Azure Identity Client Module for Go documentation](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity). The generic `max_retries` will be used as value for the `pipeline_config`'s `max_tries` and `reader_config`'s `max_retry_requests`. For more control, `max_retries` could be ignored (0) and one could set specific retry values. diff --git a/go.mod b/go.mod index 7c94f9a4f26..bcc0a7ce8f0 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/prometheus/prometheus v0.47.2-0.20231009162353-f6d9c84fde6b github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.8.4 - github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed + github.com/thanos-io/objstore v0.0.0-20231112185854-37752ee64d98 github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect @@ -107,8 +107,8 @@ require ( cloud.google.com/go v0.110.4 // indirect cloud.google.com/go/compute v1.22.0 // indirect cloud.google.com/go/iam v1.1.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect @@ -191,7 +191,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect diff --git a/go.sum b/go.sum index 3a45238946f..099ac6be2d9 100644 --- a/go.sum +++ b/go.sum @@ -69,10 +69,10 @@ cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXd dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh5k7k1LGIWLQfCjaneSj7Fc= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= @@ -522,8 +522,8 @@ github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= @@ -970,8 +970,8 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWjelVvKKSKgS7LeLkhK4VaEnQfphB9ZXA= -github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE= +github.com/thanos-io/objstore v0.0.0-20231112185854-37752ee64d98 h1:gx2MTto1UQRumGoJzY3aFPQ31Ov3nOV7NaD7j6q288k= +github.com/thanos-io/objstore v0.0.0-20231112185854-37752ee64d98/go.mod h1:JauBAcJ61tRSv9widgISVmA6akQXDeUMXBrVmWW4xog= github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591 h1:6bZbFM+Mvy2kL8BeL8TJ5+5pV3sUR2PSLaZyw911rtQ= github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591/go.mod h1:vfXJv1JXNdLfHnjsHsLLJl5tyI7KblF76Wo5lZ9YC4Q= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= From c1305d3bb3f569da33ded6502e232a21d5857e7b Mon Sep 17 00:00:00 2001 From: sinkingpoint Date: Fri, 10 Nov 2023 20:17:27 +1100 Subject: [PATCH 3/3] Allow customizing the shipper metadata file name Currently, the shipper metadata file is always called `thanos.shipper.json` in the Prometheus data directory. This precludes running multiple sidecars that upload to different object stores, as they will overwrite each other's metadata file. This commit allows the metadata file name to be customized via a flag. The default is unchanged, but it can be overridden with the `--shipper.meta-file-name` flag. As part of this, we update the signatures of `WriteMetaFile` and `ReadMetaFile` to take the full path of the metadata file, rather than just the directory, and updates the tests that go along with this. Signed-off-by: sinkingpoint --- cmd/thanos/config.go | 3 +++ cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- docs/components/rule.md | 2 ++ docs/components/sidecar.md | 2 ++ pkg/receive/multitsdb.go | 1 + pkg/shipper/shipper.go | 39 +++++++++++++++++++-------------- pkg/shipper/shipper_e2e_test.go | 18 +++++++-------- pkg/shipper/shipper_test.go | 25 +++++++++++---------- 9 files changed, 54 insertions(+), 40 deletions(-) diff --git a/cmd/thanos/config.go b/cmd/thanos/config.go index 51d0eed06f3..8ee36b333a3 100644 --- a/cmd/thanos/config.go +++ b/cmd/thanos/config.go @@ -15,6 +15,7 @@ import ( "github.com/prometheus/common/model" "github.com/thanos-io/thanos/pkg/extkingpin" + "github.com/thanos-io/thanos/pkg/shipper" ) type grpcConfig struct { @@ -140,6 +141,7 @@ type shipperConfig struct { ignoreBlockSize bool allowOutOfOrderUpload bool hashFunc string + metaFileName string } func (sc *shipperConfig) registerFlag(cmd extkingpin.FlagClause) *shipperConfig { @@ -156,6 +158,7 @@ func (sc *shipperConfig) registerFlag(cmd extkingpin.FlagClause) *shipperConfig Default("false").Hidden().BoolVar(&sc.allowOutOfOrderUpload) cmd.Flag("hash-func", "Specify which hash function to use when calculating the hashes of produced files. If no function has been specified, it does not happen. This permits avoiding downloading some files twice albeit at some performance cost. Possible values are: \"\", \"SHA256\"."). Default("").EnumVar(&sc.hashFunc, "SHA256", "") + cmd.Flag("shipper.meta-file-name", "the file to store shipper metadata in").Default(shipper.DefaultMetaFilename).StringVar(&sc.metaFileName) return sc } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index ceadf1159cd..e8f867347b7 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -745,7 +745,7 @@ func runRule( } }() - s := shipper.New(logger, reg, conf.dataDir, bkt, func() labels.Labels { return conf.lset }, metadata.RulerSource, nil, conf.shipper.allowOutOfOrderUpload, metadata.HashFunc(conf.shipper.hashFunc)) + s := shipper.New(logger, reg, conf.dataDir, bkt, func() labels.Labels { return conf.lset }, metadata.RulerSource, nil, conf.shipper.allowOutOfOrderUpload, metadata.HashFunc(conf.shipper.hashFunc), conf.shipper.metaFileName) ctx, cancel := context.WithCancel(context.Background()) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 27cf759b2ab..ab004791ef6 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -347,7 +347,7 @@ func runSidecar( uploadCompactedFunc := func() bool { return conf.shipper.uploadCompacted } s := shipper.New(logger, reg, conf.tsdb.path, bkt, m.Labels, metadata.SidecarSource, - uploadCompactedFunc, conf.shipper.allowOutOfOrderUpload, metadata.HashFunc(conf.shipper.hashFunc)) + uploadCompactedFunc, conf.shipper.allowOutOfOrderUpload, metadata.HashFunc(conf.shipper.hashFunc), conf.shipper.metaFileName) return runutil.Repeat(30*time.Second, ctx.Done(), func() error { if uploaded, err := s.Sync(ctx); err != nil { diff --git a/docs/components/rule.md b/docs/components/rule.md index a2dbdc72ec6..ec369ca82aa 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -445,6 +445,8 @@ Flags: Note that rules are not automatically detected, use SIGHUP or do HTTP POST /-/reload to re-read them. + --shipper.meta-file-name="thanos.shipper.json" + the file to store shipper metadata in --shipper.upload-compacted If true shipper will try to upload compacted blocks as well. Useful for migration purposes. diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 9bc309734a2..6dec56daeb0 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -172,6 +172,8 @@ Flags: Path to YAML file with request logging configuration. See format details: https://thanos.io/tip/thanos/logging.md/#configuration + --shipper.meta-file-name="thanos.shipper.json" + the file to store shipper metadata in --shipper.upload-compacted If true shipper will try to upload compacted blocks as well. Useful for migration purposes. diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index 7b68e8b0b00..3583ce8f3ac 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -607,6 +607,7 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant nil, t.allowOutOfOrderUpload, t.hashFunc, + shipper.DefaultMetaFilename, ) } tenant.set(store.NewTSDBStore(logger, s, component.Receive, lset), s, ship, exemplars.NewTSDB(s, lset)) diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index 69e84b963bf..a3b520d15ec 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -69,11 +69,12 @@ func newMetrics(reg prometheus.Registerer) *metrics { // Shipper watches a directory for matching files and directories and uploads // them to a remote data store. type Shipper struct { - logger log.Logger - dir string - metrics *metrics - bucket objstore.Bucket - source metadata.SourceType + logger log.Logger + dir string + metrics *metrics + bucket objstore.Bucket + source metadata.SourceType + metadataFilePath string uploadCompactedFunc func() bool allowOutOfOrderUploads bool @@ -96,6 +97,7 @@ func New( uploadCompactedFunc func() bool, allowOutOfOrderUploads bool, hashFunc metadata.HashFunc, + metaFileName string, ) *Shipper { if logger == nil { logger = log.NewNopLogger() @@ -104,6 +106,10 @@ func New( lbls = func() labels.Labels { return labels.EmptyLabels() } } + if metaFileName == "" { + metaFileName = DefaultMetaFilename + } + if uploadCompactedFunc == nil { uploadCompactedFunc = func() bool { return false @@ -119,6 +125,7 @@ func New( allowOutOfOrderUploads: allowOutOfOrderUploads, uploadCompactedFunc: uploadCompactedFunc, hashFunc: hashFunc, + metadataFilePath: filepath.Join(dir, filepath.Clean(metaFileName)), } } @@ -139,7 +146,7 @@ func (s *Shipper) getLabels() labels.Labels { // Timestamps returns the minimum timestamp for which data is available and the highest timestamp // of blocks that were successfully uploaded. func (s *Shipper) Timestamps() (minTime, maxSyncTime int64, err error) { - meta, err := ReadMetaFile(s.dir) + meta, err := ReadMetaFile(s.metadataFilePath) if err != nil { return 0, 0, errors.Wrap(err, "read shipper meta file") } @@ -247,7 +254,7 @@ func (c *lazyOverlapChecker) IsOverlapping(ctx context.Context, newMeta tsdb.Blo // // It is not concurrency-safe, however it is compactor-safe (running concurrently with compactor is ok). func (s *Shipper) Sync(ctx context.Context) (uploaded int, err error) { - meta, err := ReadMetaFile(s.dir) + meta, err := ReadMetaFile(s.metadataFilePath) if err != nil { // If we encounter any error, proceed with an empty meta file and overwrite it later. // The meta file is only used to avoid unnecessary bucket.Exists call, @@ -330,7 +337,7 @@ func (s *Shipper) Sync(ctx context.Context) (uploaded int, err error) { uploaded++ s.metrics.uploads.Inc() } - if err := WriteMetaFile(s.logger, s.dir, meta); err != nil { + if err := WriteMetaFile(s.logger, s.metadataFilePath, meta); err != nil { level.Warn(s.logger).Log("msg", "updating meta file failed", "err", err) } @@ -457,17 +464,16 @@ type Meta struct { } const ( - // MetaFilename is the known JSON filename for meta information. - MetaFilename = "thanos.shipper.json" + // DefaultMetaFilename is the default JSON filename for meta information. + DefaultMetaFilename = "thanos.shipper.json" // MetaVersion1 represents 1 version of meta. MetaVersion1 = 1 ) // WriteMetaFile writes the given meta into /thanos.shipper.json. -func WriteMetaFile(logger log.Logger, dir string, meta *Meta) error { +func WriteMetaFile(logger log.Logger, path string, meta *Meta) error { // Make any changes to the file appear atomic. - path := filepath.Join(dir, MetaFilename) tmp := path + ".tmp" f, err := os.Create(tmp) @@ -489,16 +495,15 @@ func WriteMetaFile(logger log.Logger, dir string, meta *Meta) error { } // ReadMetaFile reads the given meta from /thanos.shipper.json. -func ReadMetaFile(dir string) (*Meta, error) { - fpath := filepath.Join(dir, filepath.Clean(MetaFilename)) - b, err := os.ReadFile(fpath) +func ReadMetaFile(path string) (*Meta, error) { + b, err := os.ReadFile(path) if err != nil { - return nil, errors.Wrapf(err, "failed to read %s", fpath) + return nil, errors.Wrapf(err, "failed to read %s", path) } var m Meta if err := json.Unmarshal(b, &m); err != nil { - return nil, errors.Wrapf(err, "failed to parse %s as JSON: %q", fpath, string(b)) + return nil, errors.Wrapf(err, "failed to parse %s as JSON: %q", path, string(b)) } if m.Version != MetaVersion1 { return nil, errors.Errorf("unexpected meta file version %d", m.Version) diff --git a/pkg/shipper/shipper_e2e_test.go b/pkg/shipper/shipper_e2e_test.go index 5a9520ff430..ddb963339c5 100644 --- a/pkg/shipper/shipper_e2e_test.go +++ b/pkg/shipper/shipper_e2e_test.go @@ -44,7 +44,7 @@ func TestShipper_SyncBlocks_e2e(t *testing.T) { dir := t.TempDir() extLset := labels.FromStrings("prometheus", "prom-1") - shipper := New(log.NewLogfmtLogger(os.Stderr), nil, dir, metricsBucket, func() labels.Labels { return extLset }, metadata.TestSource, nil, false, metadata.NoneFunc) + shipper := New(log.NewLogfmtLogger(os.Stderr), nil, dir, metricsBucket, func() labels.Labels { return extLset }, metadata.TestSource, nil, false, metadata.NoneFunc, DefaultMetaFilename) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -101,7 +101,7 @@ func TestShipper_SyncBlocks_e2e(t *testing.T) { testutil.Ok(t, err) testutil.Equals(t, 0, b) - shipMeta, err := ReadMetaFile(dir) + shipMeta, err := ReadMetaFile(shipper.metadataFilePath) testutil.Ok(t, err) if len(shipMeta.Uploaded) == 0 { shipMeta.Uploaded = []ulid.ULID{} @@ -164,7 +164,7 @@ func TestShipper_SyncBlocks_e2e(t *testing.T) { testutil.Ok(t, block.Delete(ctx, log.NewNopLogger(), bkt, ids[4])) } // The shipper meta file should show all blocks as uploaded except the compacted one. - shipMeta, err = ReadMetaFile(dir) + shipMeta, err = ReadMetaFile(shipper.metadataFilePath) testutil.Ok(t, err) testutil.Equals(t, &Meta{Version: MetaVersion1, Uploaded: ids}, shipMeta) @@ -212,7 +212,7 @@ func TestShipper_SyncBlocksWithMigrating_e2e(t *testing.T) { testutil.Ok(t, p.Restart(context.Background(), logger)) uploadCompactedFunc := func() bool { return true } - shipper := New(log.NewLogfmtLogger(os.Stderr), nil, dir, bkt, func() labels.Labels { return extLset }, metadata.TestSource, uploadCompactedFunc, false, metadata.NoneFunc) + shipper := New(log.NewLogfmtLogger(os.Stderr), nil, dir, bkt, func() labels.Labels { return extLset }, metadata.TestSource, uploadCompactedFunc, false, metadata.NoneFunc, DefaultMetaFilename) // Create 10 new blocks. 9 of them (non compacted) should be actually uploaded. var ( @@ -265,7 +265,7 @@ func TestShipper_SyncBlocksWithMigrating_e2e(t *testing.T) { testutil.Ok(t, err) testutil.Equals(t, 0, b) - shipMeta, err := ReadMetaFile(dir) + shipMeta, err := ReadMetaFile(shipper.metadataFilePath) testutil.Ok(t, err) if len(shipMeta.Uploaded) == 0 { shipMeta.Uploaded = []ulid.ULID{} @@ -310,7 +310,7 @@ func TestShipper_SyncBlocksWithMigrating_e2e(t *testing.T) { testutil.Ok(t, block.Delete(ctx, log.NewNopLogger(), bkt, ids[4])) } // The shipper meta file should show all blocks as uploaded except the compacted one. - shipMeta, err = ReadMetaFile(dir) + shipMeta, err = ReadMetaFile(shipper.metadataFilePath) testutil.Ok(t, err) testutil.Equals(t, &Meta{Version: MetaVersion1, Uploaded: ids}, shipMeta) @@ -361,7 +361,7 @@ func TestShipper_SyncOverlapBlocks_e2e(t *testing.T) { uploadCompactedFunc := func() bool { return true } // Here, the allowOutOfOrderUploads flag is set to true, which allows blocks with overlaps to be uploaded. - shipper := New(log.NewLogfmtLogger(os.Stderr), nil, dir, bkt, func() labels.Labels { return extLset }, metadata.TestSource, uploadCompactedFunc, true, metadata.NoneFunc) + shipper := New(log.NewLogfmtLogger(os.Stderr), nil, dir, bkt, func() labels.Labels { return extLset }, metadata.TestSource, uploadCompactedFunc, true, metadata.NoneFunc, DefaultMetaFilename) // Creating 2 overlapping blocks - both uploaded when OOO uploads allowed. var ( @@ -421,7 +421,7 @@ func TestShipper_SyncOverlapBlocks_e2e(t *testing.T) { testutil.Ok(t, err) testutil.Equals(t, 0, b) - shipMeta, err := ReadMetaFile(dir) + shipMeta, err := ReadMetaFile(shipper.metadataFilePath) testutil.Ok(t, err) if len(shipMeta.Uploaded) == 0 { shipMeta.Uploaded = []ulid.ULID{} @@ -460,7 +460,7 @@ func TestShipper_SyncOverlapBlocks_e2e(t *testing.T) { expFiles[id[i].String()+"/chunks/0002"] = []byte("chunkcontents2") // The shipper meta file should show all blocks as uploaded except the compacted one. - shipMeta, err = ReadMetaFile(dir) + shipMeta, err = ReadMetaFile(shipper.metadataFilePath) testutil.Ok(t, err) testutil.Equals(t, &Meta{Version: MetaVersion1, Uploaded: ids}, shipMeta) } diff --git a/pkg/shipper/shipper_test.go b/pkg/shipper/shipper_test.go index b705f1649a6..64daa44a211 100644 --- a/pkg/shipper/shipper_test.go +++ b/pkg/shipper/shipper_test.go @@ -29,14 +29,14 @@ import ( func TestShipperTimestamps(t *testing.T) { dir := t.TempDir() - s := New(nil, nil, dir, nil, nil, metadata.TestSource, nil, false, metadata.NoneFunc) + s := New(nil, nil, dir, nil, nil, metadata.TestSource, nil, false, metadata.NoneFunc, DefaultMetaFilename) // Missing thanos meta file. _, _, err := s.Timestamps() testutil.NotOk(t, err) meta := &Meta{Version: MetaVersion1} - testutil.Ok(t, WriteMetaFile(log.NewNopLogger(), dir, meta)) + testutil.Ok(t, WriteMetaFile(log.NewNopLogger(), s.metadataFilePath, meta)) // Nothing uploaded, nothing in the filesystem. We assume that // we are still waiting for TSDB to dump first TSDB block. @@ -79,7 +79,7 @@ func TestShipperTimestamps(t *testing.T) { Version: MetaVersion1, Uploaded: []ulid.ULID{id1}, } - testutil.Ok(t, WriteMetaFile(log.NewNopLogger(), dir, meta)) + testutil.Ok(t, WriteMetaFile(log.NewNopLogger(), s.metadataFilePath, meta)) mint, maxt, err = s.Timestamps() testutil.Ok(t, err) testutil.Equals(t, int64(1000), mint) @@ -122,7 +122,7 @@ func TestIterBlockMetas(t *testing.T) { }, }.WriteToDir(log.NewNopLogger(), path.Join(dir, id3.String()))) - shipper := New(nil, nil, dir, nil, nil, metadata.TestSource, nil, false, metadata.NoneFunc) + shipper := New(nil, nil, dir, nil, nil, metadata.TestSource, nil, false, metadata.NoneFunc, DefaultMetaFilename) metas, err := shipper.blockMetasFromOldest() testutil.Ok(t, err) testutil.Equals(t, sort.SliceIsSorted(metas, func(i, j int) bool { @@ -153,7 +153,7 @@ func BenchmarkIterBlockMetas(b *testing.B) { }) b.ResetTimer() - shipper := New(nil, nil, dir, nil, nil, metadata.TestSource, nil, false, metadata.NoneFunc) + shipper := New(nil, nil, dir, nil, nil, metadata.TestSource, nil, false, metadata.NoneFunc, DefaultMetaFilename) _, err := shipper.blockMetasFromOldest() testutil.Ok(b, err) @@ -165,7 +165,7 @@ func TestShipperAddsSegmentFiles(t *testing.T) { inmemory := objstore.NewInMemBucket() lbls := labels.FromStrings("test", "test") - s := New(nil, nil, dir, inmemory, func() labels.Labels { return lbls }, metadata.TestSource, nil, false, metadata.NoneFunc) + s := New(nil, nil, dir, inmemory, func() labels.Labels { return lbls }, metadata.TestSource, nil, false, metadata.NoneFunc, DefaultMetaFilename) id := ulid.MustNew(1, nil) blockDir := path.Join(dir, id.String()) @@ -202,28 +202,29 @@ func TestReadMetaFile(t *testing.T) { t.Run("Missing meta file", func(t *testing.T) { // Create TSDB directory without meta file dpath := t.TempDir() + fpath := filepath.Join(dpath, DefaultMetaFilename) - _, err := ReadMetaFile(dpath) - fpath := filepath.Join(dpath, MetaFilename) + _, err := ReadMetaFile(fpath) testutil.Equals(t, fmt.Sprintf(`failed to read %s: open %s: no such file or directory`, fpath, fpath), err.Error()) }) t.Run("Non-JSON meta file", func(t *testing.T) { dpath := t.TempDir() - fpath := filepath.Join(dpath, MetaFilename) + fpath := filepath.Join(dpath, DefaultMetaFilename) + // Make an invalid JSON file testutil.Ok(t, os.WriteFile(fpath, []byte("{"), 0600)) - _, err := ReadMetaFile(dpath) + _, err := ReadMetaFile(fpath) testutil.Equals(t, fmt.Sprintf(`failed to parse %s as JSON: "{": unexpected end of JSON input`, fpath), err.Error()) }) t.Run("Wrongly versioned meta file", func(t *testing.T) { dpath := t.TempDir() - fpath := filepath.Join(dpath, MetaFilename) + fpath := filepath.Join(dpath, DefaultMetaFilename) testutil.Ok(t, os.WriteFile(fpath, []byte(`{"version": 2}`), 0600)) - _, err := ReadMetaFile(dpath) + _, err := ReadMetaFile(fpath) testutil.Equals(t, "unexpected meta file version 2", err.Error()) }) }