Merge branch 'upstream/main' into enforce-query-tenancy

Signed-off-by: Jacob Baungard Hansen <[email protected]>
thanos-io · Nov 23, 2023 · 56dd9fb · 56dd9fb
2 parents b2043ad + d7ad77e
commit 56dd9fb
Show file tree

Hide file tree

Showing 70 changed files with 748 additions and 805 deletions.
diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk
@@ -35,12 +35,6 @@ $(FAILLINT): $(BINGO_DIR)/faillint.mod
 	@echo "(re)installing $(GOBIN)/faillint-v1.11.0"
 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=faillint.mod -o=$(GOBIN)/faillint-v1.11.0 "github.com/fatih/faillint"
 
-GO_BINDATA := $(GOBIN)/go-bindata-v3.1.1+incompatible
-$(GO_BINDATA): $(BINGO_DIR)/go-bindata.mod
-	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
-	@echo "(re)installing $(GOBIN)/go-bindata-v3.1.1+incompatible"
-	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=go-bindata.mod -o=$(GOBIN)/go-bindata-v3.1.1+incompatible "github.com/go-bindata/go-bindata/go-bindata"
-
 GOIMPORTS := $(GOBIN)/goimports-v0.12.0
 $(GOIMPORTS): $(BINGO_DIR)/goimports.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.

diff --git a/.bingo/go-bindata.mod b/.bingo/go-bindata.mod
diff --git a/.bingo/go-bindata.sum b/.bingo/go-bindata.sum
diff --git a/.bingo/variables.env b/.bingo/variables.env
@@ -14,8 +14,6 @@ BINGO="${GOBIN}/bingo-v0.8.1-0.20230820182247-0568407746a2"
 
 FAILLINT="${GOBIN}/faillint-v1.11.0"
 
-GO_BINDATA="${GOBIN}/go-bindata-v3.1.1+incompatible"
-
 GOIMPORTS="${GOBIN}/goimports-v0.12.0"
 
 GOJSONTOYAML="${GOBIN}/gojsontoyaml-v0.1.0"

diff --git a/.busybox-versions b/.busybox-versions
@@ -1,7 +1,7 @@
 # Auto generated by busybox-updater.sh. DO NOT EDIT
-amd64=9990834ed59b823ba7ea1374a4c4233f692926312ac3a1ae6b13b47be07d6934
-arm64=e96ea17e7b6a5fe22366931fe90dd5307bd6f5a3612ae14490d1b5f68b291476
-arm=a84fbdcadfd8715f11877dd3d963342e427a8718b1aa2d194115c85982cb33c5
-ppc64le=32a0ce3a915ec8b7b2ed4fb4e19a85d85af68384312fc9721b8f872ca03274a8
-riscv64=8bd68bf643b76d1b55788aee51399f6f2b910c5975dbad4ca9e624f88dd2e892
-s390x=b6863ec884fcce31cd1c7c3bb3bf6db1190ccc20845cee6dc034d82687e1b36c
+amd64=393d14abb68b8b2d88304c72ac25b5ce130aa3a1d57ba7363e2c4d07d294513d
+arm64=9fe410fe5b8f283d057939a5b0a6f464ecb4bfe4a07d132d2846cfbe82cf43ea
+arm=a237b18458d6bcc8964e59ced627ea46eb9aae68875ea833c61d5050a742e624
+ppc64le=cbb9892625fd0d4c625afe8255fe35699a163bc4d74925dfcca74ee7cc43d4ba
+riscv64=fa1350d80e4481d3671d808fbe239e4075205f69c940e7e85711bdc39bf8e181
+s390x=1e3e5a05847ad67da2b148d952931cf6f716a334ab06ea00742560a2ff985c7d
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,7 +2,7 @@
 version: 2.1
 
 orbs:
-  go: circleci/[email protected].2
+  go: circleci/[email protected].3
   git-shallow-clone: guitarrapc/[email protected]
 
 executors:
@@ -21,7 +21,13 @@ jobs:
       GO111MODULE: "on"
     steps:
       - git-shallow-clone/checkout
-      - go/mod-download-cached
+      - go/load-cache
+      - go/mod-download
+      - run:
+          name: Download bingo modules
+          command: |
+            make install-tool-deps
+      - go/save-cache
       - setup_remote_docker:
           version: 20.10.12
       - run:
@@ -35,6 +41,7 @@ jobs:
             fi
       - run:
           name: "Run unit tests."
+          no_output_timeout: "30m"
           environment:
             THANOS_TEST_OBJSTORE_SKIP: GCS,S3,AZURE,COS,ALIYUNOSS,BOS,OCI,OBS
             # Variables for Swift testing.

diff --git a/.dockerignore b/.dockerignore
@@ -24,7 +24,4 @@ website/public/
 website/docs-pre-processed/
 !website/data
 
-# React build assets
-pkg/ui/static/react
-
 tmp/
diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml
@@ -70,8 +70,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        parallelism: [4]
-        index: [0, 1, 2, 3]
+        parallelism: [8]
+        index: [0, 1, 2, 3, 4, 5, 6, 7]
     runs-on: ubuntu-latest
     name: Thanos end-to-end tests
     env:

diff --git a/.gitignore b/.gitignore
@@ -35,9 +35,6 @@ website/public/
 website/docs-pre-processed/
 !website/data
 
-# React build assets
-pkg/ui/static/react
-
 tmp/bin
 examples/tmp/
 

diff --git a/.mdox.validate.yaml b/.mdox.validate.yaml
@@ -37,4 +37,6 @@ validators:
     type: 'ignore'
   # Seems like improbable.io temporarily removed their blogs.
   - regex: 'improbable\.io'
-    type: 'ignore'
+    type: 'ignore'
+  - regex: 'krisztianfekete\.org'
+    type: 'ignore'
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,9 +12,12 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ### Fixed
 
+- [#6874](https://github.com/thanos-io/thanos/pull/6874) Sidecar: fix labels returned by 'api/v1/series' in presence of conflicting external and inner labels.
+
 ### Added
 
 - [#6756](https://github.com/thanos-io/thanos/pull/6756) Query: Add the following options to allow enforcement of tenancy on the query path: `query.enable-tenancy`, `query.tenant-label-name`.
+- [#6891](https://github.com/thanos-io/thanos/pull/6891) Objstore: Bump `objstore` which adds support for Azure Workload Identity.
 
 ### Changed
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -84,7 +84,6 @@ It's key to get familiarized with the style guide and mechanics of Thanos, espec
 * To get started, create a codespace for this repository by clicking this 👉 [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=109162639)
   * A codespace will open in a web-based version of Visual Studio Code. The [dev container](.devcontainer/devcontainer.json) is fully configured with software needed for this project.
   * **Note**: Dev containers is an open spec which is supported by [GitHub Codespaces](https://github.com/codespaces) and [other tools](https://containers.dev/supporting).
-* Spin up a prebuilt dev environment using Gitpod.io [![Gitpod Ready-to-Code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/thanos-io/thanos)
 * In case you want to develop the project locally, install **Golang** in your machine. Here is a nice [gist](https://gist.github.com/nikhita/432436d570b89cab172dcf2894465753) for this purpose.
 * You can run an interactive example, which populates some data as well, by following the steps mentioned [here](https://github.com/thanos-io/thanos/blob/main/tutorials/interactive-example/README.md).
 

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -1,19 +1,20 @@
 # Core Maintainers of this repository
 
-| Name                  | Email                     | Slack                    | GitHub                                           | Company             |
-|-----------------------|---------------------------|--------------------------|--------------------------------------------------|---------------------|
-| Bartłomiej Płotka     | [email protected]        | `@bwplotka`              | [@bwplotka](https://github.com/bwplotka)         | Red Hat             |
-| Frederic Branczyk     | [email protected]       | `@brancz`                | [@brancz](https://github.com/brancz)             | Polar Signals       |
-| Giedrius Statkevičius | [email protected]    | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)       | Vinted              |
-| Kemal Akkoyun         | [email protected]        | `@kakkoyun`              | [@kakkoyun](https://github.com/kakkoyun)         | Polar Signals       |
-| Lucas Servén Marín    | [email protected]         | `@squat`                 | [@squat](https://github.com/squat)               | Red Hat             |
-| Prem Saraswat         | [email protected]        | `@Prem Saraswat`         | [@onprem](https://github.com/onprem)             | Red Hat             |
-| Matthias Loibl        | [email protected]    | `@metalmatze`            | [@metalmatze](https://github.com/metalmatze)     | Polar Signals       |
-| Ben Ye                | [email protected]     | `@yeya24`                | [@yeya24](https://github.com/yeya24)             | Amazon Web Services |
-| Wiard van Rij         | [email protected]         | `@wiard van Rij`         | [@wiardvanrij](https://github.com/wiardvanrij)   | Roku                |
-| Matej Gera            | [email protected]       | `@Matej Gera`            | [@matej-g](https://github.com/matej-g)           | Red Hat             |
-| Filip Petkovski       | [email protected] | `@Filip Petkovski`       | [@fpetkovski](https://github.com/fpetkovski)     | Shopify             |
-| Saswata Mukherjee     | [email protected]   | `@saswatamcode`          | [@saswatamcode](https://github.com/saswatamcode) | Red Hat             |
+| Name                  | Email                     | Slack                    | GitHub                                             | Company             |
+|-----------------------|---------------------------|--------------------------|----------------------------------------------------|---------------------|
+| Bartłomiej Płotka     | [email protected]        | `@bwplotka`              | [@bwplotka](https://github.com/bwplotka)           | Red Hat             |
+| Frederic Branczyk     | [email protected]       | `@brancz`                | [@brancz](https://github.com/brancz)               | Polar Signals       |
+| Giedrius Statkevičius | [email protected]    | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)         | Vinted              |
+| Kemal Akkoyun         | [email protected]        | `@kakkoyun`              | [@kakkoyun](https://github.com/kakkoyun)           | Polar Signals       |
+| Lucas Servén Marín    | [email protected]         | `@squat`                 | [@squat](https://github.com/squat)                 | Red Hat             |
+| Prem Saraswat         | [email protected]        | `@Prem Saraswat`         | [@onprem](https://github.com/onprem)               | Red Hat             |
+| Matthias Loibl        | [email protected]    | `@metalmatze`            | [@metalmatze](https://github.com/metalmatze)       | Polar Signals       |
+| Ben Ye                | [email protected]     | `@yeya24`                | [@yeya24](https://github.com/yeya24)               | Amazon Web Services |
+| Wiard van Rij         | [email protected]         | `@wiard van Rij`         | [@wiardvanrij](https://github.com/wiardvanrij)     | Roku                |
+| Matej Gera            | [email protected]       | `@Matej Gera`            | [@matej-g](https://github.com/matej-g)             | Red Hat             |
+| Filip Petkovski       | [email protected] | `@Filip Petkovski`       | [@fpetkovski](https://github.com/fpetkovski)       | Shopify             |
+| Saswata Mukherjee     | [email protected]   | `@saswatamcode`          | [@saswatamcode](https://github.com/saswatamcode)   | Red Hat             |
+| Michael Hoffmann      | [email protected]          | `@Michael Hoffmann`      | [@MichaHoffmann](https://github.com/MichaHoffmann) | Aiven               |
 
 We are bunch of people from different companies with various interests and skills. We are from different parts of the world: Germany, Holland, Lithuania, US, UK and India. We have something in common though: We all share the love for OpenSource, Go, Prometheus, :coffee: and Observability topics.
 
@@ -31,16 +32,15 @@ We also have some nice souls that help triaging issues and PRs. See [here](https
 
 Full list of triage persons is displayed below:
 
-| Name             | Slack               | GitHub                                             | Company |
-|------------------|---------------------|----------------------------------------------------|---------|
-| Adrien Fillon    | `@Adrien F`         | [@adrien-f](https://github.com/adrien-f)           |         |
-| Ian Billett      | `@billett`          | [@bill3tt](https://github.com/bill3tt)             | Red Hat |
-| Martin Chodur    | `@FUSAKLA`          | [@fusakla](https://github.com/fusakla)             |         |
-| Michael Dai      | `@jojohappy`        | [@jojohappy](https://github.com/jojohappy)         |         |
-| Xiang Dai        | `@daixiang0`        | [@daixiang0](https://github.com/daixiang0)         |         |
-| Jimmie Han       | `@hanjm`            | [@hanjm](https://github.com/hanjm)                 | Tencent |
-| Michael Hoffmann | `@Michael Hoffmann` | [@MichaHoffmann](https://github.com/MichaHoffmann) | Aiven   |
-| Douglas Camata   | `@douglascamata`    | [@douglascamata](https://github.com/douglascamata) | Red Hat |
+| Name           | Slack            | GitHub                                             | Company |
+|----------------|------------------|----------------------------------------------------|---------|
+| Adrien Fillon  | `@Adrien F`      | [@adrien-f](https://github.com/adrien-f)           |         |
+| Ian Billett    | `@billett`       | [@bill3tt](https://github.com/bill3tt)             | Red Hat |
+| Martin Chodur  | `@FUSAKLA`       | [@fusakla](https://github.com/fusakla)             |         |
+| Michael Dai    | `@jojohappy`     | [@jojohappy](https://github.com/jojohappy)         |         |
+| Xiang Dai      | `@daixiang0`     | [@daixiang0](https://github.com/daixiang0)         |         |
+| Jimmie Han     | `@hanjm`         | [@hanjm](https://github.com/hanjm)                 | Tencent |
+| Douglas Camata | `@douglascamata` | [@douglascamata](https://github.com/douglascamata) | Red Hat |
 
 Please reach any of the maintainer on slack or email if you want to help as well.
 

diff --git a/Makefile b/Makefile
@@ -118,14 +118,8 @@ $(REACT_APP_OUTPUT_DIR): $(REACT_APP_NODE_MODULES_PATH) $(REACT_APP_SOURCE_FILES
 	   @echo ">> building React app"
 	   @scripts/build-react-app.sh
 
-.PHONY: assets
-assets: # Repacks all static assets into go file for easier deploy.
-assets: $(GO_BINDATA) $(REACT_APP_OUTPUT_DIR)
-	@echo ">> deleting asset file"
-	@rm pkg/ui/bindata.go || true
-	@echo ">> writing assets"
-	@$(GO_BINDATA) $(bindata_flags) -pkg ui -o pkg/ui/bindata.go  pkg/ui/static/...
-	@$(MAKE) format
+.PHONY: react-app
+react-app: $(REACT_APP_OUTPUT_DIR)
 
 .PHONY: react-app-lint
 react-app-lint: $(REACT_APP_NODE_MODULES_PATH)
@@ -314,7 +308,7 @@ test: export THANOS_TEST_ALERTMANAGER_PATH= $(ALERTMANAGER)
 test: check-git install-tool-deps
 	@echo ">> install thanos GOOPTS=${GOOPTS}"
 	@echo ">> running unit tests (without /test/e2e). Do export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,OBS if you want to skip e2e tests against all real store buckets. Current value: ${THANOS_TEST_OBJSTORE_SKIP}"
-	@go test -timeout 15m $(shell go list ./... | grep -v /vendor/ | grep -v /test/e2e);
+	@go test -race -timeout 15m $(shell go list ./... | grep -v /vendor/ | grep -v /test/e2e);
 
 .PHONY: test-local
 test-local: ## Runs test excluding tests for ALL  object storage integrations.

diff --git a/README.md b/README.md
@@ -4,6 +4,8 @@
 
 [![CI](https://github.com/thanos-io/thanos/workflows/CI/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3ACI) [![CI](https://circleci.com/gh/thanos-io/thanos.svg?style=svg)](https://circleci.com/gh/thanos-io/thanos) [![go](https://github.com/thanos-io/thanos/workflows/go/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3Ago) [![react](https://github.com/thanos-io/thanos/workflows/react/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3Areact) [![docs](https://github.com/thanos-io/thanos/workflows/docs/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3Adocs) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/thanos-io/thanos) [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=109162639)
 
+> 📢 [ThanosCon](https://thanos.io/blog/2023-20-11-thanoscon/) is happening on 19th March as a co-located half-day on KubeCon EU in Paris. Join us there! 🤗 CFP is open until 3rd December!
+
 ## Overview
 
 Thanos is a set of components that can be composed into a highly available metric system with unlimited storage capacity, which can be added seamlessly on top of existing Prometheus deployments.

diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go
@@ -234,6 +234,7 @@ func runCompact(
 	ignoreDeletionMarkFilter := block.NewIgnoreDeletionMarkFilter(logger, insBkt, deleteDelay/2, conf.blockMetaFetchConcurrency)
 	duplicateBlocksFilter := block.NewDeduplicateFilter(conf.blockMetaFetchConcurrency)
 	noCompactMarkerFilter := compact.NewGatherNoCompactionMarkFilter(logger, insBkt, conf.blockMetaFetchConcurrency)
+	noDownsampleMarkerFilter := downsample.NewGatherNoDownsampleMarkFilter(logger, insBkt, conf.blockMetaFetchConcurrency)
 	labelShardedMetaFilter := block.NewLabelShardedMetaFilter(relabelConfig)
 	consistencyDelayMetaFilter := block.NewConsistencyDelayMetaFilter(logger, conf.consistencyDelay, extprom.WrapRegistererWithPrefix("thanos_", reg))
 	timePartitionMetaFilter := block.NewTimePartitionMetaFilter(conf.filterConf.MinTime, conf.filterConf.MaxTime)
@@ -260,18 +261,21 @@ func runCompact(
 		sy  *compact.Syncer
 	)
 	{
+		filters := []block.MetadataFilter{
+			timePartitionMetaFilter,
+			labelShardedMetaFilter,
+			consistencyDelayMetaFilter,
+			ignoreDeletionMarkFilter,
+			block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels),
+			duplicateBlocksFilter,
+			noCompactMarkerFilter,
+		}
+		if !conf.disableDownsampling {
+			filters = append(filters, noDownsampleMarkerFilter)
+		}
 		// Make sure all compactor meta syncs are done through Syncer.SyncMeta for readability.
 		cf := baseMetaFetcher.NewMetaFetcher(
-			extprom.WrapRegistererWithPrefix("thanos_", reg), []block.MetadataFilter{
-				timePartitionMetaFilter,
-				labelShardedMetaFilter,
-				consistencyDelayMetaFilter,
-				ignoreDeletionMarkFilter,
-				block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels),
-				duplicateBlocksFilter,
-				noCompactMarkerFilter,
-			},
-		)
+			extprom.WrapRegistererWithPrefix("thanos_", reg), filters)
 		cf.UpdateOnChange(func(blocks []metadata.Meta, err error) {
 			api.SetLoaded(blocks, err)
 		})
@@ -436,22 +440,53 @@ func runCompact(
 				return errors.Wrap(err, "sync before first pass of downsampling")
 			}
 
-			for _, meta := range sy.Metas() {
+			filteredMetas := sy.Metas()
+			noDownsampleBlocks := noDownsampleMarkerFilter.NoDownsampleMarkedBlocks()
+			for ul := range noDownsampleBlocks {
+				delete(filteredMetas, ul)
+			}
+
+			for _, meta := range filteredMetas {
 				groupKey := meta.Thanos.GroupKey()
 				downsampleMetrics.downsamples.WithLabelValues(groupKey)
 				downsampleMetrics.downsampleFailures.WithLabelValues(groupKey)
 			}
-			if err := downsampleBucket(ctx, logger, downsampleMetrics, insBkt, sy.Metas(), downsamplingDir, conf.downsampleConcurrency, conf.blockFilesConcurrency, metadata.HashFunc(conf.hashFunc), conf.acceptMalformedIndex); err != nil {
+
+			if err := downsampleBucket(
+				ctx,
+				logger,
+				downsampleMetrics,
+				insBkt,
+				filteredMetas,
+				downsamplingDir,
+				conf.downsampleConcurrency,
+				conf.blockFilesConcurrency,
+				metadata.HashFunc(conf.hashFunc),
+				conf.acceptMalformedIndex,
+			); err != nil {
 				return errors.Wrap(err, "first pass of downsampling failed")
 			}
 
 			level.Info(logger).Log("msg", "start second pass of downsampling")
 			if err := sy.SyncMetas(ctx); err != nil {
 				return errors.Wrap(err, "sync before second pass of downsampling")
 			}
-			if err := downsampleBucket(ctx, logger, downsampleMetrics, insBkt, sy.Metas(), downsamplingDir, conf.downsampleConcurrency, conf.blockFilesConcurrency, metadata.HashFunc(conf.hashFunc), conf.acceptMalformedIndex); err != nil {
+
+			if err := downsampleBucket(
+				ctx,
+				logger,
+				downsampleMetrics,
+				insBkt,
+				filteredMetas,
+				downsamplingDir,
+				conf.downsampleConcurrency,
+				conf.blockFilesConcurrency,
+				metadata.HashFunc(conf.hashFunc),
+				conf.acceptMalformedIndex,
+			); err != nil {
 				return errors.Wrap(err, "second pass of downsampling failed")
 			}
+
 			level.Info(logger).Log("msg", "downsampling iterations done")
 		} else {
 			level.Info(logger).Log("msg", "downsampling was explicitly disabled")