[jaeger-v2] add GRPC storage backend integration test

.codecov.yml

@@ -11,6 +11,7 @@ ignore:
   - "thrift-gen/*/*"
   - "**/thrift-0.9.2/*"
   - "**/main.go"
+  - "cmd/jaeger/internal/integration/datareceivers"
   - "examples/hotrod"
 
 coverage:

.github/workflows/ci-grpc-badger.yml → .github/workflows/ci-badger.yaml

@@ -1,4 +1,4 @@
-name: CIT gRPC And Badger
+name: CIT Badger
 
 on:
   push:
@@ -16,7 +16,7 @@ permissions:  # added using https://github.com/step-security/secure-workflows
   contents: read
 
 jobs:
-  grpc-and-badger:
+  badger:
     runs-on: ubuntu-latest
     steps:
     - name: Harden Runner
@@ -33,17 +33,14 @@ jobs:
     - name: Run Badger storage integration tests
       run: make badger-storage-integration-test
 
-    - name: Run gRPC storage integration tests
-      run: make grpc-storage-integration-test
-
     - name: Setup CODECOV_TOKEN
       uses: ./.github/actions/setup-codecov
 
     - name: Upload coverage to codecov
       uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4.1.0
       with:
-        files: cover.out,cover-badger.out
+        files: cover.out
         verbose: true
-        flags: grpc-badger
+        flags: badger
         fail_ci_if_error: true
         token: ${{ env.CODECOV_TOKEN }}

.github/workflows/ci-grpc.yml

@@ -0,0 +1,57 @@
+name: CIT gRPC
+
+on:
+  push:
+    branches: [main]
+
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ (github.event.pull_request && github.event.pull_request.number) || github.ref || github.run_id }}
+  cancel-in-progress: true
+
+# See https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions
+permissions:  # added using https://github.com/step-security/secure-workflows
+  contents: read
+
+jobs:
+  grpc:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: [v1, v2]
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
+      with:
+        egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version: 1.22.x
+
+    - name: Run gRPC storage integration tests
+      run: |
+        case ${{ matrix.version }} in
+          v1)
+            make grpc-storage-integration-test
+            ;;
+          v2)
+            bash scripts/grpc-integration-test.sh latest
+            ;;
+        esac
+
+    - name: Setup CODECOV_TOKEN
+      uses: ./.github/actions/setup-codecov
+
+    - name: Upload coverage to codecov
+      uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4.1.0
+      with:
+        files: cover.out
+        verbose: true
+        flags: grpc
+        fail_ci_if_error: true
+        token: ${{ env.CODECOV_TOKEN }}

.gitignore

@@ -27,6 +27,7 @@ cmd/collector/collector
 cmd/collector/collector-*
 cmd/ingester/ingester
 cmd/ingester/ingester-*
+cmd/jaeger/internal/integration/results
 cmd/remote-storage/remote-storage
 cmd/remote-storage/remote-storage-*
 cmd/es-index-cleaner/es-index-cleaner-*

Makefile

@@ -4,6 +4,7 @@
 SHELL := /bin/bash
 JAEGER_IMPORT_PATH = github.com/jaegertracing/jaeger
 STORAGE_PKGS = ./plugin/storage/integration/...
+JAEGER_STORAGE_PKGS = ./cmd/jaeger/internal/integration
 
 # These DOCKER_xxx vars are used when building Docker images.
 DOCKER_NAMESPACE?=jaegertracing
@@ -118,9 +119,21 @@ storage-integration-test:
 	go clean -testcache
 	bash -c "set -e; set -o pipefail; $(GOTEST) -coverpkg=./... -coverprofile cover.out $(STORAGE_PKGS) $(COLORIZE)"
 
+# A general integration tests for jaeger-v2 storage backends,
+# these tests placed at `./cmd/jaeger/internal/integration/*_test.go`.
+# The integration tests are filtered by STORAGE env,
+# currently the available STORAGE variable is:
+#  - jaeger_grpc
+.PHONY: jaeger-storage-integration-test
+jaeger-storage-integration-test:
+	# Expire tests results for jaeger storage integration tests since the environment might change
+	# even though the code remains the same.
+	go clean -testcache
+	bash -c "set -e; set -o pipefail; $(GOTEST) -coverpkg=./... -coverprofile cover.out $(JAEGER_STORAGE_PKGS) $(COLORIZE)"
+
 .PHONY: badger-storage-integration-test
 badger-storage-integration-test:
-	bash -c "set -e; set -o pipefail; $(GOTEST) -tags=badger_storage_integration -coverpkg=./... -coverprofile cover-badger.out $(STORAGE_PKGS) $(COLORIZE)"
+	bash -c "set -e; set -o pipefail; $(GOTEST) -tags=badger_storage_integration -coverpkg=./... -coverprofile cover.out $(STORAGE_PKGS) $(COLORIZE)"
 
 .PHONY: grpc-storage-integration-test
 grpc-storage-integration-test:

cmd/jaeger/internal/command.go

@@ -29,7 +29,7 @@ func Command() *cobra.Command {
 
 	settings := otelcol.CollectorSettings{
 		BuildInfo: info,
-		Factories: components,
+		Factories: Components,
 	}
 
 	cmd := otelcol.NewCommand(settings)

cmd/jaeger/internal/components.go

@@ -116,6 +116,6 @@ func (b builders) build() (otelcol.Factories, error) {
 	return factories, nil
 }
 
-func components() (otelcol.Factories, error) {
+func Components() (otelcol.Factories, error) {
 	return defaultBuilders().build()
 }

cmd/jaeger/internal/components_test.go

@@ -29,7 +29,7 @@ import (
 )
 
 func TestComponents(t *testing.T) {
-	factories, err := components()
+	factories, err := Components()
 
 	require.NoError(t, err)
 

cmd/jaeger/internal/extension/jaegerstorage/extension_test.go

@@ -184,14 +184,14 @@ func TestESStorageExtensionError(t *testing.T) {
 	ext := makeStorageExtenion(t, &Config{
 		Elasticsearch: map[string]esCfg.Configuration{
 			"foo": {
-				Servers:  []string{"http://badurl"},
+				Servers:  []string{"http://127.0.0.1:65535"},
 				LogLevel: "error",
 			},
 		},
 	})
 	err := ext.Start(context.Background(), componenttest.NewNopHost())
 	require.ErrorContains(t, err, "failed to initialize elasticsearch storage")
-	require.ErrorContains(t, err, "badurl")
+	require.ErrorContains(t, err, "http://127.0.0.1:65535")
 }
 
 func noopTelemetrySettings() component.TelemetrySettings {

cmd/jaeger/internal/integration/datareceivers/.nocover

@@ -0,0 +1 @@
+A custom testbed data receiver for integration testing purpose

cmd/jaeger/internal/integration/datareceivers/jaegerstorage.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2024 The Jaeger Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package datareceivers
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/storagetest"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/testbed/testbed"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/extension"
+	"go.opentelemetry.io/collector/receiver"
+	"go.opentelemetry.io/collector/receiver/receivertest"
+
+	"github.com/jaegertracing/jaeger/cmd/jaeger/internal/extension/jaegerstorage"
+	"github.com/jaegertracing/jaeger/cmd/jaeger/internal/integration/receivers/storagereceiver"
+)
+
+type jaegerStorageDataReceiver struct {
+	TraceStorage  string
+	StorageConfig *jaegerstorage.Config
+	host          *storagetest.StorageHost
+	receiver      receiver.Traces
+}
+
+func NewJaegerStorageDataReceiver(traceStorage string, storageConfig *jaegerstorage.Config) testbed.DataReceiver {
+	return &jaegerStorageDataReceiver{
+		TraceStorage:  traceStorage,
+		StorageConfig: storageConfig,
+	}
+}
+
+func (dr *jaegerStorageDataReceiver) Start(tc consumer.Traces, _ consumer.Metrics, _ consumer.Logs) error {
+	ctx := context.Background()
+
+	extFactory := jaegerstorage.NewFactory()
+	ext, err := extFactory.CreateExtension(ctx, extension.CreateSettings{
+		TelemetrySettings: componenttest.NewNopTelemetrySettings(),
+		BuildInfo:         component.NewDefaultBuildInfo(),
+	}, dr.StorageConfig)
+	if err != nil {
+		return err
+	}
+
+	rcvSet := receivertest.NewNopCreateSettings()
+	rcvFactory := storagereceiver.NewFactory()
+	rcvCfg := rcvFactory.CreateDefaultConfig().(*storagereceiver.Config)
+	rcvCfg.TraceStorage = dr.TraceStorage
+	rcv, err := rcvFactory.CreateTracesReceiver(ctx, rcvSet, rcvCfg, tc)
+	if err != nil {
+		return err
+	}
+	dr.receiver = rcv
+
+	dr.host = storagetest.NewStorageHost()
+	dr.host.WithExtension(jaegerstorage.ID, ext)
+
+	err = dr.host.GetExtensions()[jaegerstorage.ID].Start(ctx, dr.host)
+	if err != nil {
+		return err
+	}
+	return dr.receiver.Start(ctx, dr.host)
+}
+
+func (dr *jaegerStorageDataReceiver) Stop() error {
+	ctx := context.Background()
+	err := dr.receiver.Shutdown(ctx)
+	if err != nil {
+		return err
+	}
+	return dr.host.GetExtensions()[jaegerstorage.ID].Shutdown(ctx)
+}
+
+func (dr *jaegerStorageDataReceiver) GenConfigYAMLStr() string {
+	return fmt.Sprintf(`
+  jaeger_storage_receiver:
+    trace_storage: %s
+`, dr.TraceStorage)
+}
+
+func (dr *jaegerStorageDataReceiver) ProtocolName() string {
+	return "jaeger_storage_receiver"
+}