diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml index e56fdb154a6..83d98aa66ca 100644 --- a/google-cloud-spanner/pom.xml +++ b/google-cloud-spanner/pom.xml @@ -17,9 +17,9 @@ google-cloud-spanner 0.31.1 com.google.cloud.spanner.GceTestEnvConfig - projects/gcloud-devel/instances/spanner-testing-east1 - gcloud-devel - projects/gcloud-devel/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key + projects/span-cloud-testing/instances/surbhi-testing + span-cloud-testing + projects/span-cloud-testing/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key @@ -431,7 +431,6 @@ io.opentelemetry opentelemetry-sdk - test io.opentelemetry diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java index 179eafcf53c..c3619e72023 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java @@ -16,8 +16,10 @@ package com.google.cloud.spanner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.metrics.Aggregation; import java.util.Set; import java.util.stream.Collectors; @@ -79,4 +81,12 @@ public class BuiltInMetricsConstant { CLIENT_NAME_KEY, DIRECT_PATH_ENABLED_KEY, DIRECT_PATH_USED_KEY); + + public static Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM = + Aggregation.explicitBucketHistogram( + ImmutableList.of( + 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 8.0, 10.0, 13.0, 16.0, 20.0, 25.0, 30.0, 40.0, + 50.0, 65.0, 80.0, 100.0, 130.0, 160.0, 200.0, 250.0, 300.0, 400.0, 500.0, 650.0, + 800.0, 1000.0, 2000.0, 5000.0, 10000.0, 20000.0, 50000.0, 100000.0, 200000.0, + 400000.0, 800000.0, 1600000.0, 3200000.0)); } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java new file mode 100644 index 00000000000..b9201cf70c1 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java @@ -0,0 +1,185 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_NAME_KEY; +import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_UID_KEY; +import static com.google.cloud.spanner.BuiltInMetricsConstant.DIRECT_PATH_ENABLED_KEY; +import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_CONFIG_ID_KEY; +import static com.google.cloud.spanner.BuiltInMetricsConstant.LOCATION_ID_KEY; +import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY; + +import com.google.api.gax.core.GaxProperties; +import com.google.auth.Credentials; +import com.google.cloud.opentelemetry.detection.DetectedPlatform; +import com.google.cloud.opentelemetry.detection.GCPPlatformDetector; +import com.google.common.collect.ImmutableSet; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.Aggregation; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.View; +import io.opentelemetry.sdk.metrics.export.MetricExporter; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +class BuiltInOpenTelemetryMetricsProvider { + + private static final Logger logger = + Logger.getLogger(BuiltInOpenTelemetryMetricsProvider.class.getName()); + + private OpenTelemetry openTelemetry; + + public OpenTelemetry getOpenTelemetry(String projectId, @Nullable Credentials credentials) { + if (this.openTelemetry == null) { + + // Use custom exporter + MetricExporter metricExporter = null; + try { + metricExporter = SpannerCloudMonitoringExporter.create(projectId, credentials); + + SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder(); + registerView( + sdkMeterProviderBuilder, + BuiltInMetricsConstant.OPERATION_LATENCY_NAME, + BuiltInMetricsConstant.OPERATION_LATENCIES_NAME, + BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, + InstrumentType.HISTOGRAM, + "ms"); + registerView( + sdkMeterProviderBuilder, + BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME, + BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME, + BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, + InstrumentType.HISTOGRAM, + "ms"); + registerView( + sdkMeterProviderBuilder, + BuiltInMetricsConstant.OPERATION_COUNT_NAME, + BuiltInMetricsConstant.OPERATION_COUNT_NAME, + Aggregation.sum(), + InstrumentType.COUNTER, + "1"); + registerView( + sdkMeterProviderBuilder, + BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, + BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, + Aggregation.sum(), + InstrumentType.COUNTER, + "1"); + + SdkMeterProvider sdkMeterProvider = + sdkMeterProviderBuilder + .registerMetricReader(PeriodicMetricReader.create(metricExporter)) + .build(); + + this.openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build(); + } catch (IOException e) { + logger.log( + Level.WARNING, + "Unable to get OpenTelemetry object for client side metrics, will skip exporting client side metrics", + e); + } + } + return this.openTelemetry; + } + + public Map getClientAttributes(String projectId) { + Map clientAttributes = new HashMap<>(); + clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation()); + clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId); + clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "us-central1"); + clientAttributes.put(DIRECT_PATH_ENABLED_KEY.getKey(), "true"); + clientAttributes.put( + CLIENT_NAME_KEY.getKey(), + "spanner-java/" + + GaxProperties.getLibraryVersion(SpannerCloudMonitoringExporterUtils.class)); + clientAttributes.put(CLIENT_UID_KEY.getKey(), getDefaultTaskValue()); + return clientAttributes; + } + + private void registerView( + SdkMeterProviderBuilder sdkMeterProviderBuilder, String metricName, String metricViewName, + Aggregation aggregation, + InstrumentType type, + String unit) { + InstrumentSelector selector = + InstrumentSelector.builder() + .setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName) + .setMeterName(BuiltInMetricsConstant.GAX_METER_NAME) + .setType(type) + .setUnit(unit) + .build(); + Set attributesFilter = + ImmutableSet.builder() + .addAll( + BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream() + .map(AttributeKey::getKey) + .collect(Collectors.toSet())) + .build(); + View view = + View.builder() + .setName(BuiltInMetricsConstant.METER_NAME + '/' + metricViewName) + .setAggregation(aggregation) + .setAttributeFilter(attributesFilter) + .build(); + sdkMeterProviderBuilder.registerView(selector, view); + } + + private String detectClientLocation() { + GCPPlatformDetector detector = GCPPlatformDetector.DEFAULT_INSTANCE; + DetectedPlatform detectedPlatform = detector.detectPlatform(); + String region = detectedPlatform.getAttributes().get("cloud.region"); + return region; + } + + /** + * In most cases this should look like ${UUID}@${hostname}. The hostname will be retrieved from + * the jvm name and fallback to the local hostname. + */ + private String getDefaultTaskValue() { + // Something like '@' + final String jvmName = ManagementFactory.getRuntimeMXBean().getName(); + // If jvm doesn't have the expected format, fallback to the local hostname + if (jvmName.indexOf('@') < 1) { + String hostname = "localhost"; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + logger.log(Level.INFO, "Unable to get the hostname.", e); + } + // Generate a random number and use the same format "random_number@hostname". + return UUID.randomUUID() + "@" + hostname; + } + return UUID.randomUUID() + jvmName; + } +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index e6f60090acb..a9f7d048109 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -30,6 +30,8 @@ import com.google.api.gax.rpc.TransportChannelProvider; import com.google.api.gax.tracing.ApiTracerFactory; import com.google.api.gax.tracing.BaseApiTracerFactory; +import com.google.api.gax.tracing.MetricsTracerFactory; +import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder; import com.google.api.gax.tracing.OpencensusTracerFactory; import com.google.cloud.NoCredentials; import com.google.cloud.ServiceDefaults; @@ -157,7 +159,9 @@ public class SpannerOptions extends ServiceOptions { private final boolean useVirtualThreads; private final OpenTelemetry openTelemetry; private final boolean enableApiTracing; + private final boolean enableBuiltInMetrics; private final boolean enableExtendedTracing; + private OpenTelemetry builtInOpenTelemetry; enum TracingFramework { OPEN_CENSUS, @@ -664,6 +668,7 @@ protected SpannerOptions(Builder builder) { openTelemetry = builder.openTelemetry; enableApiTracing = builder.enableApiTracing; enableExtendedTracing = builder.enableExtendedTracing; + enableBuiltInMetrics = builder.enableBuiltInMetrics; } /** @@ -696,6 +701,10 @@ default boolean isEnableExtendedTracing() { default boolean isEnableApiTracing() { return false; } + + default boolean isEnableBuiltInMetrics() { + return true; + } } /** @@ -709,6 +718,7 @@ private static class SpannerEnvironmentImpl implements SpannerEnvironment { "SPANNER_OPTIMIZER_STATISTICS_PACKAGE"; private static final String SPANNER_ENABLE_EXTENDED_TRACING = "SPANNER_ENABLE_EXTENDED_TRACING"; private static final String SPANNER_ENABLE_API_TRACING = "SPANNER_ENABLE_API_TRACING"; + private static final String SPANNER_ENABLE_BUILTIN_METRICS = "SPANNER_ENABLE_BUILTIN_METRICS"; private SpannerEnvironmentImpl() {} @@ -734,6 +744,13 @@ public boolean isEnableExtendedTracing() { public boolean isEnableApiTracing() { return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_API_TRACING)); } + + @Override + public boolean isEnableBuiltInMetrics() { + if (System.getenv(SPANNER_ENABLE_BUILTIN_METRICS) != null) + return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_BUILTIN_METRICS)); + return true; + } } /** Builder for {@link SpannerOptions} instances. */ @@ -799,6 +816,7 @@ public static class Builder private OpenTelemetry openTelemetry; private boolean enableApiTracing = SpannerOptions.environment.isEnableApiTracing(); private boolean enableExtendedTracing = SpannerOptions.environment.isEnableExtendedTracing(); + private boolean enableBuiltInMetrics = SpannerOptions.environment.isEnableBuiltInMetrics(); private static String createCustomClientLibToken(String token) { return token + " " + ServiceOptions.getGoogApiClientLibName(); @@ -864,6 +882,7 @@ protected Builder() { this.useVirtualThreads = options.useVirtualThreads; this.enableApiTracing = options.enableApiTracing; this.enableExtendedTracing = options.enableExtendedTracing; + this.enableBuiltInMetrics = options.enableBuiltInMetrics; } @Override @@ -1391,6 +1410,12 @@ public Builder setEnableExtendedTracing(boolean enableExtendedTracing) { return this; } + /** Disables client built in metrics. */ + public Builder disableBuiltInMetrics() { + this.enableBuiltInMetrics = false; + return this; + } + @SuppressWarnings("rawtypes") @Override public SpannerOptions build() { @@ -1623,6 +1648,7 @@ public boolean isAttemptDirectPath() { public OpenTelemetry getOpenTelemetry() { if (this.openTelemetry != null) { return this.openTelemetry; + } else { return GlobalOpenTelemetry.get(); } @@ -1634,13 +1660,19 @@ public ApiTracerFactory getApiTracerFactory() { // Prefer any direct ApiTracerFactory that might have been set on the builder. apiTracerFactories.add( MoreObjects.firstNonNull(super.getApiTracerFactory(), getDefaultApiTracerFactory())); - + // Add Metrics Tracer factory + if (isEnableBuiltInMetrics()) { + ApiTracerFactory metricsTracerFactory = getMetricsApiTracerFactory(); + if (metricsTracerFactory != null) { + apiTracerFactories.add(metricsTracerFactory); + } + } return new CompositeTracerFactory(apiTracerFactories); } private ApiTracerFactory getDefaultApiTracerFactory() { - if (isEnableApiTracing()) { - if (activeTracingFramework == TracingFramework.OPEN_TELEMETRY) { + if (!isEnableApiTracing()) { + if (activeTracingFramework != TracingFramework.OPEN_TELEMETRY) { return new OpenTelemetryApiTracerFactory( getOpenTelemetry() .getTracer( @@ -1654,6 +1686,17 @@ private ApiTracerFactory getDefaultApiTracerFactory() { return BaseApiTracerFactory.getInstance(); } + private ApiTracerFactory getMetricsApiTracerFactory() { + OpenTelemetry openTelemetry = + new BuiltInOpenTelemetryMetricsProvider() + .getOpenTelemetry(getDefaultProjectId(), getCredentials()); + + return openTelemetry != null + ? new MetricsTracerFactory( + new OpenTelemetryMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME)) + : null; + } + /** * Returns true if an {@link com.google.api.gax.tracing.ApiTracer} should be created and set on * the Spanner client. Enabling this only has effect if an OpenTelemetry or OpenCensus trace @@ -1663,6 +1706,15 @@ public boolean isEnableApiTracing() { return enableApiTracing; } + /** + * Returns true if an {@link com.google.api.gax.tracing.ApiTracer} should be created and set on + * the Spanner client. Enabling this only has effect if an OpenTelemetry or OpenCensus trace + * exporter has been configured. + */ + public boolean isEnableBuiltInMetrics() { + return enableBuiltInMetrics; + } + @BetaApi public boolean isUseVirtualThreads() { return useVirtualThreads; diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java index 76b6c65a9b8..f3d9604e913 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java @@ -15,6 +15,7 @@ */ package com.google.cloud.spanner.spi.v1; +import static com.google.api.gax.grpc.GrpcCallContext.TRACER_KEY; import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.DATABASE_ID; import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.INSTANCE_ID; import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.METHOD; @@ -22,6 +23,8 @@ import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_HEADER_MISSING_COUNT; import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_LATENCY; +import com.google.cloud.spanner.BuiltInMetricsConstant; +import com.google.cloud.spanner.CompositeTracer; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerRpcMetrics; import com.google.common.cache.Cache; @@ -96,8 +99,10 @@ public void start(Listener responseListener, Metadata headers) { DatabaseName databaseName = extractDatabaseName(headers); String key = databaseName + method.getFullMethodName(); TagContext tagContext = getTagContext(key, method.getFullMethodName(), databaseName); + CompositeTracer compositeTracer = (CompositeTracer) callOptions.getOption(TRACER_KEY); Attributes attributes = getMetricAttributes(key, method.getFullMethodName(), databaseName); + addBuiltInMetricAttributes(compositeTracer, databaseName); super.start( new SimpleForwardingClientCallListener(responseListener) { @Override @@ -197,4 +202,14 @@ private Attributes getMetricAttributes(String key, String method, DatabaseName d return attributesBuilder.build(); }); } + + private void addBuiltInMetricAttributes( + CompositeTracer compositeTracer, DatabaseName databaseName) { + // Built in metrics Attributes. + compositeTracer.addAttributes( + BuiltInMetricsConstant.DATABASE_KEY.getKey(), databaseName.getDatabase()); + compositeTracer.addAttributes( + BuiltInMetricsConstant.INSTANCE_ID_KEY.getKey(), databaseName.getInstance()); + compositeTracer.addAttributes(BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), "true"); + } }