diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml
index 8e2eed31559..ef4c8b84700 100644
--- a/google-cloud-spanner/pom.xml
+++ b/google-cloud-spanner/pom.xml
@@ -17,9 +17,9 @@
<site.installationModule>google-cloud-spanner</site.installationModule>
<opencensus.version>0.31.1</opencensus.version>
<spanner.testenv.config.class>com.google.cloud.spanner.GceTestEnvConfig</spanner.testenv.config.class>
- <spanner.testenv.instance>projects/gcloud-devel/instances/spanner-testing-east1</spanner.testenv.instance>
- <spanner.gce.config.project_id>gcloud-devel</spanner.gce.config.project_id>
- <spanner.testenv.kms_key.name>projects/gcloud-devel/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key</spanner.testenv.kms_key.name>
+ <spanner.testenv.instance>projects/span-cloud-testing/instances/surbhi-testing</spanner.testenv.instance>
+ <spanner.gce.config.project_id>span-cloud-testing</spanner.gce.config.project_id>
+ <spanner.testenv.kms_key.name>projects/span-cloud-testing/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key</spanner.testenv.kms_key.name>
</properties>
@@ -413,7 +413,6 @@
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-sdk</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
@@ -433,6 +432,10 @@
<artifactId>opentelemetry-sdk-testing</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.cloud.opentelemetry</groupId>
+ <artifactId>detector-resources-support</artifactId>
+ </dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-monitoring</artifactId>
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java
new file mode 100644
index 00000000000..8179415f714
--- /dev/null
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java
@@ -0,0 +1,138 @@
+package com.google.cloud.spanner;
+
+import static com.google.cloud.spanner.SpannerMetricsConstant.CLIENT_NAME_KEY;
+import static com.google.cloud.spanner.SpannerMetricsConstant.CLIENT_UID_KEY;
+import static com.google.cloud.spanner.SpannerMetricsConstant.INSTANCE_CONFIG_ID_KEY;
+import static com.google.cloud.spanner.SpannerMetricsConstant.DIRECT_PATH_ENABLED_KEY;
+import static com.google.cloud.spanner.SpannerMetricsConstant.LOCATION_ID_KEY;
+import static com.google.cloud.spanner.SpannerMetricsConstant.PROJECT_ID_KEY;
+
+import com.google.api.gax.core.GaxProperties;
+import com.google.auth.Credentials;
+import com.google.cloud.opentelemetry.detection.DetectedPlatform;
+import com.google.cloud.opentelemetry.detection.GCPPlatformDetector;
+import com.google.common.collect.ImmutableSet;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.metrics.InstrumentSelector;
+import io.opentelemetry.sdk.metrics.InstrumentType;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
+import io.opentelemetry.sdk.metrics.View;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
+import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+class BuiltInOpenTelemetryMetricsProvider {
+
+ private static final Logger logger =
+ Logger.getLogger(BuiltInOpenTelemetryMetricsProvider.class.getName());
+
+ private OpenTelemetry openTelemetry;
+ public OpenTelemetry getOpenTelemetry(String projectId, @Nullable Credentials credentials) {
+ if (this.openTelemetry == null) {
+
+ // Use custom exporter
+ MetricExporter metricExporter = null;
+ try {
+ metricExporter = SpannerCloudMonitoringExporter.create(projectId, credentials);
+
+ SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder();
+ registerView(sdkMeterProviderBuilder, SpannerMetricsConstant.OPERATION_LATENCY_NAME, SpannerMetricsConstant.OPERATION_LATENCIES_NAME);
+ registerView(sdkMeterProviderBuilder, SpannerMetricsConstant.ATTEMPT_LATENCY_NAME, SpannerMetricsConstant.ATTEMPT_LATENCIES_NAME);
+ registerView(sdkMeterProviderBuilder, SpannerMetricsConstant.OPERATION_COUNT_NAME, SpannerMetricsConstant.OPERATION_COUNT_NAME);
+ registerView(sdkMeterProviderBuilder, SpannerMetricsConstant.ATTEMPT_COUNT_NAME, SpannerMetricsConstant.ATTEMPT_COUNT_NAME);
+
+ SdkMeterProvider sdkMeterProvider =
+ sdkMeterProviderBuilder
+ .registerMetricReader(PeriodicMetricReader.create(metricExporter))
+ .build();
+
+ this.openTelemetry =
+ OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Unable to get OpenTelemetry object for client side metrics, will skip exporting client side metrics", e);
+ }
+ }
+ return this.openTelemetry;
+ }
+
+ public Map<String, String> getClientAttributes(String projectId) {
+ Map<String, String> clientAttributes = new HashMap<>();
+ clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
+ clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId);
+ clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "us-central1");
+ clientAttributes.put(DIRECT_PATH_ENABLED_KEY.getKey(), "true");
+ clientAttributes.put(CLIENT_NAME_KEY.getKey(), "spanner-java/" + GaxProperties.getLibraryVersion(SpannerCloudMonitoringExporterUtils.class));
+ clientAttributes.put(CLIENT_UID_KEY.getKey(), getDefaultTaskValue());
+ return clientAttributes;
+ }
+
+ private void registerView(SdkMeterProviderBuilder sdkMeterProviderBuilder, String metricName, String metricViewName) {
+ InstrumentSelector selector =
+ InstrumentSelector.builder()
+ .setName(SpannerMetricsConstant.METER_NAME + '/'
+ + metricName)
+ .setMeterName(SpannerMetricsConstant.GAX_METER_NAME)
+ .setType(InstrumentType.HISTOGRAM)
+ .setUnit("ms")
+ .build();
+ Set<String> attributesFilter =
+ ImmutableSet.<String>builder()
+ .addAll(
+ SpannerMetricsConstant.COMMON_ATTRIBUTES.stream()
+ .map(AttributeKey::getKey)
+ .collect(Collectors.toSet()))
+ .build();
+ View view =
+ View.builder()
+ .setName(SpannerMetricsConstant.METER_NAME + '/'
+ + metricViewName)
+ .setAggregation(SpannerMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM)
+ .setAttributeFilter(attributesFilter)
+ .build();
+ sdkMeterProviderBuilder.registerView(selector, view);
+ }
+
+
+ private String detectClientLocation() {
+ GCPPlatformDetector detector = GCPPlatformDetector.DEFAULT_INSTANCE;
+ DetectedPlatform detectedPlatform = detector.detectPlatform();
+ String region = detectedPlatform.getAttributes().get("cloud.region");
+ return region;
+ }
+
+
+ /**
+ * In most cases this should look like ${UUID}@${hostname}. The hostname will be retrieved from
+ * the jvm name and fallback to the local hostname.
+ */
+ private String getDefaultTaskValue() {
+ // Something like '<pid>@<hostname>'
+ final String jvmName = ManagementFactory.getRuntimeMXBean().getName();
+ // If jvm doesn't have the expected format, fallback to the local hostname
+ if (jvmName.indexOf('@') < 1) {
+ String hostname = "localhost";
+ try {
+ hostname = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ logger.log(Level.INFO, "Unable to get the hostname.", e);
+ }
+ // Generate a random number and use the same format "random_number@hostname".
+ return UUID.randomUUID() + "@" + hostname;
+ }
+ return UUID.randomUUID() + jvmName;
+ }
+}
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java
index c3e75a7ed54..0002ea9f48d 100644
--- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java
@@ -30,6 +30,8 @@
import com.google.api.gax.rpc.TransportChannelProvider;
import com.google.api.gax.tracing.ApiTracerFactory;
import com.google.api.gax.tracing.BaseApiTracerFactory;
+import com.google.api.gax.tracing.MetricsTracerFactory;
+import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder;
import com.google.api.gax.tracing.OpencensusTracerFactory;
import com.google.cloud.NoCredentials;
import com.google.cloud.ServiceDefaults;
@@ -53,6 +55,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -68,7 +71,17 @@
import io.grpc.MethodDescriptor;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.metrics.Aggregation;
+import io.opentelemetry.sdk.metrics.InstrumentSelector;
+import io.opentelemetry.sdk.metrics.InstrumentType;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
+import io.opentelemetry.sdk.metrics.View;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
+import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
@@ -83,6 +96,7 @@
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
@@ -157,7 +171,9 @@ public class SpannerOptions extends ServiceOptions<Spanner, SpannerOptions> {
private final boolean useVirtualThreads;
private final OpenTelemetry openTelemetry;
private final boolean enableApiTracing;
+ private final boolean enableBuiltInMetrics;
private final boolean enableExtendedTracing;
+ private OpenTelemetry builtInOpenTelemetry;
enum TracingFramework {
OPEN_CENSUS,
@@ -664,6 +680,7 @@ protected SpannerOptions(Builder builder) {
openTelemetry = builder.openTelemetry;
enableApiTracing = builder.enableApiTracing;
enableExtendedTracing = builder.enableExtendedTracing;
+ enableBuiltInMetrics = builder.enableBuiltInMetrics;
}
/**
@@ -696,6 +713,10 @@ default boolean isEnableExtendedTracing() {
default boolean isEnableApiTracing() {
return false;
}
+
+ default boolean isEnableBuiltInMetrics() {
+ return true;
+ }
}
/**
@@ -709,6 +730,7 @@ private static class SpannerEnvironmentImpl implements SpannerEnvironment {
"SPANNER_OPTIMIZER_STATISTICS_PACKAGE";
private static final String SPANNER_ENABLE_EXTENDED_TRACING = "SPANNER_ENABLE_EXTENDED_TRACING";
private static final String SPANNER_ENABLE_API_TRACING = "SPANNER_ENABLE_API_TRACING";
+ private static final String SPANNER_ENABLE_BUILTIN_METRICS = "SPANNER_ENABLE_BUILTIN_METRICS";
private SpannerEnvironmentImpl() {}
@@ -734,6 +756,13 @@ public boolean isEnableExtendedTracing() {
public boolean isEnableApiTracing() {
return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_API_TRACING));
}
+
+ @Override
+ public boolean isEnableBuiltInMetrics() {
+ if (System.getenv(SPANNER_ENABLE_BUILTIN_METRICS) != null)
+ return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_BUILTIN_METRICS));
+ return true;
+ }
}
/** Builder for {@link SpannerOptions} instances. */
@@ -799,6 +828,7 @@ public static class Builder
private OpenTelemetry openTelemetry;
private boolean enableApiTracing = SpannerOptions.environment.isEnableApiTracing();
private boolean enableExtendedTracing = SpannerOptions.environment.isEnableExtendedTracing();
+ private boolean enableBuiltInMetrics = SpannerOptions.environment.isEnableBuiltInMetrics();
private static String createCustomClientLibToken(String token) {
return token + " " + ServiceOptions.getGoogApiClientLibName();
@@ -864,6 +894,7 @@ protected Builder() {
this.useVirtualThreads = options.useVirtualThreads;
this.enableApiTracing = options.enableApiTracing;
this.enableExtendedTracing = options.enableExtendedTracing;
+ this.enableBuiltInMetrics = options.enableBuiltInMetrics;
}
@Override
@@ -1384,6 +1415,12 @@ public Builder setEnableExtendedTracing(boolean enableExtendedTracing) {
return this;
}
+ /** Disables client built in metrics. */
+ public Builder disableBuiltInMetrics() {
+ this.enableBuiltInMetrics = false;
+ return this;
+ }
+
@SuppressWarnings("rawtypes")
@Override
public SpannerOptions build() {
@@ -1616,6 +1653,7 @@ public boolean isAttemptDirectPath() {
public OpenTelemetry getOpenTelemetry() {
if (this.openTelemetry != null) {
return this.openTelemetry;
+
} else {
return GlobalOpenTelemetry.get();
}
@@ -1627,13 +1665,19 @@ public ApiTracerFactory getApiTracerFactory() {
// Prefer any direct ApiTracerFactory that might have been set on the builder.
apiTracerFactories.add(
MoreObjects.firstNonNull(super.getApiTracerFactory(), getDefaultApiTracerFactory()));
-
+ // Add Metrics Tracer factory
+ if (isEnableBuiltInMetrics()) {
+ ApiTracerFactory metricsTracerFactory = getMetricsApiTracerFactory();
+ if (metricsTracerFactory != null) {
+ apiTracerFactories.add(metricsTracerFactory);
+ }
+ }
return new CompositeTracerFactory(apiTracerFactories);
}
private ApiTracerFactory getDefaultApiTracerFactory() {
- if (isEnableApiTracing()) {
- if (activeTracingFramework == TracingFramework.OPEN_TELEMETRY) {
+ if (!isEnableApiTracing()) {
+ if (activeTracingFramework != TracingFramework.OPEN_TELEMETRY) {
return new OpenTelemetryApiTracerFactory(
getOpenTelemetry()
.getTracer(
@@ -1647,6 +1691,15 @@ private ApiTracerFactory getDefaultApiTracerFactory() {
return BaseApiTracerFactory.getInstance();
}
+ private ApiTracerFactory getMetricsApiTracerFactory() {
+ OpenTelemetry openTelemetry = new BuiltInOpenTelemetryMetricsProvider()
+ .getOpenTelemetry(getDefaultProjectId(), getCredentials());
+
+ return openTelemetry != null ?
+ new MetricsTracerFactory(new OpenTelemetryMetricsRecorder(openTelemetry, SpannerMetricsConstant.METER_NAME)) : null;
+
+ }
+
/**
* Returns true if an {@link com.google.api.gax.tracing.ApiTracer} should be created and set on
* the Spanner client. Enabling this only has effect if an OpenTelemetry or OpenCensus trace
@@ -1656,6 +1709,15 @@ public boolean isEnableApiTracing() {
return enableApiTracing;
}
+ /**
+ * Returns true if an {@link com.google.api.gax.tracing.ApiTracer} should be created and set on
+ * the Spanner client. Enabling this only has effect if an OpenTelemetry or OpenCensus trace
+ * exporter has been configured.
+ */
+ public boolean isEnableBuiltInMetrics() {
+ return enableBuiltInMetrics;
+ }
+
@BetaApi
public boolean isUseVirtualThreads() {
return useVirtualThreads;
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java
index 76b6c65a9b8..fcc41b5829e 100644
--- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java
@@ -15,6 +15,7 @@
*/
package com.google.cloud.spanner.spi.v1;
+import static com.google.api.gax.grpc.GrpcCallContext.TRACER_KEY;
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.DATABASE_ID;
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.INSTANCE_ID;
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.METHOD;
@@ -22,7 +23,9 @@
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_HEADER_MISSING_COUNT;
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_LATENCY;
+import com.google.cloud.spanner.CompositeTracer;
import com.google.cloud.spanner.SpannerExceptionFactory;
+import com.google.cloud.spanner.SpannerMetricsConstant;
import com.google.cloud.spanner.SpannerRpcMetrics;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@@ -96,8 +99,10 @@ public void start(Listener<RespT> responseListener, Metadata headers) {
DatabaseName databaseName = extractDatabaseName(headers);
String key = databaseName + method.getFullMethodName();
TagContext tagContext = getTagContext(key, method.getFullMethodName(), databaseName);
+ CompositeTracer compositeTracer = (CompositeTracer) callOptions.getOption(TRACER_KEY);
Attributes attributes =
getMetricAttributes(key, method.getFullMethodName(), databaseName);
+ addBuiltInMetricAttributes(compositeTracer, databaseName);
super.start(
new SimpleForwardingClientCallListener<RespT>(responseListener) {
@Override
@@ -197,4 +202,11 @@ private Attributes getMetricAttributes(String key, String method, DatabaseName d
return attributesBuilder.build();
});
}
+
+ private void addBuiltInMetricAttributes(CompositeTracer compositeTracer, DatabaseName databaseName) {
+ // Built in metrics Attributes.
+ compositeTracer.addAttributes(SpannerMetricsConstant.DATABASE_KEY.getKey(), databaseName.getDatabase());
+ compositeTracer.addAttributes(SpannerMetricsConstant.INSTANCE_ID_KEY.getKey(), databaseName.getInstance());
+ compositeTracer.addAttributes(SpannerMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), "true");
+ }
}