From 122ad023fdc96a4236d1882999948a2d05596747 Mon Sep 17 00:00:00 2001 From: surbhigarg92 Date: Wed, 19 Jun 2024 15:23:04 +0530 Subject: [PATCH] feat:client metrics --- google-cloud-spanner/pom.xml | 11 +- .../BuiltInOpenTelemetryMetricsProvider.java | 138 ++++++++++++++++++ .../google/cloud/spanner/SpannerOptions.java | 68 ++++++++- .../spanner/spi/v1/HeaderInterceptor.java | 12 ++ 4 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml index 8e2eed31559..ef4c8b84700 100644 --- a/google-cloud-spanner/pom.xml +++ b/google-cloud-spanner/pom.xml @@ -17,9 +17,9 @@ google-cloud-spanner 0.31.1 com.google.cloud.spanner.GceTestEnvConfig - projects/gcloud-devel/instances/spanner-testing-east1 - gcloud-devel - projects/gcloud-devel/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key + projects/span-cloud-testing/instances/surbhi-testing + span-cloud-testing + projects/span-cloud-testing/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key @@ -413,7 +413,6 @@ io.opentelemetry opentelemetry-sdk - test io.opentelemetry @@ -433,6 +432,10 @@ opentelemetry-sdk-testing test + + com.google.cloud.opentelemetry + detector-resources-support + com.google.cloud google-cloud-monitoring diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java new file mode 100644 index 00000000000..8179415f714 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java @@ -0,0 +1,138 @@ +package com.google.cloud.spanner; + +import static com.google.cloud.spanner.SpannerMetricsConstant.CLIENT_NAME_KEY; +import static com.google.cloud.spanner.SpannerMetricsConstant.CLIENT_UID_KEY; +import static com.google.cloud.spanner.SpannerMetricsConstant.INSTANCE_CONFIG_ID_KEY; +import static com.google.cloud.spanner.SpannerMetricsConstant.DIRECT_PATH_ENABLED_KEY; +import static com.google.cloud.spanner.SpannerMetricsConstant.LOCATION_ID_KEY; +import static com.google.cloud.spanner.SpannerMetricsConstant.PROJECT_ID_KEY; + +import com.google.api.gax.core.GaxProperties; +import com.google.auth.Credentials; +import com.google.cloud.opentelemetry.detection.DetectedPlatform; +import com.google.cloud.opentelemetry.detection.GCPPlatformDetector; +import com.google.common.collect.ImmutableSet; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.View; +import io.opentelemetry.sdk.metrics.export.MetricExporter; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +class BuiltInOpenTelemetryMetricsProvider { + + private static final Logger logger = + Logger.getLogger(BuiltInOpenTelemetryMetricsProvider.class.getName()); + + private OpenTelemetry openTelemetry; + public OpenTelemetry getOpenTelemetry(String projectId, @Nullable Credentials credentials) { + if (this.openTelemetry == null) { + + // Use custom exporter + MetricExporter metricExporter = null; + try { + metricExporter = SpannerCloudMonitoringExporter.create(projectId, credentials); + + SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder(); + registerView(sdkMeterProviderBuilder, SpannerMetricsConstant.OPERATION_LATENCY_NAME, SpannerMetricsConstant.OPERATION_LATENCIES_NAME); + registerView(sdkMeterProviderBuilder, SpannerMetricsConstant.ATTEMPT_LATENCY_NAME, SpannerMetricsConstant.ATTEMPT_LATENCIES_NAME); + registerView(sdkMeterProviderBuilder, SpannerMetricsConstant.OPERATION_COUNT_NAME, SpannerMetricsConstant.OPERATION_COUNT_NAME); + registerView(sdkMeterProviderBuilder, SpannerMetricsConstant.ATTEMPT_COUNT_NAME, SpannerMetricsConstant.ATTEMPT_COUNT_NAME); + + SdkMeterProvider sdkMeterProvider = + sdkMeterProviderBuilder + .registerMetricReader(PeriodicMetricReader.create(metricExporter)) + .build(); + + this.openTelemetry = + OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build(); + } catch (IOException e) { + logger.log(Level.WARNING, "Unable to get OpenTelemetry object for client side metrics, will skip exporting client side metrics", e); + } + } + return this.openTelemetry; + } + + public Map getClientAttributes(String projectId) { + Map clientAttributes = new HashMap<>(); + clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation()); + clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId); + clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "us-central1"); + clientAttributes.put(DIRECT_PATH_ENABLED_KEY.getKey(), "true"); + clientAttributes.put(CLIENT_NAME_KEY.getKey(), "spanner-java/" + GaxProperties.getLibraryVersion(SpannerCloudMonitoringExporterUtils.class)); + clientAttributes.put(CLIENT_UID_KEY.getKey(), getDefaultTaskValue()); + return clientAttributes; + } + + private void registerView(SdkMeterProviderBuilder sdkMeterProviderBuilder, String metricName, String metricViewName) { + InstrumentSelector selector = + InstrumentSelector.builder() + .setName(SpannerMetricsConstant.METER_NAME + '/' + + metricName) + .setMeterName(SpannerMetricsConstant.GAX_METER_NAME) + .setType(InstrumentType.HISTOGRAM) + .setUnit("ms") + .build(); + Set attributesFilter = + ImmutableSet.builder() + .addAll( + SpannerMetricsConstant.COMMON_ATTRIBUTES.stream() + .map(AttributeKey::getKey) + .collect(Collectors.toSet())) + .build(); + View view = + View.builder() + .setName(SpannerMetricsConstant.METER_NAME + '/' + + metricViewName) + .setAggregation(SpannerMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM) + .setAttributeFilter(attributesFilter) + .build(); + sdkMeterProviderBuilder.registerView(selector, view); + } + + + private String detectClientLocation() { + GCPPlatformDetector detector = GCPPlatformDetector.DEFAULT_INSTANCE; + DetectedPlatform detectedPlatform = detector.detectPlatform(); + String region = detectedPlatform.getAttributes().get("cloud.region"); + return region; + } + + + /** + * In most cases this should look like ${UUID}@${hostname}. The hostname will be retrieved from + * the jvm name and fallback to the local hostname. + */ + private String getDefaultTaskValue() { + // Something like '@' + final String jvmName = ManagementFactory.getRuntimeMXBean().getName(); + // If jvm doesn't have the expected format, fallback to the local hostname + if (jvmName.indexOf('@') < 1) { + String hostname = "localhost"; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + logger.log(Level.INFO, "Unable to get the hostname.", e); + } + // Generate a random number and use the same format "random_number@hostname". + return UUID.randomUUID() + "@" + hostname; + } + return UUID.randomUUID() + jvmName; + } +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index c3e75a7ed54..0002ea9f48d 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -30,6 +30,8 @@ import com.google.api.gax.rpc.TransportChannelProvider; import com.google.api.gax.tracing.ApiTracerFactory; import com.google.api.gax.tracing.BaseApiTracerFactory; +import com.google.api.gax.tracing.MetricsTracerFactory; +import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder; import com.google.api.gax.tracing.OpencensusTracerFactory; import com.google.cloud.NoCredentials; import com.google.cloud.ServiceDefaults; @@ -53,6 +55,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -68,7 +71,17 @@ import io.grpc.MethodDescriptor; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.Aggregation; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.View; +import io.opentelemetry.sdk.metrics.export.MetricExporter; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -83,6 +96,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; @@ -157,7 +171,9 @@ public class SpannerOptions extends ServiceOptions { private final boolean useVirtualThreads; private final OpenTelemetry openTelemetry; private final boolean enableApiTracing; + private final boolean enableBuiltInMetrics; private final boolean enableExtendedTracing; + private OpenTelemetry builtInOpenTelemetry; enum TracingFramework { OPEN_CENSUS, @@ -664,6 +680,7 @@ protected SpannerOptions(Builder builder) { openTelemetry = builder.openTelemetry; enableApiTracing = builder.enableApiTracing; enableExtendedTracing = builder.enableExtendedTracing; + enableBuiltInMetrics = builder.enableBuiltInMetrics; } /** @@ -696,6 +713,10 @@ default boolean isEnableExtendedTracing() { default boolean isEnableApiTracing() { return false; } + + default boolean isEnableBuiltInMetrics() { + return true; + } } /** @@ -709,6 +730,7 @@ private static class SpannerEnvironmentImpl implements SpannerEnvironment { "SPANNER_OPTIMIZER_STATISTICS_PACKAGE"; private static final String SPANNER_ENABLE_EXTENDED_TRACING = "SPANNER_ENABLE_EXTENDED_TRACING"; private static final String SPANNER_ENABLE_API_TRACING = "SPANNER_ENABLE_API_TRACING"; + private static final String SPANNER_ENABLE_BUILTIN_METRICS = "SPANNER_ENABLE_BUILTIN_METRICS"; private SpannerEnvironmentImpl() {} @@ -734,6 +756,13 @@ public boolean isEnableExtendedTracing() { public boolean isEnableApiTracing() { return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_API_TRACING)); } + + @Override + public boolean isEnableBuiltInMetrics() { + if (System.getenv(SPANNER_ENABLE_BUILTIN_METRICS) != null) + return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_BUILTIN_METRICS)); + return true; + } } /** Builder for {@link SpannerOptions} instances. */ @@ -799,6 +828,7 @@ public static class Builder private OpenTelemetry openTelemetry; private boolean enableApiTracing = SpannerOptions.environment.isEnableApiTracing(); private boolean enableExtendedTracing = SpannerOptions.environment.isEnableExtendedTracing(); + private boolean enableBuiltInMetrics = SpannerOptions.environment.isEnableBuiltInMetrics(); private static String createCustomClientLibToken(String token) { return token + " " + ServiceOptions.getGoogApiClientLibName(); @@ -864,6 +894,7 @@ protected Builder() { this.useVirtualThreads = options.useVirtualThreads; this.enableApiTracing = options.enableApiTracing; this.enableExtendedTracing = options.enableExtendedTracing; + this.enableBuiltInMetrics = options.enableBuiltInMetrics; } @Override @@ -1384,6 +1415,12 @@ public Builder setEnableExtendedTracing(boolean enableExtendedTracing) { return this; } + /** Disables client built in metrics. */ + public Builder disableBuiltInMetrics() { + this.enableBuiltInMetrics = false; + return this; + } + @SuppressWarnings("rawtypes") @Override public SpannerOptions build() { @@ -1616,6 +1653,7 @@ public boolean isAttemptDirectPath() { public OpenTelemetry getOpenTelemetry() { if (this.openTelemetry != null) { return this.openTelemetry; + } else { return GlobalOpenTelemetry.get(); } @@ -1627,13 +1665,19 @@ public ApiTracerFactory getApiTracerFactory() { // Prefer any direct ApiTracerFactory that might have been set on the builder. apiTracerFactories.add( MoreObjects.firstNonNull(super.getApiTracerFactory(), getDefaultApiTracerFactory())); - + // Add Metrics Tracer factory + if (isEnableBuiltInMetrics()) { + ApiTracerFactory metricsTracerFactory = getMetricsApiTracerFactory(); + if (metricsTracerFactory != null) { + apiTracerFactories.add(metricsTracerFactory); + } + } return new CompositeTracerFactory(apiTracerFactories); } private ApiTracerFactory getDefaultApiTracerFactory() { - if (isEnableApiTracing()) { - if (activeTracingFramework == TracingFramework.OPEN_TELEMETRY) { + if (!isEnableApiTracing()) { + if (activeTracingFramework != TracingFramework.OPEN_TELEMETRY) { return new OpenTelemetryApiTracerFactory( getOpenTelemetry() .getTracer( @@ -1647,6 +1691,15 @@ private ApiTracerFactory getDefaultApiTracerFactory() { return BaseApiTracerFactory.getInstance(); } + private ApiTracerFactory getMetricsApiTracerFactory() { + OpenTelemetry openTelemetry = new BuiltInOpenTelemetryMetricsProvider() + .getOpenTelemetry(getDefaultProjectId(), getCredentials()); + + return openTelemetry != null ? + new MetricsTracerFactory(new OpenTelemetryMetricsRecorder(openTelemetry, SpannerMetricsConstant.METER_NAME)) : null; + + } + /** * Returns true if an {@link com.google.api.gax.tracing.ApiTracer} should be created and set on * the Spanner client. Enabling this only has effect if an OpenTelemetry or OpenCensus trace @@ -1656,6 +1709,15 @@ public boolean isEnableApiTracing() { return enableApiTracing; } + /** + * Returns true if an {@link com.google.api.gax.tracing.ApiTracer} should be created and set on + * the Spanner client. Enabling this only has effect if an OpenTelemetry or OpenCensus trace + * exporter has been configured. + */ + public boolean isEnableBuiltInMetrics() { + return enableBuiltInMetrics; + } + @BetaApi public boolean isUseVirtualThreads() { return useVirtualThreads; diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java index 76b6c65a9b8..fcc41b5829e 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java @@ -15,6 +15,7 @@ */ package com.google.cloud.spanner.spi.v1; +import static com.google.api.gax.grpc.GrpcCallContext.TRACER_KEY; import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.DATABASE_ID; import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.INSTANCE_ID; import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.METHOD; @@ -22,7 +23,9 @@ import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_HEADER_MISSING_COUNT; import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_LATENCY; +import com.google.cloud.spanner.CompositeTracer; import com.google.cloud.spanner.SpannerExceptionFactory; +import com.google.cloud.spanner.SpannerMetricsConstant; import com.google.cloud.spanner.SpannerRpcMetrics; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -96,8 +99,10 @@ public void start(Listener responseListener, Metadata headers) { DatabaseName databaseName = extractDatabaseName(headers); String key = databaseName + method.getFullMethodName(); TagContext tagContext = getTagContext(key, method.getFullMethodName(), databaseName); + CompositeTracer compositeTracer = (CompositeTracer) callOptions.getOption(TRACER_KEY); Attributes attributes = getMetricAttributes(key, method.getFullMethodName(), databaseName); + addBuiltInMetricAttributes(compositeTracer, databaseName); super.start( new SimpleForwardingClientCallListener(responseListener) { @Override @@ -197,4 +202,11 @@ private Attributes getMetricAttributes(String key, String method, DatabaseName d return attributesBuilder.build(); }); } + + private void addBuiltInMetricAttributes(CompositeTracer compositeTracer, DatabaseName databaseName) { + // Built in metrics Attributes. + compositeTracer.addAttributes(SpannerMetricsConstant.DATABASE_KEY.getKey(), databaseName.getDatabase()); + compositeTracer.addAttributes(SpannerMetricsConstant.INSTANCE_ID_KEY.getKey(), databaseName.getInstance()); + compositeTracer.addAttributes(SpannerMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), "true"); + } }