From f5f3046f2de2a77815e95711b9d641308d87c1a3 Mon Sep 17 00:00:00 2001 From: svrnm Date: Wed, 21 Dec 2022 15:22:10 +0100 Subject: [PATCH 1/7] Add otel agent with JMX metric insights to kafka Signed-off-by: svrnm --- docker-compose.yml | 6 +- pb/google/protobuf/timestamp.proto | 147 +++++++++++++++++++++++++++++ src/kafka/Dockerfile | 6 ++ 3 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 pb/google/protobuf/timestamp.proto diff --git a/docker-compose.yml b/docker-compose.yml index f870a5b895..af36238924 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -544,7 +544,11 @@ services: memory: 600M restart: always environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 + - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT + - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE + - OTEL_SERVICE_NAME=kafka logging: *logging healthcheck: test: nc -z kafka 9092 diff --git a/pb/google/protobuf/timestamp.proto b/pb/google/protobuf/timestamp.proto new file mode 100644 index 0000000000..3b2df6d911 --- /dev/null +++ b/pb/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/src/kafka/Dockerfile b/src/kafka/Dockerfile index 71396afbae..de775cc823 100644 --- a/src/kafka/Dockerfile +++ b/src/kafka/Dockerfile @@ -1,5 +1,10 @@ FROM confluentinc/cp-kafka:latest-ubi8 +USER root +ARG version=1.21.0 +ADD https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v$version/opentelemetry-javaagent.jar /tmp/opentelemetry-javaagent.jar +RUN chmod go+r /tmp/opentelemetry-javaagent.jar + USER appuser WORKDIR /tmp COPY ./src/kafka/clusterID clusterID @@ -18,5 +23,6 @@ ENV KAFKA_METADATA_LOG_MAX_RECORD_BYTES_BETWEEN_SNAPSHOTS=2800 ENV KAFKA_AUTO_CREATE_TOPICS_ENABLE=true ENV KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 ENV KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 +ENV KAFKA_OPTS="-javaagent:/tmp/opentelemetry-javaagent.jar -Dotel.jmx.target.system=kafka-broker" ENTRYPOINT ["/bin/sh", "-c", "ls -lh /tmp && /tmp/update_run.sh && /etc/confluent/docker/run"] From c42417ff1bfc6786f9ad62681bfdf972776ab919 Mon Sep 17 00:00:00 2001 From: svrnm Date: Wed, 21 Dec 2022 15:28:04 +0100 Subject: [PATCH 2/7] Increase memory limit for kafka --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index af36238924..a9461ccc88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -541,7 +541,7 @@ services: deploy: resources: limits: - memory: 600M + memory: 800M restart: always environment: - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 From 6f77fe258719aead153368afc8858b96321bfd23 Mon Sep 17 00:00:00 2001 From: svrnm Date: Wed, 21 Dec 2022 15:28:50 +0100 Subject: [PATCH 3/7] Remove pb files Signed-off-by: svrnm --- pb/demo.proto | 319 ----------------------------- pb/google/protobuf/timestamp.proto | 147 ------------- 2 files changed, 466 deletions(-) delete mode 100644 pb/demo.proto delete mode 100644 pb/google/protobuf/timestamp.proto diff --git a/pb/demo.proto b/pb/demo.proto deleted file mode 100644 index fef5cb6383..0000000000 --- a/pb/demo.proto +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -import "google/protobuf/timestamp.proto"; - -package hipstershop; - -option go_package = "genproto/hipstershop"; - -// -----------------Cart service----------------- - -service CartService { - rpc AddItem(AddItemRequest) returns (Empty) {} - rpc GetCart(GetCartRequest) returns (Cart) {} - rpc EmptyCart(EmptyCartRequest) returns (Empty) {} -} - -message CartItem { - string product_id = 1; - int32 quantity = 2; -} - -message AddItemRequest { - string user_id = 1; - CartItem item = 2; -} - -message EmptyCartRequest { - string user_id = 1; -} - -message GetCartRequest { - string user_id = 1; -} - -message Cart { - string user_id = 1; - repeated CartItem items = 2; -} - -message Empty {} - -// ---------------Recommendation service---------- - -service RecommendationService { - rpc ListRecommendations(ListRecommendationsRequest) returns (ListRecommendationsResponse){} -} - -message ListRecommendationsRequest { - string user_id = 1; - repeated string product_ids = 2; -} - -message ListRecommendationsResponse { - repeated string product_ids = 1; -} - -// ---------------Product Catalog---------------- - -service ProductCatalogService { - rpc ListProducts(Empty) returns (ListProductsResponse) {} - rpc GetProduct(GetProductRequest) returns (Product) {} - rpc SearchProducts(SearchProductsRequest) returns (SearchProductsResponse) {} -} - -message Product { - string id = 1; - string name = 2; - string description = 3; - string picture = 4; - Money price_usd = 5; - - // Categories such as "clothing" or "kitchen" that can be used to look up - // other related products. - repeated string categories = 6; -} - -message ListProductsResponse { - repeated Product products = 1; -} - -message GetProductRequest { - string id = 1; -} - -message SearchProductsRequest { - string query = 1; -} - -message SearchProductsResponse { - repeated Product results = 1; -} - -// ---------------Shipping Service---------- - -service ShippingService { - rpc GetQuote(GetQuoteRequest) returns (GetQuoteResponse) {} - rpc ShipOrder(ShipOrderRequest) returns (ShipOrderResponse) {} -} - -message GetQuoteRequest { - Address address = 1; - repeated CartItem items = 2; -} - -message GetQuoteResponse { - Money cost_usd = 1; -} - -message ShipOrderRequest { - Address address = 1; - repeated CartItem items = 2; -} - -message ShipOrderResponse { - string tracking_id = 1; -} - -message Address { - string street_address = 1; - string city = 2; - string state = 3; - string country = 4; - string zip_code = 5; -} - -// -----------------Currency service----------------- - -service CurrencyService { - rpc GetSupportedCurrencies(Empty) returns (GetSupportedCurrenciesResponse) {} - rpc Convert(CurrencyConversionRequest) returns (Money) {} -} - -// Represents an amount of money with its currency type. -message Money { - // The 3-letter currency code defined in ISO 4217. - string currency_code = 1; - - // The whole units of the amount. - // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. - int64 units = 2; - - // Number of nano (10^-9) units of the amount. - // The value must be between -999,999,999 and +999,999,999 inclusive. - // If `units` is positive, `nanos` must be positive or zero. - // If `units` is zero, `nanos` can be positive, zero, or negative. - // If `units` is negative, `nanos` must be negative or zero. - // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. - int32 nanos = 3; -} - -message GetSupportedCurrenciesResponse { - // The 3-letter currency code defined in ISO 4217. - repeated string currency_codes = 1; -} - -message CurrencyConversionRequest { - Money from = 1; - - // The 3-letter currency code defined in ISO 4217. - string to_code = 2; -} - -// -------------Payment service----------------- - -service PaymentService { - rpc Charge(ChargeRequest) returns (ChargeResponse) {} -} - -message CreditCardInfo { - string credit_card_number = 1; - int32 credit_card_cvv = 2; - int32 credit_card_expiration_year = 3; - int32 credit_card_expiration_month = 4; -} - -message ChargeRequest { - Money amount = 1; - CreditCardInfo credit_card = 2; -} - -message ChargeResponse { - string transaction_id = 1; -} - -// -------------Email service----------------- - -service EmailService { - rpc SendOrderConfirmation(SendOrderConfirmationRequest) returns (Empty) {} -} - -message OrderItem { - CartItem item = 1; - Money cost = 2; -} - -message OrderResult { - string order_id = 1; - string shipping_tracking_id = 2; - Money shipping_cost = 3; - Address shipping_address = 4; - repeated OrderItem items = 5; -} - -message SendOrderConfirmationRequest { - string email = 1; - OrderResult order = 2; -} - - -// -------------Checkout service----------------- - -service CheckoutService { - rpc PlaceOrder(PlaceOrderRequest) returns (PlaceOrderResponse) {} -} - -message PlaceOrderRequest { - string user_id = 1; - string user_currency = 2; - - Address address = 3; - string email = 5; - CreditCardInfo credit_card = 6; -} - -message PlaceOrderResponse { - OrderResult order = 1; -} - -// ------------Ad service------------------ - -service AdService { - rpc GetAds(AdRequest) returns (AdResponse) {} -} - -message AdRequest { - // List of important key words from the current page describing the context. - repeated string context_keys = 1; -} - -message AdResponse { - repeated Ad ads = 1; -} - -message Ad { - // url to redirect to when an ad is clicked. - string redirect_url = 1; - - // short advertisement text to display. - string text = 2; -} - -// ------------Feature flag service------------------ - -service FeatureFlagService { - rpc GetFlag(GetFlagRequest) returns (GetFlagResponse) {} - rpc CreateFlag(CreateFlagRequest) returns (CreateFlagResponse) {} - rpc UpdateFlag(UpdateFlagRequest) returns (UpdateFlagResponse) {} - rpc ListFlags(ListFlagsRequest) returns (ListFlagsResponse) {} - rpc DeleteFlag(DeleteFlagRequest) returns (DeleteFlagResponse) {} -} - -message Flag { - string name = 1; - string description = 2; - bool enabled = 3; - google.protobuf.Timestamp created_at = 4; - google.protobuf.Timestamp updated_at = 5; -} - -message GetFlagRequest { - string name = 1; -} - -message GetFlagResponse { - Flag flag = 1; -} - -message CreateFlagRequest { - string name = 1; - string description = 2; - bool enabled = 3; -} - -message CreateFlagResponse { - Flag flag = 1; -} - -message UpdateFlagRequest { - string name = 1; - bool enabled = 2; -} - -message UpdateFlagResponse {} - -message ListFlagsRequest {} - -message ListFlagsResponse { - repeated Flag flag = 1; -} - -message DeleteFlagRequest { - string name = 1; -} - -message DeleteFlagResponse {} diff --git a/pb/google/protobuf/timestamp.proto b/pb/google/protobuf/timestamp.proto deleted file mode 100644 index 3b2df6d911..0000000000 --- a/pb/google/protobuf/timestamp.proto +++ /dev/null @@ -1,147 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/timestamppb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "TimestampProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// A Timestamp represents a point in time independent of any time zone or local -// calendar, encoded as a count of seconds and fractions of seconds at -// nanosecond resolution. The count is relative to an epoch at UTC midnight on -// January 1, 1970, in the proleptic Gregorian calendar which extends the -// Gregorian calendar backwards to year one. -// -// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap -// second table is needed for interpretation, using a [24-hour linear -// smear](https://developers.google.com/time/smear). -// -// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By -// restricting to that range, we ensure that we can convert to and from [RFC -// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. -// -// # Examples -// -// Example 1: Compute Timestamp from POSIX `time()`. -// -// Timestamp timestamp; -// timestamp.set_seconds(time(NULL)); -// timestamp.set_nanos(0); -// -// Example 2: Compute Timestamp from POSIX `gettimeofday()`. -// -// struct timeval tv; -// gettimeofday(&tv, NULL); -// -// Timestamp timestamp; -// timestamp.set_seconds(tv.tv_sec); -// timestamp.set_nanos(tv.tv_usec * 1000); -// -// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. -// -// FILETIME ft; -// GetSystemTimeAsFileTime(&ft); -// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; -// -// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z -// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. -// Timestamp timestamp; -// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); -// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); -// -// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. -// -// long millis = System.currentTimeMillis(); -// -// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) -// .setNanos((int) ((millis % 1000) * 1000000)).build(); -// -// -// Example 5: Compute Timestamp from Java `Instant.now()`. -// -// Instant now = Instant.now(); -// -// Timestamp timestamp = -// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) -// .setNanos(now.getNano()).build(); -// -// -// Example 6: Compute Timestamp from current time in Python. -// -// timestamp = Timestamp() -// timestamp.GetCurrentTime() -// -// # JSON Mapping -// -// In JSON format, the Timestamp type is encoded as a string in the -// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the -// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" -// where {year} is always expressed using four digits while {month}, {day}, -// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional -// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), -// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone -// is required. A proto3 JSON serializer should always use UTC (as indicated by -// "Z") when printing the Timestamp type and a proto3 JSON parser should be -// able to accept both UTC and other timezones (as indicated by an offset). -// -// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past -// 01:30 UTC on January 15, 2017. -// -// In JavaScript, one can convert a Date object to this format using the -// standard -// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) -// method. In Python, a standard `datetime.datetime` object can be converted -// to this format using -// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with -// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use -// the Joda Time's [`ISODateTimeFormat.dateTime()`]( -// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D -// ) to obtain a formatter capable of generating timestamps in this format. -// -// -message Timestamp { - // Represents seconds of UTC time since Unix epoch - // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to - // 9999-12-31T23:59:59Z inclusive. - int64 seconds = 1; - - // Non-negative fractions of a second at nanosecond resolution. Negative - // second values with fractions must still have non-negative nanos values - // that count forward in time. Must be from 0 to 999,999,999 - // inclusive. - int32 nanos = 2; -} From 1fe256083332f1a7f84f9e7e1e627704ad584a47 Mon Sep 17 00:00:00 2001 From: svrnm Date: Wed, 21 Dec 2022 19:00:30 +0100 Subject: [PATCH 4/7] Add docs for kafka service Signed-off-by: svrnm --- docs/services/kafka.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docs/services/kafka.md diff --git a/docs/services/kafka.md b/docs/services/kafka.md new file mode 100644 index 0000000000..941edd20f7 --- /dev/null +++ b/docs/services/kafka.md @@ -0,0 +1,20 @@ +# Kafka + +This is used as a message queue service to connect the checkout service with +the accounting and fraud detection services. + +[Kafka service source](../../src/kafka/) + +## Auto-instrumentation + +This service relies on the OpenTelemetry Java Agent and the built in +[JMX Metric Insight Module](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/jmx-metrics/javaagent) +to capture [kafka broker metrics](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/instrumentation/jmx-metrics/javaagent/kafka-broker.md) and send them off to the collector via OTLP. + +The agent is passed into the process using the `-javaagent` command line +argument. Command line arguments are added through the `KAFKA_OPTS` +in the `Dockerfile`. + +```dockerfile +ENV KAFKA_OPTS="-javaagent:/tmp/opentelemetry-javaagent.jar -Dotel.jmx.target.system=kafka-broker" +``` From 721925116f8abfa8db2c9e4c05c0e3fe899f40d2 Mon Sep 17 00:00:00 2001 From: svrnm Date: Wed, 21 Dec 2022 19:07:30 +0100 Subject: [PATCH 5/7] Revert removal of demo.proto Signed-off-by: svrnm --- pb/demo.proto | 319 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 pb/demo.proto diff --git a/pb/demo.proto b/pb/demo.proto new file mode 100644 index 0000000000..fef5cb6383 --- /dev/null +++ b/pb/demo.proto @@ -0,0 +1,319 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +package hipstershop; + +option go_package = "genproto/hipstershop"; + +// -----------------Cart service----------------- + +service CartService { + rpc AddItem(AddItemRequest) returns (Empty) {} + rpc GetCart(GetCartRequest) returns (Cart) {} + rpc EmptyCart(EmptyCartRequest) returns (Empty) {} +} + +message CartItem { + string product_id = 1; + int32 quantity = 2; +} + +message AddItemRequest { + string user_id = 1; + CartItem item = 2; +} + +message EmptyCartRequest { + string user_id = 1; +} + +message GetCartRequest { + string user_id = 1; +} + +message Cart { + string user_id = 1; + repeated CartItem items = 2; +} + +message Empty {} + +// ---------------Recommendation service---------- + +service RecommendationService { + rpc ListRecommendations(ListRecommendationsRequest) returns (ListRecommendationsResponse){} +} + +message ListRecommendationsRequest { + string user_id = 1; + repeated string product_ids = 2; +} + +message ListRecommendationsResponse { + repeated string product_ids = 1; +} + +// ---------------Product Catalog---------------- + +service ProductCatalogService { + rpc ListProducts(Empty) returns (ListProductsResponse) {} + rpc GetProduct(GetProductRequest) returns (Product) {} + rpc SearchProducts(SearchProductsRequest) returns (SearchProductsResponse) {} +} + +message Product { + string id = 1; + string name = 2; + string description = 3; + string picture = 4; + Money price_usd = 5; + + // Categories such as "clothing" or "kitchen" that can be used to look up + // other related products. + repeated string categories = 6; +} + +message ListProductsResponse { + repeated Product products = 1; +} + +message GetProductRequest { + string id = 1; +} + +message SearchProductsRequest { + string query = 1; +} + +message SearchProductsResponse { + repeated Product results = 1; +} + +// ---------------Shipping Service---------- + +service ShippingService { + rpc GetQuote(GetQuoteRequest) returns (GetQuoteResponse) {} + rpc ShipOrder(ShipOrderRequest) returns (ShipOrderResponse) {} +} + +message GetQuoteRequest { + Address address = 1; + repeated CartItem items = 2; +} + +message GetQuoteResponse { + Money cost_usd = 1; +} + +message ShipOrderRequest { + Address address = 1; + repeated CartItem items = 2; +} + +message ShipOrderResponse { + string tracking_id = 1; +} + +message Address { + string street_address = 1; + string city = 2; + string state = 3; + string country = 4; + string zip_code = 5; +} + +// -----------------Currency service----------------- + +service CurrencyService { + rpc GetSupportedCurrencies(Empty) returns (GetSupportedCurrenciesResponse) {} + rpc Convert(CurrencyConversionRequest) returns (Money) {} +} + +// Represents an amount of money with its currency type. +message Money { + // The 3-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} + +message GetSupportedCurrenciesResponse { + // The 3-letter currency code defined in ISO 4217. + repeated string currency_codes = 1; +} + +message CurrencyConversionRequest { + Money from = 1; + + // The 3-letter currency code defined in ISO 4217. + string to_code = 2; +} + +// -------------Payment service----------------- + +service PaymentService { + rpc Charge(ChargeRequest) returns (ChargeResponse) {} +} + +message CreditCardInfo { + string credit_card_number = 1; + int32 credit_card_cvv = 2; + int32 credit_card_expiration_year = 3; + int32 credit_card_expiration_month = 4; +} + +message ChargeRequest { + Money amount = 1; + CreditCardInfo credit_card = 2; +} + +message ChargeResponse { + string transaction_id = 1; +} + +// -------------Email service----------------- + +service EmailService { + rpc SendOrderConfirmation(SendOrderConfirmationRequest) returns (Empty) {} +} + +message OrderItem { + CartItem item = 1; + Money cost = 2; +} + +message OrderResult { + string order_id = 1; + string shipping_tracking_id = 2; + Money shipping_cost = 3; + Address shipping_address = 4; + repeated OrderItem items = 5; +} + +message SendOrderConfirmationRequest { + string email = 1; + OrderResult order = 2; +} + + +// -------------Checkout service----------------- + +service CheckoutService { + rpc PlaceOrder(PlaceOrderRequest) returns (PlaceOrderResponse) {} +} + +message PlaceOrderRequest { + string user_id = 1; + string user_currency = 2; + + Address address = 3; + string email = 5; + CreditCardInfo credit_card = 6; +} + +message PlaceOrderResponse { + OrderResult order = 1; +} + +// ------------Ad service------------------ + +service AdService { + rpc GetAds(AdRequest) returns (AdResponse) {} +} + +message AdRequest { + // List of important key words from the current page describing the context. + repeated string context_keys = 1; +} + +message AdResponse { + repeated Ad ads = 1; +} + +message Ad { + // url to redirect to when an ad is clicked. + string redirect_url = 1; + + // short advertisement text to display. + string text = 2; +} + +// ------------Feature flag service------------------ + +service FeatureFlagService { + rpc GetFlag(GetFlagRequest) returns (GetFlagResponse) {} + rpc CreateFlag(CreateFlagRequest) returns (CreateFlagResponse) {} + rpc UpdateFlag(UpdateFlagRequest) returns (UpdateFlagResponse) {} + rpc ListFlags(ListFlagsRequest) returns (ListFlagsResponse) {} + rpc DeleteFlag(DeleteFlagRequest) returns (DeleteFlagResponse) {} +} + +message Flag { + string name = 1; + string description = 2; + bool enabled = 3; + google.protobuf.Timestamp created_at = 4; + google.protobuf.Timestamp updated_at = 5; +} + +message GetFlagRequest { + string name = 1; +} + +message GetFlagResponse { + Flag flag = 1; +} + +message CreateFlagRequest { + string name = 1; + string description = 2; + bool enabled = 3; +} + +message CreateFlagResponse { + Flag flag = 1; +} + +message UpdateFlagRequest { + string name = 1; + bool enabled = 2; +} + +message UpdateFlagResponse {} + +message ListFlagsRequest {} + +message ListFlagsResponse { + repeated Flag flag = 1; +} + +message DeleteFlagRequest { + string name = 1; +} + +message DeleteFlagResponse {} From 78d5114fd605b480a5d5ffef46dbd9f1cdb94e3c Mon Sep 17 00:00:00 2001 From: svrnm Date: Wed, 21 Dec 2022 19:08:46 +0100 Subject: [PATCH 6/7] Update CHANGELOG Signed-off-by: svrnm --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16a932afb1..1d2ec40742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,3 +156,5 @@ significant modifications will be credited to OpenTelemetry Authors. ([#617](https://github.com/open-telemetry/opentelemetry-demo/pull/617)) * Use `frontend-web` as service name for browser/web requests ([#628](https://github.com/open-telemetry/opentelemetry-demo/pull/628)) +* Add OTel java agent with JMX Metric Insights to kafka +([#654](https://github.com/open-telemetry/opentelemetry-demo/pull/654)) From 9e30e13ff692e55e42865b94425093bb0652e296 Mon Sep 17 00:00:00 2001 From: svrnm Date: Thu, 22 Dec 2022 08:08:48 +0100 Subject: [PATCH 7/7] Fix markdownlint error Signed-off-by: svrnm --- docs/services/kafka.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/services/kafka.md b/docs/services/kafka.md index 941edd20f7..023ffe30b8 100644 --- a/docs/services/kafka.md +++ b/docs/services/kafka.md @@ -9,7 +9,8 @@ the accounting and fraud detection services. This service relies on the OpenTelemetry Java Agent and the built in [JMX Metric Insight Module](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/jmx-metrics/javaagent) -to capture [kafka broker metrics](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/instrumentation/jmx-metrics/javaagent/kafka-broker.md) and send them off to the collector via OTLP. +to capture [kafka broker metrics](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/instrumentation/jmx-metrics/javaagent/kafka-broker.md) +and send them off to the collector via OTLP. The agent is passed into the process using the `-javaagent` command line argument. Command line arguments are added through the `KAFKA_OPTS`