From 080a46da909c55d366e4068e50b35694cd9dc5f5 Mon Sep 17 00:00:00 2001 From: taylan isikdemir Date: Wed, 14 Feb 2024 19:20:59 -0800 Subject: [PATCH] Docker compose setup for async workflow kafka queue --- Dockerfile | 10 +- config/development_async_wf_kafka_queue.yaml | 139 +++++++++++++++++++ docker/config_template.yaml | 11 ++ docker/docker-compose-async-wf-kafka.yml | 90 ++++++++++++ docker/start.sh | 39 ++++++ scripts/buildkite/docker-build.sh | 1 + 6 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 config/development_async_wf_kafka_queue.yaml create mode 100644 docker/docker-compose-async-wf-kafka.yml diff --git a/Dockerfile b/Dockerfile index 8358eecd8ab..006053b3e7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,7 +85,7 @@ ENTRYPOINT ["/docker-entrypoint.sh"] CMD /start-cadence.sh -# All-in-one Cadence server +# All-in-one Cadence server (~450mb) FROM cadence-server AS cadence-auto-setup RUN apk add --update --no-cache ca-certificates py3-pip mysql-client @@ -95,6 +95,14 @@ COPY docker/start.sh /start.sh CMD /start.sh +# All-in-one Cadence server with Kafka (~550mb) +FROM cadence-auto-setup AS cadence-auto-setup-with-kafka + +RUN apk add openjdk11 +RUN wget https://archive.apache.org/dist/kafka/2.1.1/kafka_2.12-2.1.1.tgz -O kafka.tgz +RUN mkdir -p kafka +RUN tar -xvzf kafka.tgz --strip 1 -C kafka +ENV KAFKA_HOME /etc/cadence/kafka # Cadence CLI FROM alpine AS cadence-cli diff --git a/config/development_async_wf_kafka_queue.yaml b/config/development_async_wf_kafka_queue.yaml new file mode 100644 index 00000000000..1e6599441c0 --- /dev/null +++ b/config/development_async_wf_kafka_queue.yaml @@ -0,0 +1,139 @@ +persistence: + defaultStore: cass-default + visibilityStore: cass-visibility + numHistoryShards: 4 + datastores: + cass-default: + nosql: + pluginName: "cassandra" + hosts: "127.0.0.1" + keyspace: "cadence" + cass-visibility: + nosql: + pluginName: "cassandra" + hosts: "127.0.0.1" + keyspace: "cadence_visibility" + +ringpop: + name: cadence + bootstrapMode: hosts + bootstrapHosts: [ "127.0.0.1:7933", "127.0.0.1:7934", "127.0.0.1:7935" ] + maxJoinDuration: 30s + +services: + frontend: + rpc: + port: 7933 + grpcPort: 7833 + bindOnLocalHost: true + grpcMaxMsgSize: 33554432 + metrics: + statsd: + hostPort: "127.0.0.1:8125" + prefix: "cadence" + pprof: + port: 7936 + + matching: + rpc: + port: 7935 + grpcPort: 7835 + bindOnLocalHost: true + grpcMaxMsgSize: 33554432 + metrics: + statsd: + hostPort: "127.0.0.1:8125" + prefix: "cadence" + pprof: + port: 7938 + + history: + rpc: + port: 7934 + grpcPort: 7834 + bindOnLocalHost: true + grpcMaxMsgSize: 33554432 + metrics: + statsd: + hostPort: "127.0.0.1:8125" + prefix: "cadence" + pprof: + port: 7937 + + worker: + rpc: + port: 7939 + bindOnLocalHost: true + metrics: + statsd: + hostPort: "127.0.0.1:8125" + prefix: "cadence" + pprof: + port: 7940 + +clusterGroupMetadata: + failoverVersionIncrement: 10 + primaryClusterName: "cluster0" + currentClusterName: "cluster0" + clusterGroup: + cluster0: + enabled: true + initialFailoverVersion: 0 + newInitialFailoverVersion: 1 # migrating to this new failover version + rpcAddress: "localhost:7833" # this is to let worker service and XDC replicator connected to the frontend service. In cluster setup, localhost will not work + rpcTransport: "grpc" + +dcRedirectionPolicy: + policy: "noop" + toDC: "" + +archival: + history: + status: "enabled" + enableRead: true + provider: + filestore: + fileMode: "0666" + dirMode: "0766" + gstorage: + credentialsPath: "/tmp/gcloud/keyfile.json" + visibility: + status: "enabled" + enableRead: true + provider: + filestore: + fileMode: "0666" + dirMode: "0766" + +domainDefaults: + archival: + history: + status: "enabled" + URI: "file:///tmp/cadence_archival/development" + visibility: + status: "enabled" + URI: "file:///tmp/cadence_vis_archival/development" + +dynamicconfig: + client: filebased + configstore: + pollInterval: "10s" + updateRetryAttempts: 2 + FetchTimeout: "2s" + UpdateTimeout: "2s" + filebased: + filepath: "config/dynamicconfig/development.yaml" + pollInterval: "10s" + +blobstore: + filestore: + outputDirectory: "/tmp/blobstore" + +asyncWorkflowQueues: + queue1: + type: "kafka" + config: + connection: + brokers: + - "localhost:9092" + topic: "async-wf-topic1" diff --git a/docker/config_template.yaml b/docker/config_template.yaml index d6437e2282f..dcb96da4c49 100644 --- a/docker/config_template.yaml +++ b/docker/config_template.yaml @@ -304,3 +304,14 @@ authorization: jwtCredentials: algorithm: "RS256" publicKey: {{ default .Env.OAUTH_PUBLIC_KEY "" }} + +{{- if .Env.ASYNC_WF_KAFKA_QUEUE_ENABLED }} +asyncWorkflowQueues: + queue1: + type: "kafka" + config: + connection: + brokers: + - {{ default .Env.KAFKA_SEEDS "" }}:{{ default .Env.KAFKA_PORT "9092" }} + topic: {{ default .Env.ASYNC_WF_KAFKA_QUEUE_TOPIC "default-topic" }} +{{- end }} diff --git a/docker/docker-compose-async-wf-kafka.yml b/docker/docker-compose-async-wf-kafka.yml new file mode 100644 index 00000000000..59ade158066 --- /dev/null +++ b/docker/docker-compose-async-wf-kafka.yml @@ -0,0 +1,90 @@ +version: '3' +services: + cassandra: + image: cassandra:4.1.1 + ports: + - "9042:9042" + environment: + - "MAX_HEAP_SIZE=256M" + - "HEAP_NEWSIZE=128M" + healthcheck: + test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"] + interval: 15s + timeout: 30s + retries: 10 + prometheus: + image: prom/prometheus:latest + volumes: + - ./prometheus:/etc/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - '9090:9090' + node-exporter: + image: prom/node-exporter + ports: + - '9100:9100' + cadence: + image: ubercadence/server:master-auto-setup-with-kafka + ports: + - "8000:8000" + - "8001:8001" + - "8002:8002" + - "8003:8003" + - "7933:7933" + - "7934:7934" + - "7935:7935" + - "7939:7939" + - "7833:7833" + environment: + - "ASYNC_WF_KAFKA_QUEUE_ENABLED=true" + - "ASYNC_WF_KAFKA_QUEUE_TOPIC=async-wf-topic1" # this should match the topic name in the config yml + - "KAFKA_SEEDS=kafka" + - "KAFKA_PORT=9092" + - "ZOOKEEPER_SEEDS=zookeeper" + - "ZOOKEEPER_PORT=2181" + - "CASSANDRA_SEEDS=cassandra" + - "PROMETHEUS_ENDPOINT_0=0.0.0.0:8000" + - "PROMETHEUS_ENDPOINT_1=0.0.0.0:8001" + - "PROMETHEUS_ENDPOINT_2=0.0.0.0:8002" + - "PROMETHEUS_ENDPOINT_3=0.0.0.0:8003" + - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml" + - "LOG_LEVEL=debug" + depends_on: + cassandra: + condition: service_healthy + prometheus: + condition: service_started + kafka: + condition: service_started + cadence-web: + image: ubercadence/web:latest + environment: + - "CADENCE_TCHANNEL_PEERS=cadence:7933" + ports: + - "8088:8088" + depends_on: + - cadence + grafana: + image: grafana/grafana + volumes: + - ./grafana:/etc/grafana + user: "1000" + depends_on: + - prometheus + ports: + - '3000:3000' + zookeeper: + image: wurstmeister/zookeeper:3.4.6 + ports: + - "2181:2181" + kafka: + image: wurstmeister/kafka:2.12-2.1.1 + depends_on: + - zookeeper + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 diff --git a/docker/start.sh b/docker/start.sh index a990d0907f9..23335ca6b3f 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -155,9 +155,48 @@ wait_for_db() { fi } +wait_for_async_wf_queue_kafka() { + ready="false" + while [ "$ready" != "true" ]; do + brokers=$(echo dump | nc "$ZOOKEEPER_SEEDS" "$ZOOKEEPER_PORT" | grep brokers | wc -l) + if [ "$brokers" -gt 0 ]; then + ready="true" + else + echo 'waiting for kafka broker to show up in zookeeper' + sleep 3 + fi + done + + echo 'kafka broker started' +} + +setup_async_wf_queue() { + if [ "$ASYNC_WF_KAFKA_QUEUE_ENABLED" != "true" ]; then + return + fi + + wait_for_async_wf_queue_kafka + + sh $KAFKA_HOME/bin/kafka-topics.sh --create --if-not-exists \ + --zookeeper "$ZOOKEEPER_SEEDS:$ZOOKEEPER_PORT" \ + --topic "$ASYNC_WF_KAFKA_QUEUE_TOPIC" \ + --partitions 10 \ + --replication-factor 1 + + created=$(sh $KAFKA_HOME/bin/kafka-topics.sh --describe --topic $ASYNC_WF_KAFKA_QUEUE_TOPIC --zookeeper zookeeper:2181 | grep "Topic:$ASYNC_WF_KAFKA_QUEUE_TOPIC") + if [ -z "$created" ]; then + echo 'kafka topic is not created' + exit 1 + fi + + echo "Kafka topic $ASYNC_WF_KAFKA_QUEUE_TOPIC for async workflows created" +} + wait_for_db if [ "$SKIP_SCHEMA_SETUP" != true ]; then setup_schema fi +setup_async_wf_queue + exec /start-cadence.sh diff --git a/scripts/buildkite/docker-build.sh b/scripts/buildkite/docker-build.sh index c1d0685bffc..749fd342699 100755 --- a/scripts/buildkite/docker-build.sh +++ b/scripts/buildkite/docker-build.sh @@ -6,6 +6,7 @@ echo "Building docker images for $BUILDKITE_MESSAGE" docker build . -f Dockerfile -t ubercadence/server:master --build-arg TARGET=server docker build . -f Dockerfile -t ubercadence/server:master-auto-setup --build-arg TARGET=auto-setup +docker build . -f Dockerfile -t ubercadence/server:master-auto-setup-with-kafka --build-arg TARGET=auto-setup-with-kafka docker build . -f Dockerfile -t ubercadence/cli:master --build-arg TARGET=cli docker build . -f Dockerfile -t ubercadence/cadence-bench:master --build-arg TARGET=bench docker build . -f Dockerfile -t ubercadence/cadence-canary:master --build-arg TARGET=canary