shrink cassandra memmory consumption and unify cassandra setup in all…

… docker-compose files (#5482)
cadence-workflow · Dec 18, 2023 · 07018fe · 07018fe
1 parent 40018f1
commit 07018fe
Show file tree

Hide file tree

Showing 13 changed files with 172 additions and 39 deletions.
diff --git a/docker/buildkite/docker-compose-es7.yml b/docker/buildkite/docker-compose-es7.yml
@@ -3,10 +3,18 @@ version: "3.5"
 services:
   cassandra:
     image: cassandra:4.1.3
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
     networks:
       services-network:
         aliases:
           - cassandra
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
 
   zookeeper:
     image: wurstmeister/zookeeper:3.4.6
@@ -53,9 +61,12 @@ services:
       - BUILDKITE_BUILD_ID
       - BUILDKITE_BUILD_NUMBER
     depends_on:
-      - cassandra
-      - elasticsearch
-      - kafka
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      kafka:
+        condition: service_started
     volumes:
       - ../../:/cadence
       - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent

diff --git a/docker/buildkite/docker-compose-local-es7.yml b/docker/buildkite/docker-compose-local-es7.yml
@@ -3,12 +3,20 @@ version: "3.5"
 services:
   cassandra:
     image: cassandra:4.1.3
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
     ports:
       - "9042:9042"
     networks:
       services-network:
         aliases:
           - cassandra
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
 
   zookeeper:
     image: wurstmeister/zookeeper:3.4.6
@@ -68,9 +76,12 @@ services:
       - "TEST_TAG=esintegration"
       - "ES_VERSION=v7"
     depends_on:
-      - cassandra
-      - elasticsearch
-      - kafka
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      kafka:
+        condition: service_started
     volumes:
       - ../../:/cadence
     networks:

diff --git a/docker/buildkite/docker-compose-local.yml b/docker/buildkite/docker-compose-local.yml
@@ -3,6 +3,9 @@ version: "3.5"
 services:
   cassandra:
     image: cassandra:4.1.3
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
     expose:
       - "9042"
     networks:

diff --git a/docker/buildkite/docker-compose-opensearch2.yml b/docker/buildkite/docker-compose-opensearch2.yml
@@ -3,10 +3,18 @@ version: "3.5"
 services:
   cassandra:
     image: cassandra:4.1.3
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
     networks:
       services-network:
         aliases:
           - cassandra
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
 
   zookeeper:
     image: wurstmeister/zookeeper:3.4.6
@@ -54,9 +62,12 @@ services:
       - BUILDKITE_BUILD_ID
       - BUILDKITE_BUILD_NUMBER
     depends_on:
-      - cassandra
-      - elasticsearch
-      - kafka
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      kafka:
+        condition: service_started
     volumes:
       - ../../:/cadence
       - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent

diff --git a/docker/buildkite/docker-compose.yml b/docker/buildkite/docker-compose.yml
@@ -3,6 +3,9 @@ version: "3.5"
 services:
   cassandra:
     image: cassandra:4.1.3
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
     networks:
       services-network:
         aliases:

diff --git a/docker/docker-compose-es-v7.yml b/docker/docker-compose-es-v7.yml
@@ -4,6 +4,14 @@ services:
     image: cassandra:4.1.3
     ports:
       - "9042:9042"
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
   prometheus:
     image: prom/prometheus:latest
     volumes:
@@ -56,10 +64,14 @@ services:
       - "ES_VERSION=v7"
       - "KAFKA_SEEDS=kafka"
     depends_on:
-      - cassandra
-      - prometheus
-      - kafka
-      - elasticsearch
+      cassandra:
+        condition: service_healthy
+      prometheus:
+        condition: service_started
+      kafka:
+        condition: service_started
+      elasticsearch:
+        condition: service_started
   cadence-web:
     image: ubercadence/web:latest
     environment:

diff --git a/docker/docker-compose-es.yml b/docker/docker-compose-es.yml
@@ -4,6 +4,14 @@ services:
     image: cassandra:4.1.3
     ports:
       - "9042:9042"
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
   prometheus:
     image: prom/prometheus:latest
     volumes:
@@ -55,10 +63,14 @@ services:
       - "ES_SEEDS=elasticsearch"
       - "KAFKA_SEEDS=kafka"
     depends_on:
-      - cassandra
-      - prometheus
-      - kafka
-      - elasticsearch
+      cassandra:
+        condition: service_healthy
+      prometheus:
+        condition: service_started
+      kafka:
+        condition: service_started
+      elasticsearch:
+        condition: service_started
   cadence-web:
     image: ubercadence/web:latest
     environment:

diff --git a/docker/docker-compose-http-api.yml b/docker/docker-compose-http-api.yml
@@ -4,6 +4,14 @@ services:
     image: cassandra:4.1.3
     ports:
       - "9042:9042"
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
   prometheus:
     image: prom/prometheus:latest
     volumes:
@@ -39,8 +47,10 @@ services:
       - "FRONTEND_HTTP_PORT=8800"
       - "FRONTEND_HTTP_PROCEDURES=uber.cadence.api.v1.WorkflowAPI::StartWorkflowExecution,uber.cadence.api.v1.WorkflowAPI::SignalWorkflowExecution,uber.cadence.api.v1.WorkflowAPI::QueryWorkflow,uber.cadence.api.v1.WorkflowAPI::DescribeWorkflowExecution,uber.cadence.api.v1.VisibilityAPI::ListWorkflowExecutions"
     depends_on:
-      - cassandra
-      - prometheus
+      cassandra:
+        condition: service_healthy
+      prometheus:
+        condition: service_started
   cadence-web:
     image: ubercadence/web:latest
     environment:

diff --git a/docker/docker-compose-multiclusters-cass-mysql-es.yaml b/docker/docker-compose-multiclusters-cass-mysql-es.yaml
@@ -4,6 +4,14 @@ services:
     image: cassandra:4.1.3
     ports:
       - "9042:9042"
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
   mysql:
     image: mysql:8.0
     ports:
@@ -71,10 +79,14 @@ services:
       - "VISIBILITY_NAME=cadence-visibility-primary"
       - "FRONTEND_SERVICE=cadence"
     depends_on:
-      - cassandra
-      - prometheus
-      - kafka
-      - elasticsearch
+      cassandra:
+        condition: service_healthy
+      prometheus:
+        condition: service_started
+      kafka:
+        condition: service_started
+      elasticsearch:
+        condition: service_started
   cadence-secondary:
     image: ubercadence/server:master-auto-setup
     ports:

diff --git a/docker/docker-compose-multiclusters-es.yml b/docker/docker-compose-multiclusters-es.yml
@@ -4,6 +4,14 @@ services:
     image: cassandra:4.1.3
     ports:
       - "9042:9042"
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
   prometheus:
     image: prom/prometheus:latest
     volumes:
@@ -65,10 +73,14 @@ services:
       - "VISIBILITY_NAME=cadence-visibility-primary"
       - "FRONTEND_SERVICE=cadence"
     depends_on:
-      - cassandra
-      - prometheus
-      - kafka
-      - elasticsearch
+      cassandra:
+        condition: service_healthy
+      prometheus:
+        condition: service_started
+      kafka:
+        condition: service_started
+      elasticsearch:
+        condition: service_started
   cadence-secondary:
     image: ubercadence/server:master-auto-setup
     ports:
@@ -103,10 +115,14 @@ services:
       - "VISIBILITY_NAME=cadence-visibility-secondary"
       - "FRONTEND_SERVICE=cadence-secondary"
     depends_on:
-      - cassandra
-      - prometheus
-      - kafka
-      - elasticsearch
+      cassandra:
+        condition: service_healthy
+      prometheus:
+        condition: service_started
+      kafka:
+        condition: service_started
+      elasticsearch:
+        condition: service_started
   cadence-web:
     image: ubercadence/web:latest
     environment:

diff --git a/docker/docker-compose-multiclusters.yml b/docker/docker-compose-multiclusters.yml
@@ -4,6 +4,14 @@ services:
     image: cassandra:4.1.3
     ports:
       - "9042:9042"
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
   prometheus:
     image: prom/prometheus:latest
     volumes:
@@ -39,8 +47,10 @@ services:
       - "STATSD_HISTORY_PREFIX=cadence-history-primary"
       - "STATSD_WORKER_PREFIX=cadence-worker-primary"
     depends_on:
-      - cassandra
-      - prometheus
+      cassandra:
+        condition: service_healthy
+      prometheus:
+        condition: service_started
   cadence-secondary:
     image: ubercadence/server:master-auto-setup
     ports:
@@ -69,8 +79,10 @@ services:
       - "STATSD_HISTORY_PREFIX=cadence-history-secondary"
       - "STATSD_WORKER_PREFIX=cadence-worker-secondary"
     depends_on:
-      - cassandra
-      - prometheus
+      cassandra:
+        condition: service_healthy
+      prometheus:
+        condition: service_started
   cadence-web:
     image: ubercadence/web:latest
     environment:

diff --git a/docker/docker-compose-statsd.yml b/docker/docker-compose-statsd.yml
@@ -4,6 +4,14 @@ services:
     image: cassandra:4.1.3
     ports:
       - "9042:9042"
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
   statsd:
     image: graphiteapp/graphite-statsd
     ports:
@@ -24,8 +32,10 @@ services:
       - "STATSD_ENDPOINT=statsd:8125"
       - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml"
     depends_on:
-      - cassandra
-      - statsd
+      cassandra:
+        condition: service_healthy
+      statsd:
+        condition: service_started
   cadence-web:
     image: ubercadence/web:latest
     environment: