Add async workflow integration test step to CI

.buildkite/pipeline-master.yml

@@ -138,6 +138,21 @@ steps:
           run: integration-test-ndc-postgres
           config: docker/buildkite/docker-compose.yml
 
+  - label: ":golang: async wf integration test with kafka"
+    agents:
+      queue: "workers"
+      docker: "*"
+    commands:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "go test -timeout 60s -run ^TestAsyncWFIntegrationSuite -tags asyncwfintegration -count 1 -v github.com/uber/cadence/host"
+    retry:
+      automatic:
+        limit: 1
+    plugins:
+      - docker-compose#v3.0.0:
+          run: integration-test-async-wf
+          config: docker/buildkite/docker-compose.yml
+
   - wait
 
   - label: ":golang: code-coverage"

.buildkite/pipeline-pull-request.yml

@@ -68,6 +68,7 @@ steps:
       - docker-compose#v3.0.0:
           run: integration-test-cassandra
           config: docker/buildkite/docker-compose-es7.yml
+
   - label: ":golang: integration test with cassandra with OpenSearch v2"
     agents:
       queue: "workers"
@@ -84,6 +85,7 @@ steps:
       - docker-compose#v3.0.0:
           run: integration-test-cassandra
           config: docker/buildkite/docker-compose-opensearch2.yml
+
   - label: ":golang: integration ndc test with cassandra"
     agents:
       queue: "workers"
@@ -169,6 +171,21 @@ steps:
           run: integration-test-ndc-postgres
           config: docker/buildkite/docker-compose.yml
 
+  - label: ":golang: async wf integration test with kafka"
+    agents:
+      queue: "workers"
+      docker: "*"
+    commands:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "go test -timeout 60s -run ^TestAsyncWFIntegrationSuite -tags asyncwfintegration -count 1 -v github.com/uber/cadence/host"
+    retry:
+      automatic:
+        limit: 1
+    plugins:
+      - docker-compose#v3.0.0:
+          run: integration-test-async-wf
+          config: docker/buildkite/docker-compose.yml
+
   - wait
 
   - label: ":golang: code-coverage"

docker/buildkite/docker-compose-local-async-wf.yml

@@ -67,6 +67,7 @@ services:
       - "7935:7935"
       - "7939:7939"
     environment:
+      - "ASYNC_WF_KAFKA_QUEUE_TOPIC=async-wf-topic1"
       - "CASSANDRA=1"
       - "CASSANDRA_SEEDS=cassandra"
       - "ES_SEEDS=elasticsearch"

docker/buildkite/docker-compose.yml

@@ -55,6 +55,9 @@ services:
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
       KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      # for async wf tests, create a topic with 10 partitions and 1 replica
+      # topic name must match ASYNC_WF_KAFKA_QUEUE_TOPIC
+      KAFKA_CREATE_TOPICS: "async-wf-topic1:10:1"
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.22
@@ -281,6 +284,34 @@ services:
         aliases:
           - integration-test-ndc
 
+  integration-test-async-wf:
+    build:
+      context: ../../
+      dockerfile: ./docker/buildkite/Dockerfile
+    environment:
+      - "ASYNC_WF_KAFKA_QUEUE_TOPIC=async-wf-topic1"
+      - "CASSANDRA=1"
+      - "CASSANDRA_SEEDS=cassandra"
+      - "ES_SEEDS=elasticsearch"
+      - "KAFKA_SEEDS=kafka"
+      - "KAFKA_PORT=9092"
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_BUILD_ID
+      - BUILDKITE_BUILD_NUMBER
+    depends_on:
+      cassandra:
+        condition: service_healthy
+      kafka:
+        condition: service_started
+    volumes:
+      - ../../:/cadence
+      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+    networks:
+      services-network:
+        aliases:
+          - integration-test-async-wf
+
   coverage-report:
     build:
       context: ../../

host/testdata/integration_async_wf_with_kafka_cluster.yaml

@@ -10,7 +10,7 @@ asyncwfqueues:
       connection:
         brokers:
           - "${KAFKA_SEEDS}:${KAFKA_PORT}"
-      topic: async-wf-topic1
+      topic: "${ASYNC_WF_KAFKA_QUEUE_TOPIC}"
 messagingclientconfig:
   usemock: true
 historyconfig: