feat(mojaloop/#3264): tls cert reload issue (#454)

* fix: issue with tls cert reloading * chore: updated docker stack * feat: added pm4ml tests * fix: container names * chore: update deps * fix: integration tests * fix: cert generation * fix: tests * fix: issues * chore(snapshot): 23.1.0-snapshot.0
mojaloop · Jul 7, 2023 · d5fa071 · d5fa071
1 parent 81e4bfe
commit d5fa071
Show file tree

Hide file tree

Showing 35 changed files with 6,410 additions and 1,566 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -215,6 +215,64 @@ jobs:
       # - store_test_results:
       #     path: ./test/results
 
+  test-integration-pm4ml:
+    executor: default-machine
+    environment: *defaults_environment
+    steps:
+      - checkout
+      - run:
+          <<: *defaults_configure_nvm
+      - run:
+          name: Install Yarn dependencies
+          command: yarn install --immutable
+      - attach_workspace:
+          at: /tmp
+      - run:
+          name: Load the pre-built docker image from workspace
+          command: docker load -i /tmp/docker-image.tar
+      ## This shouldn't be needed as the docker image is pre-built by build-local job
+      # - run:
+      #     name: Build docker-compose
+      #     command: |
+      #       docker-compose build
+      - run:
+          name: Generate certificates
+          command: |
+            cd docker/haproxy/tls
+            sh createSecrets.sh
+            cd $CIRCLE_WORKING_DIRECTORY
+      - run:
+          name: Start docker-compose in PM4ML mode
+          command: |
+            docker-compose -f ./docker-compose.yml -f ./docker-compose.pm4ml.yml up -d
+      - run:
+          name: Wait 4 Health Services
+          command: |
+            yarn run wait-4-docker
+      - run:
+          name: Execute Build
+          command: yarn run build
+      - run:
+          name: Execute PM4ML Integration Tests
+          command: |
+            run_int_tests() {
+              pushd /home/circleci/project/modules/$1
+              yarn run test:integration-pm4ml
+              popd
+              sleep 10
+            }
+            run_int_tests api-svc
+      ## TODO Capture results
+      # - run:
+      #     name: prepare test/results
+      #     command: |
+      #       mkdir -p test/results
+      #       mv junit.xml test/results
+      # - store_artifacts:
+      #     path: ./test/results
+      # - store_test_results:
+      #     path: ./test/results
+
   test-func:
     executor: default-machine
     environment: *defaults_environment
@@ -725,6 +783,19 @@ workflows:
                 - /feature*/
                 - /bugfix*/
 
+      - test-integration-pm4ml:
+          context: org-global
+          requires:
+            - setup
+            - build-local
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              ignore:
+                - /feature*/
+                - /bugfix*/
+
       - test-func:
           context: org-global
           requires:
@@ -793,6 +864,7 @@ workflows:
             - test-lint
             - test-unit
             - test-integration
+            - test-integration-pm4ml
             - vulnerability-check
             - audit-licenses
             - license-scan
@@ -822,6 +894,7 @@ workflows:
             - test-lint
             - test-unit
             - test-integration
+            - test-integration-pm4ml
             - vulnerability-check
             - audit-licenses
             - license-scan
@@ -842,6 +915,7 @@ workflows:
             - test-lint
             - test-unit
             - test-integration
+            - test-integration-pm4ml
             - vulnerability-check
             - audit-licenses
             - license-scan

diff --git a/.yarn/releases/yarn-3.2.0.cjs b/.yarn/releases/yarn-3.2.0.cjs
diff --git a/.yarn/releases/yarn-3.6.1.cjs b/.yarn/releases/yarn-3.6.1.cjs
diff --git a/.yarnrc.yml b/.yarnrc.yml
@@ -6,4 +6,4 @@ plugins:
   - path: .yarn/plugins/@yarnpkg/plugin-version.cjs
     spec: "@yarnpkg/plugin-version"
 
-yarnPath: .yarn/releases/yarn-3.2.0.cjs
+yarnPath: .yarn/releases/yarn-3.6.1.cjs
diff --git a/audit-ci.jsonc b/audit-ci.jsonc
@@ -32,6 +32,7 @@
         "GHSA-8x6c-cv3v-vp6g",
         // Some audit issues with api-snippets
         "GHSA-r6ch-mqf9-qc9w",
-        "GHSA-5r9g-qh6m-jxff"
+        "GHSA-5r9g-qh6m-jxff",
+        "GHSA-c2qf-rxjj-qqgw"
     ]
 }
diff --git a/docker-compose.pm4ml.yml b/docker-compose.pm4ml.yml
@@ -0,0 +1,43 @@
+version: '3.7'
+
+services:
+
+  sdk-scheme-adapter-api-svc:
+    env_file: ./modules/api-svc/test/config/integration-pm4ml.env
+    depends_on:
+      - mock-management-svc
+    command: yarn nx run modules-api-svc:start
+
+  mock-management-svc:
+    networks:
+        - mojaloop-net
+    image: mojaloop/sdk-scheme-adapter:local
+    build:
+      context: ./
+      dockerfile: ./Dockerfile
+    ports:
+      - "4005:4005"
+      - "5005:5005"
+    command: yarn nx run modules-api-svc:test:pm4ml-run-mock-servers
+    user: root # Set Root so we can install Curl for Healthchecks
+    healthcheck:
+      test: ["CMD-SHELL", "nc localhost 4005"]
+      interval: 5s
+      timeout: 10s
+      retries: 3
+
+  hubsimulator:
+    networks:
+      - mojaloop-net
+    image: haproxy:2.8.1-alpine
+    depends_on:
+      - ml-testing-toolkit
+    links:
+      - ml-testing-toolkit:ml-testing-toolkit
+    volumes:
+      - ./docker/haproxy/certs:/usr/local/etc/haproxy/certs
+      - ./docker/haproxy/tls:/usr/local/etc/haproxy/tls
+      - ./docker/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+    ports:
+      - "8443:8443"
+
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,6 +1,3 @@
-## TODOs
-# - Add Redis Insights
-
 version: '3.7'
 networks:
   mojaloop-net:
@@ -59,9 +56,11 @@ services:
 
   ml-testing-toolkit:
     networks:
-        - mojaloop-net
+      mojaloop-net:
+        aliases:
+          - ttkbackendsim
+          - ttkhubsim
     image: mojaloop/ml-testing-toolkit:v15.0.0
-    container_name: ml-testing-toolkit
     volumes:
       - "./docker/ml-testing-toolkit/spec_files:/opt/app/spec_files"
       - "./docker/ml-testing-toolkit/secrets:/opt/app/secrets"
@@ -88,36 +87,28 @@ services:
       - sh
       - /usr/share/nginx/start.sh
 
-  zookeeper:
-    networks:
-      - mojaloop-net
-    image: docker.io/bitnami/zookeeper:3.8
-    container_name: zookeeper
-    ports:
-      - 2181:2181
-    environment:
-      - ALLOW_ANONYMOUS_LOGIN=yes
-
   kafka:
     networks:
       - mojaloop-net
-    image: docker.io/bitnami/kafka:3.2
+    image: docker.io/bitnami/kafka:3.4.0
     container_name: kafka
     ports:
       - "9092:9092"
     environment:
+      # BITNAMI_DEBUG: "yes"
       ALLOW_PLAINTEXT_LISTENER: "yes"
       KAFKA_ADVERTISED_HOST_NAME: kafka
-      KAFKA_LISTENERS: LISTENER_DOCKER://kafka:29092,LISTENER_INTERN://kafka:9093,LISTENER_EXTERN://0.0.0.0:9092
-      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER://kafka:29092,LISTENER_INTERN://kafka:9093,LISTENER_EXTERN://127.0.0.1:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER:PLAINTEXT,LISTENER_INTERN:PLAINTEXT,LISTENER_EXTERN:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERN
+      KAFKA_CFG_LISTENERS: CONTROLLER://:9093,LISTENER_DOCKER://:29092,LISTENER_EXTERN://:9092
+      KAFKA_CFG_ADVERTISED_LISTENERS: LISTENER_DOCKER://kafka:29092,LISTENER_EXTERN://localhost:9092
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,LISTENER_DOCKER:PLAINTEXT,LISTENER_EXTERN:PLAINTEXT
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_CFG_MESSAGE_MAX_BYTES: 200000000
-      KAFKA_ENABLE_KRAFT: "no"
-    depends_on:
-      - zookeeper
+      KAFKA_CFG_NODE_ID: 1
+      KAFKA_CFG_PROCESS_ROLES: broker,controller
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: [email protected]:9093
+      KAFKA_ENABLE_KRAFT: "true"
     healthcheck:
       test: ["CMD" ,"/opt/bitnami/kafka/bin/kafka-broker-api-versions.sh","--bootstrap-server","kafka:9092"]
       timeout: 20s
@@ -161,19 +152,17 @@ services:
   init-kafka:
     networks:
       - mojaloop-net
-    image: docker.io/bitnami/kafka:3.2
+    image: docker.io/bitnami/kafka:3.4.0
     depends_on:
       - kafka
     entrypoint: [ '/bin/sh', '-c' ]
     command: |
       "
       # blocks until kafka is reachable
-      kafka-topics.sh --bootstrap-server kafka:9093 --list
-
+      kafka-topics.sh --bootstrap-server kafka:29092 --list
       echo -e 'Creating kafka topics'
-      kafka-topics.sh --bootstrap-server kafka:9093 --create --if-not-exists --topic topic-sdk-outbound-command-events --replication-factor 1 --partitions 1
-      kafka-topics.sh --bootstrap-server kafka:9093 --create --if-not-exists --topic topic-sdk-outbound-domain-events --replication-factor 1 --partitions 1
-
+      kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-sdk-outbound-command-events --replication-factor 1 --partitions 1
+      kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-sdk-outbound-domain-events --replication-factor 1 --partitions 1
       echo -e 'Successfully created the following topics:'
-      kafka-topics.sh --bootstrap-server kafka:9093 --list
+      kafka-topics.sh --bootstrap-server kafka:29092 --list
       "
diff --git a/docker/haproxy/haproxy.cfg b/docker/haproxy/haproxy.cfg
@@ -0,0 +1,44 @@
+global
+    log 127.0.0.1   local0
+    log 127.0.0.1   local1 debug
+    #log loghost    local0 info
+    maxconn 4096
+    tune.ssl.default-dh-param 2048
+    #chroot /usr/share/haproxy
+    #daemon
+    #debug
+    #quiet
+
+  defaults
+    log     global
+    mode    http
+    option  httplog
+    option  dontlognull
+    retries 3
+    option redispatch
+    maxconn 2000
+    timeout connect      5000
+    timeout client      50000
+    timeout server      50000
+
+  frontend localnodes
+    mode http
+    # bind *:8443 ssl crt /usr/local/etc/haproxy/certs/server/server.pem ca-file /usr/local/etc/haproxy/certs/ca/ca.pem verify required
+    bind *:8443 ssl crt /usr/local/etc/haproxy/tls/hub_server_cert.pem ca-file /usr/local/etc/haproxy/tls/hub_cacert.pem verify required
+    # bind *:8443 ssl crt /usr/local/etc/haproxy/tls/hub_server_cert.pem ca-file /usr/local/etc/haproxy/tls/hub_cacert.pem
+    default_backend nodes
+
+  backend nodes
+    mode http
+    balance roundrobin
+    option forwardfor
+    http-request set-header X-Forwarded-Port %[dst_port]
+    http-request add-header X-Forwarded-Proto https if { ssl_fc }
+    http-request set-header X-SSL                  %[ssl_fc]
+    http-request set-header X-SSL-Client-Verify    %[ssl_c_verify]
+    http-request set-header X-SSL-Client-DN        %{+Q}[ssl_c_s_dn]
+    http-request set-header X-SSL-Client-CN        %{+Q}[ssl_c_s_dn(cn)]
+    http-request set-header X-SSL-Issuer           %{+Q}[ssl_c_i_dn]
+    http-request set-header X-SSL-Client-NotBefore %{+Q}[ssl_c_notbefore]
+    http-request set-header X-SSL-Client-NotAfter  %{+Q}[ssl_c_notafter]
+    server server1 ml-testing-toolkit:4040
diff --git a/docker/haproxy/tls/createSecrets.sh b/docker/haproxy/tls/createSecrets.sh
@@ -0,0 +1,37 @@
+OUTPUT_DIR="."
+setopt +o nomatch
+rm *.key
+rm *.pem
+rm *.csr
+rm *.crt
+rm *.srl
+
+#####################
+
+## Generating Hub CA certificate
+openssl req -x509 -config openssl-hub-ca.cnf -newkey rsa:4096 -sha256 -nodes -out hub_cacert.pem -outform PEM
+
+## Generating DFSP CA certificate
+openssl req -x509 -config openssl-dfsp-ca.cnf -newkey rsa:4096 -sha256 -nodes -out dfsp_cacert.pem -outform PEM
+
+#####################
+
+## Generate Hub server csr
+openssl req -config openssl-hub-server.cnf -newkey rsa:4096 -sha256 -nodes -out hub_server.csr -outform PEM
+
+## Sign Hub server cert with DFSP CA
+openssl ca -batch -config openssl-dfsp-ca.cnf -policy signing_policy -extensions signing_req -out hub_server_cert.pem -infiles hub_server.csr
+
+## Append ca cert to PEM file
+cat dfsp_cacert.pem >> hub_server_cert.pem
+
+## Append key to PEM file
+cat hub_server_key.key >> hub_server_cert.pem
+
+#####################
+
+## Generate client csr
+openssl req -config openssl-dfsp-client.cnf -newkey rsa:4096 -sha256 -nodes -out dfsp_client.csr -outform PEM
+
+## Sign DFSP client cert with Hub CA
+openssl ca -batch -config openssl-hub-ca.cnf -policy signing_policy -extensions signing_req -out dfsp_client_cert.crt -infiles dfsp_client.csr
diff --git a/docker/haproxy/tls/index.txt b/docker/haproxy/tls/index.txt
@@ -0,0 +1,4 @@
+V	260401125934Z		09	unknown	/C=US/ST=MD/L=Baltimore/O=Hub Client/OU=Payments/CN=dfspclient
+V	260402082307Z		0A	unknown	/C=US/ST=MD/L=Baltimore/O=Hub Client/OU=Payments/CN=dfspclient
+V	260402082703Z		0B	unknown	/C=US/ST=MD/L=Baltimore/O=Test Server, Limited/OU=Payments/CN=hubsimulator
+V	260402082705Z		0C	unknown	/C=US/ST=MD/L=Baltimore/O=Hub Client/OU=Payments/CN=dfspclient
diff --git a/docker/haproxy/tls/index.txt.attr b/docker/haproxy/tls/index.txt.attr
@@ -0,0 +1 @@
+unique_subject = no