feat: added two payee sdks and ttk as hub in func tests (#413)

* chore: added improved ttk dynamic testcase

* feat: added new payee sdk and ttk as hub

* feat: added a script for calculating positions for the demo

* feat(mojaloop/2949): [sdk-scheme-adapter][private-shared-lib] Update private-shared-lib to use v0.2

feat(mojaloop/2949): [sdk-scheme-adapter][private-shared-lib] Update private-shared-lib to use v0.2.0 of @mojaloop/platform-shared-lib - mojaloop/project#2949
- upgraded @mojaloop/platform-shared-lib-nodejs-kafka-client-lib & @mojaloop/platform-shared-lib-messaging-types-lib dependencies, and re-factored messaging/kafka consumers/producers to use the raw Producers/Consumers
- removed dependencies from ncurc since we can now upgrade them as normal
- created IRawMessageHeader alias as IRawIMessageHeaders to reduce the re-factoring required

* fix to remove resolutions from main package.json and yarn.lock

* fix: ttk rule

* feat: increased-the-limits-and-2000-transfers-passed

* chore: updated ttk docker version in func tests

* chore:parameterized api server payload limit

* chore: parameterized kafka limits

* chore: updated deps

* fix: added name fields in nx project configs

* fix: integration tests

* chore: addressed pr comment

* chore: disabled the log for bulk requests

Co-authored-by: Miguel de Barros <[email protected]>

docker-compose.yml

@@ -110,6 +110,7 @@ services:
       KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERN
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CFG_MESSAGE_MAX_BYTES: 200000000
     depends_on:
       - zookeeper
     healthcheck:

modules/api-svc/.env.example

@@ -154,3 +154,7 @@ MGMT_API_WS_PORT=4005
 # Set to true to enable the use of PM4ML-related services e.g MCM, Management API service
 # when running the scheme-adapter as a mojaloop connector component within Payment Manager for Mojaloop.
 PM4ML_ENABLED=false
+
+# Maximum payload limits
+FSPIOP_API_SERVER_MAX_REQUEST_BYTES=209715200
+BACKEND_API_SERVER_MAX_REQUEST_BYTES=209715200

modules/api-svc/package.json

@@ -104,7 +104,7 @@
     "jest": "^29.3.0",
     "jest-junit": "^14.0.1",
     "nock": "^13.2.9",
-    "npm-check-updates": "^16.3.16",
+    "npm-check-updates": "^16.3.18",
     "openapi-response-validator": "^12.0.2",
     "openapi-typescript": "^5.4.1",
     "redis-mock": "^0.56.3",

modules/api-svc/project.json

@@ -1,4 +1,5 @@
 {
+  "name": "modules-api-svc",
   "root": "modules/api-svc/",
   "sourceRoot": "modules/api-svc/src",
   "projectType": "application",

modules/api-svc/src/InboundServer/index.js

@@ -95,7 +95,7 @@ class InboundApi extends EventEmitter {
 
         api.use(middlewares.createErrorHandler(logger));
         api.use(middlewares.createRequestIdGenerator());
-        api.use(middlewares.createHeaderValidator(logger));
+        api.use(middlewares.createHeaderValidator(conf, logger));
         if (conf.validateInboundJws) {
             const jwsExclusions = conf.validateInboundPutPartiesJws ? [] : ['putParties'];
             api.use(middlewares.createJwsValidator(logger, jwsVerificationKeys, jwsExclusions));

modules/api-svc/src/InboundServer/middlewares.js

@@ -174,7 +174,7 @@ const createRequestIdGenerator = () => async (ctx, next) => {
  * @return {Function}
  */
 //
-const createHeaderValidator = (logger) => async (
+const createHeaderValidator = (conf, logger) => async (
     ctx,
     next,
     resources = defaultProtocolResources,
@@ -264,7 +264,7 @@ const createHeaderValidator = (logger) => async (
     }
 
     try {
-        ctx.request.body = await coBody.json(ctx.req);
+        ctx.request.body = await coBody.json(ctx.req, { limit: conf.fspiopApiServerMaxRequestBytes });
     }
     catch(err) {
         // error parsing body
@@ -356,7 +356,9 @@ const createLogger = (logger) => async (ctx, next) => {
         path: ctx.path,
         method: ctx.method
     }});
-    if (!ctx.state.logExcludePaths.includes(ctx.path)) {
+    await ctx.state.logger.log('Request received');
+    // TODO: we need to disable the following log message based on a configurable parameter like DEBUG
+    if (!ctx.state.logExcludePaths.includes(ctx.path) && !ctx.path.startsWith('/bulk')) {
         ctx.state.logger.push({body: ctx.request.body}).log('Request received');
     }
     try {

modules/api-svc/src/OutboundServer/index.js

@@ -38,7 +38,7 @@ class OutboundApi extends EventEmitter {
 
         this._api.use(middlewares.createErrorHandler(this._logger));
         this._api.use(middlewares.createRequestIdGenerator());
-        this._api.use(koaBody()); // outbound always expects application/json
+        this._api.use(koaBody({ formidable: { maxFieldsSize: conf.backendApiServerMaxRequestBytes } })); // outbound always expects application/json
         this._api.use(middlewares.applyState({ cache, wso2, conf, metricsClient, logExcludePaths, eventProducer, eventLogger }));
         this._api.use(middlewares.createLogger(this._logger));
 

modules/api-svc/src/config.js

@@ -209,4 +209,7 @@ module.exports = {
     allowDifferentTransferTransactionId: env.get('ALLOW_DIFFERENT_TRANSFER_TRANSACTION_ID').default('false').asBool(),
 
     pm4mlEnabled: env.get('PM4ML_ENABLED').default('false').asBool(),
+
+    fspiopApiServerMaxRequestBytes: env.get('FSPIOP_API_SERVER_MAX_REQUEST_BYTES').default('209715200').asIntPositive(), // Default is 200mb
+    backendApiServerMaxRequestBytes: env.get('BACKEND_API_SERVER_MAX_REQUEST_BYTES').default('209715200').asIntPositive(), // Default is 200mb
 };

modules/outbound-command-event-handler/config/default.json

@@ -12,12 +12,15 @@
       "brokerList": "localhost:9092",
       "groupId": "command_events_consumer_group",
       "clientId": "command_events_consumer_client_id",
-      "topics": ["topic-sdk-outbound-command-events"]
+      "topics": ["topic-sdk-outbound-command-events"],
+      "messageMaxBytes": 200000000
     },
     "DOMAIN_EVENT_PRODUCER": {
       "brokerList": "localhost:9092",
       "clientId": "domain_events_producer_client_id",
-      "topic": "topic-sdk-outbound-domain-events"
+      "topic": "topic-sdk-outbound-domain-events",
+      "messageMaxBytes": 200000000,
+      "compressionCodec": "none"
     }
   }
 }

modules/outbound-command-event-handler/package.json

@@ -67,7 +67,7 @@
     "eslint": "^8.27.0",
     "jest": "^29.3.0",
     "nodemon": "^2.0.20",
-    "npm-check-updates": "^16.3.16",
+    "npm-check-updates": "^16.3.18",
     "replace": "^1.2.2",
     "standard-version": "^9.5.0",
     "ts-jest": "^29.0.3",

modules/outbound-command-event-handler/project.json

@@ -1,4 +1,5 @@
 {
+  "name": "modules-outbound-command-event-handler",
   "root": "modules/outbound-command-event-handler/",
   "sourceRoot": "modules/outbound-command-event-handler/src",
   "projectType": "application",

modules/outbound-command-event-handler/src/shared/config.ts

@@ -97,6 +97,12 @@ const config = Convict({
                 default: ['topic-sdk-outbound-command-events'],
                 env: 'COMMAND_EVENT_CONSUMER_TOPICS',
             },
+            messageMaxBytes: {
+                doc: 'messageMaxBytes',
+                format: Number,
+                default: 200000000,
+                env: 'COMMAND_EVENT_CONSUMER_MESSAGE_MAX_BYTES',
+            },
         },
         DOMAIN_EVENT_PRODUCER: {
             brokerList: {
@@ -117,6 +123,18 @@ const config = Convict({
                 default: 'topic-sdk-outbound-domain-events',
                 env: 'DOMAIN_EVENT_PRODUCER_TOPIC',
             },
+            messageMaxBytes: {
+                doc: 'messageMaxBytes',
+                format: Number,
+                default: 200000000,
+                env: 'DOMAIN_EVENT_PRODUCER_MESSAGE_MAX_BYTES',
+            },
+            compressionCodec: {
+                doc: 'compressionCodec',
+                format: String,
+                default: 'none',
+                env: 'DOMAIN_EVENT_PRODUCER_COMPRESSION_CODEC',
+            },
         },
     },
     MAX_ITEMS_PER_BATCH: {

modules/outbound-domain-event-handler/config/default.json

@@ -12,12 +12,15 @@
       "brokerList": "localhost:9092",
       "groupId": "domain_events_consumer_group",
       "clientId": "domain_events_consumer_client_id",
-      "topics": ["topic-sdk-outbound-domain-events"]
+      "topics": ["topic-sdk-outbound-domain-events"],
+      "messageMaxBytes": 200000000
     },
     "COMMAND_EVENT_PRODUCER": {
       "brokerList": "localhost:9092",
       "clientId": "command_events_producer_client_id",
-      "topic": "topic-sdk-outbound-command-events"
+      "topic": "topic-sdk-outbound-command-events",
+      "messageMaxBytes": 200000000,
+      "compressionCodec": "none"
     }
   }
 }

modules/outbound-domain-event-handler/package.json

@@ -65,7 +65,7 @@
         "eslint": "^8.27.0",
         "jest": "^29.3.0",
         "nodemon": "^2.0.20",
-        "npm-check-updates": "^16.3.16",
+        "npm-check-updates": "^16.3.18",
         "replace": "^1.2.2",
         "standard-version": "^9.5.0",
         "ts-jest": "^29.0.3",

modules/outbound-domain-event-handler/project.json

@@ -1,4 +1,5 @@
 {
+  "name": "modules-outbound-domain-event-handler",
   "root": "modules/outbound-domain-event-handler/",
   "sourceRoot": "modules/outbound-domain-event-handler/src",
   "projectType": "application",

modules/outbound-domain-event-handler/src/shared/config.ts

@@ -95,6 +95,12 @@ const config = Convict({
                 default: ['topic-sdk-outbound-domain-events'],
                 env: 'DOMAIN_EVENT_CONSUMER_TOPICS',
             },
+            messageMaxBytes: {
+                doc: 'messageMaxBytes',
+                format: Number,
+                default: 200000000,
+                env: 'DOMAIN_EVENT_CONSUMER_MESSAGE_MAX_BYTES',
+            },
         },
         COMMAND_EVENT_PRODUCER: {
             brokerList: {
@@ -115,6 +121,18 @@ const config = Convict({
                 default: 'topic-sdk-outbound-command-events',
                 env: 'COMMAND_EVENT_PRODUCER_TOPIC',
             },
+            messageMaxBytes: {
+                doc: 'messageMaxBytes',
+                format: Number,
+                default: 200000000,
+                env: 'COMMAND_EVENT_PRODUCER_MESSAGE_MAX_BYTES',
+            },
+            compressionCodec: {
+                doc: 'compressionCodec',
+                format: String,
+                default: 'none',
+                env: 'COMMAND_EVENT_PRODUCER_COMPRESSION_CODEC',
+            },
         },
     },
 });

modules/private-shared-lib/package.json

@@ -31,7 +31,7 @@
     "@mojaloop/central-services-shared": "^17.3.1",
     "@mojaloop/logging-bc-public-types-lib": "^0.1.12",
     "@mojaloop/platform-shared-lib-messaging-types-lib": "^0.2.16",
-    "@mojaloop/platform-shared-lib-nodejs-kafka-client-lib": "^0.2.13",
+    "@mojaloop/platform-shared-lib-nodejs-kafka-client-lib": "^0.2.14-snapshot.0",
     "ajv": "^8.11.0",
     "redis": "^4.4.0",
     "uuid": "^9.0.0"
@@ -40,7 +40,7 @@
     "@types/node": "^18.11.9",
     "eslint": "^8.27.0",
     "jest": "^29.3.0",
-    "npm-check-updates": "^16.3.16",
+    "npm-check-updates": "^16.3.18",
     "replace": "^1.2.2",
     "standard-version": "^9.5.0",
     "ts-jest": "^29.0.3",

modules/private-shared-lib/project.json

@@ -1,4 +1,5 @@
 {
+  "name": "modules-private-shared-lib",
   "root": "modules/private-shared-lib/",
   "sourceRoot": "modules/private-shared-lib/src",
   "projectType": "library",

modules/private-shared-lib/src/infra/kafka_command_event_consumer.ts

@@ -50,6 +50,7 @@ export class KafkaCommandEventConsumer extends KafkaEventConsumer {
             kafkaBrokerList: consumerOptions.brokerList,
             kafkaGroupId: consumerOptions.groupId,
             outputType: MLKafkaRawConsumerOutputType.Json,
+            messageMaxBytes: consumerOptions.messageMaxBytes || 200000000,
         };
         super(mlConsumerOptions, consumerOptions.topics, superHandlerFn, logger);
     }

modules/private-shared-lib/src/infra/kafka_command_event_producer.ts

@@ -28,7 +28,7 @@
 
 // TODO: Try to use the generic kafka producer from platform-shared-lib and investigate if there is any value in maintaining these classes here.
 
-import { MLKafkaRawProducerOptions } from '@mojaloop/platform-shared-lib-nodejs-kafka-client-lib';
+import { MLKafkaRawProducerOptions, MLKafkaRawProdOpCompressionCodecEnum } from '@mojaloop/platform-shared-lib-nodejs-kafka-client-lib';
 import { KafkaEventProducer } from './kafka_event_producer';
 import { ILogger } from '@mojaloop/logging-bc-public-types-lib';
 import { CommandEvent }  from '../events';
@@ -46,6 +46,8 @@ export class KafkaCommandEventProducer extends KafkaEventProducer implements ICo
             kafkaBrokerList: producerOptions.brokerList,
             producerClientId: producerOptions.clientId,
             skipAcknowledgements: true,
+            messageMaxBytes: producerOptions.messageMaxBytes || 200000000,
+            compressionCodec: producerOptions.compressionCodec || MLKafkaRawProdOpCompressionCodecEnum.none,
         };
         super(mlProducerOptions, logger);
         this._topic = producerOptions.topic;

modules/private-shared-lib/src/infra/kafka_domain_event_consumer.ts

@@ -50,6 +50,7 @@ export class KafkaDomainEventConsumer extends KafkaEventConsumer {
             kafkaBrokerList: consumerOptions.brokerList,
             kafkaGroupId: consumerOptions.groupId,
             outputType: MLKafkaRawConsumerOutputType.Json,
+            messageMaxBytes: consumerOptions.messageMaxBytes || 200000000,
         };
         super(mlConsumerOptions, consumerOptions.topics, superHandlerFn, logger);
     }

modules/private-shared-lib/src/infra/kafka_domain_event_producer.ts

@@ -28,7 +28,7 @@
 
 // TODO: Try to use the generic kafka producer from platform-shared-lib and investigate if there is any value in maintaining these classes here.
 
-import { MLKafkaRawProducerOptions } from '@mojaloop/platform-shared-lib-nodejs-kafka-client-lib';
+import { MLKafkaRawProducerOptions, MLKafkaRawProdOpCompressionCodecEnum } from '@mojaloop/platform-shared-lib-nodejs-kafka-client-lib';
 import { KafkaEventProducer } from './kafka_event_producer';
 import { ILogger } from '@mojaloop/logging-bc-public-types-lib';
 import { DomainEvent }  from '../events';
@@ -46,6 +46,8 @@ export class KafkaDomainEventProducer extends KafkaEventProducer implements IDom
             kafkaBrokerList: producerOptions.brokerList,
             producerClientId: producerOptions.clientId,
             skipAcknowledgements: true,
+            messageMaxBytes: producerOptions.messageMaxBytes || 200000000,
+            compressionCodec: producerOptions.compressionCodec || MLKafkaRawProdOpCompressionCodecEnum.none,
         };
         super(mlProducerOptions, logger);
         this._topic = producerOptions.topic;

modules/private-shared-lib/src/types/infra/ikafka_event_consumer_options.ts

@@ -28,5 +28,6 @@ export type IKafkaEventConsumerOptions = {
     brokerList: string,
     groupId: string,
     clientId: string,
-    topics: string[]
+    topics: string[],
+    messageMaxBytes?: number,
 };

modules/private-shared-lib/src/types/infra/ikafka_event_producer_options.ts

@@ -23,9 +23,12 @@
  ******/
 
 'use strict';
+import { MLKafkaRawProdOpCompressionCodecEnum } from '@mojaloop/platform-shared-lib-nodejs-kafka-client-lib';
 
 export type IKafkaEventProducerOptions = {
     brokerList: string,
     clientId: string,
-    topic: string
+    topic: string,
+    messageMaxBytes?: number,
+    compressionCodec?: MLKafkaRawProdOpCompressionCodecEnum,
 };

package.json

@@ -67,7 +67,7 @@
     "wait-4-docker": "node ./scripts/_wait4_all.js"
   },
   "dependencies": {
-    "nx": "15.0.10",
+    "nx": "15.0.12",
     "tslib": "^2.4.1"
   },
   "devDependencies": {
@@ -80,10 +80,10 @@
     "eslint": "^8.27.0",
     "eslint-config-airbnb-typescript": "^17.0.0",
     "eslint-plugin-import": "latest",
-    "husky": "^8.0.1",
+    "husky": "^8.0.2",
     "jest": "^29.3.0",
     "nodemon": "^2.0.20",
-    "npm-check-updates": "^16.3.16",
+    "npm-check-updates": "^16.3.18",
     "replace": "^1.2.2",
     "standard-version": "^9.5.0",
     "ts-jest": "^29.0.3",

test/func/common-sdk.yml

@@ -84,15 +84,3 @@ services:
     #   start_period: 40s
     #   interval: 30s
 
-  redis:
-    networks:
-        - mojaloop-net
-    image: "redis:5.0.4-alpine"
-    # ports:
-    #   - "6379:6379"
-    healthcheck:
-      test: ["CMD" ,"sh", "-c", "redis-cli","ping"]
-      timeout: 20s
-      retries: 10
-      start_period: 40s
-      interval: 30s