feat(ime-5): extension list caching changes (#530)

* feat!: extension list caching changes (#529) * chore: add log statements for debugging * ime-392:save the cache in fspiop form for payer side * ime-392:save the cache in fspiop form for inboundModel * ime-392:remove unused logs and variables * ime-406: stringify json payload for the experience api * ime-412:update cache set function to include ttl for keys * ime-416: add redisCacheTtl as env variable for ttl * ime-396: add modifyDataForUi function to OutboundTransfersModel * chore: Update @mojaloop/ml-schema-transformer-lib version to v2.5.0 * ime-396: update outboundtransfersmodel to not cache extensionlist for UI --------- Co-authored-by: v3-Panwar-Ujjwal <[email protected]> * chore: address nits * chore: address nits --------- Co-authored-by: v3-Panwar-Ujjwal <[email protected]> Co-authored-by: Kevin Leyow <[email protected]>
mojaloop · Jan 21, 2025 · 55ef1f1 · 55ef1f1
1 parent f45d550
commit 55ef1f1
Show file tree

Hide file tree

Showing 11 changed files with 497 additions and 99 deletions.
diff --git a/modules/api-svc/package.json b/modules/api-svc/package.json
@@ -68,7 +68,7 @@
     "@mojaloop/central-services-error-handling": "^13.0.3",
     "@mojaloop/central-services-logger": "^11.5.2",
     "@mojaloop/central-services-metrics": "^12.4.3",
-    "@mojaloop/central-services-shared": "^18.15.1",
+    "@mojaloop/central-services-shared": "^18.15.2",
     "@mojaloop/event-sdk": "^14.1.2",
     "@mojaloop/logging-bc-client-lib": "0.5.8",
     "@mojaloop/ml-schema-transformer-lib": "^2.5.1",

diff --git a/modules/api-svc/src/InboundServer/handlers.js b/modules/api-svc/src/InboundServer/handlers.js
@@ -178,7 +178,7 @@ const postQuotes = async (ctx) => {
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 post quotes body to FSPIOP');
         // store the original request body in the context for later use
         quoteRequest.isoPostQuote = ctx.request.body;
-        const target = await TransformFacades.FSPIOPISO20022.quotes.post({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.quotes.post({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -225,7 +225,7 @@ const postTransfers = async (ctx) => {
     if (ctx.state.conf.isIsoApi) {
         // we need to transform the incoming request body from iso20022 to fspiop
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 post transfers body to FSPIOP');
-        const target = await TransformFacades.FSPIOPISO20022.transfers.post({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.transfers.post({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -441,7 +441,7 @@ const putPartiesByTypeAndId = async (ctx) => {
     if (ctx.state.conf.isIsoApi) {
         // we need to transform the incoming request body from iso20022 to fspiop
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 put parties body to FSPIOP');
-        const target = await TransformFacades.FSPIOPISO20022.parties.put({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.parties.put({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -475,7 +475,7 @@ const putQuoteById = async (ctx) => {
         ctx.request.originalIso20022QuoteResponse = ctx.request.body;
         // we need to transform the incoming request body from iso20022 to fspiop
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 put quotes body to FSPIOP');
-        const target = await TransformFacades.FSPIOPISO20022.quotes.put({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.quotes.put({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -514,7 +514,7 @@ const putQuotesByIdError = async (ctx) => {
     if (ctx.state.conf.isIsoApi) {
         // we need to transform the incoming request body from iso20022 to fspiop
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 putError quotes body to FSPIOP');
-        const target = await TransformFacades.FSPIOPISO20022.quotes.putError({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.quotes.putError({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -646,7 +646,7 @@ const putTransfersById = async (ctx) => {
     if (ctx.state.conf.isIsoApi) {
         // we need to transform the incoming request body from iso20022 to fspiop
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 put transfers body to FSPIOP');
-        const target = await TransformFacades.FSPIOPISO20022.transfers.put({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.transfers.put({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -703,7 +703,7 @@ const putPartiesByTypeAndIdError = async(ctx) => {
     if (ctx.state.conf.isIsoApi) {
         // we need to transform the incoming request body from iso20022 to fspiop
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 putError parties body to FSPIOP');
-        const target = await TransformFacades.FSPIOPISO20022.parties.putError({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.parties.putError({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -739,7 +739,7 @@ const putTransfersByIdError = async (ctx) => {
     if (ctx.state.conf.isIsoApi) {
         // we need to transform the incoming request body from iso20022 to fspiop
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 putError transfers body to FSPIOP');
-        const target = await TransformFacades.FSPIOPISO20022.transfers.putError({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.transfers.putError({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -967,7 +967,7 @@ const healthCheck = async(ctx) => {
 const postFxQuotes = async (ctx) => {
     if (ctx.state.conf.isIsoApi) {
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 post fxQuotes body to FSPIOP');
-        const target = await TransformFacades.FSPIOPISO20022.fxQuotes.post({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.fxQuotes.post({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -993,7 +993,7 @@ const createPutFxQuotesHandler = (success) => async (ctx) => {
     if (ctx.state.conf.isIsoApi) {
         const method = success ? 'put' : 'putError';
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug(`Transforming incoming ISO20022 ${method} fxQuotes body to FSPIOP`);
-        const target = await TransformFacades.FSPIOPISO20022.fxQuotes[method]({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.fxQuotes[method]({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -1024,7 +1024,7 @@ const createPutFxQuotesHandler = (success) => async (ctx) => {
 const postFxTransfers = async (ctx) => {
     if (ctx.state.conf.isIsoApi) {
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug('Transforming incoming ISO20022 post fxTransfers body to FSPIOP');
-        const target = await TransformFacades.FSPIOPISO20022.fxTransfers.post({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.fxTransfers.post({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 
@@ -1065,7 +1065,7 @@ const createPutFxTransfersHandler = (success) => async (ctx) => {
     if (ctx.state.conf.isIsoApi) {
         const method = success ? 'put' : 'putError';
         ctx.state.logger.isDebugEnabled && ctx.state.logger.push(ctx.request.body).debug(`Transforming incoming ISO20022 ${method} fxTransfers body to FSPIOP`);
-        const target = await TransformFacades.FSPIOPISO20022.fxTransfers[method]({ body: ctx.request.body });
+        const target = await TransformFacades.FSPIOPISO20022.fxTransfers[method]({ body: ctx.request.body }, { rollUpUnmappedAsExtensions: true });
         ctx.request.body = target.body;
     }
 

diff --git a/modules/api-svc/src/config.js b/modules/api-svc/src/config.js
@@ -241,4 +241,8 @@ module.exports = {
     //    - 1
     //    - 4
     ilpVersion: env.get('ILP_VERSION').default('1').asString(),
+
+    // Redis key ttl when stored in the cache, if value is used as zero it will
+    // persist throughout the session , value used is in seconds
+    redisCacheTtl: env.get('REDIS_CACHE_TTL').default('0').asInt()
 };
diff --git a/modules/api-svc/src/lib/cache.js b/modules/api-svc/src/lib/cache.js
@@ -342,13 +342,19 @@ class Cache {
       *
       * @param key {string} - cache key
       * @param value {string} - cache value
+      * @param ttl {number} - cache ttl in seconds
       */
-    async set(key, value) {
+    async set(key, value, ttl=0 ) {
         //if we are given an object, turn it into a string
         if(typeof(value) !== 'string') {
             value = JSON.stringify(value);
         }
-        await this._client.set(key, value);
+        // If ttl is positive i.e >0 then set expiry time as ttl in seconds
+        if(ttl > 0)
+            await this._client.set(key, value, { 'EX': ttl });
+        else 
+            await this._client.set(key, value);
+
     }
 
     /**

diff --git a/modules/api-svc/src/lib/model/InboundTransfersModel.js b/modules/api-svc/src/lib/model/InboundTransfersModel.js
@@ -72,6 +72,7 @@ class InboundTransfersModel {
             secret: config.ilpSecret,
             logger: config.logger,
         });
+        this._cacheTtl = config.redisCacheTtl;
     }
 
     updateStateWithError(err) {
@@ -1061,7 +1062,7 @@ class InboundTransfersModel {
      */
     async _save() {
         try {
-            const res = await this._cache.set(`transferModel_in_${this.data.transferId}`, this.data);
+            const res = await this._cache.set(`transferModel_in_${this.data.transferId}`, this.data, this._cacheTtl);
             this._logger.push({ res }).debug('Persisted transfer model in cache');
         }
         catch(err) {
@@ -1089,7 +1090,7 @@ class InboundTransfersModel {
     // todo: combine with this._save
     async saveFxState() { // fxQuote + fxTransfer
         const key = this.makeFxQuoteCacheKey(this.data?.conversionId);
-        const res = await this._cache.set(key, this.data);
+        const res = await this._cache.set(key, this.data, this._cacheTtl);
         this._logger.push({ key, res }).log('fxState is saved in cache');
     }
 

diff --git a/modules/api-svc/src/lib/model/OutboundTransfersModel.js b/modules/api-svc/src/lib/model/OutboundTransfersModel.js
@@ -59,6 +59,8 @@ class OutboundTransfersModel {
             throw new Error('Conflicting config options provided: autoAcceptParty and multiplePartiesResponse');
         }
 
+        this._cacheTtl = config.redisCacheTtl;
+
         this._requests = new MojaloopRequests({
             logger: this._logger,
             peerEndpoint: config.peerEndpoint,
@@ -560,9 +562,9 @@ class OutboundTransfersModel {
             const subscribing = this._cache.subscribeToOneMessageWithTimer(channel);
 
             const resp = await this._requests.postFxQuotes(payload, payload.conversionTerms.counterPartyFsp);
-            
+
             const { originalRequest } = resp;
-            // Setting the fxQuoteRequest to have the fspiop payload 
+            // Setting the fxQuoteRequest to have the fspiop payload
             // If ISO20022 is required then use originalRequest
             this.data.fxQuoteRequest = {
                 body: payload,
@@ -1152,7 +1154,12 @@ class OutboundTransfersModel {
     async _save() {
         try {
             this.data.currentState = this.stateMachine.state;
-            const res = await this._cache.set(`transferModel_out_${this.data.transferId}`, this.data);
+            const res = await this._cache.set(`transferModel_out_${this.data.transferId}`, this.data, this._cacheTtl);
+            // function to modify this.data before saving to cache for UI.
+            const modifiedData = this._modifyDataForUi(this.data);
+            // save to a UI key, using a modifiedData, as we don't want any side effects to happen on original data
+            // No ttl set as it will persist throughout the session
+            await this._cache.set(`transferUI_out_${this.data.transferId}`, modifiedData);
             this._logger.isDebugEnabled && this._logger.push({ res }).debug('Persisted transfer model in cache');
         }
         catch (err) {
@@ -1161,6 +1168,33 @@ class OutboundTransfersModel {
         }
     }
 
+    /**
+     * Modifies the data being stored in the cache for UI before it is stored.
+     * Works on a copy of original object to avoid side effects
+     */
+    _modifyDataForUi( data ){
+        // deep cloning to avoid side effects
+        let modifiedData = JSON.parse(JSON.stringify(data));
+        // Removing iso quote response and extension lists
+        if(modifiedData.getPartiesResponse && modifiedData.getPartiesResponse.body && modifiedData.getPartiesResponse.body.extensionList)
+            modifiedData.getPartiesResponse.body.extensionList = undefined;
+        if(modifiedData.fxQuoteResponse && modifiedData.fxQuoteResponse.body && modifiedData.fxQuoteResponse.body.extensionList)
+            modifiedData.fxQuoteResponse.body.extensionList = undefined;
+        if(modifiedData.quoteResponse && modifiedData.quoteResponse.originalIso20022QuoteResponse){
+            modifiedData.quoteResponse.originalIso20022QuoteResponse = undefined;
+        }
+        if(modifiedData.quoteResponse && modifiedData.quoteResponse.body && modifiedData.quoteResponse.body.extensionList){
+            modifiedData.quoteResponse.body.extensionList = undefined;
+        }
+        if(modifiedData.fxTransferResponse && modifiedData.fxTransferResponse.body && modifiedData.fxTransferResponse.body.extensionList){
+            modifiedData.fxTransferResponse.body.extensionList = undefined;
+        }
+        if(modifiedData.fulfil && modifiedData.fulfil.body && modifiedData.fulfil.body.extensionList){
+            modifiedData.fulfil.body.extensionList = undefined;
+        }
+        return modifiedData;
+    }
+
 
     /**
      * Loads a transfer model from cache for resumption of the transfer process

diff --git a/modules/outbound-command-event-handler/package.json b/modules/outbound-command-event-handler/package.json
@@ -43,7 +43,7 @@
   },
   "dependencies": {
     "@mojaloop/api-snippets": "17.7.8",
-    "@mojaloop/central-services-shared": "^18.15.1",
+    "@mojaloop/central-services-shared": "^18.15.2",
     "@mojaloop/logging-bc-client-lib": "^0.5.8",
     "@mojaloop/logging-bc-public-types-lib": "^0.5.4",
     "@mojaloop/sdk-scheme-adapter-private-shared-lib": "workspace:^",
@@ -65,8 +65,8 @@
     "@types/supertest": "^6.0.2",
     "@types/swagger-ui-express": "4.1.7",
     "@types/yamljs": "^0.2.34",
-    "@typescript-eslint/eslint-plugin": "^8.20.0",
-    "@typescript-eslint/parser": "^8.20.0",
+    "@typescript-eslint/eslint-plugin": "^8.21.0",
+    "@typescript-eslint/parser": "^8.21.0",
     "copyfiles": "^2.4.1",
     "eslint": "^9.15.0",
     "jest": "^29.7.0",

diff --git a/modules/outbound-domain-event-handler/package.json b/modules/outbound-domain-event-handler/package.json
@@ -62,8 +62,8 @@
         "@types/supertest": "^6.0.2",
         "@types/swagger-ui-express": "^4.1.7",
         "@types/yamljs": "^0.2.34",
-        "@typescript-eslint/eslint-plugin": "^8.20.0",
-        "@typescript-eslint/parser": "^8.20.0",
+        "@typescript-eslint/eslint-plugin": "^8.21.0",
+        "@typescript-eslint/parser": "^8.21.0",
         "copyfiles": "^2.4.1",
         "eslint": "^9.15.0",
         "jest": "^29.7.0",

diff --git a/modules/private-shared-lib/package.json b/modules/private-shared-lib/package.json
@@ -30,7 +30,7 @@
   },
   "dependencies": {
     "@mojaloop/api-snippets": "17.7.8",
-    "@mojaloop/central-services-shared": "^18.15.1",
+    "@mojaloop/central-services-shared": "^18.15.2",
     "@mojaloop/logging-bc-public-types-lib": "^0.5.4",
     "@mojaloop/platform-shared-lib-messaging-types-lib": "^0.7.1",
     "@mojaloop/platform-shared-lib-nodejs-kafka-client-lib": "0.5.18",
@@ -42,8 +42,8 @@
     "@eslint/compat": "^1.2.5",
     "@types/node": "^22.10.7",
     "@types/uuid": "^10.0.0",
-    "@typescript-eslint/eslint-plugin": "^8.20.0",
-    "@typescript-eslint/parser": "^8.20.0",
+    "@typescript-eslint/eslint-plugin": "^8.21.0",
+    "@typescript-eslint/parser": "^8.21.0",
     "eslint": "^9.15.0",
     "jest": "^29.7.0",
     "npm-check-updates": "^16.7.10",

diff --git a/package.json b/package.json
@@ -85,8 +85,8 @@
     "@types/jest": "^29.5.14",
     "@types/node": "^22.10.7",
     "@types/node-cache": "^4.2.5",
-    "@typescript-eslint/eslint-plugin": "^8.20.0",
-    "@typescript-eslint/parser": "^8.20.0",
+    "@typescript-eslint/eslint-plugin": "^8.21.0",
+    "@typescript-eslint/parser": "^8.21.0",
     "audit-ci": "^7.1.0",
     "eslint": "^9.15.0",
     "eslint-config-airbnb-typescript": "^18.0.0",