diff --git a/.github/openapi/typescript-angular-config.json b/.github/openapi/typescript-angular-config.json index 6da9d03d6..ac031ad8e 100644 --- a/.github/openapi/typescript-angular-config.json +++ b/.github/openapi/typescript-angular-config.json @@ -2,5 +2,8 @@ "generatorName": "typescript-angular", "npmName": "@scicatproject/scicat-sdk-ts-angular", "ngVersion": "16.2.12", - "withInterfaces": true + "withInterfaces": true, + "paramNaming": "original", + "modelPropertyNaming": "original", + "enumPropertyNaming": "original" } diff --git a/.github/openapi/typescript-fetch-config.json b/.github/openapi/typescript-fetch-config.json index 2b2c3ae1a..826763d0f 100644 --- a/.github/openapi/typescript-fetch-config.json +++ b/.github/openapi/typescript-fetch-config.json @@ -1,5 +1,8 @@ { "generatorName": "typescript-fetch", "npmName": "@scicatproject/scicat-sdk-ts-fetch", - "supportsES6": true + "supportsES6": true, + "paramNaming": "original", + "modelPropertyNaming": "original", + "enumPropertyNaming": "original" } diff --git a/.gitignore b/.gitignore index eaadcb985..7edf11d70 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ functionalAccounts.json datasetTypes.json proposalTypes.json loggers.json +metricsConfig.json # Configs .env diff --git a/metricsConfig.example.json b/metricsConfig.example.json new file mode 100644 index 000000000..d6396681c --- /dev/null +++ b/metricsConfig.example.json @@ -0,0 +1,31 @@ +{ + "include": [ + { + "path": "*", + "method": 0, + "version": "3" + }, + { + "path": "datasets/fullquery", + "method": 0, + "version": "3" + }, + { + "path": "datasets/:id", + "method": 0, + "version": "3" + } + ], + "exclude": [ + { + "path": "datasets/fullfacet", + "method": 0, + "version": "3" + }, + { + "path": "datasets/metadataKeys", + "method": 0, + "version": "3" + } + ] +} diff --git a/package-lock.json b/package-lock.json index ebb870a5d..d20e6020a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,10 +14,10 @@ "@nestjs-modules/mailer": "^2.0.2", "@nestjs/axios": "^3.0.0", "@nestjs/common": "^10.3.8", - "@nestjs/config": "^3.0.0", + "@nestjs/config": "^4.0.0", "@nestjs/core": "^10.3.8", "@nestjs/elasticsearch": "^10.0.1", - "@nestjs/event-emitter": "^2.0.1", + "@nestjs/event-emitter": "^3.0.0", "@nestjs/jwt": "^10.0.1", "@nestjs/mongoose": "^10.0.0", "@nestjs/passport": "^10.0.0", @@ -2295,30 +2295,20 @@ } }, "node_modules/@nestjs/config": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/config/-/config-3.3.0.tgz", - "integrity": "sha512-pdGTp8m9d0ZCrjTpjkUbZx6gyf2IKf+7zlkrPNMsJzYZ4bFRRTpXrnj+556/5uiI6AfL5mMrJc2u7dB6bvM+VA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@nestjs/config/-/config-4.0.0.tgz", + "integrity": "sha512-hyhUMtVwlT+tavtPNyekl8iP0QTU1U6awKrgdOSxhMhp3TQMltx7hz2yqGTcARp+19zWPfgJudyxthuD3lPp/Q==", + "license": "MIT", "dependencies": { - "dotenv": "16.4.5", - "dotenv-expand": "10.0.0", + "dotenv": "16.4.7", + "dotenv-expand": "12.0.1", "lodash": "4.17.21" }, "peerDependencies": { - "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/common": "^10.0.0 || ^11.0.0", "rxjs": "^7.1.0" } }, - "node_modules/@nestjs/config/node_modules/dotenv": { - "version": "16.4.5", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", - "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, "node_modules/@nestjs/core": { "version": "10.4.15", "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-10.4.15.tgz", @@ -2367,15 +2357,16 @@ } }, "node_modules/@nestjs/event-emitter": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@nestjs/event-emitter/-/event-emitter-2.1.1.tgz", - "integrity": "sha512-6L6fBOZTyfFlL7Ih/JDdqlCzZeCW0RjCX28wnzGyg/ncv5F/EOeT1dfopQr1loBRQ3LTgu8OWM7n4zLN4xigsg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nestjs/event-emitter/-/event-emitter-3.0.0.tgz", + "integrity": "sha512-WbvzQQ9BGnj27onh2qSLND2+4iA6Pfp4K+HLlqunB0Uz0614O8lGMtcveSss2IOxsox8EhSI54WAvuAsDrX1hA==", + "license": "MIT", "dependencies": { "eventemitter2": "6.4.9" }, "peerDependencies": { - "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", - "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" } }, "node_modules/@nestjs/jwt": { @@ -3386,20 +3377,21 @@ "dev": true }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.19.0.tgz", - "integrity": "sha512-NggSaEZCdSrFddbctrVjkVZvFC6KGfKfNK0CU7mNK/iKHGKbzT4Wmgm08dKpcZECBu9f5FypndoMyRHkdqfT1Q==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.20.0.tgz", + "integrity": "sha512-naduuphVw5StFfqp4Gq4WhIBE2gN1GEmMUExpJYknZJdRnc+2gDzB8Z3+5+/Kv33hPQRDGzQO/0opHE72lZZ6A==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.19.0", - "@typescript-eslint/type-utils": "8.19.0", - "@typescript-eslint/utils": "8.19.0", - "@typescript-eslint/visitor-keys": "8.19.0", + "@typescript-eslint/scope-manager": "8.20.0", + "@typescript-eslint/type-utils": "8.20.0", + "@typescript-eslint/utils": "8.20.0", + "@typescript-eslint/visitor-keys": "8.20.0", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", - "ts-api-utils": "^1.3.0" + "ts-api-utils": "^2.0.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3415,15 +3407,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.19.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.19.1.tgz", - "integrity": "sha512-67gbfv8rAwawjYx3fYArwldTQKoYfezNUT4D5ioWetr/xCrxXxvleo3uuiFuKfejipvq+og7mjz3b0G2bVyUCw==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.20.0.tgz", + "integrity": "sha512-gKXG7A5HMyjDIedBi6bUrDcun8GIjnI8qOwVLiY3rx6T/sHP/19XLJOnIq/FgQvWLHja5JN/LSE7eklNBr612g==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.19.1", - "@typescript-eslint/types": "8.19.1", - "@typescript-eslint/typescript-estree": "8.19.1", - "@typescript-eslint/visitor-keys": "8.19.1", + "@typescript-eslint/scope-manager": "8.20.0", + "@typescript-eslint/types": "8.20.0", + "@typescript-eslint/typescript-estree": "8.20.0", + "@typescript-eslint/visitor-keys": "8.20.0", "debug": "^4.3.4" }, "engines": { @@ -3438,135 +3431,15 @@ "typescript": ">=4.8.4 <5.8.0" } }, - "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/scope-manager": { - "version": "8.19.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.19.1.tgz", - "integrity": "sha512-60L9KIuN/xgmsINzonOcMDSB8p82h95hoBfSBtXuO4jlR1R9L1xSkmVZKgCPVfavDlXihh4ARNjXhh1gGnLC7Q==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "8.19.1", - "@typescript-eslint/visitor-keys": "8.19.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/types": { - "version": "8.19.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.19.1.tgz", - "integrity": "sha512-JBVHMLj7B1K1v1051ZaMMgLW4Q/jre5qGK0Ew6UgXz1Rqh+/xPzV1aW581OM00X6iOfyr1be+QyW8LOUf19BbA==", - "dev": true, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/typescript-estree": { - "version": "8.19.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.19.1.tgz", - "integrity": "sha512-jk/TZwSMJlxlNnqhy0Eod1PNEvCkpY6MXOXE/WLlblZ6ibb32i2We4uByoKPv1d0OD2xebDv4hbs3fm11SMw8Q==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "8.19.1", - "@typescript-eslint/visitor-keys": "8.19.1", - "debug": "^4.3.4", - "fast-glob": "^3.3.2", - "is-glob": "^4.0.3", - "minimatch": "^9.0.4", - "semver": "^7.6.0", - "ts-api-utils": "^2.0.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <5.8.0" - } - }, - "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/visitor-keys": { - "version": "8.19.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.19.1.tgz", - "integrity": "sha512-fzmjU8CHK853V/avYZAvuVut3ZTfwN5YtMaoi+X9Y9MA9keaWNHC3zEQ9zvyX/7Hj+5JkNyK1l7TOR2hevHB6Q==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "8.19.1", - "eslint-visitor-keys": "^4.2.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/parser/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/@typescript-eslint/parser/node_modules/eslint-visitor-keys": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", - "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", - "dev": true, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@typescript-eslint/parser/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "dev": true, - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@typescript-eslint/parser/node_modules/ts-api-utils": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.0.0.tgz", - "integrity": "sha512-xCt/TOAc+EOHS1XPnijD3/yzpH6qg2xppZO1YDqGoVsNXfQfzHpOdNuXwrwOU8u4ITXJyDCTyt8w5g1sZv9ynQ==", - "dev": true, - "engines": { - "node": ">=18.12" - }, - "peerDependencies": { - "typescript": ">=4.8.4" - } - }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.19.0.tgz", - "integrity": "sha512-hkoJiKQS3GQ13TSMEiuNmSCvhz7ujyqD1x3ShbaETATHrck+9RaDdUbt+osXaUuns9OFwrDTTrjtwsU8gJyyRA==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.20.0.tgz", + "integrity": "sha512-J7+VkpeGzhOt3FeG1+SzhiMj9NzGD/M6KoGn9f4dbz3YzK9hvbhVTmLj/HiTp9DazIzJ8B4XcM80LrR9Dm1rJw==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.19.0", - "@typescript-eslint/visitor-keys": "8.19.0" + "@typescript-eslint/types": "8.20.0", + "@typescript-eslint/visitor-keys": "8.20.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3577,15 +3450,16 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.19.0.tgz", - "integrity": "sha512-TZs0I0OSbd5Aza4qAMpp1cdCYVnER94IziudE3JU328YUHgWu9gwiwhag+fuLeJ2LkWLXI+F/182TbG+JaBdTg==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.20.0.tgz", + "integrity": "sha512-bPC+j71GGvA7rVNAHAtOjbVXbLN5PkwqMvy1cwGeaxUoRQXVuKCebRoLzm+IPW/NtFFpstn1ummSIasD5t60GA==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "8.19.0", - "@typescript-eslint/utils": "8.19.0", + "@typescript-eslint/typescript-estree": "8.20.0", + "@typescript-eslint/utils": "8.20.0", "debug": "^4.3.4", - "ts-api-utils": "^1.3.0" + "ts-api-utils": "^2.0.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3600,10 +3474,11 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.19.0.tgz", - "integrity": "sha512-8XQ4Ss7G9WX8oaYvD4OOLCjIQYgRQxO+qCiR2V2s2GxI9AUpo7riNwo6jDhKtTcaJjT8PY54j2Yb33kWtSJsmA==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.20.0.tgz", + "integrity": "sha512-cqaMiY72CkP+2xZRrFt3ExRBu0WmVitN/rYPZErA80mHjHx/Svgp8yfbzkJmDoQ/whcytOPO9/IZXnOc+wigRA==", "dev": true, + "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, @@ -3613,19 +3488,20 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.19.0.tgz", - "integrity": "sha512-WW9PpDaLIFW9LCbucMSdYUuGeFUz1OkWYS/5fwZwTA+l2RwlWFdJvReQqMUMBw4yJWJOfqd7An9uwut2Oj8sLw==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.20.0.tgz", + "integrity": "sha512-Y7ncuy78bJqHI35NwzWol8E0X7XkRVS4K4P4TCyzWkOJih5NDvtoRDW4Ba9YJJoB2igm9yXDdYI/+fkiiAxPzA==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.19.0", - "@typescript-eslint/visitor-keys": "8.19.0", + "@typescript-eslint/types": "8.20.0", + "@typescript-eslint/visitor-keys": "8.20.0", "debug": "^4.3.4", "fast-glob": "^3.3.2", "is-glob": "^4.0.3", "minimatch": "^9.0.4", "semver": "^7.6.0", - "ts-api-utils": "^1.3.0" + "ts-api-utils": "^2.0.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3643,6 +3519,7 @@ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -3652,6 +3529,7 @@ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, + "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, @@ -3663,15 +3541,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.19.0.tgz", - "integrity": "sha512-PTBG+0oEMPH9jCZlfg07LCB2nYI0I317yyvXGfxnvGvw4SHIOuRnQ3kadyyXY6tGdChusIHIbM5zfIbp4M6tCg==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.20.0.tgz", + "integrity": "sha512-dq70RUw6UK9ei7vxc4KQtBRk7qkHZv447OUZ6RPQMQl71I3NZxQJX/f32Smr+iqWrB02pHKn2yAdHBb0KNrRMA==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.19.0", - "@typescript-eslint/types": "8.19.0", - "@typescript-eslint/typescript-estree": "8.19.0" + "@typescript-eslint/scope-manager": "8.20.0", + "@typescript-eslint/types": "8.20.0", + "@typescript-eslint/typescript-estree": "8.20.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3686,12 +3565,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.19.0.tgz", - "integrity": "sha512-mCFtBbFBJDCNCWUl5y6sZSCHXw1DEFEk3c/M3nRK2a4XUB8StGFtmcEMizdjKuBzB6e/smJAAWYug3VrdLMr1w==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.20.0.tgz", + "integrity": "sha512-v/BpkeeYAsPkKCkR8BDwcno0llhzWVqPOamQrAEMdpZav2Y9OVjd9dwJyBLJWwf335B5DmlifECIkZRJCaGaHA==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.19.0", + "@typescript-eslint/types": "8.20.0", "eslint-visitor-keys": "^4.2.0" }, "engines": { @@ -3707,6 +3587,7 @@ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", "dev": true, + "license": "Apache-2.0", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, @@ -6041,11 +5922,18 @@ } }, "node_modules/dotenv-expand": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz", - "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==", + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-12.0.1.tgz", + "integrity": "sha512-LaKRbou8gt0RNID/9RoI+J2rvXsBRPMV7p+ElHlPhcSARbCPDYcYG2s1TIzAfWv4YSgyY5taidWzzs31lNV3yQ==", + "license": "BSD-2-Clause", + "dependencies": { + "dotenv": "^16.4.5" + }, "engines": { "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" } }, "node_modules/eastasianwidth": { @@ -13531,15 +13419,16 @@ } }, "node_modules/ts-api-utils": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.2.tgz", - "integrity": "sha512-ZF5gQIQa/UmzfvxbHZI3JXN0/Jt+vnAfAviNRAMc491laiK6YCLpCW9ft8oaCRFOTxCZtUTE6XB0ZQAe3olntw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.0.0.tgz", + "integrity": "sha512-xCt/TOAc+EOHS1XPnijD3/yzpH6qg2xppZO1YDqGoVsNXfQfzHpOdNuXwrwOU8u4ITXJyDCTyt8w5g1sZv9ynQ==", "dev": true, + "license": "MIT", "engines": { - "node": ">=16" + "node": ">=18.12" }, "peerDependencies": { - "typescript": ">=4.2.0" + "typescript": ">=4.8.4" } }, "node_modules/ts-jest": { diff --git a/package.json b/package.json index 142e369d4..c94491aae 100644 --- a/package.json +++ b/package.json @@ -35,10 +35,10 @@ "@nestjs-modules/mailer": "^2.0.2", "@nestjs/axios": "^3.0.0", "@nestjs/common": "^10.3.8", - "@nestjs/config": "^3.0.0", + "@nestjs/config": "^4.0.0", "@nestjs/core": "^10.3.8", "@nestjs/elasticsearch": "^10.0.1", - "@nestjs/event-emitter": "^2.0.1", + "@nestjs/event-emitter": "^3.0.0", "@nestjs/jwt": "^10.0.1", "@nestjs/mongoose": "^10.0.0", "@nestjs/passport": "^10.0.0", diff --git a/src/app.module.ts b/src/app.module.ts index 04604d824..d76078361 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -3,7 +3,7 @@ import { MongooseModule } from "@nestjs/mongoose"; import { DatasetsModule } from "./datasets/datasets.module"; import { AuthModule } from "./auth/auth.module"; import { UsersModule } from "./users/users.module"; -import { ConfigModule, ConfigService } from "@nestjs/config"; +import { ConditionalModule, ConfigModule, ConfigService } from "@nestjs/config"; import { CaslModule } from "./casl/casl.module"; import configuration from "./config/configuration"; import { APP_GUARD, Reflector } from "@nestjs/core"; @@ -32,6 +32,7 @@ import { EventEmitterModule } from "@nestjs/event-emitter"; import { AdminModule } from "./admin/admin.module"; import { HealthModule } from "./health/health.module"; import { LoggerModule } from "./loggers/logger.module"; +import { MetricsModule } from "./metrics/metrics.module"; @Module({ imports: [ @@ -42,6 +43,13 @@ import { LoggerModule } from "./loggers/logger.module"; ConfigModule.forRoot({ load: [configuration], }), + // NOTE: `ConditionalModule.registerWhen` directly uses `process.env` as it does not support + // dependency injection for `ConfigService`. This approach ensures compatibility while + // leveraging environment variables for conditional module loading. + ConditionalModule.registerWhen( + MetricsModule, + (env: NodeJS.ProcessEnv) => env.METRICS_ENABLED === "yes", + ), LoggerModule, DatablocksModule, DatasetsModule, diff --git a/src/config/configuration.ts b/src/config/configuration.ts index f7afe539f..ae595c9df 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -48,6 +48,7 @@ const configuration = () => { loggers: process.env.LOGGERS_CONFIG_FILE || "loggers.json", datasetTypes: process.env.DATASET_TYPES_FILE || "datasetTypes.json", proposalTypes: process.env.PROPOSAL_TYPES_FILE || "proposalTypes.json", + metricsConfig: process.env.METRICS_CONFIG_FILE || "metricsConfig.json", }; Object.keys(jsonConfigFileList).forEach((key) => { const filePath = jsonConfigFileList[key]; @@ -204,7 +205,13 @@ const configuration = () => { mongoDBCollection: process.env.MONGODB_COLLECTION, defaultIndex: process.env.ES_INDEX ?? "dataset", }, - + metrics: { + // Note: `process.env.METRICS_ENABLED` is directly used for conditional module loading in + // `ConditionalModule.registerWhen` as it does not support ConfigService injection. The purpose of + // keeping `metrics.enabled` in the configuration is for other modules to use and maintain consistency. + enabled: process.env.METRICS_ENABLED || "no", + config: jsonConfigMap.metricsConfig, + }, registerDoiUri: process.env.REGISTER_DOI_URI, registerMetadataUri: process.env.REGISTER_METADATA_URI, doiUsername: process.env.DOI_USERNAME, diff --git a/src/main.ts b/src/main.ts index 327248a93..e7780f415 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,9 +15,7 @@ async function bootstrap() { const app = await NestFactory.create(AppModule, { bufferLogs: true, }); - const configService: ConfigService, false> = app.get( - ConfigService, - ); + const configService = app.get(ConfigService); const apiVersion = configService.get("versions.api"); const swaggerPath = `${configService.get("swaggerPath")}`; diff --git a/src/metrics/metrics.module.ts b/src/metrics/metrics.module.ts new file mode 100644 index 000000000..f2d758ae8 --- /dev/null +++ b/src/metrics/metrics.module.ts @@ -0,0 +1,34 @@ +import { Logger, MiddlewareConsumer, Module, NestModule } from "@nestjs/common"; +import { ConfigModule, ConfigService } from "@nestjs/config"; +import { AccessTrackingMiddleware } from "./middlewares/accessTracking.middleware"; +import { JwtModule } from "@nestjs/jwt"; + +@Module({ + imports: [ConfigModule, JwtModule], + exports: [], +}) +export class MetricsModule implements NestModule { + constructor(private readonly configService: ConfigService) {} + + configure(consumer: MiddlewareConsumer) { + const { include = [], exclude = [] } = + this.configService.get("metrics.config") || {}; + if (!include.length && !exclude.length) { + Logger.error( + 'Metrics middleware requires at least one "include" or "exclude" path in the metricsConfig.json file.', + "MetricsModule", + ); + return; + } + + try { + consumer + .apply(AccessTrackingMiddleware) + .exclude(...exclude) + .forRoutes(...include); + Logger.log("Start collecting metrics", "MetricsModule"); + } catch (error) { + Logger.error("Error configuring metrics middleware", error); + } + } +} diff --git a/src/metrics/middlewares/accessTracking.middleware.ts b/src/metrics/middlewares/accessTracking.middleware.ts new file mode 100644 index 000000000..515b0eecd --- /dev/null +++ b/src/metrics/middlewares/accessTracking.middleware.ts @@ -0,0 +1,76 @@ +import { Injectable, Logger, NestMiddleware } from "@nestjs/common"; +import { Request, Response, NextFunction } from "express"; +import { JwtService } from "@nestjs/jwt"; +import { parse } from "url"; + +@Injectable() +export class AccessTrackingMiddleware implements NestMiddleware { + private requestCache = new Map(); // Cache to store recent requests + private logIntervalDuration = 1000; // Log every 1 second to prevent spam + private cacheResetInterval = 10 * 60 * 1000; // Clear cache every 10 minutes to prevent memory leak + + constructor(private readonly jwtService: JwtService) { + this.startCacheResetInterval(); + } + use(req: Request, res: Response, next: NextFunction) { + const { query, pathname } = parse(req.originalUrl, true); + + const userAgent = req.headers["user-agent"]; + // TODO: Better to use a library for this? + const isBot = userAgent ? /bot|crawl|spider|slurp/i.test(userAgent) : false; + + if (!pathname || isBot) return; + + const startTime = Date.now(); + const authHeader = req.headers.authorization; + const originIp = req.socket.remoteAddress; + const userId = this.parseToken(authHeader); + + const cacheKeyIdentifier = `${userId}-${originIp}-${pathname}`; + + res.on("finish", () => { + const statusCode = res.statusCode; + if (statusCode === 304) return; + + const responseTime = Date.now() - startTime; + + const lastHitTime = this.requestCache.get(cacheKeyIdentifier); + + // Log only if the request was not recently logged + if (!lastHitTime || Date.now() - lastHitTime > this.logIntervalDuration) { + Logger.log("SciCatAccessLogs", { + userId, + originIp, + endpoint: pathname, + query: query, + statusCode, + responseTime, + }); + + this.requestCache.set(cacheKeyIdentifier, Date.now()); + } + }); + + next(); + } + + private parseToken(authHeader?: string) { + if (!authHeader) return "anonymous"; + const token = authHeader.split(" ")[1]; + if (!token) return "anonymous"; + + try { + const { id } = this.jwtService.decode(token); + return id; + } catch (error) { + Logger.error("Error parsing token-> AccessTrackingMiddleware", error); + return null; + } + } + + private startCacheResetInterval() { + setInterval(() => { + this.requestCache.clear(); + }, this.cacheResetInterval); + } +}