diff --git a/.commitlintrc.json b/.commitlintrc.json new file mode 100644 index 000000000..7293d5af9 --- /dev/null +++ b/.commitlintrc.json @@ -0,0 +1,28 @@ +{ + "extends": ["@commitlint/config-conventional"], + "rules": { + "subject-case": [ + 2, + "always", + ["sentence-case", "start-case", "pascal-case", "upper-case", "lower-case"] + ], + "type-enum": [ + 2, + "always", + [ + "build", + "chore", + "ci", + "docs", + "feat", + "fix", + "perf", + "refactor", + "revert", + "style", + "test", + "sample" + ] + ] + } +} diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..4ac4973fb --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 000000000..df95e3b5c --- /dev/null +++ b/.eslintignore @@ -0,0 +1,12 @@ +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# node-waf configuration +.lock-wscript + +# Dependency directory +# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git +node_modules + +# OSX +.DS_Store diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 000000000..3408a06eb --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,95 @@ +module.exports = { + root: true, + extends: [ + 'airbnb-typescript', + 'plugin:import/typescript', + 'plugin:@typescript-eslint/recommended', + 'prettier', + 'plugin:prettier/recommended', + 'plugin:promise/recommended', + ], + ignorePatterns: ['.eslintrc.js', '*.json', 'jest.config.js'], + plugins: ['import', 'promise', '@typescript-eslint', 'prettier'], + parser: '@typescript-eslint/parser', + settings: { + 'import/parsers': { + '@typescript-eslint/parser': ['.ts', '.tsx'], + }, + }, + parserOptions: { + project: './tsconfig.json', + ecmaVersion: 2020, + sourceType: 'module', + }, + rules: { + '@typescript-eslint/space-before-blocks': 'off', + '@typescript-eslint/lines-between-class-members': 'off', + 'react/jsx-wrap-multilines': 'off', + 'react/jsx-filename-extension': 'off', + 'multiline-comment-style': ['error', 'starred-block'], + 'promise/catch-or-return': 'off', + '@typescript-eslint/explicit-module-boundary-types': 'off', + '@typescript-eslint/no-unused-expressions': 'off', + '@typescript-eslint/explicit-function-return-type': 'off', + 'react/jsx-closing-bracket-location': 'off', + '@typescript-eslint/no-var-requires': 'off', + 'no-unused-vars': 'error', + '@typescript-eslint/no-unused-vars': ['error'], + 'mocha/no-mocha-arrows': 'off', + '@typescript-eslint/default-param-last': 'off', + 'no-return-await': 'off', + 'no-await-in-loop': 'off', + 'no-continue': 'off', + 'no-console': 'warn', + 'no-magic-numbers': 'error', + 'no-prototype-builtins': 'off', + 'import/no-cycle': 'off', + 'class-methods-use-this': 'off', + '@typescript-eslint/no-use-before-define': 'off', + '@typescript-eslint/no-explicit-any': 1, + 'no-restricted-syntax': 'off', + '@typescript-eslint/interface-name-prefix': 'off', + 'no-underscore-dangle': 'off', + 'import/prefer-default-export': 'off', + // A temporary hack related to IDE not resolving correct package.json + 'import/no-extraneous-dependencies': 'off', + 'react/jsx-one-expression-per-line': 'off', + 'react/jsx-no-bind': 'off', + 'lines-between-class-members': 'off', + 'max-classes-per-file': 'off', + 'react/react-in-jsx-scope': 'off', + 'max-len': ['warn', { code: 140 }], + '@typescript-eslint/return-await': 'off', + 'no-restricted-imports': [ + 'error', + { + patterns: ['@impler/shared/*', '@impler/dal/*', '!import2/good'], + }, + ], + 'padding-line-between-statements': [ + 'error', + { blankLine: 'any', prev: ['const', 'let', 'var'], next: ['if', 'for'] }, + { blankLine: 'any', prev: ['const', 'let', 'var'], next: ['const', 'let', 'var'] }, + { blankLine: 'always', prev: '*', next: 'return' }, + ], + 'id-length': ['error', { min: 2, exceptions: ['i', 'e', 'a', 'b', '_', 't'], properties: 'never' }], + '@typescript-eslint/naming-convention': [ + 'error', + + { selector: 'enumMember', format: ['UPPER_CASE'] }, + { selector: 'enum', format: ['PascalCase'], suffix: ['Enum'] }, + { selector: 'class', format: ['PascalCase'] }, + { selector: 'variableLike', format: ['camelCase', 'UPPER_CASE', 'PascalCase'], leadingUnderscore: 'allow' }, + { + selector: 'interface', + format: ['PascalCase'], + prefix: ['I'], + }, + { + selector: ['function'], + format: ['camelCase'], + leadingUnderscore: 'allow', + }, + ], + }, +}; diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml new file mode 100644 index 000000000..364671649 --- /dev/null +++ b/.github/workflows/test-build.yml @@ -0,0 +1,50 @@ +# This is a basic workflow to help you get started with Actions + +name: Test Build is happening + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main", "next", "development" ] + schedule: + - cron: '25 2 * * 4' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + environment: Development + if: "!contains(github.event.head_commit.message, 'build skip')" + # The type of runner that the job will run on + runs-on: ubuntu-latest + timeout-minutes: 80 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Setup kernel for react native, increase watchers + run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p + - uses: actions/setup-node@v2 + with: + node-version: '16.15.1' + + - name: Cache pnpm modules + uses: actions/cache@v2 + with: + path: ~/.pnpm-store + key: ${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}- + + - name: Nestjs + run: npm i -g @nestjs/cli + + - uses: pnpm/action-setup@v2.0.1 + with: + version: 7.9.4 + run_install: true \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..0b52eccd8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules +dist +build + +.env \ No newline at end of file diff --git a/.husky/commit-msg b/.husky/commit-msg new file mode 100755 index 000000000..42a70e424 --- /dev/null +++ b/.husky/commit-msg @@ -0,0 +1,4 @@ +#!/usr/bin/env sh +. "$(dirname -- "$0")/_/husky.sh" + +npx --no -- commitlint --edit "$1" \ No newline at end of file diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 000000000..a5a29d9f7 --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,4 @@ +#!/usr/bin/env sh +. "$(dirname -- "$0")/_/husky.sh" + +pnpm lint-staged diff --git a/.npmrc b/.npmrc new file mode 100644 index 000000000..4c2f52b3b --- /dev/null +++ b/.npmrc @@ -0,0 +1,2 @@ +auto-install-peers=true +strict-peer-dependencies=false diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..d9289897d --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +16.15.1 diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 000000000..0e80a3c86 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +# package.json is formatted by package managers, so we ignore it here +package.json \ No newline at end of file diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 000000000..6718863f9 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,11 @@ +{ + "printWidth": 120, + "trailingComma": "es5", + "singleQuote": true, + "semi": true, + "tabWidth": 2, + "quoteProps": "as-needed", + "jsxSingleQuote": false, + "arrowParens": "always", + "endOfLine": "lf" +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..d7f3ce132 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "typescript.tsdk": "node_modules/typescript/lib", + "npm.packageManager": "pnpm", + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.formatOnSave": true, + "eslint.format.enable": true, + "editor.codeActionsOnSave": { + "source.fixAll": true + }, +} + \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..0b42ea814 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +hello@knovator.com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/README.md b/README.md new file mode 100644 index 000000000..f4eda6504 --- /dev/null +++ b/README.md @@ -0,0 +1,172 @@ + + +[![Contributors][contributors-shield]][contributors-url] +[![Forks][forks-shield]][forks-url] +[![Stargazers][stars-shield]][stars-url] +[![Issues][issues-shield]][issues-url] +[![MIT License][license-shield]][license-url] + + + +
+
+ +

impler.io

+ +

+ Open source infrastructure for data import +
+ Explore the docs » +
+
+ View Demo + · + Report Bug + · + Request Feature +

+
+ + + + +
+ Table of Contents +
    +
  1. + About The Project + +
    2. +
  2. Setup
    3. +
  3. Usage
    4. +
  4. Roadmap
    5. +
  5. Contributing
    6. +
  6. License
    7. +
  7. Contact
    8. +
+
+ + + + +## About The Project + +All projects need to give some kind of data import facility, so that their users can import data in application through files like `.csv`, `.xls`, `.xlsx`, etc. + +At first it looks like just importing file and inserting in database, but as the app grows facilities like validating data, data mapping, becomes must. `impler` provides infrastructure to applications, so that they don't have to write code for data import. + +> impler.io is under development. + +

(back to top)

+ +### Built With + +* [Nestjs](https://nestjs.com/) +* [Typescript](https://www.typescriptlang.org/) +* [Nx](https://nx.dev/) +* [Pnpm](https://pnpm.io/) + +

(back to top)

+ +## Setup +To set up `impler.io` locally, you need the following things installed in your computer. +1. `pnpm` +2. `localstack` +3. `mongodb` + +Follow these steps to setup the project locally, +1. Clone the repo, `git clone https://github.com/knovator/impler.io`. +2. Install the dependencies, `pnpm install`. +3. Copy `.env.development` file from `apps/api/src` to `apps/api/src/.env` and do changes to variables if needed. +4. Start the application, `pnpm start:dev`. +5. Start interacting with API by visiting `http://localhost:3000/api`. + + +## Usage + +`impler` need to be communicated through **REST API**, you can easily make call through **Swagger UI** provided at `http://localhost:3000/api`, +1. You create `project`. +2. You add `template` to `project`, Template refers to set of data you want to import i.e. users data. +3. Add `columns` to `template`, Column refers to individual fields template can have, for example users Template can have firstname, lastname, address, email, phonenumber, etc. +4. Upload `file` to `template`, After columns being set well, we're ready to import `.csv`, `.xls`, `.xlsx` file to aplication. + * `Upload` response returns data with `headings` specified in uploaded file. + * Keep note of uploaded file `id`, it will be used later. + * Uploaded file headings will be mapped automatically with `columns` provided for `template`. +5. Check `mapping` done for uploaded file, and finalize mappings. +6. Get `review` data for uploaded file and confirm reivew with option whether you want ot excempt invalid data or want to keep them. + +

(back to top)

+ + +## Roadmap + +- [x] API + - [x] Project + - [x] Template + - [x] Column + - [x] Upload + - [x] Mapping + - [x] Review + - [x] Processing data +- [x] Web + - [x] Upload Phase + - [x] Mapping Phase + - [x] Review Phase + - [x] Confirm Phase +- [] Infra + - [] Docker + +

(back to top)

+ + + +## Contributing + +Contributions are what make the open source community such an amazing place to learn, inspire, and create. Any contributions you make are **greatly appreciated**. + +If you have a suggestion that would make this better, please fork the repo and create a pull request. You can also simply open an issue with the tag "enhancement". +Don't forget to give the project a star! Thanks again! + +1. Fork the Project +2. Create your Feature Branch (`git checkout -b feature/AmazingFeature`) +3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`) +4. Push to the Branch (`git push origin feature/AmazingFeature`) +5. Open a Pull Request + +

(back to top)

+ + + + +## License + +Distributed under the MIT License. See `LICENSE.txt` for more information. + +

(back to top)

+ + + + +## Contact + +Knovator - [@knovator](https://twitter.com/knovator) + +Project Link: [https://github.com/knovator/impler.io](https://github.com/knovator/impler.io) + +

(back to top)

+ + + + +[contributors-shield]: https://img.shields.io/github/contributors/knovator/impler.io.svg?style=for-the-badge +[contributors-url]: https://github.com/knovator/impler.io/graphs/contributors +[forks-shield]: https://img.shields.io/github/forks/knovator/impler.io.svg?style=for-the-badge +[forks-url]: https://github.com/knovator/impler.io/network/members +[stars-shield]: https://img.shields.io/github/stars/knovator/impler.io.svg?style=for-the-badge +[stars-url]: https://github.com/knovator/impler.io/stargazers +[issues-shield]: https://img.shields.io/github/issues/knovator/impler.io.svg?style=for-the-badge +[issues-url]: https://github.com/knovator/impler.io/issues +[license-shield]: https://img.shields.io/github/license/knovator/impler.io.svg?style=for-the-badge +[license-url]: https://github.com/knovator/impler.io/blob/master/LICENSE.txt diff --git a/apps/api/.eslintrc.js b/apps/api/.eslintrc.js new file mode 100644 index 000000000..2ebdf17bd --- /dev/null +++ b/apps/api/.eslintrc.js @@ -0,0 +1,4 @@ +module.exports = { + extends: ['../../.eslintrc.js'], + ignorePatterns: '*.spec.ts', +}; diff --git a/apps/api/nest-cli.json b/apps/api/nest-cli.json new file mode 100644 index 000000000..e4e078182 --- /dev/null +++ b/apps/api/nest-cli.json @@ -0,0 +1,49 @@ +{ + "collection": "@nestjs/schematics", + "sourceRoot": "src", + "compilerOptions": { + "assets": [ + { + "include": ".env", + "outDir": "dist/src" + }, + { + "include": ".env.development", + "outDir": "dist/src" + }, + { + "include": ".env.test", + "outDir": "dist/src" + }, + { + "include": ".env.production", + "outDir": "dist/src" + }, + { + "include": "app/content-templates/usecases/compile-template/templates/basic.handlebars", + "outDir": "dist/src" + } + ], + "plugins": [ + { + "name": "@nestjs/swagger", + "options": { + "classValidatorShim": true, + "introspectComments": true + } + } + ], + "webpack": true + }, + "projects": { + "dal": { + "type": "library", + "root": "../../../libs/dal", + "entryFile": "index", + "sourceRoot": "../../../libs/dal/src", + "compilerOptions": { + "tsConfigPath": "../../../libs/dal/tsconfig.build.json" + } + } + } +} diff --git a/apps/api/nodemon.json b/apps/api/nodemon.json new file mode 100644 index 000000000..4e2bc9177 --- /dev/null +++ b/apps/api/nodemon.json @@ -0,0 +1,8 @@ +{ + "watch": ["src", "../core/dist"], + "ext": "ts", + "delay": 2, + "ignoreRoot": [".git"], + "ignore": ["src/**/*.spec.ts"], + "exec": "ts-node -r tsconfig-paths/register src/main.ts" +} diff --git a/apps/api/package.json b/apps/api/package.json new file mode 100644 index 000000000..861121332 --- /dev/null +++ b/apps/api/package.json @@ -0,0 +1,62 @@ +{ + "name": "@impler/api", + "version": "0.1.0", + "author": "knovator", + "license": "MIT", + "private": true, + "scripts": { + "prebuild": "rimraf dist", + "preinstall": "pnpm build", + "build": "nest build", + "format": "prettier --write \"src/**/*.ts\"", + "start": "pnpm start:dev", + "start:dev": "cross-env TZ=UTC nest start --watch", + "start:test": "cross-env NODE_ENV=test PORT=1336 TZ=UTC nest start --watch", + "start:debug": "TZ=UTC nodemon --config nodemon-debug.json", + "start:prod": "TZ=UTC node dist/main.js", + "lint": "eslint src", + "lint:fix": "pnpm lint -- --fix", + "test": "cross-env TZ=UTC NODE_ENV=test E2E_RUNNER=true mocha --timeout 10000 --require ts-node/register --exit src/**/**/*.spec.ts" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.185.0", + "@impler/dal": "^0.1.0", + "@impler/shared": "^0.1.0", + "@nestjs/common": "^9.1.2", + "@nestjs/core": "^9.1.2", + "@nestjs/platform-express": "^9.1.2", + "@nestjs/swagger": "^6.1.2", + "ajv": "^8.11.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.1.0", + "body-parser": "^1.20.0", + "class-transformer": "^0.5.1", + "class-validator": "^0.13.2", + "compression": "^1.7.4", + "dotenv": "^16.0.2", + "envalid": "^7.3.1", + "fast-csv": "^4.3.6", + "rimraf": "^3.0.2", + "xlsx": "^0.18.5" + }, + "devDependencies": { + "@nestjs/cli": "^9.1.5", + "@types/chai": "^4.3.4", + "@types/express": "^4.17.14", + "@types/mocha": "^10.0.0", + "@types/multer": "^1.4.7", + "@types/node": "^18.7.18", + "chai": "^4.3.7", + "mocha": "^10.1.0", + "nodemon": "^2.0.20", + "ts-loader": "^9.4.1", + "ts-node": "^10.9.1", + "tsconfig-paths": "^4.1.0", + "typescript": "^4.8.3" + }, + "lint-staged": { + "*.{js,jsx,ts,tsx}": [ + "eslint --fix" + ] + } +} diff --git a/apps/api/src/.env.development b/apps/api/src/.env.development new file mode 100644 index 000000000..76cf09286 --- /dev/null +++ b/apps/api/src/.env.development @@ -0,0 +1,15 @@ +NODE_ENV=local +PORT=3000 +FRONT_BASE_URL=http://localhost:4200 +RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672 + +# Database +MONGO_URL=mongodb://localhost:27017/impler-db + +# Storage +S3_LOCAL_STACK=http://localhost:4566 +S3_REGION=us-east-1 +S3_BUCKET_NAME=impler + +# Security +ACCESS-KEY= diff --git a/apps/api/src/.env.production b/apps/api/src/.env.production new file mode 100644 index 000000000..76cf09286 --- /dev/null +++ b/apps/api/src/.env.production @@ -0,0 +1,15 @@ +NODE_ENV=local +PORT=3000 +FRONT_BASE_URL=http://localhost:4200 +RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672 + +# Database +MONGO_URL=mongodb://localhost:27017/impler-db + +# Storage +S3_LOCAL_STACK=http://localhost:4566 +S3_REGION=us-east-1 +S3_BUCKET_NAME=impler + +# Security +ACCESS-KEY= diff --git a/apps/api/src/.env.test b/apps/api/src/.env.test new file mode 100644 index 000000000..0e9640aef --- /dev/null +++ b/apps/api/src/.env.test @@ -0,0 +1,5 @@ +NODE_ENV=local +PORT=3000 +FRONT_BASE_URL=http://localhost:4200 + +MONGO_URL=mongodb://localhost:27017/impler-db \ No newline at end of file diff --git a/apps/api/src/.example.env b/apps/api/src/.example.env new file mode 100644 index 000000000..76cf09286 --- /dev/null +++ b/apps/api/src/.example.env @@ -0,0 +1,15 @@ +NODE_ENV=local +PORT=3000 +FRONT_BASE_URL=http://localhost:4200 +RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672 + +# Database +MONGO_URL=mongodb://localhost:27017/impler-db + +# Storage +S3_LOCAL_STACK=http://localhost:4566 +S3_REGION=us-east-1 +S3_BUCKET_NAME=impler + +# Security +ACCESS-KEY= diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts new file mode 100644 index 000000000..3452e84e9 --- /dev/null +++ b/apps/api/src/app.module.ts @@ -0,0 +1,31 @@ +import { DynamicModule, Module } from '@nestjs/common'; +import { Type } from '@nestjs/common/interfaces/type.interface'; +import { ForwardReference } from '@nestjs/common/interfaces/modules/forward-reference.interface'; +import { SharedModule } from './app/shared/shared.module'; +import { ProjectModule } from './app/project/project.module'; +import { TemplateModule } from './app/template/template.module'; +import { ColumnModule } from './app/column/column.module'; +import { UploadModule } from './app/upload/upload.module'; +import { MappingModule } from './app/mapping/mapping.module'; +import { ReviewModule } from './app/review/review.module'; +import { CommonModule } from './app/common/common.module'; + +const modules: Array | ForwardReference> = [ + ProjectModule, + SharedModule, + TemplateModule, + ColumnModule, + UploadModule, + MappingModule, + ReviewModule, + CommonModule, +]; + +const providers = []; + +@Module({ + imports: modules, + controllers: [], + providers, +}) +export class AppModule {} diff --git a/apps/api/src/app/column/column.controller.ts b/apps/api/src/app/column/column.controller.ts new file mode 100644 index 000000000..6f563f0c1 --- /dev/null +++ b/apps/api/src/app/column/column.controller.ts @@ -0,0 +1,55 @@ +import { Controller, Put, Param, Body, Get, ParseArrayPipe, UseGuards } from '@nestjs/common'; +import { ApiTags, ApiBody, ApiOperation, ApiSecurity } from '@nestjs/swagger'; +import { ValidateMongoId } from '../shared/validations/valid-mongo-id.validation'; +import { UpdateColumnRequestDto } from './dtos/update-column-request.dto'; +import { UpdateColumnCommand } from './usecases/update-columns/update-columns.command'; +import { UpdateColumns } from './usecases/update-columns/update-columns.usecase'; +import { ColumnResponseDto } from './dtos/column-response.dto'; +import { GetColumns } from './usecases/get-columns/get-columns.usecase'; +import { APIKeyGuard } from '../shared/framework/auth.gaurd'; +import { ACCESS_KEY_NAME } from '@impler/shared'; + +@Controller('/column') +@ApiTags('Column') +@ApiSecurity(ACCESS_KEY_NAME) +@UseGuards(APIKeyGuard) +export class ColumnController { + constructor(private updateColumns: UpdateColumns, private getColumns: GetColumns) {} + + @Put(':templateId') + @ApiOperation({ + summary: 'Update columns for Template', + }) + @ApiBody({ type: [UpdateColumnRequestDto] }) + async updateTemplateColumns( + @Param('templateId', ValidateMongoId) _templateId: string, + @Body(new ParseArrayPipe({ items: UpdateColumnRequestDto })) body: UpdateColumnRequestDto[] + ): Promise { + return this.updateColumns.execute( + body.map((columnData) => + UpdateColumnCommand.create({ + key: columnData.key, + alternateKeys: columnData.alternateKeys, + isRequired: columnData.isRequired, + isUnique: columnData.isUnique, + name: columnData.name, + regex: columnData.regex, + regexDescription: columnData.regexDescription, + selectValues: columnData.selectValues, + sequence: columnData.sequence, + _templateId, + type: columnData.type, + }) + ), + _templateId + ); + } + + @Get(':templateId') + @ApiOperation({ + summary: 'Get template columns', + }) + async getTemplateColumns(@Param('templateId') _templateId: string): Promise { + return this.getColumns.execute(_templateId); + } +} diff --git a/apps/api/src/app/column/column.module.ts b/apps/api/src/app/column/column.module.ts new file mode 100644 index 000000000..12faa3a6f --- /dev/null +++ b/apps/api/src/app/column/column.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { USE_CASES } from './usecases'; +import { ColumnController } from './column.controller'; +import { SharedModule } from '../shared/shared.module'; + +@Module({ + imports: [SharedModule], + providers: [...USE_CASES], + controllers: [ColumnController], +}) +export class ColumnModule {} diff --git a/apps/api/src/app/column/dtos/column-response.dto.ts b/apps/api/src/app/column/dtos/column-response.dto.ts new file mode 100644 index 000000000..eac619751 --- /dev/null +++ b/apps/api/src/app/column/dtos/column-response.dto.ts @@ -0,0 +1,57 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { ColumnTypesEnum } from '@impler/shared'; + +export class ColumnResponseDto { + @ApiProperty({ + description: 'Name of the column', + type: 'string', + }) + name: string; + + @ApiProperty({ + description: 'Key of the column', + }) + key: string; + + @ApiProperty({ + description: 'Alternative possible keys of the column', + type: Array, + }) + alternateKeys: string[]; + + @ApiPropertyOptional({ + description: 'While true, it Indicates column value should exists in data', + }) + isRequired = false; + + @ApiPropertyOptional({ + description: 'While true, it Indicates column value should be unique in data', + }) + isUnique = false; + + @ApiProperty({ + description: 'Specifies the type of column', + enum: ColumnTypesEnum, + }) + type: string; + + @ApiPropertyOptional({ + description: 'Regex if type is Regex', + }) + regex: string; + + @ApiPropertyOptional({ + description: 'Description of Regex', + }) + regexDescription: string; + + @ApiPropertyOptional({ + description: 'List of possible values for column if type is Select', + }) + selectValues: string[]; + + @ApiProperty({ + description: 'Sequence of column', + }) + sequence: number; +} diff --git a/apps/api/src/app/column/dtos/update-column-request.dto.ts b/apps/api/src/app/column/dtos/update-column-request.dto.ts new file mode 100644 index 000000000..076354b1e --- /dev/null +++ b/apps/api/src/app/column/dtos/update-column-request.dto.ts @@ -0,0 +1,99 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { + ArrayMinSize, + IsArray, + IsBoolean, + IsDefined, + IsOptional, + IsString, + IsEnum, + IsNumber, + ValidateIf, + IsNotEmpty, + Validate, +} from 'class-validator'; +import { Type } from 'class-transformer'; +import { ColumnTypesEnum } from '@impler/shared'; +import { IsValidRegex } from '../../shared/framework/is-valid-regex.validator'; + +export class UpdateColumnRequestDto { + @ApiProperty({ + description: 'Name of the column', + type: 'string', + }) + @IsString() + @IsDefined() + name: string; + + @ApiProperty({ + description: 'Key of the column', + }) + @IsString() + @IsDefined() + key: string; + + @ApiProperty({ + description: 'Alternative possible keys of the column', + type: Array, + }) + @IsArray() + @IsOptional() + @Type(() => Array) + alternateKeys: string[]; + + @ApiPropertyOptional({ + description: 'While true, it Indicates column value should exists in data', + }) + @IsBoolean() + @IsOptional() + isRequired = false; + + @ApiPropertyOptional({ + description: 'While true, it Indicates column value should be unique in data', + }) + @IsBoolean() + @IsOptional() + isUnique = false; + + @ApiProperty({ + description: 'Specifies the type of column', + enum: ColumnTypesEnum, + }) + @IsEnum(ColumnTypesEnum, { + message: `type must be one of ${Object.values(ColumnTypesEnum).join(', ')}`, + }) + @IsDefined() + type: ColumnTypesEnum; + + @ApiPropertyOptional({ + description: 'Regex if type is Regex', + }) + @ValidateIf((object) => object.type === ColumnTypesEnum.REGEX) + @Validate(IsValidRegex) + @IsNotEmpty() + regex: string; + + @ApiPropertyOptional({ + description: 'Description of Regex', + }) + @ValidateIf((object) => object.type === ColumnTypesEnum.REGEX) + @IsString() + @IsOptional() + regexDescription: string; + + @ApiPropertyOptional({ + description: 'List of possible values for column if type is Select', + }) + @ValidateIf((object) => object.type === ColumnTypesEnum.SELECT) + @IsArray() + @ArrayMinSize(1) + @Type(() => Array) + selectValues: string[]; + + @ApiProperty({ + description: 'Sequence of column', + }) + @IsNumber() + @IsOptional() + sequence: number; +} diff --git a/apps/api/src/app/column/usecases/get-columns/get-columns.usecase.ts b/apps/api/src/app/column/usecases/get-columns/get-columns.usecase.ts new file mode 100644 index 000000000..7c6716a13 --- /dev/null +++ b/apps/api/src/app/column/usecases/get-columns/get-columns.usecase.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { ColumnRepository } from '@impler/dal'; + +@Injectable() +export class GetColumns { + constructor(private columnRepository: ColumnRepository) {} + + async execute(_templateId: string) { + return this.columnRepository.find({ _templateId }); + } +} diff --git a/apps/api/src/app/column/usecases/index.ts b/apps/api/src/app/column/usecases/index.ts new file mode 100644 index 000000000..4ec139175 --- /dev/null +++ b/apps/api/src/app/column/usecases/index.ts @@ -0,0 +1,8 @@ +import { UpdateColumns } from './update-columns/update-columns.usecase'; +import { GetColumns } from './get-columns/get-columns.usecase'; + +export const USE_CASES = [ + UpdateColumns, + GetColumns, + // +]; diff --git a/apps/api/src/app/column/usecases/update-columns/update-columns.command.ts b/apps/api/src/app/column/usecases/update-columns/update-columns.command.ts new file mode 100644 index 000000000..cd0133278 --- /dev/null +++ b/apps/api/src/app/column/usecases/update-columns/update-columns.command.ts @@ -0,0 +1,51 @@ +import { IsArray, IsBoolean, IsDefined, IsMongoId, IsNumber, IsOptional, IsString } from 'class-validator'; +import { Type } from 'class-transformer'; +import { ColumnTypesEnum } from '@impler/shared'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class UpdateColumnCommand extends BaseCommand { + @IsString() + @IsDefined() + name: string; + + @IsString() + @IsDefined() + key: string; + + @IsArray() + @IsOptional() + @Type(() => Array) + alternateKeys: string[]; + + @IsBoolean() + @IsOptional() + isRequired = false; + + @IsBoolean() + @IsOptional() + isUnique = false; + + @IsDefined() + type: ColumnTypesEnum; + + @IsString() + @IsOptional() + regex: string; + + @IsString() + @IsOptional() + regexDescription: string; + + @IsArray() + @IsOptional() + @Type(() => Array) + selectValues: string[]; + + @IsNumber() + @IsOptional() + sequence: number; + + @IsDefined() + @IsMongoId() + _templateId: string; +} diff --git a/apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts b/apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts new file mode 100644 index 000000000..79f67d524 --- /dev/null +++ b/apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts @@ -0,0 +1,37 @@ +import { Injectable } from '@nestjs/common'; +import { FileMimeTypesEnum } from '@impler/shared'; +import { ColumnRepository, TemplateRepository } from '@impler/dal'; +import { UpdateColumnCommand } from './update-columns.command'; +import { StorageService } from '../../../shared/storage/storage.service'; +import { FileNameService } from '../../../shared/file/name.service'; + +@Injectable() +export class UpdateColumns { + constructor( + private columnRepository: ColumnRepository, + private storageService: StorageService, + private fileNameService: FileNameService, + private templateRepository: TemplateRepository + ) {} + + async execute(command: UpdateColumnCommand[], _templateId: string) { + await this.columnRepository.deleteMany({ _templateId }); + this.saveSampleFile(command, _templateId); + + return this.columnRepository.createMany(command); + } + + async saveSampleFile(data: UpdateColumnCommand[], templateId: string) { + const csvContent = this.createCSVFileHeadingContent(data); + const fileName = this.fileNameService.getSampleFileName(templateId); + const sampleFileUrl = this.fileNameService.getSampleFileUrl(templateId); + await this.storageService.uploadFile(fileName, csvContent, FileMimeTypesEnum.CSV, true); + await this.templateRepository.update({ _id: templateId }, { sampleFileUrl }); + } + + createCSVFileHeadingContent(data: UpdateColumnCommand[]): string { + const headings = data.map((column) => column.key); + + return headings.join(','); + } +} diff --git a/apps/api/src/app/common/common.controller.ts b/apps/api/src/app/common/common.controller.ts new file mode 100644 index 000000000..08431566c --- /dev/null +++ b/apps/api/src/app/common/common.controller.ts @@ -0,0 +1,28 @@ +import { Body, Controller, Post, UseGuards } from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiSecurity } from '@nestjs/swagger'; +import { ACCESS_KEY_NAME } from '@impler/shared'; +import { APIKeyGuard } from '../shared/framework/auth.gaurd'; +import { ValidRequestDto } from './dtos/valid.dto'; +import { ValidRequestCommand } from './usecases/valid-request/valid-request.command'; +import { ValidRequest } from './usecases/valid-request/valid-request.usecase'; + +@Controller('/common') +@ApiTags('Common') +@ApiSecurity(ACCESS_KEY_NAME) +@UseGuards(APIKeyGuard) +export class CommonController { + constructor(private validRequest: ValidRequest) {} + + @Post('/valid') + @ApiOperation({ + summary: 'Check if request is valid (Checks Auth)', + }) + async isRequestValid(@Body() body: ValidRequestDto): Promise { + return this.validRequest.execute( + ValidRequestCommand.create({ + projectId: body.projectId, + template: body.template, + }) + ); + } +} diff --git a/apps/api/src/app/common/common.module.ts b/apps/api/src/app/common/common.module.ts new file mode 100644 index 000000000..7f3fe1611 --- /dev/null +++ b/apps/api/src/app/common/common.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { USE_CASES } from './usecases'; +import { CommonController } from './common.controller'; +import { SharedModule } from '../shared/shared.module'; + +@Module({ + imports: [SharedModule], + providers: [...USE_CASES], + controllers: [CommonController], +}) +export class CommonModule {} diff --git a/apps/api/src/app/common/dtos/valid.dto.ts b/apps/api/src/app/common/dtos/valid.dto.ts new file mode 100644 index 000000000..edc221b5b --- /dev/null +++ b/apps/api/src/app/common/dtos/valid.dto.ts @@ -0,0 +1,18 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsDefined, IsOptional, IsString, IsMongoId } from 'class-validator'; + +export class ValidRequestDto { + @ApiProperty({ + description: 'Id of the project', + }) + @IsMongoId() + @IsDefined() + projectId: string; + + @ApiProperty({ + description: 'ID or Code of the template', + }) + @IsString() + @IsOptional() + template?: string; +} diff --git a/apps/api/src/app/common/usecases/index.ts b/apps/api/src/app/common/usecases/index.ts new file mode 100644 index 000000000..f04a8fb27 --- /dev/null +++ b/apps/api/src/app/common/usecases/index.ts @@ -0,0 +1,6 @@ +import { ValidRequest } from './valid-request/valid-request.usecase'; + +export const USE_CASES = [ + ValidRequest, + // +]; diff --git a/apps/api/src/app/common/usecases/valid-request/valid-request.command.ts b/apps/api/src/app/common/usecases/valid-request/valid-request.command.ts new file mode 100644 index 000000000..b726a48ce --- /dev/null +++ b/apps/api/src/app/common/usecases/valid-request/valid-request.command.ts @@ -0,0 +1,12 @@ +import { IsDefined, IsMongoId, IsOptional, IsString } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class ValidRequestCommand extends BaseCommand { + @IsMongoId() + @IsDefined() + projectId: string; + + @IsString() + @IsOptional() + template: string; +} diff --git a/apps/api/src/app/common/usecases/valid-request/valid-request.usecase.ts b/apps/api/src/app/common/usecases/valid-request/valid-request.usecase.ts new file mode 100644 index 000000000..89dd30df3 --- /dev/null +++ b/apps/api/src/app/common/usecases/valid-request/valid-request.usecase.ts @@ -0,0 +1,37 @@ +import { Injectable } from '@nestjs/common'; +import { ProjectRepository, TemplateRepository, CommonRepository } from '@impler/dal'; +import { ValidRequestCommand } from './valid-request.command'; +import { DocumentNotFoundException } from '../../../shared/exceptions/document-not-found.exception'; +import { APIMessages } from '../../../shared/constants'; + +@Injectable() +export class ValidRequest { + constructor( + private projectRepository: ProjectRepository, + private templateRepository: TemplateRepository, + private commonRepository: CommonRepository + ) {} + + async execute(command: ValidRequestCommand) { + if (command.template) { + const isMongoId = this.commonRepository.validMongoId(command.template); + const templateCount = await this.templateRepository.count( + isMongoId + ? { _id: command.template, _projectId: command.projectId } + : { code: command.template, _projectId: command.projectId } + ); + if (!templateCount) { + throw new DocumentNotFoundException('Template', command.template, APIMessages.PROJECT_WITH_TEMPLATE_MISSING); + } + } else { + const projectCount = await this.projectRepository.count({ + _id: command.projectId, + }); + if (!projectCount) { + throw new DocumentNotFoundException('Project', command.projectId); + } + } + + return true; + } +} diff --git a/apps/api/src/app/mapping/dtos/update-columns.dto.ts b/apps/api/src/app/mapping/dtos/update-columns.dto.ts new file mode 100644 index 000000000..97a1cee32 --- /dev/null +++ b/apps/api/src/app/mapping/dtos/update-columns.dto.ts @@ -0,0 +1,18 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsDefined, IsString, IsMongoId } from 'class-validator'; + +export class UpdateMappingDto { + @ApiProperty({ + description: 'Id of the column', + }) + @IsMongoId() + @IsDefined() + _columnId: string; + + @ApiProperty({ + description: 'Selected Heading for column', + }) + @IsDefined() + @IsString() + columnHeading: string; +} diff --git a/apps/api/src/app/mapping/mapping.controller.ts b/apps/api/src/app/mapping/mapping.controller.ts new file mode 100644 index 000000000..af7e7ada7 --- /dev/null +++ b/apps/api/src/app/mapping/mapping.controller.ts @@ -0,0 +1,111 @@ +import { Body, Controller, Get, Param, ParseArrayPipe, Post, UseGuards } from '@nestjs/common'; +import { ApiTags, ApiSecurity, ApiOperation, ApiBody } from '@nestjs/swagger'; +import { ACCESS_KEY_NAME, UploadStatusEnum } from '@impler/shared'; +import { MappingEntity } from '@impler/dal'; + +import { APIKeyGuard } from '../shared/framework/auth.gaurd'; +import { ValidateMongoId } from '../shared/validations/valid-mongo-id.validation'; +import { GetUploadCommand } from '../shared/usecases/get-upload/get-upload.command'; +import { DoMapping } from './usecases/do-mapping/do-mapping.usecase'; +import { DoMappingCommand } from './usecases/do-mapping/do-mapping.command'; +import { GetUpload } from '../shared/usecases/get-upload/get-upload.usecase'; +import { GetMappings } from './usecases/get-mappings/get-mappings.usecase'; +import { UpdateMappingCommand } from './usecases/update-mappings/update-mappings.command'; +import { UpdateMappings } from './usecases/update-mappings/update-mappings.usecase'; +import { FinalizeUpload } from './usecases/finalize-upload/finalize-upload.usecase'; +import { UpdateMappingDto } from './dtos/update-columns.dto'; +import { ValidateMapping } from './usecases/validate-mapping/validate-mapping.usecase'; +import { validateUploadStatus } from '../shared/helpers/upload.helpers'; +import { validateNotFound } from '../shared/helpers/common.helper'; + +@Controller('/mapping') +@ApiTags('Mappings') +@ApiSecurity(ACCESS_KEY_NAME) +@UseGuards(APIKeyGuard) +export class MappingController { + constructor( + private getUpload: GetUpload, + private doMapping: DoMapping, + private getMappings: GetMappings, + private updateMappings: UpdateMappings, + private finalizeUpload: FinalizeUpload, + private validateMapping: ValidateMapping + ) {} + + @Get(':uploadId') + @ApiOperation({ + summary: 'Get mapping information for uploaded file', + }) + async getMappingInformation(@Param('uploadId', ValidateMongoId) uploadId: string): Promise[]> { + const uploadInformation = await this.getUpload.execute( + GetUploadCommand.create({ + uploadId, + select: 'status headings _templateId', + }) + ); + + // throw error if upload information not found + validateNotFound(uploadInformation, 'upload'); + + // Get mappings can be called only when file is uploaded or it's mapping in progress + validateUploadStatus(uploadInformation.status as UploadStatusEnum, [ + UploadStatusEnum.UPLOADED, + UploadStatusEnum.MAPPING, + ]); + + if (uploadInformation.status === UploadStatusEnum.UPLOADED) { + await this.doMapping.execute( + DoMappingCommand.create({ + headings: uploadInformation.headings, + _templateId: uploadInformation._templateId, + _uploadId: uploadId, + }) + ); + } + + return this.getMappings.execute(uploadId); + } + + @Post(':uploadId/finalize') + @ApiOperation({ + summary: 'Finalize mappings for upload', + }) + @ApiBody({ type: [UpdateMappingDto] }) + async finalizeMappings( + @Param('uploadId', ValidateMongoId) _uploadId: string, + @Body(new ParseArrayPipe({ items: UpdateMappingDto, optional: true })) body: UpdateMappingDto[] + ) { + const uploadInformation = await this.getUpload.execute( + GetUploadCommand.create({ + uploadId: _uploadId, + select: 'status', + }) + ); + + // throw error if upload information not found + validateNotFound(uploadInformation, 'upload'); + + // Finalize mapping can only be called after the mapping has been completed + validateUploadStatus(uploadInformation.status as UploadStatusEnum, [UploadStatusEnum.MAPPING]); + + // validate mapping data + await this.validateMapping.execute(body, _uploadId); + + // save mapping + if (Array.isArray(body) && body.length > 0) { + this.updateMappings.execute( + body.map((updateColumnData) => + UpdateMappingCommand.create({ + _columnId: updateColumnData._columnId, + _uploadId, + columnHeading: updateColumnData.columnHeading, + }) + ), + _uploadId + ); + } + + // update mapping status + return this.finalizeUpload.execute(_uploadId); + } +} diff --git a/apps/api/src/app/mapping/mapping.module.ts b/apps/api/src/app/mapping/mapping.module.ts new file mode 100644 index 000000000..02443ad4a --- /dev/null +++ b/apps/api/src/app/mapping/mapping.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { USE_CASES } from './usecases'; +import { MappingController } from './mapping.controller'; +import { SharedModule } from '../shared/shared.module'; + +@Module({ + imports: [SharedModule], + providers: [...USE_CASES], + controllers: [MappingController], +}) +export class MappingModule {} diff --git a/apps/api/src/app/mapping/usecases/do-mapping/do-mapping.command.ts b/apps/api/src/app/mapping/usecases/do-mapping/do-mapping.command.ts new file mode 100644 index 000000000..450a96448 --- /dev/null +++ b/apps/api/src/app/mapping/usecases/do-mapping/do-mapping.command.ts @@ -0,0 +1,16 @@ +import { IsArray, IsDefined, IsMongoId } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class DoMappingCommand extends BaseCommand { + @IsDefined() + @IsMongoId() + _uploadId: string; + + @IsDefined() + @IsMongoId() + _templateId: string; + + @IsDefined() + @IsArray() + headings: string[]; +} diff --git a/apps/api/src/app/mapping/usecases/do-mapping/do-mapping.usecase.ts b/apps/api/src/app/mapping/usecases/do-mapping/do-mapping.usecase.ts new file mode 100644 index 000000000..77c7cc3d6 --- /dev/null +++ b/apps/api/src/app/mapping/usecases/do-mapping/do-mapping.usecase.ts @@ -0,0 +1,73 @@ +import { Injectable } from '@nestjs/common'; +import { UploadStatusEnum } from '@impler/shared'; +import { ColumnEntity, ColumnRepository, MappingEntity, MappingRepository, UploadRepository } from '@impler/dal'; +import { DoMappingCommand } from './do-mapping.command'; + +@Injectable() +export class DoMapping { + constructor( + private columnRepository: ColumnRepository, + private mappingRepository: MappingRepository, + private uploadRepository: UploadRepository + ) {} + + async execute(command: DoMappingCommand) { + const columns = await this.columnRepository.find( + { + _templateId: command._templateId, + }, + 'key alternateKeys sequence', + { + sort: 'sequence', + } + ); + const mapping = this.buildMapping(columns, command.headings, command._uploadId); + const createdHeadings = await this.mappingRepository.createMany(mapping); + await this.uploadRepository.update({ _id: command._uploadId }, { status: UploadStatusEnum.MAPPING }); + + return createdHeadings; + } + + private buildMapping(columns: ColumnEntity[], headings: string[], _uploadId: string) { + const mappings: MappingEntity[] = []; + for (const column of columns) { + const heading = this.findBestMatchingHeading(headings, column.key, column.alternateKeys); + if (heading) { + mappings.push(this.buildMappingItem(column._id, _uploadId, heading)); + } else { + mappings.push(this.buildMappingItem(column._id, _uploadId)); + } + } + + return mappings; + } + + private findBestMatchingHeading(headings: string[], key: string, alternateKeys: string[]): string | null { + const mappedHeading = headings.find((heading: string) => this.checkStringEqual(heading, key)); + if (mappedHeading) { + // compare key + return mappedHeading; + } else if (Array.isArray(alternateKeys) && alternateKeys.length) { + // compare alternateKeys + const intersection = headings.find( + (heading: string) => !!alternateKeys.find((altKey) => this.checkStringEqual(altKey, heading)) + ); + + return intersection; + } + + return null; + } + + private checkStringEqual(a: string, b: string): boolean { + return String(a).localeCompare(String(b), undefined, { sensitivity: 'accent' }) === 0; + } + + private buildMappingItem(columnId: string, uploadId: string, heading?: string): MappingEntity { + return { + _columnId: columnId, + _uploadId: uploadId, + columnHeading: heading || null, + }; + } +} diff --git a/apps/api/src/app/mapping/usecases/finalize-upload/finalize-upload.usecase.ts b/apps/api/src/app/mapping/usecases/finalize-upload/finalize-upload.usecase.ts new file mode 100644 index 000000000..af74880e1 --- /dev/null +++ b/apps/api/src/app/mapping/usecases/finalize-upload/finalize-upload.usecase.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@nestjs/common'; +import { UploadRepository } from '@impler/dal'; +import { UploadStatusEnum } from '@impler/shared'; + +@Injectable() +export class FinalizeUpload { + constructor(private uploadRepository: UploadRepository) {} + + async execute(_uploadId: string) { + return await this.uploadRepository.findOneAndUpdate({ _id: _uploadId }, { status: UploadStatusEnum.MAPPED }); + } +} diff --git a/apps/api/src/app/mapping/usecases/get-mappings/get-mappings.usecase.ts b/apps/api/src/app/mapping/usecases/get-mappings/get-mappings.usecase.ts new file mode 100644 index 000000000..5f0e79152 --- /dev/null +++ b/apps/api/src/app/mapping/usecases/get-mappings/get-mappings.usecase.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { MappingRepository } from '@impler/dal'; + +@Injectable() +export class GetMappings { + constructor(private mappingRepository: MappingRepository) {} + + async execute(_uploadId: string) { + return await this.mappingRepository.getMappingInfo(_uploadId); + } +} diff --git a/apps/api/src/app/mapping/usecases/index.ts b/apps/api/src/app/mapping/usecases/index.ts new file mode 100644 index 000000000..f1557f27e --- /dev/null +++ b/apps/api/src/app/mapping/usecases/index.ts @@ -0,0 +1,16 @@ +import { DoMapping } from './do-mapping/do-mapping.usecase'; +import { GetMappings } from './get-mappings/get-mappings.usecase'; +import { UpdateMappings } from './update-mappings/update-mappings.usecase'; +import { FinalizeUpload } from './finalize-upload/finalize-upload.usecase'; +import { ValidateMapping } from './validate-mapping/validate-mapping.usecase'; +import { GetUpload } from '../../shared/usecases/get-upload/get-upload.usecase'; + +export const USE_CASES = [ + DoMapping, + GetMappings, + UpdateMappings, + FinalizeUpload, + ValidateMapping, + GetUpload, + // +]; diff --git a/apps/api/src/app/mapping/usecases/update-mappings/update-mappings.command.ts b/apps/api/src/app/mapping/usecases/update-mappings/update-mappings.command.ts new file mode 100644 index 000000000..49b7c382f --- /dev/null +++ b/apps/api/src/app/mapping/usecases/update-mappings/update-mappings.command.ts @@ -0,0 +1,16 @@ +import { IsString, IsDefined, IsMongoId } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class UpdateMappingCommand extends BaseCommand { + @IsDefined() + @IsMongoId() + _columnId: string; + + @IsDefined() + @IsMongoId() + _uploadId: string; + + @IsDefined() + @IsString() + columnHeading: string; +} diff --git a/apps/api/src/app/mapping/usecases/update-mappings/update-mappings.usecase.ts b/apps/api/src/app/mapping/usecases/update-mappings/update-mappings.usecase.ts new file mode 100644 index 000000000..5a2bc0fb7 --- /dev/null +++ b/apps/api/src/app/mapping/usecases/update-mappings/update-mappings.usecase.ts @@ -0,0 +1,14 @@ +import { Injectable } from '@nestjs/common'; +import { MappingRepository } from '@impler/dal'; +import { UpdateMappingCommand } from './update-mappings.command'; + +@Injectable() +export class UpdateMappings { + constructor(private mappingRepository: MappingRepository) {} + + async execute(command: UpdateMappingCommand[], _uploadId: string) { + await this.mappingRepository.deleteMany({ _uploadId }); + + return this.mappingRepository.createMany(command); + } +} diff --git a/apps/api/src/app/mapping/usecases/validate-mapping/validate-mapping.command.ts b/apps/api/src/app/mapping/usecases/validate-mapping/validate-mapping.command.ts new file mode 100644 index 000000000..8e2815725 --- /dev/null +++ b/apps/api/src/app/mapping/usecases/validate-mapping/validate-mapping.command.ts @@ -0,0 +1,12 @@ +import { IsString, IsDefined, IsMongoId } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class ValidateMappingCommand extends BaseCommand { + @IsDefined() + @IsMongoId() + _columnId: string; + + @IsDefined() + @IsString() + columnHeading: string; +} diff --git a/apps/api/src/app/mapping/usecases/validate-mapping/validate-mapping.usecase.ts b/apps/api/src/app/mapping/usecases/validate-mapping/validate-mapping.usecase.ts new file mode 100644 index 000000000..c06301454 --- /dev/null +++ b/apps/api/src/app/mapping/usecases/validate-mapping/validate-mapping.usecase.ts @@ -0,0 +1,29 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; +import { ColumnRepository, UploadRepository } from '@impler/dal'; +import { ValidateMappingCommand } from './validate-mapping.command'; + +@Injectable() +export class ValidateMapping { + constructor(private columnRepository: ColumnRepository, private uploadRepository: UploadRepository) {} + + async execute(command: ValidateMappingCommand[], _uploadId: string) { + // check if mapping data contains duplicates + const mappings = [...new Map(command.map((item) => [item._columnId, item])).values()]; + if (mappings.length !== command.length) throw new BadRequestException('Mapping data contains duplicates'); + + // Check if mapping data _columnIds are valid + const columnIds = command.map((mapping) => ({ + _id: mapping._columnId, + })); + const count = await this.columnRepository.count({ + $or: [...columnIds], + }); + if (count !== command.length) throw new BadRequestException(`Mapping data contains invalid _columnId(s)`); + + // check if mapping data headings are valid + const columnHeadings = command.map((mapping) => mapping.columnHeading); + const uploadInfo = await this.uploadRepository.findById(_uploadId, 'headings'); + const isAllHeadingsAreValid = columnHeadings.every((heading) => uploadInfo.headings.includes(heading)); + if (!isAllHeadingsAreValid) throw new BadRequestException(`Mapping data contains invalid columnHeading values`); + } +} diff --git a/apps/api/src/app/project/dtos/create-project-request.dto.ts b/apps/api/src/app/project/dtos/create-project-request.dto.ts new file mode 100644 index 000000000..4cafa3ed0 --- /dev/null +++ b/apps/api/src/app/project/dtos/create-project-request.dto.ts @@ -0,0 +1,32 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsDefined, IsOptional, IsString, Validate } from 'class-validator'; +import { Transform } from 'class-transformer'; +import { changeToCode } from '@impler/shared'; +import { UniqueValidator } from '../../shared/framework/is-unique.validator'; + +export class CreateProjectRequestDto { + @ApiProperty({ + description: 'Name of the project', + }) + @IsString() + @IsDefined() + name: string; + + @ApiProperty({ + description: 'Code of the project', + }) + @IsString() + @IsDefined() + @Validate(UniqueValidator, ['Project', 'code'], { + message: 'Code is already taken', + }) + @Transform((value) => changeToCode(value.value)) + code: string; + + @ApiPropertyOptional({ + description: 'Name of authentication header to sent along the request', + }) + @IsString() + @IsOptional() + authHeaderName: string; +} diff --git a/apps/api/src/app/project/dtos/project-response.dto.ts b/apps/api/src/app/project/dtos/project-response.dto.ts new file mode 100644 index 000000000..f7848568b --- /dev/null +++ b/apps/api/src/app/project/dtos/project-response.dto.ts @@ -0,0 +1,32 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsDefined, IsOptional, IsString } from 'class-validator'; + +export class ProjectResponseDto { + @ApiPropertyOptional({ + description: 'Id of the project', + }) + @IsString() + @IsDefined() + _id?: string; + + @ApiProperty({ + description: 'Name of the project', + }) + @IsString() + @IsDefined() + name: string; + + @ApiProperty({ + description: 'Code of the project', + }) + @IsString() + @IsDefined() + code: string; + + @ApiPropertyOptional({ + description: 'Name of authentication header to sent along the request', + }) + @IsString() + @IsOptional() + authHeaderName?: string; +} diff --git a/apps/api/src/app/project/dtos/update-project-request.dto.ts b/apps/api/src/app/project/dtos/update-project-request.dto.ts new file mode 100644 index 000000000..15c61f6d8 --- /dev/null +++ b/apps/api/src/app/project/dtos/update-project-request.dto.ts @@ -0,0 +1,18 @@ +import { ApiPropertyOptional } from '@nestjs/swagger'; +import { IsOptional, IsString } from 'class-validator'; + +export class UpdateProjectRequestDto { + @ApiPropertyOptional({ + description: 'Name of the project', + }) + @IsString() + @IsOptional() + name: string; + + @ApiPropertyOptional({ + description: 'Name of authentication header to sent along the request', + }) + @IsString() + @IsOptional() + authHeaderName: string; +} diff --git a/apps/api/src/app/project/project.controller.ts b/apps/api/src/app/project/project.controller.ts new file mode 100644 index 000000000..91d20ed82 --- /dev/null +++ b/apps/api/src/app/project/project.controller.ts @@ -0,0 +1,94 @@ +import { Body, Controller, Delete, Get, Param, Post, Put, UseGuards } from '@nestjs/common'; +import { ApiOperation, ApiTags, ApiOkResponse, ApiSecurity } from '@nestjs/swagger'; +import { CreateProjectRequestDto } from './dtos/create-project-request.dto'; +import { ProjectResponseDto } from './dtos/project-response.dto'; +import { GetProjects } from './usecases/get-projects/get-projects.usecase'; +import { CreateProject } from './usecases/create-project/create-project.usecase'; +import { CreateProjectCommand } from './usecases/create-project/create-project.command'; +import { UpdateProjectRequestDto } from './dtos/update-project-request.dto'; +import { UpdateProject } from './usecases/update-project/update-project.usecase'; +import { UpdateProjectCommand } from './usecases/update-project/update-project.command'; +import { DeleteProject } from './usecases/delete-project/delete-project.usecase'; +import { DocumentNotFoundException } from '../shared/exceptions/document-not-found.exception'; +import { ValidateMongoId } from '../shared/validations/valid-mongo-id.validation'; +import { APIKeyGuard } from '../shared/framework/auth.gaurd'; +import { ACCESS_KEY_NAME } from '@impler/shared'; + +@Controller('/project') +@ApiTags('Project') +@ApiSecurity(ACCESS_KEY_NAME) +@UseGuards(APIKeyGuard) +export class ProjectController { + constructor( + private getProjectsUsecase: GetProjects, + private createProjectUsecase: CreateProject, + private updateProjectUsecase: UpdateProject, + private deleteProjectUsecase: DeleteProject + ) {} + + @Get('') + @ApiOperation({ + summary: 'Get projects', + }) + @ApiOkResponse({ + type: [ProjectResponseDto], + }) + getProjects(): Promise { + return this.getProjectsUsecase.execute(); + } + + @Post('') + @ApiOperation({ + summary: 'Create project', + }) + @ApiOkResponse({ + type: ProjectResponseDto, + }) + createProject(@Body() body: CreateProjectRequestDto): Promise { + return this.createProjectUsecase.execute( + CreateProjectCommand.create({ + code: body.code, + name: body.name, + authHeaderName: body.authHeaderName, + }) + ); + } + + @Put(':projectId') + @ApiOperation({ + summary: 'Update project', + }) + @ApiOkResponse({ + type: ProjectResponseDto, + }) + async updateProject( + @Body() body: UpdateProjectRequestDto, + @Param('projectId', ValidateMongoId) projectId: string + ): Promise { + const document = await this.updateProjectUsecase.execute( + UpdateProjectCommand.create({ name: body.name, authHeaderName: body.authHeaderName }), + projectId + ); + if (!document) { + throw new DocumentNotFoundException('Project', projectId); + } + + return document; + } + + @Delete(':projectId') + @ApiOperation({ + summary: 'Delete project', + }) + @ApiOkResponse({ + type: ProjectResponseDto, + }) + async deleteProject(@Param('projectId', ValidateMongoId) projectId: string): Promise { + const document = await this.deleteProjectUsecase.execute(projectId); + if (!document) { + throw new DocumentNotFoundException('Project', projectId); + } + + return document; + } +} diff --git a/apps/api/src/app/project/project.module.ts b/apps/api/src/app/project/project.module.ts new file mode 100644 index 000000000..6b404050f --- /dev/null +++ b/apps/api/src/app/project/project.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { USE_CASES } from './usecases'; +import { SharedModule } from '../shared/shared.module'; +import { ProjectController } from './project.controller'; +import { UniqueValidator } from '../shared/framework/is-unique.validator'; + +@Module({ + imports: [SharedModule, UniqueValidator], + providers: [...USE_CASES], + controllers: [ProjectController], +}) +export class ProjectModule {} diff --git a/apps/api/src/app/project/usecases/create-project/create-project.command.ts b/apps/api/src/app/project/usecases/create-project/create-project.command.ts new file mode 100644 index 000000000..0bdca9e27 --- /dev/null +++ b/apps/api/src/app/project/usecases/create-project/create-project.command.ts @@ -0,0 +1,16 @@ +import { IsDefined, IsOptional, IsString } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class CreateProjectCommand extends BaseCommand { + @IsDefined() + @IsString() + name: string; + + @IsDefined() + @IsString() + code: string; + + @IsOptional() + @IsString() + authHeaderName: string; +} diff --git a/apps/api/src/app/project/usecases/create-project/create-project.usecase.ts b/apps/api/src/app/project/usecases/create-project/create-project.usecase.ts new file mode 100644 index 000000000..61ed4030a --- /dev/null +++ b/apps/api/src/app/project/usecases/create-project/create-project.usecase.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@nestjs/common'; +import { ProjectRepository } from '@impler/dal'; +import { CreateProjectCommand } from './create-project.command'; + +@Injectable() +export class CreateProject { + constructor(private projectRepository: ProjectRepository) {} + + async execute(command: CreateProjectCommand) { + return this.projectRepository.create(command); + } +} diff --git a/apps/api/src/app/project/usecases/delete-project/delete-project.usecase.ts b/apps/api/src/app/project/usecases/delete-project/delete-project.usecase.ts new file mode 100644 index 000000000..9b43729f9 --- /dev/null +++ b/apps/api/src/app/project/usecases/delete-project/delete-project.usecase.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { ProjectRepository } from '@impler/dal'; + +@Injectable() +export class DeleteProject { + constructor(private projectRepository: ProjectRepository) {} + + async execute(id: string) { + return this.projectRepository.delete({ _id: id }); + } +} diff --git a/apps/api/src/app/project/usecases/get-projects/get-projects.usecase.ts b/apps/api/src/app/project/usecases/get-projects/get-projects.usecase.ts new file mode 100644 index 000000000..da5f7cdc5 --- /dev/null +++ b/apps/api/src/app/project/usecases/get-projects/get-projects.usecase.ts @@ -0,0 +1,21 @@ +import { Injectable } from '@nestjs/common'; +import { ProjectRepository } from '@impler/dal'; +import { ProjectResponseDto } from '../../dtos/project-response.dto'; + +@Injectable() +export class GetProjects { + constructor(private projectRepository: ProjectRepository) {} + + async execute(): Promise { + const projects = await this.projectRepository.find({}); + + return projects.map((project) => { + return { + _id: project._id, + name: project.name, + code: project.code, + authHeaderName: project.authHeaderName, + }; + }); + } +} diff --git a/apps/api/src/app/project/usecases/index.ts b/apps/api/src/app/project/usecases/index.ts new file mode 100644 index 000000000..6a77192e0 --- /dev/null +++ b/apps/api/src/app/project/usecases/index.ts @@ -0,0 +1,12 @@ +import { CreateProject } from './create-project/create-project.usecase'; +import { GetProjects } from './get-projects/get-projects.usecase'; +import { UpdateProject } from './update-project/update-project.usecase'; +import { DeleteProject } from './delete-project/delete-project.usecase'; + +export const USE_CASES = [ + GetProjects, + CreateProject, + UpdateProject, + DeleteProject, + // +]; diff --git a/apps/api/src/app/project/usecases/update-project/update-project.command.ts b/apps/api/src/app/project/usecases/update-project/update-project.command.ts new file mode 100644 index 000000000..befb7aa72 --- /dev/null +++ b/apps/api/src/app/project/usecases/update-project/update-project.command.ts @@ -0,0 +1,12 @@ +import { IsOptional, IsString } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class UpdateProjectCommand extends BaseCommand { + @IsOptional() + @IsString() + name: string; + + @IsOptional() + @IsString() + authHeaderName: string; +} diff --git a/apps/api/src/app/project/usecases/update-project/update-project.usecase.ts b/apps/api/src/app/project/usecases/update-project/update-project.usecase.ts new file mode 100644 index 000000000..01eba6063 --- /dev/null +++ b/apps/api/src/app/project/usecases/update-project/update-project.usecase.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@nestjs/common'; +import { ProjectRepository } from '@impler/dal'; +import { UpdateProjectCommand } from './update-project.command'; + +@Injectable() +export class UpdateProject { + constructor(private projectRepository: ProjectRepository) {} + + async execute(command: UpdateProjectCommand, id: string) { + return this.projectRepository.findOneAndUpdate({ _id: id }, command); + } +} diff --git a/apps/api/src/app/review/dtos/confirm-review-request.dto.ts b/apps/api/src/app/review/dtos/confirm-review-request.dto.ts new file mode 100644 index 000000000..040edc4ef --- /dev/null +++ b/apps/api/src/app/review/dtos/confirm-review-request.dto.ts @@ -0,0 +1,13 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsBoolean, IsOptional } from 'class-validator'; + +export class ConfirmReviewRequestDto { + @ApiProperty({ + description: 'Boolean value indicating whether to process the invalid data or not.', + default: false, + required: false, + }) + @IsBoolean() + @IsOptional() + processInvalidRecords: boolean; +} diff --git a/apps/api/src/app/review/review.controller.ts b/apps/api/src/app/review/review.controller.ts new file mode 100644 index 000000000..593017c4d --- /dev/null +++ b/apps/api/src/app/review/review.controller.ts @@ -0,0 +1,114 @@ +import { BadRequestException, Body, Controller, Get, Param, Post, Query, UseGuards } from '@nestjs/common'; +import { ApiOperation, ApiTags, ApiSecurity, ApiQuery, ApiOkResponse } from '@nestjs/swagger'; +import { FileEntity, UploadEntity } from '@impler/dal'; +import { ACCESS_KEY_NAME, UploadStatusEnum } from '@impler/shared'; +import { APIMessages } from '@shared/constants'; +import { APIKeyGuard } from '@shared/framework/auth.gaurd'; +import { validateUploadStatus } from '@shared/helpers/upload.helpers'; +import { DoReview } from './usecases/do-review/do-review.usecase'; +import { GetUploadInvalidData } from './usecases/get-upload-invalid-data/get-upload-invalid-data.usecase'; +import { SaveReviewData } from './usecases/save-review-data/save-review-data.usecase'; +import { GetFileInvalidData } from './usecases/get-file-invalid-data/get-file-invalid-data.usecase'; +import { ValidateMongoId } from '@shared/validations/valid-mongo-id.validation'; +import { ConfirmReviewRequestDto } from './dtos/confirm-review-request.dto'; +import { GetUploadCommand } from '@shared/usecases/get-upload/get-upload.command'; +import { GetUpload } from '@shared/usecases/get-upload/get-upload.usecase'; +import { paginateRecords, validateNotFound } from '@shared/helpers/common.helper'; +import { StartProcess } from './usecases/start-process/start-process.usecase'; +import { StartProcessCommand } from './usecases/start-process/start-process.command'; +import { PaginationResponseDto } from '@shared/dtos/pagination-response.dto'; +import { Defaults } from '@shared/constants'; + +@Controller('/review') +@ApiTags('Review') +@ApiSecurity(ACCESS_KEY_NAME) +@UseGuards(APIKeyGuard) +export class ReviewController { + constructor( + private doReview: DoReview, + private getUpload: GetUpload, + private startProcess: StartProcess, + private saveReviewData: SaveReviewData, + private getFileInvalidData: GetFileInvalidData, + private getUploadInvalidData: GetUploadInvalidData + ) {} + + @Get(':uploadId') + @ApiOperation({ + summary: 'Get Review data for uploaded file', + }) + @ApiQuery({ + name: 'page', + required: false, + type: Number, + description: 'Page index of data to return', + }) + @ApiQuery({ + name: 'limit', + required: false, + type: Number, + description: 'Size of data to return', + }) + @ApiOkResponse({ + description: 'Paginated reviewed data', + type: PaginationResponseDto, + }) + async getReview( + @Param('uploadId') _uploadId: string, + @Query('page') page = Defaults.PAGE, + @Query('limit') limit = Defaults.PAGE_LIMIT + ): Promise { + const uploadData = await this.getUploadInvalidData.execute(_uploadId); + if (!uploadData) throw new BadRequestException(APIMessages.UPLOAD_NOT_FOUND); + + // Only Mapped & Reviewing status are allowed + validateUploadStatus(uploadData.status as UploadStatusEnum, [UploadStatusEnum.MAPPED, UploadStatusEnum.REVIEWING]); + + // Get Invalid Data either from Validation or Validation Result + let invalidData = []; + if (uploadData.status === UploadStatusEnum.MAPPED) { + // uploaded file is mapped, do review + const reviewData = await this.doReview.execute(_uploadId); + // save invalid data to storage + this.saveReviewData.execute(_uploadId, reviewData.invalid, reviewData.valid); + + invalidData = reviewData.invalid; + } else { + // Uploaded file is already reviewed, return reviewed data + invalidData = await this.getFileInvalidData.execute( + (uploadData._invalidDataFileId as unknown as FileEntity).path + ); + } + + return paginateRecords(invalidData, page, limit); + } + + @Post(':uploadId/confirm') + @ApiOperation({ + summary: 'Confirm review data for uploaded file', + }) + async doConfirmReview( + @Param('uploadId', ValidateMongoId) _uploadId: string, + @Body() body: ConfirmReviewRequestDto + ): Promise { + const uploadInformation = await this.getUpload.execute( + GetUploadCommand.create({ + uploadId: _uploadId, + select: 'status', + }) + ); + + // throw error if upload information not found + validateNotFound(uploadInformation, 'upload'); + + // upload files with status reviewing can only be confirmed + validateUploadStatus(uploadInformation.status as UploadStatusEnum, [UploadStatusEnum.REVIEWING]); + + return this.startProcess.execute( + StartProcessCommand.create({ + _uploadId: _uploadId, + processInvalidRecords: body.processInvalidRecords, + }) + ); + } +} diff --git a/apps/api/src/app/review/review.module.ts b/apps/api/src/app/review/review.module.ts new file mode 100644 index 000000000..a1b78ba44 --- /dev/null +++ b/apps/api/src/app/review/review.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { USE_CASES } from './usecases'; +import { ReviewController } from './review.controller'; +import { SharedModule } from '../shared/shared.module'; +import { AJVService } from './service/AJV.service'; +import { QueueService } from '../shared/storage/queue.service'; + +@Module({ + imports: [SharedModule], + providers: [...USE_CASES, AJVService, QueueService], + controllers: [ReviewController], +}) +export class ReviewModule {} diff --git a/apps/api/src/app/review/service/AJV.service.spec.ts b/apps/api/src/app/review/service/AJV.service.spec.ts new file mode 100644 index 000000000..2c2a35550 --- /dev/null +++ b/apps/api/src/app/review/service/AJV.service.spec.ts @@ -0,0 +1,192 @@ +import { ColumnTypesEnum } from '@impler/shared'; +import { expect } from 'chai'; +import { AJVService } from './AJV.service'; + +describe('AJV Service', () => { + let ajvService = new AJVService(); + describe('isRequired', () => { + it('should mark data invalid if value is empty', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "username", isRequired: true, name: "Username", type: ColumnTypesEnum.STRING }], + [{ _columnId: "a", columnHeading: "username" }], + [{ username: "" }] + ); + expect(validationResult.invalid.length).to.equal(1); + expect(validationResult.invalid[0].message).to.equal("`username` must not be empty"); + expect(validationResult.valid.length).to.equal(0); + }); + it('should mark data invalid if value is empty for number type', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "id", isRequired: true, name: "ID", type: ColumnTypesEnum.NUMBER }], + [{ _columnId: "a", columnHeading: "id" }], + [{ id: "" }] + ); + expect(validationResult.invalid.length).to.equal(1); + expect(validationResult.invalid[0].message).to.include("`id` must not be empty"); + expect(validationResult.valid.length).to.equal(0); + }); + it('should mark data valid if value is not empty', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "username", isRequired: true, name: "Username", type: ColumnTypesEnum.STRING }], + [{ _columnId: "a", columnHeading: "username" }], + [{ username: "test" }] + ); + expect(validationResult.invalid.length).to.equal(0); + expect(validationResult.valid.length).to.equal(1); + }); + }); + describe('isUnique', () => { + it('should mark data invalid if value is not unique', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "username", isUnique: true, name: "Username", type: ColumnTypesEnum.STRING }], + [{ _columnId: "a", columnHeading: "username" }], + [{ username: "test" }, { username: "test" }] + ); + expect(validationResult.invalid.length).to.equal(1); + expect(validationResult.invalid[0].message).to.equal("`username` must be unique"); + expect(validationResult.valid.length).to.equal(1); + }); + it('should mark data valid if value is unique', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "username", isUnique: true, name: "Username", type: ColumnTypesEnum.STRING }], + [{ _columnId: "a", columnHeading: "username" }], + [{ username: "test" }, { username: "test2" }] + ); + expect(validationResult.invalid.length).to.equal(0); + expect(validationResult.valid.length).to.equal(2); + }); + }); + describe("Email", () => { + it('should mark data invalid if value is not a valid email', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "regemail", name: "Email", type: ColumnTypesEnum.EMAIL }], + [{ _columnId: "a", columnHeading: "regemail" }], + [{ regemail: "test" }] + ); + expect(validationResult.invalid.length).to.equal(1); + expect(validationResult.invalid[0].message).to.equal("`regemail` must be a valid email"); + expect(validationResult.valid.length).to.equal(0); + }); + it('should mark data valid if value is a valid email', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "regemail", name: "Email", type: ColumnTypesEnum.EMAIL }], + [{ _columnId: "a", columnHeading: "regemail" }], + [{ regemail: "test@gmail.com" }] + ); + expect(validationResult.invalid.length).to.equal(0); + expect(validationResult.valid.length).to.equal(1); + }); + }) + describe("Regex", () => { + it('should mark data invalid if value does not match regex', () => { + const pattern = "^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$"; + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "regemail", name: "Email", type: ColumnTypesEnum.REGEX, regex: "^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$" }], + [{ _columnId: "a", columnHeading: "regemail" }], + [{ regemail: "test" }] + ); + expect(validationResult.invalid.length).to.equal(1); + expect(validationResult.invalid[0].message).to.include('`regemail`' + ` must match the pattern /${pattern}/`); + expect(validationResult.valid.length).to.equal(0); + }); + it('should mark data valid if value matches regex', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "regemail", name: "Email", type: ColumnTypesEnum.REGEX, regex: "^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$" }], + [{ _columnId: "a", columnHeading: "regemail" }], + [{ regemail: "test@gmail.com" }] + ); + expect(validationResult.invalid.length).to.equal(0); + expect(validationResult.valid.length).to.equal(1); + }); + }); + describe("Number", () => { + it('should mark data invalid if value is not a number', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "id", name: "ID", type: ColumnTypesEnum.NUMBER }], + [{ _columnId: "a", columnHeading: "id" }], + [{ id: "test" }] + ); + expect(validationResult.invalid.length).to.equal(1); + expect(validationResult.invalid[0].message).to.equal("`id` must be number"); + expect(validationResult.valid.length).to.equal(0); + }); + it('should mark data valid if value is a number', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "id", name: "ID", type: ColumnTypesEnum.NUMBER }], + [{ _columnId: "a", columnHeading: "id" }], + [{ id: 1 }] + ); + expect(validationResult.invalid.length).to.equal(0); + expect(validationResult.valid.length).to.equal(1); + }); + }); + describe("Date", () => { + it('should mark data invalid if value is not a date', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "dob", name: "Date of Birth", type: ColumnTypesEnum.DATE }], + [{ _columnId: "a", columnHeading: "dob" }], + [{ dob: "test" }] + ); + expect(validationResult.invalid.length).to.equal(1); + expect(validationResult.invalid[0].message).to.equal("`dob` must be a valid date"); + expect(validationResult.valid.length).to.equal(0); + }); + it('should mark data valid if value is a date', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "dob", name: "Date of Birth", type: ColumnTypesEnum.DATE }], + [{ _columnId: "a", columnHeading: "dob" }], + [{ dob: "2020-01-01" }] + ); + expect(validationResult.invalid.length).to.equal(0); + expect(validationResult.valid.length).to.equal(1); + }); + }); + describe('Select', () => { + it('should mark data invalid if value is not in select options', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "gender", name: "Gender", type: ColumnTypesEnum.SELECT, selectValues: ['Male', 'Female'] }], + [{ _columnId: "a", columnHeading: "gender" }], + [{ gender: "abcd" }] + ); + expect(validationResult.invalid.length).to.equal(1); + expect(validationResult.invalid[0].message).to.equal("`gender` must be from [Male,Female]"); + expect(validationResult.valid.length).to.equal(0); + }); + it('should mark data valid if value is in select options', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "gender", name: "Gender", type: ColumnTypesEnum.SELECT, selectValues: ['Male', 'Female'] }], + [{ _columnId: "a", columnHeading: "gender" }], + [{ gender: "Male" }] + ); + expect(validationResult.invalid.length).to.equal(0); + expect(validationResult.valid.length).to.equal(1); + }); + }); + describe('Any', () => { + it('should mark data valid if value is any', () => { + let validationResult = ajvService.validate( + // @ts-ignore + [{ _id: "a", key: "any", name: "Any", type: ColumnTypesEnum.ANY }], + [{ _columnId: "a", columnHeading: "any" }], + [{ any: "test" }, { any: 1 }, { any: "2020-01-01" }] + ); + expect(validationResult.invalid.length).to.equal(0); + expect(validationResult.valid.length).to.equal(3); + }); + }) +}) diff --git a/apps/api/src/app/review/service/AJV.service.ts b/apps/api/src/app/review/service/AJV.service.ts new file mode 100644 index 000000000..09a063a54 --- /dev/null +++ b/apps/api/src/app/review/service/AJV.service.ts @@ -0,0 +1,216 @@ +import { Injectable } from '@nestjs/common'; +import { ColumnTypesEnum } from '@impler/shared'; +import { ColumnEntity, MappingEntity } from '@impler/dal'; +import Ajv, { ErrorObject, AnySchemaObject } from 'ajv'; +import addFormats from 'ajv-formats'; +import addKeywords from 'ajv-keywords'; + +const ajv = new Ajv({ + allErrors: true, + coerceTypes: true, + allowUnionTypes: true, + removeAdditional: true, + verbose: true, +}); +addFormats(ajv, ['email']); +addKeywords(ajv); +ajv.addFormat('custom-date-time', function (dateTimeString) { + if (typeof dateTimeString === 'object') { + dateTimeString = (dateTimeString as Date).toISOString(); + } + + return !isNaN(Date.parse(dateTimeString)); // any test that returns true/false +}); + +// Empty keyword +ajv.addKeyword({ + keyword: 'emptyCheck', + schema: false, + compile: () => { + return (data) => (data === undefined || data === null || data === '' ? false : true); + }, +}); + +// Unique keyword +let uniqueItems: Record> = {}; +ajv.addKeyword({ + keyword: 'uniqueCheck', + schema: false, // keyword value is not used, can be true + validate: function (data: any, dataPath: AnySchemaObject) { + if (uniqueItems[dataPath.parentDataProperty].has(data)) { + return false; + } + uniqueItems[dataPath.parentDataProperty].add(data); + + return true; + }, +}); + +@Injectable() +export class AJVService { + validate(columns: ColumnEntity[], mappings: MappingEntity[], data: any) { + const schema = this.buildAJVSchema(columns, mappings); + const validator = ajv.compile(schema); + + const valid = validator(data); + const returnData = { + invalid: [], + valid: [], + }; + if (!valid) { + const errors: Record = this.buildErrorRecords(validator.errors, data); + + returnData.invalid = Object.values(errors); + // eslint-disable-next-line no-magic-numbers + Object.keys(errors).forEach((index) => (data as any).splice(index as unknown as number, 1)); + } + returnData.valid = data as any; + // resetting uniqueItems + uniqueItems = {}; + + return returnData; + } + private buildAJVSchema(columns: ColumnEntity[], mappings: MappingEntity[]) { + const formattedColumns: Record = columns.reduce((acc, column) => { + acc[column._id] = { ...column }; + + return acc; + }, {}); + const properties: Record = mappings.reduce((acc, mapping) => { + acc[mapping.columnHeading] = this.getProperty(formattedColumns[mapping._columnId]); + + return acc; + }, {}); + const requiredProperties: string[] = mappings.reduce((acc, mapping) => { + if (formattedColumns[mapping._columnId].isRequired) acc.push(mapping.columnHeading); + + return acc; + }, []); + // setting uniqueItems to empty set to avoid error + mappings.forEach((mapping) => { + if (formattedColumns[mapping._columnId].isUnique) { + uniqueItems[mapping.columnHeading] = new Set(); + } + }); + const objectSchema = { + type: 'object', + properties, + required: requiredProperties, + additionalProperties: false, + }; + + return { + type: 'array', + items: objectSchema, + }; + } + private getProperty(column: ColumnEntity): Record { + let property: Record = {}; + + switch (column.type) { + case ColumnTypesEnum.STRING: + property = { + type: 'string', + }; + break; + case ColumnTypesEnum.NUMBER: + property = { + type: 'number', + }; + break; + case ColumnTypesEnum.SELECT: + property = { + type: 'string', + enum: column.selectValues || [], + }; + break; + case ColumnTypesEnum.REGEX: + const [full, pattern, flags] = column.regex.match(/\/(.*)\/(.*)|(.*)/); + + property = { type: 'string', regexp: { pattern: pattern || full, flags: flags || '' } }; + break; + case ColumnTypesEnum.EMAIL: + property = { type: 'string', format: 'email' }; + break; + case ColumnTypesEnum.DATE: + property = { type: 'string', format: 'custom-date-time' }; + break; + case ColumnTypesEnum.ANY: + property = { type: ['string', 'number', 'object'] }; + break; + } + + return { + ...property, + ...(column.isUnique && { uniqueCheck: true }), + ...(column.isRequired && { emptyCheck: true }), + }; + } + private buildErrorRecords(errors: ErrorObject[], data?: any[]) { + let index: string, field: string, message: string; + + return errors.reduce((acc, error) => { + [, index, field] = error.instancePath.split('/'); + // eslint-disable-next-line no-magic-numbers + message = this.getMessage(error, field || error.schema[0]); + + if (acc[index]) { + acc[index].message += `, ${message}`; + } else + acc[index] = { + index, + message, + ...data[index], + }; + + return acc; + }, {}); + } + private getMessage(error: ErrorObject, field: string): string { + let message = ''; + switch (true) { + // empty string case + case error.keyword === 'emptyCheck': + message = ` must not be empty`; + break; + // uniqueCheck + case error.keyword === 'uniqueCheck': + message = ` must be unique`; + break; + // custom date format + case error.keyword === 'format' && error.params.format === 'custom-date-time': + message = ` must be a valid date`; + break; + // common cases + case error.keyword === 'type': + message = ' ' + error.message; + break; + case error.keyword === 'enum': + message = ` must be from [${error.params.allowedValues}]`; + break; + case error.keyword === 'regexp': + message = ` must match the pattern ${new RegExp( + error.parentSchema?.regexp?.pattern, + error.parentSchema?.regexp?.flags || '' + ).toString()}`; + break; + case error.keyword === 'pattern': + message = ` must match the pattern ${error.params.pattern}`; + break; + case error.keyword === 'format': + message = ` must be a valid ${error.params.format}`; + break; + case error.keyword === 'required': + message = ` is required`; + break; + case error.keyword === 'uniqueItemProperties': + message = ` must be unique`; + break; + default: + message = ` contains invalid data`; + break; + } + + return '`' + field + '`' + message; + } +} diff --git a/apps/api/src/app/review/usecases/confirm-review/confirm-review.command.ts b/apps/api/src/app/review/usecases/confirm-review/confirm-review.command.ts new file mode 100644 index 000000000..80e0b97c5 --- /dev/null +++ b/apps/api/src/app/review/usecases/confirm-review/confirm-review.command.ts @@ -0,0 +1,12 @@ +import { IsBoolean, IsDefined, IsMongoId } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class ConfirmReviewCommand extends BaseCommand { + @IsDefined() + @IsMongoId() + _uploadId: string; + + @IsDefined() + @IsBoolean() + processInvalidRecords: boolean; +} diff --git a/apps/api/src/app/review/usecases/confirm-review/confirm-review.usecase.ts b/apps/api/src/app/review/usecases/confirm-review/confirm-review.usecase.ts new file mode 100644 index 000000000..681db332f --- /dev/null +++ b/apps/api/src/app/review/usecases/confirm-review/confirm-review.usecase.ts @@ -0,0 +1,16 @@ +import { Injectable } from '@nestjs/common'; +import { UploadEntity, UploadRepository } from '@impler/dal'; +import { ConfirmReviewCommand } from './confirm-review.command'; +import { UploadStatusEnum } from '@impler/shared'; + +@Injectable() +export class ConfirmReview { + constructor(private uploadRepository: UploadRepository) {} + + execute(command: ConfirmReviewCommand): Promise { + return this.uploadRepository.findOneAndUpdate( + { _id: command._uploadId }, + { status: UploadStatusEnum.CONFIRMED, processInvalidRecords: command.processInvalidRecords } + ); + } +} diff --git a/apps/api/src/app/review/usecases/do-review/do-review.usecase.ts b/apps/api/src/app/review/usecases/do-review/do-review.usecase.ts new file mode 100644 index 000000000..8faad0a82 --- /dev/null +++ b/apps/api/src/app/review/usecases/do-review/do-review.usecase.ts @@ -0,0 +1,52 @@ +import { FileEncodingsEnum, UploadStatusEnum } from '@impler/shared'; +import { Injectable, BadRequestException } from '@nestjs/common'; +import { ColumnRepository, UploadRepository, MappingRepository, FileEntity } from '@impler/dal'; +import { StorageService } from '../../../shared/storage/storage.service'; +import { AJVService } from '../../service/AJV.service'; +import { APIMessages } from '../../../shared/constants'; +import { FileNotExistError } from '../../../shared/errors/file-not-exist.error'; + +@Injectable() +export class DoReview { + constructor( + private uploadRepository: UploadRepository, + private storageService: StorageService, + private columnRepository: ColumnRepository, + private mappingRepository: MappingRepository, + private ajvService: AJVService + ) {} + + async execute(uploadId: string) { + const uploadInfo = await this.uploadRepository.getUploadInformation(uploadId); + if (!uploadInfo) { + throw new BadRequestException(APIMessages.UPLOAD_NOT_FOUND); + } + if (uploadInfo.status !== UploadStatusEnum.MAPPED) { + throw new BadRequestException(APIMessages.FILE_MAPPING_REMAINING); + } + const allDataFileInfo = uploadInfo._allDataFileId as unknown as FileEntity; + const dataContent = await this.getFileContent(allDataFileInfo.path); + const mappings = await this.mappingRepository.find({ _uploadId: uploadId }, '_columnId columnHeading'); + const columns = await this.columnRepository.find( + { _templateId: uploadInfo._templateId }, + 'isRequired isUnique selectValues type regex' + ); + const reviewData = this.ajvService.validate(columns, mappings, dataContent); + this.uploadRepository.update({ _id: uploadId }, { status: UploadStatusEnum.REVIEWING }); + + return reviewData; + } + + async getFileContent(path): Promise { + try { + const dataContent = await this.storageService.getFileContent(path, FileEncodingsEnum.JSON); + + return JSON.parse(dataContent); + } catch (error) { + if (error instanceof FileNotExistError) { + throw new BadRequestException(APIMessages.FILE_NOT_FOUND_IN_STORAGE); + } + throw error; + } + } +} diff --git a/apps/api/src/app/review/usecases/get-file-invalid-data/get-file-invalid-data.usecase.ts b/apps/api/src/app/review/usecases/get-file-invalid-data/get-file-invalid-data.usecase.ts new file mode 100644 index 000000000..bba552b15 --- /dev/null +++ b/apps/api/src/app/review/usecases/get-file-invalid-data/get-file-invalid-data.usecase.ts @@ -0,0 +1,14 @@ +import { Injectable } from '@nestjs/common'; +import { FileEncodingsEnum } from '@impler/shared'; +import { StorageService } from '../../../shared/storage/storage.service'; + +@Injectable() +export class GetFileInvalidData { + constructor(private storageService: StorageService) {} + + async execute(path: string) { + const stringContent = await this.storageService.getFileContent(path, FileEncodingsEnum.JSON); + + return JSON.parse(stringContent); + } +} diff --git a/apps/api/src/app/review/usecases/get-upload-invalid-data/get-upload-invalid-data.usecase.ts b/apps/api/src/app/review/usecases/get-upload-invalid-data/get-upload-invalid-data.usecase.ts new file mode 100644 index 000000000..e82990748 --- /dev/null +++ b/apps/api/src/app/review/usecases/get-upload-invalid-data/get-upload-invalid-data.usecase.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { UploadRepository } from '@impler/dal'; + +@Injectable() +export class GetUploadInvalidData { + constructor(private uploadRepository: UploadRepository) {} + + async execute(_uploadId: string) { + return this.uploadRepository.getUploadInvalidDataInformation(_uploadId); + } +} diff --git a/apps/api/src/app/review/usecases/index.ts b/apps/api/src/app/review/usecases/index.ts new file mode 100644 index 000000000..1d3ae1116 --- /dev/null +++ b/apps/api/src/app/review/usecases/index.ts @@ -0,0 +1,18 @@ +import { DoReview } from './do-review/do-review.usecase'; +import { SaveReviewData } from './save-review-data/save-review-data.usecase'; +import { GetUploadInvalidData } from './get-upload-invalid-data/get-upload-invalid-data.usecase'; +import { GetFileInvalidData } from './get-file-invalid-data/get-file-invalid-data.usecase'; +import { GetUpload } from '../../shared/usecases/get-upload/get-upload.usecase'; +import { ConfirmReview } from './confirm-review/confirm-review.usecase'; +import { StartProcess } from './start-process/start-process.usecase'; + +export const USE_CASES = [ + DoReview, + GetUpload, + StartProcess, + ConfirmReview, + SaveReviewData, + GetFileInvalidData, + GetUploadInvalidData, + // +]; diff --git a/apps/api/src/app/review/usecases/save-review-data/save-review-data.usecase.ts b/apps/api/src/app/review/usecases/save-review-data/save-review-data.usecase.ts new file mode 100644 index 000000000..288667f10 --- /dev/null +++ b/apps/api/src/app/review/usecases/save-review-data/save-review-data.usecase.ts @@ -0,0 +1,79 @@ +import * as XLSX from 'xlsx'; +import { Injectable } from '@nestjs/common'; +import { FileRepository, UploadRepository } from '@impler/dal'; +import { FileMimeTypesEnum } from '@impler/shared'; +import { FileNameService } from '@shared/file/name.service'; +import { StorageService } from '@shared/storage/storage.service'; +import { Defaults } from '@shared/constants'; + +@Injectable() +export class SaveReviewData { + constructor( + private fileNameService: FileNameService, + private storageService: StorageService, + private fileRepository: FileRepository, + private uploadRepository: UploadRepository + ) {} + + async execute(_uploadId: string, invalidData: any[], validData: any[]) { + const _invalidDataFileId = await this.storeInvalidFile(_uploadId, invalidData); + const _validDataFileId = await this.storeValidFile(_uploadId, validData); + const _invalidCSVDataFileId = await this.storeInvalidCSVFile(_uploadId, invalidData); + const invalidCSVDataFileUrl = this.fileNameService.getInvalidCSVDataFileUrl(_uploadId); + await this.uploadRepository.update( + { _id: _uploadId }, + { _invalidDataFileId, _validDataFileId, _invalidCSVDataFileId, invalidCSVDataFileUrl } + ); + } + + private async storeValidFile(_uploadId: string, validData: any[]): Promise { + if (validData.length < Defaults.DATA_LENGTH) return null; + + const strinValidData = JSON.stringify(validData); + const validFilePath = this.fileNameService.getValidDataFilePath(_uploadId); + await this.storeFile(validFilePath, strinValidData); + const validFileName = this.fileNameService.getValidDataFileName(); + const entry = await this.makeFileEntry(validFileName, validFilePath); + + return entry._id; + } + + private async storeInvalidFile(_uploadId: string, invalidData: any[]): Promise { + if (invalidData.length < Defaults.DATA_LENGTH) return null; + + const stringInvalidData = JSON.stringify(invalidData); + const invalidFilePath = this.fileNameService.getInvalidDataFilePath(_uploadId); + await this.storeFile(invalidFilePath, stringInvalidData); + const invalidFileName = this.fileNameService.getInvalidDataFileName(); + const entry = await this.makeFileEntry(invalidFileName, invalidFilePath); + + return entry._id; + } + + private async storeInvalidCSVFile(_uploadId: string, invalidData: any[]): Promise { + if (invalidData.length < Defaults.DATA_LENGTH) return null; + + const ws = XLSX.utils.json_to_sheet(invalidData); + const invalidCSVDataContent = XLSX.utils.sheet_to_csv(ws, { FS: ',' }); + + const invalidCSVFilePath = this.fileNameService.getInvalidCSVDataFilePath(_uploadId); + await this.storeFile(invalidCSVFilePath, invalidCSVDataContent, true); + const invalidCSVFileName = this.fileNameService.getInvalidCSVDataFileName(); + const entry = await this.makeFileEntry(invalidCSVFileName, invalidCSVFilePath); + + return entry._id; + } + + private async storeFile(invalidFilePath: string, data: string, isPublic = false) { + await this.storageService.uploadFile(invalidFilePath, data, FileMimeTypesEnum.JSON, isPublic); + } + + private async makeFileEntry(fileName: string, filePath: string) { + return await this.fileRepository.create({ + mimeType: FileMimeTypesEnum.JSON, + name: fileName, + originalName: fileName, + path: filePath, + }); + } +} diff --git a/apps/api/src/app/review/usecases/start-process/start-process.command.ts b/apps/api/src/app/review/usecases/start-process/start-process.command.ts new file mode 100644 index 000000000..c44f7c9d8 --- /dev/null +++ b/apps/api/src/app/review/usecases/start-process/start-process.command.ts @@ -0,0 +1,12 @@ +import { IsBoolean, IsDefined, IsMongoId } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class StartProcessCommand extends BaseCommand { + @IsDefined() + @IsMongoId() + _uploadId: string; + + @IsDefined() + @IsBoolean() + processInvalidRecords: boolean; +} diff --git a/apps/api/src/app/review/usecases/start-process/start-process.usecase.ts b/apps/api/src/app/review/usecases/start-process/start-process.usecase.ts new file mode 100644 index 000000000..d6e1cdf70 --- /dev/null +++ b/apps/api/src/app/review/usecases/start-process/start-process.usecase.ts @@ -0,0 +1,21 @@ +import { Injectable } from '@nestjs/common'; +import { UploadEntity, UploadRepository } from '@impler/dal'; +import { StartProcessCommand } from './start-process.command'; +import { QueuesEnum, UploadStatusEnum } from '@impler/shared'; +import { QueueService } from '../../../shared/storage/queue.service'; + +@Injectable() +export class StartProcess { + constructor(private uploadRepository: UploadRepository, private queueService: QueueService) {} + + async execute(command: StartProcessCommand): Promise { + const upload = await this.uploadRepository.findOneAndUpdate( + { _id: command._uploadId }, + { status: UploadStatusEnum.PROCESSING, processInvalidRecords: command.processInvalidRecords } + ); + + this.queueService.publishToQueue(QueuesEnum.PROCESS_FILE, { uploadId: command._uploadId }); + + return upload; + } +} diff --git a/apps/api/src/app/shared/commands/base.command.ts b/apps/api/src/app/shared/commands/base.command.ts new file mode 100644 index 000000000..0044a4b75 --- /dev/null +++ b/apps/api/src/app/shared/commands/base.command.ts @@ -0,0 +1,21 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { plainToClass } from 'class-transformer'; +import { validateSync } from 'class-validator'; +import { BadRequestException, flatten } from '@nestjs/common'; + +export abstract class BaseCommand { + static create(this: new (...args: any[]) => T, data: T): T { + const convertedObject = plainToClass(this, { + ...data, + }); + + const errors = validateSync(convertedObject as unknown as object); + if (errors?.length) { + const mappedErrors = flatten(errors.map((item) => Object.values(item.constraints))); + + throw new BadRequestException(mappedErrors); + } + + return convertedObject; + } +} diff --git a/apps/api/src/app/shared/constants.ts b/apps/api/src/app/shared/constants.ts new file mode 100644 index 000000000..cfb3ea917 --- /dev/null +++ b/apps/api/src/app/shared/constants.ts @@ -0,0 +1,23 @@ +export const APIMessages = { + FILE_TYPE_NOT_VALID: 'File type is not supported.', + FILE_IS_EMPTY: 'File is empty', + EMPTY_HEADING_PREFIX: 'Empty Heading', + INVALID_TEMPLATE_ID_CODE_SUFFIX: 'is not valid TemplateId or CODE.', + FILE_MAPPING_REMAINING: 'File mapping is not yet done, please finalize mapping before.', + UPLOAD_NOT_FOUND: 'Upload information not found with specified uploadId.', + FILE_NOT_FOUND_IN_STORAGE: + "File not found, make sure you're using the same storage provider, that you were using before.", + DO_MAPPING_FIRST: 'You may landed to wrong place, Please finalize mapping and proceed ahead.', + DO_REVIEW_FIRST: 'You may landed to wrong place, Please review data and proceed ahead.', + DO_CONFIRM_FIRST: 'You may landed to wrong place, Please confirm data and proceed ahead.', + ALREADY_CONFIRMED: '`You may landed to wrong place, This upload file is confirmed already.', + IN_PROGRESS: 'You may landed to wrong place, This uploaded file processing is started already.', + COMPLETED: 'You may landed to wrong place, This uploaded file is already completed, no more steps left to perform.', + PROJECT_WITH_TEMPLATE_MISSING: 'Template not found with provided ProjectId and Template', +}; + +export const Defaults = { + DATA_LENGTH: 1, + PAGE: 1, + PAGE_LIMIT: 100, +}; diff --git a/apps/api/src/app/shared/dtos/pagination-response.dto.ts b/apps/api/src/app/shared/dtos/pagination-response.dto.ts new file mode 100644 index 000000000..ebe43e437 --- /dev/null +++ b/apps/api/src/app/shared/dtos/pagination-response.dto.ts @@ -0,0 +1,34 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsArray, IsNumber } from 'class-validator'; + +export class PaginationResponseDto { + @ApiProperty({ + description: 'Page index of the data', + }) + @IsNumber() + page: number; + + @ApiProperty({ + description: 'Array of data', + }) + @IsArray() + data: any[]; + + @ApiProperty({ + description: 'Size of the data', + }) + @IsNumber() + limit: number; + + @ApiProperty({ + description: 'Total number of pages', + }) + @IsNumber() + totalPages: number; + + @ApiProperty({ + description: 'Total number of records', + }) + @IsNumber() + totalRecords: number; +} diff --git a/apps/api/src/app/shared/errors/file-not-exist.error.ts b/apps/api/src/app/shared/errors/file-not-exist.error.ts new file mode 100644 index 000000000..05e1c431a --- /dev/null +++ b/apps/api/src/app/shared/errors/file-not-exist.error.ts @@ -0,0 +1,6 @@ +export class FileNotExistError extends Error { + constructor() { + super('File not found for the key provided'); + this.name = 'NonExistingFileError'; + } +} diff --git a/apps/api/src/app/shared/exceptions/document-not-found.exception.ts b/apps/api/src/app/shared/exceptions/document-not-found.exception.ts new file mode 100644 index 000000000..46ba57e57 --- /dev/null +++ b/apps/api/src/app/shared/exceptions/document-not-found.exception.ts @@ -0,0 +1,7 @@ +import { NotFoundException } from '@nestjs/common'; + +export class DocumentNotFoundException extends NotFoundException { + constructor(name: string, id: string, message?: string) { + super(message || `${name} with id ${id} does not exist`); + } +} diff --git a/apps/api/src/app/shared/exceptions/empty-file.exception.ts b/apps/api/src/app/shared/exceptions/empty-file.exception.ts new file mode 100644 index 000000000..82153630f --- /dev/null +++ b/apps/api/src/app/shared/exceptions/empty-file.exception.ts @@ -0,0 +1,8 @@ +import { BadRequestException } from '@nestjs/common'; +import { APIMessages } from '../constants'; + +export class EmptyFileException extends BadRequestException { + constructor() { + super(APIMessages.FILE_IS_EMPTY); + } +} diff --git a/apps/api/src/app/shared/exceptions/file-not-valid.exception.ts b/apps/api/src/app/shared/exceptions/file-not-valid.exception.ts new file mode 100644 index 000000000..3c826a3a8 --- /dev/null +++ b/apps/api/src/app/shared/exceptions/file-not-valid.exception.ts @@ -0,0 +1,8 @@ +import { UnprocessableEntityException } from '@nestjs/common'; +import { APIMessages } from '../constants'; + +export class FileNotValidError extends UnprocessableEntityException { + constructor() { + super(APIMessages.FILE_TYPE_NOT_VALID); + } +} diff --git a/apps/api/src/app/shared/file/file.service.ts b/apps/api/src/app/shared/file/file.service.ts new file mode 100644 index 000000000..934152876 --- /dev/null +++ b/apps/api/src/app/shared/file/file.service.ts @@ -0,0 +1,75 @@ +import * as XLSX from 'xlsx'; +import { FileEncodingsEnum, IFileInformation } from '@impler/shared'; +import { ParserOptionsArgs, parseString } from 'fast-csv'; +import { StorageService } from '../storage/storage.service'; +import { EmptyFileException } from '../exceptions/empty-file.exception'; +import { APIMessages } from '../constants'; + +export abstract class FileService { + abstract getFileInformation(storageService: StorageService, storageKey: string): Promise; +} + +export class CSVFileService extends FileService { + async getFileInformation(storageService: StorageService, storageKey: string): Promise { + const fileContent = await storageService.getFileContent(storageKey, FileEncodingsEnum.CSV); + + return await this.getCSVInformation(fileContent, { headers: true }); + } + private async getCSVInformation(fileContent: string, options?: ParserOptionsArgs): Promise { + return new Promise((resolve, reject) => { + const information: IFileInformation = { + data: [], + headings: [], + totalRecords: 0, + }; + + parseString(fileContent, options) + .on('error', reject) + .on('headers', (headers) => information.headings.push(...headers)) + .on('data', (row) => information.data.push(row)) + .on('end', () => { + information.totalRecords = information.data.length; + resolve(information); + }); + }); + } +} +export class ExcelFileService extends FileService { + async getFileInformation(storageService: StorageService, storageKey: string): Promise { + const fileContent = await storageService.getFileContent(storageKey, FileEncodingsEnum.EXCEL); + + return this.getExcelInformation(fileContent); + } + async getExcelInformation(fileContent: string): Promise { + const workbookHeaders = XLSX.read(fileContent); + // Throw empty error if file do not have any sheets + if (workbookHeaders.SheetNames.length < 1) throw new EmptyFileException(); + + // get file headings + const headings = XLSX.utils.sheet_to_json(workbookHeaders.Sheets[workbookHeaders.SheetNames[0]], { + header: 1, + })[0] as string[]; + // throw error if sheet is empty + if (!headings || headings.length < 1) throw new EmptyFileException(); + + // Refine headings by replacing empty heading + let emptyHeadingCount = 0; + const updatedHeading = []; + for (const headingItem of headings) { + if (!headingItem) { + emptyHeadingCount += 1; + updatedHeading.push(`${APIMessages.EMPTY_HEADING_PREFIX} ${emptyHeadingCount}`); + } else updatedHeading.push(headingItem); + } + + const data: Record[] = XLSX.utils.sheet_to_json( + workbookHeaders.Sheets[workbookHeaders.SheetNames[0]] + ); + + return { + data, + headings: updatedHeading, + totalRecords: data.length, + }; + } +} diff --git a/apps/api/src/app/shared/file/name.service.ts b/apps/api/src/app/shared/file/name.service.ts new file mode 100644 index 000000000..e8a6aca9d --- /dev/null +++ b/apps/api/src/app/shared/file/name.service.ts @@ -0,0 +1,48 @@ +export class FileNameService { + getSampleFileName(templateId: string): string { + return `${templateId}/sample.csv`; + } + getSampleFileUrl(templateId: string): string { + const fileName = this.getSampleFileName(templateId); + + return [process.env.S3_LOCAL_STACK, process.env.S3_BUCKET_NAME, fileName].join('/'); + } + getFileExtension(fileName: string): string { + return fileName.split('.').pop(); + } + getUploadedFilePath(uploadId: string, fileName: string): string { + return `${uploadId}/${this.getUploadedFileName(fileName)}`; + } + getUploadedFileName(fileName: string): string { + return `uploaded.${this.getFileExtension(fileName)}`; + } + getAllJsonDataFileName(): string { + return `all-data.json`; + } + getAllJsonDataFilePath(uploadId: string): string { + return `${uploadId}/${this.getAllJsonDataFileName()}`; + } + getInvalidDataFileName(): string { + return `invalid-data.json`; + } + getInvalidDataFilePath(uploadId: string): string { + return `${uploadId}/${this.getInvalidDataFileName()}`; + } + getInvalidCSVDataFileName(): string { + return 'invalid-data.csv'; + } + getInvalidCSVDataFilePath(uploadId: string): string { + return `${uploadId}/${this.getInvalidCSVDataFileName()}`; + } + getInvalidCSVDataFileUrl(uploadId: string): string { + const path = this.getInvalidCSVDataFilePath(uploadId); + + return [process.env.S3_LOCAL_STACK, process.env.S3_BUCKET_NAME, path].join('/'); + } + getValidDataFileName(): string { + return `valid-data.json`; + } + getValidDataFilePath(uploadId: string): string { + return `${uploadId}/${this.getValidDataFileName()}`; + } +} diff --git a/apps/api/src/app/shared/framework/auth.gaurd.ts b/apps/api/src/app/shared/framework/auth.gaurd.ts new file mode 100644 index 000000000..2e5238e2c --- /dev/null +++ b/apps/api/src/app/shared/framework/auth.gaurd.ts @@ -0,0 +1,19 @@ +import { ExecutionContext, Injectable, CanActivate, UnauthorizedException } from '@nestjs/common'; +import { ACCESS_KEY_NAME } from '@impler/shared'; + +@Injectable() +export class APIKeyGuard implements CanActivate { + async canActivate(context: ExecutionContext) { + const request = context.switchToHttp().getRequest(); + + const authorizationHeader = request.headers[ACCESS_KEY_NAME]; + const API_KEY = + process.env[String(ACCESS_KEY_NAME).toLowerCase()] || process.env[String(ACCESS_KEY_NAME).toUpperCase()]; + + if (API_KEY && API_KEY !== authorizationHeader) { + throw new UnauthorizedException(); + } + + return true; + } +} diff --git a/apps/api/src/app/shared/framework/is-unique.validator.ts b/apps/api/src/app/shared/framework/is-unique.validator.ts new file mode 100644 index 000000000..448e0f657 --- /dev/null +++ b/apps/api/src/app/shared/framework/is-unique.validator.ts @@ -0,0 +1,21 @@ +import { ValidationArguments, ValidatorConstraint, ValidatorConstraintInterface } from 'class-validator'; +import { CommonRepository, ProjectEntity } from '@impler/dal'; + +@ValidatorConstraint({ name: 'IsUnique', async: true }) +export class UniqueValidator implements ValidatorConstraintInterface { + private commonRepository: CommonRepository; + constructor() { + this.commonRepository = new CommonRepository(); + } + + async validate(value: any, args: ValidationArguments) { + const [modelName, field] = args.constraints; + const count = await this.commonRepository.count(modelName, { [field]: value }); + + return !count; + } + + defaultMessage(args: ValidationArguments) { + return `${args.value} is already taken`; + } +} diff --git a/apps/api/src/app/shared/framework/is-valid-regex.validator.ts b/apps/api/src/app/shared/framework/is-valid-regex.validator.ts new file mode 100644 index 000000000..ec50e59fb --- /dev/null +++ b/apps/api/src/app/shared/framework/is-valid-regex.validator.ts @@ -0,0 +1,18 @@ +import { ValidatorConstraint, ValidatorConstraintInterface, ValidationArguments } from 'class-validator'; + +@ValidatorConstraint({ name: 'IsValidRegex' }) +export class IsValidRegex implements ValidatorConstraintInterface { + validate(text: string) { + try { + new RegExp(text); + + return true; + } catch (error) { + return false; + } + } + + defaultMessage(args: ValidationArguments) { + return `Text (${args.value}) is not valid Regular expression!`; + } +} diff --git a/apps/api/src/app/shared/helpers/common.helper.ts b/apps/api/src/app/shared/helpers/common.helper.ts new file mode 100644 index 000000000..966998007 --- /dev/null +++ b/apps/api/src/app/shared/helpers/common.helper.ts @@ -0,0 +1,39 @@ +import { BadRequestException } from '@nestjs/common'; +import { APIMessages } from '../constants'; +import { PaginationResult } from '@impler/shared'; + +export function validateNotFound(data: any, entityName: 'upload'): boolean { + if (data) return true; + else { + switch (entityName) { + case 'upload': + throw new BadRequestException(APIMessages.UPLOAD_NOT_FOUND); + default: + throw new BadRequestException(); + } + } +} + +export function paginateRecords(data: any[], page: number, limit: number): PaginationResult { + if (!page || page < 1) page = 1; + if (!limit || limit < 1) limit = 1; + if (!Array.isArray(data)) + return { + data: [], + limit, + page, + totalPages: 0, + totalRecords: 0, + }; + + const sliceFrom = Math.max((page - 1) * limit, 0); + const sliceTo = Math.min(page * limit, data.length); + + return { + data: data.slice(sliceFrom, sliceTo), + limit, + page, + totalPages: Math.ceil(data.length / limit), + totalRecords: data.length, + }; +} diff --git a/apps/api/src/app/shared/helpers/file.helper.ts b/apps/api/src/app/shared/helpers/file.helper.ts new file mode 100644 index 000000000..d2ef48710 --- /dev/null +++ b/apps/api/src/app/shared/helpers/file.helper.ts @@ -0,0 +1,13 @@ +import { FileMimeTypesEnum } from '@impler/shared'; +import { APIMessages } from '../constants'; +import { FileService, CSVFileService, ExcelFileService } from '../file/file.service'; + +export const getFileService = (mimeType: string): FileService => { + if (mimeType === FileMimeTypesEnum.CSV) { + return new CSVFileService(); + } else if (mimeType === FileMimeTypesEnum.EXCEL || mimeType === FileMimeTypesEnum.EXCELX) { + return new ExcelFileService(); + } + + throw new Error(APIMessages.FILE_TYPE_NOT_VALID); +}; diff --git a/apps/api/src/app/shared/helpers/upload.helpers.ts b/apps/api/src/app/shared/helpers/upload.helpers.ts new file mode 100644 index 000000000..7f4e2a5b4 --- /dev/null +++ b/apps/api/src/app/shared/helpers/upload.helpers.ts @@ -0,0 +1,22 @@ +import { UploadStatusEnum } from '@impler/shared'; +import { BadRequestException } from '@nestjs/common'; +import { APIMessages } from '../constants'; + +export function validateUploadStatus(currentStatus: UploadStatusEnum, expectedStatus: UploadStatusEnum[]): boolean { + if (expectedStatus.includes(currentStatus)) return true; + else { + if (currentStatus === UploadStatusEnum.UPLOADED) { + throw new BadRequestException(APIMessages.DO_MAPPING_FIRST); + } else if (currentStatus === UploadStatusEnum.MAPPED) { + throw new BadRequestException(APIMessages.DO_REVIEW_FIRST); + } else if (currentStatus === UploadStatusEnum.REVIEWED) { + throw new BadRequestException(APIMessages.DO_CONFIRM_FIRST); + } else if (currentStatus === UploadStatusEnum.CONFIRMED) { + throw new BadRequestException(APIMessages.ALREADY_CONFIRMED); + } else if (currentStatus === UploadStatusEnum.PROCESSING) { + throw new BadRequestException(APIMessages.IN_PROGRESS); + } else if (currentStatus === UploadStatusEnum.COMPLETED) { + throw new BadRequestException(APIMessages.COMPLETED); + } + } +} diff --git a/apps/api/src/app/shared/shared.module.ts b/apps/api/src/app/shared/shared.module.ts new file mode 100644 index 000000000..2374284e5 --- /dev/null +++ b/apps/api/src/app/shared/shared.module.ts @@ -0,0 +1,55 @@ +import { Module } from '@nestjs/common'; +import { + ColumnRepository, + CommonRepository, + DalService, + FileRepository, + MappingRepository, + ProjectRepository, + TemplateRepository, + UploadRepository, +} from '@impler/dal'; +import { S3StorageService, StorageService } from './storage/storage.service'; +import { CSVFileService } from './file/file.service'; +import { FileNameService } from './file/name.service'; + +const DAL_MODELS = [ + ProjectRepository, + TemplateRepository, + ColumnRepository, + FileRepository, + UploadRepository, + MappingRepository, + CommonRepository, +]; +const FILE_SERVICES = [CSVFileService, FileNameService]; + +const dalService = new DalService(); + +function getStorageServiceClass() { + return S3StorageService; +} + +const PROVIDERS = [ + { + provide: DalService, + useFactory: async () => { + await dalService.connect(process.env.MONGO_URL); + + return dalService; + }, + }, + ...DAL_MODELS, + { + provide: StorageService, + useClass: getStorageServiceClass(), + }, + ...FILE_SERVICES, +]; + +@Module({ + imports: [], + providers: [...PROVIDERS], + exports: [...PROVIDERS], +}) +export class SharedModule {} diff --git a/apps/api/src/app/shared/storage/queue.service.ts b/apps/api/src/app/shared/storage/queue.service.ts new file mode 100644 index 000000000..210b5ded4 --- /dev/null +++ b/apps/api/src/app/shared/storage/queue.service.ts @@ -0,0 +1,29 @@ +import { Injectable } from '@nestjs/common'; +import amqp from 'amqp-connection-manager'; +import { ProcessFileData, PublishToQueueData, QueuesEnum } from '@impler/shared'; + +@Injectable() +export class QueueService { + private connection: any; + private chanelWrapper: any; + + constructor() { + this.connection = amqp.connect([process.env.RABBITMQ_CONN_URL]); + this.connection.on('connect', () => console.log('QueueService RabbitMQ::Connected!')); + this.connection.on('disconnect', (err: Error) => console.log('RabbitMQ::Disconnected.', err)); + + this.chanelWrapper = this.connection.createChannel({ + json: true, + persistent: true, + }); + } + + publishToQueue(queueName: QueuesEnum.PROCESS_FILE, data: ProcessFileData): void; + publishToQueue(queueName: QueuesEnum, data: PublishToQueueData) { + if (this.connection.isConnected()) { + this.chanelWrapper.sendToQueue(queueName, data, { durable: false }); + } else { + throw new Error('RabbitMQ connection is not established'); + } + } +} diff --git a/apps/api/src/app/shared/storage/storage.service.ts b/apps/api/src/app/shared/storage/storage.service.ts new file mode 100644 index 000000000..50084b416 --- /dev/null +++ b/apps/api/src/app/shared/storage/storage.service.ts @@ -0,0 +1,79 @@ +import { Readable } from 'stream'; +import { + S3Client, + PutObjectCommand, + PutObjectCommandOutput, + GetObjectCommand, + DeleteObjectCommand, +} from '@aws-sdk/client-s3'; +import { FileNotExistError } from '../errors/file-not-exist.error'; + +export interface IFilePath { + path: string; + name: string; +} + +export abstract class StorageService { + abstract uploadFile( + key: string, + file: Buffer | string, + contentType: string, + isPublic?: boolean + ): Promise; + abstract getFileContent(key: string, encoding?: BufferEncoding): Promise; + abstract deleteFile(key: string): Promise; +} + +async function streamToString(stream: Readable, encoding: BufferEncoding): Promise { + return await new Promise((resolve, reject) => { + const chunks: Uint8Array[] = []; + stream.on('data', (chunk) => chunks.push(chunk)); + stream.on('error', reject); + stream.on('end', () => resolve(Buffer.concat(chunks).toString(encoding))); + }); +} + +export class S3StorageService implements StorageService { + private s3 = new S3Client({ + region: process.env.S3_REGION, + endpoint: process.env.S3_LOCAL_STACK || undefined, + forcePathStyle: true, + }); + + async uploadFile(key: string, file: Buffer, contentType: string, isPublic = false): Promise { + const command = new PutObjectCommand({ + Bucket: process.env.S3_BUCKET_NAME, + Key: key, + Body: file, + ACL: isPublic ? 'public-read' : 'private', + ContentType: contentType, + }); + + return await this.s3.send(command); + } + + async getFileContent(key: string, encoding = 'utf8' as BufferEncoding): Promise { + try { + const command = new GetObjectCommand({ + Bucket: process.env.S3_BUCKET_NAME, + Key: key, + }); + const data = await this.s3.send(command); + + return await streamToString(data.Body as Readable, encoding); + } catch (error) { + if (error.code === 404 || error.message === 'The specified key does not exist.') { + throw new FileNotExistError(); + } + throw error; + } + } + + async deleteFile(key: string): Promise { + const command = new DeleteObjectCommand({ + Bucket: process.env.S3_BUCKET_NAME, + Key: key, + }); + await this.s3.send(command); + } +} diff --git a/apps/api/src/app/shared/usecases/get-upload/get-upload.command.ts b/apps/api/src/app/shared/usecases/get-upload/get-upload.command.ts new file mode 100644 index 000000000..9abd8f9b8 --- /dev/null +++ b/apps/api/src/app/shared/usecases/get-upload/get-upload.command.ts @@ -0,0 +1,12 @@ +import { IsDefined, IsMongoId, IsOptional, IsString } from 'class-validator'; +import { BaseCommand } from '../../commands/base.command'; + +export class GetUploadCommand extends BaseCommand { + @IsDefined() + @IsMongoId() + uploadId: string; + + @IsOptional() + @IsString() + select?: string; +} diff --git a/apps/api/src/app/shared/usecases/get-upload/get-upload.usecase.ts b/apps/api/src/app/shared/usecases/get-upload/get-upload.usecase.ts new file mode 100644 index 000000000..f8609aaef --- /dev/null +++ b/apps/api/src/app/shared/usecases/get-upload/get-upload.usecase.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@nestjs/common'; +import { UploadRepository } from '@impler/dal'; +import { GetUploadCommand } from './get-upload.command'; + +@Injectable() +export class GetUpload { + constructor(private uploadRepository: UploadRepository) {} + + async execute(command: GetUploadCommand) { + return this.uploadRepository.findOne({ _id: command.uploadId }, command.select); + } +} diff --git a/apps/api/src/app/shared/validations/valid-import-file.validation.ts b/apps/api/src/app/shared/validations/valid-import-file.validation.ts new file mode 100644 index 000000000..05aed9323 --- /dev/null +++ b/apps/api/src/app/shared/validations/valid-import-file.validation.ts @@ -0,0 +1,16 @@ +// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars +import _whatever from 'multer'; +import { SupportedFileMimeTypes } from '@impler/shared'; +import { Injectable, PipeTransform } from '@nestjs/common'; +import { FileNotValidError } from '../exceptions/file-not-valid.exception'; + +@Injectable() +export class ValidImportFile implements PipeTransform { + transform(value: Express.Multer.File) { + if (!SupportedFileMimeTypes.includes(value.mimetype)) { + throw new FileNotValidError(); + } + + return value; + } +} diff --git a/apps/api/src/app/shared/validations/valid-mongo-id.validation.ts b/apps/api/src/app/shared/validations/valid-mongo-id.validation.ts new file mode 100644 index 000000000..5b1339ad8 --- /dev/null +++ b/apps/api/src/app/shared/validations/valid-mongo-id.validation.ts @@ -0,0 +1,17 @@ +import { CommonRepository } from '@impler/dal'; +import { BadRequestException, Injectable, PipeTransform } from '@nestjs/common'; + +@Injectable() +export class ValidateMongoId implements PipeTransform { + private commonRepository: CommonRepository; + constructor() { + this.commonRepository = new CommonRepository(); + } + + transform(value: string): string { + if (value && this.commonRepository.validMongoId(value)) { + return value; + } + throw new BadRequestException(); + } +} diff --git a/apps/api/src/app/shared/validations/valid-template.validation.ts b/apps/api/src/app/shared/validations/valid-template.validation.ts new file mode 100644 index 000000000..7b854aa3b --- /dev/null +++ b/apps/api/src/app/shared/validations/valid-template.validation.ts @@ -0,0 +1,16 @@ +import { CommonRepository, TemplateRepository } from '@impler/dal'; +import { BadRequestException, Injectable, PipeTransform } from '@nestjs/common'; +import { APIMessages } from '../constants'; + +@Injectable() +export class ValidateTemplate implements PipeTransform { + constructor(private commonRepository: CommonRepository, private templateRepository: TemplateRepository) {} + + async transform(value: string): Promise { + const isMongoId = this.commonRepository.validMongoId(value); + const template = await this.templateRepository.findOne(isMongoId ? { _id: value } : { code: value }); + if (!template) throw new BadRequestException(`${value} ${APIMessages.INVALID_TEMPLATE_ID_CODE_SUFFIX}`); + + return template._id; + } +} diff --git a/apps/api/src/app/template/dtos/create-template-request.dto.ts b/apps/api/src/app/template/dtos/create-template-request.dto.ts new file mode 100644 index 000000000..d4100c3ea --- /dev/null +++ b/apps/api/src/app/template/dtos/create-template-request.dto.ts @@ -0,0 +1,39 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Transform } from 'class-transformer'; +import { changeToCode } from '@impler/shared'; +import { IsDefined, IsString, Validate, IsNumber, IsUrl } from 'class-validator'; +import { UniqueValidator } from '../../shared/framework/is-unique.validator'; + +export class CreateTemplateRequestDto { + @ApiProperty({ + description: 'Name of the template', + }) + @IsString() + @IsDefined() + name: string; + + @ApiProperty({ + description: 'Code of the template', + }) + @IsString() + @IsDefined() + @Validate(UniqueValidator, ['Template', 'code'], { + message: 'Code is already taken', + }) + @Transform((value) => changeToCode(value.value)) + code: string; + + @ApiProperty({ + description: 'Callback URL of the template, gets called when sending data to the application', + }) + @IsUrl() + @IsDefined() + callbackUrl: string; + + @ApiProperty({ + description: 'Size of data in rows that gets sent to the application', + }) + @IsNumber() + @IsDefined() + chunkSize: number; +} diff --git a/apps/api/src/app/template/dtos/template-response.dto.ts b/apps/api/src/app/template/dtos/template-response.dto.ts new file mode 100644 index 000000000..99c3a155d --- /dev/null +++ b/apps/api/src/app/template/dtos/template-response.dto.ts @@ -0,0 +1,53 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsDefined, IsNumber, IsString } from 'class-validator'; + +export class TemplateResponseDto { + @ApiPropertyOptional({ + description: 'Id of the template', + }) + @IsString() + @IsDefined() + _id?: string; + + @ApiProperty({ + description: 'Name of the template', + }) + @IsString() + @IsDefined() + name: string; + + @ApiProperty({ + description: 'Code of the template', + }) + @IsString() + @IsDefined() + code: string; + + @ApiProperty({ + description: 'Callback URL of the template, gets called when sending data to the application', + }) + @IsString() + @IsDefined() + callbackUrl: string; + + @ApiProperty({ + description: 'Size of data in rows that gets sent to the application', + }) + @IsNumber() + @IsDefined() + chunkSize: number; + + @ApiProperty({ + description: 'URL to download samle csv file', + }) + @IsString() + @IsDefined() + sampleFileUrl: string; + + @ApiProperty({ + description: 'Id of project related to the template', + }) + @IsString() + @IsDefined() + _projectId: string; +} diff --git a/apps/api/src/app/template/dtos/update-template-request.dto.ts b/apps/api/src/app/template/dtos/update-template-request.dto.ts new file mode 100644 index 000000000..ef13a82c2 --- /dev/null +++ b/apps/api/src/app/template/dtos/update-template-request.dto.ts @@ -0,0 +1,39 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsString, IsOptional, IsNumber, IsNotEmpty, IsUrl } from 'class-validator'; + +export class UpdateTemplateRequestDto { + @ApiProperty({ + description: 'Name of the template', + nullable: false, + }) + @IsOptional() + name?: string; + + @ApiProperty({ + description: 'Callback URL of the template, gets called when sending data to the application', + nullable: false, + }) + @IsUrl() + @IsOptional() + callbackUrl?: string; + + @ApiProperty({ + description: 'Size of data in rows that gets sent to the application', + format: 'number', + nullable: false, + }) + @IsNumber({ + allowNaN: false, + }) + @IsOptional() + @IsNotEmpty() + chunkSize?: number; + + @ApiProperty({ + description: 'Id of project related to the template', + nullable: false, + }) + @IsString() + @IsOptional() + _projectId?: string; +} diff --git a/apps/api/src/app/template/template.controller.ts b/apps/api/src/app/template/template.controller.ts new file mode 100644 index 000000000..ec0371808 --- /dev/null +++ b/apps/api/src/app/template/template.controller.ts @@ -0,0 +1,121 @@ +import { Body, Controller, Delete, Get, Param, Post, Put, UseGuards } from '@nestjs/common'; +import { ApiOperation, ApiTags, ApiOkResponse, ApiSecurity } from '@nestjs/swagger'; +import { UploadEntity } from '@impler/dal'; +import { ACCESS_KEY_NAME } from '@impler/shared'; +import { DocumentNotFoundException } from '@shared/exceptions/document-not-found.exception'; +import { APIKeyGuard } from '@shared/framework/auth.gaurd'; +import { ValidateMongoId } from '@shared/validations/valid-mongo-id.validation'; + +import { CreateTemplateRequestDto } from './dtos/create-template-request.dto'; +import { TemplateResponseDto } from './dtos/template-response.dto'; +import { UpdateTemplateRequestDto } from './dtos/update-template-request.dto'; +import { CreateTemplateCommand } from './usecases/create-template/create-template.command'; +import { CreateTemplate } from './usecases/create-template/create-template.usecase'; +import { DeleteTemplate } from './usecases/delete-template/delete-template.usecase'; +import { GetTemplates } from './usecases/get-templates/get-templates.usecase'; +import { UpdateTemplateCommand } from './usecases/update-template/update-template.command'; +import { UpdateTemplate } from './usecases/update-template/update-template.usecase'; +import { GetUploads } from './usecases/get-uploads/get-uploads.usecase'; +import { GetUploadsCommand } from './usecases/get-uploads/get-uploads.command'; + +@Controller('/template') +@ApiTags('Template') +@ApiSecurity(ACCESS_KEY_NAME) +@UseGuards(APIKeyGuard) +export class TemplateController { + constructor( + private getTemplatesUsecase: GetTemplates, + private createTemplateUsecase: CreateTemplate, + private updateTemplateUsecase: UpdateTemplate, + private deleteTemplateUsecase: DeleteTemplate, + private getUploads: GetUploads + ) {} + + @Get(':projectId') + @ApiOperation({ + summary: 'Get project templates', + }) + @ApiOkResponse({ + type: [TemplateResponseDto], + }) + getTemplates(@Param('projectId', ValidateMongoId) projectId: string): Promise { + return this.getTemplatesUsecase.execute(projectId); + } + + @Post(':projectId') + @ApiOperation({ + summary: 'Add template in project', + }) + @ApiOkResponse({ + type: TemplateResponseDto, + }) + createTemplate( + @Param('projectId', ValidateMongoId) projectId: string, + @Body() body: CreateTemplateRequestDto + ): Promise { + return this.createTemplateUsecase.execute( + CreateTemplateCommand.create({ + _projectId: projectId, + callbackUrl: body.callbackUrl, + chunkSize: body.chunkSize, + code: body.code, + name: body.name, + }) + ); + } + + @Put(':templateId') + @ApiOperation({ + summary: 'Update template', + }) + @ApiOkResponse({ + type: TemplateResponseDto, + }) + async updateTemplate( + @Param('templateId', ValidateMongoId) templateId: string, + @Body() body: UpdateTemplateRequestDto + ): Promise { + const document = await this.updateTemplateUsecase.execute( + UpdateTemplateCommand.create({ + _projectId: body._projectId, + callbackUrl: body.callbackUrl, + chunkSize: body.chunkSize, + name: body.name, + }), + templateId + ); + if (!document) { + throw new DocumentNotFoundException('Template', templateId); + } + + return document; + } + + @Delete(':templateId') + @ApiOperation({ + summary: 'Delete template', + }) + @ApiOkResponse({ + type: TemplateResponseDto, + }) + async deleteTemplate(@Param('templateId', ValidateMongoId) templateId: string): Promise { + const document = await this.deleteTemplateUsecase.execute(templateId); + if (!document) { + throw new DocumentNotFoundException('Template', templateId); + } + + return document; + } + + @Get(':templateId/uploads') + @ApiOperation({ + summary: 'Get all uploads information for template', + }) + async getAllUploads(@Param('templateId', ValidateMongoId) templateId: string): Promise { + return this.getUploads.execute( + GetUploadsCommand.create({ + _templateId: templateId, + }) + ); + } +} diff --git a/apps/api/src/app/template/template.module.ts b/apps/api/src/app/template/template.module.ts new file mode 100644 index 000000000..acd0f4b64 --- /dev/null +++ b/apps/api/src/app/template/template.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { USE_CASES } from './usecases'; +import { TemplateController } from './template.controller'; +import { SharedModule } from '../shared/shared.module'; + +@Module({ + imports: [SharedModule], + providers: [...USE_CASES], + controllers: [TemplateController], +}) +export class TemplateModule {} diff --git a/apps/api/src/app/template/usecases/create-template/create-template.command.ts b/apps/api/src/app/template/usecases/create-template/create-template.command.ts new file mode 100644 index 000000000..694e4e8e6 --- /dev/null +++ b/apps/api/src/app/template/usecases/create-template/create-template.command.ts @@ -0,0 +1,24 @@ +import { IsDefined, IsString, IsNumber } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class CreateTemplateCommand extends BaseCommand { + @IsDefined() + @IsString() + name: string; + + @IsDefined() + @IsString() + code: string; + + @IsString() + @IsDefined() + callbackUrl: string; + + @IsNumber() + @IsDefined() + chunkSize: number; + + @IsString() + @IsDefined() + _projectId: string; +} diff --git a/apps/api/src/app/template/usecases/create-template/create-template.usecase.ts b/apps/api/src/app/template/usecases/create-template/create-template.usecase.ts new file mode 100644 index 000000000..5f8769046 --- /dev/null +++ b/apps/api/src/app/template/usecases/create-template/create-template.usecase.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@nestjs/common'; +import { TemplateRepository } from '@impler/dal'; +import { CreateTemplateCommand } from './create-template.command'; + +@Injectable() +export class CreateTemplate { + constructor(private templateRepository: TemplateRepository) {} + + async execute(command: CreateTemplateCommand) { + return this.templateRepository.create(command); + } +} diff --git a/apps/api/src/app/template/usecases/delete-template/delete-template.usecase.ts b/apps/api/src/app/template/usecases/delete-template/delete-template.usecase.ts new file mode 100644 index 000000000..d33666314 --- /dev/null +++ b/apps/api/src/app/template/usecases/delete-template/delete-template.usecase.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { TemplateRepository } from '@impler/dal'; + +@Injectable() +export class DeleteTemplate { + constructor(private templateRepository: TemplateRepository) {} + + async execute(id: string) { + return this.templateRepository.delete({ _id: id }); + } +} diff --git a/apps/api/src/app/template/usecases/get-templates/get-templates.usecase.ts b/apps/api/src/app/template/usecases/get-templates/get-templates.usecase.ts new file mode 100644 index 000000000..5589f1bbe --- /dev/null +++ b/apps/api/src/app/template/usecases/get-templates/get-templates.usecase.ts @@ -0,0 +1,22 @@ +import { Injectable } from '@nestjs/common'; +import { TemplateRepository } from '@impler/dal'; +import { TemplateResponseDto } from '../../dtos/template-response.dto'; + +@Injectable() +export class GetTemplates { + constructor(private templateRepository: TemplateRepository) {} + + async execute(projectId: string): Promise { + const templates = await this.templateRepository.find({ projectId }); + + return templates.map((template) => ({ + _projectId: template._projectId, + callbackUrl: template.callbackUrl, + chunkSize: template.chunkSize, + code: template.code, + name: template.name, + sampleFileUrl: template.sampleFileUrl, + _id: template._id, + })); + } +} diff --git a/apps/api/src/app/template/usecases/get-uploads/get-uploads.command.ts b/apps/api/src/app/template/usecases/get-uploads/get-uploads.command.ts new file mode 100644 index 000000000..fa45e3f28 --- /dev/null +++ b/apps/api/src/app/template/usecases/get-uploads/get-uploads.command.ts @@ -0,0 +1,12 @@ +import { IsDefined, IsMongoId, IsOptional, IsString } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class GetUploadsCommand extends BaseCommand { + @IsDefined() + @IsMongoId() + _templateId: string; + + @IsOptional() + @IsString() + select?: string; +} diff --git a/apps/api/src/app/template/usecases/get-uploads/get-uploads.usecase.ts b/apps/api/src/app/template/usecases/get-uploads/get-uploads.usecase.ts new file mode 100644 index 000000000..5fae99e32 --- /dev/null +++ b/apps/api/src/app/template/usecases/get-uploads/get-uploads.usecase.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@nestjs/common'; +import { UploadRepository } from '@impler/dal'; +import { GetUploadsCommand } from './get-uploads.command'; + +@Injectable() +export class GetUploads { + constructor(private uploadRepository: UploadRepository) {} + + async execute(command: GetUploadsCommand) { + return this.uploadRepository.find({ _templateId: command._templateId }, command.select); + } +} diff --git a/apps/api/src/app/template/usecases/index.ts b/apps/api/src/app/template/usecases/index.ts new file mode 100644 index 000000000..584bb55f8 --- /dev/null +++ b/apps/api/src/app/template/usecases/index.ts @@ -0,0 +1,14 @@ +import { GetTemplates } from './get-templates/get-templates.usecase'; +import { CreateTemplate } from './create-template/create-template.usecase'; +import { UpdateTemplate } from './update-template/update-template.usecase'; +import { DeleteTemplate } from './delete-template/delete-template.usecase'; +import { GetUploads } from './get-uploads/get-uploads.usecase'; + +export const USE_CASES = [ + GetTemplates, + CreateTemplate, + UpdateTemplate, + DeleteTemplate, + GetUploads, + // +]; diff --git a/apps/api/src/app/template/usecases/update-template/update-template.command.ts b/apps/api/src/app/template/usecases/update-template/update-template.command.ts new file mode 100644 index 000000000..0474e19b9 --- /dev/null +++ b/apps/api/src/app/template/usecases/update-template/update-template.command.ts @@ -0,0 +1,26 @@ +import { IsString, IsNumber, IsOptional, IsNotEmpty, IsMongoId, Min } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class UpdateTemplateCommand extends BaseCommand { + @IsString() + @IsOptional() + name?: string; + + @IsString() + @IsOptional() + callbackUrl?: string; + + @IsNumber({ + allowNaN: false, + }) + @IsOptional() + @IsNotEmpty() + @Min(1) + chunkSize?: number; + + @IsMongoId({ + message: '_projectId is not valid', + }) + @IsOptional() + _projectId?: string; +} diff --git a/apps/api/src/app/template/usecases/update-template/update-template.usecase.ts b/apps/api/src/app/template/usecases/update-template/update-template.usecase.ts new file mode 100644 index 000000000..b96f0589d --- /dev/null +++ b/apps/api/src/app/template/usecases/update-template/update-template.usecase.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@nestjs/common'; +import { TemplateRepository } from '@impler/dal'; +import { UpdateTemplateCommand } from './update-template.command'; + +@Injectable() +export class UpdateTemplate { + constructor(private templateRepository: TemplateRepository) {} + + async execute(command: UpdateTemplateCommand, id: string) { + return this.templateRepository.findOneAndUpdate({ _id: id }, command); + } +} diff --git a/apps/api/src/app/upload/dtos/upload-request.dto.ts b/apps/api/src/app/upload/dtos/upload-request.dto.ts new file mode 100644 index 000000000..fd14b31db --- /dev/null +++ b/apps/api/src/app/upload/dtos/upload-request.dto.ts @@ -0,0 +1,26 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsJSON, IsOptional, IsString } from 'class-validator'; + +export class UploadRequestDto { + @ApiProperty({ + type: 'file', + required: true, + }) + file: Express.Multer.File; + + @ApiProperty({ + description: 'Auth header value to send during webhook call', + required: false, + }) + @IsOptional() + @IsString() + authHeaderValue: string; + + @ApiProperty({ + description: 'Payload to send during webhook call', + required: false, + }) + @IsOptional() + @IsJSON() + extra: string; +} diff --git a/apps/api/src/app/upload/upload.controller.ts b/apps/api/src/app/upload/upload.controller.ts new file mode 100644 index 000000000..d81327a78 --- /dev/null +++ b/apps/api/src/app/upload/upload.controller.ts @@ -0,0 +1,74 @@ +// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars +import _whatever from 'multer'; +import { Body, Controller, Get, Param, Post, UploadedFile, UseGuards, UseInterceptors } from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { ApiTags, ApiSecurity, ApiConsumes, ApiOperation, ApiParam } from '@nestjs/swagger'; +import { ACCESS_KEY_NAME } from '@impler/shared'; + +import { APIKeyGuard } from '../shared/framework/auth.gaurd'; +import { UploadRequestDto } from './dtos/upload-request.dto'; +import { ValidImportFile } from '../shared/validations/valid-import-file.validation'; +import { MakeUploadEntry } from './usecases/make-upload-entry/make-upload-entry.usecase'; +import { MakeUploadEntryCommand } from './usecases/make-upload-entry/make-upload-entry.command'; +import { ValidateMongoId } from '../shared/validations/valid-mongo-id.validation'; +import { GetUpload } from '../shared/usecases/get-upload/get-upload.usecase'; +import { GetUploadCommand } from '../shared/usecases/get-upload/get-upload.command'; +import { ValidateTemplate } from '../shared/validations/valid-template.validation'; + +@Controller('/upload') +@ApiTags('Uploads') +@ApiSecurity(ACCESS_KEY_NAME) +@UseGuards(APIKeyGuard) +export class UploadController { + constructor(private makeUploadEntry: MakeUploadEntry, private getUpload: GetUpload) {} + + @Post(':template') + @ApiOperation({ + summary: `Upload file to template`, + }) + @ApiParam({ + name: 'template', + required: true, + description: 'ID or CODE of the template', + type: 'string', + }) + @ApiConsumes('multipart/form-data') + @UseInterceptors(FileInterceptor('file')) + async uploadFile( + @UploadedFile('file', ValidImportFile) file: Express.Multer.File, + @Body() body: UploadRequestDto, + @Param('template', ValidateTemplate) templateId: string + ) { + return await this.makeUploadEntry.execute( + MakeUploadEntryCommand.create({ + file: file, + templateId, + extra: body.extra, + authHeaderValue: body.authHeaderValue, + }) + ); + } + + @Get(':uploadId') + @ApiOperation({ + summary: 'Get Upload information', + }) + getUploadInformation(@Param('uploadId', ValidateMongoId) uploadId: string) { + return this.getUpload.execute(GetUploadCommand.create({ uploadId })); + } + + @Get(':uploadId/headings') + @ApiOperation({ + summary: 'Get headings for the uploaded file', + }) + async getHeadings(@Param('uploadId', ValidateMongoId) uploadId: string): Promise { + const uploadInfo = await this.getUpload.execute( + GetUploadCommand.create({ + uploadId, + select: 'headings', + }) + ); + + return uploadInfo.headings; + } +} diff --git a/apps/api/src/app/upload/upload.module.ts b/apps/api/src/app/upload/upload.module.ts new file mode 100644 index 000000000..f03b63a44 --- /dev/null +++ b/apps/api/src/app/upload/upload.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { USE_CASES } from './usecases'; +import { UploadController } from './upload.controller'; +import { SharedModule } from '../shared/shared.module'; + +@Module({ + imports: [SharedModule], + providers: [...USE_CASES], + controllers: [UploadController], +}) +export class UploadModule {} diff --git a/apps/api/src/app/upload/usecases/index.ts b/apps/api/src/app/upload/usecases/index.ts new file mode 100644 index 000000000..b4a0b1774 --- /dev/null +++ b/apps/api/src/app/upload/usecases/index.ts @@ -0,0 +1,8 @@ +import { MakeUploadEntry } from './make-upload-entry/make-upload-entry.usecase'; +import { GetUpload } from '../../shared/usecases/get-upload/get-upload.usecase'; + +export const USE_CASES = [ + MakeUploadEntry, + GetUpload, + // +]; diff --git a/apps/api/src/app/upload/usecases/make-upload-entry/add-upload-entry.command.ts b/apps/api/src/app/upload/usecases/make-upload-entry/add-upload-entry.command.ts new file mode 100644 index 000000000..6ba553014 --- /dev/null +++ b/apps/api/src/app/upload/usecases/make-upload-entry/add-upload-entry.command.ts @@ -0,0 +1,36 @@ +import { IsDefined, IsString, IsOptional, IsJSON, IsArray, IsNumber } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class AddUploadEntryCommand extends BaseCommand { + @IsDefined() + @IsString() + _templateId: string; + + @IsDefined() + @IsString() + _allDataFileId: string; + + @IsDefined() + @IsString() + _uploadedFileId: string; + + @IsDefined() + @IsString() + uploadId: string; + + @IsOptional() + @IsJSON() + extra?: string; + + @IsOptional() + @IsString() + authHeaderValue?: string; + + @IsOptional() + @IsArray() + headings?: string[]; + + @IsOptional() + @IsNumber() + totalRecords?: number; +} diff --git a/apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.command.ts b/apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.command.ts new file mode 100644 index 000000000..92a8183f8 --- /dev/null +++ b/apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.command.ts @@ -0,0 +1,19 @@ +import { IsDefined, IsString, IsOptional, IsJSON } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class MakeUploadEntryCommand extends BaseCommand { + @IsDefined() + file: Express.Multer.File; + + @IsDefined() + @IsString() + templateId: string; + + @IsOptional() + @IsJSON() + extra?: string; + + @IsOptional() + @IsString() + authHeaderValue?: string; +} diff --git a/apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.usecase.ts b/apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.usecase.ts new file mode 100644 index 000000000..ecf5161c9 --- /dev/null +++ b/apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.usecase.ts @@ -0,0 +1,91 @@ +import { Injectable } from '@nestjs/common'; +import { FileMimeTypesEnum, UploadStatusEnum } from '@impler/shared'; +import { CommonRepository, FileEntity, FileRepository, UploadRepository } from '@impler/dal'; +import { MakeUploadEntryCommand } from './make-upload-entry.command'; +import { FileNameService } from '../../../shared/file/name.service'; +import { StorageService } from '../../../shared/storage/storage.service'; +import { FileService } from '../../../shared/file/file.service'; +import { getFileService } from '../../../shared/helpers/file.helper'; +import { AddUploadEntryCommand } from './add-upload-entry.command'; + +@Injectable() +export class MakeUploadEntry { + constructor( + private uploadRepository: UploadRepository, + private commonRepository: CommonRepository, + private fileRepository: FileRepository, + private storageService: StorageService, + private fileNameService: FileNameService + ) {} + + async execute({ file, templateId, extra, authHeaderValue }: MakeUploadEntryCommand) { + const uploadId = this.commonRepository.generateMongoId(); + const fileEntity = await this.makeFileEntry(uploadId, file); + const fileService: FileService = getFileService(file.mimetype); + const fileInformation = await fileService.getFileInformation(this.storageService, fileEntity.path); + const allDataFile = await this.addAllDataEntry(uploadId, fileInformation.data); + + return this.addUploadEntry( + AddUploadEntryCommand.create({ + _templateId: templateId, + _uploadedFileId: fileEntity._id, + _allDataFileId: allDataFile._id, + uploadId, + extra, + authHeaderValue, + headings: fileInformation.headings, + totalRecords: fileInformation.totalRecords, + }) + ); + } + + private async makeFileEntry(uploadId: string, file: Express.Multer.File): Promise { + const uploadedFilePath = this.fileNameService.getUploadedFilePath(uploadId, file.originalname); + await this.storageService.uploadFile(uploadedFilePath, file.buffer, file.mimetype); + const uploadedFileName = this.fileNameService.getUploadedFileName(file.originalname); + const fileEntry = await this.fileRepository.create({ + mimeType: file.mimetype, + name: uploadedFileName, + originalName: file.originalname, + path: uploadedFilePath, + }); + + return fileEntry; + } + + private async addUploadEntry({ + _templateId, + _uploadedFileId, + _allDataFileId, + uploadId, + extra, + authHeaderValue, + headings, + totalRecords, + }: AddUploadEntryCommand) { + return this.uploadRepository.create({ + _id: uploadId, + _uploadedFileId, + _templateId, + _allDataFileId, + extra: extra, + headings: Array.isArray(headings) ? headings : [], + status: UploadStatusEnum.UPLOADED, + authHeaderValue: authHeaderValue, + totalRecords: totalRecords || 0, + }); + } + + private async addAllDataEntry(uploadId: string, data: Record[]): Promise { + const allDataFileName = this.fileNameService.getAllJsonDataFileName(); + const allDataFilePath = this.fileNameService.getAllJsonDataFilePath(uploadId); + await this.storageService.uploadFile(allDataFilePath, JSON.stringify(data), FileMimeTypesEnum.JSON); + + return await this.fileRepository.create({ + mimeType: FileMimeTypesEnum.JSON, + path: allDataFilePath, + name: allDataFileName, + originalName: allDataFileName, + }); + } +} diff --git a/apps/api/src/bootstrap.ts b/apps/api/src/bootstrap.ts new file mode 100644 index 000000000..4773f1b50 --- /dev/null +++ b/apps/api/src/bootstrap.ts @@ -0,0 +1,82 @@ +import './config'; + +import { INestApplication, ValidationPipe, Logger } from '@nestjs/common'; +import * as compression from 'compression'; +import { NestFactory } from '@nestjs/core'; +import * as bodyParser from 'body-parser'; +import { ExpressAdapter } from '@nestjs/platform-express'; +import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger'; +import { AppModule } from './app.module'; +import { validateEnv } from './config/env-validator'; +import { ACCESS_KEY_NAME } from '@impler/shared'; + +// Validate the ENV variables after launching SENTRY, so missing variables will report to sentry +validateEnv(); + +export async function bootstrap(expressApp?): Promise { + let app; + if (expressApp) { + app = await NestFactory.create(AppModule, new ExpressAdapter(expressApp)); + } else { + app = await NestFactory.create(AppModule); + } + + app.enableCors(corsOptionsDelegate); + + app.setGlobalPrefix('v1'); + + app.useGlobalPipes( + new ValidationPipe({ + transform: true, + }) + ); + + app.use(bodyParser.json()); + app.use(bodyParser.urlencoded({ extended: true })); + + app.use(compression()); + + const options = new DocumentBuilder() + .setTitle('Impler API') + .setDescription('The Impler API description') + .setVersion('1.0') + .addApiKey( + { + type: 'apiKey', // type + name: ACCESS_KEY_NAME, // Name of the key to expect in header + in: 'header', + }, + ACCESS_KEY_NAME // Name to show and used in swagger + ) + .addTag('Project') + .build(); + const document = SwaggerModule.createDocument(app, options); + + SwaggerModule.setup('api', app, document); + + if (expressApp) { + await app.init(); + } else { + await app.listen(process.env.PORT); + } + + Logger.log(`Started application in NODE_ENV=${process.env.NODE_ENV} on port ${process.env.PORT}`); + + return app; +} + +const corsOptionsDelegate = function (req, callback) { + const corsOptions = { + origin: false as boolean | string | string[], + preflightContinue: false, + allowedHeaders: ['Content-Type', ACCESS_KEY_NAME], + methods: ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], + }; + + if (['dev', 'test', 'local'].includes(process.env.NODE_ENV)) { + corsOptions.origin = '*'; + } else { + corsOptions.origin = [process.env.FRONT_BASE_URL]; + } + callback(null, corsOptions); +}; diff --git a/apps/api/src/config/env-validator.ts b/apps/api/src/config/env-validator.ts new file mode 100644 index 000000000..1efca9f6c --- /dev/null +++ b/apps/api/src/config/env-validator.ts @@ -0,0 +1,23 @@ +import { port, str, url, ValidatorSpec } from 'envalid'; +import * as envalid from 'envalid'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const validators: { [K in keyof any]: ValidatorSpec } = { + NODE_ENV: str({ + choices: ['dev', 'test', 'prod', 'ci', 'local'], + default: 'local', + }), + S3_LOCAL_STACK: str({ + default: '', + }), + S3_BUCKET_NAME: str(), + S3_REGION: str(), + PORT: port(), + FRONT_BASE_URL: url(), + MONGO_URL: str(), + RABBITMQ_CONN_URL: str(), +}; + +export function validateEnv() { + envalid.cleanEnv(process.env, validators); +} diff --git a/apps/api/src/config/index.ts b/apps/api/src/config/index.ts new file mode 100644 index 000000000..2e166f751 --- /dev/null +++ b/apps/api/src/config/index.ts @@ -0,0 +1,17 @@ +import * as dotenv from 'dotenv'; + +dotenv.config(); + +const envFileMapper = { + prod: '.env.production', + test: '.env.test', + ci: '.env.ci', + local: '.env', + dev: '.env.development', +}; +const selectedEnvFile = envFileMapper[process.env.NODE_ENV] || '.env'; + +const path = `${__dirname}/${process.env.E2E_RUNNER ? '..' : 'src'}/${selectedEnvFile}`; + +const { error } = dotenv.config({ path }); +if (error && !process.env.LAMBDA_TASK_ROOT) throw error; diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts new file mode 100644 index 000000000..2fb9dfe34 --- /dev/null +++ b/apps/api/src/main.ts @@ -0,0 +1,3 @@ +import { bootstrap } from './bootstrap'; + +bootstrap(); diff --git a/apps/api/src/types/env.d.ts b/apps/api/src/types/env.d.ts new file mode 100644 index 000000000..6f2dbb19e --- /dev/null +++ b/apps/api/src/types/env.d.ts @@ -0,0 +1,14 @@ +declare namespace NodeJS { + // eslint-disable-next-line @typescript-eslint/naming-convention + export interface ProcessEnv { + NODE_ENV: 'test' | 'prod' | 'dev' | 'ci' | 'local'; + PORT: number; + 'ACCESS-KEY'?: string; + FRONT_BASE_URL: string; + + MONGO_URL: string; + S3_LOCAL_STACK: string; + S3_REGION: string; + S3_BUCKET_NAME: string; + } +} diff --git a/apps/api/tsconfig.build.json b/apps/api/tsconfig.build.json new file mode 100644 index 000000000..05ea22b05 --- /dev/null +++ b/apps/api/tsconfig.build.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "declaration": true, + "noImplicitAny": false, + "removeComments": true, + "allowSyntheticDefaultImports": true, + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "target": "es6", + "esModuleInterop": false, + "sourceMap": true, + "outDir": "./dist", + "baseUrl": "src", + "paths": { + "@shared/*": ["app/shared/*"], + }, + "types": ["node"] + }, + "include": [".eslintrc.js", "src/**/*", "src/**/*.d.ts"], + "exclude": ["node_modules", "**/*.spec.ts"] +} diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json new file mode 100644 index 000000000..3b3667322 --- /dev/null +++ b/apps/api/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "allowSyntheticDefaultImports": true, + "emitDecoratorMetadata": true, + "types": ["node", "multer", "mocha", "chai"], + "target": "es2017", + "allowJs": false, + "esModuleInterop": false, + "declarationMap": true, + "baseUrl": "src", + "paths": { + "@shared/*": ["app/shared/*"], + } + } +} diff --git a/apps/queue-manager/.dockerignore b/apps/queue-manager/.dockerignore new file mode 100644 index 000000000..3c3629e64 --- /dev/null +++ b/apps/queue-manager/.dockerignore @@ -0,0 +1 @@ +node_modules diff --git a/apps/queue-manager/.eslintrc.js b/apps/queue-manager/.eslintrc.js new file mode 100644 index 000000000..5a2cc7f1e --- /dev/null +++ b/apps/queue-manager/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: ['../../.eslintrc.js'], +}; diff --git a/apps/queue-manager/nodemon.json b/apps/queue-manager/nodemon.json new file mode 100644 index 000000000..0b841991e --- /dev/null +++ b/apps/queue-manager/nodemon.json @@ -0,0 +1,6 @@ +{ + "watch": ["src"], + "ext": "ts", + "ignore": ["src/**/*.spec.ts"], + "exec": "ts-node src/index.ts" +} diff --git a/apps/queue-manager/package.json b/apps/queue-manager/package.json new file mode 100644 index 000000000..8257c499e --- /dev/null +++ b/apps/queue-manager/package.json @@ -0,0 +1,38 @@ +{ + "name": "@impler/queue-manager", + "version": "0.1.0", + "author": "knovator", + "license": "MIT", + "private": true, + "scripts": { + "prebuild": "rimraf dist", + "preinstall": "pnpm build", + "format": "prettier --write \"src/**/*.ts\"", + "build": "cross-env node_modules/.bin/tsc -p tsconfig.build.json", + "start": "cross-env TZ=UTC nodemon", + "start:prod": "cross-env TZ=UTC node ./dist/index.js", + "start:dev": "cross-env TZ=UTC nodemon", + "precommit": "lint-staged", + "lint": "eslint src", + "lint:fix": "pnpm lint -- --fix" + }, + "dependencies": { + "@impler/dal": "^0.1.0", + "@impler/shared": "^0.1.0", + "axios": "^0.26.1", + "dotenv": "^16.0.2", + "envalid": "^7.3.1" + }, + "devDependencies": { + "@types/node": "^18.7.18", + "nodemon": "^2.0.20", + "rimraf": "^3.0.2", + "ts-node": "^10.9.1", + "typescript": "^4.8.3" + }, + "lint-staged": { + "*.{js,jsx,ts,tsx}": [ + "eslint --fix" + ] + } +} diff --git a/apps/queue-manager/src/.env.development b/apps/queue-manager/src/.env.development new file mode 100644 index 000000000..fe959e4a3 --- /dev/null +++ b/apps/queue-manager/src/.env.development @@ -0,0 +1,8 @@ +NODE_ENV=local +RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672 +MONGO_URL=mongodb://localhost:27017/impler-db + +# Storage +S3_REGION=us-east-1 +S3_LOCAL_STACK=http://localhost:4566 +S3_BUCKET_NAME=impler diff --git a/apps/queue-manager/src/.env.production b/apps/queue-manager/src/.env.production new file mode 100644 index 000000000..fe959e4a3 --- /dev/null +++ b/apps/queue-manager/src/.env.production @@ -0,0 +1,8 @@ +NODE_ENV=local +RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672 +MONGO_URL=mongodb://localhost:27017/impler-db + +# Storage +S3_REGION=us-east-1 +S3_LOCAL_STACK=http://localhost:4566 +S3_BUCKET_NAME=impler diff --git a/apps/queue-manager/src/.env.test b/apps/queue-manager/src/.env.test new file mode 100644 index 000000000..fe959e4a3 --- /dev/null +++ b/apps/queue-manager/src/.env.test @@ -0,0 +1,8 @@ +NODE_ENV=local +RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672 +MONGO_URL=mongodb://localhost:27017/impler-db + +# Storage +S3_REGION=us-east-1 +S3_LOCAL_STACK=http://localhost:4566 +S3_BUCKET_NAME=impler diff --git a/apps/queue-manager/src/.example.env b/apps/queue-manager/src/.example.env new file mode 100644 index 000000000..fe959e4a3 --- /dev/null +++ b/apps/queue-manager/src/.example.env @@ -0,0 +1,8 @@ +NODE_ENV=local +RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672 +MONGO_URL=mongodb://localhost:27017/impler-db + +# Storage +S3_REGION=us-east-1 +S3_LOCAL_STACK=http://localhost:4566 +S3_BUCKET_NAME=impler diff --git a/apps/queue-manager/src/bootstrap.ts b/apps/queue-manager/src/bootstrap.ts new file mode 100644 index 000000000..8b7067f12 --- /dev/null +++ b/apps/queue-manager/src/bootstrap.ts @@ -0,0 +1,44 @@ +import './config/env-config'; +import amqp, { ChannelWrapper } from 'amqp-connection-manager'; +import { ProcessFileConsumer } from './consumers'; +import { QueuesEnum } from '@impler/shared'; +import { DalService } from '@impler/dal'; +import { IAmqpConnectionManager } from 'amqp-connection-manager/dist/esm/AmqpConnectionManager'; +import { validateEnv } from './config/env-validator'; + +let connection: IAmqpConnectionManager, chanelWrapper: ChannelWrapper; + +validateEnv(); + +export async function bootstrap() { + // conenct dal service + const dalService = new DalService(); + await dalService.connect(process.env.MONGO_URL); + + // connect to amqp rabbitmq server + connection = amqp.connect([process.env.RABBITMQ_CONN_URL]); + connection.on('connect', () => console.log('QueueManager RabbitMQ::Connected!')); + connection.on('disconnect', (err: Error) => console.log('RabbitMQ::Disconnected.', err)); + + // create channel + chanelWrapper = connection.createChannel({ + json: true, + }); + + // initialize consumers + const processFileConsumer = new ProcessFileConsumer(); + + // add queues to channel + chanelWrapper.addSetup((channel) => { + return Promise.all([ + channel.assertQueue(QueuesEnum.PROCESS_FILE, { + durable: false, + }), + channel.consume(QueuesEnum.PROCESS_FILE, processFileConsumer.message.bind(processFileConsumer), { noAck: true }), + ]); + }); +} + +export function publishToQueue(queueName: QueuesEnum, data: any) { + chanelWrapper.sendToQueue(queueName, data); +} diff --git a/apps/queue-manager/src/config/env-config.ts b/apps/queue-manager/src/config/env-config.ts new file mode 100644 index 000000000..f82be2f2e --- /dev/null +++ b/apps/queue-manager/src/config/env-config.ts @@ -0,0 +1,17 @@ +import * as dotenv from 'dotenv'; + +dotenv.config(); + +const envFileMapper = { + prod: '.env.production', + test: '.env.test', + ci: '.env.ci', + local: '.env', + dev: '.env.development', +}; +const selectedEnvFile = envFileMapper[process.env.NODE_ENV] || '.env'; + +const path = `${__dirname}/${process.env.E2E_RUNNER ? '..' : '..'}/${selectedEnvFile}`; + +const { error } = dotenv.config({ path }); +if (error && !process.env.LAMBDA_TASK_ROOT) throw error; diff --git a/apps/queue-manager/src/config/env-validator.ts b/apps/queue-manager/src/config/env-validator.ts new file mode 100644 index 000000000..5f3afb0f1 --- /dev/null +++ b/apps/queue-manager/src/config/env-validator.ts @@ -0,0 +1,21 @@ +import { str, ValidatorSpec } from 'envalid'; +import * as envalid from 'envalid'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const validators: { [K in keyof any]: ValidatorSpec } = { + NODE_ENV: str({ + choices: ['dev', 'test', 'prod', 'ci', 'local'], + default: 'local', + }), + MONGO_URL: str(), + RABBITMQ_CONN_URL: str(), + S3_LOCAL_STACK: str({ + default: '', + }), + S3_BUCKET_NAME: str(), + S3_REGION: str(), +}; + +export function validateEnv() { + envalid.cleanEnv(process.env, validators); +} diff --git a/apps/queue-manager/src/consumers/base.consumer.ts b/apps/queue-manager/src/consumers/base.consumer.ts new file mode 100644 index 000000000..09197caab --- /dev/null +++ b/apps/queue-manager/src/consumers/base.consumer.ts @@ -0,0 +1,3 @@ +export abstract class BaseConsumer { + abstract message(data: any): void; +} diff --git a/apps/queue-manager/src/consumers/index.ts b/apps/queue-manager/src/consumers/index.ts new file mode 100644 index 000000000..6846af556 --- /dev/null +++ b/apps/queue-manager/src/consumers/index.ts @@ -0,0 +1 @@ +export * from './process-file'; diff --git a/apps/queue-manager/src/consumers/process-file.ts b/apps/queue-manager/src/consumers/process-file.ts new file mode 100644 index 000000000..f82b115b5 --- /dev/null +++ b/apps/queue-manager/src/consumers/process-file.ts @@ -0,0 +1,229 @@ +import axios from 'axios'; +import { + FileEncodingsEnum, + ProcessFileCachedData, + ProcessFileData, + StorageService, + StatusEnum, + UploadStatusEnum, + QueuesEnum, +} from '@impler/shared'; +import { FileEntity, UploadRepository, TemplateRepository, WebhookLogRepository, WebhookLogEntity } from '@impler/dal'; +import { BaseConsumer } from './base.consumer'; +import { getStorageServiceClass } from '../helpers/storage.helper'; +import { publishToQueue } from '../bootstrap'; +import { + ISendDataParameters, + IBuildSendDataParameters, + IGetNextDataParameters, + ISendData, +} from '../types/file-processing.types'; + +const MIN_LIMIT = 0; +const DEFAULT_PAGE = 1; + +export class ProcessFileConsumer extends BaseConsumer { + private templateRepository: TemplateRepository = new TemplateRepository(); + private uploadRepository: UploadRepository = new UploadRepository(); + private webhookLogRepository: WebhookLogRepository = new WebhookLogRepository(); + private storageService: StorageService = getStorageServiceClass(); + + async message(message) { + const data = JSON.parse(message.content) as ProcessFileData; + const uploadId = data.uploadId; + const cachedData = data.cache || (await this.getInitialCachedData(uploadId)); + + if (cachedData) { + // Get valid data information + let validDataJSON: null | any[] = null; + if (cachedData.validDataFilePath) { + const validDataContent = await this.storageService.getFileContent( + cachedData.validDataFilePath, + FileEncodingsEnum.JSON + ); + validDataJSON = JSON.parse(validDataContent); + } + + // Get invalid data information + let invalidDataJSON: null | any[] = null; + if (cachedData.processInvalidRecords && cachedData.invalidDataFilePath) { + const invalidDataContent = await this.storageService.getFileContent( + cachedData.invalidDataFilePath, + FileEncodingsEnum.JSON + ); + invalidDataJSON = JSON.parse(invalidDataContent); + } + + const sendData = this.buildSendData({ + chunkSize: cachedData.chunkSize, + data: cachedData.isInvalidRecords ? invalidDataJSON : validDataJSON, + page: cachedData.page || DEFAULT_PAGE, + isInvalidRecords: cachedData.isInvalidRecords, + template: cachedData.code, + uploadId, + extra: cachedData.extra, + }); + + const response = await this.makeApiCall({ data: sendData, method: 'POST', url: cachedData.callbackUrl }); + this.makeResponseEntry(response); + + const nextCachedData = this.getNextData({ + validData: validDataJSON, + invalidData: invalidDataJSON, + ...cachedData, + }); + + if (nextCachedData) { + // Make next call + publishToQueue(QueuesEnum.PROCESS_FILE, { + uploadId, + cache: nextCachedData, + }); + } else { + // Processing is done + this.finalizeUpload(uploadId); + } + } + } + + private async makeApiCall({ data, method, url }: ISendDataParameters): Promise> { + const baseResponse: Partial = { + _uploadId: data.uploadId, + callDate: new Date(), + pageNumber: data.page, + }; + try { + const response = await axios({ + method, + url, + data, + }); + + baseResponse.responseStatusCode = response.status; + baseResponse.status = StatusEnum.SUCCEED; + + return baseResponse; + } catch (error) { + baseResponse.status = StatusEnum.FAILED; + if (axios.isAxiosError(error)) { + if (error.response) { + baseResponse.failedReason = 'Application Error'; + baseResponse.responseStatusCode = error.response.status; + } else if (error.request) { + baseResponse.failedReason = 'Network Error'; + baseResponse.responseStatusCode = error.request.status; + } else { + baseResponse.failedReason = error.message; + baseResponse.responseStatusCode = 400; + } + } else { + baseResponse.failedReason = error.message; + baseResponse.responseStatusCode = 400; + } + + return baseResponse; + } + } + + private buildSendData({ + data, + page = DEFAULT_PAGE, + chunkSize, + isInvalidRecords, + template, + uploadId, + extra = '', + }: IBuildSendDataParameters): ISendData { + const slicedData = data.slice( + Math.max((page - DEFAULT_PAGE) * chunkSize, MIN_LIMIT), + Math.min((page + DEFAULT_PAGE) * chunkSize, data.length) + ); + + return { + data: slicedData, + extra: extra ? JSON.parse(extra) : '', + isInvalidRecords, + page, + pageSize: slicedData.length, + template, + totalPages: this.getTotalPages(data.length, chunkSize), + totalRecords: data.length, + uploadId, + }; + } + + private getNextData({ + validData, + page, + chunkSize, + invalidData, + isInvalidRecords, + ...rest + }: IGetNextDataParameters): ProcessFileCachedData | null { + const baseData = { + chunkSize, + page: page + DEFAULT_PAGE, + isInvalidRecords: isInvalidRecords || false, + ...rest, + }; + if (!isInvalidRecords && Array.isArray(validData) && validData.length > page * chunkSize) { + // there is more valid data available to send on next page + return { + ...baseData, + page: page + DEFAULT_PAGE, + isInvalidRecords: false, + }; + } else if (!isInvalidRecords && Array.isArray(invalidData) && invalidData.length > MIN_LIMIT) { + // valid data are completed, invalid-data is available, so now move to invalid data + return { + ...baseData, + page: 1, + isInvalidRecords: true, + }; + } else if (isInvalidRecords && Array.isArray(invalidData) && invalidData.length > page * chunkSize) { + // currently processing invalid data, and there is more invalid data available to send + return { + ...baseData, + page: page + DEFAULT_PAGE, + isInvalidRecords: true, + }; + } + + return null; + } + + private getTotalPages(totalRecords, pageSize): number { + return Math.ceil(totalRecords / pageSize); + } + + private async getInitialCachedData(_uploadId: string): Promise { + // Get Upload Information + const uploadata = await this.uploadRepository.getUploadProcessInformation(_uploadId); + + if (!uploadata._validDataFileId && !uploadata._invalidDataFileId) return null; + + // Get template information + const templateData = await this.templateRepository.findById(uploadata._templateId, 'callbackUrl chunkSize code'); + + return { + _templateId: uploadata._templateId, + callbackUrl: templateData.callbackUrl, + chunkSize: templateData.chunkSize, + code: templateData.code, + isInvalidRecords: uploadata._validDataFileId ? false : true, + invalidDataFilePath: (uploadata._invalidDataFileId as unknown as FileEntity)?.path, + page: 1, + processInvalidRecords: uploadata.processInvalidRecords, + validDataFilePath: (uploadata._validDataFileId as unknown as FileEntity)?.path, + extra: uploadata.extra, + }; + } + + private async makeResponseEntry(data: Partial) { + return await this.webhookLogRepository.create(data); + } + + private async finalizeUpload(uploadId: string) { + return await this.uploadRepository.update({ _id: uploadId }, { status: UploadStatusEnum.COMPLETED }); + } +} diff --git a/apps/queue-manager/src/helpers/storage.helper.ts b/apps/queue-manager/src/helpers/storage.helper.ts new file mode 100644 index 000000000..5c0d02999 --- /dev/null +++ b/apps/queue-manager/src/helpers/storage.helper.ts @@ -0,0 +1,11 @@ +import { S3StorageService, StorageService } from '@impler/shared'; + +let storageService: StorageService; + +// Implementing singleton pattern for storage service +export function getStorageServiceClass() { + if (storageService) return storageService; + storageService = new S3StorageService(); + + return storageService; +} diff --git a/apps/queue-manager/src/index.ts b/apps/queue-manager/src/index.ts new file mode 100644 index 000000000..2fb9dfe34 --- /dev/null +++ b/apps/queue-manager/src/index.ts @@ -0,0 +1,3 @@ +import { bootstrap } from './bootstrap'; + +bootstrap(); diff --git a/apps/queue-manager/src/types/env.d.ts b/apps/queue-manager/src/types/env.d.ts new file mode 100644 index 000000000..59171da87 --- /dev/null +++ b/apps/queue-manager/src/types/env.d.ts @@ -0,0 +1,10 @@ +declare namespace NodeJS { + // eslint-disable-next-line @typescript-eslint/naming-convention + export interface ProcessEnv { + MONGO_URL: string; + RABBITMQ_CONN_URL: string; + S3_LOCAL_STACK: string; + S3_REGION: string; + S3_BUCKET_NAME: string; + } +} diff --git a/apps/queue-manager/src/types/file-processing.types.ts b/apps/queue-manager/src/types/file-processing.types.ts new file mode 100644 index 000000000..8c7b664ff --- /dev/null +++ b/apps/queue-manager/src/types/file-processing.types.ts @@ -0,0 +1,37 @@ +import { ProcessFileCachedData } from '@impler/shared'; + +export interface ISendDataParameters { + data: ISendData; + url: string; + method: 'POST'; +} +export interface IBuildSendDataParameters { + data: any[]; + page: number; + chunkSize: number; + isInvalidRecords: boolean; + template: string; + uploadId: string; + extra?: string; +} +export interface IGetNextDataParameters extends ProcessFileCachedData { + validData: any[]; + invalidData: any[]; +} + +export interface ISendDataResponse { + statusCode: number; + status: 'FAILED' | 'SUCCEED'; + failedReason?: string; +} +export interface ISendData { + template: string; + uploadId: string; + data: any[]; + totalRecords: number; + totalPages: number; + page: number; + pageSize: number; + extra: string; + isInvalidRecords: boolean; +} diff --git a/apps/queue-manager/tsconfig.build.json b/apps/queue-manager/tsconfig.build.json new file mode 100644 index 000000000..3109babaf --- /dev/null +++ b/apps/queue-manager/tsconfig.build.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "target": "es6", + "declaration": true, + "noImplicitAny": false, + "outDir": "./dist", + "rootDir": "./src", + "types": ["node"] + }, + "include": ["src/**/*"] +} diff --git a/apps/queue-manager/tsconfig.json b/apps/queue-manager/tsconfig.json new file mode 100644 index 000000000..2e0513159 --- /dev/null +++ b/apps/queue-manager/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node"] + } +} diff --git a/apps/widget-demo/.example.env b/apps/widget-demo/.example.env new file mode 100644 index 000000000..39539fe45 --- /dev/null +++ b/apps/widget-demo/.example.env @@ -0,0 +1,3 @@ +VITE_PROJECT_ID= +VITE_ACCESS_TOKEN= +VITE_TEMPLATE= \ No newline at end of file diff --git a/apps/widget-demo/.gitignore b/apps/widget-demo/.gitignore new file mode 100644 index 000000000..a547bf36d --- /dev/null +++ b/apps/widget-demo/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/apps/widget-demo/index.html b/apps/widget-demo/index.html new file mode 100644 index 000000000..0a6d08158 --- /dev/null +++ b/apps/widget-demo/index.html @@ -0,0 +1,14 @@ + + + + + + + Widget Demo + + +
+ + + + diff --git a/apps/widget-demo/package.json b/apps/widget-demo/package.json new file mode 100644 index 000000000..c69c0a18f --- /dev/null +++ b/apps/widget-demo/package.json @@ -0,0 +1,28 @@ +{ + "name": "@impler/widget-demo", + "version": "0.1.0", + "author": "knovator", + "license": "MIT", + "private": true, + "type": "module", + "scripts": { + "start": "vite", + "start:dev": "vite", + "prebuild": "rimraf dist", + "build": "tsc && vite build", + "preview": "vite preview" + }, + "dependencies": { + "@impler/react": "^0.1.0", + "react": "18.2.0", + "react-dom": "18.2.0" + }, + "devDependencies": { + "@types/react": "^18.0.21", + "@types/react-dom": "^18.0.6", + "@vitejs/plugin-react": "^2.2.0", + "rimraf": "^3.0.2", + "typescript": "^4.8.4", + "vite": "^3.2.3" + } +} \ No newline at end of file diff --git a/apps/widget-demo/public/vite.svg b/apps/widget-demo/public/vite.svg new file mode 100644 index 000000000..e7b8dfb1b --- /dev/null +++ b/apps/widget-demo/public/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/widget-demo/src/App.tsx b/apps/widget-demo/src/App.tsx new file mode 100644 index 000000000..2740f4c5b --- /dev/null +++ b/apps/widget-demo/src/App.tsx @@ -0,0 +1,13 @@ +import { Button } from '@impler/react'; + +export const App = () => { + return ( +
+
+ ); +}; diff --git a/apps/widget-demo/src/main.tsx b/apps/widget-demo/src/main.tsx new file mode 100644 index 000000000..3f2a34f65 --- /dev/null +++ b/apps/widget-demo/src/main.tsx @@ -0,0 +1,9 @@ +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import { App } from './App'; + +ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render( + + + +); diff --git a/apps/widget-demo/src/vite-env.d.ts b/apps/widget-demo/src/vite-env.d.ts new file mode 100644 index 000000000..8c012e64c --- /dev/null +++ b/apps/widget-demo/src/vite-env.d.ts @@ -0,0 +1,13 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/// + +interface ImportMetaEnv { + readonly VITE_PROJECT_ID: string; + readonly VITE_ACCESS_TOKEN: string; + readonly VITE_TEMPLATE: string; + // more env variables... +} + +interface ImportMeta { + readonly env: ImportMetaEnv; +} diff --git a/apps/widget-demo/tsconfig.json b/apps/widget-demo/tsconfig.json new file mode 100644 index 000000000..3d0a51a86 --- /dev/null +++ b/apps/widget-demo/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ESNext", + "useDefineForClassFields": true, + "lib": ["DOM", "DOM.Iterable", "ESNext"], + "allowJs": false, + "skipLibCheck": true, + "esModuleInterop": false, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "module": "ESNext", + "moduleResolution": "Node", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx" + }, + "include": ["src"], + "references": [{ "path": "./tsconfig.node.json" }] +} diff --git a/apps/widget-demo/tsconfig.node.json b/apps/widget-demo/tsconfig.node.json new file mode 100644 index 000000000..9d31e2aed --- /dev/null +++ b/apps/widget-demo/tsconfig.node.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "composite": true, + "module": "ESNext", + "moduleResolution": "Node", + "allowSyntheticDefaultImports": true + }, + "include": ["vite.config.ts"] +} diff --git a/apps/widget-demo/vite.config.ts b/apps/widget-demo/vite.config.ts new file mode 100644 index 000000000..627a31962 --- /dev/null +++ b/apps/widget-demo/vite.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; + +// https://vitejs.dev/config/ +export default defineConfig({ + plugins: [react()], +}); diff --git a/apps/widget/.eslintrc.js b/apps/widget/.eslintrc.js new file mode 100644 index 000000000..0f7d8d40d --- /dev/null +++ b/apps/widget/.eslintrc.js @@ -0,0 +1,38 @@ +module.exports = { + rules: { + 'func-names': 'off', + 'react/jsx-props-no-spreading': 'off', + 'react/no-array-index-key': 'off', + 'no-empty-pattern': 'off', + '@typescript-eslint/explicit-module-boundary-types': 'off', + 'react/no-unescaped-entities': 'off', + 'react/jsx-closing-bracket-location': 'off', + '@typescript-eslint/ban-types': 'off', + 'react/jsx-wrap-multilines': 'off', + 'jsx-a11y/anchor-is-valid': 'off', + 'promise/catch-or-return': 'off', + 'react/jsx-one-expression-per-line': 'off', + '@typescript-eslint/no-explicit-any': 'off', + 'jsx-a11y/aria-role': 'off', + 'jsx-a11y/no-static-element-interactions': 'off', + 'react/require-default-props': 'off', + 'react/no-danger': 'off', + 'jsx-a11y/click-events-have-key-events': 'off', + '@typescript-eslint/naming-convention': [ + 'error', + { + filter: '_', + selector: 'variableLike', + leadingUnderscore: 'allow', + format: ['PascalCase', 'camelCase', 'UPPER_CASE'], + }, + ], + }, + ignorePatterns: ['craco.config.js'], + extends: ['../../.eslintrc.js'], + parserOptions: { + project: './tsconfig.json', + ecmaVersion: 2020, + sourceType: 'module', + }, +}; diff --git a/apps/widget/.example.env b/apps/widget/.example.env new file mode 100644 index 000000000..b4ab84278 --- /dev/null +++ b/apps/widget/.example.env @@ -0,0 +1 @@ +REACT_APP_API_URL=http://localhost:3000 \ No newline at end of file diff --git a/apps/widget/.gitignore b/apps/widget/.gitignore new file mode 100644 index 000000000..4c9ae0ca3 --- /dev/null +++ b/apps/widget/.gitignore @@ -0,0 +1,27 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# production +/build + +storybook-static +# misc +.DS_Store +.env.local +.env.development.local +.env.test.local +.env.production.local + +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +cypress/videos +cypress/screenshots diff --git a/apps/widget/.storybook/main.js b/apps/widget/.storybook/main.js new file mode 100644 index 000000000..3742746be --- /dev/null +++ b/apps/widget/.storybook/main.js @@ -0,0 +1,8 @@ +module.exports = { + stories: ['../src/**/*.stories.mdx', '../src/**/*.stories.@(js|jsx|ts|tsx)'], + addons: ['@storybook/addon-essentials'], + framework: '@storybook/react', + features: { + emotionAlias: false, + }, +}; diff --git a/apps/widget/.storybook/preview-head.html b/apps/widget/.storybook/preview-head.html new file mode 100644 index 000000000..5935506f7 --- /dev/null +++ b/apps/widget/.storybook/preview-head.html @@ -0,0 +1,3 @@ + + + diff --git a/apps/widget/craco.config.js b/apps/widget/craco.config.js new file mode 100644 index 000000000..05e70caae --- /dev/null +++ b/apps/widget/craco.config.js @@ -0,0 +1,14 @@ +const path = require('path'); +module.exports = { + webpack: { + alias: { + '@store': path.resolve(__dirname, './src/store'), + '@config': path.resolve(__dirname, './src/config'), + '@ui': path.resolve(__dirname, './src/design-system'), + '@types': path.resolve(__dirname, './src/types'), + '@icons': path.resolve(__dirname, './src/icons/index.ts'), + '@util': path.resolve(__dirname, './src/util/index.ts'), + '@hooks': path.resolve(__dirname, './src/hooks'), + }, + }, +}; diff --git a/apps/widget/env.sh b/apps/widget/env.sh new file mode 100755 index 000000000..e8ce8621b --- /dev/null +++ b/apps/widget/env.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Recreate config file +rm -rf ./env-config.js +touch ./env-config.js + +# Add assignment +echo "window._env_ = {" >> ./env-config.js + +# Read each line in .env file +# Each line represents key=value pairs +while read -r line || [[ -n "$line" ]]; +do + # Split env variables by character `=` + if printf '%s\n' "$line" | grep -q -e '='; then + varname=$(printf '%s\n' "$line" | sed -e 's/=.*//') + varvalue=$(printf '%s\n' "$line" | sed -e 's/^[^=]*=//') + fi + + # Read value of current variable if exists as Environment variable + value=$(printf '%s\n' "${!varname}") + # Otherwise use value from .env file + [[ -z $value ]] && value=${varvalue} + + # Append configuration property to JS file + echo " $varname: \"$value\"," >> ./env-config.js +done < .env + +echo "}" >> ./env-config.js diff --git a/apps/widget/package.json b/apps/widget/package.json new file mode 100644 index 000000000..6990cefd6 --- /dev/null +++ b/apps/widget/package.json @@ -0,0 +1,71 @@ +{ + "name": "@impler/widget", + "version": "0.1.0", + "author": "knovator", + "license": "MIT", + "private": true, + "scripts": { + "start": "cross-env PORT=3500 BROWSER=none craco start", + "start:dev": "cross-env PORT=3500 BROWSER=none craco start", + "preinstall": "pnpm build", + "prebuild": "rimraf build", + "build": "craco build", + "precommit": "lint-staged", + "eject": "craco eject", + "lint": "eslint src", + "storybook": "start-storybook -p 6006 -s public", + "build-storybook": "build-storybook -s public", + "envsetup": "chmod +x ./env.sh && ./env.sh && mv env-config.js ./public/env-config.js", + "envsetup:docker": "chmod +x ./env.sh && ./env.sh && mv ./env-config.js ./build/env-config.js", + "start:static:build": "pnpm envsetup:docker && http-server build -p 4500 --proxy http://localhost:4500?", + "start:docker": "pnpm build && pnpm start:static:build" + }, + "browserslist": { + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] + }, + "lint-staged": { + "*.{js,jsx,ts,tsx}": [ + "eslint" + ] + }, + "dependencies": { + "@craco/craco": "^6.4.5", + "@emotion/react": "^11.10.5", + "@impler/client": "^0.1.0", + "@impler/shared": "^0.1.0", + "@mantine/core": "^5.6.3", + "@mantine/dropzone": "^5.6.3", + "@mantine/notifications": "5.6.3", + "@storybook/addon-essentials": "^6.5.13", + "@storybook/react": "^6.5.13", + "@tanstack/react-query": "^4.14.5", + "axios": "^0.26.1", + "cross-env": "^7.0.3", + "file-saver": "^2.0.5", + "http-server": "^14.1.1", + "react": "18.2.0", + "react-dom": "18.2.0", + "react-hook-form": "^7.39.1", + "react-router-dom": "^6.4.2", + "react-scripts": "5.0.1", + "rimraf": "^3.0.2", + "web-vitals": "^3.0.4", + "webfontloader": "^1.6.28", + "webpack-dev-server": "^4.11.1" + }, + "devDependencies": { + "@types/file-saver": "^2.0.5", + "@types/react": "^18.0.21", + "@types/react-dom": "^18.0.6", + "typescript": "^4.8.3" + } +} diff --git a/apps/widget/public/arrow.svg b/apps/widget/public/arrow.svg new file mode 100644 index 000000000..b6af7db86 --- /dev/null +++ b/apps/widget/public/arrow.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/apps/widget/public/favicon.ico b/apps/widget/public/favicon.ico new file mode 100644 index 000000000..a11777cc4 Binary files /dev/null and b/apps/widget/public/favicon.ico differ diff --git a/apps/widget/public/index.html b/apps/widget/public/index.html new file mode 100644 index 000000000..3f9e0c0c8 --- /dev/null +++ b/apps/widget/public/index.html @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + Impler Widget + + + + + +
+ + + + + diff --git a/apps/widget/public/logo192.png b/apps/widget/public/logo192.png new file mode 100644 index 000000000..fc44b0a37 Binary files /dev/null and b/apps/widget/public/logo192.png differ diff --git a/apps/widget/public/logo512.png b/apps/widget/public/logo512.png new file mode 100644 index 000000000..a4e47a654 Binary files /dev/null and b/apps/widget/public/logo512.png differ diff --git a/apps/widget/public/manifest.json b/apps/widget/public/manifest.json new file mode 100644 index 000000000..32173287a --- /dev/null +++ b/apps/widget/public/manifest.json @@ -0,0 +1,25 @@ +{ + "short_name": "Impler", + "name": "Impler", + "icons": [ + { + "src": "favicon.ico", + "sizes": "64x64 32x32 24x24 16x16", + "type": "image/x-icon" + }, + { + "src": "logo192.png", + "type": "image/png", + "sizes": "192x192" + }, + { + "src": "logo512.png", + "type": "image/png", + "sizes": "512x512" + } + ], + "start_url": ".", + "display": "standalone", + "theme_color": "#000000", + "background_color": "#ffffff" +} diff --git a/apps/widget/public/robots.txt b/apps/widget/public/robots.txt new file mode 100644 index 000000000..e9e57dc4d --- /dev/null +++ b/apps/widget/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/apps/widget/src/components/App.tsx b/apps/widget/src/components/App.tsx new file mode 100644 index 000000000..306a0b3a1 --- /dev/null +++ b/apps/widget/src/components/App.tsx @@ -0,0 +1,44 @@ +import { BrowserRouter as Router, Route, Routes } from 'react-router-dom'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { NotificationsProvider } from '@mantine/notifications'; +import { MantineProvider } from '@mantine/core'; +import { CONTEXT_PATH, mantineConfig, variables } from '@config'; +import { WidgetShell } from './ApplicationShell'; +import { Container } from './Common/Container'; +import { Widget } from './widget'; + +export function App() { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { + refetchOnWindowFocus: false, + refetchOnReconnect: false, + retry: false, + staleTime: variables.twentyFourHoursInMs, + }, + }, + }); + + return ( + + + + + + + + + + + } + /> + + + + + + ); +} diff --git a/apps/widget/src/components/ApplicationShell.tsx b/apps/widget/src/components/ApplicationShell.tsx new file mode 100644 index 000000000..bfe8abb4c --- /dev/null +++ b/apps/widget/src/components/ApplicationShell.tsx @@ -0,0 +1,52 @@ +import { useEffect } from 'react'; +import { Global } from '@emotion/react'; + +export function WidgetShell({ children }: { children: JSX.Element }) { + const WrapperComponent = inIframe() ? TransparentShell : MockPreviewShell; + + return {children}; +} + +function TransparentShell({ children }: { children: JSX.Element }) { + return
{children}
; +} + +function MockPreviewShell({ children }: { children: JSX.Element }) { + useEffect(() => { + if (document.querySelector('body')) { + (document.querySelector('body') as HTMLBodyElement).style.width = 'auto'; + } + }, []); + + return ( +
+ + {children} +
+ ); +} + +function inIframe() { + try { + return window.self !== window.top; + } catch (e) { + return true; + } +} diff --git a/apps/widget/src/components/Common/Container/Container.tsx b/apps/widget/src/components/Common/Container/Container.tsx new file mode 100644 index 000000000..884753369 --- /dev/null +++ b/apps/widget/src/components/Common/Container/Container.tsx @@ -0,0 +1,107 @@ +import { useEffect, useState, PropsWithChildren } from 'react'; +import * as WebFont from 'webfontloader'; +import { useParams } from 'react-router-dom'; +import { Global } from '@emotion/react'; +import { API_URL, colors } from '@config'; +import { Provider } from '../Provider'; +import { ParentWindow } from '@util'; +import { useAuthentication } from '@hooks/useAuthentication'; +import { ApiService } from '@impler/client'; +import { EventTypesEnum, MessageHandlerDataType } from '@types'; +import { IInitPayload, IShowPayload } from '@impler/shared'; + +let api: ApiService; + +export function Container({ children }: PropsWithChildren<{}>) { + if (!api) api = new ApiService(API_URL); + const { projectId = '' } = useParams<{ projectId: string }>(); + const [showWidget, setShowWidget] = useState(false); + const [primaryPayload, setPrimaryPayload] = useState(); + const [secondaryPayload, setSecondaryPayload] = useState(); + const { isAuthenticated, refetch } = useAuthentication({ api, projectId, template: primaryPayload?.template }); + + useEffect(() => { + WebFont.load({ + google: { + families: ['Lato'], + }, + }); + }, []); + + useEffect(() => { + if (process.env.NODE_ENV === 'test') { + // eslint-disable-next-line + (window as any).initHandler = messageEventHandler; + } + + window.addEventListener('message', messageEventHandler); + + ParentWindow.Ready(); + + return () => window.removeEventListener('message', messageEventHandler); + }, []); + + function messageEventHandler({ data }: { data?: MessageHandlerDataType }) { + if (data && data.type === EventTypesEnum.INIT_IFRAME) { + setPrimaryPayload(data.value); + if (data.value?.accessToken) { + api.setAuthorizationToken(data.value.accessToken); + } + refetch(); + } + if (data && data.type === EventTypesEnum.SHOW_WIDGET) { + setShowWidget(true); + setSecondaryPayload(data.value); + } + } + + if (!isAuthenticated || !showWidget) return null; + + return ( + <> + + {primaryPayload ? ( + + {children} + + ) : null} + + ); +} diff --git a/apps/widget/src/components/Common/Container/index.ts b/apps/widget/src/components/Common/Container/index.ts new file mode 100644 index 000000000..8a1103ffa --- /dev/null +++ b/apps/widget/src/components/Common/Container/index.ts @@ -0,0 +1 @@ +export * from './Container'; diff --git a/apps/widget/src/components/Common/Footer/Footer.tsx b/apps/widget/src/components/Common/Footer/Footer.tsx new file mode 100644 index 000000000..a0a9aaa0c --- /dev/null +++ b/apps/widget/src/components/Common/Footer/Footer.tsx @@ -0,0 +1,60 @@ +import { Group } from '@mantine/core'; +import { Button } from '@ui/Button'; +import { TEXTS } from '@config'; +import { PhasesEum } from '@types'; + +interface IFooterProps { + active: PhasesEum; + primaryButtonLoading?: boolean; + secondaryButtonLoading?: boolean; + onPrevClick: () => void; + onNextClick: () => void; +} + +export function Footer(props: IFooterProps) { + const { active, onNextClick, onPrevClick, primaryButtonLoading, secondaryButtonLoading } = props; + + const FooterActions = { + [PhasesEum.UPLOAD]: ( + + ), + [PhasesEum.MAPPING]: ( + <> + + + + ), + [PhasesEum.REVIEW]: ( + <> + + + + ), + [PhasesEum.COMPLETE]: ( + <> + + + + ), + }; + + return ( + + {FooterActions[active]} + + ); +} diff --git a/apps/widget/src/components/Common/Footer/index.ts b/apps/widget/src/components/Common/Footer/index.ts new file mode 100644 index 000000000..ddcc5a9cd --- /dev/null +++ b/apps/widget/src/components/Common/Footer/index.ts @@ -0,0 +1 @@ +export * from './Footer'; diff --git a/apps/widget/src/components/Common/Heading/Heading.tsx b/apps/widget/src/components/Common/Heading/Heading.tsx new file mode 100644 index 000000000..58e9cc30c --- /dev/null +++ b/apps/widget/src/components/Common/Heading/Heading.tsx @@ -0,0 +1,41 @@ +import { Group, Title } from '@mantine/core'; +import { Stepper } from '@ui/Stepper'; +import { TEXTS } from '@config'; +import { PhasesEum } from '@types'; + +interface IHeadingProps { + active: PhasesEum; +} + +const Titles = { + [PhasesEum.UPLOAD]: TEXTS.TITLES.UPLOAD, + [PhasesEum.MAPPING]: TEXTS.TITLES.MAPPING, + [PhasesEum.REVIEW]: TEXTS.TITLES.REVIEW, + [PhasesEum.COMPLETE]: TEXTS.TITLES.COMPLETE, +}; + +const Steps = [ + { + label: TEXTS.STEPS.UPLOAD, + }, + { + label: TEXTS.STEPS.MAPPING, + }, + { + label: TEXTS.STEPS.REVIEW, + }, + { + label: TEXTS.STEPS.COMPLETE, + }, +]; + +export function Heading(props: IHeadingProps) { + const { active } = props; + + return ( + + {Titles[active]} + + + ); +} diff --git a/apps/widget/src/components/Common/Heading/index.ts b/apps/widget/src/components/Common/Heading/index.ts new file mode 100644 index 000000000..6406e7b07 --- /dev/null +++ b/apps/widget/src/components/Common/Heading/index.ts @@ -0,0 +1 @@ +export * from './Heading'; diff --git a/apps/widget/src/components/Common/Layout/Layout.tsx b/apps/widget/src/components/Common/Layout/Layout.tsx new file mode 100644 index 000000000..c8f4919ce --- /dev/null +++ b/apps/widget/src/components/Common/Layout/Layout.tsx @@ -0,0 +1,21 @@ +import { PropsWithChildren } from 'react'; +import { Heading } from 'components/Common/Heading'; +import useStyles from './Styles'; +import { PhasesEum } from '@types'; + +interface ILayoutProps { + active: PhasesEum; +} + +export function Layout(props: PropsWithChildren) { + const { classes } = useStyles(); + const { children, active } = props; + + return ( +
+ {/* Heading */} + +
{children}
+
+ ); +} diff --git a/apps/widget/src/components/Common/Layout/Styles.tsx b/apps/widget/src/components/Common/Layout/Styles.tsx new file mode 100644 index 000000000..f7726c22e --- /dev/null +++ b/apps/widget/src/components/Common/Layout/Styles.tsx @@ -0,0 +1,28 @@ +/* eslint-disable @typescript-eslint/no-unused-vars, no-unused-vars */ +import { createStyles, MantineTheme } from '@mantine/core'; + +export const getRootStyles = (theme: MantineTheme): React.CSSProperties => ({ + paddingRight: 24, + paddingLeft: 24, + paddingBottom: 24, + border: '1px solid transparent', + display: 'flex', + flexDirection: 'column', + height: '100%', +}); + +export const getContainerStyles = (theme: MantineTheme): React.CSSProperties => ({ + flexDirection: 'column', + width: '100%', + alignItems: 'unset', + flexGrow: 1, + display: 'flex', + gap: theme.spacing.md, +}); + +export default createStyles((theme: MantineTheme, params, getRef): Record => { + return { + root: getRootStyles(theme), + container: getContainerStyles(theme), + }; +}); diff --git a/apps/widget/src/components/Common/Layout/index.ts b/apps/widget/src/components/Common/Layout/index.ts new file mode 100644 index 000000000..9877e7f4a --- /dev/null +++ b/apps/widget/src/components/Common/Layout/index.ts @@ -0,0 +1 @@ +export * from './Layout'; diff --git a/apps/widget/src/components/Common/Provider/Provider.tsx b/apps/widget/src/components/Common/Provider/Provider.tsx new file mode 100644 index 000000000..45a1e698b --- /dev/null +++ b/apps/widget/src/components/Common/Provider/Provider.tsx @@ -0,0 +1,34 @@ +import { PropsWithChildren } from 'react'; +import { ApiService } from '@impler/client'; +import ImplerContextProvider from '@store/impler.context'; +import APIContextProvider from '@store/api.context'; +import AppContextProvider from '@store/app.context'; + +interface IProviderProps { + // api-context + api: ApiService; + // impler-context + projectId: string; + template?: string; + accessToken?: string; + extra?: string; + authHeaderValue?: string; +} + +export function Provider(props: PropsWithChildren) { + const { api, projectId, template, accessToken, extra, authHeaderValue, children } = props; + + return ( + + + {children} + + + ); +} diff --git a/apps/widget/src/components/Common/Provider/index.ts b/apps/widget/src/components/Common/Provider/index.ts new file mode 100644 index 000000000..505094109 --- /dev/null +++ b/apps/widget/src/components/Common/Provider/index.ts @@ -0,0 +1 @@ +export * from './Provider'; diff --git a/apps/widget/src/components/widget/Phases/ConfirmModal/ConfirmModal.tsx b/apps/widget/src/components/widget/Phases/ConfirmModal/ConfirmModal.tsx new file mode 100644 index 000000000..7b4e6714f --- /dev/null +++ b/apps/widget/src/components/widget/Phases/ConfirmModal/ConfirmModal.tsx @@ -0,0 +1,36 @@ +import { Warning } from '@icons'; +import { colors, TEXTS } from '@config'; +import { Button } from '@ui/Button'; +import { Group, Modal as MantineModal, Text, Title } from '@mantine/core'; +import { replaceVariablesInString, numberFormatter } from '@impler/shared'; + +interface IConfirmModalProps { + opened: boolean; + wrongDataCount: number; + onClose: () => void; + onConfirm: (exempt: boolean) => void; +} + +export function ConfirmModal(props: IConfirmModalProps) { + const { opened, onClose, wrongDataCount, onConfirm } = props; + + return ( + + + + + {replaceVariablesInString(TEXTS.CONFIRM_MODAL.title, { count: numberFormatter(wrongDataCount) })} + + + {TEXTS.CONFIRM_MODAL.subTitle} + + + + + + + + ); +} diff --git a/apps/widget/src/components/widget/Phases/ConfirmModal/index.ts b/apps/widget/src/components/widget/Phases/ConfirmModal/index.ts new file mode 100644 index 000000000..353d4b1ef --- /dev/null +++ b/apps/widget/src/components/widget/Phases/ConfirmModal/index.ts @@ -0,0 +1 @@ +export * from './ConfirmModal'; diff --git a/apps/widget/src/components/widget/Phases/Phase1/Phase1.tsx b/apps/widget/src/components/widget/Phases/Phase1/Phase1.tsx new file mode 100644 index 000000000..b0228c40a --- /dev/null +++ b/apps/widget/src/components/widget/Phases/Phase1/Phase1.tsx @@ -0,0 +1,97 @@ +import { TEXTS } from '@config'; +import { Select } from '@ui/Select'; +import { Button } from '@ui/Button'; +import { Dropzone } from '@ui/Dropzone'; +import { LoadingOverlay } from '@ui/LoadingOverlay'; +import { Group } from '@mantine/core'; +import { Download } from '@icons'; +import useStyles from './Styles'; +import { Footer } from 'components/Common/Footer'; +import { usePhase1 } from '@hooks/Phase1/usePhase1'; +import { Controller } from 'react-hook-form'; +import { PhasesEum } from '@types'; + +interface IPhase1Props { + onNextClick: () => void; +} + +export function Phase1(props: IPhase1Props) { + const { classes } = useStyles(); + const { onNextClick: goNext } = props; + const { + showSelectTemplate, + onSubmit, + trigger, + control, + templates, + isInitialDataLoaded, + isUploadLoading, + onDownload, + isDownloadInProgress, + } = usePhase1({ + goNext, + }); + + return ( + <> + + + {showSelectTemplate && ( + ( + } + clearable + size={size} + searchable={searchable} + dropdownComponent="div" + onChange={(selectedValue) => onChange && onChange(selectedValue)} + ref={ref} + /> + + + {value ? : null} + {value ? mappingSucceedText : mappingFailedText} + + + ); +}); diff --git a/apps/widget/src/design-system/MappingItem/index.ts b/apps/widget/src/design-system/MappingItem/index.ts new file mode 100644 index 000000000..b8ba3fe12 --- /dev/null +++ b/apps/widget/src/design-system/MappingItem/index.ts @@ -0,0 +1 @@ +export * from './MappingItem'; diff --git a/apps/widget/src/design-system/Modal/Modal.stories.tsx b/apps/widget/src/design-system/Modal/Modal.stories.tsx new file mode 100644 index 000000000..16928e2c0 --- /dev/null +++ b/apps/widget/src/design-system/Modal/Modal.stories.tsx @@ -0,0 +1,36 @@ +import { ComponentStory, ComponentMeta } from '@storybook/react'; +import { Modal } from './Modal'; + +export default { + title: 'Modal', + component: Modal, + argTypes: { + size: { + control: { + type: 'select', + options: ['xs', 'sm', 'md', 'lg', 'xl'], + }, + }, + padding: { + control: { + type: 'select', + options: ['xs', 'sm', 'md', 'lg', 'xl'], + }, + }, + onClose: { + action: 'onClose', + }, + }, +} as ComponentMeta; + +const Template: ComponentStory = (args) => ; + +export const Simple = Template.bind({}); +Simple.args = { + title: 'Modal title', + size: '100%', + centered: true, + opened: true, + overflow: 'inside', + children: 'Content', +}; diff --git a/apps/widget/src/design-system/Modal/Modal.style.ts b/apps/widget/src/design-system/Modal/Modal.style.ts new file mode 100644 index 000000000..148abd551 --- /dev/null +++ b/apps/widget/src/design-system/Modal/Modal.style.ts @@ -0,0 +1,29 @@ +/* eslint-disable @typescript-eslint/no-unused-vars, no-unused-vars */ +import { createStyles, MantineTheme } from '@mantine/core'; +import React from 'react'; +import { colors } from '../../config/colors.config'; + +export const getHeaderStyles = (theme: MantineTheme): React.CSSProperties => ({ + marginBottom: theme.spacing.xs, + marginRight: 0, +}); + +export const getModalStyles = (theme: MantineTheme): React.CSSProperties => ({ + padding: '100px', + height: 'calc(100vh - 20%)', + width: 'calc(100vw - 20%)', + display: 'flex', + flexDirection: 'column', +}); + +export const getModalBodyStyles = (theme: MantineTheme): React.CSSProperties => ({ + flexGrow: 1, +}); + +export default createStyles((theme: MantineTheme, params, getRef): Record => { + return { + header: getHeaderStyles(theme), + modal: getModalStyles(theme), + body: getModalBodyStyles(theme), + }; +}); diff --git a/apps/widget/src/design-system/Modal/Modal.tsx b/apps/widget/src/design-system/Modal/Modal.tsx new file mode 100644 index 000000000..ec5c94641 --- /dev/null +++ b/apps/widget/src/design-system/Modal/Modal.tsx @@ -0,0 +1,34 @@ +import { PropsWithChildren } from 'react'; +import { Modal as MantineModal } from '@mantine/core'; +import useStyles from './Modal.style'; + +interface IModalProps extends JSX.ElementChildrenAttribute { + title?: string; + opened: boolean; + centered?: boolean; + onClose: () => void; + overflow?: 'inside' | 'outside'; + size?: 'sm' | 'md' | 'lg' | '100%' | number; +} + +export function Modal(props: PropsWithChildren) { + const { children, onClose, opened, title, centered = true } = props; + const { classes } = useStyles(); + + return ( + + {children} + + ); +} diff --git a/apps/widget/src/design-system/Modal/index.ts b/apps/widget/src/design-system/Modal/index.ts new file mode 100644 index 000000000..cb89ee178 --- /dev/null +++ b/apps/widget/src/design-system/Modal/index.ts @@ -0,0 +1 @@ +export * from './Modal'; diff --git a/apps/widget/src/design-system/Pagination/Pagination.stories.tsx b/apps/widget/src/design-system/Pagination/Pagination.stories.tsx new file mode 100644 index 000000000..4aa75f0ec --- /dev/null +++ b/apps/widget/src/design-system/Pagination/Pagination.stories.tsx @@ -0,0 +1,20 @@ +import { ComponentStory, ComponentMeta } from '@storybook/react'; +import { Pagination } from './Pagination'; + +export default { + title: 'Pagination', + component: Pagination, + argTypes: { + onChange: { + action: 'onChange', + }, + }, +} as ComponentMeta; + +const Template: ComponentStory = (args) => ; + +export const Simple = Template.bind({}); +Simple.args = { + total: 10, + page: 1, +}; diff --git a/apps/widget/src/design-system/Pagination/Pagination.tsx b/apps/widget/src/design-system/Pagination/Pagination.tsx new file mode 100644 index 000000000..736ed0cbf --- /dev/null +++ b/apps/widget/src/design-system/Pagination/Pagination.tsx @@ -0,0 +1,19 @@ +import { Pagination as MantinePagination, Group } from '@mantine/core'; + +interface IPaginationProps { + page?: number; + total: number; + size?: 'sm' | 'md'; + onChange?: (page: number) => void; +} + +export function Pagination(props: IPaginationProps) { + const defaultPage = 1; + const { total, page = defaultPage, size = 'md', onChange } = props; + + return ( + + + + ); +} diff --git a/apps/widget/src/design-system/Pagination/index.ts b/apps/widget/src/design-system/Pagination/index.ts new file mode 100644 index 000000000..e016c96b7 --- /dev/null +++ b/apps/widget/src/design-system/Pagination/index.ts @@ -0,0 +1 @@ +export * from './Pagination'; diff --git a/apps/widget/src/design-system/Select/Select.stories.tsx b/apps/widget/src/design-system/Select/Select.stories.tsx new file mode 100644 index 000000000..dc71189ed --- /dev/null +++ b/apps/widget/src/design-system/Select/Select.stories.tsx @@ -0,0 +1,16 @@ +import { ComponentStory, ComponentMeta } from '@storybook/react'; +import { Select } from './Select'; + +export default { + title: 'Select', + component: Select, +} as ComponentMeta; + +const Template: ComponentStory = (args) =>