From 978e0146b3fc5c7f6acc31f54b2ce70b16547d7a Mon Sep 17 00:00:00 2001 From: pauldechorgnat <23296889+pauldechorgnat@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:17:24 +0100 Subject: [PATCH 1/9] TEST: suppression du build hors Gitlab --- .gitlab-ci.yml | 93 +++++++++++++++++--------------------------------- 1 file changed, 32 insertions(+), 61 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b66a098d6..5b26939c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: docker:20.10.22 +image: docker:20.10.22 services: - docker:20.10.22-dind @@ -6,60 +6,35 @@ variables: dev: dev re7: preprod master: prod - docker_backend_image: "$CI_REGISTRY/cour-de-cassation/label/backend:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" - docker_client_image: "$CI_REGISTRY/cour-de-cassation/label/client:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" + docker_judilibre_explore_backend_image: "$CI_REGISTRY/cour-de-cassation/judilibre-explore/backend:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" + docker_judilibre_explore_client_image: "$CI_REGISTRY/cour-de-cassation/judilibre-explore/client:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" stages: - test - build - deploy -# Partie à compléter par les développeurs label -# test: -# stage: test -# services: -# - docker:20.10.22 -# script: -# - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY -# - docker build -# --build-arg http_proxy=$HTTP_PROXY -# --build-arg https_proxy=$HTTPS_PROXY -# #--target test -# -t label-test -# # docker run label-back-test /bin/sh -c "yarn install -- ; yarn compile ...." -# # -> ici on lance un conteneur un par un, donc les données générées par le yarn install ne sont pas sauvegardées. -# - docker run label-test yarn install -# - docker run label-test yarn compile -# - docker run label-test yarn test -# - docker run label-test yarn lint -# only: -# - master -# - re7 -# - dev - build_label_backend: stage: build variables: HTTP_PROXY: $HTTP_PROXY_DEV HTTPS_PROXY: $HTTPS_PROXY_DEV - script: + script: - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - - echo "$HTTP_PROXY" - - echo "$HTTPS_PROXY" - - export NODE_OPTIONS=--openssl-legacy-provider - - apk update - - apk add nodejs yarn git npm - - yarn config set proxy $HTTP_PROXY - - yarn config set https-proxy $HTTPS_PROXY - - yarn install -- - - yarn compile - - CI=false yarn buildClient # from "ugly workaround" in github workflows + # - export NODE_OPTIONS=--openssl-legacy-provider + # - apk update + # - apk add nodejs yarn git npm + # - yarn config set proxy $HTTP_PROXY + # - yarn config set https-proxy $HTTPS_PROXY + # - yarn install -- + # - yarn compile + # - CI=false yarn buildClient # from "ugly workaround" in github workflows - docker build - --build-arg http_proxy=$HTTP_PROXY - --build-arg https_proxy=$HTTPS_PROXY - --target label-backend - -f Dockerfile.label-backend - -t $docker_backend_image . + --build-arg http_proxy=$HTTP_PROXY + --build-arg https_proxy=$HTTPS_PROXY + --target label-backend + -f Dockerfile.label-backend + -t $docker_backend_image . - docker push $docker_backend_image only: - master @@ -68,7 +43,6 @@ build_label_backend: tags: - docker - build_label_client: stage: build variables: @@ -76,24 +50,24 @@ build_label_client: HTTPS_PROXY: $HTTPS_PROXY_DEV script: - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - - echo "$HTTP_PROXY" - - echo "$HTTPS_PROXY" - - export NODE_OPTIONS=--openssl-legacy-provider - - apk update - - apk add nodejs yarn git npm - - yarn config set proxy $HTTP_PROXY - - yarn config set https-proxy $HTTPS_PROXY - - yarn install -- - - yarn compile - - CI=false yarn buildClient # from "ugly workaround" in github workflows + # - echo "$HTTP_PROXY" + # - echo "$HTTPS_PROXY" + # - export NODE_OPTIONS=--openssl-legacy-provider + # - apk update + # - apk add nodejs yarn git npm + # - yarn config set proxy $HTTP_PROXY + # - yarn config set https-proxy $HTTPS_PROXY + # - yarn install -- + # - yarn compile + # - CI=false yarn buildClient # from "ugly workaround" in github workflows # - yarn test # - yarn lint - docker build - --build-arg http_proxy=$HTTP_PROXY - --build-arg https_proxy=$HTTPS_PROXY - --target label-client - -f Dockerfile.label-client - -t $docker_client_image . + --build-arg http_proxy=$HTTP_PROXY + --build-arg https_proxy=$HTTPS_PROXY + --target label-client + -f Dockerfile.label-client + -t $docker_client_image . - docker push $docker_client_image only: - master @@ -123,7 +97,6 @@ deploy_label_backend: # dependencies: # - build_label_backend - deploy_label_client: stage: deploy image: alpine/ansible:latest @@ -144,5 +117,3 @@ deploy_label_client: - docker # dependencies: # - build_label_client - - From bd9b48ffd1c7bb2dc2f8424e36ebc80b40806e1d Mon Sep 17 00:00:00 2001 From: pauldechorgnat <23296889+pauldechorgnat@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:19:24 +0100 Subject: [PATCH 2/9] TEST: modification du nom des images --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b26939c4..e0e9393de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,8 +6,8 @@ variables: dev: dev re7: preprod master: prod - docker_judilibre_explore_backend_image: "$CI_REGISTRY/cour-de-cassation/judilibre-explore/backend:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" - docker_judilibre_explore_client_image: "$CI_REGISTRY/cour-de-cassation/judilibre-explore/client:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" + docker_label_backend_image: "$CI_REGISTRY/cour-de-cassation/label/backend:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" + docker_label_client_image: "$CI_REGISTRY/cour-de-cassation/label/client:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" stages: - test From eab86a8cca79771dee80b4c49c3ecfc617246fe9 Mon Sep 17 00:00:00 2001 From: YATERA Boubacar Date: Thu, 25 Jan 2024 16:51:16 +0100 Subject: [PATCH 3/9] updating the cleanDuplicatedDocument to get the last version of duplicated doc by creationDate --- .../cleanDuplicatedDocuments.spec.ts | 24 +++++++++++++++---- .../cleanDuplicatedDocuments.ts | 10 ++++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts index 2efeb1223..6923f10c2 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts @@ -5,26 +5,42 @@ import { cleanDuplicatedDocuments } from './cleanDuplicatedDocuments'; describe('cleanDuplicatedDocuments', () => { it('should clean the DuplicatedDocuments', async () => { const firstDocument = documentModule.generator.generate(); - const secondDocument = documentModule.generator.generate(); + const secondDocument = documentModule.generator.generate({ + creationDate: 1274657452000 + }); const secondDocumentWithHigherStatus = documentModule.generator.generate({ ...secondDocument, + creationDate: 1674657452000, _id: idModule.lib.buildId(), status: 'done', }); + const documentRepository = buildDocumentRepository(); await documentRepository.insertMany([ firstDocument, secondDocument, - secondDocumentWithHigherStatus, + secondDocumentWithHigherStatus ]); await cleanDuplicatedDocuments(); const fetchedDocuments = await documentRepository.findAll(); - const fetchedIds = fetchedDocuments.map((u) => u._id).sort(); + const fetchedIds = fetchedDocuments.map((u) => { + if (u.creationDate !== undefined) { + console.log('log creation date', new Date(u.creationDate).getFullYear()); + console.log('ids --------', u._id) + } + return u._id; + }).sort(); const expectedIds = [firstDocument, secondDocumentWithHigherStatus] - .map((u) => u._id) + .map((u) => { + if (u.creationDate !== undefined) { + console.log('log expectedIds', new Date(u.creationDate).getFullYear()); + console.log('expectedIds --------', u._id) + } + return u._id; + }) .sort(); expect(fetchedIds).toEqual(expectedIds); }); diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts index 8539e1d9c..3410920c1 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts @@ -8,7 +8,7 @@ import { logger } from '../../../utils'; export { cleanDuplicatedDocuments }; /** - * Delete all doubled documents (same source, same documentNumber, same text) + * Delete all doubled documents (same source, same documentNumber, same text and creationDate ) get recent documents by creationDate */ async function cleanDuplicatedDocuments() { logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'START' }); @@ -28,8 +28,13 @@ async function cleanDuplicatedDocuments() { for (let index = 0, l = documents.length - 1; index < l; index++) { const currentDocument = sortedDocuments[index]; const nextDocument = sortedDocuments[index + 1]; + // check if document is same, if same return the last version document if (areDocumentsIdentical(currentDocument, nextDocument)) { - documentsToDelete.push(nextDocument); + if (currentDocument.creationDate && nextDocument.creationDate != undefined && currentDocument.creationDate > nextDocument.creationDate) { + documentsToDelete.push(nextDocument); + } else { + documentsToDelete.push(currentDocument); + } } } @@ -44,6 +49,7 @@ async function cleanDuplicatedDocuments() { logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'DONE' }); } +// function areDocumentsIdentical( document1: documentType, document2: documentType, From 3637a955ca8838c658e793820a50f052a75f7ae6 Mon Sep 17 00:00:00 2001 From: YATERA Boubacar Date: Thu, 25 Jan 2024 16:57:28 +0100 Subject: [PATCH 4/9] deleting console.log --- .../cleanDocuments/cleanDuplicatedDocuments.spec.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts index 6923f10c2..b28bdbffa 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts @@ -27,18 +27,10 @@ describe('cleanDuplicatedDocuments', () => { const fetchedDocuments = await documentRepository.findAll(); const fetchedIds = fetchedDocuments.map((u) => { - if (u.creationDate !== undefined) { - console.log('log creation date', new Date(u.creationDate).getFullYear()); - console.log('ids --------', u._id) - } return u._id; }).sort(); const expectedIds = [firstDocument, secondDocumentWithHigherStatus] .map((u) => { - if (u.creationDate !== undefined) { - console.log('log expectedIds', new Date(u.creationDate).getFullYear()); - console.log('expectedIds --------', u._id) - } return u._id; }) .sort(); From 3d1f5bb9739832ee3fd862df309a0413414520fa Mon Sep 17 00:00:00 2001 From: YATERA Boubacar Date: Thu, 25 Jan 2024 16:59:18 +0100 Subject: [PATCH 5/9] make yarn fix and lint --- .../cleanDocuments/cleanDuplicatedDocuments.spec.ts | 12 +++++++----- .../cleanDocuments/cleanDuplicatedDocuments.ts | 10 +++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts index b28bdbffa..b89e502f0 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts @@ -6,7 +6,7 @@ describe('cleanDuplicatedDocuments', () => { it('should clean the DuplicatedDocuments', async () => { const firstDocument = documentModule.generator.generate(); const secondDocument = documentModule.generator.generate({ - creationDate: 1274657452000 + creationDate: 1274657452000, }); const secondDocumentWithHigherStatus = documentModule.generator.generate({ ...secondDocument, @@ -19,16 +19,18 @@ describe('cleanDuplicatedDocuments', () => { await documentRepository.insertMany([ firstDocument, secondDocument, - secondDocumentWithHigherStatus + secondDocumentWithHigherStatus, ]); await cleanDuplicatedDocuments(); const fetchedDocuments = await documentRepository.findAll(); - const fetchedIds = fetchedDocuments.map((u) => { - return u._id; - }).sort(); + const fetchedIds = fetchedDocuments + .map((u) => { + return u._id; + }) + .sort(); const expectedIds = [firstDocument, secondDocumentWithHigherStatus] .map((u) => { return u._id; diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts index 3410920c1..a64a26dac 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts @@ -8,7 +8,7 @@ import { logger } from '../../../utils'; export { cleanDuplicatedDocuments }; /** - * Delete all doubled documents (same source, same documentNumber, same text and creationDate ) get recent documents by creationDate + * Delete all doubled documents (same source, same documentNumber, same text and creationDate ) get recent documents by creationDate */ async function cleanDuplicatedDocuments() { logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'START' }); @@ -30,7 +30,11 @@ async function cleanDuplicatedDocuments() { const nextDocument = sortedDocuments[index + 1]; // check if document is same, if same return the last version document if (areDocumentsIdentical(currentDocument, nextDocument)) { - if (currentDocument.creationDate && nextDocument.creationDate != undefined && currentDocument.creationDate > nextDocument.creationDate) { + if ( + currentDocument.creationDate && + nextDocument.creationDate != undefined && + currentDocument.creationDate > nextDocument.creationDate + ) { documentsToDelete.push(nextDocument); } else { documentsToDelete.push(currentDocument); @@ -49,7 +53,7 @@ async function cleanDuplicatedDocuments() { logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'DONE' }); } -// +// function areDocumentsIdentical( document1: documentType, document2: documentType, From 62b3606f66f9c9718e6f2a51f4187ba1a06c5397 Mon Sep 17 00:00:00 2001 From: Antoine Jeanneney Date: Thu, 25 Jan 2024 17:22:28 +0100 Subject: [PATCH 6/9] rollback gitlabci --- .gitlab-ci.yml | 93 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e0e9393de..b66a098d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: docker:20.10.22 +image: docker:20.10.22 services: - docker:20.10.22-dind @@ -6,35 +6,60 @@ variables: dev: dev re7: preprod master: prod - docker_label_backend_image: "$CI_REGISTRY/cour-de-cassation/label/backend:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" - docker_label_client_image: "$CI_REGISTRY/cour-de-cassation/label/client:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" + docker_backend_image: "$CI_REGISTRY/cour-de-cassation/label/backend:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" + docker_client_image: "$CI_REGISTRY/cour-de-cassation/label/client:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA" stages: - test - build - deploy +# Partie à compléter par les développeurs label +# test: +# stage: test +# services: +# - docker:20.10.22 +# script: +# - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY +# - docker build +# --build-arg http_proxy=$HTTP_PROXY +# --build-arg https_proxy=$HTTPS_PROXY +# #--target test +# -t label-test +# # docker run label-back-test /bin/sh -c "yarn install -- ; yarn compile ...." +# # -> ici on lance un conteneur un par un, donc les données générées par le yarn install ne sont pas sauvegardées. +# - docker run label-test yarn install +# - docker run label-test yarn compile +# - docker run label-test yarn test +# - docker run label-test yarn lint +# only: +# - master +# - re7 +# - dev + build_label_backend: stage: build variables: HTTP_PROXY: $HTTP_PROXY_DEV HTTPS_PROXY: $HTTPS_PROXY_DEV - script: + script: - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - # - export NODE_OPTIONS=--openssl-legacy-provider - # - apk update - # - apk add nodejs yarn git npm - # - yarn config set proxy $HTTP_PROXY - # - yarn config set https-proxy $HTTPS_PROXY - # - yarn install -- - # - yarn compile - # - CI=false yarn buildClient # from "ugly workaround" in github workflows + - echo "$HTTP_PROXY" + - echo "$HTTPS_PROXY" + - export NODE_OPTIONS=--openssl-legacy-provider + - apk update + - apk add nodejs yarn git npm + - yarn config set proxy $HTTP_PROXY + - yarn config set https-proxy $HTTPS_PROXY + - yarn install -- + - yarn compile + - CI=false yarn buildClient # from "ugly workaround" in github workflows - docker build - --build-arg http_proxy=$HTTP_PROXY - --build-arg https_proxy=$HTTPS_PROXY - --target label-backend - -f Dockerfile.label-backend - -t $docker_backend_image . + --build-arg http_proxy=$HTTP_PROXY + --build-arg https_proxy=$HTTPS_PROXY + --target label-backend + -f Dockerfile.label-backend + -t $docker_backend_image . - docker push $docker_backend_image only: - master @@ -43,6 +68,7 @@ build_label_backend: tags: - docker + build_label_client: stage: build variables: @@ -50,24 +76,24 @@ build_label_client: HTTPS_PROXY: $HTTPS_PROXY_DEV script: - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - # - echo "$HTTP_PROXY" - # - echo "$HTTPS_PROXY" - # - export NODE_OPTIONS=--openssl-legacy-provider - # - apk update - # - apk add nodejs yarn git npm - # - yarn config set proxy $HTTP_PROXY - # - yarn config set https-proxy $HTTPS_PROXY - # - yarn install -- - # - yarn compile - # - CI=false yarn buildClient # from "ugly workaround" in github workflows + - echo "$HTTP_PROXY" + - echo "$HTTPS_PROXY" + - export NODE_OPTIONS=--openssl-legacy-provider + - apk update + - apk add nodejs yarn git npm + - yarn config set proxy $HTTP_PROXY + - yarn config set https-proxy $HTTPS_PROXY + - yarn install -- + - yarn compile + - CI=false yarn buildClient # from "ugly workaround" in github workflows # - yarn test # - yarn lint - docker build - --build-arg http_proxy=$HTTP_PROXY - --build-arg https_proxy=$HTTPS_PROXY - --target label-client - -f Dockerfile.label-client - -t $docker_client_image . + --build-arg http_proxy=$HTTP_PROXY + --build-arg https_proxy=$HTTPS_PROXY + --target label-client + -f Dockerfile.label-client + -t $docker_client_image . - docker push $docker_client_image only: - master @@ -97,6 +123,7 @@ deploy_label_backend: # dependencies: # - build_label_backend + deploy_label_client: stage: deploy image: alpine/ansible:latest @@ -117,3 +144,5 @@ deploy_label_client: - docker # dependencies: # - build_label_client + + From 745261ae6c25a7916b23ba33793891dad929c98f Mon Sep 17 00:00:00 2001 From: Antoine Jeanneney Date: Wed, 31 Jan 2024 10:41:59 +0100 Subject: [PATCH 7/9] run clean documents before export scripts --- .../roles/deploy_backend/defaults/main/defaults.yml | 2 +- .../src/scripts/cleanOrphansTreatments.ts | 12 ------------ .../src/scripts/exportAllRejectedDocuments.ts | 6 ++++++ .../src/scripts/exportAllTreatedDocuments.ts | 6 ++++++ .../src/scripts/exportTreatedDocumentsSince.ts | 6 ++++++ .../src/scripts/exportTreatedPublishableDocuments.ts | 6 ++++++ packages/generic/backend/src/app/buildBackend.ts | 5 ----- .../cleanDocuments/cleanDuplicatedDocuments.ts | 5 ++--- packages/generic/backend/src/app/scripts/index.ts | 2 -- 9 files changed, 27 insertions(+), 23 deletions(-) delete mode 100644 packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts diff --git a/ansible/roles/deploy_backend/defaults/main/defaults.yml b/ansible/roles/deploy_backend/defaults/main/defaults.yml index 21aab5535..97e0410e6 100644 --- a/ansible/roles/deploy_backend/defaults/main/defaults.yml +++ b/ansible/roles/deploy_backend/defaults/main/defaults.yml @@ -33,7 +33,7 @@ jobs: active_deadline_seconds: 1800 command: "dist/scripts/importChainedDocumentsFromSder.js --count 500 --threshold 1500" - name: "export-j-4" - schedule: "30 17 * * *" + schedule: "25 17 * * *" successful_jobs_history_limit: 7 failed_jobs_history_limit: 7 backoff_limit: 2 diff --git a/packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts b/packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts deleted file mode 100644 index d0e88c30d..000000000 --- a/packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { buildBackend } from '@label/backend'; -import { parametersHandler } from '../lib/parametersHandler'; - -(async () => { - const { environment, settings } = await parametersHandler.getParameters(); - const backend = buildBackend(environment, settings); - - await backend.runScript( - () => backend.scripts.cleanOrphansTreatments.run(), - backend.scripts.cleanOrphansTreatments.option, - ); -})(); diff --git a/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts b/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts index ab10791ad..de0c153aa 100644 --- a/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts +++ b/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts @@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler'; (async () => { const { environment, settings } = await parametersHandler.getParameters(); const backend = buildBackend(environment, settings); + + await backend.runScript( + () => backend.scripts.cleanDocuments.run(), + backend.scripts.cleanDocuments.option, + ); + const sderExporter = buildSderExporter(environment, settings); backend.runScript( () => sderExporter.exportAllRejectedDocuments(environment), diff --git a/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts b/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts index bc458871d..ef924c50b 100644 --- a/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts +++ b/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts @@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler'; (async () => { const { environment, settings } = await parametersHandler.getParameters(); const backend = buildBackend(environment, settings); + + await backend.runScript( + () => backend.scripts.cleanDocuments.run(), + backend.scripts.cleanDocuments.option, + ); + const sderExporter = buildSderExporter(environment, settings); backend.runScript(() => sderExporter.exportAllTreatedDocuments(environment), { diff --git a/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts b/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts index bf29fc3d7..cdb91c4de 100644 --- a/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts +++ b/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts @@ -7,6 +7,12 @@ import { parametersHandler } from '../lib/parametersHandler'; const { environment, settings } = await parametersHandler.getParameters(); const { days } = parseArgv(); const backend = buildBackend(environment, settings); + + await backend.runScript( + () => backend.scripts.cleanDocuments.run(), + backend.scripts.cleanDocuments.option, + ); + const sderExporter = buildSderExporter(environment, settings); backend.runScript( () => sderExporter.exportTreatedDocumentsSince(days, environment), diff --git a/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts b/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts index 23e2465e7..df859f892 100644 --- a/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts +++ b/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts @@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler'; (async () => { const { environment, settings } = await parametersHandler.getParameters(); const backend = buildBackend(environment, settings); + + await backend.runScript( + () => backend.scripts.cleanDocuments.run(), + backend.scripts.cleanDocuments.option, + ); + const sderExporter = buildSderExporter(environment, settings); backend.runScript( diff --git a/packages/generic/backend/src/app/buildBackend.ts b/packages/generic/backend/src/app/buildBackend.ts index 81bc6a540..57e01c352 100644 --- a/packages/generic/backend/src/app/buildBackend.ts +++ b/packages/generic/backend/src/app/buildBackend.ts @@ -21,7 +21,6 @@ import { resetUntreatedDocumentsForTest, revertOneMigration, runNewMigrations, - cleanOrphansTreatments, } from './scripts'; export { buildBackend }; @@ -111,10 +110,6 @@ function buildBackend(environment: environmentType, settings: settingsType) { run: runNewMigrations, option: { shouldLoadDb: true, shouldExit: true }, }, - cleanOrphansTreatments: { - run: cleanOrphansTreatments, - option: { shouldLoadDb: true, shouldExit: true }, - }, }, }; } diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts index a64a26dac..318b07a28 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts @@ -8,7 +8,7 @@ import { logger } from '../../../utils'; export { cleanDuplicatedDocuments }; /** - * Delete all doubled documents (same source, same documentNumber, same text and creationDate ) get recent documents by creationDate + * Delete all doubled documents (same source, same documentNumber and same text) keep the most recent document by dateCreation */ async function cleanDuplicatedDocuments() { logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'START' }); @@ -28,11 +28,10 @@ async function cleanDuplicatedDocuments() { for (let index = 0, l = documents.length - 1; index < l; index++) { const currentDocument = sortedDocuments[index]; const nextDocument = sortedDocuments[index + 1]; - // check if document is same, if same return the last version document if (areDocumentsIdentical(currentDocument, nextDocument)) { if ( currentDocument.creationDate && - nextDocument.creationDate != undefined && + nextDocument.creationDate && currentDocument.creationDate > nextDocument.creationDate ) { documentsToDelete.push(nextDocument); diff --git a/packages/generic/backend/src/app/scripts/index.ts b/packages/generic/backend/src/app/scripts/index.ts index e0efeccea..ac91b1f32 100644 --- a/packages/generic/backend/src/app/scripts/index.ts +++ b/packages/generic/backend/src/app/scripts/index.ts @@ -20,7 +20,6 @@ import { resetUntreatedDocumentsForTest } from './resetUntreatedDocumentsForTest import { purgeDb } from './purgeDb'; import { renewCache } from './renewCache'; import { setIndexesOnAllCollections } from './setIndexesOnAllCollections'; -import { cleanOrphansTreatments } from './cleanDocuments/cleanOrphansTreatments'; export { cleanDocuments, @@ -43,5 +42,4 @@ export { revertOneMigration, runNewMigrations, setIndexesOnAllCollections, - cleanOrphansTreatments, }; From b94d0839749a0c6584bdf94043df75f6b9fb93fd Mon Sep 17 00:00:00 2001 From: Antoine Jeanneney Date: Wed, 7 Feb 2024 17:30:27 +0100 Subject: [PATCH 8/9] run ONLY cleanDuplicatedDocuments before export to avoid sides effects --- .../src/scripts/exportAllRejectedDocuments.ts | 4 ++-- .../src/scripts/exportAllTreatedDocuments.ts | 4 ++-- .../src/scripts/exportTreatedDocumentsSince.ts | 4 ++-- .../src/scripts/exportTreatedPublishableDocuments.ts | 4 ++-- packages/generic/backend/src/app/buildBackend.ts | 5 +++++ .../app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts | 1 - .../app/scripts/cleanDocuments/cleanOrphansTreatments.ts | 6 +++--- .../generic/backend/src/app/scripts/cleanDocuments/index.ts | 1 + 8 files changed, 17 insertions(+), 12 deletions(-) diff --git a/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts b/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts index de0c153aa..311b8ca5f 100644 --- a/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts +++ b/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts @@ -7,8 +7,8 @@ import { parametersHandler } from '../lib/parametersHandler'; const backend = buildBackend(environment, settings); await backend.runScript( - () => backend.scripts.cleanDocuments.run(), - backend.scripts.cleanDocuments.option, + () => backend.scripts.cleanDuplicatedDocuments.run(), + backend.scripts.cleanDuplicatedDocuments.option, ); const sderExporter = buildSderExporter(environment, settings); diff --git a/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts b/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts index ef924c50b..0a6913a17 100644 --- a/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts +++ b/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts @@ -7,8 +7,8 @@ import { parametersHandler } from '../lib/parametersHandler'; const backend = buildBackend(environment, settings); await backend.runScript( - () => backend.scripts.cleanDocuments.run(), - backend.scripts.cleanDocuments.option, + () => backend.scripts.cleanDuplicatedDocuments.run(), + backend.scripts.cleanDuplicatedDocuments.option, ); const sderExporter = buildSderExporter(environment, settings); diff --git a/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts b/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts index cdb91c4de..02f0f1edb 100644 --- a/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts +++ b/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts @@ -9,8 +9,8 @@ import { parametersHandler } from '../lib/parametersHandler'; const backend = buildBackend(environment, settings); await backend.runScript( - () => backend.scripts.cleanDocuments.run(), - backend.scripts.cleanDocuments.option, + () => backend.scripts.cleanDuplicatedDocuments.run(), + backend.scripts.cleanDuplicatedDocuments.option, ); const sderExporter = buildSderExporter(environment, settings); diff --git a/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts b/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts index df859f892..a5194265c 100644 --- a/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts +++ b/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts @@ -7,8 +7,8 @@ import { parametersHandler } from '../lib/parametersHandler'; const backend = buildBackend(environment, settings); await backend.runScript( - () => backend.scripts.cleanDocuments.run(), - backend.scripts.cleanDocuments.option, + () => backend.scripts.cleanDuplicatedDocuments.run(), + backend.scripts.cleanDuplicatedDocuments.option, ); const sderExporter = buildSderExporter(environment, settings); diff --git a/packages/generic/backend/src/app/buildBackend.ts b/packages/generic/backend/src/app/buildBackend.ts index 57e01c352..593927870 100644 --- a/packages/generic/backend/src/app/buildBackend.ts +++ b/packages/generic/backend/src/app/buildBackend.ts @@ -22,6 +22,7 @@ import { revertOneMigration, runNewMigrations, } from './scripts'; +import { cleanDuplicatedDocuments } from './scripts/cleanDocuments' export { buildBackend }; @@ -110,6 +111,10 @@ function buildBackend(environment: environmentType, settings: settingsType) { run: runNewMigrations, option: { shouldLoadDb: true, shouldExit: true }, }, + cleanDuplicatedDocuments: { + run: cleanDuplicatedDocuments, + option: { shouldLoadDb: true, shouldExit: false }, + }, }, }; } diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts index 318b07a28..868f7e603 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts @@ -52,7 +52,6 @@ async function cleanDuplicatedDocuments() { logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'DONE' }); } -// function areDocumentsIdentical( document1: documentType, document2: documentType, diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanOrphansTreatments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanOrphansTreatments.ts index 7e1497abd..f1cf71dc4 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanOrphansTreatments.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanOrphansTreatments.ts @@ -5,9 +5,9 @@ import { logger } from '../../../utils'; export { cleanOrphansTreatments }; -/* - Clean treatments with no document associated -*/ +/** + * Clean treatments with no document associated + */ async function cleanOrphansTreatments() { logger.log({ operationName: 'cleanOrphansTreatments', msg: 'START' }); diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/index.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/index.ts index 00d1b4b1b..0761e4717 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/index.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/index.ts @@ -1 +1,2 @@ export { cleanDocuments } from './cleanDocuments'; +export { cleanDuplicatedDocuments } from './cleanDuplicatedDocuments'; From 5c753f10b8f3b7827e77d3f39d0a41dd3ba9a385 Mon Sep 17 00:00:00 2001 From: Antoine Jeanneney Date: Wed, 7 Feb 2024 17:32:41 +0100 Subject: [PATCH 9/9] lint :) --- packages/generic/backend/src/app/buildBackend.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/generic/backend/src/app/buildBackend.ts b/packages/generic/backend/src/app/buildBackend.ts index 593927870..9c928f885 100644 --- a/packages/generic/backend/src/app/buildBackend.ts +++ b/packages/generic/backend/src/app/buildBackend.ts @@ -22,7 +22,7 @@ import { revertOneMigration, runNewMigrations, } from './scripts'; -import { cleanDuplicatedDocuments } from './scripts/cleanDocuments' +import { cleanDuplicatedDocuments } from './scripts/cleanDocuments'; export { buildBackend };