From 745261ae6c25a7916b23ba33793891dad929c98f Mon Sep 17 00:00:00 2001 From: Antoine Jeanneney Date: Wed, 31 Jan 2024 10:41:59 +0100 Subject: [PATCH] run clean documents before export scripts --- .../roles/deploy_backend/defaults/main/defaults.yml | 2 +- .../src/scripts/cleanOrphansTreatments.ts | 12 ------------ .../src/scripts/exportAllRejectedDocuments.ts | 6 ++++++ .../src/scripts/exportAllTreatedDocuments.ts | 6 ++++++ .../src/scripts/exportTreatedDocumentsSince.ts | 6 ++++++ .../src/scripts/exportTreatedPublishableDocuments.ts | 6 ++++++ packages/generic/backend/src/app/buildBackend.ts | 5 ----- .../cleanDocuments/cleanDuplicatedDocuments.ts | 5 ++--- packages/generic/backend/src/app/scripts/index.ts | 2 -- 9 files changed, 27 insertions(+), 23 deletions(-) delete mode 100644 packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts diff --git a/ansible/roles/deploy_backend/defaults/main/defaults.yml b/ansible/roles/deploy_backend/defaults/main/defaults.yml index 21aab5535..97e0410e6 100644 --- a/ansible/roles/deploy_backend/defaults/main/defaults.yml +++ b/ansible/roles/deploy_backend/defaults/main/defaults.yml @@ -33,7 +33,7 @@ jobs: active_deadline_seconds: 1800 command: "dist/scripts/importChainedDocumentsFromSder.js --count 500 --threshold 1500" - name: "export-j-4" - schedule: "30 17 * * *" + schedule: "25 17 * * *" successful_jobs_history_limit: 7 failed_jobs_history_limit: 7 backoff_limit: 2 diff --git a/packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts b/packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts deleted file mode 100644 index d0e88c30d..000000000 --- a/packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { buildBackend } from '@label/backend'; -import { parametersHandler } from '../lib/parametersHandler'; - -(async () => { - const { environment, settings } = await parametersHandler.getParameters(); - const backend = buildBackend(environment, settings); - - await backend.runScript( - () => backend.scripts.cleanOrphansTreatments.run(), - backend.scripts.cleanOrphansTreatments.option, - ); -})(); diff --git a/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts b/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts index ab10791ad..de0c153aa 100644 --- a/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts +++ b/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts @@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler'; (async () => { const { environment, settings } = await parametersHandler.getParameters(); const backend = buildBackend(environment, settings); + + await backend.runScript( + () => backend.scripts.cleanDocuments.run(), + backend.scripts.cleanDocuments.option, + ); + const sderExporter = buildSderExporter(environment, settings); backend.runScript( () => sderExporter.exportAllRejectedDocuments(environment), diff --git a/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts b/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts index bc458871d..ef924c50b 100644 --- a/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts +++ b/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts @@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler'; (async () => { const { environment, settings } = await parametersHandler.getParameters(); const backend = buildBackend(environment, settings); + + await backend.runScript( + () => backend.scripts.cleanDocuments.run(), + backend.scripts.cleanDocuments.option, + ); + const sderExporter = buildSderExporter(environment, settings); backend.runScript(() => sderExporter.exportAllTreatedDocuments(environment), { diff --git a/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts b/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts index bf29fc3d7..cdb91c4de 100644 --- a/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts +++ b/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts @@ -7,6 +7,12 @@ import { parametersHandler } from '../lib/parametersHandler'; const { environment, settings } = await parametersHandler.getParameters(); const { days } = parseArgv(); const backend = buildBackend(environment, settings); + + await backend.runScript( + () => backend.scripts.cleanDocuments.run(), + backend.scripts.cleanDocuments.option, + ); + const sderExporter = buildSderExporter(environment, settings); backend.runScript( () => sderExporter.exportTreatedDocumentsSince(days, environment), diff --git a/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts b/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts index 23e2465e7..df859f892 100644 --- a/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts +++ b/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts @@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler'; (async () => { const { environment, settings } = await parametersHandler.getParameters(); const backend = buildBackend(environment, settings); + + await backend.runScript( + () => backend.scripts.cleanDocuments.run(), + backend.scripts.cleanDocuments.option, + ); + const sderExporter = buildSderExporter(environment, settings); backend.runScript( diff --git a/packages/generic/backend/src/app/buildBackend.ts b/packages/generic/backend/src/app/buildBackend.ts index 81bc6a540..57e01c352 100644 --- a/packages/generic/backend/src/app/buildBackend.ts +++ b/packages/generic/backend/src/app/buildBackend.ts @@ -21,7 +21,6 @@ import { resetUntreatedDocumentsForTest, revertOneMigration, runNewMigrations, - cleanOrphansTreatments, } from './scripts'; export { buildBackend }; @@ -111,10 +110,6 @@ function buildBackend(environment: environmentType, settings: settingsType) { run: runNewMigrations, option: { shouldLoadDb: true, shouldExit: true }, }, - cleanOrphansTreatments: { - run: cleanOrphansTreatments, - option: { shouldLoadDb: true, shouldExit: true }, - }, }, }; } diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts index a64a26dac..318b07a28 100644 --- a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts +++ b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts @@ -8,7 +8,7 @@ import { logger } from '../../../utils'; export { cleanDuplicatedDocuments }; /** - * Delete all doubled documents (same source, same documentNumber, same text and creationDate ) get recent documents by creationDate + * Delete all doubled documents (same source, same documentNumber and same text) keep the most recent document by dateCreation */ async function cleanDuplicatedDocuments() { logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'START' }); @@ -28,11 +28,10 @@ async function cleanDuplicatedDocuments() { for (let index = 0, l = documents.length - 1; index < l; index++) { const currentDocument = sortedDocuments[index]; const nextDocument = sortedDocuments[index + 1]; - // check if document is same, if same return the last version document if (areDocumentsIdentical(currentDocument, nextDocument)) { if ( currentDocument.creationDate && - nextDocument.creationDate != undefined && + nextDocument.creationDate && currentDocument.creationDate > nextDocument.creationDate ) { documentsToDelete.push(nextDocument); diff --git a/packages/generic/backend/src/app/scripts/index.ts b/packages/generic/backend/src/app/scripts/index.ts index e0efeccea..ac91b1f32 100644 --- a/packages/generic/backend/src/app/scripts/index.ts +++ b/packages/generic/backend/src/app/scripts/index.ts @@ -20,7 +20,6 @@ import { resetUntreatedDocumentsForTest } from './resetUntreatedDocumentsForTest import { purgeDb } from './purgeDb'; import { renewCache } from './renewCache'; import { setIndexesOnAllCollections } from './setIndexesOnAllCollections'; -import { cleanOrphansTreatments } from './cleanDocuments/cleanOrphansTreatments'; export { cleanDocuments, @@ -43,5 +42,4 @@ export { revertOneMigration, runNewMigrations, setIndexesOnAllCollections, - cleanOrphansTreatments, };