run clean documents before export scripts

Cour-de-cassation · Jan 31, 2024 · 745261a · 745261a
1 parent 62b3606
commit 745261a
Show file tree

Hide file tree

Showing 9 changed files with 27 additions and 23 deletions.
diff --git a/ansible/roles/deploy_backend/defaults/main/defaults.yml b/ansible/roles/deploy_backend/defaults/main/defaults.yml
@@ -33,7 +33,7 @@ jobs:
     active_deadline_seconds: 1800
     command: "dist/scripts/importChainedDocumentsFromSder.js --count 500 --threshold 1500"
   - name: "export-j-4"
-    schedule: "30 17 * * *"
+    schedule: "25 17 * * *"
     successful_jobs_history_limit: 7
     failed_jobs_history_limit: 7
     backoff_limit: 2

diff --git a/packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts b/packages/courDeCassation/src/scripts/cleanOrphansTreatments.ts
diff --git a/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts b/packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts
@@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler';
 (async () => {
   const { environment, settings } = await parametersHandler.getParameters();
   const backend = buildBackend(environment, settings);
+
+  await backend.runScript(
+    () => backend.scripts.cleanDocuments.run(),
+    backend.scripts.cleanDocuments.option,
+  );
+
   const sderExporter = buildSderExporter(environment, settings);
   backend.runScript(
     () => sderExporter.exportAllRejectedDocuments(environment),

diff --git a/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts b/packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts
@@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler';
 (async () => {
   const { environment, settings } = await parametersHandler.getParameters();
   const backend = buildBackend(environment, settings);
+
+  await backend.runScript(
+    () => backend.scripts.cleanDocuments.run(),
+    backend.scripts.cleanDocuments.option,
+  );
+
   const sderExporter = buildSderExporter(environment, settings);
 
   backend.runScript(() => sderExporter.exportAllTreatedDocuments(environment), {

diff --git a/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts b/packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts
@@ -7,6 +7,12 @@ import { parametersHandler } from '../lib/parametersHandler';
   const { environment, settings } = await parametersHandler.getParameters();
   const { days } = parseArgv();
   const backend = buildBackend(environment, settings);
+
+  await backend.runScript(
+    () => backend.scripts.cleanDocuments.run(),
+    backend.scripts.cleanDocuments.option,
+  );
+
   const sderExporter = buildSderExporter(environment, settings);
   backend.runScript(
     () => sderExporter.exportTreatedDocumentsSince(days, environment),

diff --git a/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts b/packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts
@@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler';
 (async () => {
   const { environment, settings } = await parametersHandler.getParameters();
   const backend = buildBackend(environment, settings);
+
+  await backend.runScript(
+    () => backend.scripts.cleanDocuments.run(),
+    backend.scripts.cleanDocuments.option,
+  );
+
   const sderExporter = buildSderExporter(environment, settings);
 
   backend.runScript(

diff --git a/packages/generic/backend/src/app/buildBackend.ts b/packages/generic/backend/src/app/buildBackend.ts
@@ -21,7 +21,6 @@ import {
   resetUntreatedDocumentsForTest,
   revertOneMigration,
   runNewMigrations,
-  cleanOrphansTreatments,
 } from './scripts';
 
 export { buildBackend };
@@ -111,10 +110,6 @@ function buildBackend(environment: environmentType, settings: settingsType) {
         run: runNewMigrations,
         option: { shouldLoadDb: true, shouldExit: true },
       },
-      cleanOrphansTreatments: {
-        run: cleanOrphansTreatments,
-        option: { shouldLoadDb: true, shouldExit: true },
-      },
     },
   };
 }
diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts
@@ -8,7 +8,7 @@ import { logger } from '../../../utils';
 export { cleanDuplicatedDocuments };
 
 /**
- * Delete all doubled documents (same source, same documentNumber, same text and creationDate ) get recent documents by creationDate
+ * Delete all doubled documents (same source, same documentNumber and same text) keep the most recent document by dateCreation
  */
 async function cleanDuplicatedDocuments() {
   logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'START' });
@@ -28,11 +28,10 @@ async function cleanDuplicatedDocuments() {
   for (let index = 0, l = documents.length - 1; index < l; index++) {
     const currentDocument = sortedDocuments[index];
     const nextDocument = sortedDocuments[index + 1];
-    // check if document is same, if same return the last version document
     if (areDocumentsIdentical(currentDocument, nextDocument)) {
       if (
         currentDocument.creationDate &&
-        nextDocument.creationDate != undefined &&
+        nextDocument.creationDate &&
         currentDocument.creationDate > nextDocument.creationDate
       ) {
         documentsToDelete.push(nextDocument);

diff --git a/packages/generic/backend/src/app/scripts/index.ts b/packages/generic/backend/src/app/scripts/index.ts
@@ -20,7 +20,6 @@ import { resetUntreatedDocumentsForTest } from './resetUntreatedDocumentsForTest
 import { purgeDb } from './purgeDb';
 import { renewCache } from './renewCache';
 import { setIndexesOnAllCollections } from './setIndexesOnAllCollections';
-import { cleanOrphansTreatments } from './cleanDocuments/cleanOrphansTreatments';
 
 export {
   cleanDocuments,
@@ -43,5 +42,4 @@ export {
   revertOneMigration,
   runNewMigrations,
   setIndexesOnAllCollections,
-  cleanOrphansTreatments,
 };