Feature/clean duplicated documents 352

ansible/roles/deploy_backend/defaults/main/defaults.yml

@@ -33,7 +33,7 @@ jobs:
     active_deadline_seconds: 1800
     command: "dist/scripts/importChainedDocumentsFromSder.js --count 500 --threshold 1500"
   - name: "export-j-4"
-    schedule: "30 17 * * *"
+    schedule: "25 17 * * *"
     successful_jobs_history_limit: 7
     failed_jobs_history_limit: 7
     backoff_limit: 2

packages/courDeCassation/src/scripts/exportAllRejectedDocuments.ts

@@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler';
 (async () => {
   const { environment, settings } = await parametersHandler.getParameters();
   const backend = buildBackend(environment, settings);
+
+  await backend.runScript(
+    () => backend.scripts.cleanDocuments.run(),
+    backend.scripts.cleanDocuments.option,
+  );
+
   const sderExporter = buildSderExporter(environment, settings);
   backend.runScript(
     () => sderExporter.exportAllRejectedDocuments(environment),

packages/courDeCassation/src/scripts/exportAllTreatedDocuments.ts

@@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler';
 (async () => {
   const { environment, settings } = await parametersHandler.getParameters();
   const backend = buildBackend(environment, settings);
+
+  await backend.runScript(
+    () => backend.scripts.cleanDocuments.run(),
+    backend.scripts.cleanDocuments.option,
+  );
+
   const sderExporter = buildSderExporter(environment, settings);
 
   backend.runScript(() => sderExporter.exportAllTreatedDocuments(environment), {

packages/courDeCassation/src/scripts/exportTreatedDocumentsSince.ts

@@ -7,6 +7,12 @@ import { parametersHandler } from '../lib/parametersHandler';
   const { environment, settings } = await parametersHandler.getParameters();
   const { days } = parseArgv();
   const backend = buildBackend(environment, settings);
+
+  await backend.runScript(
+    () => backend.scripts.cleanDocuments.run(),
+    backend.scripts.cleanDocuments.option,
+  );
+
   const sderExporter = buildSderExporter(environment, settings);
   backend.runScript(
     () => sderExporter.exportTreatedDocumentsSince(days, environment),

packages/courDeCassation/src/scripts/exportTreatedPublishableDocuments.ts

@@ -5,6 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler';
 (async () => {
   const { environment, settings } = await parametersHandler.getParameters();
   const backend = buildBackend(environment, settings);
+
+  await backend.runScript(
+    () => backend.scripts.cleanDocuments.run(),
+    backend.scripts.cleanDocuments.option,
+  );
+
   const sderExporter = buildSderExporter(environment, settings);
 
   backend.runScript(

packages/generic/backend/src/app/buildBackend.ts

@@ -21,7 +21,6 @@ import {
   resetUntreatedDocumentsForTest,
   revertOneMigration,
   runNewMigrations,
-  cleanOrphansTreatments,
 } from './scripts';
 
 export { buildBackend };
@@ -111,10 +110,6 @@ function buildBackend(environment: environmentType, settings: settingsType) {
         run: runNewMigrations,
         option: { shouldLoadDb: true, shouldExit: true },
       },
-      cleanOrphansTreatments: {
-        run: cleanOrphansTreatments,
-        option: { shouldLoadDb: true, shouldExit: true },
-      },
     },
   };
 }

packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.spec.ts

@@ -5,12 +5,16 @@ import { cleanDuplicatedDocuments } from './cleanDuplicatedDocuments';
 describe('cleanDuplicatedDocuments', () => {
   it('should clean the DuplicatedDocuments', async () => {
     const firstDocument = documentModule.generator.generate();
-    const secondDocument = documentModule.generator.generate();
+    const secondDocument = documentModule.generator.generate({
+      creationDate: 1274657452000,
+    });
     const secondDocumentWithHigherStatus = documentModule.generator.generate({
       ...secondDocument,
+      creationDate: 1674657452000,
       _id: idModule.lib.buildId(),
       status: 'done',
     });
+
     const documentRepository = buildDocumentRepository();
     await documentRepository.insertMany([
       firstDocument,
@@ -22,9 +26,15 @@ describe('cleanDuplicatedDocuments', () => {
 
     const fetchedDocuments = await documentRepository.findAll();
 
-    const fetchedIds = fetchedDocuments.map((u) => u._id).sort();
+    const fetchedIds = fetchedDocuments
+      .map((u) => {
+        return u._id;
+      })
+      .sort();
     const expectedIds = [firstDocument, secondDocumentWithHigherStatus]
-      .map((u) => u._id)
+      .map((u) => {
+        return u._id;
+      })
       .sort();
     expect(fetchedIds).toEqual(expectedIds);
   });

packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts

@@ -8,7 +8,7 @@ import { logger } from '../../../utils';
 export { cleanDuplicatedDocuments };
 
 /**
- * Delete all doubled documents (same source, same documentNumber, same text)
+ * Delete all doubled documents (same source, same documentNumber and same text) keep the most recent document by dateCreation
  */
 async function cleanDuplicatedDocuments() {
   logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'START' });
@@ -29,7 +29,15 @@ async function cleanDuplicatedDocuments() {
     const currentDocument = sortedDocuments[index];
     const nextDocument = sortedDocuments[index + 1];
     if (areDocumentsIdentical(currentDocument, nextDocument)) {
-      documentsToDelete.push(nextDocument);
+      if (
+        currentDocument.creationDate &&
+        nextDocument.creationDate &&
+        currentDocument.creationDate > nextDocument.creationDate
+      ) {
+        documentsToDelete.push(nextDocument);
+      } else {
+        documentsToDelete.push(currentDocument);
+      }
     }
   }
 
@@ -44,6 +52,7 @@ async function cleanDuplicatedDocuments() {
   logger.log({ operationName: 'cleanDuplicatedDocuments', msg: 'DONE' });
 }
 
+//
 function areDocumentsIdentical(
   document1: documentType,
   document2: documentType,

packages/generic/backend/src/app/scripts/index.ts

@@ -20,7 +20,6 @@ import { resetUntreatedDocumentsForTest } from './resetUntreatedDocumentsForTest
 import { purgeDb } from './purgeDb';
 import { renewCache } from './renewCache';
 import { setIndexesOnAllCollections } from './setIndexesOnAllCollections';
-import { cleanOrphansTreatments } from './cleanDocuments/cleanOrphansTreatments';
 
 export {
   cleanDocuments,
@@ -43,5 +42,4 @@ export {
   revertOneMigration,
   runNewMigrations,
   setIndexesOnAllCollections,
-  cleanOrphansTreatments,
 };