ftp changes working

bcgov · Feb 15, 2025 · ae07609 · ae07609
1 parent b8fe6b1
commit ae07609
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 58 deletions.
diff --git a/backend/src/ftp/ftp.service.ts b/backend/src/ftp/ftp.service.ts
@@ -1,7 +1,6 @@
 import { Injectable, Logger } from "@nestjs/common";
 import { Cron } from "@nestjs/schedule";
 import * as ftp from "basic-ftp";
-import * as path from "path";
 import * as dotenv from "dotenv";
 import { Writable } from "stream";
 import { NotificationsService } from "src/notifications/notifications.service";
@@ -65,14 +64,17 @@ export class FtpService {
         this.remoteBasePath,
       );
 
-      this.logger.log(`~~~~~`);
       for (const folder of folders) {
         if (folder.isDirectory) {
           const folderPath: string = `${this.remoteBasePath}/${folder.name}`;
           const files: ftp.FileInfo[] = await this.client.list(folderPath);
 
           for (const file of files) {
             if (file.isFile) {
+              // lookup org in database
+              const organization = await this.prisma.ftp_users.findUnique({
+                where: { username: folder.name },
+              });
               const filePath: string = `${folderPath}/${file.name}`;
               this.logger.log(`Processing file: ${filePath}`);
 
@@ -84,11 +86,24 @@ export class FtpService {
                   // don't download the file, just send out a notification of the error
                   const ministryContact = ""; // should be obtained from file somehow
                   await this.notificationsService.notifyFtpUserOfError(
+                    // which notification is better
                     folder.name,
                     file.name,
                     errors,
                     ministryContact,
                   );
+                  await this.notificationsService.sendDataSubmitterNotification(
+                    // which notification is better
+                    organization.email,
+                    {
+                      file_name: file.name,
+                      user_account_name: organization.name,
+                      location_ids: [],
+                      file_status: "REJECTED",
+                      errors: errors.join("\n"),
+                      warnings: null,
+                    },
+                  );
                 } else {
                   const dataBuffer = [];
                   // download file to a stream that puts chunks into an array
@@ -102,22 +117,11 @@ export class FtpService {
                   await this.client.downloadTo(writableStream, filePath);
                   // convert chunk array to buffer
                   const fileBuffer = Buffer.concat(dataBuffer);
-
-                  console.log("111");
-                  console.log(folder.name);
-
-                  // lookup org in database
-                  const organization = await this.prisma.ftp_users.findUnique({
-                    where: { username: folder.name },
-                  });
-
-                  console.log(organization);
-                  // here
-                  await this.fileSubmissionsService.create(
+                  // submit the file
+                  await this.fileSubmissionsService.createWithFtp(
                     {
-                      token: "ftp-user-auth-token", // ?
-                      userID: organization.username, // ?
-                      orgGUID: organization.username, // ?
+                      userID: organization.username,
+                      orgGUID: organization.org_guid,
                       agency: organization.name,
                       operation: "IMPORT",
                     },
@@ -130,53 +134,18 @@ export class FtpService {
                       size: fileBuffer.length,
                     } as Express.Multer.File,
                   );
-                  // await this.fileValidationService.processFile(
-                  //   fileBuffer,
-                  //   filePath,
-                  //   folder.name, // username
-                  //   file.name,
-                  // );
                 }
-                // if (errors.length > 0) {
-                // this.logger.log(`Validation failure for: ${filePath}`);
-                // errors.forEach((error) => this.logger.log(error));
-                // this.logger.log(``);
-                // // send out a notification to the file submitter & ministry contact outlining the errors
-                // const ministryContact = ""; // should be obtained from file somehow
-                // await this.notificationsService.sendDataSubmitterNotification(
-                //   organizationEmail,
-                //   variables: {
-                //     file_name: file.name,
-                //     user_account_name: organizationName,
-                //     location_ids: [],
-                //     file_status: 'REJECTED',
-                //     errors: errors,
-                //     warnings: null
-                //   },
-                // )
-                //   folder.name,
-                //   file.name,
-                //   errors,
-                //   ministryContact,
-                // );
-                // } else {
-                //   this.logger.log(`Validation success for: ${filePath}`);
-                //   this.logger.log(``);
-                // pass to file validation service
-                // await this.validationService.handleFile(file); // made up function call
-                // }
-                // this.logger.log(`Cleaning up file: ${filePath}`);
-                // await this.client.remove(filePath);
               } catch (error) {
-                console.log(error);
                 this.logger.error(
                   "Error during file download or processing",
                   error,
                 );
+              } finally {
+                this.logger.log(`Cleaning up file: ${filePath}`);
+                await this.client.remove(filePath);
               }
             }
           }
-          this.logger.log(`~~~~~`);
         }
       }
     } catch (error) {

diff --git a/migrations/sql/V1.1.5__update_ftp_table.sql b/migrations/sql/V1.1.5__update_ftp_table.sql
@@ -1,14 +1,16 @@
-DROP TABLE if exists ftp_users;
-CREATE TABLE ftp_users (
+DROP TABLE enmods.ftp_users;
+
+CREATE TABLE enmods.ftp_users (
     id SERIAL PRIMARY KEY,
     username VARCHAR(255) NOT NULL UNIQUE,
     name VARCHAR(255) NOT NULL,
     email VARCHAR(255) NOT NULL,
+    org_guid UUID NOT NULL,
     create_user_id VARCHAR(255),
     create_utc_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     update_user_id VARCHAR(255),
     update_utc_timestamp TIMESTAMP
 );
 
 
-insert into ftp_users (username, name, email, create_user_id, create_utc_timestamp, update_user_id, update_utc_timestamp) values ('testorg', 'Test Organization', '[email protected]', 'system', NOW(), 'system', NOW());
+insert into enmods.ftp_users (username, name, email, org_guid, create_user_id, create_utc_timestamp, update_user_id, update_utc_timestamp) values ('testorg', 'Test Organization', '[email protected]', '56e90a60-b00f-4c78-9a88-4c32e6087829', 'system', NOW(), 'system', NOW());