From 3b760c9eacccf97f6f029d9a7ba57c954ba89586 Mon Sep 17 00:00:00 2001 From: Carlos Quintero Date: Mon, 17 Apr 2023 22:20:43 -0500 Subject: [PATCH 1/3] refactor adminbro redis for queries when exporting general resources --- src/server/adminJs/adminJs.ts | 19 ++++++++++++------- src/server/adminJs/tabs/projectsTab.ts | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/server/adminJs/adminJs.ts b/src/server/adminJs/adminJs.ts index a0896b2a9..9380ddecb 100644 --- a/src/server/adminJs/adminJs.ts +++ b/src/server/adminJs/adminJs.ts @@ -73,24 +73,29 @@ export const getAdminJsRouter = async () => { // Express Middleware to save query of a search export const adminJsQueryCache = async (req, res, next) => { if ( - req.url.startsWith('/admin/api/resources/Project/actions/list') && - req.headers.cookie.includes('adminjs') + req.url.startsWith('/admin/api/resources/') && + req.headers.cookie.includes('adminbro') ) { const admin = await getCurrentAdminJsSession(req); if (!admin) return next(); // skip saving queries + const matches = req.url.match(/\/admin\/api\/resources\/(.+?)(\/|$)/); + if (!matches) return next(); // invalid URL + + const resourceName = matches[1]; const queryStrings = {}; - // get URL query strings + + // Extract filter names and values from URL query string parameters for (const key of Object.keys(req.query)) { const [_, filter] = key.split('.'); if (!filter) continue; queryStrings[filter] = req.query[key]; } - // save query string for later use with an expiration - await redis.set( - `adminjs_${admin.id}_qs`, - JSON.stringify(queryStrings), + // Save query strings to Redis hash with an expiration + await redis.hset( + `adminbro:${admin.id}:${resourceName}`, + queryStrings, 'ex', 1800, ); diff --git a/src/server/adminJs/tabs/projectsTab.ts b/src/server/adminJs/tabs/projectsTab.ts index e17eb09fa..7a0c17dc0 100644 --- a/src/server/adminJs/tabs/projectsTab.ts +++ b/src/server/adminJs/tabs/projectsTab.ts @@ -525,7 +525,7 @@ export const exportProjectsWithFiltersToCsv = async ( try { const { records } = context; const rawQueryStrings = await redis.get( - `adminjs_${context.currentAdmin.id}_qs`, + `adminbro:${context.currentAdmin.id}:Project`, ); const queryStrings = rawQueryStrings ? JSON.parse(rawQueryStrings) : {}; const projectsQuery = buildProjectsQuery(queryStrings); From 65275d63fa31fac4be62582cca8954a827016149 Mon Sep 17 00:00:00 2001 From: Carlos Quintero Date: Thu, 20 Apr 2023 01:25:38 -0500 Subject: [PATCH 2/3] refactor and add export by csv of donation --- src/server/adminJs/adminJs-types.ts | 42 +++- src/server/adminJs/adminJsPermissions.ts | 3 + src/server/adminJs/tabs/donationTab.ts | 229 ++++++++++++++++++-- src/server/adminJs/tabs/projectUpdateTab.ts | 3 + src/server/adminJs/tabs/projectsTab.ts | 10 +- src/services/googleSheets.ts | 101 ++++++--- 6 files changed, 336 insertions(+), 52 deletions(-) diff --git a/src/server/adminJs/adminJs-types.ts b/src/server/adminJs/adminJs-types.ts index 1a7f4a093..e23a9b47a 100644 --- a/src/server/adminJs/adminJs-types.ts +++ b/src/server/adminJs/adminJs-types.ts @@ -30,8 +30,22 @@ export interface AdminJsProjectsQuery { reviewStatus: ReviewStatus; } +export interface AdminJsDonationsQuery { + projectId?: string; + contactEmail?: string; + referrerWallet?: string; + userId?: string; + fromWalletAddress?: string; + toWalletAddress?: string; + status?: string; + createdAt?: string; + currency?: string; + transactionNetworkId?: string; + isProjectVerified?: string; +} + // headers defined by the verification team for exporting -export const headers = [ +export const projectHeaders = [ 'id', 'title', 'slug', @@ -54,3 +68,29 @@ export const headers = [ 'secondWalletAddress', 'secondWalletAddressNetwork', ]; + +export const donationHeaders = [ + 'id', + 'transactionId', + 'transactionNetworkId', + 'isProjectVerified', + 'status', + 'toWalletAddress', + 'fromWalletAddress', + 'tokenAddress', + 'currency', + 'anonymous', + 'amount', + 'isFiat', + 'isCustomToken', + 'valueEth', + 'valueUsd', + 'priceEth', + 'priceUsd', + 'projectId', + 'userId', + 'contactEmail', + 'createdAt', + 'referrerWallet', + 'isTokenEligibleForGivback', +]; diff --git a/src/server/adminJs/adminJsPermissions.ts b/src/server/adminJs/adminJsPermissions.ts index 372772939..ebd91b715 100644 --- a/src/server/adminJs/adminJsPermissions.ts +++ b/src/server/adminJs/adminJsPermissions.ts @@ -273,12 +273,15 @@ const donationPermissions = { new: true, show: true, edit: true, + exportFilterToCsv: true, }, [UserRole.OPERATOR]: { show: true, + exportFilterToCsv: true, }, [UserRole.VERIFICATION_FORM_REVIEWER]: { show: true, + exportFilterToCsv: true, }, [UserRole.CAMPAIGN_MANAGER]: { show: true, diff --git a/src/server/adminJs/tabs/donationTab.ts b/src/server/adminJs/tabs/donationTab.ts index c319f47f5..73cffec1b 100644 --- a/src/server/adminJs/tabs/donationTab.ts +++ b/src/server/adminJs/tabs/donationTab.ts @@ -10,6 +10,8 @@ import { import { AdminJsContextInterface, AdminJsRequestInterface, + AdminJsDonationsQuery, + donationHeaders, } from '../adminJs-types'; import { messages } from '../../../utils/messages'; import { logger } from '../../../utils/logger'; @@ -22,16 +24,19 @@ import { getCsvAirdropTransactions, getGnosisSafeTransactions, } from '../../../services/transactionService'; -import { - i18n, - translationErrorMessagesKeys, -} from '../../../utils/errorMessages'; +import { i18n, translationErrorMessagesKeys } from '../../../utils/errorMessages'; import { Project } from '../../../entities/project'; import { calculateGivbackFactor } from '../../../services/givbackService'; import { findUserByWalletAddress } from '../../../repositories/userRepository'; import { updateTotalDonationsOfProject } from '../../../services/donationService'; import { updateUserTotalDonated } from '../../../services/userService'; -import { NETWORK_IDS } from '../../../provider'; +import { NETWORK_IDS, NETWORKS_IDS_TO_NAME } from '../../../provider'; +import { redis } from '../../../redis'; +import { + initExportSpreadsheet, + addDonationsSheetToSpreadsheet, +} from '../../../services/googleSheets'; +import { SelectQueryBuilder } from 'typeorm'; export const createDonation = async ( request: AdminJsRequestInterface, @@ -168,6 +173,156 @@ export const createDonation = async ( }; }; +// add queries depending on which filters were selected +export const buildDonationsQuery = ( + queryStrings: AdminJsDonationsQuery, +): SelectQueryBuilder => { + const query = Donation.createQueryBuilder('donation') + .leftJoinAndSelect('donation.user', 'user') + .leftJoinAndSelect('donation.project', 'project') + .where('donation.amount > 0') + .addOrderBy('donation.createdAt', 'DESC'); + + if (queryStrings.projectId) + query.andWhere('donation.projectId = :projectId', { + projectId: queryStrings.projectId, + }); + + if (queryStrings.userId) + query.andWhere('donation.userId = :userId', { + userId: queryStrings.userId, + }); + + if (queryStrings.currency) + query.andWhere('donation.currency = :currency', { + currency: queryStrings.currency, + }); + + if (queryStrings.status) + query.andWhere('donation.status = :status', { + status: queryStrings.status, + }); + + if (queryStrings.transactionNetworkId) + query.andWhere('donation.transactionNetworkId = :transactionNetworkId', { + transactionNetworkId: Number(queryStrings.transactionNetworkId), + }); + + if (queryStrings.fromWalletAddress) + query.andWhere('donation.fromWalletAddress = :fromWalletAddress', { + fromWalletAddress: queryStrings.fromWalletAddress, + }); + + if (queryStrings.toWalletAddress) + query.andWhere('donation.toWalletAddress = :toWalletAddress', { + toWalletAddress: queryStrings.toWalletAddress, + }); + + if (queryStrings.contactEmail) + query.andWhere('donation.contactEmail = :contactEmail', { + contactEmail: queryStrings.contactEmail, + }); + + if (queryStrings.referrerWallet) + query.andWhere('donation.referrerWallet = :referrerWallet', { + referrerWallet: queryStrings.referrerWallet, + }); + + if (queryStrings.isProjectVerified) + query.andWhere('donation.isProjectVerified = :isProjectVerified', { + isProjectVerified: queryStrings.isProjectVerified === 'true', + }); + + if (queryStrings['createdAt~~from']) + query.andWhere('donation."createdAt" >= :createdFrom', { + createdAt: queryStrings['createdAt~~from'], + }); + + if (queryStrings['createdAt~~to']) + query.andWhere('donation."createdAt" <= :createdTo', { + createdAt: queryStrings['createdAt~~to'], + }); + + return query; +}; + +export const exportDonationsWithFiltersToCsv = async ( + _request: AdminJsRequestInterface, + _response, + context: AdminJsContextInterface, +) => { + try { + const { records } = context; + const rawQueryStrings = await redis.get( + `adminbro:${context.currentAdmin.id}:Donation`, + ); + const queryStrings = rawQueryStrings ? JSON.parse(rawQueryStrings) : {}; + const projectsQuery = buildDonationsQuery(queryStrings); + const projects = await projectsQuery.getMany(); + + await sendDonationsToGoogleSheet(projects); + + return { + redirectUrl: '/admin/resources/Donation', + records, + notice: { + message: `Donation(s) successfully exported`, + type: 'success', + }, + }; + } catch (e) { + return { + redirectUrl: '/admin/resources/Donation', + record: {}, + notice: { + message: e.message, + type: 'danger', + }, + }; + } +}; + +const sendDonationsToGoogleSheet = async ( + donations: Donation[], +): Promise => { + const spreadsheet = await initExportSpreadsheet(); + + // parse data and set headers + const donationRows = donations.map((donation: Donation) => { + return { + id: donation.id, + transactionId: donation.transactionId, + transactionNetworkId: donation.transactionNetworkId, + isProjectVerified: Boolean(donation.isProjectVerified), + status: donation.status, + toWalletAddress: donation.toWalletAddress, + fromWalletAddress: donation.fromWalletAddress, + tokenAddress: donation.tokenAddress || '', + currency: donation.currency, + anonymous: Boolean(donation.anonymous), + amount: donation.amount, + isFiat: Boolean(donation.isFiat), + isCustomToken: Boolean(donation.isCustomToken), + valueEth: donation.valueEth, + valueUsd: donation.valueUsd, + priceEth: donation.priceEth, + priceUsd: donation.priceUsd, + projectId: donation?.project?.id || '', + userId: donation?.user?.id || '', + contactEmail: donation?.contactEmail || '', + createdAt: donation?.createdAt.toISOString(), + referrerWallet: donation?.referrerWallet || '', + isTokenEligibleForGivback: Boolean(donation?.isTokenEligibleForGivback), + }; + }); + + await addDonationsSheetToSpreadsheet( + spreadsheet, + donationHeaders, + donationRows, + ); +}; + export const donationTab = { resource: Donation, options: { @@ -202,7 +357,7 @@ export const donationTab = { onramperTransactionStatus: { isVisible: { list: false, - filter: true, + filter: false, show: true, edit: false, new: false, @@ -212,7 +367,7 @@ export const donationTab = { onramperId: { isVisible: { list: false, - filter: true, + filter: false, show: true, edit: false, new: false, @@ -233,7 +388,7 @@ export const donationTab = { referrerWallet: { isVisible: { list: false, - filter: false, + filter: true, show: true, edit: false, new: false, @@ -242,7 +397,7 @@ export const donationTab = { verifyErrorMessage: { isVisible: { list: false, - filter: true, + filter: false, show: true, edit: false, new: false, @@ -263,8 +418,8 @@ export const donationTab = { transakStatus: { isVisible: { list: false, - filter: true, - show: true, + filter: false, + show: false, edit: false, new: false, }, @@ -273,7 +428,13 @@ export const donationTab = { isVisible: false, }, anonymous: { - isVisible: false, + isVisible: { + list: false, + filter: false, + show: true, + edit: false, + new: false, + }, }, userId: { isVisible: { @@ -307,8 +468,8 @@ export const donationTab = { }, amount: { isVisible: { - list: true, - filter: true, + list: false, + filter: false, show: true, edit: false, new: false, @@ -322,8 +483,8 @@ export const donationTab = { }, valueUsd: { isVisible: { - list: true, - filter: true, + list: false, + filter: false, show: true, edit: false, new: false, @@ -348,7 +509,13 @@ export const donationTab = { }, }, currency: { - isVisible: true, + isVisible: { + list: true, + filter: true, + show: true, + edit: false, + new: false, + }, }, transactionNetworkId: { availableValues: [ @@ -359,7 +526,13 @@ export const donationTab = { { value: NETWORK_IDS.CELO, label: 'Celo' }, { value: NETWORK_IDS.CELO_ALFAJORES, label: 'Alfajores' }, ], - isVisible: true, + isVisible: { + list: true, + filter: true, + show: true, + edit: false, + new: false, + }, }, txType: { availableValues: [ @@ -368,6 +541,7 @@ export const donationTab = { { value: 'gnosisSafe', label: 'Using gnosis safe multi sig' }, ], isVisible: { + filter: false, list: false, show: false, new: true, @@ -375,7 +549,13 @@ export const donationTab = { }, }, priceUsd: { - isVisible: true, + isVisible: { + list: false, + filter: false, + show: true, + edit: false, + new: false, + }, type: 'number', }, }, @@ -404,6 +584,17 @@ export const donationTab = { isAccessible: ({ currentAdmin }) => canAccessDonationAction({ currentAdmin }, ResourceActions.NEW), }, + exportFilterToCsv: { + actionType: 'resource', + isVisible: true, + isAccessible: ({ currentAdmin }) => + canAccessDonationAction( + { currentAdmin }, + ResourceActions.EXPORT_FILTER_TO_CSV, + ), + handler: exportDonationsWithFiltersToCsv, + component: false, + }, }, }, }; diff --git a/src/server/adminJs/tabs/projectUpdateTab.ts b/src/server/adminJs/tabs/projectUpdateTab.ts index b501d13ca..a9e510d7d 100644 --- a/src/server/adminJs/tabs/projectUpdateTab.ts +++ b/src/server/adminJs/tabs/projectUpdateTab.ts @@ -187,6 +187,9 @@ export const projectUpdateTab = { edit: false, }, }, + contentSummary: { + isVisible: false, + }, mission: { isVisible: { list: false, diff --git a/src/server/adminJs/tabs/projectsTab.ts b/src/server/adminJs/tabs/projectsTab.ts index 7a0c17dc0..a2a51e2e9 100644 --- a/src/server/adminJs/tabs/projectsTab.ts +++ b/src/server/adminJs/tabs/projectsTab.ts @@ -34,13 +34,13 @@ import { AdminJsContextInterface, AdminJsProjectsQuery, AdminJsRequestInterface, - headers, + projectHeaders, } from '../adminJs-types'; import { ProjectStatus } from '../../../entities/projectStatus'; import { messages } from '../../../utils/messages'; import { - addSheetWithRows, - projectExportSpreadsheet, + addProjectsSheetToSpreadsheet, + initExportSpreadsheet, } from '../../../services/googleSheets'; import { NETWORKS_IDS_TO_NAME } from '../../../provider'; import { @@ -402,7 +402,7 @@ export const setSocialProfiles: After = async ( const sendProjectsToGoogleSheet = async ( projects: Project[], ): Promise => { - const spreadsheet = await projectExportSpreadsheet(); + const spreadsheet = await initExportSpreadsheet(); // parse data and set headers const projectRows = projects.map((project: Project) => { @@ -435,7 +435,7 @@ const sendProjectsToGoogleSheet = async ( }; }); - await addSheetWithRows(spreadsheet, headers, projectRows); + await addProjectsSheetToSpreadsheet(spreadsheet, projectHeaders, projectRows); }; export const listDelist = async ( diff --git a/src/services/googleSheets.ts b/src/services/googleSheets.ts index 9248e0201..e5b91866e 100644 --- a/src/services/googleSheets.ts +++ b/src/services/googleSheets.ts @@ -6,7 +6,57 @@ import { ReviewStatus } from '../entities/project'; // tslint:disable-next-line:no-var-requires const moment = require('moment'); -export const projectExportSpreadsheet = async (): Promise< +interface ProjectExport { + id: number; + title: string; + slug?: string | null; + admin?: string | null; + creationDate: Date; + updatedAt: Date; + impactLocation?: string | null; + walletAddress?: string | null; + statusId: number; + qualityScore: number; + verified: boolean; + listed: boolean; + reviewStatus: ReviewStatus; + totalDonations: number; + totalProjectUpdates: number; + website: string; + email: string; + firstWalletAddress: string; + firstWalletAddressNetwork: string; + secondWalletAddress: string; + secondWalletAddressNetwork: string; +} + +interface DonationExport { + id: number; + transactionId: string; + transactionNetworkId: number; + isProjectVerified: boolean; + status: string; + toWalletAddress: string; + fromWalletAddress: string; + tokenAddress: string; + currency: string; + anonymous: boolean; + amount: number; + isFiat: boolean; + isCustomToken: boolean; + valueEth: number; + valueUsd: number; + priceEth: number; + priceUsd: number; + projectId: string | number; + userId: string | number; + contactEmail: string; + createdAt: string; + referrerWallet: string | null; + isTokenEligibleForGivback: boolean; +} + +export const initExportSpreadsheet = async (): Promise< typeof GoogleSpreadsheet > => { // Initialize the sheet - document ID is the long id in the sheets URL @@ -25,43 +75,40 @@ export const projectExportSpreadsheet = async (): Promise< return spreadSheet; }; -export const addSheetWithRows = async ( +export const addDonationsSheetToSpreadsheet = async ( + spreadSheet: GoogleSpreadsheet, + headers: string[], + rows: DonationExport[], +): Promise => { + try { + const currentDate = moment().toDate(); + + const sheet = await spreadSheet.addSheet({ + headerValues: headers, + title: `Donations ${currentDate.toDateString()} ${currentDate.getTime()}`, + }); + await sheet.addRows(rows); + } catch (e) { + logger.error('addDonationsSheetToSpreadsheet error', e); + throw e; + } +}; + +export const addProjectsSheetToSpreadsheet = async ( spreadSheet: GoogleSpreadsheet, headers: string[], - rows: { - id: number; - title: string; - slug?: string | null; - admin?: string | null; - creationDate: Date; - updatedAt: Date; - impactLocation?: string | null; - walletAddress?: string | null; - statusId: number; - qualityScore: number; - verified: boolean; - listed: boolean; - reviewStatus: ReviewStatus; - totalDonations: number; - totalProjectUpdates: number; - website: string; - email: string; - firstWalletAddress: string; - firstWalletAddressNetwork: string; - secondWalletAddress: string; - secondWalletAddressNetwork: string; - }[], + rows: ProjectExport[], ): Promise => { try { const currentDate = moment().toDate(); const sheet = await spreadSheet.addSheet({ headerValues: headers, - title: `export ${currentDate.toDateString()} ${currentDate.getTime()}`, + title: `Projects ${currentDate.toDateString()} ${currentDate.getTime()}`, }); await sheet.addRows(rows); } catch (e) { - logger.error('addSheetWithRows error', e); + logger.error('addProjectsSheetToSpreadsheet error', e); throw e; } }; From 01e9a8ecd1150ef06127ffdbbfa370f430017fbe Mon Sep 17 00:00:00 2001 From: Carlos Quintero Date: Thu, 20 Apr 2023 17:04:39 -0500 Subject: [PATCH 3/3] add adminJs renaming to branch, fix merge conflicts --- src/server/adminJs/tabs/donationTab.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server/adminJs/tabs/donationTab.ts b/src/server/adminJs/tabs/donationTab.ts index 73cffec1b..3fc280503 100644 --- a/src/server/adminJs/tabs/donationTab.ts +++ b/src/server/adminJs/tabs/donationTab.ts @@ -24,7 +24,10 @@ import { getCsvAirdropTransactions, getGnosisSafeTransactions, } from '../../../services/transactionService'; -import { i18n, translationErrorMessagesKeys } from '../../../utils/errorMessages'; +import { + i18n, + translationErrorMessagesKeys, +} from '../../../utils/errorMessages'; import { Project } from '../../../entities/project'; import { calculateGivbackFactor } from '../../../services/givbackService'; import { findUserByWalletAddress } from '../../../repositories/userRepository';