From c65db93ca6653d946f3648d6aa6658b603c22d59 Mon Sep 17 00:00:00 2001 From: QuiiBz Date: Thu, 1 Jun 2023 21:28:52 +0200 Subject: [PATCH 1/6] refactor(dashboard,serverless): replace assets table with json column --- crates/serverless/src/deployments/mod.rs | 32 +++++++-------- packages/dashboard/lib/api/deployments.ts | 41 ++++--------------- .../dashboard/lib/trpc/deploymentsRouter.ts | 2 +- .../dashboard/lib/trpc/functionsRouter.ts | 13 ++---- .../migration.sql | 12 ++++++ packages/dashboard/prisma/schema.prisma | 13 +----- 6 files changed, 41 insertions(+), 72 deletions(-) create mode 100644 packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql diff --git a/crates/serverless/src/deployments/mod.rs b/crates/serverless/src/deployments/mod.rs index 4dffc7157..0e5a4105b 100644 --- a/crates/serverless/src/deployments/mod.rs +++ b/crates/serverless/src/deployments/mod.rs @@ -6,6 +6,7 @@ use lagon_runtime_utils::{Deployment, DEPLOYMENTS_DIR}; use lagon_serverless_downloader::Downloader; use log::{error, info, warn}; use mysql::{prelude::Queryable, PooledConn}; +use serde::Deserialize; use std::{ collections::{HashMap, HashSet}, fs, @@ -59,17 +60,20 @@ where } } +#[derive(Deserialize)] +struct AssetObj(Vec); + type QueryResult = ( String, bool, String, String, + String, usize, usize, usize, Option, Option, - Option, ); pub async fn get_deployments( @@ -90,22 +94,20 @@ where SELECT Deployment.id, Deployment.isProduction, + Deployment.assets, Function.id, Function.name, Function.memory, Function.tickTimeout, Function.totalTimeout, Function.cron, - Domain.domain, - Asset.name + Domain.domain FROM Deployment INNER JOIN Function ON Deployment.functionId = Function.id LEFT JOIN Domain ON Function.id = Domain.functionId -LEFT JOIN Asset - ON Deployment.id = Asset.deploymentId WHERE Function.cron IS NULL OR @@ -116,6 +118,7 @@ OR |( id, is_production, + assets, function_id, function_name, memory, @@ -123,8 +126,13 @@ OR total_timeout, cron, domain, - asset, ): QueryResult| { + let assets = serde_json::from_str::(&assets) + .map(|asset_obj| asset_obj.0) + .unwrap_or_default(); + + println!("found assets: {:?}", assets); + deployments_list .entry(id.clone()) .and_modify(|deployment| { @@ -132,9 +140,7 @@ OR deployment.domains.insert(domain); } - if let Some(asset) = asset.clone() { - deployment.assets.insert(asset); - } + deployment.assets.extend(assets.clone()); }) .or_insert(Deployment { id, @@ -147,13 +153,7 @@ OR domains }) .unwrap_or_default(), - assets: asset - .map(|asset| { - let mut assets = HashSet::new(); - assets.insert(asset); - assets - }) - .unwrap_or_default(), + assets: HashSet::from_iter(assets.iter().cloned()), environment_variables: HashMap::new(), memory, tick_timeout, diff --git a/packages/dashboard/lib/api/deployments.ts b/packages/dashboard/lib/api/deployments.ts index 36bc2253d..000ed63fa 100644 --- a/packages/dashboard/lib/api/deployments.ts +++ b/packages/dashboard/lib/api/deployments.ts @@ -24,13 +24,7 @@ export async function createDeployment( return prisma.deployment.create({ data: { isProduction: false, - assets: { - createMany: { - data: assets.map(name => ({ - name, - })), - }, - }, + assets, functionId: func.id, triggerer, }, @@ -39,11 +33,6 @@ export async function createDeployment( createdAt: true, updatedAt: true, isProduction: true, - assets: { - select: { - name: true, - }, - }, functionId: true, }, }); @@ -63,12 +52,6 @@ export async function removeDeployment( }, deploymentId: string, ) { - await prisma.asset.deleteMany({ - where: { - deploymentId, - }, - }); - const deployment = await prisma.deployment.delete({ where: { id: deploymentId, @@ -79,11 +62,7 @@ export async function removeDeployment( updatedAt: true, functionId: true, isProduction: true, - assets: { - select: { - name: true, - }, - }, + assets: true, }, }); @@ -96,7 +75,7 @@ export async function removeDeployment( ), ]; - if (deployment.assets.length > 0) { + if (Array.isArray(deployment.assets) && deployment.assets.length > 0) { deletePromises.push( s3.send( new DeleteObjectsCommand({ @@ -127,15 +106,15 @@ export async function removeDeployment( cronRegion: func.cronRegion, env: envStringToObject(func.env), isProduction: deployment.isProduction, - assets: deployment.assets.map(({ name }) => name), + assets: deployment.assets, }), ); } export async function unpromoteProductionDeployment(functionId: string): Promise< | { - id: string; - } + id: string; + } | undefined > { const currentDeployment = await prisma.deployment.findFirst({ @@ -208,11 +187,7 @@ export async function promoteProductionDeployment(functionId: string, newDeploym createdAt: true, updatedAt: true, isProduction: true, - assets: { - select: { - name: true, - }, - }, + assets: true, }, }); @@ -231,7 +206,7 @@ export async function promoteProductionDeployment(functionId: string, newDeploym cronRegion: func.cronRegion, env: envStringToObject(func.env), isProduction: true, - assets: deployment.assets.map(({ name }) => name), + assets: deployment.assets, }), ); } diff --git a/packages/dashboard/lib/trpc/deploymentsRouter.ts b/packages/dashboard/lib/trpc/deploymentsRouter.ts index 03f204f80..eff551519 100644 --- a/packages/dashboard/lib/trpc/deploymentsRouter.ts +++ b/packages/dashboard/lib/trpc/deploymentsRouter.ts @@ -159,7 +159,7 @@ export const deploymentsRouter = (t: T) => cronRegion: func.cronRegion, env: envStringToObject(func.env), isProduction: deployment.isProduction, - assets: deployment.assets.map(({ name }) => name), + assets: deployment.assets, }), ); diff --git a/packages/dashboard/lib/trpc/functionsRouter.ts b/packages/dashboard/lib/trpc/functionsRouter.ts index a1e8a9713..e77d66a7c 100644 --- a/packages/dashboard/lib/trpc/functionsRouter.ts +++ b/packages/dashboard/lib/trpc/functionsRouter.ts @@ -141,9 +141,8 @@ FROM serverless.logs WHERE function_id = '${input.functionId}' AND - timestamp >= toDateTime(now() - INTERVAL ${ - input.timeframe === 'Last hour' ? '1 HOUR' : input.timeframe === 'Last 24 hours' ? '1 DAY' : '1 WEEK' - }) + timestamp >= toDateTime(now() - INTERVAL ${input.timeframe === 'Last hour' ? '1 HOUR' : input.timeframe === 'Last 24 hours' ? '1 DAY' : '1 WEEK' + }) ${input.level !== 'all' ? `AND level = '${input.level}'` : ''} ORDER BY timestamp DESC LIMIT 100`, @@ -437,7 +436,7 @@ LIMIT 100`, }, { ...deployment, - assets: deployment.assets.map(({ name }) => name), + assets: deployment.assets as string[], }, oldDomains, ); @@ -490,12 +489,6 @@ LIMIT 100`, deployments: { select: { id: true, - triggerer: true, - commit: true, - isProduction: true, - assets: true, - createdAt: true, - updatedAt: true, }, }, }, diff --git a/packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql b/packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql new file mode 100644 index 000000000..34db5b72f --- /dev/null +++ b/packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql @@ -0,0 +1,12 @@ +/* + Warnings: + + - You are about to drop the `Asset` table. If the table is not empty, all the data it contains will be lost. + - Added the required column `assets` to the `Deployment` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE `Deployment` ADD COLUMN `assets` JSON NOT NULL; + +-- DropTable +DROP TABLE `Asset`; diff --git a/packages/dashboard/prisma/schema.prisma b/packages/dashboard/prisma/schema.prisma index 21d40af31..746d4b181 100644 --- a/packages/dashboard/prisma/schema.prisma +++ b/packages/dashboard/prisma/schema.prisma @@ -144,22 +144,11 @@ model Deployment { commit String? isProduction Boolean @default(false) function Function @relation(fields: [functionId], references: [id]) - assets Asset[] + assets Json @@index([functionId]) } -model Asset { - id String @id @default(cuid()) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - name String - deploymentId String - deployment Deployment @relation(fields: [deploymentId], references: [id]) - - @@index([deploymentId]) -} - model Token { id String @id @default(cuid()) createdAt DateTime @default(now()) From adbc600ab6c9773620caa5c4ecf65dd3bc82886d Mon Sep 17 00:00:00 2001 From: QuiiBz Date: Fri, 2 Jun 2023 07:53:52 +0200 Subject: [PATCH 2/6] fix(dashboard): lint errors --- packages/dashboard/lib/api/deployments.ts | 4 ++-- packages/dashboard/lib/trpc/functionsRouter.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/dashboard/lib/api/deployments.ts b/packages/dashboard/lib/api/deployments.ts index 000ed63fa..27bb0e40a 100644 --- a/packages/dashboard/lib/api/deployments.ts +++ b/packages/dashboard/lib/api/deployments.ts @@ -113,8 +113,8 @@ export async function removeDeployment( export async function unpromoteProductionDeployment(functionId: string): Promise< | { - id: string; - } + id: string; + } | undefined > { const currentDeployment = await prisma.deployment.findFirst({ diff --git a/packages/dashboard/lib/trpc/functionsRouter.ts b/packages/dashboard/lib/trpc/functionsRouter.ts index e77d66a7c..e82c0dafd 100644 --- a/packages/dashboard/lib/trpc/functionsRouter.ts +++ b/packages/dashboard/lib/trpc/functionsRouter.ts @@ -141,8 +141,9 @@ FROM serverless.logs WHERE function_id = '${input.functionId}' AND - timestamp >= toDateTime(now() - INTERVAL ${input.timeframe === 'Last hour' ? '1 HOUR' : input.timeframe === 'Last 24 hours' ? '1 DAY' : '1 WEEK' - }) + timestamp >= toDateTime(now() - INTERVAL ${ + input.timeframe === 'Last hour' ? '1 HOUR' : input.timeframe === 'Last 24 hours' ? '1 DAY' : '1 WEEK' + }) ${input.level !== 'all' ? `AND level = '${input.level}'` : ''} ORDER BY timestamp DESC LIMIT 100`, From fef61a836447bb34d680de0d6d97c8420dd11fb5 Mon Sep 17 00:00:00 2001 From: QuiiBz Date: Fri, 2 Jun 2023 07:56:58 +0200 Subject: [PATCH 3/6] feat(dashboard): add migration sql command --- .../migration.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql b/packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql index 34db5b72f..320319f89 100644 --- a/packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql +++ b/packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql @@ -8,5 +8,19 @@ -- AlterTable ALTER TABLE `Deployment` ADD COLUMN `assets` JSON NOT NULL; +-- Migrate Asset rows to Deployment's assets column +UPDATE + Deployment AS deployment + INNER JOIN ( + SELECT + JSON_ARRAYAGG(name) AS assets, + deploymentId + FROM + Asset + GROUP BY + deploymentId) AS res SET deployment.assets = res.assets +WHERE + deployment.id = res.deploymentId + -- DropTable DROP TABLE `Asset`; From 285ed012dc1e549358b92902a763b7b9859cb729 Mon Sep 17 00:00:00 2001 From: QuiiBz Date: Fri, 2 Jun 2023 07:59:11 +0200 Subject: [PATCH 4/6] chore: add changeset --- .changeset/violet-panthers-remember.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/violet-panthers-remember.md diff --git a/.changeset/violet-panthers-remember.md b/.changeset/violet-panthers-remember.md new file mode 100644 index 000000000..73267b138 --- /dev/null +++ b/.changeset/violet-panthers-remember.md @@ -0,0 +1,6 @@ +--- +'@lagon/serverless': patch +'@lagon/dashboard': patch +--- + +Replace assets table with a json column From bcb35992f1eae72838e49b309ef28d5bf48e9b7d Mon Sep 17 00:00:00 2001 From: QuiiBz Date: Fri, 2 Jun 2023 07:59:51 +0200 Subject: [PATCH 5/6] fix(serverless): remove println --- crates/serverless/src/deployments/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/serverless/src/deployments/mod.rs b/crates/serverless/src/deployments/mod.rs index 0e5a4105b..a1e93608f 100644 --- a/crates/serverless/src/deployments/mod.rs +++ b/crates/serverless/src/deployments/mod.rs @@ -131,8 +131,6 @@ OR .map(|asset_obj| asset_obj.0) .unwrap_or_default(); - println!("found assets: {:?}", assets); - deployments_list .entry(id.clone()) .and_modify(|deployment| { From 9643de7024ebd0ac4814418b8d1d00236cf4e23e Mon Sep 17 00:00:00 2001 From: QuiiBz Date: Fri, 2 Jun 2023 08:18:38 +0200 Subject: [PATCH 6/6] feat(dashboard): set json default value --- .../migration.sql | 1 - packages/dashboard/prisma/schema.prisma | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) rename packages/dashboard/prisma/migrations/{20230601181224_remove_assets_table_use_json_instead => 20230602061526_remove_assets_table_use_json_instead}/migration.sql (79%) diff --git a/packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql b/packages/dashboard/prisma/migrations/20230602061526_remove_assets_table_use_json_instead/migration.sql similarity index 79% rename from packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql rename to packages/dashboard/prisma/migrations/20230602061526_remove_assets_table_use_json_instead/migration.sql index 320319f89..ef1bba6fd 100644 --- a/packages/dashboard/prisma/migrations/20230601181224_remove_assets_table_use_json_instead/migration.sql +++ b/packages/dashboard/prisma/migrations/20230602061526_remove_assets_table_use_json_instead/migration.sql @@ -2,7 +2,6 @@ Warnings: - You are about to drop the `Asset` table. If the table is not empty, all the data it contains will be lost. - - Added the required column `assets` to the `Deployment` table without a default value. This is not possible if the table is not empty. */ -- AlterTable diff --git a/packages/dashboard/prisma/schema.prisma b/packages/dashboard/prisma/schema.prisma index 746d4b181..fb27ba5f8 100644 --- a/packages/dashboard/prisma/schema.prisma +++ b/packages/dashboard/prisma/schema.prisma @@ -144,7 +144,7 @@ model Deployment { commit String? isProduction Boolean @default(false) function Function @relation(fields: [functionId], references: [id]) - assets Json + assets Json @default("[]") @@index([functionId]) }