diff --git a/prisma/migrations/20250214010811_arc_without_conflict/migration.sql b/prisma/migrations/20250214010811_arc_without_conflict/migration.sql new file mode 100644 index 000000000..1bfcbd6ab --- /dev/null +++ b/prisma/migrations/20250214010811_arc_without_conflict/migration.sql @@ -0,0 +1,25 @@ +/* + Warnings: + + - The primary key for the `Arc` table will be changed. If it partially fails, the table could be left without primary key constraint. + - A unique constraint covering the columns `[fromId,toId,withoutConflict]` on the table `Arc` will be added. If there are existing duplicate values, this will fail. + - A unique constraint covering the columns `[toId,fromId,withoutConflict]` on the table `Arc` will be added. If there are existing duplicate values, this will fail. + +*/ +-- DropIndex +DROP INDEX "Arc_toId_fromId_idx"; + +-- AlterTable +ALTER TABLE "Arc" DROP CONSTRAINT "Arc_pkey", +ADD COLUMN "id" SERIAL NOT NULL, +ADD COLUMN "withoutConflict" BOOLEAN NOT NULL DEFAULT false, +ADD CONSTRAINT "Arc_pkey" PRIMARY KEY ("id"); + +-- AlterTable +ALTER TABLE "users" ADD COLUMN "trustWithoutConflict" DOUBLE PRECISION NOT NULL DEFAULT 0; + +-- CreateIndex +CREATE UNIQUE INDEX "Arc_fromId_toId_withoutConflict_key" ON "Arc"("fromId", "toId", "withoutConflict"); + +-- CreateIndex +CREATE UNIQUE INDEX "Arc_toId_fromId_withoutConflict_key" ON "Arc"("toId", "fromId", "withoutConflict"); diff --git a/prisma/migrations/20250214015347_zap_rank_personal_noconflicts/migration.sql b/prisma/migrations/20250214015347_zap_rank_personal_noconflicts/migration.sql new file mode 100644 index 000000000..3a8b943d0 --- /dev/null +++ b/prisma/migrations/20250214015347_zap_rank_personal_noconflicts/migration.sql @@ -0,0 +1,69 @@ +DROP MATERIALIZED VIEW IF EXISTS zap_rank_personal_view; +CREATE MATERIALIZED VIEW IF NOT EXISTS zap_rank_personal_view AS +WITH item_votes AS ( + SELECT "Item".id, "Item"."parentId", "Item".boost, "Item".created_at, "Item"."weightedComments", "ItemAct"."userId" AS "voterId", + LOG((SUM("ItemAct".msats) FILTER (WHERE "ItemAct".act IN ('TIP', 'FEE'))) / 1000.0) AS "vote", + GREATEST(LOG((SUM("ItemAct".msats) FILTER (WHERE "ItemAct".act = 'DONT_LIKE_THIS')) / 1000.0), 0) AS "downVote" + FROM "Item" + CROSS JOIN zap_rank_personal_constants + JOIN "ItemAct" ON "ItemAct"."itemId" = "Item".id + WHERE ( + ("ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID') + AND + ( + ("ItemAct"."userId" <> "Item"."userId" AND "ItemAct".act IN ('TIP', 'FEE', 'DONT_LIKE_THIS')) + OR + ("ItemAct".act = 'BOOST' AND "ItemAct"."userId" = "Item"."userId") + ) + ) + AND "Item".created_at >= now_utc() - item_age_bound + GROUP BY "Item".id, "Item"."parentId", "Item".boost, "Item".created_at, "Item"."weightedComments", "ItemAct"."userId" + HAVING SUM("ItemAct".msats) > 1000 +), viewer_votes AS ( + SELECT item_votes.id, item_votes."parentId", item_votes.boost, item_votes.created_at, + item_votes."weightedComments", "Arc"."fromId" AS "viewerId", + GREATEST("Arc"."zapTrust", g."zapTrust", 0) * item_votes."vote" AS "weightedVote", + GREATEST("Arc"."zapTrust", g."zapTrust", 0) * item_votes."downVote" AS "weightedDownVote" + FROM item_votes + CROSS JOIN zap_rank_personal_constants + LEFT JOIN "Arc" ON "Arc"."toId" = item_votes."voterId" + LEFT JOIN "Arc" g ON g."fromId" = global_viewer_id AND g."toId" = item_votes."voterId" + AND ("Arc"."zapTrust" IS NOT NULL OR g."zapTrust" IS NOT NULL) +), viewer_weighted_votes AS ( + SELECT viewer_votes.id, viewer_votes."parentId", viewer_votes.boost, viewer_votes.created_at, viewer_votes."viewerId", + viewer_votes."weightedComments", SUM(viewer_votes."weightedVote") AS "weightedVotes", + SUM(viewer_votes."weightedDownVote") AS "weightedDownVotes" + FROM viewer_votes + GROUP BY viewer_votes.id, viewer_votes."parentId", viewer_votes.boost, viewer_votes.created_at, viewer_votes."viewerId", viewer_votes."weightedComments" +), viewer_zaprank AS ( + SELECT l.id, l."parentId", l.boost, l.created_at, l."viewerId", l."weightedComments", + GREATEST(l."weightedVotes", g."weightedVotes") AS "weightedVotes", GREATEST(l."weightedDownVotes", g."weightedDownVotes") AS "weightedDownVotes" + FROM viewer_weighted_votes l + CROSS JOIN zap_rank_personal_constants + JOIN users ON users.id = l."viewerId" + JOIN viewer_weighted_votes g ON l.id = g.id AND g."viewerId" = global_viewer_id + WHERE (l."weightedVotes" > min_viewer_votes + AND g."weightedVotes" / l."weightedVotes" <= max_personal_viewer_vote_ratio + AND users."lastSeenAt" >= now_utc() - user_last_seen_bound) + OR l."viewerId" = global_viewer_id + GROUP BY l.id, l."parentId", l.boost, l.created_at, l."viewerId", l."weightedVotes", l."weightedComments", + g."weightedVotes", l."weightedDownVotes", g."weightedDownVotes", min_viewer_votes + HAVING GREATEST(l."weightedVotes", g."weightedVotes") > min_viewer_votes OR l.boost > 0 +), viewer_fractions_zaprank AS ( + SELECT z.*, + (CASE WHEN z."weightedVotes" - z."weightedDownVotes" > 0 THEN + GREATEST(z."weightedVotes" - z."weightedDownVotes", POWER(z."weightedVotes" - z."weightedDownVotes", vote_power)) + ELSE + z."weightedVotes" - z."weightedDownVotes" + END + z."weightedComments" * CASE WHEN z."parentId" IS NULL THEN comment_scaler ELSE 0 END) AS tf_numerator, + POWER(GREATEST(age_wait_hours, EXTRACT(EPOCH FROM (now_utc() - z.created_at))/3600), vote_decay) AS decay_denominator, + (POWER(z.boost/boost_per_vote, boost_power) + / + POWER(GREATEST(age_wait_hours, EXTRACT(EPOCH FROM (now_utc() - z.created_at))/3600), boost_decay)) AS boost_addend + FROM viewer_zaprank z, zap_rank_personal_constants +) +SELECT z.id, z."parentId", z."viewerId", + COALESCE(tf_numerator, 0) / decay_denominator + boost_addend AS tf_hot_score, + COALESCE(tf_numerator, 0) AS tf_top_score +FROM viewer_fractions_zaprank z +WHERE tf_numerator > 0 OR boost_addend > 0; \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index a3f986b62..738b70099 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -37,6 +37,7 @@ model User { tipPopover Boolean @default(false) upvotePopover Boolean @default(false) trust Float @default(0) + trustWithoutConflict Float @default(0) lastSeenAt DateTime? stackedMsats BigInt @default(0) stackedMcredits BigInt @default(0) @@ -343,14 +344,18 @@ model Mute { } model Arc { - fromId Int - fromUser User @relation("fromUser", fields: [fromId], references: [id], onDelete: Cascade) - toId Int - toUser User @relation("toUser", fields: [toId], references: [id], onDelete: Cascade) - zapTrust Float - - @@id([fromId, toId]) - @@index([toId, fromId]) + id Int @id @default(autoincrement()) + fromId Int + fromUser User @relation("fromUser", fields: [fromId], references: [id], onDelete: Cascade) + toId Int + toUser User @relation("toUser", fields: [toId], references: [id], onDelete: Cascade) + zapTrust Float + // this is used to store trust without conflicts + // it's a temporary solution until we create trust graphs for each territory + withoutConflict Boolean @default(false) + + @@unique([fromId, toId, withoutConflict]) + @@unique([toId, fromId, withoutConflict]) } enum StreakType { diff --git a/worker/trust.js b/worker/trust.js index 1e2a7f645..efe66ff8f 100644 --- a/worker/trust.js +++ b/worker/trust.js @@ -1,8 +1,13 @@ import * as math from 'mathjs' import { USER_ID, SN_ADMIN_IDS } from '@/lib/constants' +import { Prisma } from '@prisma/client' export async function trust ({ boss, models }) { + // for simplicity, until this is completely rewritten, we want to do two trust runs + // one run not checking for conflicts, computing global AND personal trust + // one run removing conflicts, recording unconflicted arcs try { + // first run console.time('trust') console.timeLog('trust', 'getting graph') const graph = await getGraph(models) @@ -10,8 +15,18 @@ export async function trust ({ boss, models }) { const [vGlobal, mPersonal] = await trustGivenGraph(graph) console.timeLog('trust', 'storing trust') await storeTrust(models, graph, vGlobal, mPersonal) + + // second run + console.time('trust without conflicts') + console.timeLog('trust without conflicts', 'getting graph') + const graphWithoutConflicts = await getGraph(models, true) + console.timeLog('trust without conflicts', 'computing trust') + const [vGlobalWithoutConflicts, mPersonalWithoutConflicts] = await trustGivenGraph(graphWithoutConflicts) + console.timeLog('trust without conflicts', 'storing trust') + await storeTrust(models, graphWithoutConflicts, vGlobalWithoutConflicts, mPersonalWithoutConflicts, true) } finally { console.timeEnd('trust') + console.timeEnd('trust without conflicts') } } @@ -26,7 +41,7 @@ const Z_CONFIDENCE = 6.109410204869 // 99.9999999% confidence const GLOBAL_ROOT = 616 const SEED_WEIGHT = 0.25 const AGAINST_MSAT_MIN = 1000 -const MSAT_MIN = 1000 +const MSAT_MIN = 10001 // 10001 is the minimum for a tip to be counted in trust const SIG_DIFF = 0.1 // need to differ by at least 10 percent /* @@ -114,7 +129,7 @@ function trustGivenGraph (graph) { ... ] */ -async function getGraph (models) { +async function getGraph (models, withoutConflict = false) { return await models.$queryRaw` SELECT id, json_agg(json_build_object( 'node', oid, @@ -128,7 +143,11 @@ async function getGraph (models) { JOIN "Item" ON "Item".id = "ItemAct"."itemId" AND "ItemAct".act IN ('FEE', 'TIP', 'DONT_LIKE_THIS') AND "Item"."parentId" IS NULL AND NOT "Item".bio AND "Item"."userId" <> "ItemAct"."userId" JOIN users ON "ItemAct"."userId" = users.id AND users.id <> ${USER_ID.anon} - WHERE "ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID' + JOIN "Sub" ON "Sub".name = "Item"."subName" + WHERE ("ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID') + ${withoutConflict + ? Prisma.sql` AND ("Sub"."userId" <> "ItemAct"."userId" OR "Sub"."userId" = ANY (${SN_ADMIN_IDS}))` + : Prisma.empty} GROUP BY user_id, name, item_id, user_at, against HAVING CASE WHEN "ItemAct".act = 'DONT_LIKE_THIS' THEN sum("ItemAct".msats) > ${AGAINST_MSAT_MIN} @@ -167,7 +186,7 @@ async function getGraph (models) { ORDER BY id ASC` } -async function storeTrust (models, graph, vGlobal, mPersonal) { +async function storeTrust (models, graph, vGlobal, mPersonal, withoutConflict = false) { // convert nodeTrust into table literal string let globalValues = '' let personalValues = '' @@ -176,29 +195,36 @@ async function storeTrust (models, graph, vGlobal, mPersonal) { if (globalValues) globalValues += ',' globalValues += `(${graph[idx].id}, ${val}::FLOAT)` if (personalValues) personalValues += ',' - personalValues += `(${GLOBAL_ROOT}, ${graph[idx].id}, ${val}::FLOAT)` + personalValues += `(${GLOBAL_ROOT}, ${graph[idx].id}, ${val}::FLOAT, ${withoutConflict}::BOOLEAN)` }) math.forEach(mPersonal, (val, [fromIdx, toIdx]) => { const globalVal = vGlobal.get([toIdx]) if (isNaN(val) || val - globalVal <= SIG_DIFF) return if (personalValues) personalValues += ',' - personalValues += `(${graph[fromIdx].id}, ${graph[toIdx].id}, ${val}::FLOAT)` + personalValues += `(${graph[fromIdx].id}, ${graph[toIdx].id}, ${val}::FLOAT, ${withoutConflict}::BOOLEAN)` }) // update the trust of each user in graph await models.$transaction([ - models.$executeRaw`UPDATE users SET trust = 0`, + models.$executeRaw`UPDATE users SET ${withoutConflict ? 'trustWithoutConflict' : 'trust'} = 0`, models.$executeRawUnsafe( `UPDATE users - SET trust = g.trust + SET ${withoutConflict ? 'trustWithoutConflict' : 'trust'} = g.trust FROM (values ${globalValues}) g(id, trust) WHERE users.id = g.id`), models.$executeRawUnsafe( - `INSERT INTO "Arc" ("fromId", "toId", "zapTrust") - SELECT id, oid, trust - FROM (values ${personalValues}) g(id, oid, trust) - ON CONFLICT ("fromId", "toId") DO UPDATE SET "zapTrust" = EXCLUDED."zapTrust"` + `INSERT INTO "Arc" ("fromId", "toId", "zapTrust", "withoutConflict") + SELECT id, oid, trust, "withoutConflict" + FROM (values ${personalValues}) g(id, oid, trust, "withoutConflict") + ON CONFLICT ("fromId", "toId", "withoutConflict") DO UPDATE SET "zapTrust" = EXCLUDED."zapTrust"` + ), + // select all arcs that don't exist in personalValues and delete them + models.$executeRawUnsafe( + `DELETE FROM "Arc" + WHERE "Arc"."fromId" NOT IN (SELECT id FROM (values ${personalValues}) g(id, oid, trust, "withoutConflict")) + AND "Arc"."toId" NOT IN (SELECT oid FROM (values ${personalValues}) g(id, oid, trust, "withoutConflict")) + AND "Arc"."withoutConflict" = ${withoutConflict}::BOOLEAN` ) ]) }