Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove aggregate tables (fixes #4869) #5407

Draft
wants to merge 45 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 39 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
5ca40a7
migration
Nutomic Feb 7, 2025
a3b835f
update code
Nutomic Feb 7, 2025
549e51f
tests
Nutomic Feb 7, 2025
43ef0b5
triggers
Nutomic Feb 7, 2025
2ea59bf
Merge branch 'main' into remove-aggregate-tables
Nutomic Feb 7, 2025
31cbe18
fix
Nutomic Feb 7, 2025
cd29e85
fmt
Nutomic Feb 7, 2025
61c5038
clippy
Nutomic Feb 7, 2025
b27aea0
post aggregate migration
Nutomic Feb 7, 2025
bb4bfe3
changes for post aggregate code
Nutomic Feb 10, 2025
17d8d98
wip: update tests for post aggregate
Nutomic Feb 10, 2025
4124f03
format
Nutomic Feb 10, 2025
c9aaaf0
fix partialeq
Nutomic Feb 10, 2025
068aded
Merge branch 'main' into remove-aggregate-tables
Nutomic Feb 10, 2025
a88845e
trigger fix
Nutomic Feb 10, 2025
1a1c366
fix post insert trigger
Nutomic Feb 10, 2025
af3035b
wip
Nutomic Feb 10, 2025
41112a9
reorder
Nutomic Feb 11, 2025
bb30add
fixes
Nutomic Feb 11, 2025
8cd04b4
Merge branch 'main' into remove-aggregate-tables
Nutomic Feb 11, 2025
f2d2852
Merge branch 'main' into remove-aggregate-tables
Nutomic Feb 11, 2025
794240c
community aggregate migration
Nutomic Feb 12, 2025
082cdfd
update code
Nutomic Feb 12, 2025
261c0a1
triggers
Nutomic Feb 12, 2025
f145565
person aggregate migration
Nutomic Feb 12, 2025
1ee4e63
person aggregate code
Nutomic Feb 12, 2025
f29c21d
person triggers
Nutomic Feb 12, 2025
8c92d25
test fixes
Nutomic Feb 12, 2025
707c418
fix scheduled task
Nutomic Feb 12, 2025
85fc71d
update api tests
Nutomic Feb 12, 2025
42015ab
Merge branch 'main' into remove-aggregate-tables
Nutomic Feb 12, 2025
cf86bc7
site_aggregates to local_site migration
Nutomic Feb 12, 2025
c9da44d
site_aggregates code changes
Nutomic Feb 12, 2025
7badf7c
triggers, tests
Nutomic Feb 12, 2025
ac7b305
more fixes
Nutomic Feb 12, 2025
47414e3
Merge branch 'main' into remove-aggregate-tables
Nutomic Feb 13, 2025
f732f63
Rename PersonPostAggregates to PostActions
Nutomic Feb 13, 2025
25aa10a
Merge local_user_vote_display_mode into local_user
Nutomic Feb 13, 2025
416ad1b
fix schema
Nutomic Feb 13, 2025
b0bc927
remove duplicate fields
Nutomic Feb 17, 2025
4a6a5e6
remove "aggregates" from index names
Nutomic Feb 17, 2025
4b7422a
uncomment indices
Nutomic Feb 17, 2025
f758ed8
if count = 0
Nutomic Feb 17, 2025
1cf4a9c
Merge branch 'main' into remove-aggregate-tables
Nutomic Feb 20, 2025
44c5990
remove commentaggregates
Nutomic Feb 25, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api_tests/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
"eslint": "^9.20.0",
"eslint-plugin-prettier": "^5.2.3",
"jest": "^29.5.0",
"lemmy-js-client": "0.20.0-show-mod-reports.2",
"lemmy-js-client": "0.20.0-remove-aggregate-tables.4",
"prettier": "^3.5.0",
"ts-jest": "^29.1.0",
"tsoa": "^6.6.0",
Expand Down
10 changes: 5 additions & 5 deletions api_tests/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

58 changes: 29 additions & 29 deletions api_tests/src/comment.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,19 +81,19 @@ test("Create a comment", async () => {
expect(commentRes.comment_view.comment.content).toBeDefined();
expect(commentRes.comment_view.community.local).toBe(false);
expect(commentRes.comment_view.creator.local).toBe(true);
expect(commentRes.comment_view.counts.score).toBe(1);
expect(commentRes.comment_view.comment.score).toBe(1);

// Make sure that comment is liked on beta
let betaComment = (
await waitUntil(
() => resolveComment(beta, commentRes.comment_view.comment),
c => c.comment?.counts.score === 1,
c => c.comment?.comment.score === 1,
)
).comment;
expect(betaComment).toBeDefined();
expect(betaComment?.community.local).toBe(true);
expect(betaComment?.creator.local).toBe(false);
expect(betaComment?.counts.score).toBe(1);
expect(betaComment?.comment.score).toBe(1);
assertCommentFederation(betaComment, commentRes.comment_view);
});

Expand Down Expand Up @@ -293,48 +293,48 @@ test("Unlike a comment", async () => {
let gammaComment1 = (
await waitUntil(
() => resolveComment(gamma, commentRes.comment_view.comment),
c => c.comment?.counts.score === 1,
c => c.comment?.comment.score === 1,
)
).comment;
expect(gammaComment1).toBeDefined();
expect(gammaComment1?.community.local).toBe(false);
expect(gammaComment1?.creator.local).toBe(false);
expect(gammaComment1?.counts.score).toBe(1);
expect(gammaComment1?.comment.score).toBe(1);

let unlike = await likeComment(alpha, 0, commentRes.comment_view.comment);
expect(unlike.comment_view.counts.score).toBe(0);
expect(unlike.comment_view.comment.score).toBe(0);

// Make sure that comment is unliked on beta
let betaComment = (
await waitUntil(
() => resolveComment(beta, commentRes.comment_view.comment),
c => c.comment?.counts.score === 0,
c => c.comment?.comment.score === 0,
)
).comment;
expect(betaComment).toBeDefined();
expect(betaComment?.community.local).toBe(true);
expect(betaComment?.creator.local).toBe(false);
expect(betaComment?.counts.score).toBe(0);
expect(betaComment?.comment.score).toBe(0);

// Make sure that comment is unliked on gamma, downstream peer
// This is testing replication from remote-home-remote (alpha-beta-gamma)
let gammaComment = (
await waitUntil(
() => resolveComment(gamma, commentRes.comment_view.comment),
c => c.comment?.counts.score === 0,
c => c.comment?.comment.score === 0,
)
).comment;
expect(gammaComment).toBeDefined();
expect(gammaComment?.community.local).toBe(false);
expect(gammaComment?.creator.local).toBe(false);
expect(gammaComment?.counts.score).toBe(0);
expect(gammaComment?.comment.score).toBe(0);
});

test("Federated comment like", async () => {
let commentRes = await createComment(alpha, postOnAlphaRes.post_view.post.id);
await waitUntil(
() => resolveComment(beta, commentRes.comment_view.comment),
c => c.comment?.counts.score === 1,
c => c.comment?.comment.score === 1,
);
// Find the comment on beta
let betaComment = (
Expand All @@ -346,14 +346,14 @@ test("Federated comment like", async () => {
}

let like = await likeComment(beta, 1, betaComment.comment);
expect(like.comment_view.counts.score).toBe(2);
expect(like.comment_view.comment.score).toBe(2);

// Get the post from alpha, check the likes
let postComments = await waitUntil(
() => getComments(alpha, postOnAlphaRes.post_view.post.id),
c => c.comments[0].counts.score === 2,
c => c.comments[0].comment.score === 2,
);
expect(postComments.comments[0].counts.score).toBe(2);
expect(postComments.comments[0].comment.score).toBe(2);
});

test("Reply to a comment from another instance, get notification", async () => {
Expand All @@ -377,7 +377,7 @@ test("Reply to a comment from another instance, get notification", async () => {
let betaComment = (
await waitUntil(
() => resolveComment(beta, commentRes.comment_view.comment),
c => c.comment?.counts.score === 1,
c => c.comment?.comment.score === 1,
)
).comment;

Expand All @@ -397,12 +397,12 @@ test("Reply to a comment from another instance, get notification", async () => {
expect(getCommentParentId(replyRes.comment_view.comment)).toBe(
betaComment.comment.id,
);
expect(replyRes.comment_view.counts.score).toBe(1);
expect(replyRes.comment_view.comment.score).toBe(1);

// Make sure that reply comment is seen on alpha
let commentSearch = await waitUntil(
() => resolveComment(alpha, replyRes.comment_view.comment),
c => c.comment?.counts.score === 1,
c => c.comment?.comment.score === 1,
);
let alphaComment = commentSearch.comment!;
let postComments = await waitUntil(
Expand All @@ -418,7 +418,7 @@ test("Reply to a comment from another instance, get notification", async () => {
);
expect(alphaComment.community.local).toBe(false);
expect(alphaComment.creator.local).toBe(false);
expect(alphaComment.counts.score).toBe(1);
expect(alphaComment.comment.score).toBe(1);
assertCommentFederation(alphaComment, replyRes.comment_view);

// Did alpha get notified of the reply from beta?
Expand All @@ -441,7 +441,7 @@ test("Reply to a comment from another instance, get notification", async () => {
expect(alphaReply.comment.content).toBeDefined();
expect(alphaReply.community.local).toBe(false);
expect(alphaReply.creator.local).toBe(false);
expect(alphaReply.counts.score).toBe(1);
expect(alphaReply.comment.score).toBe(1);
// ToDo: interesting alphaRepliesRes.replies[0].comment_reply.id is 1, meaning? how did that come about?
expect(alphaReply.comment.id).toBe(alphaComment.comment.id);
// this is a new notification, getReplies fetch was for read/unread both, confirm it is unread.
Expand Down Expand Up @@ -515,7 +515,7 @@ test("Mention beta from alpha comment", async () => {
expect(mentionRes.comment_view.comment.content).toBeDefined();
expect(mentionRes.comment_view.community.local).toBe(false);
expect(mentionRes.comment_view.creator.local).toBe(true);
expect(mentionRes.comment_view.counts.score).toBe(1);
expect(mentionRes.comment_view.comment.score).toBe(1);

// get beta's localized copy of the alpha post
let betaPost = await waitForPost(beta, postOnAlphaRes.post_view.post);
Expand All @@ -528,7 +528,7 @@ test("Mention beta from alpha comment", async () => {
// Make sure that both new comments are seen on beta and have parent/child relationship
let betaPostComments = await waitUntil(
() => getComments(beta, betaPost!.post.id),
c => c.comments[1]?.counts.score === 1,
c => c.comments[1]?.comment.score === 1,
);
expect(betaPostComments.comments.length).toEqual(2);
// the trunk-branch root comment will be older than the mention reply comment, so index 1
Expand All @@ -542,7 +542,7 @@ test("Mention beta from alpha comment", async () => {
);
expect(betaRootComment.community.local).toBe(true);
expect(betaRootComment.creator.local).toBe(false);
expect(betaRootComment.counts.score).toBe(1);
expect(betaRootComment.comment.score).toBe(1);
assertCommentFederation(betaRootComment, commentRes.comment_view);

let mentionsRes = await waitUntil(
Expand All @@ -554,7 +554,7 @@ test("Mention beta from alpha comment", async () => {
expect(firstMention.comment.content).toBeDefined();
expect(firstMention.community.local).toBe(true);
expect(firstMention.creator.local).toBe(false);
expect(firstMention.counts.score).toBe(1);
expect(firstMention.comment.score).toBe(1);
// the reply comment with mention should be the most fresh, newest, index 0
expect(firstMention.person_comment_mention.comment_id).toBe(
betaPostComments.comments[0].comment.id,
Expand Down Expand Up @@ -606,17 +606,17 @@ test("A and G subscribe to B (center) A posts, G mentions B, it gets announced t
expect(commentRes.comment_view.comment.content).toBe(commentContent);
expect(commentRes.comment_view.community.local).toBe(false);
expect(commentRes.comment_view.creator.local).toBe(true);
expect(commentRes.comment_view.counts.score).toBe(1);
expect(commentRes.comment_view.comment.score).toBe(1);

// Make sure alpha sees it
let alphaPostComments2 = await waitUntil(
() => getComments(alpha, alphaPost.post_view.post.id),
e => e.comments[0]?.counts.score === 1,
e => e.comments[0]?.comment.score === 1,
);
expect(alphaPostComments2.comments[0].comment.content).toBe(commentContent);
expect(alphaPostComments2.comments[0].community.local).toBe(true);
expect(alphaPostComments2.comments[0].creator.local).toBe(false);
expect(alphaPostComments2.comments[0].counts.score).toBe(1);
expect(alphaPostComments2.comments[0].comment.score).toBe(1);
assertCommentFederation(
alphaPostComments2.comments[0],
commentRes.comment_view,
Expand Down Expand Up @@ -688,17 +688,17 @@ test("Check that activity from another instance is sent to third instance", asyn
expect(commentRes.comment_view.comment.content).toBe(commentContent);
expect(commentRes.comment_view.community.local).toBe(false);
expect(commentRes.comment_view.creator.local).toBe(true);
expect(commentRes.comment_view.counts.score).toBe(1);
expect(commentRes.comment_view.comment.score).toBe(1);

// Make sure alpha sees it
let alphaPostComments2 = await waitUntil(
() => getComments(alpha, alphaPost!.post.id),
e => e.comments[0]?.counts.score === 1,
e => e.comments[0]?.comment.score === 1,
);
expect(alphaPostComments2.comments[0].comment.content).toBe(commentContent);
expect(alphaPostComments2.comments[0].community.local).toBe(false);
expect(alphaPostComments2.comments[0].creator.local).toBe(false);
expect(alphaPostComments2.comments[0].counts.score).toBe(1);
expect(alphaPostComments2.comments[0].comment.score).toBe(1);
assertCommentFederation(
alphaPostComments2.comments[0],
commentRes.comment_view,
Expand Down
14 changes: 7 additions & 7 deletions api_tests/src/community.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ test.skip("Community follower count is federated", async () => {
).community;

// Make sure there is 1 subscriber
expect(followed?.counts.subscribers).toBe(1);
expect(followed?.community.subscribers).toBe(1);

// Follow the community from gamma
resolved = await resolveCommunity(gamma, communityActorId);
Expand All @@ -413,7 +413,7 @@ test.skip("Community follower count is federated", async () => {
).community;

// Make sure there are 2 subscribers
expect(followed?.counts?.subscribers).toBe(2);
expect(followed?.community?.subscribers).toBe(2);

// Follow the community from delta
resolved = await resolveCommunity(delta, communityActorId);
Expand All @@ -430,13 +430,13 @@ test.skip("Community follower count is federated", async () => {
).community;

// Make sure there are 3 subscribers
expect(followed?.counts?.subscribers).toBe(3);
expect(followed?.community?.subscribers).toBe(3);
});

test("Dont receive community activities after unsubscribe", async () => {
let communityRes = await createCommunity(alpha);
expect(communityRes.community_view.community.name).toBeDefined();
expect(communityRes.community_view.counts.subscribers).toBe(1);
expect(communityRes.community_view.community.subscribers).toBe(1);

let betaCommunity = (
await resolveCommunity(beta, communityRes.community_view.community.ap_id)
Expand All @@ -451,7 +451,7 @@ test("Dont receive community activities after unsubscribe", async () => {
alpha,
communityRes.community_view.community.id,
);
expect(communityRes1.community_view.counts.subscribers).toBe(2);
expect(communityRes1.community_view.community.subscribers).toBe(2);

// temporarily block alpha, so that it doesn't know about unfollow
var allow_instance_params: AdminAllowInstanceParams = {
Expand All @@ -470,7 +470,7 @@ test("Dont receive community activities after unsubscribe", async () => {
alpha,
communityRes.community_view.community.id,
);
expect(communityRes2.community_view.counts.subscribers).toBe(2);
expect(communityRes2.community_view.community.subscribers).toBe(2);

// unblock alpha
allow_instance_params.allow = true;
Expand All @@ -492,7 +492,7 @@ test("Dont receive community activities after unsubscribe", async () => {
test("Fetch community, includes posts", async () => {
let communityRes = await createCommunity(alpha);
expect(communityRes.community_view.community.name).toBeDefined();
expect(communityRes.community_view.counts.subscribers).toBe(1);
expect(communityRes.community_view.community.subscribers).toBe(1);

let postRes = await createPost(
alpha,
Expand Down
36 changes: 18 additions & 18 deletions api_tests/src/follow.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,21 @@ test("Follow local community", async () => {
// Make sure the follow response went through
expect(follow.community_view.community.local).toBe(true);
expect(follow.community_view.subscribed).toBe("Subscribed");
expect(follow.community_view.counts.subscribers).toBe(
community.counts.subscribers + 1,
expect(follow.community_view.community.subscribers).toBe(
community.community.subscribers + 1,
);
expect(follow.community_view.counts.subscribers_local).toBe(
community.counts.subscribers_local + 1,
expect(follow.community_view.community.subscribers_local).toBe(
community.community.subscribers_local + 1,
);

// Test an unfollow
let unfollow = await followCommunity(user, false, community.community.id);
expect(unfollow.community_view.subscribed).toBe("NotSubscribed");
expect(unfollow.community_view.counts.subscribers).toBe(
community.counts.subscribers,
expect(unfollow.community_view.community.subscribers).toBe(
community.community.subscribers,
);
expect(unfollow.community_view.counts.subscribers_local).toBe(
community.counts.subscribers_local,
expect(unfollow.community_view.community.subscribers_local).toBe(
community.community.subscribers_local,
);
});

Expand All @@ -51,7 +51,7 @@ test("Follow federated community", async () => {
const betaCommunityInitial = (
await waitUntil(
() => resolveBetaCommunity(alpha),
c => !!c.community && c.community?.counts.subscribers >= 1,
c => !!c.community && c.community?.community.subscribers >= 1,
)
).community;
if (!betaCommunityInitial) {
Expand All @@ -74,14 +74,14 @@ test("Follow federated community", async () => {
expect(betaCommunity?.community.local).toBe(false);
expect(betaCommunity?.community.name).toBe("main");
expect(betaCommunity?.subscribed).toBe("Subscribed");
expect(betaCommunity?.counts.subscribers_local).toBe(
betaCommunityInitial.counts.subscribers_local + 1,
expect(betaCommunity?.community.subscribers_local).toBe(
betaCommunityInitial.community.subscribers_local + 1,
);

// check that unfollow was federated
let communityOnBeta1 = await resolveBetaCommunity(beta);
expect(communityOnBeta1.community?.counts.subscribers).toBe(
betaCommunityInitial.counts.subscribers + 1,
expect(communityOnBeta1.community?.community.subscribers).toBe(
betaCommunityInitial.community.subscribers + 1,
);

// Check it from local
Expand Down Expand Up @@ -113,11 +113,11 @@ test("Follow federated community", async () => {
let communityOnBeta2 = await waitUntil(
() => resolveBetaCommunity(beta),
c =>
c.community?.counts.subscribers ===
betaCommunityInitial.counts.subscribers,
c.community?.community.subscribers ===
betaCommunityInitial.community.subscribers,
);
expect(communityOnBeta2.community?.counts.subscribers).toBe(
betaCommunityInitial.counts.subscribers,
expect(communityOnBeta2.community?.community.subscribers).toBe(
betaCommunityInitial.community.subscribers,
);
expect(communityOnBeta2.community?.counts.subscribers_local).toBe(1);
expect(communityOnBeta2.community?.community.subscribers_local).toBe(1);
});
Loading