From 5ca40a760cc34a4e0b00cbfb0f3b8361a0f44957 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 7 Feb 2025 12:16:32 +0100 Subject: [PATCH 01/39] migration --- .../down.sql | 36 +++++++++++++++++++ .../up.sql | 28 +++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 migrations/2025-02-07-105516_remove-aggregate-tables/down.sql create mode 100644 migrations/2025-02-07-105516_remove-aggregate-tables/up.sql diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql new file mode 100644 index 0000000000..c81582cfd3 --- /dev/null +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql @@ -0,0 +1,36 @@ +CREATE TABLE comment_aggregates ( + comment_id integer NOT NULL, + score bigint NOT NULL DEFAULT 0, + upvotes bigint NOT NULL DEFAULT 0, + downvotes bigint NOT NULL DEFAULT 0, + published timestamp with time zone NOT NULL DEFAULT now(), + child_count integer NOT NULL DEFAULT 0, + hot_rank double precision NOT NULL DEFAULT 0.0001, + controversy_rank double precision NOT NULL DEFAULT 0, + report_count smallint NOT NULL DEFAULT 0, + unresolved_report_count smallint NOT NULL DEFAULT 0 +); + +INSERT INTO comment_aggregates +SELECT + id AS comment_id, + score, + upvotes, + downvotes, + published, + child_count, + hot_rank, + controversy_rank, + report_count, + unresolved_report_count from comment; + +ALTER TABLE comment + DROP COLUMN score, + DROP COLUMN upvotes, + DROP COLUMN downvotes, + DROP COLUMN child_count, + DROP COLUMN hot_rank, + DROP COLUMN controversy_rank, + DROP COLUMN report_count, + DROP COLUMN unresolved_report_count; + diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql new file mode 100644 index 0000000000..4fa9c83741 --- /dev/null +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -0,0 +1,28 @@ +ALTER TABLE comment + ADD COLUMN score bigint NOT NULL DEFAULT 0, + ADD COLUMN upvotes bigint NOT NULL DEFAULT 0, + ADD COLUMN downvotes bigint NOT NULL DEFAULT 0, + ADD COLUMN child_count integer NOT NULL DEFAULT 0, + ADD COLUMN hot_rank double precision NOT NULL DEFAULT 0.0001, + ADD COLUMN controversy_rank double precision NOT NULL DEFAULT 0, + ADD COLUMN report_count smallint NOT NULL DEFAULT 0, + ADD COLUMN unresolved_report_count smallint NOT NULL DEFAULT 0; + +UPDATE + comment +SET + score = ca.score, + upvotes = ca.upvotes, + downvotes = ca.downvotes, + child_count = ca.child_count, + hot_rank = ca.hot_rank, + controversy_rank = ca.controversy_rank, + report_count = ca.report_count, + unresolved_report_count = ca.unresolved_report_count +FROM + comment_aggregates AS ca +WHERE + comment.id = ca.comment_id; + +DROP TABLE comment_aggregates; + From a3b835f815032a890b25e3fbf7f6ecd9db2eccb1 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 7 Feb 2025 12:52:51 +0100 Subject: [PATCH 02/39] update code --- .../activities/create_or_update/comment.rs | 3 +- .../src/aggregates/comment_aggregates.rs | 153 ----------- crates/db_schema/src/aggregates/mod.rs | 2 - crates/db_schema/src/aggregates/structs.rs | 29 +-- crates/db_schema/src/impls/comment.rs | 20 +- crates/db_schema/src/schema.rs | 241 +++++++++--------- crates/db_schema/src/source/comment.rs | 47 +++- .../src/combined/inbox_combined_view.rs | 19 +- .../combined/person_content_combined_view.rs | 13 +- .../combined/person_saved_combined_view.rs | 2 - .../src/combined/report_combined_view.rs | 9 - .../src/combined/search_combined_view.rs | 10 +- crates/db_views/src/comment/comment_view.rs | 16 +- .../src/reports/comment_report_view.rs | 6 - crates/db_views/src/structs.rs | 18 +- 15 files changed, 193 insertions(+), 395 deletions(-) delete mode 100644 crates/db_schema/src/aggregates/comment_aggregates.rs diff --git a/crates/apub/src/activities/create_or_update/comment.rs b/crates/apub/src/activities/create_or_update/comment.rs index 0b6936f3be..4474d24765 100644 --- a/crates/apub/src/activities/create_or_update/comment.rs +++ b/crates/apub/src/activities/create_or_update/comment.rs @@ -28,7 +28,6 @@ use lemmy_api_common::{ utils::{check_post_deleted_or_removed, is_mod_or_admin}, }; use lemmy_db_schema::{ - aggregates::structs::CommentAggregates, newtypes::{PersonId, PostOrCommentId}, source::{ activity::ActivitySendTargets, @@ -160,7 +159,7 @@ impl ActivityHandler for CreateOrUpdateNote { CommentLike::like(&mut context.pool(), &like_form).await?; // Calculate initial hot_rank - CommentAggregates::update_hot_rank(&mut context.pool(), comment.id).await?; + Comment::update_hot_rank(&mut context.pool(), comment.id).await?; let do_send_email = self.kind == CreateOrUpdateType::Create; let actor = self.actor.dereference(context).await?; diff --git a/crates/db_schema/src/aggregates/comment_aggregates.rs b/crates/db_schema/src/aggregates/comment_aggregates.rs deleted file mode 100644 index b26d277363..0000000000 --- a/crates/db_schema/src/aggregates/comment_aggregates.rs +++ /dev/null @@ -1,153 +0,0 @@ -use crate::{ - aggregates::structs::CommentAggregates, - newtypes::CommentId, - schema::comment_aggregates, - utils::{functions::hot_rank, get_conn, DbPool}, -}; -use diesel::{result::Error, ExpressionMethods, QueryDsl}; -use diesel_async::RunQueryDsl; - -impl CommentAggregates { - pub async fn read(pool: &mut DbPool<'_>, comment_id: CommentId) -> Result { - let conn = &mut get_conn(pool).await?; - comment_aggregates::table.find(comment_id).first(conn).await - } - - pub async fn update_hot_rank( - pool: &mut DbPool<'_>, - comment_id: CommentId, - ) -> Result { - let conn = &mut get_conn(pool).await?; - - diesel::update(comment_aggregates::table.find(comment_id)) - .set(comment_aggregates::hot_rank.eq(hot_rank( - comment_aggregates::score, - comment_aggregates::published, - ))) - .get_result::(conn) - .await - } -} - -#[cfg(test)] -mod tests { - - use crate::{ - aggregates::comment_aggregates::CommentAggregates, - source::{ - comment::{Comment, CommentInsertForm, CommentLike, CommentLikeForm}, - community::{Community, CommunityInsertForm}, - instance::Instance, - person::{Person, PersonInsertForm}, - post::{Post, PostInsertForm}, - }, - traits::{Crud, Likeable}, - utils::build_db_pool_for_tests, - }; - use diesel::result::Error; - use pretty_assertions::assert_eq; - use serial_test::serial; - - #[tokio::test] - #[serial] - async fn test_crud() -> Result<(), Error> { - let pool = &build_db_pool_for_tests(); - let pool = &mut pool.into(); - - let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; - - let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_comment_agg"); - - let inserted_person = Person::create(pool, &new_person).await?; - - let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_comment_agg"); - - let another_inserted_person = Person::create(pool, &another_person).await?; - - let new_community = CommunityInsertForm::new( - inserted_instance.id, - "TIL_comment_agg".into(), - "nada".to_owned(), - "pubkey".to_string(), - ); - let inserted_community = Community::create(pool, &new_community).await?; - - let new_post = PostInsertForm::new( - "A test post".into(), - inserted_person.id, - inserted_community.id, - ); - let inserted_post = Post::create(pool, &new_post).await?; - - let comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - let inserted_comment = Comment::create(pool, &comment_form, None).await?; - - let child_comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - let _inserted_child_comment = - Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; - - let comment_like = CommentLikeForm { - comment_id: inserted_comment.id, - person_id: inserted_person.id, - score: 1, - }; - - CommentLike::like(pool, &comment_like).await?; - - let comment_aggs_before_delete = CommentAggregates::read(pool, inserted_comment.id).await?; - - assert_eq!(1, comment_aggs_before_delete.score); - assert_eq!(1, comment_aggs_before_delete.upvotes); - assert_eq!(0, comment_aggs_before_delete.downvotes); - - // Add a post dislike from the other person - let comment_dislike = CommentLikeForm { - comment_id: inserted_comment.id, - person_id: another_inserted_person.id, - score: -1, - }; - - CommentLike::like(pool, &comment_dislike).await?; - - let comment_aggs_after_dislike = CommentAggregates::read(pool, inserted_comment.id).await?; - - assert_eq!(0, comment_aggs_after_dislike.score); - assert_eq!(1, comment_aggs_after_dislike.upvotes); - assert_eq!(1, comment_aggs_after_dislike.downvotes); - - // Remove the first comment like - CommentLike::remove(pool, inserted_person.id, inserted_comment.id).await?; - let after_like_remove = CommentAggregates::read(pool, inserted_comment.id).await?; - assert_eq!(-1, after_like_remove.score); - assert_eq!(0, after_like_remove.upvotes); - assert_eq!(1, after_like_remove.downvotes); - - // Remove the parent post - Post::delete(pool, inserted_post.id).await?; - - // Should be none found, since the post was deleted - let after_delete = CommentAggregates::read(pool, inserted_comment.id).await; - assert!(after_delete.is_err()); - - // This should delete all the associated rows, and fire triggers - Person::delete(pool, another_inserted_person.id).await?; - let person_num_deleted = Person::delete(pool, inserted_person.id).await?; - assert_eq!(1, person_num_deleted); - - // Delete the community - let community_num_deleted = Community::delete(pool, inserted_community.id).await?; - assert_eq!(1, community_num_deleted); - - Instance::delete(pool, inserted_instance.id).await?; - - Ok(()) - } -} diff --git a/crates/db_schema/src/aggregates/mod.rs b/crates/db_schema/src/aggregates/mod.rs index d55f188f3f..0df9ffc744 100644 --- a/crates/db_schema/src/aggregates/mod.rs +++ b/crates/db_schema/src/aggregates/mod.rs @@ -1,6 +1,4 @@ #[cfg(feature = "full")] -pub mod comment_aggregates; -#[cfg(feature = "full")] pub mod community_aggregates; #[cfg(feature = "full")] pub mod person_aggregates; diff --git a/crates/db_schema/src/aggregates/structs.rs b/crates/db_schema/src/aggregates/structs.rs index 86176c14c2..89c28d9958 100644 --- a/crates/db_schema/src/aggregates/structs.rs +++ b/crates/db_schema/src/aggregates/structs.rs @@ -1,7 +1,6 @@ -use crate::newtypes::{CommentId, CommunityId, InstanceId, PersonId, PostId, SiteId}; +use crate::newtypes::{CommunityId, InstanceId, PersonId, PostId, SiteId}; #[cfg(feature = "full")] use crate::schema::{ - comment_aggregates, community_aggregates, person_aggregates, post_actions, @@ -16,32 +15,6 @@ use i_love_jesus::CursorKeysModule; use serde::{Deserialize, Serialize}; #[cfg(feature = "full")] use ts_rs::TS; -#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] -#[cfg_attr( - feature = "full", - derive(Queryable, Selectable, Associations, Identifiable, TS) -)] -#[cfg_attr(feature = "full", diesel(table_name = comment_aggregates))] -#[cfg_attr(feature = "full", diesel(belongs_to(crate::source::comment::Comment)))] -#[cfg_attr(feature = "full", diesel(primary_key(comment_id)))] -#[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))] -#[cfg_attr(feature = "full", ts(export))] -/// Aggregate data for a comment. -pub struct CommentAggregates { - pub comment_id: CommentId, - pub score: i64, - pub upvotes: i64, - pub downvotes: i64, - pub published: DateTime, - /// The total number of children in this comment branch. - pub child_count: i32, - #[serde(skip)] - pub hot_rank: f64, - #[serde(skip)] - pub controversy_rank: f64, - pub report_count: i16, - pub unresolved_report_count: i16, -} #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] #[cfg_attr( diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 17cd6ce5cd..8f0ad1bf62 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -12,7 +12,14 @@ use crate::{ CommentUpdateForm, }, traits::{Crud, Likeable, Saveable}, - utils::{functions::coalesce, get_conn, now, uplete, DbPool, DELETED_REPLACEMENT_TEXT}, + utils::{ + functions::{coalesce, hot_rank}, + get_conn, + now, + uplete, + DbPool, + DELETED_REPLACEMENT_TEXT, + }, }; use chrono::{DateTime, Utc}; use diesel::{ @@ -116,6 +123,17 @@ impl Comment { None } } + pub async fn update_hot_rank( + pool: &mut DbPool<'_>, + comment_id: CommentId, + ) -> Result { + let conn = &mut get_conn(pool).await?; + + diesel::update(comment::table.find(comment_id)) + .set(comment::hot_rank.eq(hot_rank(comment::score, comment::published))) + .get_result::(conn) + .await + } } #[async_trait] diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 3fecdd7378..9f1f1037fa 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -1,45 +1,45 @@ // @generated automatically by Diesel CLI. pub mod sql_types { - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "actor_type_enum"))] - pub struct ActorTypeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "actor_type_enum"))] + pub struct ActorTypeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "comment_sort_type_enum"))] - pub struct CommentSortTypeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "comment_sort_type_enum"))] + pub struct CommentSortTypeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "community_follower_state"))] - pub struct CommunityFollowerState; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "community_follower_state"))] + pub struct CommunityFollowerState; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "community_visibility"))] - pub struct CommunityVisibility; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "community_visibility"))] + pub struct CommunityVisibility; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "federation_mode_enum"))] - pub struct FederationModeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "federation_mode_enum"))] + pub struct FederationModeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "listing_type_enum"))] - pub struct ListingTypeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "listing_type_enum"))] + pub struct ListingTypeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "ltree"))] - pub struct Ltree; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "ltree"))] + pub struct Ltree; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "post_listing_mode_enum"))] - pub struct PostListingModeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "post_listing_mode_enum"))] + pub struct PostListingModeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "post_sort_type_enum"))] - pub struct PostSortTypeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "post_sort_type_enum"))] + pub struct PostSortTypeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "registration_mode_enum"))] - pub struct RegistrationModeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "registration_mode_enum"))] + pub struct RegistrationModeEnum; } diesel::table! { @@ -130,6 +130,14 @@ diesel::table! { path -> Ltree, distinguished -> Bool, language_id -> Int4, + score -> Int8, + upvotes -> Int8, + downvotes -> Int8, + child_count -> Int4, + hot_rank -> Float8, + controversy_rank -> Float8, + report_count -> Int2, + unresolved_report_count -> Int2, } } @@ -143,21 +151,6 @@ diesel::table! { } } -diesel::table! { - comment_aggregates (comment_id) { - comment_id -> Int4, - score -> Int8, - upvotes -> Int8, - downvotes -> Int8, - published -> Timestamptz, - child_count -> Int4, - hot_rank -> Float8, - controversy_rank -> Float8, - report_count -> Int2, - unresolved_report_count -> Int2, - } -} - diesel::table! { comment_reply (id) { id -> Int4, @@ -1114,7 +1107,6 @@ diesel::joinable!(comment -> person (creator_id)); diesel::joinable!(comment -> post (post_id)); diesel::joinable!(comment_actions -> comment (comment_id)); diesel::joinable!(comment_actions -> person (person_id)); -diesel::joinable!(comment_aggregates -> comment (comment_id)); diesel::joinable!(comment_reply -> comment (comment_id)); diesel::joinable!(comment_reply -> person (recipient_id)); diesel::joinable!(comment_report -> comment (comment_id)); @@ -1220,83 +1212,82 @@ diesel::joinable!(site_language -> site (site_id)); diesel::joinable!(tag -> community (community_id)); diesel::allow_tables_to_appear_in_same_query!( - admin_allow_instance, - admin_block_instance, - admin_purge_comment, - admin_purge_community, - admin_purge_person, - admin_purge_post, - captcha_answer, - comment, - comment_actions, - comment_aggregates, - comment_reply, - comment_report, - community, - community_actions, - community_aggregates, - community_language, - community_report, - custom_emoji, - custom_emoji_keyword, - email_verification, - federation_allowlist, - federation_blocklist, - federation_queue_state, - image_details, - inbox_combined, - instance, - instance_actions, - language, - local_image, - local_site, - local_site_rate_limit, - local_site_url_blocklist, - local_user, - local_user_language, - local_user_vote_display_mode, - login_token, - mod_add, - mod_add_community, - mod_ban, - mod_ban_from_community, - mod_feature_post, - mod_hide_community, - mod_lock_post, - mod_remove_comment, - mod_remove_community, - mod_remove_post, - mod_transfer_community, - modlog_combined, - oauth_account, - oauth_provider, - password_reset_request, - person, - person_actions, - person_aggregates, - person_ban, - person_comment_mention, - person_content_combined, - person_post_mention, - person_saved_combined, - post, - post_actions, - post_aggregates, - post_report, - post_tag, - previously_run_sql, - private_message, - private_message_report, - received_activity, - registration_application, - remote_image, - report_combined, - search_combined, - secret, - sent_activity, - site, - site_aggregates, - site_language, - tag, - tagline, + admin_allow_instance, + admin_block_instance, + admin_purge_comment, + admin_purge_community, + admin_purge_person, + admin_purge_post, + captcha_answer, + comment, + comment_actions, + comment_reply, + comment_report, + community, + community_actions, + community_aggregates, + community_language, + community_report, + custom_emoji, + custom_emoji_keyword, + email_verification, + federation_allowlist, + federation_blocklist, + federation_queue_state, + image_details, + inbox_combined, + instance, + instance_actions, + language, + local_image, + local_site, + local_site_rate_limit, + local_site_url_blocklist, + local_user, + local_user_language, + local_user_vote_display_mode, + login_token, + mod_add, + mod_add_community, + mod_ban, + mod_ban_from_community, + mod_feature_post, + mod_hide_community, + mod_lock_post, + mod_remove_comment, + mod_remove_community, + mod_remove_post, + mod_transfer_community, + modlog_combined, + oauth_account, + oauth_provider, + password_reset_request, + person, + person_actions, + person_aggregates, + person_ban, + person_comment_mention, + person_content_combined, + person_post_mention, + person_saved_combined, + post, + post_actions, + post_aggregates, + post_report, + post_tag, + previously_run_sql, + private_message, + private_message_report, + received_activity, + registration_application, + remote_image, + report_combined, + search_combined, + secret, + sent_activity, + site, + site_aggregates, + site_language, + tag, + tagline, ); diff --git a/crates/db_schema/src/source/comment.rs b/crates/db_schema/src/source/comment.rs index cc5d8c20c1..d30f0a9e8b 100644 --- a/crates/db_schema/src/source/comment.rs +++ b/crates/db_schema/src/source/comment.rs @@ -14,7 +14,7 @@ use serde_with::skip_serializing_none; use ts_rs::TS; #[skip_serializing_none] -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[cfg_attr( feature = "full", derive(Queryable, Selectable, Associations, Identifiable, TS) @@ -51,8 +51,53 @@ pub struct Comment { /// Whether the comment has been distinguished(speaking officially) by a mod. pub distinguished: bool, pub language_id: LanguageId, + pub score: i64, + pub upvotes: i64, + pub downvotes: i64, + /// The total number of children in this comment branch. + pub child_count: i32, + #[serde(skip)] + pub hot_rank: f64, + #[serde(skip)] + pub controversy_rank: f64, + pub report_count: i16, + pub unresolved_report_count: i16, } +impl PartialEq for Comment { + fn eq(&self, _other: &Self) -> bool { + // TODO: is this really needed? seems only for tests, so we could rewrite + // them to compare individual fields. or use `derivative` crate + todo!() + /* + self.id == other.id + && self.creator_id == other.creator_id + && self.post_id == other.post_id + && self.content == other.content + && self.removed == other.removed + && self.published == other.published + && self.updated == other.updated + && self.deleted == other.deleted + && self.ap_id == other.ap_id + && self.local == other.local + && self.path == other.path + && self.path == other.path + && self.distinguished == other.distinguished + && self.language_id == other.language_id + && self.score == other.score + && self.upvotes == other.upvotes + && self.downvotes == other.downvotes + && self.child_count == other.child_count + && self.hot_rank == other.hot_rank + && self.controversy_rank == other.controversy_rank + && self.report_count == other.report_count + && self.unresolved_report_count == other.unresolved_report_count + */ + } +} + +impl Eq for Comment {} + #[derive(Debug, Clone, derive_new::new)] #[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] #[cfg_attr(feature = "full", diesel(table_name = comment))] diff --git a/crates/db_views/src/combined/inbox_combined_view.rs b/crates/db_views/src/combined/inbox_combined_view.rs index 7e7bc1f206..66a0e4ed02 100644 --- a/crates/db_views/src/combined/inbox_combined_view.rs +++ b/crates/db_views/src/combined/inbox_combined_view.rs @@ -25,7 +25,6 @@ use lemmy_db_schema::{ schema::{ comment, comment_actions, - comment_aggregates, comment_reply, community, community_actions, @@ -105,8 +104,6 @@ impl InboxCombinedViewInternal { ); let post_aggregates_join = post_aggregates::table.on(post::id.eq(post_aggregates::post_id)); - let comment_aggregates_join = - comment_aggregates::table.on(comment::id.eq(comment_aggregates::comment_id)); let image_details_join = image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())); @@ -164,7 +161,6 @@ impl InboxCombinedViewInternal { .inner_join(recipient_join) .left_join(image_details_join) .left_join(post_aggregates_join) - .left_join(comment_aggregates_join) .left_join(creator_community_actions_join) .left_join(local_user_join) .left_join(community_actions_join) @@ -304,7 +300,6 @@ impl InboxCombinedQuery { post::all_columns.nullable(), community::all_columns.nullable(), comment::all_columns.nullable(), - comment_aggregates::all_columns.nullable(), comment_actions::saved.nullable().is_not_null(), comment_actions::like_score.nullable(), CommunityFollower::select_subscribed_type(), @@ -424,17 +419,15 @@ impl InternalToCombinedView for InboxCombinedViewInternal { // Use for a short alias let v = self; - if let (Some(comment_reply), Some(comment), Some(counts), Some(post), Some(community)) = ( + if let (Some(comment_reply), Some(comment), Some(post), Some(community)) = ( v.comment_reply, v.comment.clone(), - v.comment_counts.clone(), v.post.clone(), v.community.clone(), ) { Some(InboxCombinedView::CommentReply(CommentReplyView { comment_reply, comment, - counts, recipient: v.item_recipient, post, community, @@ -448,16 +441,9 @@ impl InternalToCombinedView for InboxCombinedViewInternal { my_vote: v.my_comment_vote, banned_from_community: v.banned_from_community, })) - } else if let ( - Some(person_comment_mention), - Some(comment), - Some(counts), - Some(post), - Some(community), - ) = ( + } else if let (Some(person_comment_mention), Some(comment), Some(post), Some(community)) = ( v.person_comment_mention, v.comment, - v.comment_counts, v.post.clone(), v.community.clone(), ) { @@ -465,7 +451,6 @@ impl InternalToCombinedView for InboxCombinedViewInternal { PersonCommentMentionView { person_comment_mention, comment, - counts, recipient: v.item_recipient, post, community, diff --git a/crates/db_views/src/combined/person_content_combined_view.rs b/crates/db_views/src/combined/person_content_combined_view.rs index 414f847542..67a4064c56 100644 --- a/crates/db_views/src/combined/person_content_combined_view.rs +++ b/crates/db_views/src/combined/person_content_combined_view.rs @@ -23,7 +23,6 @@ use lemmy_db_schema::{ schema::{ comment, comment_actions, - comment_aggregates, community, community_actions, image_details, @@ -119,9 +118,6 @@ impl PersonContentCombinedViewInternal { let post_aggregates_join = post_aggregates::table.on(post::id.eq(post_aggregates::post_id)); - let comment_aggregates_join = comment_aggregates::table - .on(person_content_combined::comment_id.eq(comment_aggregates::comment_id.nullable())); - let image_details_join = image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())); @@ -136,7 +132,6 @@ impl PersonContentCombinedViewInternal { .left_join(post_actions_join) .left_join(person_actions_join) .inner_join(post_aggregates_join) - .left_join(comment_aggregates_join) .left_join(comment_actions_join) .left_join(image_details_join) } @@ -211,9 +206,6 @@ impl PersonContentCombinedViewInternal { let post_aggregates_join = post_aggregates::table.on(post::id.eq(post_aggregates::post_id)); - let comment_aggregates_join = comment_aggregates::table - .on(person_saved_combined::comment_id.eq(comment_aggregates::comment_id.nullable())); - let image_details_join = image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())); @@ -228,7 +220,6 @@ impl PersonContentCombinedViewInternal { .left_join(post_actions_join) .left_join(person_actions_join) .inner_join(post_aggregates_join) - .left_join(comment_aggregates_join) .left_join(comment_actions_join) .left_join(image_details_join) } @@ -320,7 +311,6 @@ impl PersonContentCombinedQuery { post_tags, // Comment-specific comment::all_columns.nullable(), - comment_aggregates::all_columns.nullable(), comment_actions::saved.nullable(), comment_actions::like_score.nullable(), // Shared @@ -389,10 +379,9 @@ impl InternalToCombinedView for PersonContentCombinedViewInternal { // Use for a short alias let v = self; - if let (Some(comment), Some(counts)) = (v.comment, v.comment_counts) { + if let Some(comment) = v.comment { Some(PersonContentCombinedView::Comment(CommentView { comment, - counts, post: v.post, community: v.community, creator: v.item_creator, diff --git a/crates/db_views/src/combined/person_saved_combined_view.rs b/crates/db_views/src/combined/person_saved_combined_view.rs index 11356f802a..7add423207 100644 --- a/crates/db_views/src/combined/person_saved_combined_view.rs +++ b/crates/db_views/src/combined/person_saved_combined_view.rs @@ -18,7 +18,6 @@ use lemmy_db_schema::{ schema::{ comment, comment_actions, - comment_aggregates, community, community_actions, image_details, @@ -117,7 +116,6 @@ impl PersonSavedCombinedQuery { post_tags, // Comment-specific comment::all_columns.nullable(), - comment_aggregates::all_columns.nullable(), comment_actions::saved.nullable(), comment_actions::like_score.nullable(), // Shared diff --git a/crates/db_views/src/combined/report_combined_view.rs b/crates/db_views/src/combined/report_combined_view.rs index 115c8413a5..e4e4775aeb 100644 --- a/crates/db_views/src/combined/report_combined_view.rs +++ b/crates/db_views/src/combined/report_combined_view.rs @@ -26,7 +26,6 @@ use lemmy_db_schema::{ schema::{ comment, comment_actions, - comment_aggregates, comment_report, community, community_actions, @@ -143,9 +142,6 @@ impl ReportCombinedViewInternal { let post_aggregates_join = post_aggregates::table.on(post_report::post_id.eq(post_aggregates::post_id)); - let comment_aggregates_join = - comment_aggregates::table.on(comment_report::comment_id.eq(comment_aggregates::comment_id)); - let community_aggregates_join = community_aggregates::table .on(community_report::community_id.eq(community_aggregates::community_id)); @@ -167,7 +163,6 @@ impl ReportCombinedViewInternal { .left_join(post_actions_join) .left_join(person_actions_join) .left_join(post_aggregates_join) - .left_join(comment_aggregates_join) .left_join(community_aggregates_join) .left_join(comment_actions_join) } @@ -288,7 +283,6 @@ impl ReportCombinedQuery { // Comment-specific comment_report::all_columns.nullable(), comment::all_columns.nullable(), - comment_aggregates::all_columns.nullable(), comment_actions::saved.nullable().is_not_null(), comment_actions::like_score.nullable(), // Private-message-specific @@ -436,14 +430,12 @@ impl InternalToCombinedView for ReportCombinedViewInternal { } else if let ( Some(comment_report), Some(comment), - Some(counts), Some(post), Some(community), Some(comment_creator), ) = ( v.comment_report, v.comment, - v.comment_counts, v.post, v.community.clone(), v.item_creator.clone(), @@ -451,7 +443,6 @@ impl InternalToCombinedView for ReportCombinedViewInternal { Some(ReportCombinedView::Comment(CommentReportView { comment_report, comment, - counts, post, community, creator: v.report_creator, diff --git a/crates/db_views/src/combined/search_combined_view.rs b/crates/db_views/src/combined/search_combined_view.rs index adeea855b4..558577ece5 100644 --- a/crates/db_views/src/combined/search_combined_view.rs +++ b/crates/db_views/src/combined/search_combined_view.rs @@ -27,7 +27,6 @@ use lemmy_db_schema::{ schema::{ comment, comment_actions, - comment_aggregates, community, community_actions, community_aggregates, @@ -142,9 +141,6 @@ impl SearchCombinedViewInternal { let post_aggregates_join = post_aggregates::table.on(post::id.eq(post_aggregates::post_id)); - let comment_aggregates_join = comment_aggregates::table - .on(search_combined::comment_id.eq(comment_aggregates::comment_id.nullable())); - let community_aggregates_join = community_aggregates::table .on(search_combined::community_id.eq(community_aggregates::community_id.nullable())); @@ -166,7 +162,6 @@ impl SearchCombinedViewInternal { .left_join(person_actions_join) .left_join(person_aggregates_join) .left_join(post_aggregates_join) - .left_join(comment_aggregates_join) .left_join(community_aggregates_join) .left_join(comment_actions_join) .left_join(image_details_join) @@ -263,7 +258,6 @@ impl SearchCombinedQuery { post_tags, // Comment-specific comment::all_columns.nullable(), - comment_aggregates::all_columns.nullable(), comment_actions::saved.nullable(), comment_actions::like_score.nullable(), // Community-specific @@ -424,16 +418,14 @@ impl InternalToCombinedView for SearchCombinedViewInternal { // Use for a short alias let v = self; - if let (Some(comment), Some(counts), Some(creator), Some(post), Some(community)) = ( + if let (Some(comment), Some(creator), Some(post), Some(community)) = ( v.comment, - v.comment_counts, v.item_creator.clone(), v.post.clone(), v.community.clone(), ) { Some(SearchCombinedView::Comment(CommentView { comment, - counts, post, community, creator, diff --git a/crates/db_views/src/comment/comment_view.rs b/crates/db_views/src/comment/comment_view.rs index 8b27942173..7cbab97deb 100644 --- a/crates/db_views/src/comment/comment_view.rs +++ b/crates/db_views/src/comment/comment_view.rs @@ -19,7 +19,6 @@ use lemmy_db_schema::{ schema::{ comment, comment_actions, - comment_aggregates, community, community_actions, instance_actions, @@ -48,7 +47,7 @@ impl CommentView { let comment_actions_join = comment_actions::table.on( comment_actions::comment_id - .eq(comment_aggregates::comment_id) + .eq(comment::id) .and(comment_actions::person_id.nullable().eq(my_person_id)), ); @@ -79,7 +78,6 @@ impl CommentView { .inner_join(person::table) .inner_join(post::table) .inner_join(community_join) - .inner_join(comment_aggregates::table) .left_join(community_actions_join) .left_join(comment_actions_join) .left_join(person_actions_join) @@ -129,7 +127,6 @@ impl CommentView { CommentSlimView { comment: self.comment, creator: self.creator, - counts: self.counts, creator_banned_from_community: self.creator_banned_from_community, banned_from_community: self.banned_from_community, creator_is_moderator: self.creator_is_moderator, @@ -306,14 +303,12 @@ impl CommentQuery<'_> { query = match o.sort.unwrap_or(CommentSortType::Hot) { CommentSortType::Hot => query - .then_order_by(comment_aggregates::hot_rank.desc()) - .then_order_by(comment_aggregates::score.desc()), - CommentSortType::Controversial => { - query.then_order_by(comment_aggregates::controversy_rank.desc()) - } + .then_order_by(comment::hot_rank.desc()) + .then_order_by(comment::score.desc()), + CommentSortType::Controversial => query.then_order_by(comment::controversy_rank.desc()), CommentSortType::New => query.then_order_by(comment::published.desc()), CommentSortType::Old => query.then_order_by(comment::published.asc()), - CommentSortType::Top => query.then_order_by(comment_aggregates::score.desc()), + CommentSortType::Top => query.then_order_by(comment::score.desc()), }; let res = query @@ -351,7 +346,6 @@ mod tests { structs::LocalUserView, }; use lemmy_db_schema::{ - aggregates::structs::CommentAggregates, assert_length, impls::actor_language::UNDETERMINED_ID, newtypes::LanguageId, diff --git a/crates/db_views/src/reports/comment_report_view.rs b/crates/db_views/src/reports/comment_report_view.rs index a2fa22a26f..7ae89aab1a 100644 --- a/crates/db_views/src/reports/comment_report_view.rs +++ b/crates/db_views/src/reports/comment_report_view.rs @@ -15,7 +15,6 @@ use lemmy_db_schema::{ schema::{ comment, comment_actions, - comment_aggregates, comment_report, community, community_actions, @@ -48,9 +47,6 @@ impl CommentReportView { let comment_creator_join = aliases::person1.on(comment::creator_id.eq(recipient_id)); - let comment_aggregates_join = - comment_aggregates::table.on(comment_report::comment_id.eq(comment_aggregates::comment_id)); - let comment_actions_join = comment_actions::table.on( comment_actions::comment_id .eq(comment_report::comment_id) @@ -88,7 +84,6 @@ impl CommentReportView { .inner_join(community_join) .inner_join(report_creator_join) .inner_join(comment_creator_join) - .inner_join(comment_aggregates_join) .left_join(comment_actions_join) .left_join(resolver_join) .left_join(creator_community_actions_join) @@ -115,7 +110,6 @@ impl CommentReportView { community::all_columns, person::all_columns, aliases::person1.fields(person::all_columns), - comment_aggregates::all_columns, coalesce( creator_community_actions .field(community_actions::received_ban) diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index fe420a6451..728dfa502b 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -14,13 +14,7 @@ use diesel::{ Selectable, }; use lemmy_db_schema::{ - aggregates::structs::{ - CommentAggregates, - CommunityAggregates, - PersonAggregates, - PostAggregates, - SiteAggregates, - }, + aggregates::structs::{CommunityAggregates, PersonAggregates, PostAggregates, SiteAggregates}, source::{ comment::Comment, comment_reply::CommentReply, @@ -97,7 +91,6 @@ pub struct CommentReportView { pub community: Community, pub creator: Person, pub comment_creator: Person, - pub counts: CommentAggregates, pub creator_banned_from_community: bool, pub creator_is_moderator: bool, pub creator_is_admin: bool, @@ -125,8 +118,6 @@ pub struct CommentView { pub post: Post, #[cfg_attr(feature = "full", diesel(embed))] pub community: Community, - #[cfg_attr(feature = "full", diesel(embed))] - pub counts: CommentAggregates, #[cfg_attr(feature = "full", diesel( select_expression = @@ -209,7 +200,6 @@ pub struct CommentView { pub struct CommentSlimView { pub comment: Comment, pub creator: Person, - pub counts: CommentAggregates, pub creator_banned_from_community: bool, pub banned_from_community: bool, pub creator_is_moderator: bool, @@ -443,7 +433,6 @@ pub struct ReportCombinedViewInternal { // Comment-specific pub comment_report: Option, pub comment: Option, - pub comment_counts: Option, pub comment_saved: bool, pub my_comment_vote: Option, // Private-message-specific @@ -492,7 +481,6 @@ pub(crate) struct PersonContentCombinedViewInternal { pub post_tags: PostTags, // Comment-specific pub comment: Option, - pub comment_counts: Option, pub comment_saved: Option>, pub my_comment_vote: Option, // Shared @@ -617,7 +605,6 @@ pub struct PersonCommentMentionView { pub post: Post, pub community: Community, pub recipient: Person, - pub counts: CommentAggregates, pub creator_banned_from_community: bool, pub banned_from_community: bool, pub creator_is_moderator: bool, @@ -672,7 +659,6 @@ pub struct CommentReplyView { pub post: Post, pub community: Community, pub recipient: Person, - pub counts: CommentAggregates, pub creator_banned_from_community: bool, pub banned_from_community: bool, pub creator_is_moderator: bool, @@ -764,7 +750,6 @@ pub struct InboxCombinedViewInternal { pub post: Option, pub community: Option, pub comment: Option, - pub comment_counts: Option, pub comment_saved: bool, pub my_comment_vote: Option, pub subscribed: SubscribedType, @@ -1136,7 +1121,6 @@ pub(crate) struct SearchCombinedViewInternal { pub post_tags: PostTags, // // Comment-specific pub comment: Option, - pub comment_counts: Option, pub comment_saved: Option>, pub my_comment_vote: Option, // // Community-specific From 549e51f664fdffd72e6430b5cd05dffd033cea4b Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 7 Feb 2025 13:19:59 +0100 Subject: [PATCH 03/39] tests --- crates/db_schema/src/impls/comment.rs | 60 +++++++++---------- .../src/combined/report_combined_view.rs | 8 +-- crates/db_views/src/comment/comment_view.rs | 27 ++++----- 3 files changed, 42 insertions(+), 53 deletions(-) diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 8f0ad1bf62..40c879823b 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -228,18 +228,10 @@ impl Saveable for CommentSaved { #[cfg(test)] mod tests { + use super::*; use crate::{ newtypes::LanguageId, source::{ - comment::{ - Comment, - CommentInsertForm, - CommentLike, - CommentLikeForm, - CommentSaved, - CommentSavedForm, - CommentUpdateForm, - }, community::{Community, CommunityInsertForm}, instance::Instance, person::{Person, PersonInsertForm}, @@ -288,26 +280,6 @@ mod tests { ); let inserted_comment = Comment::create(pool, &comment_form, None).await?; - let expected_comment = Comment { - id: inserted_comment.id, - content: "A test comment".into(), - creator_id: inserted_person.id, - post_id: inserted_post.id, - removed: false, - deleted: false, - path: Ltree(format!("0.{}", inserted_comment.id)), - published: inserted_comment.published, - updated: None, - ap_id: Url::parse(&format!( - "https://lemmy-alpha/comment/{}", - inserted_comment.id - ))? - .into(), - distinguished: false, - local: true, - language_id: LanguageId::default(), - }; - let child_comment_form = CommentInsertForm::new( inserted_person.id, inserted_post.id, @@ -359,13 +331,12 @@ mod tests { Person::delete(pool, inserted_person.id).await?; Instance::delete(pool, inserted_instance.id).await?; - assert_eq!(expected_comment, read_comment); - assert_eq!(expected_comment, inserted_comment); - assert_eq!(expected_comment, updated_comment); + expect_comment(&inserted_comment, &read_comment)?; + expect_comment(&inserted_comment, &updated_comment)?; assert_eq!(expected_comment_like, inserted_comment_like); assert_eq!(expected_comment_saved, inserted_comment_saved); assert_eq!( - format!("0.{}.{}", expected_comment.id, inserted_child_comment.id), + format!("0.{}.{}", read_comment.id, inserted_child_comment.id), inserted_child_comment.path.0, ); assert_eq!(uplete::Count::only_updated(1), like_removed); @@ -374,4 +345,27 @@ mod tests { Ok(()) } + + fn expect_comment(inserted_comment: &Comment, comment: &Comment) -> LemmyResult<()> { + assert_eq!(inserted_comment.id, comment.id); + assert_eq!("A test comment".to_string(), comment.content); + assert_eq!(inserted_comment.post_id, comment.post_id); + assert_eq!(inserted_comment.creator_id, comment.creator_id); + assert!(!comment.removed); + assert!(!comment.deleted); + assert_eq!(Ltree(format!("0.{}", inserted_comment.id)), comment.path); + assert_eq!(inserted_comment.published, comment.published); + assert_eq!(None, comment.updated); + assert_eq!( + &Url::parse(&format!( + "https://lemmy-alpha/comment/{}", + inserted_comment.id + ))?, + comment.ap_id.inner() + ); + assert!(!comment.distinguished); + assert!(comment.local); + assert_eq!(LanguageId::default(), comment.language_id); + Ok(()) + } } diff --git a/crates/db_views/src/combined/report_combined_view.rs b/crates/db_views/src/combined/report_combined_view.rs index e4e4775aeb..5fc64e13d0 100644 --- a/crates/db_views/src/combined/report_combined_view.rs +++ b/crates/db_views/src/combined/report_combined_view.rs @@ -503,7 +503,7 @@ mod tests { }, }; use lemmy_db_schema::{ - aggregates::structs::{CommentAggregates, PostAggregates}, + aggregates::structs::PostAggregates, assert_length, source::{ comment::{Comment, CommentInsertForm}, @@ -1011,12 +1011,12 @@ mod tests { let inserted_jessica_report = CommentReport::report(pool, &jessica_report_form).await?; - let agg = CommentAggregates::read(pool, data.comment.id).await?; - assert_eq!(agg.report_count, 2); + let comment = Comment::read(pool, data.comment.id).await?; + assert_eq!(comment.report_count, 2); let read_jessica_report_view = CommentReportView::read(pool, inserted_jessica_report.id, data.timmy.id).await?; - assert_eq!(read_jessica_report_view.counts.unresolved_report_count, 2); + assert_eq!(read_jessica_report_view.comment.unresolved_report_count, 2); // Do a batch read of timmys reports let reports = ReportCombinedQuery::default() diff --git a/crates/db_views/src/comment/comment_view.rs b/crates/db_views/src/comment/comment_view.rs index 7cbab97deb..a3e4f29e44 100644 --- a/crates/db_views/src/comment/comment_view.rs +++ b/crates/db_views/src/comment/comment_view.rs @@ -547,7 +547,7 @@ mod tests { let pool = &mut pool.into(); let data = init_data(pool).await?; - let expected_comment_view_no_person = expected_comment_view(&data, pool).await?; + let expected_comment_view_no_person = expected_comment_view(&data).await?; let mut expected_comment_view_with_person = expected_comment_view_no_person.clone(); expected_comment_view_with_person.my_vote = Some(1); @@ -693,7 +693,7 @@ mod tests { // Make sure a depth limited one only has the top comment assert_eq!( - expected_comment_view(&data, pool).await?, + expected_comment_view(&data).await?, read_comment_views_top_max_depth[0] ); assert_length!(1, read_comment_views_top_max_depth); @@ -879,8 +879,7 @@ mod tests { Ok(()) } - async fn expected_comment_view(data: &Data, pool: &mut DbPool<'_>) -> LemmyResult { - let agg = CommentAggregates::read(pool, data.inserted_comment_0.id).await?; + async fn expected_comment_view(data: &Data) -> LemmyResult { Ok(CommentView { creator_banned_from_community: false, banned_from_community: false, @@ -904,6 +903,14 @@ mod tests { distinguished: false, path: data.inserted_comment_0.clone().path, language_id: LanguageId(37), + score: 1, + upvotes: 1, + downvotes: 0, + child_count: 5, + hot_rank: RANK_DEFAULT, + controversy_rank: 0.0, + report_count: 0, + unresolved_report_count: 0, }, creator: Person { id: data.timmy_local_user_view.person.id, @@ -981,18 +988,6 @@ mod tests { visibility: CommunityVisibility::Public, random_number: data.inserted_community.random_number, }, - counts: CommentAggregates { - comment_id: data.inserted_comment_0.id, - score: 1, - upvotes: 1, - downvotes: 0, - published: agg.published, - child_count: 5, - hot_rank: RANK_DEFAULT, - controversy_rank: 0.0, - report_count: 0, - unresolved_report_count: 0, - }, }) } From 43ef0b594456b98c5a7f3d6d0ba7e231776cf63c Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 7 Feb 2025 14:46:15 +0100 Subject: [PATCH 04/39] triggers --- .../db_schema/replaceable_schema/triggers.sql | 35 ++----------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index adf06c32b6..baedddc9c2 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -22,16 +22,6 @@ CREATE FUNCTION r.creator_id_from_post_aggregates (agg post_aggregates) RETURNS int IMMUTABLE PARALLEL SAFE RETURN agg.creator_id; -CREATE FUNCTION r.creator_id_from_comment_aggregates (agg comment_aggregates) - RETURNS int IMMUTABLE PARALLEL SAFE RETURN ( - SELECT - creator_id - FROM - comment - WHERE - comment.id = agg.comment_id LIMIT 1 -); - CREATE PROCEDURE r.post_or_comment (table_name text) LANGUAGE plpgsql AS $a$ @@ -109,7 +99,7 @@ WHERE AND diff.comment_count != 0; UPDATE - comment_aggregates AS a + comment AS a SET child_count = a.child_count + diff.child_count FROM ( @@ -406,7 +396,7 @@ $$); CALL r.create_triggers ('comment_report', $$ BEGIN UPDATE - comment_aggregates AS a + comment AS a SET report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count FROM ( @@ -443,25 +433,6 @@ $$); -- These triggers create and update rows in each aggregates table to match its associated table's rows. -- Deleting rows and updating IDs are already handled by `CASCADE` in foreign key constraints. -CREATE FUNCTION r.comment_aggregates_from_comment () - RETURNS TRIGGER - LANGUAGE plpgsql - AS $$ -BEGIN - INSERT INTO comment_aggregates (comment_id, published) - SELECT - id, - published - FROM - new_comment; - RETURN NULL; -END; -$$; - -CREATE TRIGGER aggregates - AFTER INSERT ON comment REFERENCING NEW TABLE AS new_comment - FOR EACH STATEMENT - EXECUTE FUNCTION r.comment_aggregates_from_comment (); CREATE FUNCTION r.community_aggregates_from_community () RETURNS TRIGGER @@ -1004,7 +975,7 @@ END $$; CREATE TRIGGER search_combined_comment_score - AFTER UPDATE OF score ON comment_aggregates + AFTER UPDATE OF score ON comment FOR EACH ROW EXECUTE FUNCTION r.search_combined_comment_score_update (); From 31cbe186c174bbaf5a3ad4348bcbba9698bee2b7 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 7 Feb 2025 15:05:32 +0100 Subject: [PATCH 05/39] fix --- crates/db_schema/replaceable_schema/triggers.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index baedddc9c2..4ee39af063 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -133,7 +133,7 @@ FROM ( GROUP BY parent_id) AS diff WHERE - a.comment_id = diff.parent_id + a.id = diff.parent_id AND diff.child_count != 0; WITH post_diff AS ( @@ -404,7 +404,7 @@ BEGIN (comment_report).comment_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (comment_report).resolved), 0) AS unresolved_report_count FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (comment_report).comment_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) - AND a.comment_id = diff.comment_id; + AND a.id = diff.comment_id; RETURN NULL; From cd29e85880e8d2f2eaea3016b57811ca560d175d Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 7 Feb 2025 15:14:08 +0100 Subject: [PATCH 06/39] fmt --- .../db_schema/replaceable_schema/triggers.sql | 1 - .../down.sql | 20 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index 4ee39af063..27de7ee1be 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -433,7 +433,6 @@ $$); -- These triggers create and update rows in each aggregates table to match its associated table's rows. -- Deleting rows and updating IDs are already handled by `CASCADE` in foreign key constraints. - CREATE FUNCTION r.community_aggregates_from_community () RETURNS TRIGGER LANGUAGE plpgsql diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql index c81582cfd3..6709056d9b 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql @@ -14,15 +14,17 @@ CREATE TABLE comment_aggregates ( INSERT INTO comment_aggregates SELECT id AS comment_id, - score, - upvotes, - downvotes, - published, - child_count, - hot_rank, - controversy_rank, - report_count, - unresolved_report_count from comment; + score, + upvotes, + downvotes, + published, + child_count, + hot_rank, + controversy_rank, + report_count, + unresolved_report_count +FROM + comment; ALTER TABLE comment DROP COLUMN score, From 61c5038415215fdc693091f561c9674ded920123 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 7 Feb 2025 15:31:21 +0100 Subject: [PATCH 07/39] clippy --- crates/db_views/src/comment/comment_view.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/db_views/src/comment/comment_view.rs b/crates/db_views/src/comment/comment_view.rs index a0419144e9..fb85101ca0 100644 --- a/crates/db_views/src/comment/comment_view.rs +++ b/crates/db_views/src/comment/comment_view.rs @@ -547,7 +547,7 @@ mod tests { let pool = &mut pool.into(); let data = init_data(pool).await?; - let expected_comment_view_no_person = expected_comment_view(&data).await?; + let expected_comment_view_no_person = expected_comment_view(&data); let mut expected_comment_view_with_person = expected_comment_view_no_person.clone(); expected_comment_view_with_person.my_vote = Some(1); @@ -693,7 +693,7 @@ mod tests { // Make sure a depth limited one only has the top comment assert_eq!( - expected_comment_view(&data).await?, + expected_comment_view(&data), read_comment_views_top_max_depth[0] ); assert_length!(1, read_comment_views_top_max_depth); @@ -879,8 +879,8 @@ mod tests { Ok(()) } - async fn expected_comment_view(data: &Data) -> LemmyResult { - Ok(CommentView { + fn expected_comment_view(data: &Data) -> CommentView { + CommentView { creator_banned_from_community: false, banned_from_community: false, creator_is_moderator: false, @@ -988,7 +988,7 @@ mod tests { visibility: CommunityVisibility::Public, random_number: data.inserted_community.random_number, }, - }) + } } #[tokio::test] From b27aea0e7bce5373e5a87e7f308ae664fff9ec67 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 7 Feb 2025 16:40:29 +0100 Subject: [PATCH 08/39] post aggregate migration --- .../down.sql | 143 +++++++++++++++++- .../up.sql | 114 ++++++++++++++ 2 files changed, 256 insertions(+), 1 deletion(-) diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql index 6709056d9b..e752289d78 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql @@ -1,5 +1,6 @@ +-- move comment_aggregates back into separate table CREATE TABLE comment_aggregates ( - comment_id integer NOT NULL, + comment_id int PRIMARY KEY NOT NULL REFERENCES COMMENT ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, score bigint NOT NULL DEFAULT 0, upvotes bigint NOT NULL DEFAULT 0, downvotes bigint NOT NULL DEFAULT 0, @@ -36,3 +37,143 @@ ALTER TABLE comment DROP COLUMN report_count, DROP COLUMN unresolved_report_count; +CREATE INDEX idx_comment_aggregates_controversy ON comment_aggregates USING btree (controversy_rank DESC); + +CREATE INDEX idx_comment_aggregates_hot ON comment_aggregates USING btree (hot_rank DESC, score DESC); + +CREATE INDEX idx_comment_aggregates_nonzero_hotrank ON comment_aggregates USING btree (published) +WHERE (hot_rank <> (0)::double precision); + +CREATE INDEX idx_comment_aggregates_published ON comment_aggregates USING btree (published DESC); + +CREATE INDEX idx_comment_aggregates_score ON comment_aggregates USING btree (score DESC); + +-- move comment_aggregates back into separate table +CREATE TABLE post_aggregates ( + post_id int PRIMARY KEY NOT NULL REFERENCES post ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + comments bigint NOT NULL DEFAULT 0, + score bigint NOT NULL DEFAULT 0, + upvotes bigint NOT NULL DEFAULT 0, + downvotes bigint NOT NULL DEFAULT 0, + published timestamp with time zone NOT NULL DEFAULT now(), + newest_comment_time_necro timestamp with time zone NOT NULL DEFAULT now(), + newest_comment_time timestamp with time zone NOT NULL DEFAULT now(), + featured_community boolean NOT NULL DEFAULT FALSE, + featured_local boolean NOT NULL DEFAULT FALSE, + hot_rank double precision NOT NULL DEFAULT 0.0001, + hot_rank_active double precision NOT NULL DEFAULT 0.0001, + community_id integer NOT NULL REFERENCES community (id) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + creator_id integer NOT NULL REFERENCES person (id) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + controversy_rank double precision NOT NULL DEFAULT 0, + instance_id integer NOT NULL REFERENCES instance (id) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + scaled_rank double precision NOT NULL DEFAULT 0.0001, + report_count smallint NOT NULL DEFAULT 0, + unresolved_report_count smallint NOT NULL DEFAULT 0 +); + +INSERT INTO post_aggregates +SELECT + id AS post_id, + comments, + score, + upvotes, + downvotes, + published, + newest_comment_time_necro, + newest_comment_time, + featured_community, + featured_local, + hot_rank, + hot_rank_active, + community_id, + creator_id, + controversy_rank, + instance_id, + scaled_rank, + report_count, + unresolved_report_count +FROM + post; + +ALTER TABLE post + DROP COLUMN comments, + DROP COLUMN score, + DROP COLUMN upvotes, + DROP COLUMN downvotes, + DROP COLUMN newest_comment_time_necro, + DROP COLUMN newest_comment_time, + DROP COLUMN hot_rank, + DROP COLUMN hot_rank_active, + DROP COLUMN controversy_rank, + DROP COLUMN instance_id, + DROP COLUMN scaled_rank, + DROP COLUMN report_count, + DROP COLUMN unresolved_report_count; + +CREATE INDEX idx_post_aggregates_community_active ON post_aggregates USING btree (community_id, featured_local DESC, hot_rank_active DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_controversy ON post_aggregates USING btree (community_id, featured_local DESC, controversy_rank DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_hot ON post_aggregates USING btree (community_id, featured_local DESC, hot_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_most_comments ON post_aggregates USING btree (community_id, featured_local DESC, comments DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_newest_comment_time ON post_aggregates USING btree (community_id, featured_local DESC, newest_comment_time DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_newest_comment_time_necro ON post_aggregates USING btree (community_id, featured_local DESC, newest_comment_time_necro DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_published ON post_aggregates USING btree (community_id, featured_local DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_published_asc ON post_aggregates USING btree (community_id, featured_local DESC, reverse_timestamp_sort (published) DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_scaled ON post_aggregates USING btree (community_id, featured_local DESC, scaled_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_score ON post_aggregates USING btree (community_id, featured_local DESC, score DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_active ON post_aggregates USING btree (community_id, featured_community DESC, hot_rank_active DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_controversy ON post_aggregates USING btree (community_id, featured_community DESC, controversy_rank DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_hot ON post_aggregates USING btree (community_id, featured_community DESC, hot_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_most_comments ON post_aggregates USING btree (community_id, featured_community DESC, comments DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time ON post_aggregates USING btree (community_id, featured_community DESC, newest_comment_time DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time_necr ON post_aggregates USING btree (community_id, featured_community DESC, newest_comment_time_necro DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_published ON post_aggregates USING btree (community_id, featured_community DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_published_asc ON post_aggregates USING btree (community_id, featured_community DESC, reverse_timestamp_sort (published) DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_scaled ON post_aggregates USING btree (community_id, featured_community DESC, scaled_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_score ON post_aggregates USING btree (community_id, featured_community DESC, score DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_active ON post_aggregates USING btree (featured_local DESC, hot_rank_active DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_controversy ON post_aggregates USING btree (featured_local DESC, controversy_rank DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_hot ON post_aggregates USING btree (featured_local DESC, hot_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_most_comments ON post_aggregates USING btree (featured_local DESC, comments DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time ON post_aggregates USING btree (featured_local DESC, newest_comment_time DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time_necro ON post_aggregates USING btree (featured_local DESC, newest_comment_time_necro DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_published ON post_aggregates USING btree (featured_local DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_published_asc ON post_aggregates USING btree (featured_local DESC, reverse_timestamp_sort (published) DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_scaled ON post_aggregates USING btree (featured_local DESC, scaled_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_score ON post_aggregates USING btree (featured_local DESC, score DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_nonzero_hotrank ON post_aggregates USING btree (published DESC) +WHERE ((hot_rank <> (0)::double precision) OR (hot_rank_active <> (0)::double precision)); + +CREATE INDEX idx_post_aggregates_published ON post_aggregates USING btree (published DESC); + +CREATE INDEX idx_post_aggregates_published_asc ON post_aggregates USING btree (reverse_timestamp_sort (published) DESC); + diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql index 4fa9c83741..6140f27350 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -1,3 +1,4 @@ +-- merge comment_aggregates into comment table ALTER TABLE comment ADD COLUMN score bigint NOT NULL DEFAULT 0, ADD COLUMN upvotes bigint NOT NULL DEFAULT 0, @@ -26,3 +27,116 @@ WHERE DROP TABLE comment_aggregates; +CREATE INDEX idx_comment_aggregates_controversy ON comment USING btree (controversy_rank DESC); + +CREATE INDEX idx_comment_aggregates_hot ON comment USING btree (hot_rank DESC, score DESC); + +CREATE INDEX idx_comment_aggregates_nonzero_hotrank ON comment USING btree (published) +WHERE (hot_rank <> (0)::double precision); + +-- some indexes commented out because they already exist +--CREATE INDEX idx_comment_aggregates_published on comment USING btree (published DESC); +CREATE INDEX idx_comment_aggregates_score ON comment USING btree (score DESC); + +-- merge post_aggregates into post table +ALTER TABLE post + ADD COLUMN comments bigint NOT NULL DEFAULT 0, + ADD COLUMN score bigint NOT NULL DEFAULT 0, + ADD COLUMN upvotes bigint NOT NULL DEFAULT 0, + ADD COLUMN downvotes bigint NOT NULL DEFAULT 0, + ADD COLUMN newest_comment_time_necro timestamp with time zone NOT NULL DEFAULT now(), + ADD COLUMN newest_comment_time timestamp with time zone NOT NULL DEFAULT now(), + ADD COLUMN hot_rank double precision NOT NULL DEFAULT 0.0001, + ADD COLUMN hot_rank_active double precision NOT NULL DEFAULT 0.0001, + ADD COLUMN controversy_rank double precision NOT NULL DEFAULT 0, + ADD COLUMN instance_id int NOT NULL DEFAULT 0 REFERENCES instance (id) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + ADD COLUMN scaled_rank double precision NOT NULL DEFAULT 0.0001, + ADD COLUMN report_count smallint NOT NULL DEFAULT 0, + ADD COLUMN unresolved_report_count smallint NOT NULL DEFAULT 0; + +UPDATE + post +SET + comments = pa.comments, + score = pa.score, + upvotes = pa.upvotes, + downvotes = pa.downvotes, + newest_comment_time_necro = pa.newest_comment_time_necro, + newest_comment_time = pa.newest_comment_time, + hot_rank = pa.hot_rank, + hot_rank_active = pa.hot_rank_active, + controversy_rank = pa.controversy_rank, + instance_id = pa.instance_id, + scaled_rank = pa.scaled_rank, + report_count = pa.report_count, + unresolved_report_count = pa.unresolved_report_count +FROM + post_aggregates AS pa +WHERE + post.id = pa.post_id; + +ALTER TABLE post + ALTER COLUMN instance_id DROP NOT NULL; + +DROP TABLE post_aggregates; + +-- Note, removed `post_id DESC` from all these +CREATE INDEX idx_post_aggregates_community_active ON post USING btree (community_id, featured_local DESC, hot_rank_active DESC, published DESC); + +CREATE INDEX idx_post_aggregates_community_controversy ON post USING btree (community_id, featured_local DESC, controversy_rank DESC); + +CREATE INDEX idx_post_aggregates_community_hot ON post USING btree (community_id, featured_local DESC, hot_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_community_most_comments ON post USING btree (community_id, featured_local DESC, comments DESC, published DESC); + +CREATE INDEX idx_post_aggregates_community_newest_comment_time ON post USING btree (community_id, featured_local DESC, newest_comment_time DESC); + +CREATE INDEX idx_post_aggregates_community_newest_comment_time_necro ON post USING btree (community_id, featured_local DESC, newest_comment_time_necro DESC); + +--CREATE INDEX idx_post_aggregates_community_published on post USING btree (community_id, featured_local DESC, published DESC); +--CREATE INDEX idx_post_aggregates_community_published_asc on post USING btree (community_id, featured_local DESC, reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_aggregates_community_scaled ON post USING btree (community_id, featured_local DESC, scaled_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_community_score ON post USING btree (community_id, featured_local DESC, score DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_active ON post USING btree (community_id, featured_community DESC, hot_rank_active DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_controversy ON post USING btree (community_id, featured_community DESC, controversy_rank DESC); + +CREATE INDEX idx_post_aggregates_featured_community_hot ON post USING btree (community_id, featured_community DESC, hot_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_most_comments ON post USING btree (community_id, featured_community DESC, comments DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time ON post USING btree (community_id, featured_community DESC, newest_comment_time DESC); + +CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time_necr ON post USING btree (community_id, featured_community DESC, newest_comment_time_necro DESC); + +--CREATE INDEX idx_post_aggregates_featured_community_published on post USING btree (community_id, featured_community DESC, published DESC); +--CREATE INDEX idx_post_aggregates_featured_community_published_asc on post USING btree (community_id, featured_community DESC, reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_aggregates_featured_community_scaled ON post USING btree (community_id, featured_community DESC, scaled_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_score ON post USING btree (community_id, featured_community DESC, score DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_active ON post USING btree (featured_local DESC, hot_rank_active DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_controversy ON post USING btree (featured_local DESC, controversy_rank DESC); + +CREATE INDEX idx_post_aggregates_featured_local_hot ON post USING btree (featured_local DESC, hot_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_most_comments ON post USING btree (featured_local DESC, comments DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time ON post USING btree (featured_local DESC, newest_comment_time DESC); + +CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time_necro ON post USING btree (featured_local DESC, newest_comment_time_necro DESC); + +--CREATE INDEX idx_post_aggregates_featured_local_published on post USING btree (featured_local DESC, published DESC); +--CREATE INDEX idx_post_aggregates_featured_local_published_asc on post USING btree (featured_local DESC, reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_aggregates_featured_local_scaled ON post USING btree (featured_local DESC, scaled_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_score ON post USING btree (featured_local DESC, score DESC, published DESC); + +CREATE INDEX idx_post_aggregates_nonzero_hotrank ON post USING btree (published DESC) +WHERE ((hot_rank <> (0)::double precision) OR (hot_rank_active <> (0)::double precision)); + +--CREATE INDEX idx_post_aggregates_published on post USING btree (published DESC); +--CREATE INDEX idx_post_aggregates_published_asc on post USING btree (reverse_timestamp_sort (published) DESC); From bb4bfe3e8689f1e6f1b3f2c78d9fcd22fdb292e9 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 10 Feb 2025 12:48:55 +0100 Subject: [PATCH 09/39] changes for post aggregate code --- crates/api_crud/src/comment/create.rs | 2 +- crates/api_crud/src/post/read.rs | 8 +- .../src/activities/create_or_update/post.rs | 3 +- crates/db_schema/Cargo.toml | 2 +- .../src/aggregates/post_aggregates.rs | 33 +------- crates/db_schema/src/aggregates/structs.rs | 76 ++--------------- crates/db_schema/src/impls/post.rs | 33 +++++++- crates/db_schema/src/schema.rs | 42 +++------- crates/db_schema/src/source/post.rs | 56 ++++++++++--- .../src/combined/inbox_combined_view.rs | 12 +-- .../combined/person_content_combined_view.rs | 13 +-- .../combined/person_saved_combined_view.rs | 6 +- .../src/combined/report_combined_view.rs | 14 +--- .../src/combined/search_combined_view.rs | 12 +-- crates/db_views/src/post/post_view.rs | 83 ++++++++----------- .../db_views/src/reports/post_report_view.rs | 10 +-- crates/db_views/src/structs.rs | 9 +- crates/routes/src/feeds.rs | 4 +- 18 files changed, 150 insertions(+), 268 deletions(-) diff --git a/crates/api_crud/src/comment/create.rs b/crates/api_crud/src/comment/create.rs index a9ddda3b1d..208c52b179 100644 --- a/crates/api_crud/src/comment/create.rs +++ b/crates/api_crud/src/comment/create.rs @@ -145,7 +145,7 @@ pub async fn create_comment( update_read_comments( local_user_view.person.id, post_id, - post_view.counts.comments + 1, + post.comments + 1, &mut context.pool(), ) .await?; diff --git a/crates/api_crud/src/post/read.rs b/crates/api_crud/src/post/read.rs index 043db4f40b..d806533c46 100644 --- a/crates/api_crud/src/post/read.rs +++ b/crates/api_crud/src/post/read.rs @@ -66,13 +66,7 @@ pub async fn get_post( let read_form = PostReadForm::new(post_id, person_id); PostRead::mark_as_read(&mut context.pool(), &read_form).await?; - update_read_comments( - person_id, - post_id, - post_view.counts.comments, - &mut context.pool(), - ) - .await?; + update_read_comments(person_id, post_id, post_view.post.comments, &mut context.pool()).await?; } // Necessary for the sidebar subscribed diff --git a/crates/apub/src/activities/create_or_update/post.rs b/crates/apub/src/activities/create_or_update/post.rs index 4dc8ea4b27..d263c6f315 100644 --- a/crates/apub/src/activities/create_or_update/post.rs +++ b/crates/apub/src/activities/create_or_update/post.rs @@ -22,7 +22,6 @@ use activitypub_federation::{ }; use lemmy_api_common::{build_response::send_local_notifs, context::LemmyContext}; use lemmy_db_schema::{ - aggregates::structs::PostAggregates, newtypes::{PersonId, PostOrCommentId}, source::{ activity::ActivitySendTargets, @@ -121,7 +120,7 @@ impl ActivityHandler for CreateOrUpdatePage { PostLike::like(&mut context.pool(), &like_form).await?; // Calculate initial hot_rank for post - PostAggregates::update_ranks(&mut context.pool(), post.id).await?; + Post::update_ranks(&mut context.pool(), post.id).await?; let do_send_email = self.kind == CreateOrUpdateType::Create; let actor = self.actor.dereference(context).await?; diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index b2b1dbb1b1..89ed13e26c 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -54,7 +54,7 @@ diesel = { workspace = true, features = [ "chrono", "postgres", "serde_json", - "uuid", + "uuid","64-column-tables" ], optional = true } diesel-derive-newtype = { workspace = true, optional = true } diesel-derive-enum = { workspace = true, optional = true } diff --git a/crates/db_schema/src/aggregates/post_aggregates.rs b/crates/db_schema/src/aggregates/post_aggregates.rs index af59fe761c..8f4e29e31b 100644 --- a/crates/db_schema/src/aggregates/post_aggregates.rs +++ b/crates/db_schema/src/aggregates/post_aggregates.rs @@ -1,4 +1,4 @@ -use crate::{ +/*use crate::{ aggregates::structs::PostAggregates, newtypes::PostId, schema::{community_aggregates, post, post_aggregates}, @@ -17,36 +17,6 @@ impl PostAggregates { post_aggregates::table.find(post_id).first(conn).await } - pub async fn update_ranks(pool: &mut DbPool<'_>, post_id: PostId) -> Result { - let conn = &mut get_conn(pool).await?; - - // Diesel can't update based on a join, which is necessary for the scaled_rank - // https://github.com/diesel-rs/diesel/issues/1478 - // Just select the metrics we need manually, for now, since its a single post anyway - - let interactions_month = community_aggregates::table - .select(community_aggregates::interactions_month) - .inner_join(post::table.on(community_aggregates::community_id.eq(post::community_id))) - .filter(post::id.eq(post_id)) - .first::(conn) - .await?; - - diesel::update(post_aggregates::table.find(post_id)) - .set(( - post_aggregates::hot_rank.eq(hot_rank(post_aggregates::score, post_aggregates::published)), - post_aggregates::hot_rank_active.eq(hot_rank( - post_aggregates::score, - post_aggregates::newest_comment_time_necro, - )), - post_aggregates::scaled_rank.eq(scaled_rank( - post_aggregates::score, - post_aggregates::published, - interactions_month, - )), - )) - .get_result::(conn) - .await - } } #[cfg(test)] @@ -268,3 +238,4 @@ mod tests { Ok(()) } } +*/ diff --git a/crates/db_schema/src/aggregates/structs.rs b/crates/db_schema/src/aggregates/structs.rs index 89c28d9958..e3501d4072 100644 --- a/crates/db_schema/src/aggregates/structs.rs +++ b/crates/db_schema/src/aggregates/structs.rs @@ -1,20 +1,12 @@ -use crate::newtypes::{CommunityId, InstanceId, PersonId, PostId, SiteId}; -#[cfg(feature = "full")] -use crate::schema::{ - community_aggregates, - person_aggregates, - post_actions, - post_aggregates, - site_aggregates, -}; +use crate::newtypes::{CommunityId, PersonId, PostId, SiteId}; use chrono::{DateTime, Utc}; -#[cfg(feature = "full")] -use diesel::{dsl, expression_methods::NullableExpressionMethods}; -#[cfg(feature = "full")] -use i_love_jesus::CursorKeysModule; use serde::{Deserialize, Serialize}; #[cfg(feature = "full")] -use ts_rs::TS; +use { + crate::schema::{community_aggregates, person_aggregates, post_actions, site_aggregates}, + diesel::{dsl, expression_methods::NullableExpressionMethods}, + ts_rs::TS, +}; #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] #[cfg_attr( @@ -74,62 +66,6 @@ pub struct PersonAggregates { pub comment_score: i64, } -#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] -#[cfg_attr( - feature = "full", - derive( - Queryable, - Selectable, - Associations, - Identifiable, - TS, - CursorKeysModule - ) -)] -#[cfg_attr(feature = "full", diesel(table_name = post_aggregates))] -#[cfg_attr(feature = "full", diesel(belongs_to(crate::source::post::Post)))] -#[cfg_attr(feature = "full", diesel(primary_key(post_id)))] -#[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))] -#[cfg_attr(feature = "full", ts(export))] -#[cfg_attr(feature = "full", cursor_keys_module(name = post_aggregates_keys))] -/// Aggregate data for a post. -pub struct PostAggregates { - pub post_id: PostId, - pub comments: i64, - pub score: i64, - pub upvotes: i64, - pub downvotes: i64, - pub published: DateTime, - #[serde(skip)] - /// A newest comment time, limited to 2 days, to prevent necrobumping - pub newest_comment_time_necro: DateTime, - /// The time of the newest comment in the post. - pub newest_comment_time: DateTime, - /// If the post is featured on the community. - #[serde(skip)] - pub featured_community: bool, - /// If the post is featured on the site / to local. - #[serde(skip)] - pub featured_local: bool, - #[serde(skip)] - pub hot_rank: f64, - #[serde(skip)] - pub hot_rank_active: f64, - #[serde(skip)] - pub community_id: CommunityId, - #[serde(skip)] - pub creator_id: PersonId, - #[serde(skip)] - pub controversy_rank: f64, - #[serde(skip)] - pub instance_id: InstanceId, - /// A rank that amplifies smaller communities - #[serde(skip)] - pub scaled_rank: f64, - pub report_count: i16, - pub unresolved_report_count: i16, -} - #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)] #[cfg_attr( feature = "full", diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index 96818ec6d7..c3369bf9dd 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -1,7 +1,7 @@ use crate::{ diesel::{BoolExpressionMethods, NullableExpressionMethods, OptionalExtension}, newtypes::{CommunityId, DbUrl, PersonId, PostId}, - schema::{community, person, post, post_actions}, + schema::{community, community_aggregates, person, post, post_actions}, source::post::{ Post, PostActionsCursor, @@ -18,7 +18,7 @@ use crate::{ }, traits::{Crud, Likeable, Saveable}, utils::{ - functions::coalesce, + functions::{coalesce, hot_rank, scaled_rank}, get_conn, now, uplete, @@ -37,6 +37,7 @@ use diesel::{ result::Error, DecoratableTarget, ExpressionMethods, + JoinOnDsl, QueryDsl, TextExpressionMethods, }; @@ -270,6 +271,34 @@ impl Post { .first::(conn) .await } + + pub async fn update_ranks(pool: &mut DbPool<'_>, post_id: PostId) -> Result { + let conn = &mut get_conn(pool).await?; + + // Diesel can't update based on a join, which is necessary for the scaled_rank + // https://github.com/diesel-rs/diesel/issues/1478 + // Just select the metrics we need manually, for now, since its a single post anyway + + let interactions_month = community_aggregates::table + .select(community_aggregates::interactions_month) + .inner_join(post::table.on(community_aggregates::community_id.eq(post::community_id))) + .filter(post::id.eq(post_id)) + .first::(conn) + .await?; + + diesel::update(post::table.find(post_id)) + .set(( + post::hot_rank.eq(hot_rank(post::score, post::published)), + post::hot_rank_active.eq(hot_rank(post::score, post::newest_comment_time_necro)), + post::scaled_rank.eq(scaled_rank( + post::score, + post::published, + interactions_month, + )), + )) + .get_result::(conn) + .await + } } #[async_trait] diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 9f1f1037fa..ac4ae88034 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -850,6 +850,19 @@ diesel::table! { url_content_type -> Nullable, alt_text -> Nullable, scheduled_publish_time -> Nullable, + comments -> Int8, + score -> Int8, + upvotes -> Int8, + downvotes -> Int8, + newest_comment_time_necro -> Timestamptz, + newest_comment_time -> Timestamptz, + hot_rank -> Float8, + hot_rank_active -> Float8, + controversy_rank -> Float8, + instance_id -> Int4, + scaled_rank -> Float8, + report_count -> Int2, + unresolved_report_count -> Int2, } } @@ -867,30 +880,6 @@ diesel::table! { } } -diesel::table! { - post_aggregates (post_id) { - post_id -> Int4, - comments -> Int8, - score -> Int8, - upvotes -> Int8, - downvotes -> Int8, - published -> Timestamptz, - newest_comment_time_necro -> Timestamptz, - newest_comment_time -> Timestamptz, - featured_community -> Bool, - featured_local -> Bool, - hot_rank -> Float8, - hot_rank_active -> Float8, - community_id -> Int4, - creator_id -> Int4, - controversy_rank -> Float8, - instance_id -> Int4, - scaled_rank -> Float8, - report_count -> Int2, - unresolved_report_count -> Int2, - } -} - diesel::table! { post_report (id) { id -> Int4, @@ -1187,10 +1176,6 @@ diesel::joinable!(post -> language (language_id)); diesel::joinable!(post -> person (creator_id)); diesel::joinable!(post_actions -> person (person_id)); diesel::joinable!(post_actions -> post (post_id)); -diesel::joinable!(post_aggregates -> community (community_id)); -diesel::joinable!(post_aggregates -> instance (instance_id)); -diesel::joinable!(post_aggregates -> person (creator_id)); -diesel::joinable!(post_aggregates -> post (post_id)); diesel::joinable!(post_report -> post (post_id)); diesel::joinable!(post_tag -> post (post_id)); diesel::joinable!(post_tag -> tag (tag_id)); @@ -1272,7 +1257,6 @@ diesel::allow_tables_to_appear_in_same_query!( person_saved_combined, post, post_actions, - post_aggregates, post_report, post_tag, previously_run_sql, diff --git a/crates/db_schema/src/source/post.rs b/crates/db_schema/src/source/post.rs index 8280fe8fea..c6e407c46d 100644 --- a/crates/db_schema/src/source/post.rs +++ b/crates/db_schema/src/source/post.rs @@ -1,22 +1,25 @@ -use crate::newtypes::{CommunityId, DbUrl, LanguageId, PersonId, PostId}; -#[cfg(feature = "full")] -use crate::schema::{post, post_actions}; +use crate::newtypes::{CommunityId, DbUrl, InstanceId, LanguageId, PersonId, PostId}; use chrono::{DateTime, Utc}; -#[cfg(feature = "full")] -use diesel::{dsl, expression_methods::NullableExpressionMethods}; -#[cfg(feature = "full")] -use i_love_jesus::CursorKeysModule; use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; #[cfg(feature = "full")] -use ts_rs::TS; +use { + crate::schema::{post, post_actions}, + diesel::{dsl, expression_methods::NullableExpressionMethods}, + i_love_jesus::CursorKeysModule, + ts_rs::TS, +}; #[skip_serializing_none] -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] -#[cfg_attr(feature = "full", derive(Queryable, Selectable, Identifiable, TS))] +#[derive(Clone, Debug, Serialize, Deserialize)] +#[cfg_attr( + feature = "full", + derive(Queryable, Selectable, Identifiable, TS, CursorKeysModule) +)] #[cfg_attr(feature = "full", diesel(table_name = post))] #[cfg_attr(feature = "full", ts(export))] #[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))] +#[cfg_attr(feature = "full", cursor_keys_module(name = post_keys))] /// A post. pub struct Post { pub id: PostId, @@ -69,8 +72,41 @@ pub struct Post { /// Time at which the post will be published. None means publish immediately. #[cfg_attr(feature = "full", ts(optional))] pub scheduled_publish_time: Option>, + pub comments: i64, + pub score: i64, + pub upvotes: i64, + pub downvotes: i64, + #[serde(skip)] + /// A newest comment time, limited to 2 days, to prevent necrobumping + pub newest_comment_time_necro: DateTime, + /// The time of the newest comment in the post. + pub newest_comment_time: DateTime, + #[serde(skip)] + pub hot_rank: f64, + #[serde(skip)] + pub hot_rank_active: f64, + #[serde(skip)] + pub controversy_rank: f64, + #[serde(skip)] + pub instance_id: InstanceId, + /// A rank that amplifies smaller communities + #[serde(skip)] + pub scaled_rank: f64, + pub report_count: i16, + pub unresolved_report_count: i16, +} + +impl PartialEq for Post { + fn eq(&self, other: &Self) -> bool { + todo!() + /* + self.id == other.id && self.name == other.name && self.url == other.url && self.body == other.body && self.creator_id == other.creator_id && self.community_id == other.community_id && self.removed == other.removed && self.locked == other.locked && self.published == other.published && self.updated == other.updated && self.deleted == other.deleted && self.nsfw == other.nsfw && self.embed_title == other.embed_title && self.embed_description == other.embed_description && self.thumbnail_url == other.thumbnail_url && self.ap_id == other.ap_id && self.local == other.local && self.embed_video_url == other.embed_video_url && self.language_id == other.language_id && self.featured_community == other.featured_community && self.featured_local == other.featured_local && self.url_content_type == other.url_content_type && self.alt_text == other.alt_text && self.scheduled_publish_time == other.scheduled_publish_time && self.comments == other.comments && self.score == other.score && self.upvotes == other.upvotes && self.downvotes == other.downvotes && self.newest_comment_time_necro == other.newest_comment_time_necro && self.newest_comment_time == other.newest_comment_time && self.hot_rank == other.hot_rank && self.hot_rank_active == other.hot_rank_active && self.controversy_rank == other.controversy_rank && self.instance_id == other.instance_id && self.scaled_rank == other.scaled_rank && self.report_count == other.report_count && self.unresolved_report_count == other.unresolved_report_count + */ + } } +impl Eq for Post {} + #[derive(Debug, Clone, derive_new::new)] #[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] #[cfg_attr(feature = "full", diesel(table_name = post))] diff --git a/crates/db_views/src/combined/inbox_combined_view.rs b/crates/db_views/src/combined/inbox_combined_view.rs index 979911680d..c004f00d5f 100644 --- a/crates/db_views/src/combined/inbox_combined_view.rs +++ b/crates/db_views/src/combined/inbox_combined_view.rs @@ -38,7 +38,6 @@ use lemmy_db_schema::{ person_post_mention, post, post_actions, - post_aggregates, post_tag, private_message, tag, @@ -103,8 +102,6 @@ impl InboxCombinedViewInternal { .and(local_user::admin.eq(true)), ); - let post_aggregates_join = post_aggregates::table.on(post::id.eq(post_aggregates::post_id)); - let image_details_join = image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())); @@ -160,7 +157,6 @@ impl InboxCombinedViewInternal { .inner_join(person::table.on(item_creator_join)) .inner_join(recipient_join) .left_join(image_details_join) - .left_join(post_aggregates_join) .left_join(creator_community_actions_join) .left_join(local_user_join) .left_join(community_actions_join) @@ -283,10 +279,9 @@ impl InboxCombinedQuery { comment_reply::all_columns.nullable(), person_comment_mention::all_columns.nullable(), person_post_mention::all_columns.nullable(), - post_aggregates::all_columns.nullable(), coalesce( - post_aggregates::comments.nullable() - post_actions::read_comments_amount.nullable(), - post_aggregates::comments, + post::comments.nullable() - post_actions::read_comments_amount.nullable(), + post::comments, ) .nullable(), post_actions::saved.nullable(), @@ -468,19 +463,16 @@ impl InternalToCombinedView for InboxCombinedViewInternal { } else if let ( Some(person_post_mention), Some(post), - Some(counts), Some(unread_comments), Some(community), ) = ( v.person_post_mention, v.post, - v.post_counts, v.post_unread_comments, v.community, ) { Some(InboxCombinedView::PostMention(PersonPostMentionView { person_post_mention, - counts, post, community, recipient: v.item_recipient, diff --git a/crates/db_views/src/combined/person_content_combined_view.rs b/crates/db_views/src/combined/person_content_combined_view.rs index 67a4064c56..7adc416328 100644 --- a/crates/db_views/src/combined/person_content_combined_view.rs +++ b/crates/db_views/src/combined/person_content_combined_view.rs @@ -33,7 +33,6 @@ use lemmy_db_schema::{ person_saved_combined, post, post_actions, - post_aggregates, post_tag, tag, }, @@ -116,8 +115,6 @@ impl PersonContentCombinedViewInternal { .and(comment_actions::person_id.nullable().eq(my_person_id)), ); - let post_aggregates_join = post_aggregates::table.on(post::id.eq(post_aggregates::post_id)); - let image_details_join = image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())); @@ -131,7 +128,6 @@ impl PersonContentCombinedViewInternal { .left_join(community_actions_join) .left_join(post_actions_join) .left_join(person_actions_join) - .inner_join(post_aggregates_join) .left_join(comment_actions_join) .left_join(image_details_join) } @@ -204,8 +200,6 @@ impl PersonContentCombinedViewInternal { .and(comment_actions::person_id.eq(my_person_id)), ); - let post_aggregates_join = post_aggregates::table.on(post::id.eq(post_aggregates::post_id)); - let image_details_join = image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())); @@ -219,7 +213,6 @@ impl PersonContentCombinedViewInternal { .left_join(community_actions_join) .left_join(post_actions_join) .left_join(person_actions_join) - .inner_join(post_aggregates_join) .left_join(comment_actions_join) .left_join(image_details_join) } @@ -298,10 +291,9 @@ impl PersonContentCombinedQuery { .filter(item_creator.eq(self.creator_id)) .select(( // Post-specific - post_aggregates::all_columns, coalesce( - post_aggregates::comments.nullable() - post_actions::read_comments_amount.nullable(), - post_aggregates::comments, + post::comments.nullable() - post_actions::read_comments_amount.nullable(), + post::comments, ), post_actions::saved.nullable(), post_actions::read.nullable().is_not_null(), @@ -399,7 +391,6 @@ impl InternalToCombinedView for PersonContentCombinedViewInternal { post: v.post, community: v.community, unread_comments: v.post_unread_comments, - counts: v.post_counts, creator: v.item_creator, creator_banned_from_community: v.item_creator_banned_from_community, creator_is_moderator: v.item_creator_is_moderator, diff --git a/crates/db_views/src/combined/person_saved_combined_view.rs b/crates/db_views/src/combined/person_saved_combined_view.rs index 7add423207..76922ee3fb 100644 --- a/crates/db_views/src/combined/person_saved_combined_view.rs +++ b/crates/db_views/src/combined/person_saved_combined_view.rs @@ -27,7 +27,6 @@ use lemmy_db_schema::{ person_saved_combined, post, post_actions, - post_aggregates, post_tag, tag, }, @@ -103,10 +102,9 @@ impl PersonSavedCombinedQuery { .filter(person_saved_combined::person_id.eq(my_person_id)) .select(( // Post-specific - post_aggregates::all_columns, coalesce( - post_aggregates::comments.nullable() - post_actions::read_comments_amount.nullable(), - post_aggregates::comments, + post::comments.nullable() - post_actions::read_comments_amount.nullable(), + post::comments, ), post_actions::saved.nullable(), post_actions::read.nullable().is_not_null(), diff --git a/crates/db_views/src/combined/report_combined_view.rs b/crates/db_views/src/combined/report_combined_view.rs index 5b5872d0d4..c6fa3a628a 100644 --- a/crates/db_views/src/combined/report_combined_view.rs +++ b/crates/db_views/src/combined/report_combined_view.rs @@ -36,7 +36,6 @@ use lemmy_db_schema::{ person_actions, post, post_actions, - post_aggregates, post_report, private_message, private_message_report, @@ -139,9 +138,6 @@ impl ReportCombinedViewInternal { .and(comment_actions::person_id.eq(my_person_id)), ); - let post_aggregates_join = - post_aggregates::table.on(post_report::post_id.eq(post_aggregates::post_id)); - let community_aggregates_join = community_aggregates::table .on(community_report::community_id.eq(community_aggregates::community_id)); @@ -162,7 +158,6 @@ impl ReportCombinedViewInternal { .left_join(community_actions_join) .left_join(post_actions_join) .left_join(person_actions_join) - .left_join(post_aggregates_join) .left_join(community_aggregates_join) .left_join(comment_actions_join) } @@ -270,10 +265,9 @@ impl ReportCombinedQuery { // Post-specific post_report::all_columns.nullable(), post::all_columns.nullable(), - post_aggregates::all_columns.nullable(), coalesce( - post_aggregates::comments.nullable() - post_actions::read_comments_amount.nullable(), - post_aggregates::comments, + post::comments.nullable() - post_actions::read_comments_amount.nullable(), + post::comments, ) .nullable(), post_actions::saved.nullable(), @@ -398,14 +392,12 @@ impl InternalToCombinedView for ReportCombinedViewInternal { Some(post), Some(community), Some(unread_comments), - Some(counts), Some(post_creator), ) = ( v.post_report, v.post.clone(), v.community.clone(), v.post_unread_comments, - v.post_counts, v.item_creator.clone(), ) { Some(ReportCombinedView::Post(PostReportView { @@ -413,7 +405,6 @@ impl InternalToCombinedView for ReportCombinedViewInternal { post, community, unread_comments, - counts, creator: v.report_creator, post_creator, creator_banned_from_community: v.item_creator_banned_from_community, @@ -503,7 +494,6 @@ mod tests { }, }; use lemmy_db_schema::{ - aggregates::structs::PostAggregates, assert_length, source::{ comment::{Comment, CommentInsertForm}, diff --git a/crates/db_views/src/combined/search_combined_view.rs b/crates/db_views/src/combined/search_combined_view.rs index 558577ece5..b6f59c091f 100644 --- a/crates/db_views/src/combined/search_combined_view.rs +++ b/crates/db_views/src/combined/search_combined_view.rs @@ -37,7 +37,6 @@ use lemmy_db_schema::{ person_aggregates, post, post_actions, - post_aggregates, post_tag, search_combined, tag, @@ -139,8 +138,6 @@ impl SearchCombinedViewInternal { .and(comment_actions::person_id.nullable().eq(my_person_id)), ); - let post_aggregates_join = post_aggregates::table.on(post::id.eq(post_aggregates::post_id)); - let community_aggregates_join = community_aggregates::table .on(search_combined::community_id.eq(community_aggregates::community_id.nullable())); @@ -161,7 +158,6 @@ impl SearchCombinedViewInternal { .left_join(post_actions_join) .left_join(person_actions_join) .left_join(person_aggregates_join) - .left_join(post_aggregates_join) .left_join(community_aggregates_join) .left_join(comment_actions_join) .left_join(image_details_join) @@ -244,10 +240,9 @@ impl SearchCombinedQuery { .select(( // Post-specific post::all_columns.nullable(), - post_aggregates::all_columns.nullable(), coalesce( - post_aggregates::comments.nullable() - post_actions::read_comments_amount.nullable(), - post_aggregates::comments, + post::comments.nullable() - post_actions::read_comments_amount.nullable(), + post::comments, ) .nullable(), post_actions::saved.nullable(), @@ -440,13 +435,11 @@ impl InternalToCombinedView for SearchCombinedViewInternal { })) } else if let ( Some(post), - Some(counts), Some(creator), Some(community), Some(unread_comments), ) = ( v.post, - v.post_counts, v.item_creator.clone(), v.community.clone(), v.post_unread_comments, @@ -454,7 +447,6 @@ impl InternalToCombinedView for SearchCombinedViewInternal { Some(SearchCombinedView::Post(PostView { post, community, - counts, unread_comments, creator, creator_banned_from_community: v.item_creator_banned_from_community, diff --git a/crates/db_views/src/post/post_view.rs b/crates/db_views/src/post/post_view.rs index de7350c133..582619fc60 100644 --- a/crates/db_views/src/post/post_view.rs +++ b/crates/db_views/src/post/post_view.rs @@ -16,11 +16,7 @@ use diesel::{ }; use diesel_async::RunQueryDsl; use lemmy_db_schema::{ - aggregates::structs::{post_aggregates_keys as key, PostAggregates}, - aliases::creator_community_actions, - impls::local_user::LocalUserOptionHelper, - newtypes::{CommunityId, PersonId, PostId}, - schema::{ + aliases::creator_community_actions, impls::local_user::LocalUserOptionHelper, newtypes::{CommunityId, PersonId, PostId}, schema::{ community, community_actions, image_details, @@ -31,17 +27,14 @@ use lemmy_db_schema::{ person_actions, post, post_actions, - post_aggregates, post_tag, tag, - }, - source::{ + }, source::{ community::{CommunityFollower, CommunityFollowerState}, local_user::LocalUser, - post::{post_actions_keys, PostActionsCursor}, + post::{post_actions_keys, post_keys as key, Post, PostActionsCursor}, site::Site, - }, - utils::{ + }, traits::Crud, utils::{ functions::coalesce, fuzzy_search, get_conn, @@ -51,10 +44,7 @@ use lemmy_db_schema::{ Commented, DbPool, ReverseTimestampKey, - }, - CommunityVisibility, - ListingType, - PostSortType, + }, CommunityVisibility, ListingType, PostSortType }; use tracing::debug; use PostSortType::*; @@ -68,46 +58,45 @@ impl PostView { fn joins(my_person_id: Option) -> _ { let community_actions_join = community_actions::table.on( community_actions::community_id - .eq(post_aggregates::community_id) + .eq(post::community_id) .and(community_actions::person_id.nullable().eq(my_person_id)), ); let person_actions_join = person_actions::table.on( person_actions::target_id - .eq(post_aggregates::creator_id) + .eq(post::creator_id) .and(person_actions::person_id.nullable().eq(my_person_id)), ); let post_actions_join = post_actions::table.on( post_actions::post_id - .eq(post_aggregates::post_id) + .eq(post::id) .and(post_actions::person_id.nullable().eq(my_person_id)), ); let instance_actions_join = instance_actions::table.on( instance_actions::instance_id - .eq(post_aggregates::instance_id) + .eq(post::instance_id) .and(instance_actions::person_id.nullable().eq(my_person_id)), ); let post_creator_community_actions_join = creator_community_actions.on( creator_community_actions .field(community_actions::community_id) - .eq(post_aggregates::community_id) + .eq(post::community_id) .and( creator_community_actions .field(community_actions::person_id) - .eq(post_aggregates::creator_id), + .eq(post::creator_id), ), ); let image_details_join = image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())); - post_aggregates::table + post::table .inner_join(person::table) .inner_join(community::table) - .inner_join(post::table) .left_join(image_details_join) .left_join(community_actions_join) .left_join(person_actions_join) @@ -120,7 +109,7 @@ impl PostView { fn creator_is_admin() -> _ { exists( local_user::table.filter( - post_aggregates::creator_id + post::creator_id .eq(local_user::person_id) .and(local_user::admin.eq(true)), ), @@ -141,12 +130,12 @@ impl PostView { .select(diesel::dsl::sql::( "json_agg(tag.*)", )) - .filter(post_tag::post_id.eq(post_aggregates::post_id)) + .filter(post_tag::post_id.eq(post::id)) .filter(tag::deleted.eq(false)) .single_value(); let mut query = Self::joins(my_person_id) - .filter(post_aggregates::post_id.eq(post_id)) + .filter(post::id.eq(post_id)) .select(( post::all_columns, person::all_columns, @@ -162,7 +151,6 @@ impl PostView { .nullable() .is_not_null(), Self::creator_is_admin(), - post_aggregates::all_columns, CommunityFollower::select_subscribed_type(), post_actions::saved.nullable(), post_actions::read.nullable().is_not_null(), @@ -170,8 +158,8 @@ impl PostView { person_actions::blocked.nullable().is_not_null(), post_actions::like_score.nullable(), coalesce( - post_aggregates::comments.nullable() - post_actions::read_comments_amount.nullable(), - post_aggregates::comments, + post::comments.nullable() - post_actions::read_comments_amount.nullable(), + post::comments, ), post_tags, )) @@ -222,7 +210,7 @@ impl PaginationCursor { // get cursor for page that starts immediately after the given post pub fn after_post(view: &PostView) -> PaginationCursor { // hex encoding to prevent ossification - PaginationCursor(format!("P{:x}", view.counts.post_id.0)) + PaginationCursor(format!("P{:x}", view.post.id.0)) } pub async fn read( &self, @@ -237,13 +225,10 @@ impl PaginationCursor { .and_then(|e| i32::from_str_radix(e, 16).ok()) .ok_or_else(err_msg)?, ); - let post_aggregates = PostAggregates::read(pool, post_id).await?; + let post = Post::read(pool, post_id).await?; let post_actions = PostActionsCursor::read(pool, post_id, local_user.person_id()).await?; - Ok(PaginationCursorData { - post_aggregates, - post_actions, - }) + Ok(PaginationCursorData { post, post_actions }) } } @@ -251,7 +236,7 @@ impl PaginationCursor { // we only use some of the properties, depending on which sort type we page by #[derive(Clone)] pub struct PaginationCursorData { - post_aggregates: PostAggregates, + post: Post, post_actions: PostActionsCursor, } @@ -274,7 +259,7 @@ pub struct PostQuery<'a> { pub page: Option, pub limit: Option, pub page_after: Option, - pub page_before_or_equal: Option, + pub page_before_or_equal: Option, pub page_back: Option, pub show_hidden: Option, pub show_read: Option, @@ -353,7 +338,7 @@ impl<'a> PostQuery<'a> { } else { v.pop() }; - let limit_cursor = Some(item.expect("else case").counts); + let limit_cursor = Some(item.expect("else case").post); Ok(Some(PostQuery { page_before_or_equal: limit_cursor, ..self.clone() @@ -389,7 +374,7 @@ impl<'a> PostQuery<'a> { .select(diesel::dsl::sql::( "json_agg(tag.*)", )) - .filter(post_tag::post_id.eq(post_aggregates::post_id)) + .filter(post_tag::post_id.eq(post::id)) .filter(tag::deleted.eq(false)) .single_value(); @@ -409,7 +394,6 @@ impl<'a> PostQuery<'a> { .nullable() .is_not_null(), PostView::creator_is_admin(), - post_aggregates::all_columns, CommunityFollower::select_subscribed_type(), post_actions::saved.nullable(), post_actions::read.nullable().is_not_null(), @@ -417,8 +401,8 @@ impl<'a> PostQuery<'a> { person_actions::blocked.nullable().is_not_null(), post_actions::like_score.nullable(), coalesce( - post_aggregates::comments.nullable() - post_actions::read_comments_amount.nullable(), - post_aggregates::comments, + post::comments.nullable() - post_actions::read_comments_amount.nullable(), + post::comments, ), post_tags, )) @@ -448,11 +432,11 @@ impl<'a> PostQuery<'a> { .filter(post::removed.eq(false)); } if let Some(community_id) = o.community_id { - query = query.filter(post_aggregates::community_id.eq(community_id)); + query = query.filter(post::community_id.eq(community_id)); } if let Some(creator_id) = o.creator_id { - query = query.filter(post_aggregates::creator_id.eq(creator_id)); + query = query.filter(post::creator_id.eq(creator_id)); } let is_subscribed = community_actions::followed.is_not_null(); @@ -497,7 +481,7 @@ impl<'a> PostQuery<'a> { // Filter to show only posts with no comments if o.no_comments_only.unwrap_or_default() { - query = query.filter(post_aggregates::comments.eq(0)); + query = query.filter(post::comments.eq(0)); }; if !o.show_read.unwrap_or(o.local_user.show_read_posts()) { @@ -524,7 +508,7 @@ impl<'a> PostQuery<'a> { } if let Some(my_id) = o.local_user.person_id() { - let not_creator_filter = post_aggregates::creator_id.ne(my_id); + let not_creator_filter = post::creator_id.ne(my_id); if o.liked_only.unwrap_or_default() { query = query .filter(not_creator_filter) @@ -583,7 +567,7 @@ impl<'a> PostQuery<'a> { } else { let mut query = paginate( query, - o.page_after.map(|c| c.post_aggregates), + o.page_after.map(|c| c.post), o.page_before_or_equal, o.page_back.unwrap_or_default(), ); @@ -595,7 +579,7 @@ impl<'a> PostQuery<'a> { query.then_desc(key::featured_community) }; - let time = |interval| post_aggregates::published.gt(now() - interval); + let time = |interval| post::published.gt(now() - interval); // then use the main sort query = match o.sort.unwrap_or(Hot) { @@ -629,7 +613,7 @@ impl<'a> PostQuery<'a> { }; // finally use unique post id as tie breaker - query = query.then_desc(key::post_id); + query = query.then_desc(key::id); query.as_query() }; @@ -658,7 +642,6 @@ mod tests { use chrono::Utc; use diesel_async::SimpleAsyncConnection; use lemmy_db_schema::{ - aggregates::structs::PostAggregates, impls::actor_language::UNDETERMINED_ID, newtypes::LanguageId, source::{ diff --git a/crates/db_views/src/reports/post_report_view.rs b/crates/db_views/src/reports/post_report_view.rs index 5f246be9d1..e1212588db 100644 --- a/crates/db_views/src/reports/post_report_view.rs +++ b/crates/db_views/src/reports/post_report_view.rs @@ -19,7 +19,6 @@ use lemmy_db_schema::{ person_actions, post, post_actions, - post_aggregates, post_report, }, source::community::CommunityFollower, @@ -73,9 +72,6 @@ impl PostReportView { .and(person_actions::person_id.eq(my_person_id)), ); - let post_aggregates_join = - post_aggregates::table.on(post_report::post_id.eq(post_aggregates::post_id)); - let resolver_join = aliases::person2.on(post_report::resolver_id.eq(resolver_id.nullable())); post_report::table @@ -88,7 +84,6 @@ impl PostReportView { .left_join(local_user_join) .left_join(post_actions_join) .left_join(person_actions_join) - .inner_join(post_aggregates_join) .left_join(resolver_join) } @@ -126,10 +121,9 @@ impl PostReportView { person_actions::blocked.nullable().is_not_null(), post_actions::like_score.nullable(), coalesce( - post_aggregates::comments.nullable() - post_actions::read_comments_amount.nullable(), - post_aggregates::comments, + post::comments.nullable() - post_actions::read_comments_amount.nullable(), + post::comments, ), - post_aggregates::all_columns, aliases::person2.fields(person::all_columns.nullable()), )) .first(conn) diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index f818094b55..57922e2985 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -14,7 +14,7 @@ use diesel::{ Selectable, }; use lemmy_db_schema::{ - aggregates::structs::{CommunityAggregates, PersonAggregates, PostAggregates, SiteAggregates}, + aggregates::structs::{CommunityAggregates, PersonAggregates, SiteAggregates}, source::{ comment::Comment, comment_reply::CommentReply, @@ -273,7 +273,6 @@ pub struct PostReportView { #[cfg_attr(feature = "full", ts(optional))] pub my_vote: Option, pub unread_comments: i64, - pub counts: PostAggregates, #[cfg_attr(feature = "full", ts(optional))] pub resolver: Option, } @@ -321,7 +320,6 @@ pub struct PostView { pub banned_from_community: bool, pub creator_is_moderator: bool, pub creator_is_admin: bool, - pub counts: PostAggregates, pub subscribed: SubscribedType, #[cfg_attr(feature = "full", ts(optional))] /// The time when the post was saved. @@ -432,7 +430,6 @@ pub struct ReportCombinedViewInternal { // Post-specific pub post_report: Option, pub post: Option, - pub post_counts: Option, pub post_unread_comments: Option, pub post_saved: Option>, pub post_read: bool, @@ -479,7 +476,6 @@ pub enum ReportCombinedView { /// A combined person_content view pub(crate) struct PersonContentCombinedViewInternal { // Post-specific - pub post_counts: PostAggregates, pub post_unread_comments: i64, pub post_saved: Option>, pub post_read: bool, @@ -640,7 +636,6 @@ pub struct PersonPostMentionView { #[cfg_attr(feature = "full", ts(optional))] pub image_details: Option, pub recipient: Person, - pub counts: PostAggregates, pub creator_banned_from_community: bool, pub banned_from_community: bool, pub creator_is_moderator: bool, @@ -750,7 +745,6 @@ pub struct InboxCombinedViewInternal { pub person_comment_mention: Option, // Person post mention pub person_post_mention: Option, - pub post_counts: Option, pub post_unread_comments: Option, pub post_saved: Option>, pub post_read: bool, @@ -1125,7 +1119,6 @@ pub struct SearchCombinedPaginationCursor(pub String); pub(crate) struct SearchCombinedViewInternal { // Post-specific pub post: Option, - pub post_counts: Option, pub post_unread_comments: Option, pub post_saved: Option>, pub post_read: bool, diff --git a/crates/routes/src/feeds.rs b/crates/routes/src/feeds.rs index fc4f234569..421415fce7 100644 --- a/crates/routes/src/feeds.rs +++ b/crates/routes/src/feeds.rs @@ -480,9 +480,9 @@ fn create_post_items(posts: Vec, protocol_and_hostname: &str) -> Lemmy &p.creator.name, community_url, &p.community.name, - p.counts.score, + p.post.score, post_url, - p.counts.comments); + p.post.comments); // If its a url post, add it to the description // and see if we can parse it as a media enclosure. From 17d8d98f03f98b5d24ce64f4a7bc337c1c7a689f Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 10 Feb 2025 13:16:08 +0100 Subject: [PATCH 10/39] wip: update tests for post aggregate --- .../db_schema/replaceable_schema/triggers.sql | 34 +-- crates/db_schema/replaceable_schema/utils.sql | 2 +- crates/db_schema/src/aggregates/mod.rs | 2 - .../src/aggregates/post_aggregates.rs | 241 ---------------- crates/db_schema/src/impls/comment.rs | 103 +++++++ crates/db_schema/src/impls/post.rs | 267 ++++++++++++++++-- crates/db_views/src/post/post_view.rs | 17 +- 7 files changed, 371 insertions(+), 295 deletions(-) delete mode 100644 crates/db_schema/src/aggregates/post_aggregates.rs diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index 27de7ee1be..47bc9bf834 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -6,7 +6,7 @@ -- before the automatic deletion of the row that references it. This is not a problem for insert or delete. -- -- Triggers that update multiple tables should use this order: person_aggregates, comment_aggregates, --- post_aggregates, community_aggregates, site_aggregates +-- post, community_aggregates, site_aggregates -- * The order matters because the updated rows are locked until the end of the transaction, and statements -- in a trigger don't use separate transactions. This means that updates closer to the beginning cause -- longer locks because the duration of each update extends the durations of the locks caused by previous @@ -19,7 +19,7 @@ -- -- -- Create triggers for both post and comments -CREATE FUNCTION r.creator_id_from_post_aggregates (agg post_aggregates) +CREATE FUNCTION r.creator_id_from_post (agg post) RETURNS int IMMUTABLE PARALLEL SAFE RETURN agg.creator_id; CREATE PROCEDURE r.post_or_comment (table_name text) @@ -138,7 +138,7 @@ WHERE WITH post_diff AS ( UPDATE - post_aggregates AS a + post AS a SET comments = a.comments + diff.comments, newest_comment_time = GREATEST (a.newest_comment_time, diff.newest_comment_time), @@ -163,7 +163,7 @@ WITH post_diff AS ( GROUP BY post.id) AS diff WHERE - a.post_id = diff.post_id + a.id = diff.post_id AND (diff.comments, GREATEST (a.newest_comment_time, diff.newest_comment_time), GREATEST (a.newest_comment_time_necro, diff.newest_comment_time_necro)) != (0, @@ -328,12 +328,12 @@ BEGIN 1 ELSE -1 - END) * post_aggregates.comments) AS comments + END) * post.comments) AS comments FROM new_post INNER JOIN old_post ON new_post.id = old_post.id AND (r.is_counted (new_post.*) != r.is_counted (old_post.*)) - INNER JOIN post_aggregates ON post_aggregates.post_id = new_post.id + INNER JOIN post ON post.id = new_post.id GROUP BY old_post.community_id) AS diff WHERE @@ -377,7 +377,7 @@ $$); CALL r.create_triggers ('post_report', $$ BEGIN UPDATE - post_aggregates AS a + post AS a SET report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count FROM ( @@ -385,7 +385,7 @@ BEGIN (post_report).post_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (post_report).resolved), 0) AS unresolved_report_count FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (post_report).post_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) - AND a.post_id = diff.post_id; + AND a.id = diff.post_id; RETURN NULL; @@ -472,12 +472,12 @@ CREATE TRIGGER aggregates FOR EACH STATEMENT EXECUTE FUNCTION r.person_aggregates_from_person (); -CREATE FUNCTION r.post_aggregates_from_post () +CREATE FUNCTION r.post_from_post () RETURNS TRIGGER LANGUAGE plpgsql AS $$ BEGIN - INSERT INTO post_aggregates (post_id, published, newest_comment_time, newest_comment_time_necro, community_id, creator_id, instance_id, featured_community, featured_local) + INSERT INTO post (post_id, published, newest_comment_time, newest_comment_time_necro, community_id, creator_id, instance_id, featured_community, featured_local) SELECT new_post.id, new_post.published, @@ -498,15 +498,15 @@ $$; CREATE TRIGGER aggregates AFTER INSERT ON post REFERENCING NEW TABLE AS new_post FOR EACH STATEMENT - EXECUTE FUNCTION r.post_aggregates_from_post (); + EXECUTE FUNCTION r.post_from_post (); -CREATE FUNCTION r.post_aggregates_from_post_update () +CREATE FUNCTION r.post_from_post_update () RETURNS TRIGGER LANGUAGE plpgsql AS $$ BEGIN UPDATE - post_aggregates + post SET featured_community = new_post.featured_community, featured_local = new_post.featured_local @@ -517,7 +517,7 @@ BEGIN old_post.featured_local) != (new_post.featured_community, new_post.featured_local) WHERE - post_aggregates.post_id = new_post.id; + post.id = new_post.id; RETURN NULL; END; $$; @@ -525,7 +525,7 @@ $$; CREATE TRIGGER aggregates_update AFTER UPDATE ON post REFERENCING OLD TABLE AS old_post NEW TABLE AS new_post FOR EACH STATEMENT - EXECUTE FUNCTION r.post_aggregates_from_post_update (); + EXECUTE FUNCTION r.post_from_post_update (); CREATE FUNCTION r.site_aggregates_from_site () RETURNS TRIGGER @@ -931,7 +931,7 @@ CALL r.create_search_combined_trigger ('community'); CALL r.create_search_combined_trigger ('person'); -- You also need to triggers to update the `score` column. --- post | post_aggregates::score +-- post | post::score -- comment | comment_aggregates::score -- community | community_aggregates::users_active_monthly -- person | person_aggregates::post_score @@ -953,7 +953,7 @@ END $$; CREATE TRIGGER search_combined_post_score - AFTER UPDATE OF score ON post_aggregates + AFTER UPDATE OF score ON post FOR EACH ROW EXECUTE FUNCTION r.search_combined_post_score_update (); diff --git a/crates/db_schema/replaceable_schema/utils.sql b/crates/db_schema/replaceable_schema/utils.sql index 8c3c96ce12..a951f1a206 100644 --- a/crates/db_schema/replaceable_schema/utils.sql +++ b/crates/db_schema/replaceable_schema/utils.sql @@ -225,7 +225,7 @@ BEGIN COALESCE(sum(comments + upvotes + downvotes)::bigint, 0) AS count_, community_id AS community_id_ FROM - post_aggregates + post WHERE published >= (CURRENT_TIMESTAMP - i::interval) GROUP BY diff --git a/crates/db_schema/src/aggregates/mod.rs b/crates/db_schema/src/aggregates/mod.rs index 0df9ffc744..8e72aace7d 100644 --- a/crates/db_schema/src/aggregates/mod.rs +++ b/crates/db_schema/src/aggregates/mod.rs @@ -5,7 +5,5 @@ pub mod person_aggregates; #[cfg(feature = "full")] pub mod person_post_aggregates; #[cfg(feature = "full")] -pub mod post_aggregates; -#[cfg(feature = "full")] pub mod site_aggregates; pub mod structs; diff --git a/crates/db_schema/src/aggregates/post_aggregates.rs b/crates/db_schema/src/aggregates/post_aggregates.rs deleted file mode 100644 index 8f4e29e31b..0000000000 --- a/crates/db_schema/src/aggregates/post_aggregates.rs +++ /dev/null @@ -1,241 +0,0 @@ -/*use crate::{ - aggregates::structs::PostAggregates, - newtypes::PostId, - schema::{community_aggregates, post, post_aggregates}, - utils::{ - functions::{hot_rank, scaled_rank}, - get_conn, - DbPool, - }, -}; -use diesel::{result::Error, ExpressionMethods, JoinOnDsl, QueryDsl}; -use diesel_async::RunQueryDsl; - -impl PostAggregates { - pub async fn read(pool: &mut DbPool<'_>, post_id: PostId) -> Result { - let conn = &mut get_conn(pool).await?; - post_aggregates::table.find(post_id).first(conn).await - } - -} - -#[cfg(test)] -mod tests { - - use crate::{ - aggregates::post_aggregates::PostAggregates, - source::{ - comment::{Comment, CommentInsertForm, CommentUpdateForm}, - community::{Community, CommunityInsertForm}, - instance::Instance, - person::{Person, PersonInsertForm}, - post::{Post, PostInsertForm, PostLike, PostLikeForm}, - }, - traits::{Crud, Likeable}, - utils::build_db_pool_for_tests, - }; - use diesel::result::Error; - use pretty_assertions::assert_eq; - use serial_test::serial; - - #[tokio::test] - #[serial] - async fn test_crud() -> Result<(), Error> { - let pool = &build_db_pool_for_tests(); - let pool = &mut pool.into(); - - let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; - - let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_community_agg"); - - let inserted_person = Person::create(pool, &new_person).await?; - - let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_community_agg"); - - let another_inserted_person = Person::create(pool, &another_person).await?; - - let new_community = CommunityInsertForm::new( - inserted_instance.id, - "TIL_community_agg".into(), - "nada".to_owned(), - "pubkey".to_string(), - ); - let inserted_community = Community::create(pool, &new_community).await?; - - let new_post = PostInsertForm::new( - "A test post".into(), - inserted_person.id, - inserted_community.id, - ); - let inserted_post = Post::create(pool, &new_post).await?; - - let comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - let inserted_comment = Comment::create(pool, &comment_form, None).await?; - - let child_comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - let inserted_child_comment = - Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; - - let post_like = PostLikeForm::new(inserted_post.id, inserted_person.id, 1); - - PostLike::like(pool, &post_like).await?; - - let post_aggs_before_delete = PostAggregates::read(pool, inserted_post.id).await?; - - assert_eq!(2, post_aggs_before_delete.comments); - assert_eq!(1, post_aggs_before_delete.score); - assert_eq!(1, post_aggs_before_delete.upvotes); - assert_eq!(0, post_aggs_before_delete.downvotes); - - // Add a post dislike from the other person - let post_dislike = PostLikeForm::new(inserted_post.id, another_inserted_person.id, -1); - - PostLike::like(pool, &post_dislike).await?; - - let post_aggs_after_dislike = PostAggregates::read(pool, inserted_post.id).await?; - - assert_eq!(2, post_aggs_after_dislike.comments); - assert_eq!(0, post_aggs_after_dislike.score); - assert_eq!(1, post_aggs_after_dislike.upvotes); - assert_eq!(1, post_aggs_after_dislike.downvotes); - - // Remove the comments - Comment::delete(pool, inserted_comment.id).await?; - Comment::delete(pool, inserted_child_comment.id).await?; - let after_comment_delete = PostAggregates::read(pool, inserted_post.id).await?; - assert_eq!(0, after_comment_delete.comments); - assert_eq!(0, after_comment_delete.score); - assert_eq!(1, after_comment_delete.upvotes); - assert_eq!(1, after_comment_delete.downvotes); - - // Remove the first post like - PostLike::remove(pool, inserted_person.id, inserted_post.id).await?; - let after_like_remove = PostAggregates::read(pool, inserted_post.id).await?; - assert_eq!(0, after_like_remove.comments); - assert_eq!(-1, after_like_remove.score); - assert_eq!(0, after_like_remove.upvotes); - assert_eq!(1, after_like_remove.downvotes); - - // This should delete all the associated rows, and fire triggers - Person::delete(pool, another_inserted_person.id).await?; - let person_num_deleted = Person::delete(pool, inserted_person.id).await?; - assert_eq!(1, person_num_deleted); - - // Delete the community - let community_num_deleted = Community::delete(pool, inserted_community.id).await?; - assert_eq!(1, community_num_deleted); - - // Should be none found, since the creator was deleted - let after_delete = PostAggregates::read(pool, inserted_post.id).await; - assert!(after_delete.is_err()); - - Instance::delete(pool, inserted_instance.id).await?; - - Ok(()) - } - - #[tokio::test] - #[serial] - async fn test_soft_delete() -> Result<(), Error> { - let pool = &build_db_pool_for_tests(); - let pool = &mut pool.into(); - - let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; - - let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_community_agg"); - - let inserted_person = Person::create(pool, &new_person).await?; - - let new_community = CommunityInsertForm::new( - inserted_instance.id, - "TIL_community_agg".into(), - "nada".to_owned(), - "pubkey".to_string(), - ); - let inserted_community = Community::create(pool, &new_community).await?; - - let new_post = PostInsertForm::new( - "A test post".into(), - inserted_person.id, - inserted_community.id, - ); - let inserted_post = Post::create(pool, &new_post).await?; - - let comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - - let inserted_comment = Comment::create(pool, &comment_form, None).await?; - - let post_aggregates_before = PostAggregates::read(pool, inserted_post.id).await?; - assert_eq!(1, post_aggregates_before.comments); - - Comment::update( - pool, - inserted_comment.id, - &CommentUpdateForm { - removed: Some(true), - ..Default::default() - }, - ) - .await?; - - let post_aggregates_after_remove = PostAggregates::read(pool, inserted_post.id).await?; - assert_eq!(0, post_aggregates_after_remove.comments); - - Comment::update( - pool, - inserted_comment.id, - &CommentUpdateForm { - removed: Some(false), - ..Default::default() - }, - ) - .await?; - - Comment::update( - pool, - inserted_comment.id, - &CommentUpdateForm { - deleted: Some(true), - ..Default::default() - }, - ) - .await?; - - let post_aggregates_after_delete = PostAggregates::read(pool, inserted_post.id).await?; - assert_eq!(0, post_aggregates_after_delete.comments); - - Comment::update( - pool, - inserted_comment.id, - &CommentUpdateForm { - removed: Some(true), - ..Default::default() - }, - ) - .await?; - - let post_aggregates_after_delete_remove = PostAggregates::read(pool, inserted_post.id).await?; - assert_eq!(0, post_aggregates_after_delete_remove.comments); - - Comment::delete(pool, inserted_comment.id).await?; - Post::delete(pool, inserted_post.id).await?; - Person::delete(pool, inserted_person.id).await?; - Community::delete(pool, inserted_community.id).await?; - Instance::delete(pool, inserted_instance.id).await?; - - Ok(()) - } -} -*/ diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 40c879823b..99ab1725f2 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -368,4 +368,107 @@ mod tests { assert_eq!(LanguageId::default(), comment.language_id); Ok(()) } + + #[tokio::test] + #[serial] + async fn test_aggregates() -> Result<(), Error> { + let pool = &build_db_pool_for_tests(); + let pool = &mut pool.into(); + + let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; + + let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_comment_agg"); + + let inserted_person = Person::create(pool, &new_person).await?; + + let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_comment_agg"); + + let another_inserted_person = Person::create(pool, &another_person).await?; + + let new_community = CommunityInsertForm::new( + inserted_instance.id, + "TIL_comment_agg".into(), + "nada".to_owned(), + "pubkey".to_string(), + ); + let inserted_community = Community::create(pool, &new_community).await?; + + let new_post = PostInsertForm::new( + "A test post".into(), + inserted_person.id, + inserted_community.id, + ); + let inserted_post = Post::create(pool, &new_post).await?; + + let comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + let inserted_comment = Comment::create(pool, &comment_form, None).await?; + + let child_comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + let _inserted_child_comment = + Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; + + let comment_like = CommentLikeForm { + comment_id: inserted_comment.id, + person_id: inserted_person.id, + score: 1, + }; + + CommentLike::like(pool, &comment_like).await?; + + let comment_aggs_before_delete = Comment::read(pool, inserted_comment.id).await?; + + assert_eq!(1, comment_aggs_before_delete.score); + assert_eq!(1, comment_aggs_before_delete.upvotes); + assert_eq!(0, comment_aggs_before_delete.downvotes); + + // Add a post dislike from the other person + let comment_dislike = CommentLikeForm { + comment_id: inserted_comment.id, + person_id: another_inserted_person.id, + score: -1, + }; + + CommentLike::like(pool, &comment_dislike).await?; + + let comment_aggs_after_dislike = Comment::read(pool, inserted_comment.id).await?; + + assert_eq!(0, comment_aggs_after_dislike.score); + assert_eq!(1, comment_aggs_after_dislike.upvotes); + assert_eq!(1, comment_aggs_after_dislike.downvotes); + + // Remove the first comment like + CommentLike::remove(pool, inserted_person.id, inserted_comment.id).await?; + let after_like_remove = Comment::read(pool, inserted_comment.id).await?; + assert_eq!(-1, after_like_remove.score); + assert_eq!(0, after_like_remove.upvotes); + assert_eq!(1, after_like_remove.downvotes); + + // Remove the parent post + Post::delete(pool, inserted_post.id).await?; + + // Should be none found, since the post was deleted + let after_delete = Comment::read(pool, inserted_comment.id).await; + assert!(after_delete.is_err()); + + // This should delete all the associated rows, and fire triggers + Person::delete(pool, another_inserted_person.id).await?; + let person_num_deleted = Person::delete(pool, inserted_person.id).await?; + assert_eq!(1, person_num_deleted); + + // Delete the community + let community_num_deleted = Community::delete(pool, inserted_community.id).await?; + assert_eq!(1, community_num_deleted); + + Instance::delete(pool, inserted_instance.id).await?; + + Ok(()) + } } diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index c3369bf9dd..3d40ba840f 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -466,6 +466,7 @@ mod tests { use crate::{ source::{ + comment::{Comment, CommentInsertForm, CommentUpdateForm}, community::{Community, CommunityInsertForm}, instance::Instance, person::{Person, PersonInsertForm}, @@ -485,6 +486,7 @@ mod tests { utils::{build_db_pool_for_tests, uplete}, }; use chrono::DateTime; + use diesel::result::Error; use lemmy_utils::error::LemmyResult; use pretty_assertions::assert_eq; use serial_test::serial; @@ -531,33 +533,6 @@ mod tests { }; let inserted_scheduled_post = Post::create(pool, &new_scheduled_post).await?; - let expected_post = Post { - id: inserted_post.id, - name: "A test post".into(), - url: None, - body: None, - alt_text: None, - creator_id: inserted_person.id, - community_id: inserted_community.id, - published: inserted_post.published, - removed: false, - locked: false, - nsfw: false, - deleted: false, - updated: None, - embed_title: None, - embed_description: None, - embed_video_url: None, - thumbnail_url: None, - ap_id: Url::parse(&format!("https://lemmy-alpha/post/{}", inserted_post.id))?.into(), - local: true, - language_id: Default::default(), - featured_community: false, - featured_local: false, - url_content_type: None, - scheduled_publish_time: None, - }; - // Post Like let post_like_form = PostLikeForm::new(inserted_post.id, inserted_person.id, 1); @@ -621,12 +596,244 @@ mod tests { Person::delete(pool, inserted_person.id).await?; Instance::delete(pool, inserted_instance.id).await?; - assert_eq!(expected_post, read_post); - assert_eq!(expected_post, inserted_post); - assert_eq!(expected_post, updated_post); + expect_post(&inserted_post, &read_post); + expect_post(&inserted_post, &updated_post); assert_eq!(expected_post_like, inserted_post_like); assert_eq!(expected_post_saved, inserted_post_saved); Ok(()) } + + fn expect_post(inserted_post: &Post, expected_post: &Post) { + assert_eq!(inserted_post.id, expected_post.id); + todo!() + /* + let expected_post = Post { + id: inserted_post.id, + name: "A test post".into(), + url: None, + body: None, + alt_text: None, + creator_id: inserted_person.id, + community_id: inserted_community.id, + published: inserted_post.published, + removed: false, + locked: false, + nsfw: false, + deleted: false, + updated: None, + embed_title: None, + embed_description: None, + embed_video_url: None, + thumbnail_url: None, + ap_id: Url::parse(&format!("https://lemmy-alpha/post/{}", inserted_post.id))?.into(), + local: true, + language_id: Default::default(), + featured_community: false, + featured_local: false, + url_content_type: None, + scheduled_publish_time: None, + }; + */ + } + + #[tokio::test] + #[serial] + async fn test_aggregates() -> Result<(), Error> { + let pool = &build_db_pool_for_tests(); + let pool = &mut pool.into(); + + let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; + + let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_community_agg"); + + let inserted_person = Person::create(pool, &new_person).await?; + + let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_community_agg"); + + let another_inserted_person = Person::create(pool, &another_person).await?; + + let new_community = CommunityInsertForm::new( + inserted_instance.id, + "TIL_community_agg".into(), + "nada".to_owned(), + "pubkey".to_string(), + ); + let inserted_community = Community::create(pool, &new_community).await?; + + let new_post = PostInsertForm::new( + "A test post".into(), + inserted_person.id, + inserted_community.id, + ); + let inserted_post = Post::create(pool, &new_post).await?; + + let comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + let inserted_comment = Comment::create(pool, &comment_form, None).await?; + + let child_comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + let inserted_child_comment = + Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; + + let post_like = PostLikeForm::new(inserted_post.id, inserted_person.id, 1); + + PostLike::like(pool, &post_like).await?; + + let post_aggs_before_delete = Post::read(pool, inserted_post.id).await?; + + assert_eq!(2, post_aggs_before_delete.comments); + assert_eq!(1, post_aggs_before_delete.score); + assert_eq!(1, post_aggs_before_delete.upvotes); + assert_eq!(0, post_aggs_before_delete.downvotes); + + // Add a post dislike from the other person + let post_dislike = PostLikeForm::new(inserted_post.id, another_inserted_person.id, -1); + + PostLike::like(pool, &post_dislike).await?; + + let post_aggs_after_dislike = Post::read(pool, inserted_post.id).await?; + + assert_eq!(2, post_aggs_after_dislike.comments); + assert_eq!(0, post_aggs_after_dislike.score); + assert_eq!(1, post_aggs_after_dislike.upvotes); + assert_eq!(1, post_aggs_after_dislike.downvotes); + + // Remove the comments + Comment::delete(pool, inserted_comment.id).await?; + Comment::delete(pool, inserted_child_comment.id).await?; + let after_comment_delete = Post::read(pool, inserted_post.id).await?; + assert_eq!(0, after_comment_delete.comments); + assert_eq!(0, after_comment_delete.score); + assert_eq!(1, after_comment_delete.upvotes); + assert_eq!(1, after_comment_delete.downvotes); + + // Remove the first post like + PostLike::remove(pool, inserted_person.id, inserted_post.id).await?; + let after_like_remove = Post::read(pool, inserted_post.id).await?; + assert_eq!(0, after_like_remove.comments); + assert_eq!(-1, after_like_remove.score); + assert_eq!(0, after_like_remove.upvotes); + assert_eq!(1, after_like_remove.downvotes); + + // This should delete all the associated rows, and fire triggers + Person::delete(pool, another_inserted_person.id).await?; + let person_num_deleted = Person::delete(pool, inserted_person.id).await?; + assert_eq!(1, person_num_deleted); + + // Delete the community + let community_num_deleted = Community::delete(pool, inserted_community.id).await?; + assert_eq!(1, community_num_deleted); + + // Should be none found, since the creator was deleted + let after_delete = Post::read(pool, inserted_post.id).await; + assert!(after_delete.is_err()); + + Instance::delete(pool, inserted_instance.id).await?; + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_aggregates_soft_delete() -> Result<(), Error> { + let pool = &build_db_pool_for_tests(); + let pool = &mut pool.into(); + + let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; + + let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_community_agg"); + + let inserted_person = Person::create(pool, &new_person).await?; + + let new_community = CommunityInsertForm::new( + inserted_instance.id, + "TIL_community_agg".into(), + "nada".to_owned(), + "pubkey".to_string(), + ); + let inserted_community = Community::create(pool, &new_community).await?; + + let new_post = PostInsertForm::new( + "A test post".into(), + inserted_person.id, + inserted_community.id, + ); + let inserted_post = Post::create(pool, &new_post).await?; + + let comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + + let inserted_comment = Comment::create(pool, &comment_form, None).await?; + + let post_aggregates_before = Post::read(pool, inserted_post.id).await?; + assert_eq!(1, post_aggregates_before.comments); + + Comment::update( + pool, + inserted_comment.id, + &CommentUpdateForm { + removed: Some(true), + ..Default::default() + }, + ) + .await?; + + let post_aggregates_after_remove = Post::read(pool, inserted_post.id).await?; + assert_eq!(0, post_aggregates_after_remove.comments); + + Comment::update( + pool, + inserted_comment.id, + &CommentUpdateForm { + removed: Some(false), + ..Default::default() + }, + ) + .await?; + + Comment::update( + pool, + inserted_comment.id, + &CommentUpdateForm { + deleted: Some(true), + ..Default::default() + }, + ) + .await?; + + let post_aggregates_after_delete = Post::read(pool, inserted_post.id).await?; + assert_eq!(0, post_aggregates_after_delete.comments); + + Comment::update( + pool, + inserted_comment.id, + &CommentUpdateForm { + removed: Some(true), + ..Default::default() + }, + ) + .await?; + + let post_aggregates_after_delete_remove = Post::read(pool, inserted_post.id).await?; + assert_eq!(0, post_aggregates_after_delete_remove.comments); + + Comment::delete(pool, inserted_comment.id).await?; + Post::delete(pool, inserted_post.id).await?; + Person::delete(pool, inserted_person.id).await?; + Community::delete(pool, inserted_community.id).await?; + Instance::delete(pool, inserted_instance.id).await?; + + Ok(()) + } } diff --git a/crates/db_views/src/post/post_view.rs b/crates/db_views/src/post/post_view.rs index 582619fc60..ff6c0f1073 100644 --- a/crates/db_views/src/post/post_view.rs +++ b/crates/db_views/src/post/post_view.rs @@ -16,7 +16,10 @@ use diesel::{ }; use diesel_async::RunQueryDsl; use lemmy_db_schema::{ - aliases::creator_community_actions, impls::local_user::LocalUserOptionHelper, newtypes::{CommunityId, PersonId, PostId}, schema::{ + aliases::creator_community_actions, + impls::local_user::LocalUserOptionHelper, + newtypes::{CommunityId, PersonId, PostId}, + schema::{ community, community_actions, image_details, @@ -29,12 +32,15 @@ use lemmy_db_schema::{ post_actions, post_tag, tag, - }, source::{ + }, + source::{ community::{CommunityFollower, CommunityFollowerState}, local_user::LocalUser, post::{post_actions_keys, post_keys as key, Post, PostActionsCursor}, site::Site, - }, traits::Crud, utils::{ + }, + traits::Crud, + utils::{ functions::coalesce, fuzzy_search, get_conn, @@ -44,7 +50,10 @@ use lemmy_db_schema::{ Commented, DbPool, ReverseTimestampKey, - }, CommunityVisibility, ListingType, PostSortType + }, + CommunityVisibility, + ListingType, + PostSortType, }; use tracing::debug; use PostSortType::*; From 4124f03e12b7afdfc4b16f8531dfb0e8ba73f900 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 10 Feb 2025 14:06:22 +0100 Subject: [PATCH 11/39] format --- crates/api_crud/src/post/read.rs | 8 +++++++- crates/db_schema/Cargo.toml | 3 ++- crates/db_views/src/combined/inbox_combined_view.rs | 7 +------ crates/db_views/src/combined/search_combined_view.rs | 7 +------ 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/api_crud/src/post/read.rs b/crates/api_crud/src/post/read.rs index d806533c46..eeaa5ce593 100644 --- a/crates/api_crud/src/post/read.rs +++ b/crates/api_crud/src/post/read.rs @@ -66,7 +66,13 @@ pub async fn get_post( let read_form = PostReadForm::new(post_id, person_id); PostRead::mark_as_read(&mut context.pool(), &read_form).await?; - update_read_comments(person_id, post_id, post_view.post.comments, &mut context.pool()).await?; + update_read_comments( + person_id, + post_id, + post_view.post.comments, + &mut context.pool(), + ) + .await?; } // Necessary for the sidebar subscribed diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index 89ed13e26c..9a907e19dc 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -54,7 +54,8 @@ diesel = { workspace = true, features = [ "chrono", "postgres", "serde_json", - "uuid","64-column-tables" + "uuid", + "64-column-tables", ], optional = true } diesel-derive-newtype = { workspace = true, optional = true } diesel-derive-enum = { workspace = true, optional = true } diff --git a/crates/db_views/src/combined/inbox_combined_view.rs b/crates/db_views/src/combined/inbox_combined_view.rs index c004f00d5f..61b6f8023a 100644 --- a/crates/db_views/src/combined/inbox_combined_view.rs +++ b/crates/db_views/src/combined/inbox_combined_view.rs @@ -460,12 +460,7 @@ impl InternalToCombinedView for InboxCombinedViewInternal { banned_from_community: v.banned_from_community, }, )) - } else if let ( - Some(person_post_mention), - Some(post), - Some(unread_comments), - Some(community), - ) = ( + } else if let (Some(person_post_mention), Some(post), Some(unread_comments), Some(community)) = ( v.person_post_mention, v.post, v.post_unread_comments, diff --git a/crates/db_views/src/combined/search_combined_view.rs b/crates/db_views/src/combined/search_combined_view.rs index b6f59c091f..617241d876 100644 --- a/crates/db_views/src/combined/search_combined_view.rs +++ b/crates/db_views/src/combined/search_combined_view.rs @@ -433,12 +433,7 @@ impl InternalToCombinedView for SearchCombinedViewInternal { my_vote: v.my_comment_vote, banned_from_community: v.banned_from_community, })) - } else if let ( - Some(post), - Some(creator), - Some(community), - Some(unread_comments), - ) = ( + } else if let (Some(post), Some(creator), Some(community), Some(unread_comments)) = ( v.post, v.item_creator.clone(), v.community.clone(), From c9aaaf04694af96375d78cbed644c931487d6f09 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 10 Feb 2025 14:34:20 +0100 Subject: [PATCH 12/39] fix partialeq --- crates/db_schema/src/impls/comment.rs | 54 ++++++++--------- crates/db_schema/src/impls/post.rs | 80 ++++++++++++++------------ crates/db_schema/src/source/comment.rs | 36 +----------- crates/db_schema/src/source/post.rs | 13 +---- 4 files changed, 73 insertions(+), 110 deletions(-) diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 99ab1725f2..e706e568f9 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -280,6 +280,30 @@ mod tests { ); let inserted_comment = Comment::create(pool, &comment_form, None).await?; + let expected_comment = Comment { + id: inserted_comment.id, + content: "A test comment".into(), + creator_id: inserted_person.id, + post_id: inserted_post.id, + removed: false, + deleted: false, + path: Ltree(format!("0.{}", inserted_comment.id)), + published: inserted_comment.published, + updated: None, + ap_id: Url::parse(&format!( + "https://lemmy-alpha/comment/{}", + inserted_comment.id + ))? + .into(), + distinguished: false, + local: true, + language_id: LanguageId::default(), + child_count: 0, + controversy_rank: 0.0, + downvotes: 0, + upvotes: 0,score: 0, hot_rank: 0.0, report_count: 0, unresolved_report_count: 0 + }; + let child_comment_form = CommentInsertForm::new( inserted_person.id, inserted_post.id, @@ -331,12 +355,13 @@ mod tests { Person::delete(pool, inserted_person.id).await?; Instance::delete(pool, inserted_instance.id).await?; - expect_comment(&inserted_comment, &read_comment)?; - expect_comment(&inserted_comment, &updated_comment)?; + assert_eq!(expected_comment, read_comment); + assert_eq!(expected_comment, inserted_comment); + assert_eq!(expected_comment, updated_comment); assert_eq!(expected_comment_like, inserted_comment_like); assert_eq!(expected_comment_saved, inserted_comment_saved); assert_eq!( - format!("0.{}.{}", read_comment.id, inserted_child_comment.id), + format!("0.{}.{}", expected_comment.id, inserted_child_comment.id), inserted_child_comment.path.0, ); assert_eq!(uplete::Count::only_updated(1), like_removed); @@ -346,29 +371,6 @@ mod tests { Ok(()) } - fn expect_comment(inserted_comment: &Comment, comment: &Comment) -> LemmyResult<()> { - assert_eq!(inserted_comment.id, comment.id); - assert_eq!("A test comment".to_string(), comment.content); - assert_eq!(inserted_comment.post_id, comment.post_id); - assert_eq!(inserted_comment.creator_id, comment.creator_id); - assert!(!comment.removed); - assert!(!comment.deleted); - assert_eq!(Ltree(format!("0.{}", inserted_comment.id)), comment.path); - assert_eq!(inserted_comment.published, comment.published); - assert_eq!(None, comment.updated); - assert_eq!( - &Url::parse(&format!( - "https://lemmy-alpha/comment/{}", - inserted_comment.id - ))?, - comment.ap_id.inner() - ); - assert!(!comment.distinguished); - assert!(comment.local); - assert_eq!(LanguageId::default(), comment.language_id); - Ok(()) - } - #[tokio::test] #[serial] async fn test_aggregates() -> Result<(), Error> { diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index 3d40ba840f..e58637dfd3 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -463,8 +463,8 @@ impl PostActionsCursor { #[cfg(test)] mod tests { - use crate::{ + newtypes::InstanceId, source::{ comment::{Comment, CommentInsertForm, CommentUpdateForm}, community::{Community, CommunityInsertForm}, @@ -485,7 +485,7 @@ mod tests { traits::{Crud, Likeable, Saveable}, utils::{build_db_pool_for_tests, uplete}, }; - use chrono::DateTime; + use chrono::{DateTime, Utc}; use diesel::result::Error; use lemmy_utils::error::LemmyResult; use pretty_assertions::assert_eq; @@ -533,6 +533,44 @@ mod tests { }; let inserted_scheduled_post = Post::create(pool, &new_scheduled_post).await?; + let expected_post = Post { + id: inserted_post.id, + name: "A test post".into(), + url: None, + body: None, + alt_text: None, + creator_id: inserted_person.id, + community_id: inserted_community.id, + published: inserted_post.published, + removed: false, + locked: false, + nsfw: false, + deleted: false, + updated: None, + embed_title: None, + embed_description: None, + embed_video_url: None, + thumbnail_url: None, + ap_id: Url::parse(&format!("https://lemmy-alpha/post/{}", inserted_post.id))?.into(), + local: true, + language_id: Default::default(), + featured_community: false, + featured_local: false, + url_content_type: None, + scheduled_publish_time: None, + comments: 0, + controversy_rank: 0.0, + downvotes: 0, + upvotes: 0, + score: 0, + hot_rank: 0.0, + hot_rank_active: 0.0, + instance_id: InstanceId(0), + newest_comment_time: Utc::now(), + newest_comment_time_necro: Utc::now(), + report_count: 0,scaled_rank: 0.0,unresolved_report_count: 0 + }; + // Post Like let post_like_form = PostLikeForm::new(inserted_post.id, inserted_person.id, 1); @@ -596,47 +634,15 @@ mod tests { Person::delete(pool, inserted_person.id).await?; Instance::delete(pool, inserted_instance.id).await?; - expect_post(&inserted_post, &read_post); - expect_post(&inserted_post, &updated_post); + assert_eq!(expected_post, read_post); + assert_eq!(expected_post, inserted_post); + assert_eq!(expected_post, updated_post); assert_eq!(expected_post_like, inserted_post_like); assert_eq!(expected_post_saved, inserted_post_saved); Ok(()) } - fn expect_post(inserted_post: &Post, expected_post: &Post) { - assert_eq!(inserted_post.id, expected_post.id); - todo!() - /* - let expected_post = Post { - id: inserted_post.id, - name: "A test post".into(), - url: None, - body: None, - alt_text: None, - creator_id: inserted_person.id, - community_id: inserted_community.id, - published: inserted_post.published, - removed: false, - locked: false, - nsfw: false, - deleted: false, - updated: None, - embed_title: None, - embed_description: None, - embed_video_url: None, - thumbnail_url: None, - ap_id: Url::parse(&format!("https://lemmy-alpha/post/{}", inserted_post.id))?.into(), - local: true, - language_id: Default::default(), - featured_community: false, - featured_local: false, - url_content_type: None, - scheduled_publish_time: None, - }; - */ - } - #[tokio::test] #[serial] async fn test_aggregates() -> Result<(), Error> { diff --git a/crates/db_schema/src/source/comment.rs b/crates/db_schema/src/source/comment.rs index d30f0a9e8b..a66dad02cb 100644 --- a/crates/db_schema/src/source/comment.rs +++ b/crates/db_schema/src/source/comment.rs @@ -14,7 +14,7 @@ use serde_with::skip_serializing_none; use ts_rs::TS; #[skip_serializing_none] -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[cfg_attr( feature = "full", derive(Queryable, Selectable, Associations, Identifiable, TS) @@ -64,40 +64,6 @@ pub struct Comment { pub unresolved_report_count: i16, } -impl PartialEq for Comment { - fn eq(&self, _other: &Self) -> bool { - // TODO: is this really needed? seems only for tests, so we could rewrite - // them to compare individual fields. or use `derivative` crate - todo!() - /* - self.id == other.id - && self.creator_id == other.creator_id - && self.post_id == other.post_id - && self.content == other.content - && self.removed == other.removed - && self.published == other.published - && self.updated == other.updated - && self.deleted == other.deleted - && self.ap_id == other.ap_id - && self.local == other.local - && self.path == other.path - && self.path == other.path - && self.distinguished == other.distinguished - && self.language_id == other.language_id - && self.score == other.score - && self.upvotes == other.upvotes - && self.downvotes == other.downvotes - && self.child_count == other.child_count - && self.hot_rank == other.hot_rank - && self.controversy_rank == other.controversy_rank - && self.report_count == other.report_count - && self.unresolved_report_count == other.unresolved_report_count - */ - } -} - -impl Eq for Comment {} - #[derive(Debug, Clone, derive_new::new)] #[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] #[cfg_attr(feature = "full", diesel(table_name = comment))] diff --git a/crates/db_schema/src/source/post.rs b/crates/db_schema/src/source/post.rs index c6e407c46d..f466378d45 100644 --- a/crates/db_schema/src/source/post.rs +++ b/crates/db_schema/src/source/post.rs @@ -11,7 +11,7 @@ use { }; #[skip_serializing_none] -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[cfg_attr( feature = "full", derive(Queryable, Selectable, Identifiable, TS, CursorKeysModule) @@ -96,17 +96,6 @@ pub struct Post { pub unresolved_report_count: i16, } -impl PartialEq for Post { - fn eq(&self, other: &Self) -> bool { - todo!() - /* - self.id == other.id && self.name == other.name && self.url == other.url && self.body == other.body && self.creator_id == other.creator_id && self.community_id == other.community_id && self.removed == other.removed && self.locked == other.locked && self.published == other.published && self.updated == other.updated && self.deleted == other.deleted && self.nsfw == other.nsfw && self.embed_title == other.embed_title && self.embed_description == other.embed_description && self.thumbnail_url == other.thumbnail_url && self.ap_id == other.ap_id && self.local == other.local && self.embed_video_url == other.embed_video_url && self.language_id == other.language_id && self.featured_community == other.featured_community && self.featured_local == other.featured_local && self.url_content_type == other.url_content_type && self.alt_text == other.alt_text && self.scheduled_publish_time == other.scheduled_publish_time && self.comments == other.comments && self.score == other.score && self.upvotes == other.upvotes && self.downvotes == other.downvotes && self.newest_comment_time_necro == other.newest_comment_time_necro && self.newest_comment_time == other.newest_comment_time && self.hot_rank == other.hot_rank && self.hot_rank_active == other.hot_rank_active && self.controversy_rank == other.controversy_rank && self.instance_id == other.instance_id && self.scaled_rank == other.scaled_rank && self.report_count == other.report_count && self.unresolved_report_count == other.unresolved_report_count - */ - } -} - -impl Eq for Post {} - #[derive(Debug, Clone, derive_new::new)] #[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] #[cfg_attr(feature = "full", diesel(table_name = post))] From a88845e776ec2f512c8ea6f1ba4d4f1ffd87937c Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 10 Feb 2025 15:15:07 +0100 Subject: [PATCH 13/39] trigger fix --- crates/db_schema/replaceable_schema/triggers.sql | 11 ++++------- crates/db_schema/replaceable_schema/utils.sql | 3 +++ crates/db_schema/src/impls/post.rs | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index 47bc9bf834..ee477127be 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -19,9 +19,6 @@ -- -- -- Create triggers for both post and comments -CREATE FUNCTION r.creator_id_from_post (agg post) - RETURNS int IMMUTABLE PARALLEL SAFE RETURN agg.creator_id; - CREATE PROCEDURE r.post_or_comment (table_name text) LANGUAGE plpgsql AS $a$ @@ -31,7 +28,7 @@ BEGIN CALL r.create_triggers ('thing_actions', $$ BEGIN WITH thing_diff AS ( UPDATE - thing_aggregates AS a + thing AS a SET score = a.score + diff.upvotes - diff.downvotes, upvotes = a.upvotes + diff.upvotes, downvotes = a.downvotes + diff.downvotes, controversy_rank = r.controversy_rank ((a.upvotes + diff.upvotes)::numeric, (a.downvotes + diff.downvotes)::numeric) FROM ( @@ -39,10 +36,10 @@ BEGIN (thing_actions).thing_id, coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score = 1), 0) AS upvotes, coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score != 1), 0) AS downvotes FROM select_old_and_new_rows AS old_and_new_rows WHERE (thing_actions).like_score IS NOT NULL GROUP BY (thing_actions).thing_id) AS diff WHERE - a.thing_id = diff.thing_id + a.id = diff.thing_id AND (diff.upvotes, diff.downvotes) != (0, 0) RETURNING - r.creator_id_from_thing_aggregates (a.*) AS creator_id, diff.upvotes - diff.downvotes AS score) + a.creator_id AS creator_id, diff.upvotes - diff.downvotes AS score) UPDATE person_aggregates AS a SET @@ -947,7 +944,7 @@ BEGIN SET score = NEW.score WHERE - post_id = NEW.post_id; + post_id = NEW.id; RETURN NULL; END $$; diff --git a/crates/db_schema/replaceable_schema/utils.sql b/crates/db_schema/replaceable_schema/utils.sql index a951f1a206..996b8b3afa 100644 --- a/crates/db_schema/replaceable_schema/utils.sql +++ b/crates/db_schema/replaceable_schema/utils.sql @@ -92,6 +92,7 @@ DECLARE CREATE TRIGGER delete_statement AFTER DELETE ON thing REFERENCING OLD TABLE AS select_old_rows FOR EACH STATEMENT + WHEN (pg_trigger_depth( ) = 0 ) EXECUTE FUNCTION r.thing_delete_statement ( ); -- Insert CREATE FUNCTION r.thing_insert_statement ( ) @@ -101,6 +102,7 @@ DECLARE CREATE TRIGGER insert_statement AFTER INSERT ON thing REFERENCING NEW TABLE AS select_new_rows FOR EACH STATEMENT + WHEN (pg_trigger_depth( ) = 0 ) EXECUTE FUNCTION r.thing_insert_statement ( ); -- Update CREATE FUNCTION r.thing_update_statement ( ) @@ -110,6 +112,7 @@ DECLARE CREATE TRIGGER update_statement AFTER UPDATE ON thing REFERENCING OLD TABLE AS select_old_rows NEW TABLE AS select_new_rows FOR EACH STATEMENT + WHEN (pg_trigger_depth( ) = 0 ) EXECUTE FUNCTION r.thing_update_statement ( ); $$; select_old_and_new_rows text := $$ ( diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index a3d46333a0..792e647eda 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -301,7 +301,7 @@ impl Post { )) .get_result::(conn) .await - } + } pub fn local_url(&self, settings: &Settings) -> LemmyResult { let domain = settings.get_protocol_and_hostname(); Ok(Url::parse(&format!("{domain}/post/{}", self.id))?.into()) @@ -575,7 +575,9 @@ mod tests { instance_id: InstanceId(0), newest_comment_time: Utc::now(), newest_comment_time_necro: Utc::now(), - report_count: 0,scaled_rank: 0.0,unresolved_report_count: 0 + report_count: 0, + scaled_rank: 0.0, + unresolved_report_count: 0, }; // Post Like From 1a1c3666a7c1cf1232a5f01dcf8d364e548987b2 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 10 Feb 2025 15:47:31 +0100 Subject: [PATCH 14/39] fix post insert trigger --- .../db_schema/replaceable_schema/triggers.sql | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index ee477127be..8f787b2299 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -474,20 +474,22 @@ CREATE FUNCTION r.post_from_post () LANGUAGE plpgsql AS $$ BEGIN - INSERT INTO post (post_id, published, newest_comment_time, newest_comment_time_necro, community_id, creator_id, instance_id, featured_community, featured_local) - SELECT - new_post.id, - new_post.published, - new_post.published, - new_post.published, - new_post.community_id, - new_post.creator_id, - community.instance_id, - new_post.featured_community, - new_post.featured_local + UPDATE + post + SET + published = new_post.published, + newest_comment_time = new_post.published, + newest_comment_time_necro = new_post.published, + community_id = new_post.community_id, + creator_id = new_post.creator_id, + instance_id = community.instance_id, + featured_community = new_post.featured_community, + featured_local = new_post.featured_local FROM new_post - INNER JOIN community ON community.id = new_post.community_id; + INNER JOIN community ON community.id = new_post.community_id + WHERE + post.id = new_post.id; RETURN NULL; END; $$; @@ -495,6 +497,7 @@ $$; CREATE TRIGGER aggregates AFTER INSERT ON post REFERENCING NEW TABLE AS new_post FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) EXECUTE FUNCTION r.post_from_post (); CREATE FUNCTION r.post_from_post_update () @@ -522,6 +525,7 @@ $$; CREATE TRIGGER aggregates_update AFTER UPDATE ON post REFERENCING OLD TABLE AS old_post NEW TABLE AS new_post FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) EXECUTE FUNCTION r.post_from_post_update (); CREATE FUNCTION r.site_aggregates_from_site () @@ -965,7 +969,7 @@ BEGIN SET score = NEW.score WHERE - comment_id = NEW.comment_id; + comment_id = NEW.id; RETURN NULL; END $$; From af3035b17ec83b1b3791c22de7bed32c38c2c946 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 10 Feb 2025 16:40:34 +0100 Subject: [PATCH 15/39] wip --- crates/db_schema/src/impls/post.rs | 8 +-- .../src/combined/report_combined_view.rs | 10 ++-- crates/db_views/src/comment/comment_view.rs | 27 +++++++-- crates/db_views/src/post/post_view.rs | 55 ++++++++----------- crates/routes/src/utils/scheduled_tasks.rs | 19 +++---- 5 files changed, 62 insertions(+), 57 deletions(-) diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index 792e647eda..7665037429 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -490,7 +490,7 @@ mod tests { }, }, traits::{Crud, Likeable, Saveable}, - utils::{build_db_pool_for_tests, uplete}, + utils::{build_db_pool_for_tests, uplete, RANK_DEFAULT}, }; use chrono::{DateTime, Utc}; use diesel::result::Error; @@ -570,13 +570,13 @@ mod tests { downvotes: 0, upvotes: 0, score: 0, - hot_rank: 0.0, - hot_rank_active: 0.0, + hot_rank: RANK_DEFAULT, + hot_rank_active: RANK_DEFAULT, instance_id: InstanceId(0), newest_comment_time: Utc::now(), newest_comment_time_necro: Utc::now(), report_count: 0, - scaled_rank: 0.0, + scaled_rank: RANK_DEFAULT, unresolved_report_count: 0, }; diff --git a/crates/db_views/src/combined/report_combined_view.rs b/crates/db_views/src/combined/report_combined_view.rs index c6fa3a628a..1c2b4e06e2 100644 --- a/crates/db_views/src/combined/report_combined_view.rs +++ b/crates/db_views/src/combined/report_combined_view.rs @@ -878,14 +878,14 @@ mod tests { PostReportView::read(pool, inserted_jessica_report.id, data.timmy.id).await?; // Make sure the triggers are reading the aggregates correctly. - let agg_1 = PostAggregates::read(pool, data.post.id).await?; - let agg_2 = PostAggregates::read(pool, data.post_2.id).await?; + let agg_1 = Post::read(pool, data.post.id).await?; + let agg_2 = Post::read(pool, data.post_2.id).await?; assert_eq!( read_jessica_report_view.post_report, inserted_jessica_report ); - assert_eq!(read_jessica_report_view.post, data.post_2); + assert_eq!(read_jessica_report_view.post.id, data.post_2.id); assert_eq!(read_jessica_report_view.community.id, data.community.id); assert_eq!(read_jessica_report_view.creator.id, data.jessica.id); assert_eq!(read_jessica_report_view.post_creator.id, data.timmy.id); @@ -939,12 +939,12 @@ mod tests { ); // Make sure the unresolved_post report got decremented in the trigger - let agg_2 = PostAggregates::read(pool, data.post_2.id).await?; + let agg_2 = Post::read(pool, data.post_2.id).await?; assert_eq!(agg_2.report_count, 1); assert_eq!(agg_2.unresolved_report_count, 0); // Make sure the other unresolved report isn't changed - let agg_1 = PostAggregates::read(pool, data.post.id).await?; + let agg_1 = Post::read(pool, data.post.id).await?; assert_eq!(agg_1.report_count, 1); assert_eq!(agg_1.unresolved_report_count, 1); diff --git a/crates/db_views/src/comment/comment_view.rs b/crates/db_views/src/comment/comment_view.rs index fb85101ca0..6afbe5185c 100644 --- a/crates/db_views/src/comment/comment_view.rs +++ b/crates/db_views/src/comment/comment_view.rs @@ -348,7 +348,7 @@ mod tests { use lemmy_db_schema::{ assert_length, impls::actor_language::UNDETERMINED_ID, - newtypes::LanguageId, + newtypes::{CommentId, LanguageId}, source::{ actor_language::LocalUserLanguage, comment::{Comment, CommentInsertForm, CommentLike, CommentLikeForm, CommentUpdateForm}, @@ -387,6 +387,7 @@ mod tests { inserted_comment_0: Comment, inserted_comment_1: Comment, inserted_comment_2: Comment, + inserted_comment_5: Comment, inserted_post: Post, timmy_local_user_view: LocalUserView, inserted_sara_person: Person, @@ -494,7 +495,7 @@ mod tests { inserted_post.id, "Comment 5".into(), ); - let _inserted_comment_5 = + let inserted_comment_5 = Comment::create(pool, &comment_form_5, Some(&inserted_comment_4.path)).await?; let timmy_blocks_sara_form = PersonBlockForm { @@ -532,6 +533,7 @@ mod tests { inserted_comment_0, inserted_comment_1, inserted_comment_2, + inserted_comment_5, inserted_post, timmy_local_user_view, inserted_sara_person, @@ -678,10 +680,10 @@ mod tests { // Make sure it contains the parent, but not the comment from the other tree let child_comments = read_comment_views_child_path .into_iter() - .map(|c| c.comment) - .collect::>(); - assert!(child_comments.contains(&data.inserted_comment_1)); - assert!(!child_comments.contains(&data.inserted_comment_2)); + .map(|c| c.comment.id) + .collect::>(); + assert!(child_comments.contains(&data.inserted_comment_1.id)); + assert!(!child_comments.contains(&data.inserted_comment_2.id)); let read_comment_views_top_max_depth = CommentQuery { post_id: (Some(data.inserted_post.id)), @@ -959,6 +961,19 @@ mod tests { featured_local: false, url_content_type: None, scheduled_publish_time: None, + comments: 6, + score: 0, + upvotes: 0, + downvotes: 0, + newest_comment_time_necro: data.inserted_comment_1.published, + newest_comment_time: data.inserted_comment_5.published, + hot_rank: RANK_DEFAULT, + hot_rank_active: RANK_DEFAULT, + controversy_rank: 0.0, + scaled_rank: RANK_DEFAULT, + instance_id: data.inserted_instance.id, + report_count: 0, + unresolved_report_count: 0, }, community: Community { id: data.inserted_community.id, diff --git a/crates/db_views/src/post/post_view.rs b/crates/db_views/src/post/post_view.rs index ff6c0f1073..ebf5c9988c 100644 --- a/crates/db_views/src/post/post_view.rs +++ b/crates/db_views/src/post/post_view.rs @@ -966,7 +966,7 @@ mod tests { ) .await?; - let expected_post_listing_with_user = expected_post_view(data, pool).await?; + let expected_post_listing_with_user = expected_post_view(data)?; // Should be only one person, IE the bot post, and blocked should be missing assert_eq!( @@ -1017,7 +1017,7 @@ mod tests { let read_post_listing_single_no_person = PostView::read(pool, data.inserted_post.id, None, false).await?; - let expected_post_listing_no_person = expected_post_view(data, pool).await?; + let expected_post_listing_no_person = expected_post_view(data)?; // Should be 2 posts, with the bot post, and the blocked assert_eq!( @@ -1150,10 +1150,10 @@ mod tests { ) .await?; - let mut expected_post_with_upvote = expected_post_view(data, pool).await?; + let mut expected_post_with_upvote = expected_post_view(data)?; expected_post_with_upvote.my_vote = Some(1); - expected_post_with_upvote.counts.score = 1; - expected_post_with_upvote.counts.upvotes = 1; + expected_post_with_upvote.post.score = 1; + expected_post_with_upvote.post.upvotes = 1; assert_eq!(expected_post_with_upvote, post_listing_single_with_person); let local_user_form = LocalUserUpdateForm { @@ -1600,7 +1600,7 @@ mod tests { loop { let post_listings = PostQuery { page_after: page_after.map(|p| PaginationCursorData { - post_aggregates: p, + post: p, post_actions: Default::default(), }), ..options.clone() @@ -1611,7 +1611,7 @@ mod tests { listed_post_ids.extend(post_listings.iter().map(|p| p.post.id)); if let Some(p) = post_listings.into_iter().next_back() { - page_after = Some(p.counts); + page_after = Some(p.post); } else { break; } @@ -1623,7 +1623,7 @@ mod tests { loop { let post_listings = PostQuery { page_after: page_before.map(|p| PaginationCursorData { - post_aggregates: p, + post: p, post_actions: Default::default(), }), page_back: Some(true), @@ -1642,7 +1642,7 @@ mod tests { listed_post_ids_forward.truncate(index); if let Some(p) = post_listings.into_iter().next() { - page_before = Some(p.counts); + page_before = Some(p.post); } else { break; } @@ -1793,13 +1793,12 @@ mod tests { Ok(()) } - async fn expected_post_view(data: &Data, pool: &mut DbPool<'_>) -> LemmyResult { + fn expected_post_view(data: &Data) -> LemmyResult { let (inserted_person, inserted_community, inserted_post) = ( &data.local_user_view.person, &data.inserted_community, &data.inserted_post, ); - let agg = PostAggregates::read(pool, inserted_post.id).await?; Ok(PostView { post: Post { @@ -1827,6 +1826,19 @@ mod tests { featured_local: false, url_content_type: None, scheduled_publish_time: None, + comments: 0, + score: 0, + upvotes: 0, + downvotes: 0, + newest_comment_time_necro: inserted_post.published, + newest_comment_time: inserted_post.published, + hot_rank: RANK_DEFAULT, + hot_rank_active: RANK_DEFAULT, + controversy_rank: 0.0, + scaled_rank: RANK_DEFAULT, + instance_id: data.inserted_instance.id, + report_count: 0, + unresolved_report_count: 0, }, my_vote: None, unread_comments: 0, @@ -1885,27 +1897,6 @@ mod tests { visibility: CommunityVisibility::Public, random_number: inserted_community.random_number, }, - counts: PostAggregates { - post_id: inserted_post.id, - comments: 0, - score: 0, - upvotes: 0, - downvotes: 0, - published: agg.published, - newest_comment_time_necro: inserted_post.published, - newest_comment_time: inserted_post.published, - featured_community: false, - featured_local: false, - hot_rank: RANK_DEFAULT, - hot_rank_active: RANK_DEFAULT, - controversy_rank: 0.0, - scaled_rank: RANK_DEFAULT, - community_id: inserted_post.community_id, - creator_id: inserted_post.creator_id, - instance_id: data.inserted_instance.id, - report_count: 0, - unresolved_report_count: 0, - }, subscribed: SubscribedType::NotSubscribed, read: false, hidden: false, diff --git a/crates/routes/src/utils/scheduled_tasks.rs b/crates/routes/src/utils/scheduled_tasks.rs index 3d50116a4b..1afa636101 100644 --- a/crates/routes/src/utils/scheduled_tasks.rs +++ b/crates/routes/src/utils/scheduled_tasks.rs @@ -207,23 +207,22 @@ async fn process_ranks_in_batches( // Raw `sql_query` is used as a performance optimization - Diesel does not support doing this // in a single query (neither as a CTE, nor using a subquery) let updated_rows = sql_query(format!( - r#"WITH batch AS (SELECT a.{id_column} - FROM {aggregates_table} a + r#"WITH batch AS (SELECT a.id + FROM {table_name} a WHERE a.published > $1 AND ({where_clause}) ORDER BY a.published LIMIT $2 FOR UPDATE SKIP LOCKED) - UPDATE {aggregates_table} a {set_clause} - FROM batch WHERE a.{id_column} = batch.{id_column} RETURNING a.published; + UPDATE {table_name} a {set_clause} + FROM batch WHERE a.id = batch.id RETURNING a.published; "#, - id_column = format_args!("{table_name}_id"), - aggregates_table = format_args!("{table_name}_aggregates"), )) .bind::(previous_batch_last_published) .bind::(update_batch_size) .get_results::(conn) .await .map_err(|e| { + // TODO: need to remove community aggregates to get this working LemmyErrorType::Unknown(format!("Failed to update {} hot_ranks: {}", table_name, e)) })?; @@ -247,19 +246,19 @@ async fn process_post_aggregates_ranks_in_batches(conn: &mut AsyncPgConnection) let mut previous_batch_result = Some(process_start_time); while let Some(previous_batch_last_published) = previous_batch_result { let updated_rows = sql_query( - r#"WITH batch AS (SELECT pa.post_id - FROM post_aggregates pa + r#"WITH batch AS (SELECT pa.id + FROM post pa WHERE pa.published > $1 AND (pa.hot_rank != 0 OR pa.hot_rank_active != 0) ORDER BY pa.published LIMIT $2 FOR UPDATE SKIP LOCKED) - UPDATE post_aggregates pa + UPDATE post pa SET hot_rank = r.hot_rank(pa.score, pa.published), hot_rank_active = r.hot_rank(pa.score, pa.newest_comment_time_necro), scaled_rank = r.scaled_rank(pa.score, pa.published, ca.interactions_month) FROM batch, community_aggregates ca - WHERE pa.post_id = batch.post_id + WHERE pa.id = batch.id AND pa.community_id = ca.community_id RETURNING pa.published; "#, From 41112a9ae6ba09258af217cdd6af6b1faa19032c Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 11 Feb 2025 11:21:23 +0100 Subject: [PATCH 16/39] reorder --- crates/db_schema/src/aggregates/community_aggregates.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/db_schema/src/aggregates/community_aggregates.rs b/crates/db_schema/src/aggregates/community_aggregates.rs index 3ec56d73d6..44bc42b4c9 100644 --- a/crates/db_schema/src/aggregates/community_aggregates.rs +++ b/crates/db_schema/src/aggregates/community_aggregates.rs @@ -167,8 +167,8 @@ mod tests { // Remove a parent post (the comment count should also be 0) Post::delete(pool, inserted_post.id).await?; let after_parent_post_delete = CommunityAggregates::read(pool, inserted_community.id).await?; - assert_eq!(0, after_parent_post_delete.comments); assert_eq!(0, after_parent_post_delete.posts); + assert_eq!(0, after_parent_post_delete.comments); // Remove the 2nd person Person::delete(pool, another_inserted_person.id).await?; From bb30add244c59ac355759a76f610ecd5f316bef9 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 11 Feb 2025 11:33:10 +0100 Subject: [PATCH 17/39] fixes --- crates/db_schema/src/impls/comment.rs | 11 +++++------ crates/db_schema/src/impls/post.rs | 14 ++++++-------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 16301eea7e..bb502ba0d4 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -243,7 +243,7 @@ mod tests { post::{Post, PostInsertForm}, }, traits::{Crud, Likeable, Saveable}, - utils::{build_db_pool_for_tests, uplete}, + utils::{build_db_pool_for_tests, uplete, RANK_DEFAULT}, }; use diesel_ltree::Ltree; use lemmy_utils::error::LemmyResult; @@ -303,12 +303,12 @@ mod tests { distinguished: false, local: true, language_id: LanguageId::default(), - child_count: 0, + child_count: 1, controversy_rank: 0.0, downvotes: 0, - upvotes: 0, - score: 0, - hot_rank: 0.0, + upvotes: 1, + score: 1, + hot_rank: RANK_DEFAULT, report_count: 0, unresolved_report_count: 0, }; @@ -365,7 +365,6 @@ mod tests { Instance::delete(pool, inserted_instance.id).await?; assert_eq!(expected_comment, read_comment); - assert_eq!(expected_comment, inserted_comment); assert_eq!(expected_comment, updated_comment); assert_eq!(expected_comment_like, inserted_comment_like); assert_eq!(expected_comment_saved, inserted_comment_saved); diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index 7665037429..0a7f5a2b03 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -471,7 +471,6 @@ impl PostActionsCursor { #[cfg(test)] mod tests { use crate::{ - newtypes::InstanceId, source::{ comment::{Comment, CommentInsertForm, CommentUpdateForm}, community::{Community, CommunityInsertForm}, @@ -492,7 +491,7 @@ mod tests { traits::{Crud, Likeable, Saveable}, utils::{build_db_pool_for_tests, uplete, RANK_DEFAULT}, }; - use chrono::{DateTime, Utc}; + use chrono::DateTime; use diesel::result::Error; use lemmy_utils::error::LemmyResult; use pretty_assertions::assert_eq; @@ -568,13 +567,13 @@ mod tests { comments: 0, controversy_rank: 0.0, downvotes: 0, - upvotes: 0, - score: 0, + upvotes: 1, + score: 1, hot_rank: RANK_DEFAULT, hot_rank_active: RANK_DEFAULT, - instance_id: InstanceId(0), - newest_comment_time: Utc::now(), - newest_comment_time_necro: Utc::now(), + instance_id: inserted_instance.id, + newest_comment_time: inserted_post.published, + newest_comment_time_necro: inserted_post.published, report_count: 0, scaled_rank: RANK_DEFAULT, unresolved_report_count: 0, @@ -644,7 +643,6 @@ mod tests { Instance::delete(pool, inserted_instance.id).await?; assert_eq!(expected_post, read_post); - assert_eq!(expected_post, inserted_post); assert_eq!(expected_post, updated_post); assert_eq!(expected_post_like, inserted_post_like); assert_eq!(expected_post_saved, inserted_post_saved); From 794240c94748da0f5d3170532354d18575a2e30b Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 11:40:55 +0100 Subject: [PATCH 18/39] community aggregate migration --- .../down.sql | 62 +++++++++++++++++++ .../up.sql | 46 ++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql index e752289d78..87af16be49 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql @@ -177,3 +177,65 @@ CREATE INDEX idx_post_aggregates_published ON post_aggregates USING btree (publi CREATE INDEX idx_post_aggregates_published_asc ON post_aggregates USING btree (reverse_timestamp_sort (published) DESC); +-- move community_aggregates back into separate table +CREATE TABLE community_aggregates ( + community_id int PRIMARY KEY NOT NULL REFERENCES COMMunity ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + subscribers bigint NOT NULL DEFAULT 0, + posts bigint NOT NULL DEFAULT 0, + comments bigint NOT NULL DEFAULT 0, + published timestamp with time zone DEFAULT now() NOT NULL, + users_active_day bigint NOT NULL DEFAULT 0, + users_active_week bigint NOT NULL DEFAULT 0, + users_active_month bigint NOT NULL DEFAULT 0, + users_active_half_year bigint NOT NULL DEFAULT 0, + hot_rank double precision NOT NULL DEFAULT 0.0001, + subscribers_local bigint NOT NULL DEFAULT 0, + report_count smallint NOT NULL DEFAULT 0, + unresolved_report_count smallint NOT NULL DEFAULT 0, + interactions_month bigint NOT NULL DEFAULT 0 +); + +INSERT INTO community_aggregates +SELECT + id AS comment_id, + subscribers, + posts, + comments, + published, + users_active_day, + users_active_week, + users_active_month, + users_active_half_year, + hot_rank, + subscribers_local, + report_count, + unresolved_report_count, + interactions_month +FROM + community; + +ALTER TABLE community + DROP COLUMN subscribers, + DROP COLUMN posts, + DROP COLUMN comments, + DROP COLUMN users_active_day, + DROP COLUMN users_active_week, + DROP COLUMN users_active_month, + DROP COLUMN users_active_half_year, + DROP COLUMN hot_rank, + DROP COLUMN subscribers_local, + DROP COLUMN report_count, + DROP COLUMN unresolved_report_count, + DROP COLUMN interactions_month; + +CREATE INDEX idx_community_aggregates_hot ON public.community_aggregates USING btree (hot_rank DESC); + +CREATE INDEX idx_community_aggregates_nonzero_hotrank ON public.community_aggregates USING btree (published) +WHERE (hot_rank <> (0)::double precision); + +CREATE INDEX idx_community_aggregates_published ON public.community_aggregates USING btree (published DESC); + +CREATE INDEX idx_community_aggregates_subscribers ON public.community_aggregates USING btree (subscribers DESC); + +CREATE INDEX idx_community_aggregates_users_active_month ON public.community_aggregates USING btree (users_active_month DESC); + diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql index 6140f27350..6f535f9dbc 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -140,3 +140,49 @@ WHERE ((hot_rank <> (0)::double precision) OR (hot_rank_active <> (0)::double pr --CREATE INDEX idx_post_aggregates_published on post USING btree (published DESC); --CREATE INDEX idx_post_aggregates_published_asc on post USING btree (reverse_timestamp_sort (published) DESC); +-- merge community_aggregates into community table +ALTER TABLE community + ADD COLUMN subscribers bigint NOT NULL DEFAULT 0, + ADD COLUMN posts bigint NOT NULL DEFAULT 0, + ADD COLUMN comments bigint NOT NULL DEFAULT 0, + ADD COLUMN users_active_day bigint NOT NULL DEFAULT 0, + ADD COLUMN users_active_week bigint NOT NULL DEFAULT 0, + ADD COLUMN users_active_month bigint NOT NULL DEFAULT 0, + ADD COLUMN users_active_half_year bigint NOT NULL DEFAULT 0, + ADD COLUMN hot_rank double precision NOT NULL DEFAULT 0.0001, + ADD COLUMN subscribers_local bigint NOT NULL DEFAULT 0, + ADD COLUMN report_count smallint NOT NULL DEFAULT 0, + ADD COLUMN unresolved_report_count smallint NOT NULL DEFAULT 0, + ADD COLUMN interactions_month bigint NOT NULL DEFAULT 0; + +UPDATE + community +SET + subscribers = ca.subscribers, + posts = ca.posts, + comments = ca.comments, + users_active_day = ca.users_active_day, + users_active_week = ca.users_active_week, + users_active_month = ca.users_active_month, + users_active_half_year = ca.users_active_half_year, + hot_rank = ca.hot_rank, + subscribers_local = ca.subscribers_local, + report_count = ca.report_count, + unresolved_report_count = ca.unresolved_report_count, + interactions_month = ca.interactions_month +FROM + community_aggregates AS ca +WHERE + community.id = ca.community_id; + +DROP TABLE community_aggregates; + +CREATE INDEX idx_community_aggregates_hot ON public.community USING btree (hot_rank DESC); + +CREATE INDEX idx_community_aggregates_nonzero_hotrank ON public.community USING btree (published) +WHERE (hot_rank <> (0)::double precision); + +CREATE INDEX idx_community_aggregates_subscribers ON public.community USING btree (subscribers DESC); + +CREATE INDEX idx_community_aggregates_users_active_month ON public.community USING btree (users_active_month DESC); + From 082cdfdc993e3feb8191b875873c02f5e32a3b6e Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 12:18:16 +0100 Subject: [PATCH 19/39] update code --- .../src/collections/community_follower.rs | 10 +- .../src/aggregates/community_aggregates.rs | 197 ------------------ crates/db_schema/src/aggregates/mod.rs | 2 - crates/db_schema/src/aggregates/structs.rs | 41 +--- crates/db_schema/src/impls/community.rs | 167 ++++++++++++++- crates/db_schema/src/impls/post.rs | 8 +- crates/db_schema/src/schema.rs | 33 ++- crates/db_schema/src/source/community.rs | 21 +- .../src/combined/report_combined_view.rs | 11 +- .../src/combined/search_combined_view.rs | 9 +- crates/db_views/src/comment/comment_view.rs | 12 ++ .../db_views/src/community/community_view.rs | 28 ++- crates/db_views/src/post/post_view.rs | 23 +- crates/db_views/src/structs.rs | 7 +- 14 files changed, 249 insertions(+), 320 deletions(-) delete mode 100644 crates/db_schema/src/aggregates/community_aggregates.rs diff --git a/crates/apub/src/collections/community_follower.rs b/crates/apub/src/collections/community_follower.rs index a95016c94a..5862a359ed 100644 --- a/crates/apub/src/collections/community_follower.rs +++ b/crates/apub/src/collections/community_follower.rs @@ -9,7 +9,7 @@ use activitypub_federation::{ traits::Collection, }; use lemmy_api_common::{context::LemmyContext, utils::generate_followers_url}; -use lemmy_db_schema::aggregates::structs::CommunityAggregates; +use lemmy_db_schema::source::community::Community; use lemmy_db_views::structs::CommunityFollowerView; use lemmy_utils::error::LemmyError; use url::Url; @@ -54,12 +54,8 @@ impl Collection for ApubCommunityFollower { community: &Self::Owner, context: &Data, ) -> Result { - CommunityAggregates::update_federated_followers( - &mut context.pool(), - community.id, - json.total_items, - ) - .await?; + Community::update_federated_followers(&mut context.pool(), community.id, json.total_items) + .await?; Ok(ApubCommunityFollower(())) } diff --git a/crates/db_schema/src/aggregates/community_aggregates.rs b/crates/db_schema/src/aggregates/community_aggregates.rs deleted file mode 100644 index 44bc42b4c9..0000000000 --- a/crates/db_schema/src/aggregates/community_aggregates.rs +++ /dev/null @@ -1,197 +0,0 @@ -use crate::{ - aggregates::structs::CommunityAggregates, - newtypes::CommunityId, - schema::{community_aggregates, community_aggregates::subscribers}, - utils::{get_conn, DbPool}, -}; -use diesel::{result::Error, ExpressionMethods, QueryDsl}; -use diesel_async::RunQueryDsl; - -impl CommunityAggregates { - pub async fn read(pool: &mut DbPool<'_>, for_community_id: CommunityId) -> Result { - let conn = &mut get_conn(pool).await?; - community_aggregates::table - .find(for_community_id) - .first(conn) - .await - } - - pub async fn update_federated_followers( - pool: &mut DbPool<'_>, - for_community_id: CommunityId, - new_subscribers: i32, - ) -> Result { - let conn = &mut get_conn(pool).await?; - let new_subscribers: i64 = new_subscribers.into(); - diesel::update(community_aggregates::table.find(for_community_id)) - .set(subscribers.eq(new_subscribers)) - .get_result(conn) - .await - } -} - -#[cfg(test)] -mod tests { - - use crate::{ - aggregates::community_aggregates::CommunityAggregates, - source::{ - comment::{Comment, CommentInsertForm}, - community::{ - Community, - CommunityFollower, - CommunityFollowerForm, - CommunityFollowerState, - CommunityInsertForm, - }, - instance::Instance, - person::{Person, PersonInsertForm}, - post::{Post, PostInsertForm}, - }, - traits::{Crud, Followable}, - utils::build_db_pool_for_tests, - }; - use diesel::result::Error; - use pretty_assertions::assert_eq; - use serial_test::serial; - - #[tokio::test] - #[serial] - async fn test_crud() -> Result<(), Error> { - let pool = &build_db_pool_for_tests(); - let pool = &mut pool.into(); - - let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; - - let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_community_agg"); - - let inserted_person = Person::create(pool, &new_person).await?; - - let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_community_agg"); - - let another_inserted_person = Person::create(pool, &another_person).await?; - - let new_community = CommunityInsertForm::new( - inserted_instance.id, - "TIL_community_agg".into(), - "nada".to_owned(), - "pubkey".to_string(), - ); - let inserted_community = Community::create(pool, &new_community).await?; - - let another_community = CommunityInsertForm::new( - inserted_instance.id, - "TIL_community_agg_2".into(), - "nada".to_owned(), - "pubkey".to_string(), - ); - let another_inserted_community = Community::create(pool, &another_community).await?; - - let first_person_follow = CommunityFollowerForm { - community_id: inserted_community.id, - person_id: inserted_person.id, - state: Some(CommunityFollowerState::Accepted), - approver_id: None, - }; - - CommunityFollower::follow(pool, &first_person_follow).await?; - - let second_person_follow = CommunityFollowerForm { - community_id: inserted_community.id, - person_id: another_inserted_person.id, - state: Some(CommunityFollowerState::Accepted), - approver_id: None, - }; - - CommunityFollower::follow(pool, &second_person_follow).await?; - - let another_community_follow = CommunityFollowerForm { - community_id: another_inserted_community.id, - person_id: inserted_person.id, - state: Some(CommunityFollowerState::Accepted), - approver_id: None, - }; - - CommunityFollower::follow(pool, &another_community_follow).await?; - - let new_post = PostInsertForm::new( - "A test post".into(), - inserted_person.id, - inserted_community.id, - ); - let inserted_post = Post::create(pool, &new_post).await?; - - let comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - let inserted_comment = Comment::create(pool, &comment_form, None).await?; - - let child_comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - let _inserted_child_comment = - Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; - - let community_aggregates_before_delete = - CommunityAggregates::read(pool, inserted_community.id).await?; - - assert_eq!(2, community_aggregates_before_delete.subscribers); - assert_eq!(2, community_aggregates_before_delete.subscribers_local); - assert_eq!(1, community_aggregates_before_delete.posts); - assert_eq!(2, community_aggregates_before_delete.comments); - - // Test the other community - let another_community_aggs = - CommunityAggregates::read(pool, another_inserted_community.id).await?; - assert_eq!(1, another_community_aggs.subscribers); - assert_eq!(1, another_community_aggs.subscribers_local); - assert_eq!(0, another_community_aggs.posts); - assert_eq!(0, another_community_aggs.comments); - - // Unfollow test - CommunityFollower::unfollow(pool, &second_person_follow).await?; - let after_unfollow = CommunityAggregates::read(pool, inserted_community.id).await?; - assert_eq!(1, after_unfollow.subscribers); - assert_eq!(1, after_unfollow.subscribers_local); - - // Follow again just for the later tests - CommunityFollower::follow(pool, &second_person_follow).await?; - let after_follow_again = CommunityAggregates::read(pool, inserted_community.id).await?; - assert_eq!(2, after_follow_again.subscribers); - assert_eq!(2, after_follow_again.subscribers_local); - - // Remove a parent post (the comment count should also be 0) - Post::delete(pool, inserted_post.id).await?; - let after_parent_post_delete = CommunityAggregates::read(pool, inserted_community.id).await?; - assert_eq!(0, after_parent_post_delete.posts); - assert_eq!(0, after_parent_post_delete.comments); - - // Remove the 2nd person - Person::delete(pool, another_inserted_person.id).await?; - let after_person_delete = CommunityAggregates::read(pool, inserted_community.id).await?; - assert_eq!(1, after_person_delete.subscribers); - assert_eq!(1, after_person_delete.subscribers_local); - - // This should delete all the associated rows, and fire triggers - let person_num_deleted = Person::delete(pool, inserted_person.id).await?; - assert_eq!(1, person_num_deleted); - - // Delete the community - let community_num_deleted = Community::delete(pool, inserted_community.id).await?; - assert_eq!(1, community_num_deleted); - - let another_community_num_deleted = - Community::delete(pool, another_inserted_community.id).await?; - assert_eq!(1, another_community_num_deleted); - - // Should be none found, since the creator was deleted - let after_delete = CommunityAggregates::read(pool, inserted_community.id).await; - assert!(after_delete.is_err()); - - Ok(()) - } -} diff --git a/crates/db_schema/src/aggregates/mod.rs b/crates/db_schema/src/aggregates/mod.rs index 8e72aace7d..5c7adfcb8b 100644 --- a/crates/db_schema/src/aggregates/mod.rs +++ b/crates/db_schema/src/aggregates/mod.rs @@ -1,6 +1,4 @@ #[cfg(feature = "full")] -pub mod community_aggregates; -#[cfg(feature = "full")] pub mod person_aggregates; #[cfg(feature = "full")] pub mod person_post_aggregates; diff --git a/crates/db_schema/src/aggregates/structs.rs b/crates/db_schema/src/aggregates/structs.rs index 20e936b584..67b31c526f 100644 --- a/crates/db_schema/src/aggregates/structs.rs +++ b/crates/db_schema/src/aggregates/structs.rs @@ -1,50 +1,13 @@ -use crate::newtypes::{CommunityId, PersonId, PostId, SiteId}; +use crate::newtypes::{PersonId, PostId, SiteId}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; #[cfg(feature = "full")] use { - crate::schema::{community_aggregates, person_aggregates, post_actions, site_aggregates}, + crate::schema::{person_aggregates, post_actions, site_aggregates}, diesel::{dsl, expression_methods::NullableExpressionMethods}, ts_rs::TS, }; -#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] -#[cfg_attr( - feature = "full", - derive(Queryable, Selectable, Associations, Identifiable, TS) -)] -#[cfg_attr(feature = "full", diesel(table_name = community_aggregates))] -#[cfg_attr( - feature = "full", - diesel(belongs_to(crate::source::community::Community)) -)] -#[cfg_attr(feature = "full", diesel(primary_key(community_id)))] -#[cfg_attr(feature = "full", ts(export))] -/// Aggregate data for a community. -pub struct CommunityAggregates { - pub community_id: CommunityId, - pub subscribers: i64, - pub posts: i64, - pub comments: i64, - pub published: DateTime, - /// The number of users with any activity in the last day. - pub users_active_day: i64, - /// The number of users with any activity in the last week. - pub users_active_week: i64, - /// The number of users with any activity in the last month. - pub users_active_month: i64, - /// The number of users with any activity in the last year. - pub users_active_half_year: i64, - #[serde(skip)] - pub hot_rank: f64, - pub subscribers_local: i64, - pub report_count: i16, - pub unresolved_report_count: i16, - /// Number of any interactions over the last month. - #[serde(skip)] - pub interactions_month: i64, -} - #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone, Default)] #[cfg_attr( feature = "full", diff --git a/crates/db_schema/src/impls/community.rs b/crates/db_schema/src/impls/community.rs index 12578d5991..6fa40abee4 100644 --- a/crates/db_schema/src/impls/community.rs +++ b/crates/db_schema/src/impls/community.rs @@ -282,6 +282,19 @@ impl Community { let domain = settings.get_protocol_and_hostname(); Ok(Url::parse(&format!("{domain}/c/{name}"))?.into()) } + + pub async fn update_federated_followers( + pool: &mut DbPool<'_>, + for_community_id: CommunityId, + new_subscribers: i32, + ) -> Result { + let conn = &mut get_conn(pool).await?; + let new_subscribers: i64 = new_subscribers.into(); + diesel::update(community::table.find(for_community_id)) + .set(community::dsl::subscribers.eq(new_subscribers)) + .get_result(conn) + .await + } } impl CommunityModerator { @@ -556,6 +569,7 @@ impl ApubActor for Community { mod tests { use crate::{ source::{ + comment::{Comment, CommentInsertForm}, community::{ Community, CommunityFollower, @@ -571,9 +585,10 @@ mod tests { instance::Instance, local_user::LocalUser, person::{Person, PersonInsertForm}, + post::{Post, PostInsertForm}, }, traits::{Bannable, Crud, Followable, Joinable}, - utils::{build_db_pool_for_tests, uplete}, + utils::{build_db_pool_for_tests, uplete, RANK_DEFAULT}, CommunityVisibility, }; use lemmy_utils::error::LemmyResult; @@ -629,6 +644,18 @@ mod tests { instance_id: inserted_instance.id, visibility: CommunityVisibility::Public, random_number: inserted_community.random_number, + subscribers: 0, + posts: 0, + comments: 0, + users_active_day: 0, + users_active_week: 0, + users_active_month: 0, + users_active_half_year: 0, + hot_rank: RANK_DEFAULT, + subscribers_local: 0, + report_count: 0, + unresolved_report_count: 0, + interactions_month: 0, }; let community_follower_form = CommunityFollowerForm { @@ -749,4 +776,142 @@ mod tests { Ok(()) } + + #[tokio::test] + #[serial] + async fn test_aggregates() -> LemmyResult<()> { + let pool = &build_db_pool_for_tests(); + let pool = &mut pool.into(); + + let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; + + let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_community_agg"); + + let inserted_person = Person::create(pool, &new_person).await?; + + let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_community_agg"); + + let another_inserted_person = Person::create(pool, &another_person).await?; + + let new_community = CommunityInsertForm::new( + inserted_instance.id, + "TIL_community_agg".into(), + "nada".to_owned(), + "pubkey".to_string(), + ); + let inserted_community = Community::create(pool, &new_community).await?; + + let another_community = CommunityInsertForm::new( + inserted_instance.id, + "TIL_community_agg_2".into(), + "nada".to_owned(), + "pubkey".to_string(), + ); + let another_inserted_community = Community::create(pool, &another_community).await?; + + let first_person_follow = CommunityFollowerForm { + community_id: inserted_community.id, + person_id: inserted_person.id, + state: Some(CommunityFollowerState::Accepted), + approver_id: None, + }; + + CommunityFollower::follow(pool, &first_person_follow).await?; + + let second_person_follow = CommunityFollowerForm { + community_id: inserted_community.id, + person_id: another_inserted_person.id, + state: Some(CommunityFollowerState::Accepted), + approver_id: None, + }; + + CommunityFollower::follow(pool, &second_person_follow).await?; + + let another_community_follow = CommunityFollowerForm { + community_id: another_inserted_community.id, + person_id: inserted_person.id, + state: Some(CommunityFollowerState::Accepted), + approver_id: None, + }; + + CommunityFollower::follow(pool, &another_community_follow).await?; + + let new_post = PostInsertForm::new( + "A test post".into(), + inserted_person.id, + inserted_community.id, + ); + let inserted_post = Post::create(pool, &new_post).await?; + + let comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + let inserted_comment = Comment::create(pool, &comment_form, None).await?; + + let child_comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + let _inserted_child_comment = + Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; + + let community_aggregates_before_delete = Community::read(pool, inserted_community.id).await?; + + assert_eq!(2, community_aggregates_before_delete.subscribers); + assert_eq!(2, community_aggregates_before_delete.subscribers_local); + assert_eq!(1, community_aggregates_before_delete.posts); + assert_eq!(2, community_aggregates_before_delete.comments); + + // Test the other community + let another_community_aggs = Community::read(pool, another_inserted_community.id).await?; + assert_eq!(1, another_community_aggs.subscribers); + assert_eq!(1, another_community_aggs.subscribers_local); + assert_eq!(0, another_community_aggs.posts); + assert_eq!(0, another_community_aggs.comments); + + // Unfollow test + CommunityFollower::unfollow(pool, &second_person_follow).await?; + let after_unfollow = Community::read(pool, inserted_community.id).await?; + assert_eq!(1, after_unfollow.subscribers); + assert_eq!(1, after_unfollow.subscribers_local); + + // Follow again just for the later tests + CommunityFollower::follow(pool, &second_person_follow).await?; + let after_follow_again = Community::read(pool, inserted_community.id).await?; + assert_eq!(2, after_follow_again.subscribers); + assert_eq!(2, after_follow_again.subscribers_local); + + // Remove a parent post (the comment count should also be 0) + Post::delete(pool, inserted_post.id).await?; + let after_parent_post_delete = Community::read(pool, inserted_community.id).await?; + assert_eq!(0, after_parent_post_delete.posts); + assert_eq!(0, after_parent_post_delete.comments); + + // Remove the 2nd person + Person::delete(pool, another_inserted_person.id).await?; + let after_person_delete = Community::read(pool, inserted_community.id).await?; + assert_eq!(1, after_person_delete.subscribers); + assert_eq!(1, after_person_delete.subscribers_local); + + // This should delete all the associated rows, and fire triggers + let person_num_deleted = Person::delete(pool, inserted_person.id).await?; + assert_eq!(1, person_num_deleted); + + // Delete the community + let community_num_deleted = Community::delete(pool, inserted_community.id).await?; + assert_eq!(1, community_num_deleted); + + let another_community_num_deleted = + Community::delete(pool, another_inserted_community.id).await?; + assert_eq!(1, another_community_num_deleted); + + // Should be none found, since the creator was deleted + let after_delete = Community::read(pool, inserted_community.id).await; + assert!(after_delete.is_err()); + + Ok(()) + } } diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index 0a7f5a2b03..d2fceff7f5 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -1,7 +1,7 @@ use crate::{ diesel::{BoolExpressionMethods, NullableExpressionMethods, OptionalExtension}, newtypes::{CommunityId, DbUrl, PersonId, PostId}, - schema::{community, community_aggregates, person, post, post_actions}, + schema::{community, person, post, post_actions}, source::post::{ Post, PostActionsCursor, @@ -282,9 +282,9 @@ impl Post { // https://github.com/diesel-rs/diesel/issues/1478 // Just select the metrics we need manually, for now, since its a single post anyway - let interactions_month = community_aggregates::table - .select(community_aggregates::interactions_month) - .inner_join(post::table.on(community_aggregates::community_id.eq(post::community_id))) + let interactions_month = community::table + .select(community::interactions_month) + .inner_join(post::table.on(community::id.eq(post::community_id))) .filter(post::id.eq(post_id)) .first::(conn) .await?; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index d6f15cbbe2..c1bab4e794 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -214,6 +214,18 @@ diesel::table! { #[max_length = 150] description -> Nullable, random_number -> Int2, + subscribers -> Int8, + posts -> Int8, + comments -> Int8, + users_active_day -> Int8, + users_active_week -> Int8, + users_active_month -> Int8, + users_active_half_year -> Int8, + hot_rank -> Float8, + subscribers_local -> Int8, + report_count -> Int2, + unresolved_report_count -> Int2, + interactions_month -> Int8, } } @@ -234,25 +246,6 @@ diesel::table! { } } -diesel::table! { - community_aggregates (community_id) { - community_id -> Int4, - subscribers -> Int8, - posts -> Int8, - comments -> Int8, - published -> Timestamptz, - users_active_day -> Int8, - users_active_week -> Int8, - users_active_month -> Int8, - users_active_half_year -> Int8, - hot_rank -> Float8, - subscribers_local -> Int8, - report_count -> Int2, - unresolved_report_count -> Int2, - interactions_month -> Int8, - } -} - diesel::table! { community_language (community_id, language_id) { community_id -> Int4, @@ -1106,7 +1099,6 @@ diesel::joinable!(comment_reply -> person (recipient_id)); diesel::joinable!(comment_report -> comment (comment_id)); diesel::joinable!(community -> instance (instance_id)); diesel::joinable!(community_actions -> community (community_id)); -diesel::joinable!(community_aggregates -> community (community_id)); diesel::joinable!(community_language -> community (community_id)); diesel::joinable!(community_language -> language (language_id)); diesel::joinable!(community_report -> community (community_id)); @@ -1215,7 +1207,6 @@ diesel::allow_tables_to_appear_in_same_query!( comment_report, community, community_actions, - community_aggregates, community_language, community_report, custom_emoji, diff --git a/crates/db_schema/src/source/community.rs b/crates/db_schema/src/source/community.rs index f5dbd6915c..33f62edce1 100644 --- a/crates/db_schema/src/source/community.rs +++ b/crates/db_schema/src/source/community.rs @@ -16,7 +16,7 @@ use strum::{Display, EnumString}; use ts_rs::TS; #[skip_serializing_none] -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)] #[cfg_attr(feature = "full", derive(Queryable, Selectable, Identifiable, TS))] #[cfg_attr(feature = "full", diesel(table_name = community))] #[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))] @@ -78,6 +78,25 @@ pub struct Community { pub description: Option, #[serde(skip)] pub random_number: i16, + pub subscribers: i64, + pub posts: i64, + pub comments: i64, + /// The number of users with any activity in the last day. + pub users_active_day: i64, + /// The number of users with any activity in the last week. + pub users_active_week: i64, + /// The number of users with any activity in the last month. + pub users_active_month: i64, + /// The number of users with any activity in the last year. + pub users_active_half_year: i64, + #[serde(skip)] + pub hot_rank: f64, + pub subscribers_local: i64, + pub report_count: i16, + pub unresolved_report_count: i16, + /// Number of any interactions over the last month. + #[serde(skip)] + pub interactions_month: i64, } #[derive(Debug, Clone, derive_new::new)] diff --git a/crates/db_views/src/combined/report_combined_view.rs b/crates/db_views/src/combined/report_combined_view.rs index bd9665433d..3a40364da8 100644 --- a/crates/db_views/src/combined/report_combined_view.rs +++ b/crates/db_views/src/combined/report_combined_view.rs @@ -30,7 +30,6 @@ use lemmy_db_schema::{ comment_report, community, community_actions, - community_aggregates, community_report, local_user, person, @@ -136,9 +135,6 @@ impl ReportCombinedViewInternal { .and(comment_actions::person_id.eq(my_person_id)), ); - let community_aggregates_join = community_aggregates::table - .on(community_report::community_id.eq(community_aggregates::community_id)); - report_combined::table .left_join(post_report::table) .left_join(comment_report::table) @@ -156,7 +152,6 @@ impl ReportCombinedViewInternal { .left_join(community_actions_join) .left_join(post_actions_join) .left_join(person_actions_join) - .left_join(community_aggregates_join) .left_join(comment_actions_join) } @@ -282,7 +277,6 @@ impl ReportCombinedQuery { private_message::all_columns.nullable(), // Community-specific community_report::all_columns.nullable(), - community_aggregates::all_columns.nullable(), // Shared person::all_columns, aliases::person1.fields(person::all_columns.nullable()), @@ -460,14 +454,11 @@ impl InternalToCombinedView for ReportCombinedViewInternal { resolver: v.resolver, }, )) - } else if let (Some(community), Some(community_report), Some(counts)) = - (v.community, v.community_report, v.community_counts) - { + } else if let (Some(community), Some(community_report)) = (v.community, v.community_report) { Some(ReportCombinedView::Community(CommunityReportView { community_report, community, creator: v.report_creator, - counts, subscribed: v.subscribed, resolver: v.resolver, })) diff --git a/crates/db_views/src/combined/search_combined_view.rs b/crates/db_views/src/combined/search_combined_view.rs index d574f2a1d5..10dba8ea46 100644 --- a/crates/db_views/src/combined/search_combined_view.rs +++ b/crates/db_views/src/combined/search_combined_view.rs @@ -30,7 +30,6 @@ use lemmy_db_schema::{ comment_actions, community, community_actions, - community_aggregates, image_details, local_user, person, @@ -147,9 +146,6 @@ impl SearchCombinedViewInternal { .and(comment_actions::person_id.nullable().eq(my_person_id)), ); - let community_aggregates_join = community_aggregates::table - .on(search_combined::community_id.eq(community_aggregates::community_id.nullable())); - let image_details_join = image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())); @@ -168,7 +164,6 @@ impl SearchCombinedViewInternal { .left_join(post_actions_join) .left_join(person_actions_join) .left_join(person_aggregates_join) - .left_join(community_aggregates_join) .left_join(comment_actions_join) .left_join(image_details_join) } @@ -268,7 +263,6 @@ impl SearchCombinedQuery { comment_actions::like_score.nullable(), // Community-specific community::all_columns.nullable(), - community_aggregates::all_columns.nullable(), community_actions::blocked.nullable().is_not_null(), community_follower_select_subscribed_type(), // Person @@ -478,10 +472,9 @@ impl InternalToCombinedView for SearchCombinedViewInternal { tags: v.post_tags, can_mod: v.can_mod, })) - } else if let (Some(community), Some(counts)) = (v.community, v.community_counts) { + } else if let Some(community) = v.community { Some(SearchCombinedView::Community(CommunityView { community, - counts, subscribed: v.subscribed, blocked: v.community_blocked, banned_from_community: v.banned_from_community, diff --git a/crates/db_views/src/comment/comment_view.rs b/crates/db_views/src/comment/comment_view.rs index 77afb2d6d1..67effc1f5c 100644 --- a/crates/db_views/src/comment/comment_view.rs +++ b/crates/db_views/src/comment/comment_view.rs @@ -989,6 +989,18 @@ mod tests { featured_url: data.inserted_community.featured_url.clone(), visibility: CommunityVisibility::Public, random_number: data.inserted_community.random_number, + subscribers: 0, + posts: 0, + comments: 0, + users_active_day: 0, + users_active_week: 0, + users_active_month: 0, + users_active_half_year: 0, + hot_rank: RANK_DEFAULT, + subscribers_local: 0, + report_count: 0, + unresolved_report_count: 0, + interactions_month: 0, }, } } diff --git a/crates/db_views/src/community/community_view.rs b/crates/db_views/src/community/community_view.rs index b5061e927d..e7245acd75 100644 --- a/crates/db_views/src/community/community_view.rs +++ b/crates/db_views/src/community/community_view.rs @@ -12,7 +12,7 @@ use diesel_async::RunQueryDsl; use lemmy_db_schema::{ impls::local_user::LocalUserOptionHelper, newtypes::{CommunityId, PersonId}, - schema::{community, community_actions, community_aggregates, instance_actions, local_user}, + schema::{community, community_actions, instance_actions, local_user}, source::{ community::{Community, CommunityFollowerState}, local_user::LocalUser, @@ -41,7 +41,6 @@ impl CommunityView { let local_user_join = local_user::table.on(local_user::person_id.nullable().eq(person_id)); community::table - .inner_join(community_aggregates::table) .left_join(community_actions_join) .left_join(instance_actions_join) .left_join(local_user_join) @@ -164,27 +163,24 @@ impl CommunityQuery<'_> { query = o.local_user.visible_communities_only(query); match o.sort.unwrap_or_default() { - Hot => query = query.order_by(community_aggregates::hot_rank.desc()), - Comments => query = query.order_by(community_aggregates::comments.desc()), - Posts => query = query.order_by(community_aggregates::posts.desc()), + Hot => query = query.order_by(community::hot_rank.desc()), + Comments => query = query.order_by(community::comments.desc()), + Posts => query = query.order_by(community::posts.desc()), New => query = query.order_by(community::published.desc()), Old => query = query.order_by(community::published.asc()), - Subscribers => query = query.order_by(community_aggregates::subscribers.desc()), - SubscribersLocal => query = query.order_by(community_aggregates::subscribers_local.desc()), - ActiveSixMonths => { - query = query.order_by(community_aggregates::users_active_half_year.desc()) - } - ActiveMonthly => query = query.order_by(community_aggregates::users_active_month.desc()), - ActiveWeekly => query = query.order_by(community_aggregates::users_active_week.desc()), - ActiveDaily => query = query.order_by(community_aggregates::users_active_day.desc()), + Subscribers => query = query.order_by(community::subscribers.desc()), + SubscribersLocal => query = query.order_by(community::subscribers_local.desc()), + ActiveSixMonths => query = query.order_by(community::users_active_half_year.desc()), + ActiveMonthly => query = query.order_by(community::users_active_month.desc()), + ActiveWeekly => query = query.order_by(community::users_active_week.desc()), + ActiveDaily => query = query.order_by(community::users_active_day.desc()), NameAsc => query = query.order_by(lower(community::name).asc()), NameDesc => query = query.order_by(lower(community::name).desc()), }; // Filter by the time range if let Some(time_range_seconds) = o.time_range_seconds { - query = query.filter( - community_aggregates::published.gt(now() - seconds_to_pg_interval(time_range_seconds)), - ); + query = + query.filter(community::published.gt(now() - seconds_to_pg_interval(time_range_seconds))); } let (limit, offset) = limit_and_offset(o.page, o.limit)?; diff --git a/crates/db_views/src/post/post_view.rs b/crates/db_views/src/post/post_view.rs index d0dff018c9..2dead793f2 100644 --- a/crates/db_views/src/post/post_view.rs +++ b/crates/db_views/src/post/post_view.rs @@ -304,11 +304,6 @@ impl<'a> PostQuery<'a> { // covers the "worst case" of the whole page consisting of posts from one community // but using the largest community decreases the pagination-frame so make the real query more // efficient. - use lemmy_db_schema::schema::community_aggregates::dsl::{ - community_aggregates, - community_id, - users_active_month, - }; let (limit, offset) = limit_and_offset(self.page, self.limit)?; if offset != 0 && self.page_after.is_some() { return Err(Error::QueryBuilderError( @@ -321,9 +316,9 @@ impl<'a> PostQuery<'a> { community_actions::table .filter(community_actions::followed.is_not_null()) .filter(community_actions::person_id.eq(self_person_id)) - .inner_join(community_aggregates.on(community_id.eq(community_actions::community_id))) - .order_by(users_active_month.desc()) - .select(community_id) + .inner_join(community::table.on(community::id.eq(community_actions::community_id))) + .order_by(community::users_active_month.desc()) + .select(community::id) .limit(1) .get_result::(conn) .await @@ -2023,6 +2018,18 @@ mod tests { featured_url: inserted_community.featured_url.clone(), visibility: CommunityVisibility::Public, random_number: inserted_community.random_number, + subscribers: 0, + posts: 0, + comments: 0, + users_active_day: 0, + users_active_week: 0, + users_active_month: 0, + users_active_half_year: 0, + hot_rank: RANK_DEFAULT, + subscribers_local: 0, + report_count: 0, + unresolved_report_count: 0, + interactions_month: 0, }, subscribed: SubscribedType::NotSubscribed, read: false, diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index c0722d0167..4b1a725d9e 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -16,7 +16,7 @@ use diesel::{ Selectable, }; use lemmy_db_schema::{ - aggregates::structs::{CommunityAggregates, PersonAggregates, SiteAggregates}, + aggregates::structs::{PersonAggregates, SiteAggregates}, source::{ comment::Comment, comment_reply::CommentReply, @@ -237,7 +237,6 @@ pub struct CommunityReportView { pub community_report: CommunityReport, pub community: Community, pub creator: Person, - pub counts: CommunityAggregates, pub subscribed: SubscribedType, #[cfg_attr(feature = "full", ts(optional))] pub resolver: Option, @@ -457,7 +456,6 @@ pub struct ReportCombinedViewInternal { pub private_message: Option, // Community-specific pub community_report: Option, - pub community_counts: Option, // Shared pub report_creator: Person, pub item_creator: Option, @@ -577,8 +575,6 @@ pub struct CommunityView { ) )] pub blocked: bool, - #[cfg_attr(feature = "full", diesel(embed))] - pub counts: CommunityAggregates, #[cfg_attr(feature = "full", diesel( select_expression = community_actions::received_ban.nullable().is_not_null() @@ -1156,7 +1152,6 @@ pub(crate) struct SearchCombinedViewInternal { pub my_comment_vote: Option, // // Community-specific pub community: Option, - pub community_counts: Option, pub community_blocked: bool, pub subscribed: SubscribedType, // Person From 261c0a109773e3c388639d1065809ac5fc2bc09e Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 12:21:12 +0100 Subject: [PATCH 20/39] triggers --- .../db_schema/replaceable_schema/triggers.sql | 46 +++++-------------- crates/db_schema/src/impls/community.rs | 5 +- 2 files changed, 14 insertions(+), 37 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index 8f787b2299..ed38a13848 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -171,7 +171,7 @@ WITH post_diff AS ( diff.comments, diff.include_in_community_aggregates) UPDATE - community_aggregates AS a + community AS a SET comments = a.comments + diff.comments FROM ( @@ -185,7 +185,7 @@ FROM ( GROUP BY community_id) AS diff WHERE - a.community_id = diff.community_id + a.id = diff.community_id AND diff.comments != 0; UPDATE @@ -227,7 +227,7 @@ WHERE AND diff.post_count != 0; UPDATE - community_aggregates AS a + community AS a SET posts = a.posts + diff.posts FROM ( @@ -241,7 +241,7 @@ FROM ( GROUP BY (post).community_id) AS diff WHERE - a.community_id = diff.community_id + a.id = diff.community_id AND diff.posts != 0; UPDATE @@ -307,14 +307,14 @@ END; $$); --- For community_aggregates.comments, don't include comments of deleted or removed posts +-- For community.comments, don't include comments of deleted or removed posts CREATE FUNCTION r.update_comment_count_from_post () RETURNS TRIGGER LANGUAGE plpgsql AS $$ BEGIN UPDATE - community_aggregates AS a + community AS a SET comments = a.comments + diff.comments FROM ( @@ -334,7 +334,7 @@ BEGIN GROUP BY old_post.community_id) AS diff WHERE - a.community_id = diff.community_id + a.id = diff.community_id AND diff.comments != 0; RETURN NULL; END; @@ -351,7 +351,7 @@ CREATE TRIGGER comment_count CALL r.create_triggers ('community_actions', $$ BEGIN UPDATE - community_aggregates AS a + community AS a SET subscribers = a.subscribers + diff.subscribers, subscribers_local = a.subscribers_local + diff.subscribers_local FROM ( @@ -362,7 +362,7 @@ BEGIN LEFT JOIN person ON person.id = (community_actions).person_id WHERE (community_actions).followed IS NOT NULL GROUP BY (community_actions).community_id) AS diff WHERE - a.community_id = diff.community_id + a.id = diff.community_id AND (diff.subscribers, diff.subscribers_local) != (0, 0); RETURN NULL; @@ -412,7 +412,7 @@ $$); CALL r.create_triggers ('community_report', $$ BEGIN UPDATE - community_aggregates AS a + community AS a SET report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count FROM ( @@ -420,7 +420,7 @@ BEGIN (community_report).community_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (community_report).resolved), 0) AS unresolved_report_count FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (community_report).community_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) - AND a.community_id = diff.community_id; + AND a.id = diff.community_id; RETURN NULL; @@ -428,28 +428,6 @@ END; $$); --- These triggers create and update rows in each aggregates table to match its associated table's rows. --- Deleting rows and updating IDs are already handled by `CASCADE` in foreign key constraints. -CREATE FUNCTION r.community_aggregates_from_community () - RETURNS TRIGGER - LANGUAGE plpgsql - AS $$ -BEGIN - INSERT INTO community_aggregates (community_id, published) - SELECT - id, - published - FROM - new_community; - RETURN NULL; -END; -$$; - -CREATE TRIGGER aggregates - AFTER INSERT ON community REFERENCING NEW TABLE AS new_community - FOR EACH STATEMENT - EXECUTE FUNCTION r.community_aggregates_from_community (); - CREATE FUNCTION r.person_aggregates_from_person () RETURNS TRIGGER LANGUAGE plpgsql @@ -1017,7 +995,7 @@ END $$; CREATE TRIGGER search_combined_community_score - AFTER UPDATE OF users_active_month ON community_aggregates + AFTER UPDATE OF users_active_month ON community FOR EACH ROW EXECUTE FUNCTION r.search_combined_community_score_update (); diff --git a/crates/db_schema/src/impls/community.rs b/crates/db_schema/src/impls/community.rs index 6fa40abee4..97fd3ad39e 100644 --- a/crates/db_schema/src/impls/community.rs +++ b/crates/db_schema/src/impls/community.rs @@ -644,7 +644,7 @@ mod tests { instance_id: inserted_instance.id, visibility: CommunityVisibility::Public, random_number: inserted_community.random_number, - subscribers: 0, + subscribers: 1, posts: 0, comments: 0, users_active_day: 0, @@ -652,7 +652,7 @@ mod tests { users_active_month: 0, users_active_half_year: 0, hot_rank: RANK_DEFAULT, - subscribers_local: 0, + subscribers_local: 1, report_count: 0, unresolved_report_count: 0, interactions_month: 0, @@ -763,7 +763,6 @@ mod tests { Instance::delete(pool, inserted_instance.id).await?; assert_eq!(expected_community, read_community); - assert_eq!(expected_community, inserted_community); assert_eq!(expected_community, updated_community); assert_eq!(expected_community_follower, inserted_community_follower); assert_eq!(expected_community_moderator, inserted_bobby_moderator); From f1455656b888a2230469d7aab448e6b8fbf1fd29 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 12:33:10 +0100 Subject: [PATCH 21/39] person aggregate migration --- .../down.sql | 31 +++++++++++++++++++ .../up.sql | 21 +++++++++++++ 2 files changed, 52 insertions(+) diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql index 87af16be49..581d476923 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql @@ -239,3 +239,34 @@ CREATE INDEX idx_community_aggregates_subscribers ON public.community_aggregates CREATE INDEX idx_community_aggregates_users_active_month ON public.community_aggregates USING btree (users_active_month DESC); +-- move person_aggregates back into separate table +CREATE TABLE person_aggregates ( + person_id int PRIMARY KEY NOT NULL REFERENCES person ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + post_count bigint NOT NULL DEFAULT 0, + post_score bigint NOT NULL DEFAULT 0, + comment_count bigint NOT NULL DEFAULT 0, + comment_score bigint NOT NULL DEFAULT 0, + published timestamp with time zone DEFAULT now() NOT NULL +); + +INSERT INTO person_aggregates +SELECT + id AS person_id, + post_count, + post_score, + comment_count, + comment_score, + published +FROM + person; + +ALTER TABLE person + DROP COLUMN post_count, + DROP COLUMN post_score, + DROP COLUMN comment_count, + DROP COLUMN comment_score; + +CREATE INDEX idx_person_aggregates_comment_score ON public.person_aggregates USING btree (comment_score DESC); + +CREATE INDEX idx_person_aggregates_person ON public.person_aggregates USING btree (person_id); + diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql index 6f535f9dbc..f7b43cf959 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -186,3 +186,24 @@ CREATE INDEX idx_community_aggregates_subscribers ON public.community USING btre CREATE INDEX idx_community_aggregates_users_active_month ON public.community USING btree (users_active_month DESC); +-- merge person_aggregates into person table +ALTER TABLE person + ADD COLUMN post_count bigint NOT NULL DEFAULT 0, + ADD COLUMN post_score bigint NOT NULL DEFAULT 0, + ADD COLUMN comment_count bigint NOT NULL DEFAULT 0, + ADD COLUMN comment_score bigint NOT NULL DEFAULT 0; + +UPDATE + person +SET + post_count = pa.post_count, + post_score = pa.post_score, + comment_count = pa.comment_count, + comment_score = pa.comment_score +FROM + person_aggregates AS pa +WHERE + person.id = pa.person_id; + +DROP TABLE person_aggregates; + From 1ee4e635257e3ded022fc6eb1cd994d43cbc1c4e Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 13:09:24 +0100 Subject: [PATCH 22/39] person aggregate code --- crates/api_crud/src/user/create.rs | 2 - crates/db_schema/src/aggregates/mod.rs | 2 - .../src/aggregates/person_aggregates.rs | 182 ------------------ crates/db_schema/src/aggregates/structs.rs | 24 +-- crates/db_schema/src/impls/person.rs | 157 ++++++++++++++- crates/db_schema/src/lib.rs | 4 + crates/db_schema/src/schema.rs | 17 +- crates/db_schema/src/source/person.rs | 6 + .../combined/person_saved_combined_view.rs | 1 - .../src/combined/report_combined_view.rs | 2 - .../src/combined/search_combined_view.rs | 11 +- crates/db_views/src/comment/comment_view.rs | 5 +- .../src/local_user/local_user_view.rs | 12 +- crates/db_views/src/person/person_view.rs | 6 +- crates/db_views/src/post/post_view.rs | 7 +- .../registration_application_view.rs | 8 + crates/db_views/src/structs.rs | 8 +- 17 files changed, 195 insertions(+), 259 deletions(-) delete mode 100644 crates/db_schema/src/aggregates/person_aggregates.rs diff --git a/crates/api_crud/src/user/create.rs b/crates/api_crud/src/user/create.rs index 43a854e62c..3ae541aebd 100644 --- a/crates/api_crud/src/user/create.rs +++ b/crates/api_crud/src/user/create.rs @@ -18,7 +18,6 @@ use lemmy_api_common::{ }, }; use lemmy_db_schema::{ - aggregates::structs::PersonAggregates, newtypes::{InstanceId, OAuthProviderId}, source::{ actor_language::SiteLanguage, @@ -488,7 +487,6 @@ async fn send_verification_email_if_required( local_user: local_user.clone(), local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: person.clone(), - counts: PersonAggregates::default(), }; send_verification_email( diff --git a/crates/db_schema/src/aggregates/mod.rs b/crates/db_schema/src/aggregates/mod.rs index 5c7adfcb8b..fb58cdd7eb 100644 --- a/crates/db_schema/src/aggregates/mod.rs +++ b/crates/db_schema/src/aggregates/mod.rs @@ -1,6 +1,4 @@ #[cfg(feature = "full")] -pub mod person_aggregates; -#[cfg(feature = "full")] pub mod person_post_aggregates; #[cfg(feature = "full")] pub mod site_aggregates; diff --git a/crates/db_schema/src/aggregates/person_aggregates.rs b/crates/db_schema/src/aggregates/person_aggregates.rs deleted file mode 100644 index df7004d0eb..0000000000 --- a/crates/db_schema/src/aggregates/person_aggregates.rs +++ /dev/null @@ -1,182 +0,0 @@ -use crate::{ - aggregates::structs::PersonAggregates, - newtypes::PersonId, - schema::person_aggregates, - utils::{get_conn, DbPool}, -}; -use diesel::{result::Error, QueryDsl}; -use diesel_async::RunQueryDsl; - -impl PersonAggregates { - pub async fn read(pool: &mut DbPool<'_>, person_id: PersonId) -> Result { - let conn = &mut get_conn(pool).await?; - person_aggregates::table.find(person_id).first(conn).await - } -} - -#[cfg(test)] -mod tests { - - use crate::{ - aggregates::person_aggregates::PersonAggregates, - source::{ - comment::{Comment, CommentInsertForm, CommentLike, CommentLikeForm, CommentUpdateForm}, - community::{Community, CommunityInsertForm}, - instance::Instance, - person::{Person, PersonInsertForm}, - post::{Post, PostInsertForm, PostLike, PostLikeForm}, - }, - traits::{Crud, Likeable}, - utils::build_db_pool_for_tests, - }; - use diesel::result::Error; - use pretty_assertions::assert_eq; - use serial_test::serial; - - #[tokio::test] - #[serial] - async fn test_crud() -> Result<(), Error> { - let pool = &build_db_pool_for_tests(); - let pool = &mut pool.into(); - - let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; - - let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_user_agg"); - - let inserted_person = Person::create(pool, &new_person).await?; - - let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_user_agg"); - - let another_inserted_person = Person::create(pool, &another_person).await?; - - let new_community = CommunityInsertForm::new( - inserted_instance.id, - "TIL_site_agg".into(), - "nada".to_owned(), - "pubkey".to_string(), - ); - - let inserted_community = Community::create(pool, &new_community).await?; - - let new_post = PostInsertForm::new( - "A test post".into(), - inserted_person.id, - inserted_community.id, - ); - let inserted_post = Post::create(pool, &new_post).await?; - - let post_like = PostLikeForm::new(inserted_post.id, inserted_person.id, 1); - let _inserted_post_like = PostLike::like(pool, &post_like).await?; - - let comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - let inserted_comment = Comment::create(pool, &comment_form, None).await?; - - let mut comment_like = CommentLikeForm { - comment_id: inserted_comment.id, - person_id: inserted_person.id, - score: 1, - }; - - let _inserted_comment_like = CommentLike::like(pool, &comment_like).await?; - - let child_comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - let inserted_child_comment = - Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; - - let child_comment_like = CommentLikeForm { - comment_id: inserted_child_comment.id, - person_id: another_inserted_person.id, - score: 1, - }; - - let _inserted_child_comment_like = CommentLike::like(pool, &child_comment_like).await?; - - let person_aggregates_before_delete = PersonAggregates::read(pool, inserted_person.id).await?; - - assert_eq!(1, person_aggregates_before_delete.post_count); - assert_eq!(1, person_aggregates_before_delete.post_score); - assert_eq!(2, person_aggregates_before_delete.comment_count); - assert_eq!(2, person_aggregates_before_delete.comment_score); - - // Remove a post like - PostLike::remove(pool, inserted_person.id, inserted_post.id).await?; - let after_post_like_remove = PersonAggregates::read(pool, inserted_person.id).await?; - assert_eq!(0, after_post_like_remove.post_score); - - Comment::update( - pool, - inserted_comment.id, - &CommentUpdateForm { - removed: Some(true), - ..Default::default() - }, - ) - .await?; - Comment::update( - pool, - inserted_child_comment.id, - &CommentUpdateForm { - removed: Some(true), - ..Default::default() - }, - ) - .await?; - - let after_parent_comment_removed = PersonAggregates::read(pool, inserted_person.id).await?; - assert_eq!(0, after_parent_comment_removed.comment_count); - // TODO: fix person aggregate comment score calculation - // assert_eq!(0, after_parent_comment_removed.comment_score); - - // Remove a parent comment (the scores should also be removed) - Comment::delete(pool, inserted_comment.id).await?; - Comment::delete(pool, inserted_child_comment.id).await?; - let after_parent_comment_delete = PersonAggregates::read(pool, inserted_person.id).await?; - assert_eq!(0, after_parent_comment_delete.comment_count); - // TODO: fix person aggregate comment score calculation - // assert_eq!(0, after_parent_comment_delete.comment_score); - - // Add in the two comments again, then delete the post. - let new_parent_comment = Comment::create(pool, &comment_form, None).await?; - let _new_child_comment = - Comment::create(pool, &child_comment_form, Some(&new_parent_comment.path)).await?; - comment_like.comment_id = new_parent_comment.id; - CommentLike::like(pool, &comment_like).await?; - let after_comment_add = PersonAggregates::read(pool, inserted_person.id).await?; - assert_eq!(2, after_comment_add.comment_count); - // TODO: fix person aggregate comment score calculation - // assert_eq!(1, after_comment_add.comment_score); - - Post::delete(pool, inserted_post.id).await?; - let after_post_delete = PersonAggregates::read(pool, inserted_person.id).await?; - // TODO: fix person aggregate comment score calculation - // assert_eq!(0, after_post_delete.comment_score); - assert_eq!(0, after_post_delete.comment_count); - assert_eq!(0, after_post_delete.post_score); - assert_eq!(0, after_post_delete.post_count); - - // This should delete all the associated rows, and fire triggers - let person_num_deleted = Person::delete(pool, inserted_person.id).await?; - assert_eq!(1, person_num_deleted); - Person::delete(pool, another_inserted_person.id).await?; - - // Delete the community - let community_num_deleted = Community::delete(pool, inserted_community.id).await?; - assert_eq!(1, community_num_deleted); - - // Should be none found - let after_delete = PersonAggregates::read(pool, inserted_person.id).await; - assert!(after_delete.is_err()); - - Instance::delete(pool, inserted_instance.id).await?; - - Ok(()) - } -} diff --git a/crates/db_schema/src/aggregates/structs.rs b/crates/db_schema/src/aggregates/structs.rs index 67b31c526f..096cfa55ad 100644 --- a/crates/db_schema/src/aggregates/structs.rs +++ b/crates/db_schema/src/aggregates/structs.rs @@ -3,33 +3,11 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; #[cfg(feature = "full")] use { - crate::schema::{person_aggregates, post_actions, site_aggregates}, + crate::schema::{post_actions, site_aggregates}, diesel::{dsl, expression_methods::NullableExpressionMethods}, ts_rs::TS, }; -#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone, Default)] -#[cfg_attr( - feature = "full", - derive(Queryable, Selectable, Associations, Identifiable, TS) -)] -#[cfg_attr(feature = "full", diesel(table_name = person_aggregates))] -#[cfg_attr(feature = "full", diesel(belongs_to(crate::source::person::Person)))] -#[cfg_attr(feature = "full", diesel(primary_key(person_id)))] -#[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))] -#[cfg_attr(feature = "full", ts(export))] -/// Aggregate data for a person. -pub struct PersonAggregates { - pub person_id: PersonId, - pub post_count: i64, - #[serde(skip)] - pub post_score: i64, - pub comment_count: i64, - #[serde(skip)] - pub comment_score: i64, - pub published: DateTime, -} - #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)] #[cfg_attr( feature = "full", diff --git a/crates/db_schema/src/impls/person.rs b/crates/db_schema/src/impls/person.rs index 8a0b9e6571..c41e5d418e 100644 --- a/crates/db_schema/src/impls/person.rs +++ b/crates/db_schema/src/impls/person.rs @@ -259,12 +259,16 @@ mod tests { use crate::{ source::{ + comment::{Comment, CommentInsertForm, CommentLike, CommentLikeForm, CommentUpdateForm}, + community::{Community, CommunityInsertForm}, instance::Instance, person::{Person, PersonFollower, PersonFollowerForm, PersonInsertForm, PersonUpdateForm}, + post::{Post, PostInsertForm, PostLike, PostLikeForm}, }, - traits::{Crud, Followable}, + traits::{Crud, Followable, Likeable}, utils::{build_db_pool_for_tests, uplete}, }; + use diesel::result::Error; use lemmy_utils::error::LemmyResult; use pretty_assertions::assert_eq; use serial_test::serial; @@ -302,6 +306,10 @@ mod tests { matrix_user_id: None, ban_expires: None, instance_id: inserted_instance.id, + post_count: 0, + post_score: 0, + comment_count: 0, + comment_score: 0, }; let read_person = Person::read(pool, inserted_person.id).await?; @@ -353,4 +361,151 @@ mod tests { Ok(()) } + + #[tokio::test] + #[serial] + async fn test_aggregates() -> Result<(), Error> { + let pool = &build_db_pool_for_tests(); + let pool = &mut pool.into(); + + let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; + + let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_user_agg"); + + let inserted_person = Person::create(pool, &new_person).await?; + + let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_user_agg"); + + let another_inserted_person = Person::create(pool, &another_person).await?; + + let new_community = CommunityInsertForm::new( + inserted_instance.id, + "TIL_site_agg".into(), + "nada".to_owned(), + "pubkey".to_string(), + ); + + let inserted_community = Community::create(pool, &new_community).await?; + + let new_post = PostInsertForm::new( + "A test post".into(), + inserted_person.id, + inserted_community.id, + ); + let inserted_post = Post::create(pool, &new_post).await?; + + let post_like = PostLikeForm::new(inserted_post.id, inserted_person.id, 1); + let _inserted_post_like = PostLike::like(pool, &post_like).await?; + + let comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + let inserted_comment = Comment::create(pool, &comment_form, None).await?; + + let mut comment_like = CommentLikeForm { + comment_id: inserted_comment.id, + person_id: inserted_person.id, + score: 1, + }; + + let _inserted_comment_like = CommentLike::like(pool, &comment_like).await?; + + let child_comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + let inserted_child_comment = + Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; + + let child_comment_like = CommentLikeForm { + comment_id: inserted_child_comment.id, + person_id: another_inserted_person.id, + score: 1, + }; + + let _inserted_child_comment_like = CommentLike::like(pool, &child_comment_like).await?; + + let person_aggregates_before_delete = Person::read(pool, inserted_person.id).await?; + + assert_eq!(1, person_aggregates_before_delete.post_count); + assert_eq!(1, person_aggregates_before_delete.post_score); + assert_eq!(2, person_aggregates_before_delete.comment_count); + assert_eq!(2, person_aggregates_before_delete.comment_score); + + // Remove a post like + PostLike::remove(pool, inserted_person.id, inserted_post.id).await?; + let after_post_like_remove = Person::read(pool, inserted_person.id).await?; + assert_eq!(0, after_post_like_remove.post_score); + + Comment::update( + pool, + inserted_comment.id, + &CommentUpdateForm { + removed: Some(true), + ..Default::default() + }, + ) + .await?; + Comment::update( + pool, + inserted_child_comment.id, + &CommentUpdateForm { + removed: Some(true), + ..Default::default() + }, + ) + .await?; + + let after_parent_comment_removed = Person::read(pool, inserted_person.id).await?; + assert_eq!(0, after_parent_comment_removed.comment_count); + // TODO: fix person aggregate comment score calculation + // assert_eq!(0, after_parent_comment_removed.comment_score); + + // Remove a parent comment (the scores should also be removed) + Comment::delete(pool, inserted_comment.id).await?; + Comment::delete(pool, inserted_child_comment.id).await?; + let after_parent_comment_delete = Person::read(pool, inserted_person.id).await?; + assert_eq!(0, after_parent_comment_delete.comment_count); + // TODO: fix person aggregate comment score calculation + // assert_eq!(0, after_parent_comment_delete.comment_score); + + // Add in the two comments again, then delete the post. + let new_parent_comment = Comment::create(pool, &comment_form, None).await?; + let _new_child_comment = + Comment::create(pool, &child_comment_form, Some(&new_parent_comment.path)).await?; + comment_like.comment_id = new_parent_comment.id; + CommentLike::like(pool, &comment_like).await?; + let after_comment_add = Person::read(pool, inserted_person.id).await?; + assert_eq!(2, after_comment_add.comment_count); + // TODO: fix person aggregate comment score calculation + // assert_eq!(1, after_comment_add.comment_score); + + Post::delete(pool, inserted_post.id).await?; + let after_post_delete = Person::read(pool, inserted_person.id).await?; + // TODO: fix person aggregate comment score calculation + // assert_eq!(0, after_post_delete.comment_score); + assert_eq!(0, after_post_delete.comment_count); + assert_eq!(0, after_post_delete.post_score); + assert_eq!(0, after_post_delete.post_count); + + // This should delete all the associated rows, and fire triggers + let person_num_deleted = Person::delete(pool, inserted_person.id).await?; + assert_eq!(1, person_num_deleted); + Person::delete(pool, another_inserted_person.id).await?; + + // Delete the community + let community_num_deleted = Community::delete(pool, inserted_community.id).await?; + assert_eq!(1, community_num_deleted); + + // Should be none found + let after_delete = Person::read(pool, inserted_person.id).await; + assert!(after_delete.is_err()); + + Instance::delete(pool, inserted_instance.id).await?; + + Ok(()) + } } diff --git a/crates/db_schema/src/lib.rs b/crates/db_schema/src/lib.rs index e72514fa9d..521bb12ac2 100644 --- a/crates/db_schema/src/lib.rs +++ b/crates/db_schema/src/lib.rs @@ -345,4 +345,8 @@ pub type Person1AliasAllColumnsTuple = ( AliasedField, AliasedField, AliasedField, + AliasedField, + AliasedField, + AliasedField, + AliasedField, ); diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index c1bab4e794..663e1cd448 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -749,6 +749,10 @@ diesel::table! { bot_account -> Bool, ban_expires -> Nullable, instance_id -> Int4, + post_count -> Int8, + post_score -> Int8, + comment_count -> Int8, + comment_score -> Int8, } } @@ -762,17 +766,6 @@ diesel::table! { } } -diesel::table! { - person_aggregates (person_id) { - person_id -> Int4, - post_count -> Int8, - post_score -> Int8, - comment_count -> Int8, - comment_score -> Int8, - published -> Timestamptz, - } -} - diesel::table! { person_ban (person_id) { person_id -> Int4, @@ -1157,7 +1150,6 @@ diesel::joinable!(oauth_account -> local_user (local_user_id)); diesel::joinable!(oauth_account -> oauth_provider (oauth_provider_id)); diesel::joinable!(password_reset_request -> local_user (local_user_id)); diesel::joinable!(person -> instance (instance_id)); -diesel::joinable!(person_aggregates -> person (person_id)); diesel::joinable!(person_ban -> person (person_id)); diesel::joinable!(person_comment_mention -> comment (comment_id)); diesel::joinable!(person_comment_mention -> person (recipient_id)); @@ -1245,7 +1237,6 @@ diesel::allow_tables_to_appear_in_same_query!( password_reset_request, person, person_actions, - person_aggregates, person_ban, person_comment_mention, person_content_combined, diff --git a/crates/db_schema/src/source/person.rs b/crates/db_schema/src/source/person.rs index 49f9cd1b8a..db78421ef7 100644 --- a/crates/db_schema/src/source/person.rs +++ b/crates/db_schema/src/source/person.rs @@ -64,6 +64,12 @@ pub struct Person { #[cfg_attr(feature = "full", ts(optional))] pub ban_expires: Option>, pub instance_id: InstanceId, + pub post_count: i64, + #[serde(skip)] + pub post_score: i64, + pub comment_count: i64, + #[serde(skip)] + pub comment_score: i64, } #[derive(Clone, derive_new::new)] diff --git a/crates/db_views/src/combined/person_saved_combined_view.rs b/crates/db_views/src/combined/person_saved_combined_view.rs index c70c19ae83..4b4f220093 100644 --- a/crates/db_views/src/combined/person_saved_combined_view.rs +++ b/crates/db_views/src/combined/person_saved_combined_view.rs @@ -223,7 +223,6 @@ mod tests { local_user: timmy_local_user, local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: timmy.clone(), - counts: Default::default(), }; let sara_form = PersonInsertForm::test_form(instance.id, "sara_pcv"); diff --git a/crates/db_views/src/combined/report_combined_view.rs b/crates/db_views/src/combined/report_combined_view.rs index 3a40364da8..5f4867852d 100644 --- a/crates/db_views/src/combined/report_combined_view.rs +++ b/crates/db_views/src/combined/report_combined_view.rs @@ -530,7 +530,6 @@ mod tests { local_user: timmy_local_user, local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_timmy.clone(), - counts: Default::default(), }; // Make an admin, to be able to see private message reports. @@ -542,7 +541,6 @@ mod tests { local_user: admin_local_user, local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_admin.clone(), - counts: Default::default(), }; let sara_form = PersonInsertForm::test_form(inserted_instance.id, "sara_rcv"); diff --git a/crates/db_views/src/combined/search_combined_view.rs b/crates/db_views/src/combined/search_combined_view.rs index 10dba8ea46..f540a259dc 100644 --- a/crates/db_views/src/combined/search_combined_view.rs +++ b/crates/db_views/src/combined/search_combined_view.rs @@ -34,7 +34,6 @@ use lemmy_db_schema::{ local_user, person, person_actions, - person_aggregates, post, post_actions, post_tag, @@ -149,9 +148,6 @@ impl SearchCombinedViewInternal { let image_details_join = image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())); - let person_aggregates_join = person_aggregates::table - .on(search_combined::person_id.eq(person_aggregates::person_id.nullable())); - search_combined::table .left_join(comment_join) .left_join(post_join) @@ -163,7 +159,6 @@ impl SearchCombinedViewInternal { .left_join(community_actions_join) .left_join(post_actions_join) .left_join(person_actions_join) - .left_join(person_aggregates_join) .left_join(comment_actions_join) .left_join(image_details_join) } @@ -266,7 +261,7 @@ impl SearchCombinedQuery { community_actions::blocked.nullable().is_not_null(), community_follower_select_subscribed_type(), // Person - person_aggregates::all_columns.nullable(), + person::all_columns.nullable(), // // Shared person::all_columns.nullable(), local_user::admin.nullable().is_not_null(), @@ -480,10 +475,9 @@ impl InternalToCombinedView for SearchCombinedViewInternal { banned_from_community: v.banned_from_community, can_mod: v.can_mod, })) - } else if let (Some(person), Some(counts)) = (v.item_creator, v.item_creator_counts) { + } else if let Some(person) = v.item_creator { Some(SearchCombinedView::Person(PersonView { person, - counts, is_admin: v.item_creator_is_admin, })) } else { @@ -550,7 +544,6 @@ mod tests { local_user: timmy_local_user, local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: timmy.clone(), - counts: Default::default(), }; let community_form = CommunityInsertForm { diff --git a/crates/db_views/src/comment/comment_view.rs b/crates/db_views/src/comment/comment_view.rs index 67effc1f5c..33f1710d14 100644 --- a/crates/db_views/src/comment/comment_view.rs +++ b/crates/db_views/src/comment/comment_view.rs @@ -509,7 +509,6 @@ mod tests { local_user: inserted_timmy_local_user.clone(), local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_timmy_person.clone(), - counts: Default::default(), }; let site_form = SiteInsertForm::new("test site".to_string(), inserted_instance.id); let site = Site::create(pool, &site_form).await?; @@ -922,6 +921,10 @@ mod tests { private_key: data.timmy_local_user_view.person.private_key.clone(), public_key: data.timmy_local_user_view.person.public_key.clone(), last_refreshed_at: data.timmy_local_user_view.person.last_refreshed_at, + post_count: 0, + post_score: 0, + comment_count: 0, + comment_score: 0, }, post: Post { id: data.inserted_post.id, diff --git a/crates/db_views/src/local_user/local_user_view.rs b/crates/db_views/src/local_user/local_user_view.rs index 6525d5ca79..7aac376cc0 100644 --- a/crates/db_views/src/local_user/local_user_view.rs +++ b/crates/db_views/src/local_user/local_user_view.rs @@ -1,17 +1,10 @@ use crate::structs::LocalUserView; use actix_web::{dev::Payload, FromRequest, HttpMessage, HttpRequest}; -use diesel::{ - result::Error, - BoolExpressionMethods, - ExpressionMethods, - JoinOnDsl, - QueryDsl, - SelectableHelper, -}; +use diesel::{result::Error, BoolExpressionMethods, ExpressionMethods, QueryDsl, SelectableHelper}; use diesel_async::RunQueryDsl; use lemmy_db_schema::{ newtypes::{LocalUserId, OAuthProviderId, PersonId}, - schema::{local_user, local_user_vote_display_mode, oauth_account, person, person_aggregates}, + schema::{local_user, local_user_vote_display_mode, oauth_account, person}, source::{ instance::Instance, local_user::{LocalUser, LocalUserInsertForm}, @@ -33,7 +26,6 @@ impl LocalUserView { local_user::table .inner_join(local_user_vote_display_mode::table) .inner_join(person::table) - .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) } pub async fn read(pool: &mut DbPool<'_>, local_user_id: LocalUserId) -> Result { diff --git a/crates/db_views/src/person/person_view.rs b/crates/db_views/src/person/person_view.rs index e0a2160641..9957169471 100644 --- a/crates/db_views/src/person/person_view.rs +++ b/crates/db_views/src/person/person_view.rs @@ -10,16 +10,14 @@ use diesel::{ use diesel_async::RunQueryDsl; use lemmy_db_schema::{ newtypes::PersonId, - schema::{local_user, person, person_aggregates}, + schema::{local_user, person}, utils::{get_conn, now, DbPool}, }; impl PersonView { #[diesel::dsl::auto_type(no_type_alias)] fn joins() -> _ { - person::table - .inner_join(person_aggregates::table) - .left_join(local_user::table) + person::table.left_join(local_user::table) } pub async fn read( diff --git a/crates/db_views/src/post/post_view.rs b/crates/db_views/src/post/post_view.rs index 2dead793f2..1d613a3be4 100644 --- a/crates/db_views/src/post/post_view.rs +++ b/crates/db_views/src/post/post_view.rs @@ -871,20 +871,17 @@ mod tests { local_user: inserted_tegan_local_user, local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_tegan_person, - counts: Default::default(), }; let john_local_user_view = LocalUserView { local_user: inserted_john_local_user, local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_john_person, - counts: Default::default(), }; let bot_local_user_view = LocalUserView { local_user: inserted_bot_local_user, local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_bot_person, - counts: Default::default(), }; let site = Site { @@ -1984,6 +1981,10 @@ mod tests { private_key: inserted_person.private_key.clone(), public_key: inserted_person.public_key.clone(), last_refreshed_at: inserted_person.last_refreshed_at, + post_count: 0, + post_score: 0, + comment_count: 0, + comment_score: 0, }, image_details: None, creator_banned_from_community: false, diff --git a/crates/db_views/src/registration_applications/registration_application_view.rs b/crates/db_views/src/registration_applications/registration_application_view.rs index df2aa7b6c8..63456247f0 100644 --- a/crates/db_views/src/registration_applications/registration_application_view.rs +++ b/crates/db_views/src/registration_applications/registration_application_view.rs @@ -240,6 +240,10 @@ mod tests { private_key: inserted_sara_person.private_key, public_key: inserted_sara_person.public_key, last_refreshed_at: inserted_sara_person.last_refreshed_at, + post_count: 0, + post_score: 0, + comment_count: 0, + comment_score: 0, }, admin: None, }; @@ -309,6 +313,10 @@ mod tests { private_key: inserted_timmy_person.private_key, public_key: inserted_timmy_person.public_key, last_refreshed_at: inserted_timmy_person.last_refreshed_at, + post_count: 0, + post_score: 0, + comment_count: 0, + comment_score: 0, }); assert_eq!(read_sara_app_view_after_approve, expected_sara_app_view); diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index 4b1a725d9e..c225f1baa3 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -16,7 +16,7 @@ use diesel::{ Selectable, }; use lemmy_db_schema::{ - aggregates::structs::{PersonAggregates, SiteAggregates}, + aggregates::structs::SiteAggregates, source::{ comment::Comment, comment_reply::CommentReply, @@ -254,8 +254,6 @@ pub struct LocalUserView { pub local_user_vote_display_mode: LocalUserVoteDisplayMode, #[cfg_attr(feature = "full", diesel(embed))] pub person: Person, - #[cfg_attr(feature = "full", diesel(embed))] - pub counts: PersonAggregates, } #[skip_serializing_none] @@ -706,8 +704,6 @@ pub struct CommentReplyView { pub struct PersonView { #[cfg_attr(feature = "full", diesel(embed))] pub person: Person, - #[cfg_attr(feature = "full", diesel(embed))] - pub counts: PersonAggregates, #[cfg_attr(feature = "full", diesel( select_expression_type = coalesce, bool>, @@ -1155,7 +1151,7 @@ pub(crate) struct SearchCombinedViewInternal { pub community_blocked: bool, pub subscribed: SubscribedType, // Person - pub item_creator_counts: Option, + pub item_creator_counts: Option, // Shared pub item_creator: Option, pub item_creator_is_admin: bool, From f29c21d06ac0f25dc28b1338fab0019ca1c78932 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 14:29:37 +0100 Subject: [PATCH 23/39] person triggers --- .../db_schema/replaceable_schema/triggers.sql | 70 +++---------------- 1 file changed, 9 insertions(+), 61 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index ed38a13848..c17c95e5b0 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -41,13 +41,13 @@ BEGIN RETURNING a.creator_id AS creator_id, diff.upvotes - diff.downvotes AS score) UPDATE - person_aggregates AS a + person AS a SET thing_score = a.thing_score + diff.score FROM ( SELECT creator_id, sum(score) AS score FROM thing_diff GROUP BY creator_id) AS diff WHERE - a.person_id = diff.creator_id + a.id = diff.creator_id AND diff.score != 0; RETURN NULL; END; @@ -81,7 +81,7 @@ END; CALL r.create_triggers ('comment', $$ BEGIN UPDATE - person_aggregates AS a + person AS a SET comment_count = a.comment_count + diff.comment_count FROM ( @@ -92,7 +92,7 @@ BEGIN r.is_counted (comment) GROUP BY (comment).creator_id) AS diff WHERE - a.person_id = diff.creator_id + a.id = diff.creator_id AND diff.comment_count != 0; UPDATE @@ -212,7 +212,7 @@ $$); CALL r.create_triggers ('post', $$ BEGIN UPDATE - person_aggregates AS a + person AS a SET post_count = a.post_count + diff.post_count FROM ( @@ -223,7 +223,7 @@ BEGIN r.is_counted (post) GROUP BY (post).creator_id) AS diff WHERE - a.person_id = diff.creator_id + a.id = diff.creator_id AND diff.post_count != 0; UPDATE @@ -428,25 +428,6 @@ END; $$); -CREATE FUNCTION r.person_aggregates_from_person () - RETURNS TRIGGER - LANGUAGE plpgsql - AS $$ -BEGIN - INSERT INTO person_aggregates (person_id) - SELECT - id - FROM - new_person; - RETURN NULL; -END; -$$; - -CREATE TRIGGER aggregates - AFTER INSERT ON person REFERENCING NEW TABLE AS new_person - FOR EACH STATEMENT - EXECUTE FUNCTION r.person_aggregates_from_person (); - CREATE FUNCTION r.post_from_post () RETURNS TRIGGER LANGUAGE plpgsql @@ -455,14 +436,9 @@ BEGIN UPDATE post SET - published = new_post.published, newest_comment_time = new_post.published, newest_comment_time_necro = new_post.published, - community_id = new_post.community_id, - creator_id = new_post.creator_id, - instance_id = community.instance_id, - featured_community = new_post.featured_community, - featured_local = new_post.featured_local + instance_id = community.instance_id FROM new_post INNER JOIN community ON community.id = new_post.community_id @@ -478,34 +454,6 @@ CREATE TRIGGER aggregates WHEN (pg_trigger_depth() = 0) EXECUTE FUNCTION r.post_from_post (); -CREATE FUNCTION r.post_from_post_update () - RETURNS TRIGGER - LANGUAGE plpgsql - AS $$ -BEGIN - UPDATE - post - SET - featured_community = new_post.featured_community, - featured_local = new_post.featured_local - FROM - new_post - INNER JOIN old_post ON old_post.id = new_post.id - AND (old_post.featured_community, - old_post.featured_local) != (new_post.featured_community, - new_post.featured_local) - WHERE - post.id = new_post.id; - RETURN NULL; -END; -$$; - -CREATE TRIGGER aggregates_update - AFTER UPDATE ON post REFERENCING OLD TABLE AS old_post NEW TABLE AS new_post - FOR EACH STATEMENT - WHEN (pg_trigger_depth() = 0) - EXECUTE FUNCTION r.post_from_post_update (); - CREATE FUNCTION r.site_aggregates_from_site () RETURNS TRIGGER LANGUAGE plpgsql @@ -968,13 +916,13 @@ BEGIN SET score = NEW.post_score WHERE - person_id = NEW.person_id; + person_id = NEW.id; RETURN NULL; END $$; CREATE TRIGGER search_combined_person_score - AFTER UPDATE OF post_score ON person_aggregates + AFTER UPDATE OF post_score ON person FOR EACH ROW EXECUTE FUNCTION r.search_combined_person_score_update (); From 8c92d259d310a25873ada429aea63c2ca071ccec Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 15:08:01 +0100 Subject: [PATCH 24/39] test fixes --- crates/db_views/src/comment/comment_view.rs | 10 +++++----- crates/db_views/src/post/post_view.rs | 5 +++-- crates/db_views/src/site/vote_view.rs | 8 ++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/db_views/src/comment/comment_view.rs b/crates/db_views/src/comment/comment_view.rs index 33f1710d14..dc0e80f06f 100644 --- a/crates/db_views/src/comment/comment_view.rs +++ b/crates/db_views/src/comment/comment_view.rs @@ -921,10 +921,10 @@ mod tests { private_key: data.timmy_local_user_view.person.private_key.clone(), public_key: data.timmy_local_user_view.person.public_key.clone(), last_refreshed_at: data.timmy_local_user_view.person.last_refreshed_at, - post_count: 0, + post_count: 1, post_score: 0, - comment_count: 0, - comment_score: 0, + comment_count: 5, + comment_score: 1, }, post: Post { id: data.inserted_post.id, @@ -993,8 +993,8 @@ mod tests { visibility: CommunityVisibility::Public, random_number: data.inserted_community.random_number, subscribers: 0, - posts: 0, - comments: 0, + posts: 1, + comments: 6, users_active_day: 0, users_active_week: 0, users_active_month: 0, diff --git a/crates/db_views/src/post/post_view.rs b/crates/db_views/src/post/post_view.rs index 1d613a3be4..b0b077ea6a 100644 --- a/crates/db_views/src/post/post_view.rs +++ b/crates/db_views/src/post/post_view.rs @@ -1169,6 +1169,7 @@ mod tests { expected_post_with_upvote.my_vote = Some(1); expected_post_with_upvote.post.score = 1; expected_post_with_upvote.post.upvotes = 1; + expected_post_with_upvote.creator.post_score = 1; assert_eq!(expected_post_with_upvote, post_listing_single_with_person); let local_user_form = LocalUserUpdateForm { @@ -1981,7 +1982,7 @@ mod tests { private_key: inserted_person.private_key.clone(), public_key: inserted_person.public_key.clone(), last_refreshed_at: inserted_person.last_refreshed_at, - post_count: 0, + post_count: 2, post_score: 0, comment_count: 0, comment_score: 0, @@ -2020,7 +2021,7 @@ mod tests { visibility: CommunityVisibility::Public, random_number: inserted_community.random_number, subscribers: 0, - posts: 0, + posts: 4, comments: 0, users_active_day: 0, users_active_week: 0, diff --git a/crates/db_views/src/site/vote_view.rs b/crates/db_views/src/site/vote_view.rs index dac7857539..b6dd61fb53 100644 --- a/crates/db_views/src/site/vote_view.rs +++ b/crates/db_views/src/site/vote_view.rs @@ -164,7 +164,7 @@ mod tests { let sara_post_vote_form = PostLikeForm::new(inserted_post.id, inserted_sara.id, -1); PostLike::like(pool, &sara_post_vote_form).await?; - let expected_post_vote_views = [ + let mut expected_post_vote_views = [ VoteView { creator: inserted_sara.clone(), creator_banned_from_community: false, @@ -176,6 +176,8 @@ mod tests { score: 1, }, ]; + expected_post_vote_views[1].creator.post_count = 1; + expected_post_vote_views[1].creator.comment_count = 1; let read_post_vote_views = VoteView::list_for_post(pool, inserted_post.id, None, None).await?; assert_eq!(read_post_vote_views, expected_post_vote_views); @@ -196,7 +198,7 @@ mod tests { }; CommentLike::like(pool, &sara_comment_vote_form).await?; - let expected_comment_vote_views = [ + let mut expected_comment_vote_views = [ VoteView { creator: inserted_timmy.clone(), creator_banned_from_community: false, @@ -208,6 +210,8 @@ mod tests { score: 1, }, ]; + expected_comment_vote_views[0].creator.post_count = 1; + expected_comment_vote_views[0].creator.comment_count = 1; let read_comment_vote_views = VoteView::list_for_comment(pool, inserted_comment.id, None, None).await?; From 707c418d730c6ac5f84b9ce8b1b41090416abe23 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 15:19:57 +0100 Subject: [PATCH 25/39] fix scheduled task --- crates/routes/src/utils/scheduled_tasks.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/routes/src/utils/scheduled_tasks.rs b/crates/routes/src/utils/scheduled_tasks.rs index 1afa636101..61e31f9b32 100644 --- a/crates/routes/src/utils/scheduled_tasks.rs +++ b/crates/routes/src/utils/scheduled_tasks.rs @@ -222,7 +222,6 @@ async fn process_ranks_in_batches( .get_results::(conn) .await .map_err(|e| { - // TODO: need to remove community aggregates to get this working LemmyErrorType::Unknown(format!("Failed to update {} hot_ranks: {}", table_name, e)) })?; @@ -257,9 +256,9 @@ async fn process_post_aggregates_ranks_in_batches(conn: &mut AsyncPgConnection) SET hot_rank = r.hot_rank(pa.score, pa.published), hot_rank_active = r.hot_rank(pa.score, pa.newest_comment_time_necro), scaled_rank = r.scaled_rank(pa.score, pa.published, ca.interactions_month) - FROM batch, community_aggregates ca + FROM batch, community ca WHERE pa.id = batch.id - AND pa.community_id = ca.community_id + AND pa.community_id = ca.id RETURNING pa.published; "#, ) @@ -372,11 +371,11 @@ async fn active_counts(pool: &mut DbPool<'_>) -> LemmyResult<()> { ); sql_query(update_site_stmt).execute(&mut conn).await?; - let update_community_stmt = format!("update community_aggregates ca set users_active_{} = mv.count_ from r.community_aggregates_activity('{}') mv where ca.community_id = mv.community_id_", abbr, full_form); + let update_community_stmt = format!("update community ca set users_active_{} = mv.count_ from r.community_aggregates_activity('{}') mv where ca.id = mv.community_id_", abbr, full_form); sql_query(update_community_stmt).execute(&mut conn).await?; } - let update_interactions_stmt = "update community_aggregates ca set interactions_month = mv.count_ from r.community_aggregates_interactions('1 month') mv where ca.community_id = mv.community_id_"; + let update_interactions_stmt = "update community ca set interactions_month = mv.count_ from r.community_aggregates_interactions('1 month') mv where ca.id = mv.community_id_"; sql_query(update_interactions_stmt) .execute(&mut conn) .await?; From 85fc71d01a2c11f6fdb2a1d0fd54252663eee824 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 15:43:27 +0100 Subject: [PATCH 26/39] update api tests --- api_tests/package.json | 2 +- api_tests/pnpm-lock.yaml | 10 +++--- api_tests/src/comment.spec.ts | 58 ++++++++++++++++----------------- api_tests/src/community.spec.ts | 14 ++++---- api_tests/src/follow.spec.ts | 36 ++++++++++---------- api_tests/src/post.spec.ts | 22 ++++++------- 6 files changed, 71 insertions(+), 71 deletions(-) diff --git a/api_tests/package.json b/api_tests/package.json index e2953d6b33..6535d81709 100644 --- a/api_tests/package.json +++ b/api_tests/package.json @@ -29,7 +29,7 @@ "eslint": "^9.20.0", "eslint-plugin-prettier": "^5.2.3", "jest": "^29.5.0", - "lemmy-js-client": "0.20.0-ap-id.1", + "lemmy-js-client": "0.20.0-remove-aggregate-tables.1", "prettier": "^3.5.0", "ts-jest": "^29.1.0", "tsoa": "^6.6.0", diff --git a/api_tests/pnpm-lock.yaml b/api_tests/pnpm-lock.yaml index cd2112bb45..389e05d0eb 100644 --- a/api_tests/pnpm-lock.yaml +++ b/api_tests/pnpm-lock.yaml @@ -33,8 +33,8 @@ importers: specifier: ^29.5.0 version: 29.7.0(@types/node@22.13.1) lemmy-js-client: - specifier: 0.20.0-ap-id.1 - version: 0.20.0-ap-id.1 + specifier: 0.20.0-remove-aggregate-tables.1 + version: 0.20.0-remove-aggregate-tables.1 prettier: specifier: ^3.5.0 version: 3.5.0 @@ -1528,8 +1528,8 @@ packages: resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} engines: {node: '>=6'} - lemmy-js-client@0.20.0-ap-id.1: - resolution: {integrity: sha512-HzY005mhbINXa5i+GabuJSrwN27ExZKj2XxM1cAnfTWJ4ZqvbLuz4i26JDeE8pj6GGKbXBIj2VX4aOhKgCjkSA==} + lemmy-js-client@0.20.0-remove-aggregate-tables.1: + resolution: {integrity: sha512-O+zsHAEijbrT6da+x5HZAdb8WqDnoCVUQOuG7qsdJGc/ZlkUnyr0n0isuJDCmP3SOy2JnrGIUD1vUSJVMwMtrg==} leven@3.1.0: resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} @@ -4169,7 +4169,7 @@ snapshots: kleur@3.0.3: {} - lemmy-js-client@0.20.0-ap-id.1: {} + lemmy-js-client@0.20.0-remove-aggregate-tables.1: {} leven@3.1.0: {} diff --git a/api_tests/src/comment.spec.ts b/api_tests/src/comment.spec.ts index 8fcf25abfb..5a532d4889 100644 --- a/api_tests/src/comment.spec.ts +++ b/api_tests/src/comment.spec.ts @@ -81,19 +81,19 @@ test("Create a comment", async () => { expect(commentRes.comment_view.comment.content).toBeDefined(); expect(commentRes.comment_view.community.local).toBe(false); expect(commentRes.comment_view.creator.local).toBe(true); - expect(commentRes.comment_view.counts.score).toBe(1); + expect(commentRes.comment_view.comment.score).toBe(1); // Make sure that comment is liked on beta let betaComment = ( await waitUntil( () => resolveComment(beta, commentRes.comment_view.comment), - c => c.comment?.counts.score === 1, + c => c.comment?.comment.score === 1, ) ).comment; expect(betaComment).toBeDefined(); expect(betaComment?.community.local).toBe(true); expect(betaComment?.creator.local).toBe(false); - expect(betaComment?.counts.score).toBe(1); + expect(betaComment?.comment.score).toBe(1); assertCommentFederation(betaComment, commentRes.comment_view); }); @@ -293,48 +293,48 @@ test("Unlike a comment", async () => { let gammaComment1 = ( await waitUntil( () => resolveComment(gamma, commentRes.comment_view.comment), - c => c.comment?.counts.score === 1, + c => c.comment?.comment.score === 1, ) ).comment; expect(gammaComment1).toBeDefined(); expect(gammaComment1?.community.local).toBe(false); expect(gammaComment1?.creator.local).toBe(false); - expect(gammaComment1?.counts.score).toBe(1); + expect(gammaComment1?.comment.score).toBe(1); let unlike = await likeComment(alpha, 0, commentRes.comment_view.comment); - expect(unlike.comment_view.counts.score).toBe(0); + expect(unlike.comment_view.comment.score).toBe(0); // Make sure that comment is unliked on beta let betaComment = ( await waitUntil( () => resolveComment(beta, commentRes.comment_view.comment), - c => c.comment?.counts.score === 0, + c => c.comment?.comment.score === 0, ) ).comment; expect(betaComment).toBeDefined(); expect(betaComment?.community.local).toBe(true); expect(betaComment?.creator.local).toBe(false); - expect(betaComment?.counts.score).toBe(0); + expect(betaComment?.comment.score).toBe(0); // Make sure that comment is unliked on gamma, downstream peer // This is testing replication from remote-home-remote (alpha-beta-gamma) let gammaComment = ( await waitUntil( () => resolveComment(gamma, commentRes.comment_view.comment), - c => c.comment?.counts.score === 0, + c => c.comment?.comment.score === 0, ) ).comment; expect(gammaComment).toBeDefined(); expect(gammaComment?.community.local).toBe(false); expect(gammaComment?.creator.local).toBe(false); - expect(gammaComment?.counts.score).toBe(0); + expect(gammaComment?.comment.score).toBe(0); }); test("Federated comment like", async () => { let commentRes = await createComment(alpha, postOnAlphaRes.post_view.post.id); await waitUntil( () => resolveComment(beta, commentRes.comment_view.comment), - c => c.comment?.counts.score === 1, + c => c.comment?.comment.score === 1, ); // Find the comment on beta let betaComment = ( @@ -346,14 +346,14 @@ test("Federated comment like", async () => { } let like = await likeComment(beta, 1, betaComment.comment); - expect(like.comment_view.counts.score).toBe(2); + expect(like.comment_view.comment.score).toBe(2); // Get the post from alpha, check the likes let postComments = await waitUntil( () => getComments(alpha, postOnAlphaRes.post_view.post.id), - c => c.comments[0].counts.score === 2, + c => c.comments[0].comment.score === 2, ); - expect(postComments.comments[0].counts.score).toBe(2); + expect(postComments.comments[0].comment.score).toBe(2); }); test("Reply to a comment from another instance, get notification", async () => { @@ -377,7 +377,7 @@ test("Reply to a comment from another instance, get notification", async () => { let betaComment = ( await waitUntil( () => resolveComment(beta, commentRes.comment_view.comment), - c => c.comment?.counts.score === 1, + c => c.comment?.comment.score === 1, ) ).comment; @@ -397,12 +397,12 @@ test("Reply to a comment from another instance, get notification", async () => { expect(getCommentParentId(replyRes.comment_view.comment)).toBe( betaComment.comment.id, ); - expect(replyRes.comment_view.counts.score).toBe(1); + expect(replyRes.comment_view.comment.score).toBe(1); // Make sure that reply comment is seen on alpha let commentSearch = await waitUntil( () => resolveComment(alpha, replyRes.comment_view.comment), - c => c.comment?.counts.score === 1, + c => c.comment?.comment.score === 1, ); let alphaComment = commentSearch.comment!; let postComments = await waitUntil( @@ -418,7 +418,7 @@ test("Reply to a comment from another instance, get notification", async () => { ); expect(alphaComment.community.local).toBe(false); expect(alphaComment.creator.local).toBe(false); - expect(alphaComment.counts.score).toBe(1); + expect(alphaComment.comment.score).toBe(1); assertCommentFederation(alphaComment, replyRes.comment_view); // Did alpha get notified of the reply from beta? @@ -441,7 +441,7 @@ test("Reply to a comment from another instance, get notification", async () => { expect(alphaReply.comment.content).toBeDefined(); expect(alphaReply.community.local).toBe(false); expect(alphaReply.creator.local).toBe(false); - expect(alphaReply.counts.score).toBe(1); + expect(alphaReply.comment.score).toBe(1); // ToDo: interesting alphaRepliesRes.replies[0].comment_reply.id is 1, meaning? how did that come about? expect(alphaReply.comment.id).toBe(alphaComment.comment.id); // this is a new notification, getReplies fetch was for read/unread both, confirm it is unread. @@ -515,7 +515,7 @@ test("Mention beta from alpha comment", async () => { expect(mentionRes.comment_view.comment.content).toBeDefined(); expect(mentionRes.comment_view.community.local).toBe(false); expect(mentionRes.comment_view.creator.local).toBe(true); - expect(mentionRes.comment_view.counts.score).toBe(1); + expect(mentionRes.comment_view.comment.score).toBe(1); // get beta's localized copy of the alpha post let betaPost = await waitForPost(beta, postOnAlphaRes.post_view.post); @@ -528,7 +528,7 @@ test("Mention beta from alpha comment", async () => { // Make sure that both new comments are seen on beta and have parent/child relationship let betaPostComments = await waitUntil( () => getComments(beta, betaPost!.post.id), - c => c.comments[1]?.counts.score === 1, + c => c.comments[1]?.comment.score === 1, ); expect(betaPostComments.comments.length).toEqual(2); // the trunk-branch root comment will be older than the mention reply comment, so index 1 @@ -542,7 +542,7 @@ test("Mention beta from alpha comment", async () => { ); expect(betaRootComment.community.local).toBe(true); expect(betaRootComment.creator.local).toBe(false); - expect(betaRootComment.counts.score).toBe(1); + expect(betaRootComment.comment.score).toBe(1); assertCommentFederation(betaRootComment, commentRes.comment_view); let mentionsRes = await waitUntil( @@ -554,7 +554,7 @@ test("Mention beta from alpha comment", async () => { expect(firstMention.comment.content).toBeDefined(); expect(firstMention.community.local).toBe(true); expect(firstMention.creator.local).toBe(false); - expect(firstMention.counts.score).toBe(1); + expect(firstMention.comment.score).toBe(1); // the reply comment with mention should be the most fresh, newest, index 0 expect(firstMention.person_comment_mention.comment_id).toBe( betaPostComments.comments[0].comment.id, @@ -606,17 +606,17 @@ test("A and G subscribe to B (center) A posts, G mentions B, it gets announced t expect(commentRes.comment_view.comment.content).toBe(commentContent); expect(commentRes.comment_view.community.local).toBe(false); expect(commentRes.comment_view.creator.local).toBe(true); - expect(commentRes.comment_view.counts.score).toBe(1); + expect(commentRes.comment_view.comment.score).toBe(1); // Make sure alpha sees it let alphaPostComments2 = await waitUntil( () => getComments(alpha, alphaPost.post_view.post.id), - e => e.comments[0]?.counts.score === 1, + e => e.comments[0]?.comment.score === 1, ); expect(alphaPostComments2.comments[0].comment.content).toBe(commentContent); expect(alphaPostComments2.comments[0].community.local).toBe(true); expect(alphaPostComments2.comments[0].creator.local).toBe(false); - expect(alphaPostComments2.comments[0].counts.score).toBe(1); + expect(alphaPostComments2.comments[0].comment.score).toBe(1); assertCommentFederation( alphaPostComments2.comments[0], commentRes.comment_view, @@ -688,17 +688,17 @@ test("Check that activity from another instance is sent to third instance", asyn expect(commentRes.comment_view.comment.content).toBe(commentContent); expect(commentRes.comment_view.community.local).toBe(false); expect(commentRes.comment_view.creator.local).toBe(true); - expect(commentRes.comment_view.counts.score).toBe(1); + expect(commentRes.comment_view.comment.score).toBe(1); // Make sure alpha sees it let alphaPostComments2 = await waitUntil( () => getComments(alpha, alphaPost!.post.id), - e => e.comments[0]?.counts.score === 1, + e => e.comments[0]?.comment.score === 1, ); expect(alphaPostComments2.comments[0].comment.content).toBe(commentContent); expect(alphaPostComments2.comments[0].community.local).toBe(false); expect(alphaPostComments2.comments[0].creator.local).toBe(false); - expect(alphaPostComments2.comments[0].counts.score).toBe(1); + expect(alphaPostComments2.comments[0].comment.score).toBe(1); assertCommentFederation( alphaPostComments2.comments[0], commentRes.comment_view, diff --git a/api_tests/src/community.spec.ts b/api_tests/src/community.spec.ts index 06a317b47c..016574eaa8 100644 --- a/api_tests/src/community.spec.ts +++ b/api_tests/src/community.spec.ts @@ -396,7 +396,7 @@ test.skip("Community follower count is federated", async () => { ).community; // Make sure there is 1 subscriber - expect(followed?.counts.subscribers).toBe(1); + expect(followed?.community.subscribers).toBe(1); // Follow the community from gamma resolved = await resolveCommunity(gamma, communityActorId); @@ -413,7 +413,7 @@ test.skip("Community follower count is federated", async () => { ).community; // Make sure there are 2 subscribers - expect(followed?.counts?.subscribers).toBe(2); + expect(followed?.community?.subscribers).toBe(2); // Follow the community from delta resolved = await resolveCommunity(delta, communityActorId); @@ -430,13 +430,13 @@ test.skip("Community follower count is federated", async () => { ).community; // Make sure there are 3 subscribers - expect(followed?.counts?.subscribers).toBe(3); + expect(followed?.community?.subscribers).toBe(3); }); test("Dont receive community activities after unsubscribe", async () => { let communityRes = await createCommunity(alpha); expect(communityRes.community_view.community.name).toBeDefined(); - expect(communityRes.community_view.counts.subscribers).toBe(1); + expect(communityRes.community_view.community.subscribers).toBe(1); let betaCommunity = ( await resolveCommunity(beta, communityRes.community_view.community.ap_id) @@ -451,7 +451,7 @@ test("Dont receive community activities after unsubscribe", async () => { alpha, communityRes.community_view.community.id, ); - expect(communityRes1.community_view.counts.subscribers).toBe(2); + expect(communityRes1.community_view.community.subscribers).toBe(2); // temporarily block alpha, so that it doesn't know about unfollow var allow_instance_params: AdminAllowInstanceParams = { @@ -470,7 +470,7 @@ test("Dont receive community activities after unsubscribe", async () => { alpha, communityRes.community_view.community.id, ); - expect(communityRes2.community_view.counts.subscribers).toBe(2); + expect(communityRes2.community_view.community.subscribers).toBe(2); // unblock alpha allow_instance_params.allow = true; @@ -492,7 +492,7 @@ test("Dont receive community activities after unsubscribe", async () => { test("Fetch community, includes posts", async () => { let communityRes = await createCommunity(alpha); expect(communityRes.community_view.community.name).toBeDefined(); - expect(communityRes.community_view.counts.subscribers).toBe(1); + expect(communityRes.community_view.community.subscribers).toBe(1); let postRes = await createPost( alpha, diff --git a/api_tests/src/follow.spec.ts b/api_tests/src/follow.spec.ts index c447e14cd7..824c34f4ae 100644 --- a/api_tests/src/follow.spec.ts +++ b/api_tests/src/follow.spec.ts @@ -27,21 +27,21 @@ test("Follow local community", async () => { // Make sure the follow response went through expect(follow.community_view.community.local).toBe(true); expect(follow.community_view.subscribed).toBe("Subscribed"); - expect(follow.community_view.counts.subscribers).toBe( - community.counts.subscribers + 1, + expect(follow.community_view.community.subscribers).toBe( + community.community.subscribers + 1, ); - expect(follow.community_view.counts.subscribers_local).toBe( - community.counts.subscribers_local + 1, + expect(follow.community_view.community.subscribers_local).toBe( + community.community.subscribers_local + 1, ); // Test an unfollow let unfollow = await followCommunity(user, false, community.community.id); expect(unfollow.community_view.subscribed).toBe("NotSubscribed"); - expect(unfollow.community_view.counts.subscribers).toBe( - community.counts.subscribers, + expect(unfollow.community_view.community.subscribers).toBe( + community.community.subscribers, ); - expect(unfollow.community_view.counts.subscribers_local).toBe( - community.counts.subscribers_local, + expect(unfollow.community_view.community.subscribers_local).toBe( + community.community.subscribers_local, ); }); @@ -51,7 +51,7 @@ test("Follow federated community", async () => { const betaCommunityInitial = ( await waitUntil( () => resolveBetaCommunity(alpha), - c => !!c.community && c.community?.counts.subscribers >= 1, + c => !!c.community && c.community?.community.subscribers >= 1, ) ).community; if (!betaCommunityInitial) { @@ -74,14 +74,14 @@ test("Follow federated community", async () => { expect(betaCommunity?.community.local).toBe(false); expect(betaCommunity?.community.name).toBe("main"); expect(betaCommunity?.subscribed).toBe("Subscribed"); - expect(betaCommunity?.counts.subscribers_local).toBe( - betaCommunityInitial.counts.subscribers_local + 1, + expect(betaCommunity?.community.subscribers_local).toBe( + betaCommunityInitial.community.subscribers_local + 1, ); // check that unfollow was federated let communityOnBeta1 = await resolveBetaCommunity(beta); - expect(communityOnBeta1.community?.counts.subscribers).toBe( - betaCommunityInitial.counts.subscribers + 1, + expect(communityOnBeta1.community?.community.subscribers).toBe( + betaCommunityInitial.community.subscribers + 1, ); // Check it from local @@ -113,11 +113,11 @@ test("Follow federated community", async () => { let communityOnBeta2 = await waitUntil( () => resolveBetaCommunity(beta), c => - c.community?.counts.subscribers === - betaCommunityInitial.counts.subscribers, + c.community?.community.subscribers === + betaCommunityInitial.community.subscribers, ); - expect(communityOnBeta2.community?.counts.subscribers).toBe( - betaCommunityInitial.counts.subscribers, + expect(communityOnBeta2.community?.community.subscribers).toBe( + betaCommunityInitial.community.subscribers, ); - expect(communityOnBeta2.community?.counts.subscribers_local).toBe(1); + expect(communityOnBeta2.community?.community.subscribers_local).toBe(1); }); diff --git a/api_tests/src/post.spec.ts b/api_tests/src/post.spec.ts index 18419cf264..069b6cf01a 100644 --- a/api_tests/src/post.spec.ts +++ b/api_tests/src/post.spec.ts @@ -123,19 +123,19 @@ test("Create a post", async () => { expect(postRes.post_view.post).toBeDefined(); expect(postRes.post_view.community.local).toBe(false); expect(postRes.post_view.creator.local).toBe(true); - expect(postRes.post_view.counts.score).toBe(1); + expect(postRes.post_view.post.score).toBe(1); // Make sure that post is liked on beta const betaPost = await waitForPost( beta, postRes.post_view.post, - res => res?.counts.score === 1, + res => res?.post.score === 1, ); expect(betaPost).toBeDefined(); expect(betaPost?.community.local).toBe(true); expect(betaPost?.creator.local).toBe(false); - expect(betaPost?.counts.score).toBe(1); + expect(betaPost?.post.score).toBe(1); await assertPostFederation(betaPost, postRes.post_view); // Delta only follows beta, so it should not see an alpha ap_id @@ -163,23 +163,23 @@ test("Unlike a post", async () => { } let postRes = await createPost(alpha, betaCommunity.community.id); let unlike = await likePost(alpha, 0, postRes.post_view.post); - expect(unlike.post_view.counts.score).toBe(0); + expect(unlike.post_view.post.score).toBe(0); // Try to unlike it again, make sure it stays at 0 let unlike2 = await likePost(alpha, 0, postRes.post_view.post); - expect(unlike2.post_view.counts.score).toBe(0); + expect(unlike2.post_view.post.score).toBe(0); // Make sure that post is unliked on beta const betaPost = await waitForPost( beta, postRes.post_view.post, - post => post?.counts.score === 0, + post => post?.post.score === 0, ); expect(betaPost).toBeDefined(); expect(betaPost?.community.local).toBe(true); expect(betaPost?.creator.local).toBe(false); - expect(betaPost?.counts.score).toBe(0); + expect(betaPost?.post.score).toBe(0); await assertPostFederation(betaPost, postRes.post_view); }); @@ -658,7 +658,7 @@ test("Enforce community ban for federated user", async () => { expect(postRes3.post_view.post).toBeDefined(); expect(postRes3.post_view.community.local).toBe(false); expect(postRes3.post_view.creator.local).toBe(true); - expect(postRes3.post_view.counts.score).toBe(1); + expect(postRes3.post_view.post.score).toBe(1); // Make sure that post makes it to beta community let postRes4 = await waitForPost(beta, postRes3.post_view.post); @@ -798,7 +798,7 @@ test("Fetch post via redirect", async () => { const betaPost = await waitForPost( beta, alphaPost.post_view.post, - res => res?.counts.score === 1, + res => res?.post.score === 1, ); expect(betaPost).toBeDefined(); @@ -871,7 +871,7 @@ test("Mention beta from alpha post body", async () => { expect(postOnAlphaRes.post_view.post.body).toBeDefined(); expect(postOnAlphaRes.post_view.community.local).toBe(false); expect(postOnAlphaRes.post_view.creator.local).toBe(true); - expect(postOnAlphaRes.post_view.counts.score).toBe(1); + expect(postOnAlphaRes.post_view.post.score).toBe(1); // get beta's localized copy of the alpha post let betaPost = await waitForPost(beta, postOnAlphaRes.post_view.post); @@ -891,7 +891,7 @@ test("Mention beta from alpha post body", async () => { expect(firstMention.post.body).toBeDefined(); expect(firstMention.community.local).toBe(true); expect(firstMention.creator.local).toBe(false); - expect(firstMention.counts.score).toBe(1); + expect(firstMention.post.score).toBe(1); expect(firstMention.person_post_mention.post_id).toBe(betaPost.post.id); }); From cf86bc7f23d80c06045fa0c5ca7e67a33c4262dc Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 16:01:01 +0100 Subject: [PATCH 27/39] site_aggregates to local_site migration --- .../down.sql | 37 +++++++++++++++++++ .../up.sql | 29 +++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql index 581d476923..d272298905 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql @@ -270,3 +270,40 @@ CREATE INDEX idx_person_aggregates_comment_score ON public.person_aggregates USI CREATE INDEX idx_person_aggregates_person ON public.person_aggregates USING btree (person_id); +-- move site_aggregates back into separate table +CREATE TABLE person_aggregates ( + site_id int PRIMARY KEY NOT NULL REFERENCES person ON UPDATE CASCADE ON DELETE CASCADE users bigint NOT NULL DEFAULT 1, + posts bigint NOT NULL DEFAULT 0, + comments bigint NOT NULL DEFAULT 0, + communities bigint NOT NULL DEFAULT 0, + users_active_day bigint NOT NULL DEFAULT 0, + users_active_week bigint NOT NULL DEFAULT 0, + users_active_month bigint NOT NULL DEFAULT 0, + users_active_half_year bigint NOT NULL DEFAULT 0; + +); + +INSERT INTO site_aggregates +SELECT + id AS site_id, + users, + posts, + comments, + communities, + users_active_day, + users_active_week, + users_active_month, + users_active_half_year +FROM + local_site; + +ALTER TABLE local_site + DROP COLUMN users, + DROP COLUMN posts, + DROP COLUMN comments, + DROP COLUMN communities, + DROP COLUMN users_active_day, + DROP COLUMN users_active_week, + DROP COLUMN users_active_month, + DROP COLUMN users_active_half_year; + diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql index f7b43cf959..eda201e6d1 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -207,3 +207,32 @@ WHERE DROP TABLE person_aggregates; +-- merge site_aggregates into person table +ALTER TABLE local_site + ADD COLUMN users bigint NOT NULL DEFAULT 1, + ADD COLUMN posts bigint NOT NULL DEFAULT 0, + ADD COLUMN comments bigint NOT NULL DEFAULT 0, + ADD COLUMN communities bigint NOT NULL DEFAULT 0, + ADD COLUMN users_active_day bigint NOT NULL DEFAULT 0, + ADD COLUMN users_active_week bigint NOT NULL DEFAULT 0, + ADD COLUMN users_active_month bigint NOT NULL DEFAULT 0, + ADD COLUMN users_active_half_year bigint NOT NULL DEFAULT 0; + +UPDATE + local_site +SET + users = sa.users, + posts = sa.posts, + comments = sa.comments, + communities = sa.communities, + users_active_day = sa.users_active_day, + users_active_week = sa.users_active_week, + users_active_month = sa.users_active_month, + users_active_half_year = sa.users_active_half_year +FROM + site_aggregates AS sa +WHERE + local_site.site_id = sa.site_id; + +DROP TABLE site_aggregates; + From c9da44d7c79c93ba8a2d0fd7394fa102861cd156 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 16:13:59 +0100 Subject: [PATCH 28/39] site_aggregates code changes --- crates/db_schema/src/aggregates/mod.rs | 2 - .../src/aggregates/site_aggregates.rs | 204 ------------------ crates/db_schema/src/aggregates/structs.rs | 32 +-- crates/db_schema/src/impls/local_site.rs | 189 ++++++++++++++++ crates/db_schema/src/schema.rs | 24 +-- crates/db_schema/src/source/local_site.rs | 12 ++ crates/db_views/src/site/site_view.rs | 3 +- crates/db_views/src/structs.rs | 3 - crates/routes/src/nodeinfo.rs | 10 +- 9 files changed, 217 insertions(+), 262 deletions(-) delete mode 100644 crates/db_schema/src/aggregates/site_aggregates.rs diff --git a/crates/db_schema/src/aggregates/mod.rs b/crates/db_schema/src/aggregates/mod.rs index fb58cdd7eb..6a8ef5ad3b 100644 --- a/crates/db_schema/src/aggregates/mod.rs +++ b/crates/db_schema/src/aggregates/mod.rs @@ -1,5 +1,3 @@ #[cfg(feature = "full")] pub mod person_post_aggregates; -#[cfg(feature = "full")] -pub mod site_aggregates; pub mod structs; diff --git a/crates/db_schema/src/aggregates/site_aggregates.rs b/crates/db_schema/src/aggregates/site_aggregates.rs deleted file mode 100644 index 2df5662907..0000000000 --- a/crates/db_schema/src/aggregates/site_aggregates.rs +++ /dev/null @@ -1,204 +0,0 @@ -use crate::{ - aggregates::structs::SiteAggregates, - schema::site_aggregates, - utils::{get_conn, DbPool}, -}; -use diesel::result::Error; -use diesel_async::RunQueryDsl; - -impl SiteAggregates { - pub async fn read(pool: &mut DbPool<'_>) -> Result { - let conn = &mut get_conn(pool).await?; - site_aggregates::table.first(conn).await - } -} - -#[cfg(test)] -mod tests { - - use crate::{ - aggregates::site_aggregates::SiteAggregates, - source::{ - comment::{Comment, CommentInsertForm}, - community::{Community, CommunityInsertForm, CommunityUpdateForm}, - instance::Instance, - person::{Person, PersonInsertForm}, - post::{Post, PostInsertForm}, - site::{Site, SiteInsertForm}, - }, - traits::Crud, - utils::{build_db_pool_for_tests, DbPool}, - }; - use diesel::result::Error; - use pretty_assertions::assert_eq; - use serial_test::serial; - - async fn prepare_site_with_community( - pool: &mut DbPool<'_>, - ) -> Result<(Instance, Person, Site, Community), Error> { - let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; - - let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_site_agg"); - - let inserted_person = Person::create(pool, &new_person).await?; - - let site_form = SiteInsertForm::new("test_site".into(), inserted_instance.id); - let inserted_site = Site::create(pool, &site_form).await?; - - let new_community = CommunityInsertForm::new( - inserted_instance.id, - "TIL_site_agg".into(), - "nada".to_owned(), - "pubkey".to_string(), - ); - - let inserted_community = Community::create(pool, &new_community).await?; - - Ok(( - inserted_instance, - inserted_person, - inserted_site, - inserted_community, - )) - } - - #[tokio::test] - #[serial] - async fn test_crud() -> Result<(), Error> { - let pool = &build_db_pool_for_tests(); - let pool = &mut pool.into(); - - let (inserted_instance, inserted_person, inserted_site, inserted_community) = - prepare_site_with_community(pool).await?; - - let new_post = PostInsertForm::new( - "A test post".into(), - inserted_person.id, - inserted_community.id, - ); - - // Insert two of those posts - let inserted_post = Post::create(pool, &new_post).await?; - let _inserted_post_again = Post::create(pool, &new_post).await?; - - let comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - - // Insert two of those comments - let inserted_comment = Comment::create(pool, &comment_form, None).await?; - - let child_comment_form = CommentInsertForm::new( - inserted_person.id, - inserted_post.id, - "A test comment".into(), - ); - let _inserted_child_comment = - Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; - - let site_aggregates_before_delete = SiteAggregates::read(pool).await?; - - // TODO: this is unstable, sometimes it returns 0 users, sometimes 1 - //assert_eq!(0, site_aggregates_before_delete.users); - assert_eq!(1, site_aggregates_before_delete.communities); - assert_eq!(2, site_aggregates_before_delete.posts); - assert_eq!(2, site_aggregates_before_delete.comments); - - // Try a post delete - Post::delete(pool, inserted_post.id).await?; - let site_aggregates_after_post_delete = SiteAggregates::read(pool).await?; - assert_eq!(1, site_aggregates_after_post_delete.posts); - assert_eq!(0, site_aggregates_after_post_delete.comments); - - // This shouuld delete all the associated rows, and fire triggers - let person_num_deleted = Person::delete(pool, inserted_person.id).await?; - assert_eq!(1, person_num_deleted); - - // Delete the community - let community_num_deleted = Community::delete(pool, inserted_community.id).await?; - assert_eq!(1, community_num_deleted); - - // Site should still exist, it can without a site creator. - let after_delete_creator = SiteAggregates::read(pool).await; - assert!(after_delete_creator.is_ok()); - - Site::delete(pool, inserted_site.id).await?; - let after_delete_site = SiteAggregates::read(pool).await; - assert!(after_delete_site.is_err()); - - Instance::delete(pool, inserted_instance.id).await?; - - Ok(()) - } - - #[tokio::test] - #[serial] - async fn test_soft_delete() -> Result<(), Error> { - let pool = &build_db_pool_for_tests(); - let pool = &mut pool.into(); - - let (inserted_instance, inserted_person, inserted_site, inserted_community) = - prepare_site_with_community(pool).await?; - - let site_aggregates_before = SiteAggregates::read(pool).await?; - assert_eq!(1, site_aggregates_before.communities); - - Community::update( - pool, - inserted_community.id, - &CommunityUpdateForm { - deleted: Some(true), - ..Default::default() - }, - ) - .await?; - - let site_aggregates_after_delete = SiteAggregates::read(pool).await?; - assert_eq!(0, site_aggregates_after_delete.communities); - - Community::update( - pool, - inserted_community.id, - &CommunityUpdateForm { - deleted: Some(false), - ..Default::default() - }, - ) - .await?; - - Community::update( - pool, - inserted_community.id, - &CommunityUpdateForm { - removed: Some(true), - ..Default::default() - }, - ) - .await?; - - let site_aggregates_after_remove = SiteAggregates::read(pool).await?; - assert_eq!(0, site_aggregates_after_remove.communities); - - Community::update( - pool, - inserted_community.id, - &CommunityUpdateForm { - deleted: Some(true), - ..Default::default() - }, - ) - .await?; - - let site_aggregates_after_remove_delete = SiteAggregates::read(pool).await?; - assert_eq!(0, site_aggregates_after_remove_delete.communities); - - Community::delete(pool, inserted_community.id).await?; - Site::delete(pool, inserted_site.id).await?; - Person::delete(pool, inserted_person.id).await?; - Instance::delete(pool, inserted_instance.id).await?; - - Ok(()) - } -} diff --git a/crates/db_schema/src/aggregates/structs.rs b/crates/db_schema/src/aggregates/structs.rs index 096cfa55ad..9c815a68e9 100644 --- a/crates/db_schema/src/aggregates/structs.rs +++ b/crates/db_schema/src/aggregates/structs.rs @@ -1,11 +1,10 @@ -use crate::newtypes::{PersonId, PostId, SiteId}; +use crate::newtypes::{PersonId, PostId}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; #[cfg(feature = "full")] use { - crate::schema::{post_actions, site_aggregates}, + crate::schema::post_actions, diesel::{dsl, expression_methods::NullableExpressionMethods}, - ts_rs::TS, }; #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)] @@ -41,30 +40,3 @@ pub struct PersonPostAggregatesForm { #[cfg_attr(feature = "full", diesel(column_name = read_comments_amount))] pub read_comments: i64, } - -#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone, Copy, Hash)] -#[cfg_attr( - feature = "full", - derive(Queryable, Selectable, Associations, Identifiable, TS) -)] -#[cfg_attr(feature = "full", diesel(table_name = site_aggregates))] -#[cfg_attr(feature = "full", diesel(belongs_to(crate::source::site::Site)))] -#[cfg_attr(feature = "full", diesel(primary_key(site_id)))] -#[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))] -#[cfg_attr(feature = "full", ts(export))] -/// Aggregate data for a site. -pub struct SiteAggregates { - pub site_id: SiteId, - pub users: i64, - pub posts: i64, - pub comments: i64, - pub communities: i64, - /// The number of users with any activity in the last day. - pub users_active_day: i64, - /// The number of users with any activity in the last week. - pub users_active_week: i64, - /// The number of users with any activity in the last month. - pub users_active_month: i64, - /// The number of users with any activity in the last half year. - pub users_active_half_year: i64, -} diff --git a/crates/db_schema/src/impls/local_site.rs b/crates/db_schema/src/impls/local_site.rs index bdbe4ac6ca..2846340d97 100644 --- a/crates/db_schema/src/impls/local_site.rs +++ b/crates/db_schema/src/impls/local_site.rs @@ -39,3 +39,192 @@ impl LocalSite { diesel::delete(local_site::table).execute(conn).await } } + +#[cfg(test)] +mod tests { + + use super::*; + use crate::{ + source::{ + comment::{Comment, CommentInsertForm}, + community::{Community, CommunityInsertForm, CommunityUpdateForm}, + instance::Instance, + person::{Person, PersonInsertForm}, + post::{Post, PostInsertForm}, + site::{Site, SiteInsertForm}, + }, + traits::Crud, + utils::{build_db_pool_for_tests, DbPool}, + }; + use pretty_assertions::assert_eq; + use serial_test::serial; + + async fn prepare_site_with_community( + pool: &mut DbPool<'_>, + ) -> LemmyResult<(Instance, Person, Site, Community)> { + let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; + + let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_site_agg"); + + let inserted_person = Person::create(pool, &new_person).await?; + + let site_form = SiteInsertForm::new("test_site".into(), inserted_instance.id); + let inserted_site = Site::create(pool, &site_form).await?; + + let new_community = CommunityInsertForm::new( + inserted_instance.id, + "TIL_site_agg".into(), + "nada".to_owned(), + "pubkey".to_string(), + ); + + let inserted_community = Community::create(pool, &new_community).await?; + + Ok(( + inserted_instance, + inserted_person, + inserted_site, + inserted_community, + )) + } + + #[tokio::test] + #[serial] + async fn test_aggregates() -> LemmyResult<()> { + let pool = &build_db_pool_for_tests(); + let pool = &mut pool.into(); + + let (inserted_instance, inserted_person, inserted_site, inserted_community) = + prepare_site_with_community(pool).await?; + + let new_post = PostInsertForm::new( + "A test post".into(), + inserted_person.id, + inserted_community.id, + ); + + // Insert two of those posts + let inserted_post = Post::create(pool, &new_post).await?; + let _inserted_post_again = Post::create(pool, &new_post).await?; + + let comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + + // Insert two of those comments + let inserted_comment = Comment::create(pool, &comment_form, None).await?; + + let child_comment_form = CommentInsertForm::new( + inserted_person.id, + inserted_post.id, + "A test comment".into(), + ); + let _inserted_child_comment = + Comment::create(pool, &child_comment_form, Some(&inserted_comment.path)).await?; + + let site_aggregates_before_delete = LocalSite::read(pool).await?; + + // TODO: this is unstable, sometimes it returns 0 users, sometimes 1 + //assert_eq!(0, site_aggregates_before_delete.users); + assert_eq!(1, site_aggregates_before_delete.communities); + assert_eq!(2, site_aggregates_before_delete.posts); + assert_eq!(2, site_aggregates_before_delete.comments); + + // Try a post delete + Post::delete(pool, inserted_post.id).await?; + let site_aggregates_after_post_delete = LocalSite::read(pool).await?; + assert_eq!(1, site_aggregates_after_post_delete.posts); + assert_eq!(0, site_aggregates_after_post_delete.comments); + + // This shouuld delete all the associated rows, and fire triggers + let person_num_deleted = Person::delete(pool, inserted_person.id).await?; + assert_eq!(1, person_num_deleted); + + // Delete the community + let community_num_deleted = Community::delete(pool, inserted_community.id).await?; + assert_eq!(1, community_num_deleted); + + // Site should still exist, it can without a site creator. + let after_delete_creator = LocalSite::read(pool).await; + assert!(after_delete_creator.is_ok()); + + Site::delete(pool, inserted_site.id).await?; + let after_delete_site = LocalSite::read(pool).await; + assert!(after_delete_site.is_err()); + + Instance::delete(pool, inserted_instance.id).await?; + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_soft_delete() -> LemmyResult<()> { + let pool = &build_db_pool_for_tests(); + let pool = &mut pool.into(); + + let (inserted_instance, inserted_person, inserted_site, inserted_community) = + prepare_site_with_community(pool).await?; + + let site_aggregates_before = LocalSite::read(pool).await?; + assert_eq!(1, site_aggregates_before.communities); + + Community::update( + pool, + inserted_community.id, + &CommunityUpdateForm { + deleted: Some(true), + ..Default::default() + }, + ) + .await?; + + let site_aggregates_after_delete = LocalSite::read(pool).await?; + assert_eq!(0, site_aggregates_after_delete.communities); + + Community::update( + pool, + inserted_community.id, + &CommunityUpdateForm { + deleted: Some(false), + ..Default::default() + }, + ) + .await?; + + Community::update( + pool, + inserted_community.id, + &CommunityUpdateForm { + removed: Some(true), + ..Default::default() + }, + ) + .await?; + + let site_aggregates_after_remove = LocalSite::read(pool).await?; + assert_eq!(0, site_aggregates_after_remove.communities); + + Community::update( + pool, + inserted_community.id, + &CommunityUpdateForm { + deleted: Some(true), + ..Default::default() + }, + ) + .await?; + + let site_aggregates_after_remove_delete = LocalSite::read(pool).await?; + assert_eq!(0, site_aggregates_after_remove_delete.communities); + + Community::delete(pool, inserted_community.id).await?; + Site::delete(pool, inserted_site.id).await?; + Person::delete(pool, inserted_person.id).await?; + Instance::delete(pool, inserted_instance.id).await?; + + Ok(()) + } +} diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 73ac5c374f..58c3af50cc 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -434,6 +434,14 @@ diesel::table! { comment_downvotes -> FederationModeEnum, disable_donation_dialog -> Bool, default_post_time_range_seconds -> Nullable, + users -> Int8, + posts -> Int8, + comments -> Int8, + communities -> Int8, + users_active_day -> Int8, + users_active_week -> Int8, + users_active_month -> Int8, + users_active_half_year -> Int8, } } @@ -1031,20 +1039,6 @@ diesel::table! { } } -diesel::table! { - site_aggregates (site_id) { - site_id -> Int4, - users -> Int8, - posts -> Int8, - comments -> Int8, - communities -> Int8, - users_active_day -> Int8, - users_active_week -> Int8, - users_active_month -> Int8, - users_active_half_year -> Int8, - } -} - diesel::table! { site_language (site_id, language_id) { site_id -> Int4, @@ -1181,7 +1175,6 @@ diesel::joinable!(search_combined -> community (community_id)); diesel::joinable!(search_combined -> person (person_id)); diesel::joinable!(search_combined -> post (post_id)); diesel::joinable!(site -> instance (instance_id)); -diesel::joinable!(site_aggregates -> site (site_id)); diesel::joinable!(site_language -> language (language_id)); diesel::joinable!(site_language -> site (site_id)); diesel::joinable!(tag -> community (community_id)); @@ -1258,7 +1251,6 @@ diesel::allow_tables_to_appear_in_same_query!( secret, sent_activity, site, - site_aggregates, site_language, tag, tagline, diff --git a/crates/db_schema/src/source/local_site.rs b/crates/db_schema/src/source/local_site.rs index 78d5da2f34..bbfb555829 100644 --- a/crates/db_schema/src/source/local_site.rs +++ b/crates/db_schema/src/source/local_site.rs @@ -89,6 +89,18 @@ pub struct LocalSite { #[cfg_attr(feature = "full", ts(optional))] /// A default time range limit to apply to post sorts, in seconds. pub default_post_time_range_seconds: Option, + pub users: i64, + pub posts: i64, + pub comments: i64, + pub communities: i64, + /// The number of users with any activity in the last day. + pub users_active_day: i64, + /// The number of users with any activity in the last week. + pub users_active_week: i64, + /// The number of users with any activity in the last month. + pub users_active_month: i64, + /// The number of users with any activity in the last half year. + pub users_active_half_year: i64, } #[derive(Clone, derive_new::new)] diff --git a/crates/db_views/src/site/site_view.rs b/crates/db_views/src/site/site_view.rs index ecf730e883..86e1dc962e 100644 --- a/crates/db_views/src/site/site_view.rs +++ b/crates/db_views/src/site/site_view.rs @@ -2,7 +2,7 @@ use crate::structs::SiteView; use diesel::{ExpressionMethods, JoinOnDsl, OptionalExtension, QueryDsl, SelectableHelper}; use diesel_async::RunQueryDsl; use lemmy_db_schema::{ - schema::{local_site, local_site_rate_limit, site, site_aggregates}, + schema::{local_site, local_site_rate_limit, site}, utils::{get_conn, DbPool}, }; use lemmy_utils::error::{LemmyErrorType, LemmyResult}; @@ -16,7 +16,6 @@ impl SiteView { .inner_join( local_site_rate_limit::table.on(local_site::id.eq(local_site_rate_limit::local_site_id)), ) - .inner_join(site_aggregates::table) .select(Self::as_select()) .first(conn) .await diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index c225f1baa3..6a3e4a27bc 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -16,7 +16,6 @@ use diesel::{ Selectable, }; use lemmy_db_schema::{ - aggregates::structs::SiteAggregates, source::{ comment::Comment, comment_reply::CommentReply, @@ -392,8 +391,6 @@ pub struct SiteView { pub local_site: LocalSite, #[cfg_attr(feature = "full", diesel(embed))] pub local_site_rate_limit: LocalSiteRateLimit, - #[cfg_attr(feature = "full", diesel(embed))] - pub counts: SiteAggregates, } #[derive(Debug, Serialize, Deserialize, Clone)] diff --git a/crates/routes/src/nodeinfo.rs b/crates/routes/src/nodeinfo.rs index e5b183a0b8..b6401d5239 100644 --- a/crates/routes/src/nodeinfo.rs +++ b/crates/routes/src/nodeinfo.rs @@ -59,12 +59,12 @@ async fn node_info(context: web::Data) -> Result Date: Wed, 12 Feb 2025 16:34:18 +0100 Subject: [PATCH 29/39] triggers, tests --- .../db_schema/replaceable_schema/triggers.sql | 29 +++---------------- crates/db_schema/src/impls/local_site.rs | 3 ++ crates/db_views/src/structs.rs | 20 ++++++------- 3 files changed, 17 insertions(+), 35 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index c17c95e5b0..5ee13b3f89 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -189,7 +189,7 @@ WHERE AND diff.comments != 0; UPDATE - site_aggregates AS a + local_site AS a SET comments = a.comments + diff.comments FROM ( @@ -245,7 +245,7 @@ WHERE AND diff.posts != 0; UPDATE - site_aggregates AS a + local_site AS a SET posts = a.posts + diff.posts FROM ( @@ -268,7 +268,7 @@ $$); CALL r.create_triggers ('community', $$ BEGIN UPDATE - site_aggregates AS a + local_site AS a SET communities = a.communities + diff.communities FROM ( @@ -290,7 +290,7 @@ $$); CALL r.create_triggers ('person', $$ BEGIN UPDATE - site_aggregates AS a + local_site AS a SET users = a.users + diff.users FROM ( @@ -454,27 +454,6 @@ CREATE TRIGGER aggregates WHEN (pg_trigger_depth() = 0) EXECUTE FUNCTION r.post_from_post (); -CREATE FUNCTION r.site_aggregates_from_site () - RETURNS TRIGGER - LANGUAGE plpgsql - AS $$ -BEGIN - -- only 1 row can be in site_aggregates because of the index idx_site_aggregates_1_row_only. - -- we only ever want to have a single value in site_aggregate because the site_aggregate triggers update all rows in that table. - -- a cleaner check would be to insert it for the local_site but that would break assumptions at least in the tests - INSERT INTO site_aggregates (site_id) - VALUES (NEW.id) - ON CONFLICT ((TRUE)) - DO NOTHING; - RETURN NULL; -END; -$$; - -CREATE TRIGGER aggregates - AFTER INSERT ON site - FOR EACH ROW - EXECUTE FUNCTION r.site_aggregates_from_site (); - -- Change the order of some cascading deletions to make deletion triggers run before the deletion of rows that the triggers need to read CREATE FUNCTION r.delete_comments_before_post () RETURNS TRIGGER diff --git a/crates/db_schema/src/impls/local_site.rs b/crates/db_schema/src/impls/local_site.rs index 2846340d97..85ba82bf96 100644 --- a/crates/db_schema/src/impls/local_site.rs +++ b/crates/db_schema/src/impls/local_site.rs @@ -71,6 +71,9 @@ mod tests { let site_form = SiteInsertForm::new("test_site".into(), inserted_instance.id); let inserted_site = Site::create(pool, &site_form).await?; + let local_site_form = LocalSiteInsertForm::new(inserted_site.id); + LocalSite::create(pool, &local_site_form).await?; + let new_community = CommunityInsertForm::new( inserted_instance.id, "TIL_site_agg".into(), diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index 6a3e4a27bc..6335930870 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -15,6 +15,16 @@ use diesel::{ Queryable, Selectable, }; +#[cfg(feature = "full")] +use lemmy_db_schema::{ + aliases::{creator_community_actions, creator_local_user, person1}, + impls::comment::comment_select_remove_deletes, + impls::community::community_follower_select_subscribed_type, + impls::local_user::local_user_can_mod, + schema::{comment, comment_actions, community_actions, local_user, person, person_actions}, + utils::functions::coalesce, + Person1AliasAllColumnsTuple, +}; use lemmy_db_schema::{ source::{ comment::Comment, @@ -66,16 +76,6 @@ use lemmy_db_schema::{ }, SubscribedType, }; -#[cfg(feature = "full")] -use lemmy_db_schema::{ - aliases::{creator_community_actions, creator_local_user, person1}, - impls::comment::comment_select_remove_deletes, - impls::community::community_follower_select_subscribed_type, - impls::local_user::local_user_can_mod, - schema::{comment, comment_actions, community_actions, local_user, person, person_actions}, - utils::functions::coalesce, - Person1AliasAllColumnsTuple, -}; use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; #[cfg(feature = "full")] From ac7b305f8b76d5d87ad515e79dc0b9b075c5b28c Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 12 Feb 2025 16:42:27 +0100 Subject: [PATCH 30/39] more fixes --- api_tests/package.json | 2 +- api_tests/pnpm-lock.yaml | 10 +++++----- crates/routes/src/utils/scheduled_tasks.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api_tests/package.json b/api_tests/package.json index 6535d81709..6f7c49188e 100644 --- a/api_tests/package.json +++ b/api_tests/package.json @@ -29,7 +29,7 @@ "eslint": "^9.20.0", "eslint-plugin-prettier": "^5.2.3", "jest": "^29.5.0", - "lemmy-js-client": "0.20.0-remove-aggregate-tables.1", + "lemmy-js-client": "0.20.0-remove-aggregate-tables.2", "prettier": "^3.5.0", "ts-jest": "^29.1.0", "tsoa": "^6.6.0", diff --git a/api_tests/pnpm-lock.yaml b/api_tests/pnpm-lock.yaml index 389e05d0eb..b65d1a03f1 100644 --- a/api_tests/pnpm-lock.yaml +++ b/api_tests/pnpm-lock.yaml @@ -33,8 +33,8 @@ importers: specifier: ^29.5.0 version: 29.7.0(@types/node@22.13.1) lemmy-js-client: - specifier: 0.20.0-remove-aggregate-tables.1 - version: 0.20.0-remove-aggregate-tables.1 + specifier: 0.20.0-remove-aggregate-tables.2 + version: 0.20.0-remove-aggregate-tables.2 prettier: specifier: ^3.5.0 version: 3.5.0 @@ -1528,8 +1528,8 @@ packages: resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} engines: {node: '>=6'} - lemmy-js-client@0.20.0-remove-aggregate-tables.1: - resolution: {integrity: sha512-O+zsHAEijbrT6da+x5HZAdb8WqDnoCVUQOuG7qsdJGc/ZlkUnyr0n0isuJDCmP3SOy2JnrGIUD1vUSJVMwMtrg==} + lemmy-js-client@0.20.0-remove-aggregate-tables.2: + resolution: {integrity: sha512-ai5OVMGANm4VcxnnZqi9cz4U0ir9YvMpl47D2RT/MErRQNUruoZGl+p5yCPaqF7rWhTd/T/oAJiNBZSJ1HLwJA==} leven@3.1.0: resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} @@ -4169,7 +4169,7 @@ snapshots: kleur@3.0.3: {} - lemmy-js-client@0.20.0-remove-aggregate-tables.1: {} + lemmy-js-client@0.20.0-remove-aggregate-tables.2: {} leven@3.1.0: {} diff --git a/crates/routes/src/utils/scheduled_tasks.rs b/crates/routes/src/utils/scheduled_tasks.rs index 61e31f9b32..de7348b942 100644 --- a/crates/routes/src/utils/scheduled_tasks.rs +++ b/crates/routes/src/utils/scheduled_tasks.rs @@ -366,7 +366,7 @@ async fn active_counts(pool: &mut DbPool<'_>) -> LemmyResult<()> { for (full_form, abbr) in &intervals { let update_site_stmt = format!( - "update site_aggregates set users_active_{} = (select r.site_aggregates_activity('{}')) where site_id = 1", + "update local_site set users_active_{} = (select r.site_aggregates_activity('{}')) where site_id = 1", abbr, full_form ); sql_query(update_site_stmt).execute(&mut conn).await?; From f732f63da5b5fef2d6c08ddb58b666d023e6232e Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Thu, 13 Feb 2025 10:53:29 +0100 Subject: [PATCH 31/39] Rename PersonPostAggregates to PostActions --- crates/api_common/src/utils.rs | 6 +++--- crates/db_schema/src/aggregates/mod.rs | 3 --- crates/db_schema/src/impls/mod.rs | 1 + .../person_post_aggregates.rs => impls/post_actions.rs} | 9 +++------ crates/db_schema/src/lib.rs | 1 - crates/db_schema/src/source/mod.rs | 1 + .../{aggregates/structs.rs => source/post_actions.rs} | 4 ++-- 7 files changed, 10 insertions(+), 15 deletions(-) delete mode 100644 crates/db_schema/src/aggregates/mod.rs rename crates/db_schema/src/{aggregates/person_post_aggregates.rs => impls/post_actions.rs} (82%) rename crates/db_schema/src/{aggregates/structs.rs => source/post_actions.rs} (96%) diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index a7fd73f110..caf4be8f64 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -13,7 +13,6 @@ use actix_web_httpauth::headers::authorization::{Authorization, Bearer}; use chrono::{DateTime, Days, Local, TimeZone, Utc}; use enum_map::{enum_map, EnumMap}; use lemmy_db_schema::{ - aggregates::structs::{PersonPostAggregates, PersonPostAggregatesForm}, newtypes::{CommentId, CommunityId, DbUrl, InstanceId, PersonId, PostId, PostOrCommentId}, source::{ comment::{Comment, CommentLike, CommentUpdateForm}, @@ -38,6 +37,7 @@ use lemmy_db_schema::{ person::{Person, PersonUpdateForm}, person_block::PersonBlock, post::{Post, PostLike}, + post_actions::{PostActions, PostActionsForm}, private_message::PrivateMessage, registration_application::RegistrationApplication, site::Site, @@ -158,13 +158,13 @@ pub async fn update_read_comments( read_comments: i64, pool: &mut DbPool<'_>, ) -> LemmyResult<()> { - let person_post_agg_form = PersonPostAggregatesForm { + let person_post_agg_form = PostActionsForm { person_id, post_id, read_comments, }; - PersonPostAggregates::upsert(pool, &person_post_agg_form).await?; + PostActions::upsert(pool, &person_post_agg_form).await?; Ok(()) } diff --git a/crates/db_schema/src/aggregates/mod.rs b/crates/db_schema/src/aggregates/mod.rs deleted file mode 100644 index 6a8ef5ad3b..0000000000 --- a/crates/db_schema/src/aggregates/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -#[cfg(feature = "full")] -pub mod person_post_aggregates; -pub mod structs; diff --git a/crates/db_schema/src/impls/mod.rs b/crates/db_schema/src/impls/mod.rs index f6a01f06a8..d87d1746d0 100644 --- a/crates/db_schema/src/impls/mod.rs +++ b/crates/db_schema/src/impls/mod.rs @@ -31,6 +31,7 @@ pub mod person_block; pub mod person_comment_mention; pub mod person_post_mention; pub mod post; +pub mod post_actions; pub mod post_report; pub mod private_message; pub mod private_message_report; diff --git a/crates/db_schema/src/aggregates/person_post_aggregates.rs b/crates/db_schema/src/impls/post_actions.rs similarity index 82% rename from crates/db_schema/src/aggregates/person_post_aggregates.rs rename to crates/db_schema/src/impls/post_actions.rs index 9f40b71b8f..ac3ef22e77 100644 --- a/crates/db_schema/src/aggregates/person_post_aggregates.rs +++ b/crates/db_schema/src/impls/post_actions.rs @@ -1,8 +1,8 @@ use crate::{ - aggregates::structs::{PersonPostAggregates, PersonPostAggregatesForm}, diesel::OptionalExtension, newtypes::{PersonId, PostId}, schema::post_actions, + source::post_actions::{PostActions, PostActionsForm}, utils::{get_conn, now, DbPool}, }; use diesel::{ @@ -15,11 +15,8 @@ use diesel::{ }; use diesel_async::RunQueryDsl; -impl PersonPostAggregates { - pub async fn upsert( - pool: &mut DbPool<'_>, - form: &PersonPostAggregatesForm, - ) -> Result { +impl PostActions { + pub async fn upsert(pool: &mut DbPool<'_>, form: &PostActionsForm) -> Result { let conn = &mut get_conn(pool).await?; let form = (form, post_actions::read_comments.eq(now().nullable())); insert_into(post_actions::table) diff --git a/crates/db_schema/src/lib.rs b/crates/db_schema/src/lib.rs index 521bb12ac2..f9f8ab299e 100644 --- a/crates/db_schema/src/lib.rs +++ b/crates/db_schema/src/lib.rs @@ -13,7 +13,6 @@ extern crate diesel_derive_enum; #[macro_use] extern crate async_trait; -pub mod aggregates; #[cfg(feature = "full")] pub mod impls; pub mod newtypes; diff --git a/crates/db_schema/src/source/mod.rs b/crates/db_schema/src/source/mod.rs index c34be3726e..ecf475bd00 100644 --- a/crates/db_schema/src/source/mod.rs +++ b/crates/db_schema/src/source/mod.rs @@ -37,6 +37,7 @@ pub mod person_block; pub mod person_comment_mention; pub mod person_post_mention; pub mod post; +pub mod post_actions; pub mod post_report; pub mod private_message; pub mod private_message_report; diff --git a/crates/db_schema/src/aggregates/structs.rs b/crates/db_schema/src/source/post_actions.rs similarity index 96% rename from crates/db_schema/src/aggregates/structs.rs rename to crates/db_schema/src/source/post_actions.rs index 9c815a68e9..3c43789517 100644 --- a/crates/db_schema/src/aggregates/structs.rs +++ b/crates/db_schema/src/source/post_actions.rs @@ -17,7 +17,7 @@ use { #[cfg_attr(feature = "full", diesel(belongs_to(crate::source::person::Person)))] #[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))] /// Aggregate data for a person's post. -pub struct PersonPostAggregates { +pub struct PostActions { pub person_id: PersonId, pub post_id: PostId, /// The number of comments they've read on that post. @@ -34,7 +34,7 @@ pub struct PersonPostAggregates { #[derive(Clone, Default)] #[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] #[cfg_attr(feature = "full", diesel(table_name = post_actions))] -pub struct PersonPostAggregatesForm { +pub struct PostActionsForm { pub person_id: PersonId, pub post_id: PostId, #[cfg_attr(feature = "full", diesel(column_name = read_comments_amount))] From 25aa10a5077a59ca4e8d8a5a7586d9d75e3b21fd Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Thu, 13 Feb 2025 11:26:56 +0100 Subject: [PATCH 32/39] Merge local_user_vote_display_mode into local_user --- api_tests/package.json | 2 +- api_tests/pnpm-lock.yaml | 10 +- crates/api/src/local_user/save_settings.rs | 16 +-- crates/apub/src/api/user_settings_backup.rs | 28 +--- .../db_schema/replaceable_schema/triggers.sql | 4 +- crates/db_schema/src/impls/local_user.rs | 5 - .../src/impls/local_user_vote_display_mode.rs | 60 -------- crates/db_schema/src/impls/mod.rs | 1 - crates/db_schema/src/schema.rs | 16 +-- crates/db_schema/src/source/local_user.rs | 16 +++ .../source/local_user_vote_display_mode.rs | 53 -------- crates/db_schema/src/source/mod.rs | 1 - .../combined/person_saved_combined_view.rs | 2 - .../src/combined/report_combined_view.rs | 3 - .../src/combined/search_combined_view.rs | 2 - crates/db_views/src/comment/comment_view.rs | 2 - .../src/local_user/local_user_view.rs | 6 +- crates/db_views/src/post/post_view.rs | 4 - .../registration_application_view.rs | 128 +++++++++--------- crates/db_views/src/structs.rs | 3 - .../down.sql | 27 +++- .../up.sql | 21 +++ 22 files changed, 150 insertions(+), 260 deletions(-) delete mode 100644 crates/db_schema/src/impls/local_user_vote_display_mode.rs delete mode 100644 crates/db_schema/src/source/local_user_vote_display_mode.rs diff --git a/api_tests/package.json b/api_tests/package.json index ad7a1a9ee4..425dc4b6ec 100644 --- a/api_tests/package.json +++ b/api_tests/package.json @@ -29,7 +29,7 @@ "eslint": "^9.20.0", "eslint-plugin-prettier": "^5.2.3", "jest": "^29.5.0", - "lemmy-js-client": "0.20.0-remove-aggregate-tables.3", + "lemmy-js-client": "0.20.0-remove-aggregate-tables.4", "prettier": "^3.5.0", "ts-jest": "^29.1.0", "tsoa": "^6.6.0", diff --git a/api_tests/pnpm-lock.yaml b/api_tests/pnpm-lock.yaml index 2c8d1651c6..451796ac80 100644 --- a/api_tests/pnpm-lock.yaml +++ b/api_tests/pnpm-lock.yaml @@ -33,8 +33,8 @@ importers: specifier: ^29.5.0 version: 29.7.0(@types/node@22.13.1) lemmy-js-client: - specifier: 0.20.0-remove-aggregate-tables.3 - version: 0.20.0-remove-aggregate-tables.3 + specifier: 0.20.0-remove-aggregate-tables.4 + version: 0.20.0-remove-aggregate-tables.4 prettier: specifier: ^3.5.0 version: 3.5.0 @@ -1528,8 +1528,8 @@ packages: resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} engines: {node: '>=6'} - lemmy-js-client@0.20.0-remove-aggregate-tables.3: - resolution: {integrity: sha512-4zOHRaqkA+YHKSsweicPD7kg2wydgJfmcplWKNW1TL3hDeCFwdRb0Il/7ohd/cG358nXYfBSqhW1V8GJc6lWqQ==} + lemmy-js-client@0.20.0-remove-aggregate-tables.4: + resolution: {integrity: sha512-W5s2vEFMOkieVNrcUbeXoAqId6/GBVLFKzlJGTzMpAkmENzEutTXGSa683vbCrofj8aV+Eg4AAZZzrKYLet05w==} leven@3.1.0: resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} @@ -4169,7 +4169,7 @@ snapshots: kleur@3.0.3: {} - lemmy-js-client@0.20.0-remove-aggregate-tables.3: {} + lemmy-js-client@0.20.0-remove-aggregate-tables.4: {} leven@3.1.0: {} diff --git a/crates/api/src/local_user/save_settings.rs b/crates/api/src/local_user/save_settings.rs index 289012c251..69da6e04b2 100644 --- a/crates/api/src/local_user/save_settings.rs +++ b/crates/api/src/local_user/save_settings.rs @@ -15,7 +15,6 @@ use lemmy_db_schema::{ source::{ actor_language::LocalUserLanguage, local_user::{LocalUser, LocalUserUpdateForm}, - local_user_vote_display_mode::{LocalUserVoteDisplayMode, LocalUserVoteDisplayModeUpdateForm}, person::{Person, PersonUpdateForm}, }, traits::Crud, @@ -137,20 +136,15 @@ pub async fn save_user_settings( collapse_bot_comments: data.collapse_bot_comments, auto_mark_fetched_posts_as_read: data.auto_mark_fetched_posts_as_read, hide_media: data.hide_media, + // Update the vote display modes + show_score: data.show_scores, + show_upvotes: data.show_upvotes, + show_downvotes: data.show_downvotes, + show_upvote_percentage: data.show_upvote_percentage, ..Default::default() }; LocalUser::update(&mut context.pool(), local_user_id, &local_user_form).await?; - // Update the vote display modes - let vote_display_modes_form = LocalUserVoteDisplayModeUpdateForm { - score: data.show_scores, - upvotes: data.show_upvotes, - downvotes: data.show_downvotes, - upvote_percentage: data.show_upvote_percentage, - }; - LocalUserVoteDisplayMode::update(&mut context.pool(), local_user_id, &vote_display_modes_form) - .await?; - Ok(Json(SuccessResponse::default())) } diff --git a/crates/apub/src/api/user_settings_backup.rs b/crates/apub/src/api/user_settings_backup.rs index 8bfeb813ce..2dcdcf3df4 100644 --- a/crates/apub/src/api/user_settings_backup.rs +++ b/crates/apub/src/api/user_settings_backup.rs @@ -18,7 +18,6 @@ use lemmy_db_schema::{ instance::Instance, instance_block::{InstanceBlock, InstanceBlockForm}, local_user::{LocalUser, LocalUserUpdateForm}, - local_user_vote_display_mode::{LocalUserVoteDisplayMode, LocalUserVoteDisplayModeUpdateForm}, person::{Person, PersonUpdateForm}, person_block::{PersonBlock, PersonBlockForm}, post::{PostSaved, PostSavedForm}, @@ -55,7 +54,6 @@ pub struct UserSettingsBackup { // TODO: might be worth making a separate struct for settings backup, to avoid breakage in case // fields are renamed, and to avoid storing unnecessary fields like person_id or email pub settings: Option, - pub vote_display_mode_settings: Option, #[serde(default)] pub followed_communities: Vec>, #[serde(default)] @@ -85,7 +83,6 @@ pub async fn export_settings( matrix_id: local_user_view.person.matrix_user_id, bot_account: local_user_view.person.bot_account.into(), settings: Some(local_user_view.local_user), - vote_display_mode_settings: Some(local_user_view.local_user_vote_display_mode), followed_communities: vec_into(lists.followed_communities), blocked_communities: vec_into(lists.blocked_communities), blocked_instances: lists.blocked_instances, @@ -130,6 +127,10 @@ pub async fn import_settings( blur_nsfw: data.settings.as_ref().map(|s| s.blur_nsfw), infinite_scroll_enabled: data.settings.as_ref().map(|s| s.infinite_scroll_enabled), post_listing_mode: data.settings.as_ref().map(|s| s.post_listing_mode), + show_score: data.settings.as_ref().map(|s| s.show_score), + show_upvotes: data.settings.as_ref().map(|s| s.show_upvotes), + show_downvotes: data.settings.as_ref().map(|s| s.show_downvotes), + show_upvote_percentage: data.settings.as_ref().map(|s| s.show_upvote_percentage), ..Default::default() }; LocalUser::update( @@ -139,27 +140,6 @@ pub async fn import_settings( ) .await?; - // Update the vote display mode settings - let vote_display_mode_form = LocalUserVoteDisplayModeUpdateForm { - score: data.vote_display_mode_settings.as_ref().map(|s| s.score), - upvotes: data.vote_display_mode_settings.as_ref().map(|s| s.upvotes), - downvotes: data - .vote_display_mode_settings - .as_ref() - .map(|s| s.downvotes), - upvote_percentage: data - .vote_display_mode_settings - .as_ref() - .map(|s| s.upvote_percentage), - }; - - LocalUserVoteDisplayMode::update( - &mut context.pool(), - local_user_view.local_user.id, - &vote_display_mode_form, - ) - .await?; - let url_count = data.followed_communities.len() + data.blocked_communities.len() + data.blocked_users.len() diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index ad8b13decc..b86de3ca46 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -383,7 +383,7 @@ BEGIN AND NOT (post_report).violates_instance_rules), 0) AS unresolved_report_count FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (post_report).post_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) - AND a.id = diff.post_id; +AND a.id = diff.post_id; RETURN NULL; @@ -403,7 +403,7 @@ BEGIN AND NOT (comment_report).violates_instance_rules), 0) AS unresolved_report_count FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (comment_report).comment_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) - AND a.id = diff.comment_id; +AND a.id = diff.comment_id; RETURN NULL; diff --git a/crates/db_schema/src/impls/local_user.rs b/crates/db_schema/src/impls/local_user.rs index 44978b1903..e643427c18 100644 --- a/crates/db_schema/src/impls/local_user.rs +++ b/crates/db_schema/src/impls/local_user.rs @@ -5,7 +5,6 @@ use crate::{ source::{ actor_language::LocalUserLanguage, local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm}, - local_user_vote_display_mode::{LocalUserVoteDisplayMode, LocalUserVoteDisplayModeInsertForm}, site::Site, }, utils::{ @@ -55,10 +54,6 @@ impl LocalUser { LocalUserLanguage::update(pool, languages, local_user_.id).await?; - // Create their vote_display_modes - let vote_display_mode_form = LocalUserVoteDisplayModeInsertForm::new(local_user_.id); - LocalUserVoteDisplayMode::create(pool, &vote_display_mode_form).await?; - Ok(local_user_) } diff --git a/crates/db_schema/src/impls/local_user_vote_display_mode.rs b/crates/db_schema/src/impls/local_user_vote_display_mode.rs deleted file mode 100644 index 2d169f81bc..0000000000 --- a/crates/db_schema/src/impls/local_user_vote_display_mode.rs +++ /dev/null @@ -1,60 +0,0 @@ -use crate::{ - diesel::OptionalExtension, - newtypes::LocalUserId, - schema::local_user_vote_display_mode, - source::local_user_vote_display_mode::{ - LocalUserVoteDisplayMode, - LocalUserVoteDisplayModeInsertForm, - LocalUserVoteDisplayModeUpdateForm, - }, - utils::{get_conn, DbPool}, -}; -use diesel::{dsl::insert_into, result::Error, QueryDsl}; -use diesel_async::RunQueryDsl; - -impl LocalUserVoteDisplayMode { - pub async fn read(pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - local_user_vote_display_mode::table - .first(conn) - .await - .optional() - } - - pub async fn create( - pool: &mut DbPool<'_>, - form: &LocalUserVoteDisplayModeInsertForm, - ) -> Result { - let conn = &mut get_conn(pool).await?; - insert_into(local_user_vote_display_mode::table) - .values(form) - .get_result::(conn) - .await - } - - pub async fn update( - pool: &mut DbPool<'_>, - local_user_id: LocalUserId, - form: &LocalUserVoteDisplayModeUpdateForm, - ) -> Result<(), Error> { - // avoid error "There are no changes to save. This query cannot be built" - if form.is_empty() { - return Ok(()); - } - let conn = &mut get_conn(pool).await?; - diesel::update(local_user_vote_display_mode::table.find(local_user_id)) - .set(form) - .get_result::(conn) - .await?; - Ok(()) - } -} - -impl LocalUserVoteDisplayModeUpdateForm { - fn is_empty(&self) -> bool { - self.score.is_none() - && self.upvotes.is_none() - && self.downvotes.is_none() - && self.upvote_percentage.is_none() - } -} diff --git a/crates/db_schema/src/impls/mod.rs b/crates/db_schema/src/impls/mod.rs index d87d1746d0..d384afdd0c 100644 --- a/crates/db_schema/src/impls/mod.rs +++ b/crates/db_schema/src/impls/mod.rs @@ -20,7 +20,6 @@ pub mod local_site; pub mod local_site_rate_limit; pub mod local_site_url_blocklist; pub mod local_user; -pub mod local_user_vote_display_mode; pub mod login_token; pub mod mod_log; pub mod oauth_account; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 34a9933873..6c7727c665 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -517,6 +517,10 @@ diesel::table! { last_donation_notification -> Timestamptz, hide_media -> Bool, default_post_time_range_seconds -> Nullable, + show_score -> Bool, + show_upvotes -> Bool, + show_downvotes -> Bool, + show_upvote_percentage -> Bool, } } @@ -527,16 +531,6 @@ diesel::table! { } } -diesel::table! { - local_user_vote_display_mode (local_user_id) { - local_user_id -> Int4, - score -> Bool, - upvotes -> Bool, - downvotes -> Bool, - upvote_percentage -> Bool, - } -} - diesel::table! { login_token (token) { token -> Text, @@ -1109,7 +1103,6 @@ diesel::joinable!(local_site_rate_limit -> local_site (local_site_id)); diesel::joinable!(local_user -> person (person_id)); diesel::joinable!(local_user_language -> language (language_id)); diesel::joinable!(local_user_language -> local_user (local_user_id)); -diesel::joinable!(local_user_vote_display_mode -> local_user (local_user_id)); diesel::joinable!(login_token -> local_user (user_id)); diesel::joinable!(mod_add_community -> community (community_id)); diesel::joinable!(mod_ban_from_community -> community (community_id)); @@ -1214,7 +1207,6 @@ diesel::allow_tables_to_appear_in_same_query!( local_site_url_blocklist, local_user, local_user_language, - local_user_vote_display_mode, login_token, mod_add, mod_add_community, diff --git a/crates/db_schema/src/source/local_user.rs b/crates/db_schema/src/source/local_user.rs index 4ebb184f0d..a3adf2834b 100644 --- a/crates/db_schema/src/source/local_user.rs +++ b/crates/db_schema/src/source/local_user.rs @@ -79,6 +79,10 @@ pub struct LocalUser { #[cfg_attr(feature = "full", ts(optional))] /// A default time range limit to apply to post sorts, in seconds. pub default_post_time_range_seconds: Option, + pub show_score: bool, + pub show_upvotes: bool, + pub show_downvotes: bool, + pub show_upvote_percentage: bool, } #[derive(Clone, derive_new::new)] @@ -143,6 +147,14 @@ pub struct LocalUserInsertForm { pub hide_media: Option, #[new(default)] pub default_post_time_range_seconds: Option>, + #[new(default)] + pub show_score: Option, + #[new(default)] + pub show_upvotes: Option, + #[new(default)] + pub show_downvotes: Option, + #[new(default)] + pub show_upvote_percentage: Option, } #[derive(Clone, Default)] @@ -178,4 +190,8 @@ pub struct LocalUserUpdateForm { pub last_donation_notification: Option>, pub hide_media: Option, pub default_post_time_range_seconds: Option>, + pub show_score: Option, + pub show_upvotes: Option, + pub show_downvotes: Option, + pub show_upvote_percentage: Option, } diff --git a/crates/db_schema/src/source/local_user_vote_display_mode.rs b/crates/db_schema/src/source/local_user_vote_display_mode.rs deleted file mode 100644 index 06a4330346..0000000000 --- a/crates/db_schema/src/source/local_user_vote_display_mode.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::newtypes::LocalUserId; -#[cfg(feature = "full")] -use crate::schema::local_user_vote_display_mode; -use serde::{Deserialize, Serialize}; -use serde_with::skip_serializing_none; -#[cfg(feature = "full")] -use ts_rs::TS; - -#[skip_serializing_none] -#[derive(PartialEq, Eq, Debug, Clone, Default, Serialize, Deserialize)] -#[cfg_attr(feature = "full", derive(Queryable, Selectable, Identifiable, TS))] -#[cfg_attr(feature = "full", diesel(table_name = local_user_vote_display_mode))] -#[cfg_attr(feature = "full", diesel(primary_key(local_user_id)))] -#[cfg_attr( - feature = "full", - diesel(belongs_to(crate::source::local_site::LocalUser)) -)] -#[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))] -#[cfg_attr(feature = "full", ts(export))] -/// The vote display settings for your user. -pub struct LocalUserVoteDisplayMode { - #[serde(skip)] - pub local_user_id: LocalUserId, - pub score: bool, - pub upvotes: bool, - pub downvotes: bool, - pub upvote_percentage: bool, -} - -#[derive(Clone, derive_new::new)] -#[cfg_attr(feature = "full", derive(Insertable))] -#[cfg_attr(feature = "full", diesel(table_name = local_user_vote_display_mode))] -pub struct LocalUserVoteDisplayModeInsertForm { - pub local_user_id: LocalUserId, - #[new(default)] - pub score: Option, - #[new(default)] - pub upvotes: Option, - #[new(default)] - pub downvotes: Option, - #[new(default)] - pub upvote_percentage: Option, -} - -#[derive(Clone, Default)] -#[cfg_attr(feature = "full", derive(AsChangeset))] -#[cfg_attr(feature = "full", diesel(table_name = local_user_vote_display_mode))] -pub struct LocalUserVoteDisplayModeUpdateForm { - pub score: Option, - pub upvotes: Option, - pub downvotes: Option, - pub upvote_percentage: Option, -} diff --git a/crates/db_schema/src/source/mod.rs b/crates/db_schema/src/source/mod.rs index ecf475bd00..a9a99c433b 100644 --- a/crates/db_schema/src/source/mod.rs +++ b/crates/db_schema/src/source/mod.rs @@ -26,7 +26,6 @@ pub mod local_site; pub mod local_site_rate_limit; pub mod local_site_url_blocklist; pub mod local_user; -pub mod local_user_vote_display_mode; pub mod login_token; pub mod mod_log; pub mod oauth_account; diff --git a/crates/db_views/src/combined/person_saved_combined_view.rs b/crates/db_views/src/combined/person_saved_combined_view.rs index 4b4f220093..627fff5e66 100644 --- a/crates/db_views/src/combined/person_saved_combined_view.rs +++ b/crates/db_views/src/combined/person_saved_combined_view.rs @@ -191,7 +191,6 @@ mod tests { community::{Community, CommunityInsertForm}, instance::Instance, local_user::{LocalUser, LocalUserInsertForm}, - local_user_vote_display_mode::LocalUserVoteDisplayMode, person::{Person, PersonInsertForm}, post::{Post, PostInsertForm, PostSaved, PostSavedForm}, }, @@ -221,7 +220,6 @@ mod tests { let timmy_local_user = LocalUser::create(pool, &timmy_local_user_form, vec![]).await?; let timmy_view = LocalUserView { local_user: timmy_local_user, - local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: timmy.clone(), }; diff --git a/crates/db_views/src/combined/report_combined_view.rs b/crates/db_views/src/combined/report_combined_view.rs index ee5d425293..3e6e79b87e 100644 --- a/crates/db_views/src/combined/report_combined_view.rs +++ b/crates/db_views/src/combined/report_combined_view.rs @@ -527,7 +527,6 @@ mod tests { community_report::{CommunityReport, CommunityReportForm}, instance::Instance, local_user::{LocalUser, LocalUserInsertForm}, - local_user_vote_display_mode::LocalUserVoteDisplayMode, person::{Person, PersonInsertForm}, post::{Post, PostInsertForm}, post_report::{PostReport, PostReportForm}, @@ -564,7 +563,6 @@ mod tests { let timmy_local_user = LocalUser::create(pool, &timmy_local_user_form, vec![]).await?; let timmy_view = LocalUserView { local_user: timmy_local_user, - local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_timmy.clone(), }; @@ -575,7 +573,6 @@ mod tests { let admin_local_user = LocalUser::create(pool, &admin_local_user_form, vec![]).await?; let admin_view = LocalUserView { local_user: admin_local_user, - local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_admin.clone(), }; diff --git a/crates/db_views/src/combined/search_combined_view.rs b/crates/db_views/src/combined/search_combined_view.rs index f540a259dc..425e681d14 100644 --- a/crates/db_views/src/combined/search_combined_view.rs +++ b/crates/db_views/src/combined/search_combined_view.rs @@ -501,7 +501,6 @@ mod tests { community::{Community, CommunityInsertForm}, instance::Instance, local_user::{LocalUser, LocalUserInsertForm}, - local_user_vote_display_mode::LocalUserVoteDisplayMode, person::{Person, PersonInsertForm}, post::{Post, PostInsertForm, PostLike, PostLikeForm, PostUpdateForm}, }, @@ -542,7 +541,6 @@ mod tests { let timmy_local_user = LocalUser::create(pool, &timmy_local_user_form, vec![]).await?; let timmy_view = LocalUserView { local_user: timmy_local_user, - local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: timmy.clone(), }; diff --git a/crates/db_views/src/comment/comment_view.rs b/crates/db_views/src/comment/comment_view.rs index 25b047d818..9aa51270bf 100644 --- a/crates/db_views/src/comment/comment_view.rs +++ b/crates/db_views/src/comment/comment_view.rs @@ -348,7 +348,6 @@ mod tests { instance::Instance, language::Language, local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm}, - local_user_vote_display_mode::LocalUserVoteDisplayMode, person::{Person, PersonInsertForm}, person_block::{PersonBlock, PersonBlockForm}, post::{Post, PostInsertForm, PostUpdateForm}, @@ -503,7 +502,6 @@ mod tests { let timmy_local_user_view = LocalUserView { local_user: inserted_timmy_local_user.clone(), - local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_timmy_person.clone(), }; let site_form = SiteInsertForm::new("test site".to_string(), inserted_instance.id); diff --git a/crates/db_views/src/local_user/local_user_view.rs b/crates/db_views/src/local_user/local_user_view.rs index 7aac376cc0..416fdff40c 100644 --- a/crates/db_views/src/local_user/local_user_view.rs +++ b/crates/db_views/src/local_user/local_user_view.rs @@ -4,7 +4,7 @@ use diesel::{result::Error, BoolExpressionMethods, ExpressionMethods, QueryDsl, use diesel_async::RunQueryDsl; use lemmy_db_schema::{ newtypes::{LocalUserId, OAuthProviderId, PersonId}, - schema::{local_user, local_user_vote_display_mode, oauth_account, person}, + schema::{local_user, oauth_account, person}, source::{ instance::Instance, local_user::{LocalUser, LocalUserInsertForm}, @@ -23,9 +23,7 @@ use std::future::{ready, Ready}; impl LocalUserView { #[diesel::dsl::auto_type(no_type_alias)] fn joins() -> _ { - local_user::table - .inner_join(local_user_vote_display_mode::table) - .inner_join(person::table) + local_user::table.inner_join(person::table) } pub async fn read(pool: &mut DbPool<'_>, local_user_id: LocalUserId) -> Result { diff --git a/crates/db_views/src/post/post_view.rs b/crates/db_views/src/post/post_view.rs index b0b077ea6a..ebf41303ab 100644 --- a/crates/db_views/src/post/post_view.rs +++ b/crates/db_views/src/post/post_view.rs @@ -672,7 +672,6 @@ mod tests { instance_block::{InstanceBlock, InstanceBlockForm}, language::Language, local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm}, - local_user_vote_display_mode::LocalUserVoteDisplayMode, person::{Person, PersonInsertForm}, person_block::{PersonBlock, PersonBlockForm}, post::{ @@ -869,18 +868,15 @@ mod tests { let tegan_local_user_view = LocalUserView { local_user: inserted_tegan_local_user, - local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_tegan_person, }; let john_local_user_view = LocalUserView { local_user: inserted_john_local_user, - local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_john_person, }; let bot_local_user_view = LocalUserView { local_user: inserted_bot_local_user, - local_user_vote_display_mode: LocalUserVoteDisplayMode::default(), person: inserted_bot_person, }; diff --git a/crates/db_views/src/registration_applications/registration_application_view.rs b/crates/db_views/src/registration_applications/registration_application_view.rs index 63456247f0..f3bac8f713 100644 --- a/crates/db_views/src/registration_applications/registration_application_view.rs +++ b/crates/db_views/src/registration_applications/registration_application_view.rs @@ -140,28 +140,28 @@ mod tests { let pool = &build_db_pool_for_tests(); let pool = &mut pool.into(); - let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; + let instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; - let timmy_person_form = PersonInsertForm::test_form(inserted_instance.id, "timmy_rav"); + let timmy_person_form = PersonInsertForm::test_form(instance.id, "timmy_rav"); - let inserted_timmy_person = Person::create(pool, &timmy_person_form).await?; + let timmy_person = Person::create(pool, &timmy_person_form).await?; - let timmy_local_user_form = LocalUserInsertForm::test_form_admin(inserted_timmy_person.id); + let timmy_local_user_form = LocalUserInsertForm::test_form_admin(timmy_person.id); let _inserted_timmy_local_user = LocalUser::create(pool, &timmy_local_user_form, vec![]).await?; - let sara_person_form = PersonInsertForm::test_form(inserted_instance.id, "sara_rav"); + let sara_person_form = PersonInsertForm::test_form(instance.id, "sara_rav"); - let inserted_sara_person = Person::create(pool, &sara_person_form).await?; + let sara_person = Person::create(pool, &sara_person_form).await?; - let sara_local_user_form = LocalUserInsertForm::test_form(inserted_sara_person.id); + let sara_local_user_form = LocalUserInsertForm::test_form(sara_person.id); - let inserted_sara_local_user = LocalUser::create(pool, &sara_local_user_form, vec![]).await?; + let sara_local_user = LocalUser::create(pool, &sara_local_user_form, vec![]).await?; // Sara creates an application let sara_app_form = RegistrationApplicationInsertForm { - local_user_id: inserted_sara_local_user.id, + local_user_id: sara_local_user.id, answer: "LET ME IIIIINN".to_string(), }; @@ -169,17 +169,17 @@ mod tests { let read_sara_app_view = RegistrationApplicationView::read(pool, sara_app.id).await?; - let jess_person_form = PersonInsertForm::test_form(inserted_instance.id, "jess_rav"); + let jess_person_form = PersonInsertForm::test_form(instance.id, "jess_rav"); let inserted_jess_person = Person::create(pool, &jess_person_form).await?; let jess_local_user_form = LocalUserInsertForm::test_form(inserted_jess_person.id); - let inserted_jess_local_user = LocalUser::create(pool, &jess_local_user_form, vec![]).await?; + let jess_local_user = LocalUser::create(pool, &jess_local_user_form, vec![]).await?; // Sara creates an application let jess_app_form = RegistrationApplicationInsertForm { - local_user_id: inserted_jess_local_user.id, + local_user_id: jess_local_user.id, answer: "LET ME IIIIINN".to_string(), }; @@ -190,42 +190,44 @@ mod tests { let mut expected_sara_app_view = RegistrationApplicationView { registration_application: sara_app.clone(), creator_local_user: LocalUser { - id: inserted_sara_local_user.id, - person_id: inserted_sara_local_user.person_id, - email: inserted_sara_local_user.email, - show_nsfw: inserted_sara_local_user.show_nsfw, - blur_nsfw: inserted_sara_local_user.blur_nsfw, - theme: inserted_sara_local_user.theme, - default_post_sort_type: inserted_sara_local_user.default_post_sort_type, - default_comment_sort_type: inserted_sara_local_user.default_comment_sort_type, - default_listing_type: inserted_sara_local_user.default_listing_type, - interface_language: inserted_sara_local_user.interface_language, - show_avatars: inserted_sara_local_user.show_avatars, - send_notifications_to_email: inserted_sara_local_user.send_notifications_to_email, - show_bot_accounts: inserted_sara_local_user.show_bot_accounts, - show_read_posts: inserted_sara_local_user.show_read_posts, - email_verified: inserted_sara_local_user.email_verified, - accepted_application: inserted_sara_local_user.accepted_application, - totp_2fa_secret: inserted_sara_local_user.totp_2fa_secret, - password_encrypted: inserted_sara_local_user.password_encrypted, - open_links_in_new_tab: inserted_sara_local_user.open_links_in_new_tab, - infinite_scroll_enabled: inserted_sara_local_user.infinite_scroll_enabled, - post_listing_mode: inserted_sara_local_user.post_listing_mode, - totp_2fa_enabled: inserted_sara_local_user.totp_2fa_enabled, - enable_keyboard_navigation: inserted_sara_local_user.enable_keyboard_navigation, - enable_animated_images: inserted_sara_local_user.enable_animated_images, - enable_private_messages: inserted_sara_local_user.enable_private_messages, - collapse_bot_comments: inserted_sara_local_user.collapse_bot_comments, - last_donation_notification: inserted_sara_local_user.last_donation_notification, + id: sara_local_user.id, + person_id: sara_local_user.person_id, + email: sara_local_user.email, + show_nsfw: sara_local_user.show_nsfw, + blur_nsfw: sara_local_user.blur_nsfw, + theme: sara_local_user.theme, + default_post_sort_type: sara_local_user.default_post_sort_type, + default_comment_sort_type: sara_local_user.default_comment_sort_type, + default_listing_type: sara_local_user.default_listing_type, + interface_language: sara_local_user.interface_language, + show_avatars: sara_local_user.show_avatars, + send_notifications_to_email: sara_local_user.send_notifications_to_email, + show_bot_accounts: sara_local_user.show_bot_accounts, + show_read_posts: sara_local_user.show_read_posts, + email_verified: sara_local_user.email_verified, + accepted_application: sara_local_user.accepted_application, + totp_2fa_secret: sara_local_user.totp_2fa_secret, + password_encrypted: sara_local_user.password_encrypted, + open_links_in_new_tab: sara_local_user.open_links_in_new_tab, + infinite_scroll_enabled: sara_local_user.infinite_scroll_enabled, + post_listing_mode: sara_local_user.post_listing_mode, + totp_2fa_enabled: sara_local_user.totp_2fa_enabled, + enable_keyboard_navigation: sara_local_user.enable_keyboard_navigation, + enable_animated_images: sara_local_user.enable_animated_images, + enable_private_messages: sara_local_user.enable_private_messages, + collapse_bot_comments: sara_local_user.collapse_bot_comments, + last_donation_notification: sara_local_user.last_donation_notification, + show_upvotes: sara_local_user.show_upvotes, + show_downvotes: sara_local_user.show_downvotes, ..Default::default() }, creator: Person { - id: inserted_sara_person.id, - name: inserted_sara_person.name.clone(), + id: sara_person.id, + name: sara_person.name.clone(), display_name: None, - published: inserted_sara_person.published, + published: sara_person.published, avatar: None, - ap_id: inserted_sara_person.ap_id.clone(), + ap_id: sara_person.ap_id.clone(), local: true, banned: false, ban_expires: None, @@ -234,12 +236,12 @@ mod tests { bio: None, banner: None, updated: None, - inbox_url: inserted_sara_person.inbox_url.clone(), + inbox_url: sara_person.inbox_url.clone(), matrix_user_id: None, - instance_id: inserted_instance.id, - private_key: inserted_sara_person.private_key, - public_key: inserted_sara_person.public_key, - last_refreshed_at: inserted_sara_person.last_refreshed_at, + instance_id: instance.id, + private_key: sara_person.private_key, + public_key: sara_person.public_key, + last_refreshed_at: sara_person.last_refreshed_at, post_count: 0, post_score: 0, comment_count: 0, @@ -269,7 +271,7 @@ mod tests { // Approve the application let approve_form = RegistrationApplicationUpdateForm { - admin_id: Some(Some(inserted_timmy_person.id)), + admin_id: Some(Some(timmy_person.id)), deny_reason: None, }; @@ -281,7 +283,7 @@ mod tests { ..Default::default() }; - LocalUser::update(pool, inserted_sara_local_user.id, &approve_local_user_form).await?; + LocalUser::update(pool, sara_local_user.id, &approve_local_user_form).await?; let read_sara_app_view_after_approve = RegistrationApplicationView::read(pool, sara_app.id).await?; @@ -290,15 +292,15 @@ mod tests { expected_sara_app_view .creator_local_user .accepted_application = true; - expected_sara_app_view.registration_application.admin_id = Some(inserted_timmy_person.id); + expected_sara_app_view.registration_application.admin_id = Some(timmy_person.id); expected_sara_app_view.admin = Some(Person { - id: inserted_timmy_person.id, - name: inserted_timmy_person.name.clone(), + id: timmy_person.id, + name: timmy_person.name.clone(), display_name: None, - published: inserted_timmy_person.published, + published: timmy_person.published, avatar: None, - ap_id: inserted_timmy_person.ap_id.clone(), + ap_id: timmy_person.ap_id.clone(), local: true, banned: false, ban_expires: None, @@ -307,12 +309,12 @@ mod tests { bio: None, banner: None, updated: None, - inbox_url: inserted_timmy_person.inbox_url.clone(), + inbox_url: timmy_person.inbox_url.clone(), matrix_user_id: None, - instance_id: inserted_instance.id, - private_key: inserted_timmy_person.private_key, - public_key: inserted_timmy_person.public_key, - last_refreshed_at: inserted_timmy_person.last_refreshed_at, + instance_id: instance.id, + private_key: timmy_person.private_key, + public_key: timmy_person.public_key, + last_refreshed_at: timmy_person.last_refreshed_at, post_count: 0, post_score: 0, comment_count: 0, @@ -339,10 +341,10 @@ mod tests { let all_apps = RegistrationApplicationQuery::default().list(pool).await?; assert_eq!(all_apps.len(), 2); - Person::delete(pool, inserted_timmy_person.id).await?; - Person::delete(pool, inserted_sara_person.id).await?; + Person::delete(pool, timmy_person.id).await?; + Person::delete(pool, sara_person.id).await?; Person::delete(pool, inserted_jess_person.id).await?; - Instance::delete(pool, inserted_instance.id).await?; + Instance::delete(pool, instance.id).await?; Ok(()) } diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index 6335930870..2e5fc00526 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -39,7 +39,6 @@ use lemmy_db_schema::{ local_site::LocalSite, local_site_rate_limit::LocalSiteRateLimit, local_user::LocalUser, - local_user_vote_display_mode::LocalUserVoteDisplayMode, mod_log::{ admin::{ AdminAllowInstance, @@ -250,8 +249,6 @@ pub struct LocalUserView { #[cfg_attr(feature = "full", diesel(embed))] pub local_user: LocalUser, #[cfg_attr(feature = "full", diesel(embed))] - pub local_user_vote_display_mode: LocalUserVoteDisplayMode, - #[cfg_attr(feature = "full", diesel(embed))] pub person: Person, } diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql index d272298905..06f6fdc0c2 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql @@ -279,8 +279,7 @@ CREATE TABLE person_aggregates ( users_active_day bigint NOT NULL DEFAULT 0, users_active_week bigint NOT NULL DEFAULT 0, users_active_month bigint NOT NULL DEFAULT 0, - users_active_half_year bigint NOT NULL DEFAULT 0; - + users_active_half_year bigint NOT NULL DEFAULT 0 ); INSERT INTO site_aggregates @@ -307,3 +306,27 @@ ALTER TABLE local_site DROP COLUMN users_active_month, DROP COLUMN users_active_half_year; +-- move local_user_vote_display_mode back into separate table +CREATE TABLE local_user_vote_display_mode ( + local_user_id int PRIMARY KEY NOT NULL REFERENCES person ON UPDATE CASCADE ON DELETE CASCADE users score boolean NOT NULL DEFAULT FALSE, + upvotes boolean NOT NULL DEFAULT TRUE, + downvotes boolean NOT NULL DEFAULT TRUE, + upvote_percentage boolean NOT NULL DEFAULT FALSE +); + +INSERT INTO local_user_vote_display_mode +SELECT + id AS local_user_id, + show_score AS score, + show_upvotes AS upvotes, + show_downvotes AS downvotes, + show_upvote_percentage AS upvote_percentage +FROM + local_user; + +ALTER TABLE local_user + DROP COLUMN show_score, + DROP COLUMN show_upvotes, + DROP COLUMN show_downvotes, + DROP COLUMN show_upvote_percentage; + diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql index eda201e6d1..17536198b2 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -236,3 +236,24 @@ WHERE DROP TABLE site_aggregates; +-- merge local_user_vote_display_mode into local_user table +ALTER TABLE local_user + ADD COLUMN show_score boolean NOT NULL DEFAULT FALSE, + ADD COLUMN show_upvotes boolean NOT NULL DEFAULT TRUE, + ADD COLUMN show_downvotes boolean NOT NULL DEFAULT TRUE, + ADD COLUMN show_upvote_percentage boolean NOT NULL DEFAULT FALSE; + +UPDATE + local_user +SET + show_score = v.score, + show_upvotes = v.upvotes, + show_downvotes = v.downvotes, + show_upvote_percentage = v.upvote_percentage +FROM + local_user_vote_display_mode AS v +WHERE + local_user.id = v.local_user_id; + +DROP TABLE local_user_vote_display_mode; + From 416ad1bf5c5753590f80c693e1a70b3d27885799 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Thu, 13 Feb 2025 12:12:07 +0100 Subject: [PATCH 33/39] fix schema --- crates/db_schema/src/schema.rs | 207 +++++++++--------- .../up.sql | 3 - 2 files changed, 104 insertions(+), 106 deletions(-) diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 6c7727c665..aadcd474ba 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -1,45 +1,45 @@ // @generated automatically by Diesel CLI. pub mod sql_types { - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "actor_type_enum"))] - pub struct ActorTypeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "actor_type_enum"))] + pub struct ActorTypeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "comment_sort_type_enum"))] - pub struct CommentSortTypeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "comment_sort_type_enum"))] + pub struct CommentSortTypeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "community_follower_state"))] - pub struct CommunityFollowerState; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "community_follower_state"))] + pub struct CommunityFollowerState; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "community_visibility"))] - pub struct CommunityVisibility; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "community_visibility"))] + pub struct CommunityVisibility; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "federation_mode_enum"))] - pub struct FederationModeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "federation_mode_enum"))] + pub struct FederationModeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "listing_type_enum"))] - pub struct ListingTypeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "listing_type_enum"))] + pub struct ListingTypeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "ltree"))] - pub struct Ltree; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "ltree"))] + pub struct Ltree; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "post_listing_mode_enum"))] - pub struct PostListingModeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "post_listing_mode_enum"))] + pub struct PostListingModeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "post_sort_type_enum"))] - pub struct PostSortTypeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "post_sort_type_enum"))] + pub struct PostSortTypeEnum; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "registration_mode_enum"))] - pub struct RegistrationModeEnum; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "registration_mode_enum"))] + pub struct RegistrationModeEnum; } diesel::table! { @@ -1151,6 +1151,7 @@ diesel::joinable!(person_saved_combined -> comment (comment_id)); diesel::joinable!(person_saved_combined -> person (person_id)); diesel::joinable!(person_saved_combined -> post (post_id)); diesel::joinable!(post -> community (community_id)); +diesel::joinable!(post -> instance (instance_id)); diesel::joinable!(post -> language (language_id)); diesel::joinable!(post -> person (creator_id)); diesel::joinable!(post_actions -> person (person_id)); @@ -1175,77 +1176,77 @@ diesel::joinable!(site_language -> site (site_id)); diesel::joinable!(tag -> community (community_id)); diesel::allow_tables_to_appear_in_same_query!( - admin_allow_instance, - admin_block_instance, - admin_purge_comment, - admin_purge_community, - admin_purge_person, - admin_purge_post, - captcha_answer, - comment, - comment_actions, - comment_reply, - comment_report, - community, - community_actions, - community_language, - community_report, - custom_emoji, - custom_emoji_keyword, - email_verification, - federation_allowlist, - federation_blocklist, - federation_queue_state, - image_details, - inbox_combined, - instance, - instance_actions, - language, - local_image, - local_site, - local_site_rate_limit, - local_site_url_blocklist, - local_user, - local_user_language, - login_token, - mod_add, - mod_add_community, - mod_ban, - mod_ban_from_community, - mod_feature_post, - mod_hide_community, - mod_lock_post, - mod_remove_comment, - mod_remove_community, - mod_remove_post, - mod_transfer_community, - modlog_combined, - oauth_account, - oauth_provider, - password_reset_request, - person, - person_actions, - person_ban, - person_comment_mention, - person_content_combined, - person_post_mention, - person_saved_combined, - post, - post_actions, - post_report, - post_tag, - previously_run_sql, - private_message, - private_message_report, - received_activity, - registration_application, - remote_image, - report_combined, - search_combined, - secret, - sent_activity, - site, - site_language, - tag, - tagline, + admin_allow_instance, + admin_block_instance, + admin_purge_comment, + admin_purge_community, + admin_purge_person, + admin_purge_post, + captcha_answer, + comment, + comment_actions, + comment_reply, + comment_report, + community, + community_actions, + community_language, + community_report, + custom_emoji, + custom_emoji_keyword, + email_verification, + federation_allowlist, + federation_blocklist, + federation_queue_state, + image_details, + inbox_combined, + instance, + instance_actions, + language, + local_image, + local_site, + local_site_rate_limit, + local_site_url_blocklist, + local_user, + local_user_language, + login_token, + mod_add, + mod_add_community, + mod_ban, + mod_ban_from_community, + mod_feature_post, + mod_hide_community, + mod_lock_post, + mod_remove_comment, + mod_remove_community, + mod_remove_post, + mod_transfer_community, + modlog_combined, + oauth_account, + oauth_provider, + password_reset_request, + person, + person_actions, + person_ban, + person_comment_mention, + person_content_combined, + person_post_mention, + person_saved_combined, + post, + post_actions, + post_report, + post_tag, + previously_run_sql, + private_message, + private_message_report, + received_activity, + registration_application, + remote_image, + report_combined, + search_combined, + secret, + sent_activity, + site, + site_language, + tag, + tagline, ); diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql index 17536198b2..8edbc9a884 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -75,9 +75,6 @@ FROM WHERE post.id = pa.post_id; -ALTER TABLE post - ALTER COLUMN instance_id DROP NOT NULL; - DROP TABLE post_aggregates; -- Note, removed `post_id DESC` from all these From b0bc9271926121b0225f50bff78fdce0a6b5bc78 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 17 Feb 2025 11:51:04 +0100 Subject: [PATCH 34/39] remove duplicate fields --- crates/db_views/src/combined/search_combined_view.rs | 2 -- crates/db_views/src/structs.rs | 2 -- 2 files changed, 4 deletions(-) diff --git a/crates/db_views/src/combined/search_combined_view.rs b/crates/db_views/src/combined/search_combined_view.rs index 425e681d14..d9a0ca0c2c 100644 --- a/crates/db_views/src/combined/search_combined_view.rs +++ b/crates/db_views/src/combined/search_combined_view.rs @@ -260,8 +260,6 @@ impl SearchCombinedQuery { community::all_columns.nullable(), community_actions::blocked.nullable().is_not_null(), community_follower_select_subscribed_type(), - // Person - person::all_columns.nullable(), // // Shared person::all_columns.nullable(), local_user::admin.nullable().is_not_null(), diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index 2e5fc00526..f2e55f2f40 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -1144,8 +1144,6 @@ pub(crate) struct SearchCombinedViewInternal { pub community: Option, pub community_blocked: bool, pub subscribed: SubscribedType, - // Person - pub item_creator_counts: Option, // Shared pub item_creator: Option, pub item_creator_is_admin: bool, From 4a6a5e61fcb7c3e5f6491ff5873d49806aa7e2f5 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 17 Feb 2025 11:53:33 +0100 Subject: [PATCH 35/39] remove "aggregates" from index names --- .../up.sql | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql index 8edbc9a884..b193b6ed99 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -27,16 +27,16 @@ WHERE DROP TABLE comment_aggregates; -CREATE INDEX idx_comment_aggregates_controversy ON comment USING btree (controversy_rank DESC); +CREATE INDEX idx_comment_controversy ON comment USING btree (controversy_rank DESC); -CREATE INDEX idx_comment_aggregates_hot ON comment USING btree (hot_rank DESC, score DESC); +CREATE INDEX idx_comment_hot ON comment USING btree (hot_rank DESC, score DESC); -CREATE INDEX idx_comment_aggregates_nonzero_hotrank ON comment USING btree (published) +CREATE INDEX idx_comment_nonzero_hotrank ON comment USING btree (published) WHERE (hot_rank <> (0)::double precision); -- some indexes commented out because they already exist ---CREATE INDEX idx_comment_aggregates_published on comment USING btree (published DESC); -CREATE INDEX idx_comment_aggregates_score ON comment USING btree (score DESC); +--CREATE INDEX idx_comment_published on comment USING btree (published DESC); +CREATE INDEX idx_comment_score ON comment USING btree (score DESC); -- merge post_aggregates into post table ALTER TABLE post @@ -78,65 +78,65 @@ WHERE DROP TABLE post_aggregates; -- Note, removed `post_id DESC` from all these -CREATE INDEX idx_post_aggregates_community_active ON post USING btree (community_id, featured_local DESC, hot_rank_active DESC, published DESC); +CREATE INDEX idx_post_community_active ON post USING btree (community_id, featured_local DESC, hot_rank_active DESC, published DESC); -CREATE INDEX idx_post_aggregates_community_controversy ON post USING btree (community_id, featured_local DESC, controversy_rank DESC); +CREATE INDEX idx_post_community_controversy ON post USING btree (community_id, featured_local DESC, controversy_rank DESC); -CREATE INDEX idx_post_aggregates_community_hot ON post USING btree (community_id, featured_local DESC, hot_rank DESC, published DESC); +CREATE INDEX idx_post_community_hot ON post USING btree (community_id, featured_local DESC, hot_rank DESC, published DESC); -CREATE INDEX idx_post_aggregates_community_most_comments ON post USING btree (community_id, featured_local DESC, comments DESC, published DESC); +CREATE INDEX idx_post_community_most_comments ON post USING btree (community_id, featured_local DESC, comments DESC, published DESC); -CREATE INDEX idx_post_aggregates_community_newest_comment_time ON post USING btree (community_id, featured_local DESC, newest_comment_time DESC); +CREATE INDEX idx_post_community_newest_comment_time ON post USING btree (community_id, featured_local DESC, newest_comment_time DESC); -CREATE INDEX idx_post_aggregates_community_newest_comment_time_necro ON post USING btree (community_id, featured_local DESC, newest_comment_time_necro DESC); +CREATE INDEX idx_post_community_newest_comment_time_necro ON post USING btree (community_id, featured_local DESC, newest_comment_time_necro DESC); ---CREATE INDEX idx_post_aggregates_community_published on post USING btree (community_id, featured_local DESC, published DESC); ---CREATE INDEX idx_post_aggregates_community_published_asc on post USING btree (community_id, featured_local DESC, reverse_timestamp_sort (published) DESC); -CREATE INDEX idx_post_aggregates_community_scaled ON post USING btree (community_id, featured_local DESC, scaled_rank DESC, published DESC); +--CREATE INDEX idx_post_community_published on post USING btree (community_id, featured_local DESC, published DESC); +--CREATE INDEX idx_post_community_published_asc on post USING btree (community_id, featured_local DESC, reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_community_scaled ON post USING btree (community_id, featured_local DESC, scaled_rank DESC, published DESC); -CREATE INDEX idx_post_aggregates_community_score ON post USING btree (community_id, featured_local DESC, score DESC, published DESC); +CREATE INDEX idx_post_community_score ON post USING btree (community_id, featured_local DESC, score DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_community_active ON post USING btree (community_id, featured_community DESC, hot_rank_active DESC, published DESC); +CREATE INDEX idx_post_featured_community_active ON post USING btree (community_id, featured_community DESC, hot_rank_active DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_community_controversy ON post USING btree (community_id, featured_community DESC, controversy_rank DESC); +CREATE INDEX idx_post_featured_community_controversy ON post USING btree (community_id, featured_community DESC, controversy_rank DESC); -CREATE INDEX idx_post_aggregates_featured_community_hot ON post USING btree (community_id, featured_community DESC, hot_rank DESC, published DESC); +CREATE INDEX idx_post_featured_community_hot ON post USING btree (community_id, featured_community DESC, hot_rank DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_community_most_comments ON post USING btree (community_id, featured_community DESC, comments DESC, published DESC); +CREATE INDEX idx_post_featured_community_most_comments ON post USING btree (community_id, featured_community DESC, comments DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time ON post USING btree (community_id, featured_community DESC, newest_comment_time DESC); +CREATE INDEX idx_post_featured_community_newest_comment_time ON post USING btree (community_id, featured_community DESC, newest_comment_time DESC); -CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time_necr ON post USING btree (community_id, featured_community DESC, newest_comment_time_necro DESC); +CREATE INDEX idx_post_featured_community_newest_comment_time_necr ON post USING btree (community_id, featured_community DESC, newest_comment_time_necro DESC); ---CREATE INDEX idx_post_aggregates_featured_community_published on post USING btree (community_id, featured_community DESC, published DESC); ---CREATE INDEX idx_post_aggregates_featured_community_published_asc on post USING btree (community_id, featured_community DESC, reverse_timestamp_sort (published) DESC); -CREATE INDEX idx_post_aggregates_featured_community_scaled ON post USING btree (community_id, featured_community DESC, scaled_rank DESC, published DESC); +--CREATE INDEX idx_post_featured_community_published on post USING btree (community_id, featured_community DESC, published DESC); +--CREATE INDEX idx_post_featured_community_published_asc on post USING btree (community_id, featured_community DESC, reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_featured_community_scaled ON post USING btree (community_id, featured_community DESC, scaled_rank DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_community_score ON post USING btree (community_id, featured_community DESC, score DESC, published DESC); +CREATE INDEX idx_post_featured_community_score ON post USING btree (community_id, featured_community DESC, score DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_local_active ON post USING btree (featured_local DESC, hot_rank_active DESC, published DESC); +CREATE INDEX idx_post_featured_local_active ON post USING btree (featured_local DESC, hot_rank_active DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_local_controversy ON post USING btree (featured_local DESC, controversy_rank DESC); +CREATE INDEX idx_post_featured_local_controversy ON post USING btree (featured_local DESC, controversy_rank DESC); -CREATE INDEX idx_post_aggregates_featured_local_hot ON post USING btree (featured_local DESC, hot_rank DESC, published DESC); +CREATE INDEX idx_post_featured_local_hot ON post USING btree (featured_local DESC, hot_rank DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_local_most_comments ON post USING btree (featured_local DESC, comments DESC, published DESC); +CREATE INDEX idx_post_featured_local_most_comments ON post USING btree (featured_local DESC, comments DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time ON post USING btree (featured_local DESC, newest_comment_time DESC); +CREATE INDEX idx_post_featured_local_newest_comment_time ON post USING btree (featured_local DESC, newest_comment_time DESC); -CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time_necro ON post USING btree (featured_local DESC, newest_comment_time_necro DESC); +CREATE INDEX idx_post_featured_local_newest_comment_time_necro ON post USING btree (featured_local DESC, newest_comment_time_necro DESC); ---CREATE INDEX idx_post_aggregates_featured_local_published on post USING btree (featured_local DESC, published DESC); ---CREATE INDEX idx_post_aggregates_featured_local_published_asc on post USING btree (featured_local DESC, reverse_timestamp_sort (published) DESC); -CREATE INDEX idx_post_aggregates_featured_local_scaled ON post USING btree (featured_local DESC, scaled_rank DESC, published DESC); +--CREATE INDEX idx_post_featured_local_published on post USING btree (featured_local DESC, published DESC); +--CREATE INDEX idx_post_featured_local_published_asc on post USING btree (featured_local DESC, reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_featured_local_scaled ON post USING btree (featured_local DESC, scaled_rank DESC, published DESC); -CREATE INDEX idx_post_aggregates_featured_local_score ON post USING btree (featured_local DESC, score DESC, published DESC); +CREATE INDEX idx_post_featured_local_score ON post USING btree (featured_local DESC, score DESC, published DESC); -CREATE INDEX idx_post_aggregates_nonzero_hotrank ON post USING btree (published DESC) +CREATE INDEX idx_post_nonzero_hotrank ON post USING btree (published DESC) WHERE ((hot_rank <> (0)::double precision) OR (hot_rank_active <> (0)::double precision)); ---CREATE INDEX idx_post_aggregates_published on post USING btree (published DESC); ---CREATE INDEX idx_post_aggregates_published_asc on post USING btree (reverse_timestamp_sort (published) DESC); +--CREATE INDEX idx_post_published on post USING btree (published DESC); +--CREATE INDEX idx_post_published_asc on post USING btree (reverse_timestamp_sort (published) DESC); -- merge community_aggregates into community table ALTER TABLE community ADD COLUMN subscribers bigint NOT NULL DEFAULT 0, @@ -174,14 +174,14 @@ WHERE DROP TABLE community_aggregates; -CREATE INDEX idx_community_aggregates_hot ON public.community USING btree (hot_rank DESC); +CREATE INDEX idx_community_hot ON public.community USING btree (hot_rank DESC); -CREATE INDEX idx_community_aggregates_nonzero_hotrank ON public.community USING btree (published) +CREATE INDEX idx_community_nonzero_hotrank ON public.community USING btree (published) WHERE (hot_rank <> (0)::double precision); -CREATE INDEX idx_community_aggregates_subscribers ON public.community USING btree (subscribers DESC); +CREATE INDEX idx_community_subscribers ON public.community USING btree (subscribers DESC); -CREATE INDEX idx_community_aggregates_users_active_month ON public.community USING btree (users_active_month DESC); +CREATE INDEX idx_community_users_active_month ON public.community USING btree (users_active_month DESC); -- merge person_aggregates into person table ALTER TABLE person From 4b7422aec906724f0dc2ff22319b43cad16bffde Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 17 Feb 2025 12:02:02 +0100 Subject: [PATCH 36/39] uncomment indices --- .../up.sql | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql index b193b6ed99..295fca18db 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -34,7 +34,6 @@ CREATE INDEX idx_comment_hot ON comment USING btree (hot_rank DESC, score DESC); CREATE INDEX idx_comment_nonzero_hotrank ON comment USING btree (published) WHERE (hot_rank <> (0)::double precision); --- some indexes commented out because they already exist --CREATE INDEX idx_comment_published on comment USING btree (published DESC); CREATE INDEX idx_comment_score ON comment USING btree (score DESC); @@ -90,8 +89,10 @@ CREATE INDEX idx_post_community_newest_comment_time ON post USING btree (communi CREATE INDEX idx_post_community_newest_comment_time_necro ON post USING btree (community_id, featured_local DESC, newest_comment_time_necro DESC); ---CREATE INDEX idx_post_community_published on post USING btree (community_id, featured_local DESC, published DESC); ---CREATE INDEX idx_post_community_published_asc on post USING btree (community_id, featured_local DESC, reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_community_published ON post USING btree (community_id, featured_local DESC, published DESC); + +CREATE INDEX idx_post_community_published_asc ON post USING btree (community_id, featured_local DESC, reverse_timestamp_sort (published) DESC); + CREATE INDEX idx_post_community_scaled ON post USING btree (community_id, featured_local DESC, scaled_rank DESC, published DESC); CREATE INDEX idx_post_community_score ON post USING btree (community_id, featured_local DESC, score DESC, published DESC); @@ -108,8 +109,10 @@ CREATE INDEX idx_post_featured_community_newest_comment_time ON post USING btree CREATE INDEX idx_post_featured_community_newest_comment_time_necr ON post USING btree (community_id, featured_community DESC, newest_comment_time_necro DESC); ---CREATE INDEX idx_post_featured_community_published on post USING btree (community_id, featured_community DESC, published DESC); ---CREATE INDEX idx_post_featured_community_published_asc on post USING btree (community_id, featured_community DESC, reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_featured_community_published ON post USING btree (community_id, featured_community DESC, published DESC); + +CREATE INDEX idx_post_featured_community_published_asc ON post USING btree (community_id, featured_community DESC, reverse_timestamp_sort (published) DESC); + CREATE INDEX idx_post_featured_community_scaled ON post USING btree (community_id, featured_community DESC, scaled_rank DESC, published DESC); CREATE INDEX idx_post_featured_community_score ON post USING btree (community_id, featured_community DESC, score DESC, published DESC); @@ -126,8 +129,10 @@ CREATE INDEX idx_post_featured_local_newest_comment_time ON post USING btree (fe CREATE INDEX idx_post_featured_local_newest_comment_time_necro ON post USING btree (featured_local DESC, newest_comment_time_necro DESC); ---CREATE INDEX idx_post_featured_local_published on post USING btree (featured_local DESC, published DESC); ---CREATE INDEX idx_post_featured_local_published_asc on post USING btree (featured_local DESC, reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_featured_local_published ON post USING btree (featured_local DESC, published DESC); + +CREATE INDEX idx_post_featured_local_published_asc ON post USING btree (featured_local DESC, reverse_timestamp_sort (published) DESC); + CREATE INDEX idx_post_featured_local_scaled ON post USING btree (featured_local DESC, scaled_rank DESC, published DESC); CREATE INDEX idx_post_featured_local_score ON post USING btree (featured_local DESC, score DESC, published DESC); @@ -135,8 +140,10 @@ CREATE INDEX idx_post_featured_local_score ON post USING btree (featured_local D CREATE INDEX idx_post_nonzero_hotrank ON post USING btree (published DESC) WHERE ((hot_rank <> (0)::double precision) OR (hot_rank_active <> (0)::double precision)); ---CREATE INDEX idx_post_published on post USING btree (published DESC); ---CREATE INDEX idx_post_published_asc on post USING btree (reverse_timestamp_sort (published) DESC); +CREATE INDEX idx_post_published ON post USING btree (published DESC); + +CREATE INDEX idx_post_published_asc ON post USING btree (reverse_timestamp_sort (published) DESC); + -- merge community_aggregates into community table ALTER TABLE community ADD COLUMN subscribers bigint NOT NULL DEFAULT 0, From f758ed88561262ae859a6e35f58a40b4d2819fb6 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 17 Feb 2025 16:21:44 +0100 Subject: [PATCH 37/39] if count = 0 --- .../db_schema/replaceable_schema/triggers.sql | 314 +++++++++++++----- crates/db_schema/replaceable_schema/utils.sql | 3 - 2 files changed, 222 insertions(+), 95 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index b86de3ca46..b15e452cc2 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -27,28 +27,53 @@ BEGIN -- When a thing gets a vote, update its aggregates and its creator's aggregates CALL r.create_triggers ('thing_actions', $$ BEGIN - WITH thing_diff AS ( UPDATE + IF ( + SELECT + count(*) + FROM select_old_and_new_rows) = 0 THEN + RETURN NULL; + END IF; + WITH thing_diff AS ( + UPDATE thing AS a SET - score = a.score + diff.upvotes - diff.downvotes, upvotes = a.upvotes + diff.upvotes, downvotes = a.downvotes + diff.downvotes, controversy_rank = r.controversy_rank ((a.upvotes + diff.upvotes)::numeric, (a.downvotes + diff.downvotes)::numeric) + score = a.score + diff.upvotes - diff.downvotes, + upvotes = a.upvotes + diff.upvotes, + downvotes = a.downvotes + diff.downvotes, + controversy_rank = r.controversy_rank ((a.upvotes + diff.upvotes)::numeric, (a.downvotes + diff.downvotes)::numeric) FROM ( SELECT - (thing_actions).thing_id, coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score = 1), 0) AS upvotes, coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score != 1), 0) AS downvotes FROM select_old_and_new_rows AS old_and_new_rows - WHERE (thing_actions).like_score IS NOT NULL GROUP BY (thing_actions).thing_id) AS diff - WHERE - a.id = diff.thing_id - AND (diff.upvotes, diff.downvotes) != (0, 0) - RETURNING - a.creator_id AS creator_id, diff.upvotes - diff.downvotes AS score) - UPDATE - person AS a - SET - thing_score = a.thing_score + diff.score FROM ( - SELECT - creator_id, sum(score) AS score FROM thing_diff GROUP BY creator_id) AS diff + (thing_actions).thing_id, + coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score = 1), 0) AS upvotes, + coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score != 1), 0) AS downvotes + FROM + select_old_and_new_rows AS old_and_new_rows + WHERE (thing_actions).like_score IS NOT NULL + GROUP BY + (thing_actions).thing_id) AS diff WHERE - a.id = diff.creator_id - AND diff.score != 0; + a.id = diff.thing_id + AND (diff.upvotes, + diff.downvotes) != (0, + 0) + RETURNING + a.creator_id AS creator_id, + diff.upvotes - diff.downvotes AS score) + UPDATE + person AS a + SET + thing_score = a.thing_score + diff.score + FROM ( + SELECT + creator_id, + sum(score) AS score + FROM + thing_diff + GROUP BY + creator_id) AS diff + WHERE + a.id = diff.creator_id + AND diff.score != 0; RETURN NULL; END; $$); @@ -80,20 +105,32 @@ END; CALL r.create_triggers ('comment', $$ BEGIN - UPDATE - person AS a - SET - comment_count = a.comment_count + diff.comment_count - FROM ( + IF ( SELECT - (comment).creator_id, coalesce(sum(count_diff), 0) AS comment_count - FROM select_old_and_new_rows AS old_and_new_rows - WHERE - r.is_counted (comment) - GROUP BY (comment).creator_id) AS diff + count(*) +FROM + select_old_and_new_rows) = 0 THEN + RETURN NULL; + +END IF; + +UPDATE + person AS a +SET + comment_count = a.comment_count + diff.comment_count +FROM ( + SELECT + (comment).creator_id, + coalesce(sum(count_diff), 0) AS comment_count + FROM + select_old_and_new_rows AS old_and_new_rows + WHERE + r.is_counted (comment) + GROUP BY + (comment).creator_id) AS diff WHERE a.id = diff.creator_id - AND diff.comment_count != 0; + AND diff.comment_count != 0; UPDATE comment AS a @@ -211,20 +248,32 @@ $$); CALL r.create_triggers ('post', $$ BEGIN - UPDATE - person AS a - SET - post_count = a.post_count + diff.post_count - FROM ( + IF ( SELECT - (post).creator_id, coalesce(sum(count_diff), 0) AS post_count - FROM select_old_and_new_rows AS old_and_new_rows - WHERE - r.is_counted (post) - GROUP BY (post).creator_id) AS diff + count(*) +FROM + select_old_and_new_rows) = 0 THEN + RETURN NULL; + +END IF; + +UPDATE + person AS a +SET + post_count = a.post_count + diff.post_count +FROM ( + SELECT + (post).creator_id, + coalesce(sum(count_diff), 0) AS post_count + FROM + select_old_and_new_rows AS old_and_new_rows + WHERE + r.is_counted (post) + GROUP BY + (post).creator_id) AS diff WHERE a.id = diff.creator_id - AND diff.post_count != 0; + AND diff.post_count != 0; UPDATE community AS a @@ -267,17 +316,27 @@ $$); CALL r.create_triggers ('community', $$ BEGIN - UPDATE - local_site AS a - SET - communities = a.communities + diff.communities - FROM ( + IF ( SELECT - coalesce(sum(count_diff), 0) AS communities - FROM select_old_and_new_rows AS old_and_new_rows - WHERE - r.is_counted (community) - AND (community).local) AS diff + count(*) +FROM + select_old_and_new_rows) = 0 THEN + RETURN NULL; + +END IF; + +UPDATE + local_site AS a +SET + communities = a.communities + diff.communities +FROM ( + SELECT + coalesce(sum(count_diff), 0) AS communities + FROM + select_old_and_new_rows AS old_and_new_rows + WHERE + r.is_counted (community) + AND (community).local) AS diff WHERE diff.communities != 0; @@ -289,15 +348,25 @@ $$); CALL r.create_triggers ('person', $$ BEGIN - UPDATE - local_site AS a - SET - users = a.users + diff.users - FROM ( + IF ( SELECT - coalesce(sum(count_diff), 0) AS users - FROM select_old_and_new_rows AS old_and_new_rows - WHERE (person).local) AS diff + count(*) +FROM + select_old_and_new_rows) = 0 THEN + RETURN NULL; + +END IF; + +UPDATE + local_site AS a +SET + users = a.users + diff.users +FROM ( + SELECT + coalesce(sum(count_diff), 0) AS users + FROM + select_old_and_new_rows AS old_and_new_rows + WHERE (person).local) AS diff WHERE diff.users != 0; @@ -350,20 +419,37 @@ CREATE TRIGGER comment_count -- subscribers_local should be updated only when a local person follows a local or remote community. CALL r.create_triggers ('community_actions', $$ BEGIN - UPDATE - community AS a - SET - subscribers = a.subscribers + diff.subscribers, subscribers_local = a.subscribers_local + diff.subscribers_local - FROM ( + IF ( SELECT - (community_actions).community_id, coalesce(sum(count_diff) FILTER (WHERE community.local), 0) AS subscribers, coalesce(sum(count_diff) FILTER (WHERE person.local), 0) AS subscribers_local - FROM select_old_and_new_rows AS old_and_new_rows + count(*) +FROM + select_old_and_new_rows) = 0 THEN + RETURN NULL; + +END IF; + +UPDATE + community AS a +SET + subscribers = a.subscribers + diff.subscribers, + subscribers_local = a.subscribers_local + diff.subscribers_local +FROM ( + SELECT + (community_actions).community_id, + coalesce(sum(count_diff) FILTER (WHERE community.local), 0) AS subscribers, + coalesce(sum(count_diff) FILTER (WHERE person.local), 0) AS subscribers_local + FROM + select_old_and_new_rows AS old_and_new_rows LEFT JOIN community ON community.id = (community_actions).community_id LEFT JOIN person ON person.id = (community_actions).person_id - WHERE (community_actions).followed IS NOT NULL GROUP BY (community_actions).community_id) AS diff +WHERE (community_actions).followed IS NOT NULL +GROUP BY + (community_actions).community_id) AS diff WHERE a.id = diff.community_id - AND (diff.subscribers, diff.subscribers_local) != (0, 0); + AND (diff.subscribers, + diff.subscribers_local) != (0, + 0); RETURN NULL; @@ -373,17 +459,32 @@ $$); CALL r.create_triggers ('post_report', $$ BEGIN - UPDATE - post AS a - SET - report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count - FROM ( + IF ( SELECT - (post_report).post_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (post_report).resolved - AND NOT (post_report).violates_instance_rules), 0) AS unresolved_report_count -FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (post_report).post_id) AS diff + count(*) +FROM + select_old_and_new_rows) = 0 THEN + RETURN NULL; + +END IF; + +UPDATE + post AS a +SET + report_count = a.report_count + diff.report_count, + unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count +FROM ( + SELECT + (post_report).post_id, + coalesce(sum(count_diff), 0) AS report_count, + coalesce(sum(count_diff) FILTER (WHERE NOT (post_report).resolved + AND NOT (post_report).violates_instance_rules), 0) AS unresolved_report_count + FROM + select_old_and_new_rows AS old_and_new_rows + GROUP BY + (post_report).post_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) -AND a.id = diff.post_id; + AND a.id = diff.post_id; RETURN NULL; @@ -393,17 +494,32 @@ $$); CALL r.create_triggers ('comment_report', $$ BEGIN - UPDATE - comment AS a - SET - report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count - FROM ( + IF ( SELECT - (comment_report).comment_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (comment_report).resolved - AND NOT (comment_report).violates_instance_rules), 0) AS unresolved_report_count -FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (comment_report).comment_id) AS diff + count(*) +FROM + select_old_and_new_rows) = 0 THEN + RETURN NULL; + +END IF; + +UPDATE + comment AS a +SET + report_count = a.report_count + diff.report_count, + unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count +FROM ( + SELECT + (comment_report).comment_id, + coalesce(sum(count_diff), 0) AS report_count, + coalesce(sum(count_diff) FILTER (WHERE NOT (comment_report).resolved + AND NOT (comment_report).violates_instance_rules), 0) AS unresolved_report_count + FROM + select_old_and_new_rows AS old_and_new_rows + GROUP BY + (comment_report).comment_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) -AND a.id = diff.comment_id; + AND a.id = diff.comment_id; RETURN NULL; @@ -413,14 +529,29 @@ $$); CALL r.create_triggers ('community_report', $$ BEGIN - UPDATE - community AS a - SET - report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count - FROM ( + IF ( SELECT - (community_report).community_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (community_report).resolved), 0) AS unresolved_report_count - FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (community_report).community_id) AS diff + count(*) +FROM + select_old_and_new_rows) = 0 THEN + RETURN NULL; + +END IF; + +UPDATE + community AS a +SET + report_count = a.report_count + diff.report_count, + unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count +FROM ( + SELECT + (community_report).community_id, + coalesce(sum(count_diff), 0) AS report_count, + coalesce(sum(count_diff) FILTER (WHERE NOT (community_report).resolved), 0) AS unresolved_report_count + FROM + select_old_and_new_rows AS old_and_new_rows + GROUP BY + (community_report).community_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) AND a.id = diff.community_id; @@ -453,7 +584,6 @@ $$; CREATE TRIGGER aggregates AFTER INSERT ON post REFERENCING NEW TABLE AS new_post FOR EACH STATEMENT - WHEN (pg_trigger_depth() = 0) EXECUTE FUNCTION r.post_from_post (); -- Change the order of some cascading deletions to make deletion triggers run before the deletion of rows that the triggers need to read diff --git a/crates/db_schema/replaceable_schema/utils.sql b/crates/db_schema/replaceable_schema/utils.sql index 996b8b3afa..a951f1a206 100644 --- a/crates/db_schema/replaceable_schema/utils.sql +++ b/crates/db_schema/replaceable_schema/utils.sql @@ -92,7 +92,6 @@ DECLARE CREATE TRIGGER delete_statement AFTER DELETE ON thing REFERENCING OLD TABLE AS select_old_rows FOR EACH STATEMENT - WHEN (pg_trigger_depth( ) = 0 ) EXECUTE FUNCTION r.thing_delete_statement ( ); -- Insert CREATE FUNCTION r.thing_insert_statement ( ) @@ -102,7 +101,6 @@ DECLARE CREATE TRIGGER insert_statement AFTER INSERT ON thing REFERENCING NEW TABLE AS select_new_rows FOR EACH STATEMENT - WHEN (pg_trigger_depth( ) = 0 ) EXECUTE FUNCTION r.thing_insert_statement ( ); -- Update CREATE FUNCTION r.thing_update_statement ( ) @@ -112,7 +110,6 @@ DECLARE CREATE TRIGGER update_statement AFTER UPDATE ON thing REFERENCING OLD TABLE AS select_old_rows NEW TABLE AS select_new_rows FOR EACH STATEMENT - WHEN (pg_trigger_depth( ) = 0 ) EXECUTE FUNCTION r.thing_update_statement ( ); $$; select_old_and_new_rows text := $$ ( From 44c5990fc00b9378d20367fa5a8a41b2f7f05903 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 25 Feb 2025 10:27:02 +0100 Subject: [PATCH 38/39] remove commentaggregates --- crates/db_views/src/combined/report_combined_view.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/db_views/src/combined/report_combined_view.rs b/crates/db_views/src/combined/report_combined_view.rs index e002fc7b10..aae7dcac13 100644 --- a/crates/db_views/src/combined/report_combined_view.rs +++ b/crates/db_views/src/combined/report_combined_view.rs @@ -1309,7 +1309,7 @@ mod tests { }; CommentReport::report(pool, &timmy_report_form).await?; - let agg = CommentAggregates::read(pool, data.comment.id).await?; + let agg = Comment::read(pool, data.comment.id).await?; assert_eq!(agg.report_count, 2); // Do a batch read of timmys reports, it should only show his own From 2b7a0b3908247e9d91fdcd7677032c7c819eb3d0 Mon Sep 17 00:00:00 2001 From: dullbananas Date: Mon, 3 Mar 2025 09:00:28 -0700 Subject: [PATCH 39/39] Fix triggers in remove aggregates tables pr (#5451) * prevent all db_schema test errors * fix the delete_comments_before_post problem in a way that doesn't affect the returned number of affected rows * remove unnecessary recursion checks and add comment to remaining check * clean up * Fixing SQL format. * Update triggers.sql * Update triggers.sql * Update triggers.sql * Update triggers.sql * remove update of deleted column --------- Co-authored-by: Dessalines --- .../db_schema/replaceable_schema/triggers.sql | 459 +++++------------- crates/utils/src/lib.rs | 3 + .../down.sql | 3 +- .../up.sql | 3 +- 4 files changed, 133 insertions(+), 335 deletions(-) diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index b15e452cc2..63fc1bd7a6 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -27,53 +27,28 @@ BEGIN -- When a thing gets a vote, update its aggregates and its creator's aggregates CALL r.create_triggers ('thing_actions', $$ BEGIN - IF ( - SELECT - count(*) - FROM select_old_and_new_rows) = 0 THEN - RETURN NULL; - END IF; - WITH thing_diff AS ( - UPDATE + WITH thing_diff AS ( UPDATE thing AS a SET - score = a.score + diff.upvotes - diff.downvotes, - upvotes = a.upvotes + diff.upvotes, - downvotes = a.downvotes + diff.downvotes, - controversy_rank = r.controversy_rank ((a.upvotes + diff.upvotes)::numeric, (a.downvotes + diff.downvotes)::numeric) + score = a.score + diff.upvotes - diff.downvotes, upvotes = a.upvotes + diff.upvotes, downvotes = a.downvotes + diff.downvotes, controversy_rank = r.controversy_rank ((a.upvotes + diff.upvotes)::numeric, (a.downvotes + diff.downvotes)::numeric) FROM ( SELECT - (thing_actions).thing_id, - coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score = 1), 0) AS upvotes, - coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score != 1), 0) AS downvotes - FROM - select_old_and_new_rows AS old_and_new_rows - WHERE (thing_actions).like_score IS NOT NULL - GROUP BY - (thing_actions).thing_id) AS diff - WHERE - a.id = diff.thing_id - AND (diff.upvotes, - diff.downvotes) != (0, - 0) + (thing_actions).thing_id, coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score = 1), 0) AS upvotes, coalesce(sum(count_diff) FILTER (WHERE (thing_actions).like_score != 1), 0) AS downvotes FROM select_old_and_new_rows AS old_and_new_rows + WHERE (thing_actions).like_score IS NOT NULL GROUP BY (thing_actions).thing_id) AS diff + WHERE + a.id = diff.thing_id + AND (diff.upvotes, diff.downvotes) != (0, 0) RETURNING - a.creator_id AS creator_id, - diff.upvotes - diff.downvotes AS score) - UPDATE - person AS a - SET - thing_score = a.thing_score + diff.score - FROM ( - SELECT - creator_id, - sum(score) AS score - FROM - thing_diff - GROUP BY - creator_id) AS diff - WHERE - a.id = diff.creator_id - AND diff.score != 0; + a.creator_id AS creator_id, diff.upvotes - diff.downvotes AS score) + UPDATE + person AS a + SET + thing_score = a.thing_score + diff.score FROM ( + SELECT + creator_id, sum(score) AS score FROM thing_diff GROUP BY creator_id) AS diff + WHERE + a.id = diff.creator_id + AND diff.score != 0; RETURN NULL; END; $$); @@ -105,11 +80,11 @@ END; CALL r.create_triggers ('comment', $$ BEGIN + -- Prevent infinite recursion IF ( SELECT count(*) -FROM - select_old_and_new_rows) = 0 THEN + FROM select_old_and_new_rows AS old_and_new_rows) = 0 THEN RETURN NULL; END IF; @@ -170,60 +145,37 @@ WHERE a.id = diff.parent_id AND diff.child_count != 0; -WITH post_diff AS ( - UPDATE - post AS a - SET - comments = a.comments + diff.comments, - newest_comment_time = GREATEST (a.newest_comment_time, diff.newest_comment_time), - newest_comment_time_necro = GREATEST (a.newest_comment_time_necro, diff.newest_comment_time_necro) - FROM ( - SELECT - post.id AS post_id, - coalesce(sum(count_diff), 0) AS comments, - -- Old rows are excluded using `count_diff = 1` - max((comment).published) FILTER (WHERE count_diff = 1) AS newest_comment_time, - max((comment).published) FILTER (WHERE count_diff = 1 - -- Ignore comments from the post's creator - AND post.creator_id != (comment).creator_id - -- Ignore comments on old posts - AND post.published > ((comment).published - '2 days'::interval)) AS newest_comment_time_necro, - r.is_counted (post.*) AS include_in_community_aggregates - FROM - select_old_and_new_rows AS old_and_new_rows - LEFT JOIN post ON post.id = (comment).post_id - WHERE - r.is_counted (comment) - GROUP BY - post.id) AS diff - WHERE - a.id = diff.post_id - AND (diff.comments, - GREATEST (a.newest_comment_time, diff.newest_comment_time), - GREATEST (a.newest_comment_time_necro, diff.newest_comment_time_necro)) != (0, - a.newest_comment_time, - a.newest_comment_time_necro) - RETURNING - a.community_id, - diff.comments, - diff.include_in_community_aggregates) UPDATE - community AS a + post AS a SET - comments = a.comments + diff.comments + comments = a.comments + diff.comments, + newest_comment_time = GREATEST (a.newest_comment_time, diff.newest_comment_time), + newest_comment_time_necro = GREATEST (a.newest_comment_time_necro, diff.newest_comment_time_necro) FROM ( SELECT - community_id, - sum(comments) AS comments - FROM - post_diff - WHERE - post_diff.include_in_community_aggregates - GROUP BY - community_id) AS diff + post.id AS post_id, + coalesce(sum(count_diff), 0) AS comments, + -- Old rows are excluded using `count_diff = 1` + max((comment).published) FILTER (WHERE count_diff = 1) AS newest_comment_time, + max((comment).published) FILTER (WHERE count_diff = 1 + -- Ignore comments from the post's creator + AND post.creator_id != (comment).creator_id + -- Ignore comments on old posts + AND post.published > ((comment).published - '2 days'::interval)) AS newest_comment_time_necro +FROM + select_old_and_new_rows AS old_and_new_rows + LEFT JOIN post ON post.id = (comment).post_id WHERE - a.id = diff.community_id - AND diff.comments != 0; + r.is_counted (comment) +GROUP BY + post.id) AS diff +WHERE + a.id = diff.post_id + AND (diff.comments, + GREATEST (a.newest_comment_time, diff.newest_comment_time), + GREATEST (a.newest_comment_time_necro, diff.newest_comment_time_necro)) != (0, + a.newest_comment_time, + a.newest_comment_time_necro); UPDATE local_site AS a @@ -248,41 +200,31 @@ $$); CALL r.create_triggers ('post', $$ BEGIN - IF ( + UPDATE + person AS a + SET + post_count = a.post_count + diff.post_count + FROM ( SELECT - count(*) -FROM - select_old_and_new_rows) = 0 THEN - RETURN NULL; - -END IF; - -UPDATE - person AS a -SET - post_count = a.post_count + diff.post_count -FROM ( - SELECT - (post).creator_id, - coalesce(sum(count_diff), 0) AS post_count - FROM - select_old_and_new_rows AS old_and_new_rows - WHERE - r.is_counted (post) - GROUP BY - (post).creator_id) AS diff + (post).creator_id, coalesce(sum(count_diff), 0) AS post_count + FROM select_old_and_new_rows AS old_and_new_rows + WHERE + r.is_counted (post) + GROUP BY (post).creator_id) AS diff WHERE a.id = diff.creator_id - AND diff.post_count != 0; + AND diff.post_count != 0; UPDATE community AS a SET - posts = a.posts + diff.posts + posts = a.posts + diff.posts, + comments = a.comments + diff.comments FROM ( SELECT (post).community_id, - coalesce(sum(count_diff), 0) AS posts + coalesce(sum(count_diff), 0) AS posts, + coalesce(sum(count_diff * (post).comments), 0) AS comments FROM select_old_and_new_rows AS old_and_new_rows WHERE @@ -291,7 +233,9 @@ FROM ( (post).community_id) AS diff WHERE a.id = diff.community_id - AND diff.posts != 0; + AND (diff.posts, + diff.comments) != (0, + 0); UPDATE local_site AS a @@ -316,27 +260,17 @@ $$); CALL r.create_triggers ('community', $$ BEGIN - IF ( + UPDATE + local_site AS a + SET + communities = a.communities + diff.communities + FROM ( SELECT - count(*) -FROM - select_old_and_new_rows) = 0 THEN - RETURN NULL; - -END IF; - -UPDATE - local_site AS a -SET - communities = a.communities + diff.communities -FROM ( - SELECT - coalesce(sum(count_diff), 0) AS communities - FROM - select_old_and_new_rows AS old_and_new_rows - WHERE - r.is_counted (community) - AND (community).local) AS diff + coalesce(sum(count_diff), 0) AS communities + FROM select_old_and_new_rows AS old_and_new_rows + WHERE + r.is_counted (community) + AND (community).local) AS diff WHERE diff.communities != 0; @@ -348,25 +282,15 @@ $$); CALL r.create_triggers ('person', $$ BEGIN - IF ( + UPDATE + local_site AS a + SET + users = a.users + diff.users + FROM ( SELECT - count(*) -FROM - select_old_and_new_rows) = 0 THEN - RETURN NULL; - -END IF; - -UPDATE - local_site AS a -SET - users = a.users + diff.users -FROM ( - SELECT - coalesce(sum(count_diff), 0) AS users - FROM - select_old_and_new_rows AS old_and_new_rows - WHERE (person).local) AS diff + coalesce(sum(count_diff), 0) AS users + FROM select_old_and_new_rows AS old_and_new_rows + WHERE (person).local) AS diff WHERE diff.users != 0; @@ -376,80 +300,25 @@ END; $$); --- For community.comments, don't include comments of deleted or removed posts -CREATE FUNCTION r.update_comment_count_from_post () - RETURNS TRIGGER - LANGUAGE plpgsql - AS $$ -BEGIN - UPDATE - community AS a - SET - comments = a.comments + diff.comments - FROM ( - SELECT - old_post.community_id, - sum(( - CASE WHEN r.is_counted (new_post.*) THEN - 1 - ELSE - -1 - END) * post.comments) AS comments - FROM - new_post - INNER JOIN old_post ON new_post.id = old_post.id - AND (r.is_counted (new_post.*) != r.is_counted (old_post.*)) - INNER JOIN post ON post.id = new_post.id - GROUP BY - old_post.community_id) AS diff -WHERE - a.id = diff.community_id - AND diff.comments != 0; - RETURN NULL; -END; -$$; - -CREATE TRIGGER comment_count - AFTER UPDATE ON post REFERENCING OLD TABLE AS old_post NEW TABLE AS new_post - FOR EACH STATEMENT - EXECUTE FUNCTION r.update_comment_count_from_post (); - -- Count subscribers for communities. -- subscribers should be updated only when a local community is followed by a local or remote person. -- subscribers_local should be updated only when a local person follows a local or remote community. CALL r.create_triggers ('community_actions', $$ BEGIN - IF ( + UPDATE + community AS a + SET + subscribers = a.subscribers + diff.subscribers, subscribers_local = a.subscribers_local + diff.subscribers_local + FROM ( SELECT - count(*) -FROM - select_old_and_new_rows) = 0 THEN - RETURN NULL; - -END IF; - -UPDATE - community AS a -SET - subscribers = a.subscribers + diff.subscribers, - subscribers_local = a.subscribers_local + diff.subscribers_local -FROM ( - SELECT - (community_actions).community_id, - coalesce(sum(count_diff) FILTER (WHERE community.local), 0) AS subscribers, - coalesce(sum(count_diff) FILTER (WHERE person.local), 0) AS subscribers_local - FROM - select_old_and_new_rows AS old_and_new_rows + (community_actions).community_id, coalesce(sum(count_diff) FILTER (WHERE community.local), 0) AS subscribers, coalesce(sum(count_diff) FILTER (WHERE person.local), 0) AS subscribers_local + FROM select_old_and_new_rows AS old_and_new_rows LEFT JOIN community ON community.id = (community_actions).community_id LEFT JOIN person ON person.id = (community_actions).person_id -WHERE (community_actions).followed IS NOT NULL -GROUP BY - (community_actions).community_id) AS diff + WHERE (community_actions).followed IS NOT NULL GROUP BY (community_actions).community_id) AS diff WHERE a.id = diff.community_id - AND (diff.subscribers, - diff.subscribers_local) != (0, - 0); + AND (diff.subscribers, diff.subscribers_local) != (0, 0); RETURN NULL; @@ -459,32 +328,17 @@ $$); CALL r.create_triggers ('post_report', $$ BEGIN - IF ( + UPDATE + post AS a + SET + report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count + FROM ( SELECT - count(*) -FROM - select_old_and_new_rows) = 0 THEN - RETURN NULL; - -END IF; - -UPDATE - post AS a -SET - report_count = a.report_count + diff.report_count, - unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count -FROM ( - SELECT - (post_report).post_id, - coalesce(sum(count_diff), 0) AS report_count, - coalesce(sum(count_diff) FILTER (WHERE NOT (post_report).resolved - AND NOT (post_report).violates_instance_rules), 0) AS unresolved_report_count - FROM - select_old_and_new_rows AS old_and_new_rows - GROUP BY - (post_report).post_id) AS diff + (post_report).post_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (post_report).resolved + AND NOT (post_report).violates_instance_rules), 0) AS unresolved_report_count +FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (post_report).post_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) - AND a.id = diff.post_id; +AND a.id = diff.post_id; RETURN NULL; @@ -494,32 +348,17 @@ $$); CALL r.create_triggers ('comment_report', $$ BEGIN - IF ( + UPDATE + comment AS a + SET + report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count + FROM ( SELECT - count(*) -FROM - select_old_and_new_rows) = 0 THEN - RETURN NULL; - -END IF; - -UPDATE - comment AS a -SET - report_count = a.report_count + diff.report_count, - unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count -FROM ( - SELECT - (comment_report).comment_id, - coalesce(sum(count_diff), 0) AS report_count, - coalesce(sum(count_diff) FILTER (WHERE NOT (comment_report).resolved - AND NOT (comment_report).violates_instance_rules), 0) AS unresolved_report_count - FROM - select_old_and_new_rows AS old_and_new_rows - GROUP BY - (comment_report).comment_id) AS diff + (comment_report).comment_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (comment_report).resolved + AND NOT (comment_report).violates_instance_rules), 0) AS unresolved_report_count +FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (comment_report).comment_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) - AND a.id = diff.comment_id; +AND a.id = diff.comment_id; RETURN NULL; @@ -529,29 +368,14 @@ $$); CALL r.create_triggers ('community_report', $$ BEGIN - IF ( + UPDATE + community AS a + SET + report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count + FROM ( SELECT - count(*) -FROM - select_old_and_new_rows) = 0 THEN - RETURN NULL; - -END IF; - -UPDATE - community AS a -SET - report_count = a.report_count + diff.report_count, - unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count -FROM ( - SELECT - (community_report).community_id, - coalesce(sum(count_diff), 0) AS report_count, - coalesce(sum(count_diff) FILTER (WHERE NOT (community_report).resolved), 0) AS unresolved_report_count - FROM - select_old_and_new_rows AS old_and_new_rows - GROUP BY - (community_report).community_id) AS diff + (community_report).community_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (community_report).resolved), 0) AS unresolved_report_count + FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (community_report).community_id) AS diff WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) AND a.id = diff.community_id; @@ -561,48 +385,7 @@ END; $$); -CREATE FUNCTION r.post_from_post () - RETURNS TRIGGER - LANGUAGE plpgsql - AS $$ -BEGIN - UPDATE - post - SET - newest_comment_time = new_post.published, - newest_comment_time_necro = new_post.published, - instance_id = community.instance_id - FROM - new_post - INNER JOIN community ON community.id = new_post.community_id - WHERE - post.id = new_post.id; - RETURN NULL; -END; -$$; - -CREATE TRIGGER aggregates - AFTER INSERT ON post REFERENCING NEW TABLE AS new_post - FOR EACH STATEMENT - EXECUTE FUNCTION r.post_from_post (); - -- Change the order of some cascading deletions to make deletion triggers run before the deletion of rows that the triggers need to read -CREATE FUNCTION r.delete_comments_before_post () - RETURNS TRIGGER - LANGUAGE plpgsql - AS $$ -BEGIN - DELETE FROM comment AS c - WHERE c.post_id = OLD.id; - RETURN OLD; -END; -$$; - -CREATE TRIGGER delete_comments - BEFORE DELETE ON post - FOR EACH ROW - EXECUTE FUNCTION r.delete_comments_before_post (); - CREATE FUNCTION r.delete_follow_before_person () RETURNS TRIGGER LANGUAGE plpgsql @@ -653,6 +436,16 @@ BEGIN IF NEW.local THEN NEW.ap_id = coalesce(NEW.ap_id, r.local_url ('/post/' || NEW.id::text)); END IF; + -- Set aggregates + NEW.newest_comment_time = NEW.published; + NEW.newest_comment_time_necro = NEW.published; + NEW.instance_id = ( + SELECT + community.instance_id + FROM + community + WHERE + community.id = NEW.community_id); RETURN NEW; END $$; @@ -1048,7 +841,7 @@ BEGIN SET score = NEW.users_active_month WHERE - community_id = NEW.community_id; + community_id = NEW.id; RETURN NULL; END $$; diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 3367c91bb6..ab449c4272 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -26,6 +26,9 @@ pub const CACHE_DURATION_FEDERATION: Duration = Duration::from_millis(500); #[cfg(not(debug_assertions))] pub const CACHE_DURATION_FEDERATION: Duration = Duration::from_secs(60); +#[cfg(debug_assertions)] +pub const CACHE_DURATION_API: Duration = Duration::from_secs(0); +#[cfg(not(debug_assertions))] pub const CACHE_DURATION_API: Duration = Duration::from_secs(1); pub const MAX_COMMENT_DEPTH_LIMIT: usize = 50; diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql index 06f6fdc0c2..783cba99cc 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/down.sql @@ -226,7 +226,8 @@ ALTER TABLE community DROP COLUMN subscribers_local, DROP COLUMN report_count, DROP COLUMN unresolved_report_count, - DROP COLUMN interactions_month; + DROP COLUMN interactions_month, + ALTER CONSTRAINT community_instance_id_fkey NOT DEFERRABLE INITIALLY IMMEDIATE; CREATE INDEX idx_community_aggregates_hot ON public.community_aggregates USING btree (hot_rank DESC); diff --git a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql index 295fca18db..e17563f1fa 100644 --- a/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql +++ b/migrations/2025-02-07-105516_remove-aggregate-tables/up.sql @@ -157,7 +157,8 @@ ALTER TABLE community ADD COLUMN subscribers_local bigint NOT NULL DEFAULT 0, ADD COLUMN report_count smallint NOT NULL DEFAULT 0, ADD COLUMN unresolved_report_count smallint NOT NULL DEFAULT 0, - ADD COLUMN interactions_month bigint NOT NULL DEFAULT 0; + ADD COLUMN interactions_month bigint NOT NULL DEFAULT 0, + ALTER CONSTRAINT community_instance_id_fkey DEFERRABLE INITIALLY DEFERRED; UPDATE community