Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify slur regex #5442

Merged
merged 6 commits into from
Feb 20, 2025
Merged

Simplify slur regex #5442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
07a6373
Simplify logic for slur regex
Nutomic Feb 20, 2025
df1c499
Fix tests
Nutomic Feb 20, 2025
d6c405b
clippy
Nutomic Feb 20, 2025
5588d33
fix test
Nutomic Feb 20, 2025
9ed3063
cache regex
Nutomic Feb 20, 2025
9d9725c
longer cache duration
Nutomic Feb 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/api/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ anyhow = { workspace = true }
tracing = { workspace = true }
chrono = { workspace = true }
url = { workspace = true }
regex = { workspace = true }
hound = "3.5.1"
sitemap-rs = "0.2.2"
totp-rs = { version = "5.6.0", features = ["gen_secret", "otpauth"] }
Expand Down
8 changes: 3 additions & 5 deletions crates/api/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use lemmy_api_common::{
community::BanFromCommunity,
context::LemmyContext,
send_activity::{ActivityChannel, SendActivityData},
utils::{check_expire_time, local_site_to_slur_regex},
utils::check_expire_time,
};
use lemmy_db_schema::{
source::{
Expand All @@ -15,7 +15,6 @@ use lemmy_db_schema::{
CommunityPersonBan,
CommunityPersonBanForm,
},
local_site::LocalSite,
mod_log::moderator::{ModBanFromCommunity, ModBanFromCommunityForm},
person::Person,
},
Expand All @@ -26,6 +25,7 @@ use lemmy_utils::{
error::{LemmyErrorExt, LemmyErrorType, LemmyResult},
utils::slurs::check_slurs,
};
use regex::Regex;
use std::io::Cursor;
use totp_rs::{Secret, TOTP};

Expand Down Expand Up @@ -79,9 +79,7 @@ pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> LemmyResult<String> {
}

/// Check size of report
pub(crate) fn check_report_reason(reason: &str, local_site: &LocalSite) -> LemmyResult<()> {
let slur_regex = &local_site_to_slur_regex(local_site);

pub(crate) fn check_report_reason(reason: &str, slur_regex: &Regex) -> LemmyResult<()> {
check_slurs(reason, slur_regex)?;
if reason.is_empty() {
Err(LemmyErrorType::ReportReasonRequired)?
Expand Down
9 changes: 2 additions & 7 deletions crates/api/src/local_user/save_settings.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,7 @@ use actix_web::web::Json;
use lemmy_api_common::{
context::LemmyContext,
person::SaveUserSettings,
utils::{
get_url_blocklist,
local_site_to_slur_regex,
process_markdown_opt,
send_verification_email,
},
utils::{get_url_blocklist, process_markdown_opt, send_verification_email, slur_regex},
SuccessResponse,
};
use lemmy_db_schema::{
Expand All @@ -35,7 +30,7 @@ pub async fn save_user_settings(
) -> LemmyResult<Json<SuccessResponse>> {
let site_view = SiteView::read_local(&mut context.pool()).await?;

let slur_regex = local_site_to_slur_regex(&site_view.local_site);
let slur_regex = slur_regex(&context).await?;
let url_blocklist = get_url_blocklist(&context).await?;
let bio = diesel_string_update(
process_markdown_opt(&data.bio, &slur_regex, &url_blocklist, &context)
Expand Down
7 changes: 4 additions & 3 deletions crates/api/src/reports/comment_report/create.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use lemmy_api_common::{
check_comment_deleted_or_removed,
check_community_user_action,
send_new_report_email_to_admins,
slur_regex,
},
};
use lemmy_db_schema::{
Expand All @@ -27,10 +28,9 @@ pub async fn create_comment_report(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<CommentReportResponse>> {
let local_site = LocalSite::read(&mut context.pool()).await?;

let reason = data.reason.trim().to_string();
check_report_reason(&reason, &local_site)?;
let slur_regex = slur_regex(&context).await?;
check_report_reason(&reason, &slur_regex)?;

let person_id = local_user_view.person.id;
let comment_id = data.comment_id;
Expand Down Expand Up @@ -67,6 +67,7 @@ pub async fn create_comment_report(
CommentReportView::read(&mut context.pool(), report.id, person_id).await?;

// Email the admins
let local_site = LocalSite::read(&mut context.pool()).await?;
if local_site.reports_email_admins {
send_new_report_email_to_admins(
&comment_report_view.creator.name,
Expand Down
7 changes: 4 additions & 3 deletions crates/api/src/reports/post_report/create.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use lemmy_api_common::{
check_community_user_action,
check_post_deleted_or_removed,
send_new_report_email_to_admins,
slur_regex,
},
};
use lemmy_db_schema::{
Expand All @@ -27,10 +28,9 @@ pub async fn create_post_report(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<PostReportResponse>> {
let local_site = LocalSite::read(&mut context.pool()).await?;

let reason = data.reason.trim().to_string();
check_report_reason(&reason, &local_site)?;
let slur_regex = slur_regex(&context).await?;
check_report_reason(&reason, &slur_regex)?;

let person_id = local_user_view.person.id;
let post_id = data.post_id;
Expand Down Expand Up @@ -62,6 +62,7 @@ pub async fn create_post_report(
let post_report_view = PostReportView::read(&mut context.pool(), report.id, person_id).await?;

// Email the admins
let local_site = LocalSite::read(&mut context.pool()).await?;
if local_site.reports_email_admins {
send_new_report_email_to_admins(
&post_report_view.creator.name,
Expand Down
8 changes: 4 additions & 4 deletions crates/api/src/reports/private_message_report/create.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use actix_web::web::{Data, Json};
use lemmy_api_common::{
context::LemmyContext,
reports::private_message::{CreatePrivateMessageReport, PrivateMessageReportResponse},
utils::send_new_report_email_to_admins,
utils::{send_new_report_email_to_admins, slur_regex},
};
use lemmy_db_schema::{
source::{
Expand All @@ -21,10 +21,9 @@ pub async fn create_pm_report(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<PrivateMessageReportResponse>> {
let local_site = LocalSite::read(&mut context.pool()).await?;

let reason = data.reason.trim().to_string();
check_report_reason(&reason, &local_site)?;
let slur_regex = slur_regex(&context).await?;
check_report_reason(&reason, &slur_regex)?;

let person_id = local_user_view.person.id;
let private_message_id = data.private_message_id;
Expand All @@ -50,6 +49,7 @@ pub async fn create_pm_report(
PrivateMessageReportView::read(&mut context.pool(), report.id).await?;

// Email the admins
let local_site = LocalSite::read(&mut context.pool()).await?;
if local_site.reports_email_admins {
send_new_report_email_to_admins(
&private_message_report_view.creator.name,
Expand Down
33 changes: 20 additions & 13 deletions crates/api_common/src/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ use lemmy_utils::{
spawn_try_task,
utils::{
markdown::{image_links::markdown_rewrite_image_links, markdown_check_for_blocked_urls},
slurs::{build_slur_regex, remove_slurs},
validation::clean_urls_in_text,
slurs::remove_slurs,
validation::{build_and_check_regex, clean_urls_in_text},
},
CacheLock,
CACHE_DURATION_FEDERATION,
Expand Down Expand Up @@ -540,15 +540,22 @@ pub fn local_site_rate_limit_to_rate_limit_config(
})
}

pub fn local_site_to_slur_regex(local_site: &LocalSite) -> Option<LemmyResult<Regex>> {
build_slur_regex(local_site.slur_filter_regex.as_deref())
}

pub fn local_site_opt_to_slur_regex(local_site: &Option<LocalSite>) -> Option<LemmyResult<Regex>> {
local_site
.as_ref()
.map(local_site_to_slur_regex)
.unwrap_or(None)
pub async fn slur_regex(context: &LemmyContext) -> LemmyResult<Regex> {
static CACHE: CacheLock<Regex> = LazyLock::new(|| {
Cache::builder()
.max_capacity(1)
.time_to_live(CACHE_DURATION_FEDERATION)
.build()
});
Ok(
CACHE
.try_get_with((), async {
let local_site = LocalSite::read(&mut context.pool()).await.ok();
build_and_check_regex(local_site.and_then(|s| s.slur_filter_regex).as_deref())
})
.await
.map_err(|e| anyhow::anyhow!("Failed to construct regex: {e}"))?,
)
}

pub async fn get_url_blocklist(context: &LemmyContext) -> LemmyResult<RegexSet> {
Expand Down Expand Up @@ -1037,7 +1044,7 @@ pub fn check_conflicting_like_filters(

pub async fn process_markdown(
text: &str,
slur_regex: &Option<LemmyResult<Regex>>,
slur_regex: &Regex,
Copy link
Member

@dessalines dessalines Feb 20, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now that this is no longer an option, isn't it going to throw the new lemmyerror you created? Or maybe since it matches nothing it should be ok.

I feel like Option<Regex (which is the reality in the DB) is probably better and more explicit than a missing regex that default matches nothing, but its up to you I spose.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The error would only be thrown if the regex is invalid, and that shouldnt happen as its checked before create/update.

Option<Regex> also works, but this way the code is a bit simpler.

url_blocklist: &RegexSet,
context: &LemmyContext,
) -> LemmyResult<String> {
Expand Down Expand Up @@ -1069,7 +1076,7 @@ pub async fn process_markdown(

pub async fn process_markdown_opt(
text: &Option<String>,
slur_regex: &Option<LemmyResult<Regex>>,
slur_regex: &Regex,
url_blocklist: &RegexSet,
context: &LemmyContext,
) -> LemmyResult<Option<String>> {
Expand Down
7 changes: 2 additions & 5 deletions crates/api_crud/src/comment/create.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use lemmy_api_common::{
check_post_deleted_or_removed,
get_url_blocklist,
is_mod_or_admin,
local_site_to_slur_regex,
process_markdown,
slur_regex,
update_read_comments,
},
};
Expand All @@ -21,7 +21,6 @@ use lemmy_db_schema::{
source::{
comment::{Comment, CommentInsertForm, CommentLike, CommentLikeForm},
comment_reply::{CommentReply, CommentReplyUpdateForm},
local_site::LocalSite,
person_comment_mention::{PersonCommentMention, PersonCommentMentionUpdateForm},
},
traits::{Crud, Likeable},
Expand All @@ -38,9 +37,7 @@ pub async fn create_comment(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<CommentResponse>> {
let local_site = LocalSite::read(&mut context.pool()).await?;

let slur_regex = local_site_to_slur_regex(&local_site);
let slur_regex = slur_regex(&context).await?;
let url_blocklist = get_url_blocklist(&context).await?;
let content = process_markdown(&data.content, &slur_regex, &url_blocklist, &context).await?;
is_valid_body_field(&content, false)?;
Expand Down
16 changes: 3 additions & 13 deletions crates/api_crud/src/comment/update.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,12 @@ use lemmy_api_common::{
comment::{CommentResponse, EditComment},
context::LemmyContext,
send_activity::{ActivityChannel, SendActivityData},
utils::{
check_community_user_action,
get_url_blocklist,
local_site_to_slur_regex,
process_markdown_opt,
},
utils::{check_community_user_action, get_url_blocklist, process_markdown_opt, slur_regex},
};
use lemmy_db_schema::{
impls::actor_language::validate_post_language,
newtypes::PostOrCommentId,
source::{
comment::{Comment, CommentUpdateForm},
local_site::LocalSite,
},
source::comment::{Comment, CommentUpdateForm},
traits::Crud,
};
use lemmy_db_views::structs::{CommentView, LocalUserView};
Expand All @@ -33,8 +25,6 @@ pub async fn update_comment(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<CommentResponse>> {
let local_site = LocalSite::read(&mut context.pool()).await?;

let comment_id = data.comment_id;
let orig_comment = CommentView::read(
&mut context.pool(),
Expand Down Expand Up @@ -63,7 +53,7 @@ pub async fn update_comment(
)
.await?;

let slur_regex = local_site_to_slur_regex(&local_site);
let slur_regex = slur_regex(&context).await?;
let url_blocklist = get_url_blocklist(&context).await?;
let content = process_markdown_opt(&data.content, &slur_regex, &url_blocklist, &context).await?;
if let Some(content) = &content {
Expand Down
4 changes: 2 additions & 2 deletions crates/api_crud/src/community/create.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use lemmy_api_common::{
generate_inbox_url,
get_url_blocklist,
is_admin,
local_site_to_slur_regex,
process_markdown_opt,
slur_regex,
},
};
use lemmy_db_schema::{
Expand Down Expand Up @@ -54,7 +54,7 @@ pub async fn create_community(
Err(LemmyErrorType::OnlyAdminsCanCreateCommunities)?
}

let slur_regex = local_site_to_slur_regex(&local_site);
let slur_regex = slur_regex(&context).await?;
let url_blocklist = get_url_blocklist(&context).await?;
check_slurs(&data.name, &slur_regex)?;
check_slurs(&data.title, &slur_regex)?;
Expand Down
12 changes: 2 additions & 10 deletions crates/api_crud/src/community/update.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,12 @@ use lemmy_api_common::{
community::{CommunityResponse, EditCommunity},
context::LemmyContext,
send_activity::{ActivityChannel, SendActivityData},
utils::{
check_community_mod_action,
get_url_blocklist,
local_site_to_slur_regex,
process_markdown_opt,
},
utils::{check_community_mod_action, get_url_blocklist, process_markdown_opt, slur_regex},
};
use lemmy_db_schema::{
source::{
actor_language::{CommunityLanguage, SiteLanguage},
community::{Community, CommunityUpdateForm},
local_site::LocalSite,
},
traits::Crud,
utils::diesel_string_update,
Expand All @@ -34,9 +28,7 @@ pub async fn update_community(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<CommunityResponse>> {
let local_site = LocalSite::read(&mut context.pool()).await?;

let slur_regex = local_site_to_slur_regex(&local_site);
let slur_regex = slur_regex(&context).await?;
let url_blocklist = get_url_blocklist(&context).await?;
check_slurs_opt(&data.title, &slur_regex)?;

Expand Down
7 changes: 2 additions & 5 deletions crates/api_crud/src/post/create.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,16 @@ use lemmy_api_common::{
check_community_user_action,
get_url_blocklist,
honeypot_check,
local_site_to_slur_regex,
process_markdown_opt,
send_webmention,
slur_regex,
},
};
use lemmy_db_schema::{
impls::actor_language::validate_post_language,
newtypes::PostOrCommentId,
source::{
community::Community,
local_site::LocalSite,
post::{Post, PostInsertForm, PostLike, PostLikeForm, PostRead, PostReadForm},
},
traits::{Crud, Likeable},
Expand All @@ -48,11 +47,9 @@ pub async fn create_post(
context: Data<LemmyContext>,
local_user_view: LocalUserView,
) -> LemmyResult<Json<PostResponse>> {
let local_site = LocalSite::read(&mut context.pool()).await?;

honeypot_check(&data.honeypot)?;

let slur_regex = local_site_to_slur_regex(&local_site);
let slur_regex = slur_regex(&context).await?;
check_slurs(&data.name, &slur_regex)?;
let url_blocklist = get_url_blocklist(&context).await?;

Expand Down
Loading