Persistent, performant, reliable federation queue (#3605)

* persistent activity queue

* fixes

* fixes

* make federation workers function callable from outside

* log federation instances

* dead instance detection not needed here

* taplo fmt

* split federate bin/lib

* minor fix

* better logging

* log

* create struct to hold cancellable task for readability

* use boxfuture for readability

* reset submodule

* fix

* fix lint

* swap

* remove json column, use separate array columns instead

* some review comments

* make worker a struct for readability

* minor readability

* add local filter to community follower view

* remove separate lemmy_federate entry point

* fix remaining duration

* address review comments mostly

* fix lint

* upgrade actitypub-fed to simpler interface

* fix sql format

* increase delays a bit

* fixes after merge

* remove selectable

* fix instance selectable

* add comment

* start federation based on latest id at the time

* rename federate process args

* dead instances in one query

* filter follow+report activities by local

* remove synchronous federation

remove activity sender queue

* lint

* fix federation tests by waiting for results to change

* fix fed test

* fix comment report

* wait some more

* Apply suggestions from code review

Co-authored-by: SorteKanin <[email protected]>

* fix most remaining tests

* wait until private messages

* fix community tests

* fix community tests

* move arg parse

* use instance_id instead of domain in federation_queue_state table

---------

Co-authored-by: Dessalines <[email protected]>
Co-authored-by: SorteKanin <[email protected]>

Cargo.toml

@@ -54,6 +54,7 @@ members = [
   "crates/db_views_actor",
   "crates/db_views_actor",
   "crates/routes",
+  "crates/federate",
 ]
 
 [workspace.dependencies]
@@ -67,7 +68,7 @@ lemmy_routes = { version = "=0.18.1", path = "./crates/routes" }
 lemmy_db_views = { version = "=0.18.1", path = "./crates/db_views" }
 lemmy_db_views_actor = { version = "=0.18.1", path = "./crates/db_views_actor" }
 lemmy_db_views_moderator = { version = "=0.18.1", path = "./crates/db_views_moderator" }
-activitypub_federation = { version = "0.5.0-beta.2", default-features = false, features = [
+activitypub_federation = { version = "0.5.0-beta.3", default-features = false, features = [
   "actix-web",
 ] }
 diesel = "2.1.0"
@@ -88,7 +89,6 @@ tracing-error = "0.2.0"
 tracing-log = "0.1.3"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
 url = { version = "2.4.0", features = ["serde"] }
-url_serde = "0.2.0"
 reqwest = { version = "0.11.18", features = ["json", "blocking", "gzip"] }
 reqwest-middleware = "0.2.2"
 reqwest-tracing = "0.4.5"
@@ -119,7 +119,6 @@ futures = "0.3.28"
 http = "0.2.9"
 percent-encoding = "2.3.0"
 rosetta-i18n = "0.1.3"
-rand = "0.8.5"
 opentelemetry = { version = "0.19.0", features = ["rt-tokio"] }
 tracing-opentelemetry = { version = "0.19.0" }
 ts-rs = { version = "7.0.0", features = ["serde-compat", "chrono-impl"] }
@@ -167,3 +166,5 @@ tokio-postgres-rustls = { workspace = true }
 chrono = { workspace = true }
 prometheus = { version = "0.13.3", features = ["process"], optional = true }
 actix-web-prom = { version = "0.6.0", optional = true }
+clap = { version = "4.3.19", features = ["derive"] }
+lemmy_federate = { version = "0.18.1", path = "crates/federate" }

api_tests/prepare-drone-federation-test.sh

@@ -4,7 +4,7 @@
 set -e
 
 export RUST_BACKTRACE=1
-export RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
+export RUST_LOG="warn,lemmy_server=debug,lemmy_federate=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
 
 for INSTANCE in lemmy_alpha lemmy_beta lemmy_gamma lemmy_delta lemmy_epsilon; do
   echo "DB URL: ${LEMMY_DATABASE_URL} INSTANCE: $INSTANCE"
@@ -26,7 +26,7 @@ if [ -z "$DO_WRITE_HOSTS_FILE" ]; then
   fi
 else
   for INSTANCE in lemmy-alpha lemmy-beta lemmy-gamma lemmy-delta lemmy-epsilon; do
-    echo "127.0.0.1 $INSTANCE" >> /etc/hosts
+    echo "127.0.0.1 $INSTANCE" >>/etc/hosts
   done
 fi
 

api_tests/run-federation-test.sh

@@ -2,7 +2,6 @@
 set -e
 
 export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432
-export LEMMY_SYNCHRONOUS_FEDERATION=1 # currently this is true in debug by default, but still.
 pushd ..
 cargo build
 rm target/lemmy_server || true