From dd2e1f1c7dac6a633ef36300a7bdace006320cfd Mon Sep 17 00:00:00 2001 From: densumesh Date: Sat, 15 Feb 2025 18:50:30 -0700 Subject: [PATCH 1/2] feature: seperate into broccoli-queue and broccoli-cli --- .vscode/settings.json | 7 +- Cargo.lock | 301 +- Cargo.toml | 50 +- broccoli-cli/.env | 1 + broccoli-cli/Cargo.toml | 20 + broccoli-cli/src/lib.rs | 15 + broccoli-cli/src/main.rs | 126 + broccoli-queue/Cargo.lock | 3028 +++++++++++++++++ broccoli-queue/Cargo.toml | 59 + .../Dockerfile.rabbitmq | 0 .../benches}/queue_benchmark.rs | 0 .../examples}/consumer.rs | 0 .../examples}/publisher.rs | 0 run-tests.sh => broccoli-queue/run-tests.sh | 0 {src => broccoli-queue/src}/brokers/broker.rs | 0 .../src}/brokers/connect.rs | 70 +- broccoli-queue/src/brokers/management.rs | 54 + broccoli-queue/src/brokers/mod.rs | 13 + .../src}/brokers/rabbitmq/broker.rs | 37 +- .../src/brokers/rabbitmq/management.rs | 110 + .../src}/brokers/rabbitmq/mod.rs | 3 +- broccoli-queue/src/brokers/rabbitmq/utils.rs | 128 + .../src}/brokers/redis/broker.rs | 0 .../src/brokers/redis/management.rs | 174 + broccoli-queue/src/brokers/redis/mod.rs | 9 + broccoli-queue/src/brokers/redis/utils.rs | 274 ++ {src => broccoli-queue/src}/error.rs | 14 + {src => broccoli-queue/src}/lib.rs | 0 {src => broccoli-queue/src}/queue.rs | 65 + {tests => broccoli-queue/tests}/common/mod.rs | 0 broccoli-queue/tests/edge_cases.rs | 507 +++ broccoli-queue/tests/fairness.rs | 382 +++ broccoli-queue/tests/happy_path.rs | 710 ++++ errors.txt | 0 src/brokers/mod.rs | 11 +- src/brokers/rabbitmq/utils.rs | 166 +- src/brokers/redis/mod.rs | 6 - src/brokers/redis/utils.rs | 318 +- 38 files changed, 6096 insertions(+), 562 deletions(-) create mode 100644 broccoli-cli/.env create mode 100644 broccoli-cli/Cargo.toml create mode 100644 broccoli-cli/src/lib.rs create mode 100644 broccoli-cli/src/main.rs create mode 100644 broccoli-queue/Cargo.lock create mode 100644 broccoli-queue/Cargo.toml rename Dockerfile.rabbitmq => broccoli-queue/Dockerfile.rabbitmq (100%) rename {benches => broccoli-queue/benches}/queue_benchmark.rs (100%) rename {examples => broccoli-queue/examples}/consumer.rs (100%) rename {examples => broccoli-queue/examples}/publisher.rs (100%) rename run-tests.sh => broccoli-queue/run-tests.sh (100%) rename {src => broccoli-queue/src}/brokers/broker.rs (100%) rename {src => broccoli-queue/src}/brokers/connect.rs (50%) create mode 100644 broccoli-queue/src/brokers/management.rs create mode 100644 broccoli-queue/src/brokers/mod.rs rename {src => broccoli-queue/src}/brokers/rabbitmq/broker.rs (89%) create mode 100644 broccoli-queue/src/brokers/rabbitmq/management.rs rename {src => broccoli-queue/src}/brokers/rabbitmq/mod.rs (53%) create mode 100644 broccoli-queue/src/brokers/rabbitmq/utils.rs rename {src => broccoli-queue/src}/brokers/redis/broker.rs (100%) create mode 100644 broccoli-queue/src/brokers/redis/management.rs create mode 100644 broccoli-queue/src/brokers/redis/mod.rs create mode 100644 broccoli-queue/src/brokers/redis/utils.rs rename {src => broccoli-queue/src}/error.rs (87%) rename {src => broccoli-queue/src}/lib.rs (100%) rename {src => broccoli-queue/src}/queue.rs (93%) rename {tests => broccoli-queue/tests}/common/mod.rs (100%) create mode 100644 broccoli-queue/tests/edge_cases.rs create mode 100644 broccoli-queue/tests/fairness.rs create mode 100644 broccoli-queue/tests/happy_path.rs delete mode 100644 errors.txt delete mode 100644 src/brokers/redis/mod.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index 8d25c7d..29a1270 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,9 @@ { "rust-analyzer.showUnlinkedFileNotification": false, - "rust-analyzer.cargo.features": ["redis", "rabbitmq", "test-fairness"] + "rust-analyzer.cargo.features": [ + "redis", + "rabbitmq", + "test-fairness", + "management" + ] } diff --git a/Cargo.lock b/Cargo.lock index 66142e4..3ac9310 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,6 +156,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + [[package]] name = "arc-swap" version = "1.7.1" @@ -164,9 +170,9 @@ checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "asn1-rs" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +checksum = "607495ec7113b178fbba7a6166a27f99e774359ef4823adbefd756b5b81d7970" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -174,15 +180,15 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 1.0.69", + "thiserror", "time", ] [[package]] name = "asn1-rs-derive" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", @@ -331,9 +337,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.85" +version = "0.1.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" +checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" dependencies = [ "proc-macro2", "quote", @@ -437,6 +443,24 @@ dependencies = [ "piper", ] +[[package]] +name = "broccoli-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "broccoli_queue", + "clap", + "colored", + "env_logger", + "humansize", + "log", + "redis", + "serde_json", + "tabwriter", + "thiserror", + "tokio", +] + [[package]] name = "broccoli_queue" version = "0.3.3" @@ -457,7 +481,7 @@ dependencies = [ "serde", "serde_json", "sha256", - "thiserror 2.0.11", + "thiserror", "time", "tokio", "uuid", @@ -465,9 +489,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "byteorder" @@ -477,9 +501,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" [[package]] name = "cast" @@ -498,9 +522,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.10" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" dependencies = [ "shlex", ] @@ -565,21 +589,36 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.27" +version = "4.5.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" +checksum = "8acebd8ad879283633b343856142139f2da2317c96b05b4dd6181c61e2480184" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] name = "clap_builder" -version = "4.5.27" +version = "4.5.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +checksum = "f6ba32cbda51c7e1dfd49acc1457ba1a7dec5b64fe360e828acb13ca8dc9c2f9" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -606,6 +645,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + [[package]] name = "combine" version = "4.6.7" @@ -761,15 +810,15 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" +checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" [[package]] name = "deadpool" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6541a3916932fe57768d4be0b1ffb5ec7cbf74ca8c903fdfd5c0fe8aa958f0ed" +checksum = "5ed5957ff93768adf7a65ab167a17835c3d2c3c50d084fe305174c112f468e2f" dependencies = [ "deadpool-runtime", "num_cpus", @@ -811,9 +860,9 @@ dependencies = [ [[package]] name = "der-parser" -version = "9.0.0" +version = "10.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ "asn1-rs", "displaydoc", @@ -1131,7 +1180,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", ] [[package]] @@ -1156,6 +1217,12 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -1183,6 +1250,15 @@ dependencies = [ "digest", ] +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "humantime" version = "2.1.0" @@ -1466,6 +1542,12 @@ version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -1514,9 +1596,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b" dependencies = [ "adler2", ] @@ -1528,7 +1610,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -1597,18 +1679,18 @@ dependencies = [ [[package]] name = "oid-registry" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" dependencies = [ "asn1-rs", ] [[package]] name = "once_cell" -version = "1.20.2" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "oorandom" @@ -1624,9 +1706,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "p12-keystore" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7b60d0b2dcace322e6e8c4499c4c8bdf331c1bae046a54be5e4191c3610286" +checksum = "a09eaa3a6d8884c204c2ab17e313f563b524362e62567f09ba27857a6e31257f" dependencies = [ "cbc", "cms", @@ -1640,7 +1722,7 @@ dependencies = [ "rc2", "sha1", "sha2", - "thiserror 1.0.69", + "thiserror", "x509-parser", ] @@ -1834,7 +1916,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -1857,20 +1939,20 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ - "libc", "rand_chacha", "rand_core", + "zerocopy 0.8.18", ] [[package]] name = "rand_chacha" -version = "0.3.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", "rand_core", @@ -1878,11 +1960,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" dependencies = [ - "getrandom", + "getrandom 0.3.1", + "zerocopy 0.8.18", ] [[package]] @@ -1994,15 +2077,14 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "ring" -version = "0.17.8" +version = "0.17.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "e75ec5e92c4d8aede845126adc388046234541629e76029599ed35a003c7ed24" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] @@ -2051,9 +2133,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.21" +version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "once_cell", "ring", @@ -2100,9 +2182,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" [[package]] name = "rustls-webpki" @@ -2123,9 +2205,9 @@ checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "salsa20" @@ -2216,9 +2298,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.137" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -2293,9 +2375,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "socket2" @@ -2342,6 +2424,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2350,9 +2438,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.96" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -2370,6 +2458,15 @@ dependencies = [ "syn", ] +[[package]] +name = "tabwriter" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce91f2f0ec87dff7e6bcbbeb267439aa1188703003c6055193c821487400432" +dependencies = [ + "unicode-width", +] + [[package]] name = "tcp-stream" version = "0.28.0" @@ -2382,33 +2479,13 @@ dependencies = [ "rustls-pemfile", ] -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - [[package]] name = "thiserror" version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.11", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "thiserror-impl", ] [[package]] @@ -2560,9 +2637,15 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "untrusted" @@ -2601,11 +2684,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.12.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" +checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" dependencies = [ - "getrandom", + "getrandom 0.3.1", "serde", ] @@ -2637,6 +2720,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -2893,6 +2985,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "write16" version = "1.0.0" @@ -2918,9 +3019,9 @@ dependencies = [ [[package]] name = "x509-parser" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +checksum = "4569f339c0c402346d4a75a9e39cf8dad310e287eef1ff56d4c68e5067f53460" dependencies = [ "asn1-rs", "data-encoding", @@ -2929,7 +3030,7 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror 1.0.69", + "thiserror", "time", ] @@ -2964,7 +3065,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" +dependencies = [ + "zerocopy-derive 0.8.18", ] [[package]] @@ -2978,6 +3088,17 @@ dependencies = [ "syn", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index 59ce948..8b00cd9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,47 +1,3 @@ -[package] -name = "broccoli_queue" -version = "0.3.3" -edition = "2021" -license = "MIT" -description = "Broccoli is a simple, fast, and reliable job queue for Rust." -repository = "https://github.com/densumesh/broccoli" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] - -serde = { version = "1.0.216", features = ["derive"] } -serde_json = "1.0.133" -tokio = { version = "1.42.0", features = ["full"] } -uuid = { version = "1", features = ["v4", "serde"] } -async-trait = "0.1.83" -thiserror = "2.0.7" -futures = "0.3.31" -sha256 = "1.5.0" -log = "0.4.22" -time = "0.3.37" -lapin = { version = "2.5.0", optional = true } -deadpool = { version = "0.12.1", optional = true } -deadpool-lapin = { version = "0.12.1", optional = true } -bb8-redis = { version = "0.18.0", optional = true } -redis = { version = "0.27.5", features = [ - "tokio-rustls-comp", - "aio", -], optional = true } -dashmap = "6.1.0" - -[dev-dependencies] -chrono = { version = "0.4.39", features = ["serde"] } -env_logger = "0.11.5" -lazy_static = "1.4" -criterion = { version = "0.5", features = ["async_tokio"] } - -[features] -default = [] -redis = ["dep:bb8-redis", "dep:redis"] -rabbitmq = ["dep:lapin", "dep:deadpool", "dep:deadpool-lapin"] -test-fairness = [] - -[[bench]] -name = "queue_benchmark" -harness = false +[workspace] +resolver = "2" +members = ["broccoli-queue", "broccoli-cli"] diff --git a/broccoli-cli/.env b/broccoli-cli/.env new file mode 100644 index 0000000..abdc5d0 --- /dev/null +++ b/broccoli-cli/.env @@ -0,0 +1 @@ +BROCCOLI_QUEUE_URL=redis://localhost:6380 \ No newline at end of file diff --git a/broccoli-cli/Cargo.toml b/broccoli-cli/Cargo.toml new file mode 100644 index 0000000..79978f9 --- /dev/null +++ b/broccoli-cli/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "broccoli-cli" +version = "0.1.0" +edition = "2021" +description = "CLI tool for managing Broccoli queues" + +[dependencies] +broccoli_queue = { path = "../broccoli-queue", features = [ + "redis", + "rabbitmq", + "management", +] } +clap = { version = "4.4", features = ["derive", "env"] } +tokio = { version = "1.42.0", features = ["full"] } +redis = { version = "0.27.5", features = ["tokio-rustls-comp"] } +serde_json = "1.0.108" +anyhow = "1.0" +thiserror = "2.0.11" +dotenv = "0.15.0" +prettytable = "0.10.0" diff --git a/broccoli-cli/src/lib.rs b/broccoli-cli/src/lib.rs new file mode 100644 index 0000000..ea704c1 --- /dev/null +++ b/broccoli-cli/src/lib.rs @@ -0,0 +1,15 @@ +pub mod errors { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum CliError { + #[error("Redis error: {0}")] + Redis(#[from] redis::RedisError), + #[error("Environment error: {0}")] + Env(#[from] std::env::VarError), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + } +} + +pub use errors::CliError; diff --git a/broccoli-cli/src/main.rs b/broccoli-cli/src/main.rs new file mode 100644 index 0000000..fdeb69d --- /dev/null +++ b/broccoli-cli/src/main.rs @@ -0,0 +1,126 @@ +use anyhow::Result; +use broccoli_queue::{ + brokers::management::{QueueStatus, QueueType}, + queue::BroccoliQueue, +}; +use clap::{Parser, Subcommand, ValueEnum}; +use prettytable::{format, Cell, Row, Table}; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Clone, ValueEnum)] +enum QueueTypeCli { + Failed, + Processing, +} + +impl From for QueueType { + fn from(queue_type: QueueTypeCli) -> Self { + match queue_type { + QueueTypeCli::Failed => QueueType::Failed, + QueueTypeCli::Processing => QueueType::Processing, + } + } +} + +#[derive(Subcommand)] +enum Commands { + /// Retry messages from failed or processing queues + Retry { + /// Queue URL + #[arg(long, env)] + broccoli_queue_url: String, + /// Queue name + queue_name: String, + /// Source queue type (failed or processing) + #[arg(value_enum)] + source: QueueTypeCli, + }, + /// Show status of queues + Status { + /// Queue URL + #[arg(long, env)] + broccoli_queue_url: String, + /// Queue name. If not provided, all queues will be shown + queue_name: Option, + }, +} + +async fn retry_queue(queue: BroccoliQueue, queue_name: String, source: QueueType) -> Result<()> { + let count = queue.retry_queue(queue_name, source).await?; + if count == 0 { + println!("No messages to retry"); + } else { + println!("Successfully retried {} messages", count); + } + Ok(()) +} + +async fn show_status(queue: BroccoliQueue, queue_name: Option) -> Result<()> { + let statuses = queue.queue_status(queue_name).await?; + print_status_table(&statuses)?; + Ok(()) +} + +fn print_status_table(statuses: &[QueueStatus]) -> Result<()> { + let mut table = Table::new(); + + table.set_format(*format::consts::FORMAT_BOX_CHARS); + + table.set_format(*format::consts::FORMAT_BOX_CHARS); + + table.add_row(Row::new(vec![ + Cell::new("Queue").style_spec("Fb"), + Cell::new("Type").style_spec("Fb"), + Cell::new("Main Queue Size").style_spec("Fb"), + Cell::new("Processing Queue Size").style_spec("Fb"), + Cell::new("Failed Queue Size").style_spec("Fb"), + Cell::new("Disambiguators").style_spec("Fb"), + ])); + + for status in statuses { + table.add_row(Row::new(vec![ + Cell::new(&status.name), + Cell::new(&format!("{:?}", status.queue_type)), + Cell::new(&status.size.to_string()), + Cell::new(&status.processing.to_string()), + Cell::new(&status.failed.to_string()), + Cell::new(&status.disambiguator_count.unwrap_or(0).to_string()), + ])); + } + + table.printstd(); + + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + dotenv::dotenv().ok(); + let cli = Cli::parse(); + + match cli.command { + Commands::Retry { + broccoli_queue_url, + queue_name, + source, + } => { + let queue = BroccoliQueue::builder(broccoli_queue_url).build().await?; + retry_queue(queue, queue_name, source.into()).await?; + } + Commands::Status { + broccoli_queue_url, + queue_name, + } => { + let queue = BroccoliQueue::builder(broccoli_queue_url).build().await?; + show_status(queue, queue_name).await?; + } + } + + Ok(()) +} diff --git a/broccoli-queue/Cargo.lock b/broccoli-queue/Cargo.lock new file mode 100644 index 0000000..66142e4 --- /dev/null +++ b/broccoli-queue/Cargo.lock @@ -0,0 +1,3028 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "amq-protocol" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a41c091e49edfcc098b4f90d4d7706a8cf9158034e84ebfee7ff346092f67c" +dependencies = [ + "amq-protocol-tcp", + "amq-protocol-types", + "amq-protocol-uri", + "cookie-factory", + "nom", + "serde", +] + +[[package]] +name = "amq-protocol-tcp" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed7a4a662472f88823ed2fc81babb0b00562f2c54284e3e7bffc02b6df649bf" +dependencies = [ + "amq-protocol-uri", + "tcp-stream", + "tracing", +] + +[[package]] +name = "amq-protocol-types" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6484fdc918c1b6e2ae8eda2914d19a5873e1975f93ad8d33d6a24d1d98df05" +dependencies = [ + "cookie-factory", + "nom", + "serde", + "serde_json", +] + +[[package]] +name = "amq-protocol-uri" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7f2da69e0e1182765bf33407cd8a843f20791b5af2b57a2645818c4776c56c" +dependencies = [ + "amq-protocol-types", + "percent-encoding", + "url", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys 0.59.0", +] + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ca9a001c1e8ba5149f91a74362376cc6bc5b919d92d988668657bd570bdcec" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand 2.3.0", + "futures-lite 2.6.0", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel", + "async-executor", + "async-io 2.4.0", + "async-lock 3.4.0", + "blocking", + "futures-lite 2.6.0", + "once_cell", +] + +[[package]] +name = "async-global-executor-trait" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80f19936c1a84fb48ceb8899b642d2a72572587d1021cc561bfb24de9f33ee89" +dependencies = [ + "async-global-executor", + "async-trait", + "executor-trait", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite 1.13.0", + "log", + "parking", + "polling 2.8.0", + "rustix 0.37.28", + "slab", + "socket2 0.4.10", + "waker-fn", +] + +[[package]] +name = "async-io" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" +dependencies = [ + "async-lock 3.4.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.6.0", + "parking", + "polling 3.7.4", + "rustix 0.38.44", + "slab", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener 5.4.0", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-reactor-trait" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6012d170ad00de56c9ee354aef2e358359deb1ec504254e0e5a3774771de0e" +dependencies = [ + "async-io 1.13.0", + "async-trait", + "futures-core", + "reactor-trait", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bb8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d8b8e1a22743d9241575c6ba822cf9c8fef34771c86ab7e477a4fbfd254e5" +dependencies = [ + "futures-util", + "parking_lot", + "tokio", +] + +[[package]] +name = "bb8-redis" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8416aa3639520757fd0ed659c8c12f7cd9f0ed638fa0cdd52a13d3b19946df2a" +dependencies = [ + "bb8", + "redis", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite 2.6.0", + "piper", +] + +[[package]] +name = "broccoli_queue" +version = "0.3.3" +dependencies = [ + "async-trait", + "bb8-redis", + "chrono", + "criterion", + "dashmap", + "deadpool", + "deadpool-lapin", + "env_logger", + "futures", + "lapin", + "lazy_static", + "log", + "redis", + "serde", + "serde_json", + "sha256", + "thiserror 2.0.11", + "time", + "tokio", + "uuid", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.6", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.5.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "cms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" +dependencies = [ + "const-oid", + "der", + "spki", + "x509-cert", +] + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "futures", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" + +[[package]] +name = "deadpool" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6541a3916932fe57768d4be0b1ffb5ec7cbf74ca8c903fdfd5c0fe8aa958f0ed" +dependencies = [ + "deadpool-runtime", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-lapin" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c7b14064f854a3969735e7c948c677a57ef17ca7f0bc029da8fe2e5e0fc1eb" +dependencies = [ + "deadpool", + "lapin", + "tokio-executor-trait", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +dependencies = [ + "tokio", +] + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "der_derive", + "flagset", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "des" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +dependencies = [ + "cipher", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" +dependencies = [ + "event-listener 5.4.0", + "pin-project-lite", +] + +[[package]] +name = "executor-trait" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c39dff9342e4e0e16ce96be751eb21a94e94a87bb2f6e63ad1961c2ce109bf" +dependencies = [ + "async-trait", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "flagset" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ea1ec5f8307826a5b71094dd91fc04d4ae75d5709b20ad351c7fb4815c86ec" + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-lite" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" +dependencies = [ + "fastrand 2.3.0", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lapin" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209b09a06f4bd4952a0fd0594f90d53cf4496b062f59acc838a2823e1bb7d95c" +dependencies = [ + "amq-protocol", + "async-global-executor-trait", + "async-reactor-trait", + "async-trait", + "executor-trait", + "flume", + "futures-core", + "futures-io", + "parking_lot", + "pinky-swear", + "reactor-trait", + "serde", + "tracing", + "waker-fn", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "p12-keystore" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df7b60d0b2dcace322e6e8c4499c4c8bdf331c1bae046a54be5e4191c3610286" +dependencies = [ + "cbc", + "cms", + "der", + "des", + "hex", + "hmac", + "pkcs12", + "pkcs5", + "rand", + "rc2", + "sha1", + "sha2", + "thiserror 1.0.69", + "x509-parser", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pinky-swear" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfae3ead413ca051a681152bd266438d3bfa301c9bdf836939a14c721bb2a21" +dependencies = [ + "doc-comment", + "flume", + "parking_lot", + "tracing", +] + +[[package]] +name = "piper" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" +dependencies = [ + "atomic-waker", + "fastrand 2.3.0", + "futures-io", +] + +[[package]] +name = "pkcs12" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" +dependencies = [ + "cms", + "const-oid", + "der", + "digest", + "spki", + "x509-cert", + "zeroize", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2", + "spki", +] + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.4.0", + "pin-project-lite", + "rustix 0.38.44", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rc2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c64daa8e9438b84aaae55010a93f396f8e60e3911590fcba770d04643fc1dd" +dependencies = [ + "cipher", +] + +[[package]] +name = "reactor-trait" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "438a4293e4d097556730f4711998189416232f009c137389e0f961d2bc0ddc58" +dependencies = [ + "async-trait", + "futures-core", + "futures-io", +] + +[[package]] +name = "redis" +version = "0.27.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d8f99a4090c89cc489a94833c901ead69bfbf3877b4867d5482e321ee875bc" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "combine", + "futures-util", + "itertools 0.13.0", + "itoa", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", + "ryu", + "sha1_smol", + "socket2 0.5.8", + "tokio", + "tokio-rustls", + "tokio-util", + "url", +] + +[[package]] +name = "redox_syscall" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +dependencies = [ + "bitflags 2.8.0", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "0.37.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519165d378b97752ca44bbe15047d5d3409e875f39327546b42ac81d7e18c1b6" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.8.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-connector" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a980454b497c439c274f2feae2523ed8138bbd3d323684e1435fec62f800481" +dependencies = [ + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "rustls-webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.8.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha256" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18278f6a914fa3070aa316493f7d2ddfb9ac86ebc06fa3b83bffda487e9065b0" +dependencies = [ + "async-trait", + "bytes", + "hex", + "sha2", + "tokio", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tcp-stream" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "495b0abdce3dc1f8fd27240651c9e68890c14e9d9c61527b1ce44d8a5a7bd3d5" +dependencies = [ + "cfg-if", + "p12-keystore", + "rustls-connector", + "rustls-pemfile", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +dependencies = [ + "thiserror-impl 2.0.11", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tokio" +version = "1.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.5.8", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-executor-trait" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a1593beae7759f592e1100c5997fe9e9ebf4b5968062f1fbcd807989cd1b79" +dependencies = [ + "async-trait", + "executor-trait", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-ident" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "waker-fn" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid", + "der", + "spki", +] + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/broccoli-queue/Cargo.toml b/broccoli-queue/Cargo.toml new file mode 100644 index 0000000..dc4d2ac --- /dev/null +++ b/broccoli-queue/Cargo.toml @@ -0,0 +1,59 @@ +[package] +name = "broccoli_queue" +version = "0.3.3" +edition = "2021" +license = "MIT" +description = "Broccoli is a simple, fast, and reliable job queue for Rust." +repository = "https://github.com/densumesh/broccoli" + +[package.metadata.docs.rs] +all-features = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] + +serde = { version = "1.0.216", features = ["derive"] } +serde_json = "1.0.133" +tokio = { version = "1.42.0", features = ["full"] } +uuid = { version = "1", features = ["v4", "serde"] } +async-trait = "0.1.83" +thiserror = "2.0.7" +futures = "0.3.31" +sha256 = "1.5.0" +log = "0.4.22" +time = "0.3.37" +lapin = { version = "2.5.0", optional = true } +deadpool = { version = "0.12.1", optional = true } +deadpool-lapin = { version = "0.12.1", optional = true } +bb8-redis = { version = "0.18.0", optional = true } +redis = { version = "0.27.5", features = [ + "tokio-rustls-comp", + "aio", +], optional = true } +dashmap = "6.1.0" + +[dev-dependencies] +chrono = { version = "0.4.39", features = ["serde"] } +env_logger = "0.11.5" +lazy_static = "1.4" +criterion = { version = "0.5", features = ["async_tokio"] } + +[features] +default = [] + +# Features for optional dependencies + +# Turns on redis support using the `bb8-redis` and `redis` crates. +redis = ["dep:bb8-redis", "dep:redis"] + +# Turns on rabbitmq support using the `lapin`, `deadpool`, and `deadpool-lapin` crates. +rabbitmq = ["dep:lapin", "dep:deadpool", "dep:deadpool-lapin"] +test-fairness = [] + +# Allows for access to the management API for the queue. +management = [] + +[[bench]] +name = "queue_benchmark" +harness = false diff --git a/Dockerfile.rabbitmq b/broccoli-queue/Dockerfile.rabbitmq similarity index 100% rename from Dockerfile.rabbitmq rename to broccoli-queue/Dockerfile.rabbitmq diff --git a/benches/queue_benchmark.rs b/broccoli-queue/benches/queue_benchmark.rs similarity index 100% rename from benches/queue_benchmark.rs rename to broccoli-queue/benches/queue_benchmark.rs diff --git a/examples/consumer.rs b/broccoli-queue/examples/consumer.rs similarity index 100% rename from examples/consumer.rs rename to broccoli-queue/examples/consumer.rs diff --git a/examples/publisher.rs b/broccoli-queue/examples/publisher.rs similarity index 100% rename from examples/publisher.rs rename to broccoli-queue/examples/publisher.rs diff --git a/run-tests.sh b/broccoli-queue/run-tests.sh similarity index 100% rename from run-tests.sh rename to broccoli-queue/run-tests.sh diff --git a/src/brokers/broker.rs b/broccoli-queue/src/brokers/broker.rs similarity index 100% rename from src/brokers/broker.rs rename to broccoli-queue/src/brokers/broker.rs diff --git a/src/brokers/connect.rs b/broccoli-queue/src/brokers/connect.rs similarity index 50% rename from src/brokers/connect.rs rename to broccoli-queue/src/brokers/connect.rs index e03ddf3..6147220 100644 --- a/src/brokers/connect.rs +++ b/broccoli-queue/src/brokers/connect.rs @@ -1,12 +1,13 @@ -use crate::{ - brokers::broker::{Broker, BrokerConfig, BrokerType}, - error::BroccoliError, -}; - +#[cfg(feature = "management")] +use super::management::BrokerWithManagement; #[cfg(feature = "rabbitmq")] use crate::brokers::rabbitmq::RabbitMQBroker; #[cfg(feature = "redis")] use crate::brokers::redis::RedisBroker; +use crate::{ + brokers::broker::{Broker, BrokerConfig, BrokerType}, + error::BroccoliError, +}; /// Returns a boxed broker implementation based on the broker URL. /// @@ -16,6 +17,7 @@ use crate::brokers::redis::RedisBroker; /// /// # Returns /// A `Result` containing a boxed broker implementation, or a `BroccoliError` on failure. +#[cfg(not(feature = "management"))] pub async fn connect_to_broker( broker_url: &str, config: Option, @@ -64,3 +66,61 @@ pub async fn connect_to_broker( Ok(broker) } + +/// Returns a boxed broker implementation based on the broker URL. +/// +/// # Arguments +/// * `broker_url` - The URL of the broker. +/// * `config` - Optional broker configuration. +/// +/// # Returns +/// A `Result` containing a boxed broker implementation, or a `BroccoliError` on failure. +#[cfg(feature = "management")] +pub async fn connect_to_broker( + broker_url: &str, + config: Option, +) -> Result, BroccoliError> { + #[cfg(all(feature = "redis", feature = "rabbitmq"))] + let broker_type = if broker_url.starts_with("redis") { + BrokerType::Redis + } else if broker_url.starts_with("amqp") { + BrokerType::RabbitMQ + } else { + return Err(BroccoliError::Broker( + "Unsupported broker URL scheme".to_string(), + )); + }; + + #[cfg(all(feature = "redis", not(feature = "rabbitmq")))] + let broker_type = if broker_url.starts_with("redis") { + BrokerType::Redis + } else { + return Err(BroccoliError::Broker( + "Unsupported broker URL scheme".to_string(), + )); + }; + + #[cfg(all(not(feature = "redis"), feature = "rabbitmq"))] + let broker_type = if broker_url.starts_with("amqp") { + BrokerType::RabbitMQ + } else { + return Err(BroccoliError::Broker( + "Unsupported broker URL scheme".to_string(), + )); + }; + + let mut broker: Box = match broker_type { + #[cfg(feature = "redis")] + BrokerType::Redis => Box::new(config.map_or_else(RedisBroker::new, |config| { + RedisBroker::new_with_config(config) + })), + #[cfg(feature = "rabbitmq")] + BrokerType::RabbitMQ => Box::new(config.map_or_else(RabbitMQBroker::new, |config| { + RabbitMQBroker::new_with_config(config) + })), + }; + + broker.connect(broker_url).await?; + + Ok(broker) +} diff --git a/broccoli-queue/src/brokers/management.rs b/broccoli-queue/src/brokers/management.rs new file mode 100644 index 0000000..2acba56 --- /dev/null +++ b/broccoli-queue/src/brokers/management.rs @@ -0,0 +1,54 @@ +use crate::error::BroccoliError; + +use super::broker::Broker; + +#[async_trait::async_trait] +/// Trait for managing queues. +pub trait QueueManagement { + /// Retries all messages in the queue. + async fn retry_queue( + &self, + queue_name: &str, + disambiguator: Option, + source_type: QueueType, + ) -> Result; + /// Gets the size of the queue. + async fn get_queue_size( + &self, + queue_name: &str, + queue_type: QueueType, + ) -> Result; + /// Gets the status of specific or all queues. If `queue_name` is `None`, returns the status of all queues. + async fn get_queue_status( + &self, + queue_name: Option<&str>, + ) -> Result, BroccoliError>; +} + +pub(crate) trait BrokerWithManagement: Broker + QueueManagement {} + +#[derive(Debug, Clone)] +/// Enum representing the type of queue. +pub enum QueueType { + /// Failed queue. + Failed, + /// Processing queue. + Processing, + /// Main queue. + Main, +} + +#[derive(Debug, Clone)] +/// Struct representing the status of a queue. +pub struct QueueStatus { + /// Name of the queue. + pub name: String, + /// Type of the queue. + pub queue_type: QueueType, + /// Size of the queue. + pub size: usize, + /// Number of messages that are being processed. + pub processing: usize, + /// Number of messages that failed to be processed. + pub failed: usize, +} diff --git a/broccoli-queue/src/brokers/mod.rs b/broccoli-queue/src/brokers/mod.rs new file mode 100644 index 0000000..a8438a5 --- /dev/null +++ b/broccoli-queue/src/brokers/mod.rs @@ -0,0 +1,13 @@ +/// Contains the generic interfaces for brokers +pub mod broker; +/// Contains functions to connect to a broker +pub(crate) mod connect; +#[cfg(feature = "management")] +/// Contains the management interface for brokers +pub mod management; +/// Contains the `RabbitMQ` broker implementation +#[cfg(feature = "rabbitmq")] +pub mod rabbitmq; +/// Contains the Redis broker implementation +#[cfg(feature = "redis")] +pub mod redis; diff --git a/src/brokers/rabbitmq/broker.rs b/broccoli-queue/src/brokers/rabbitmq/broker.rs similarity index 89% rename from src/brokers/rabbitmq/broker.rs rename to broccoli-queue/src/brokers/rabbitmq/broker.rs index 3c7fcc3..7263888 100644 --- a/src/brokers/rabbitmq/broker.rs +++ b/broccoli-queue/src/brokers/rabbitmq/broker.rs @@ -63,10 +63,7 @@ impl Broker for RabbitMQBroker { BroccoliError::Broker(format!("Failed to get connection from pool: {e}")) })?; - let channel = conn - .create_channel() - .await - .map_err(|e| BroccoliError::Broker(format!("Failed to create channel: {e}")))?; + let channel = conn.create_channel().await?; self.setup_exchange(&channel, "broccoli").await?; @@ -87,10 +84,7 @@ impl Broker for RabbitMQBroker { BroccoliError::Broker(format!("Failed to get connection from pool: {e}")) })?; - let channel = conn - .create_channel() - .await - .map_err(|e| BroccoliError::Broker(format!("Failed to create channel: {e}")))?; + let channel = conn.create_channel().await?; self.setup_queue(&channel, queue_name).await?; @@ -156,8 +150,7 @@ impl Broker for RabbitMQBroker { message.payload.as_bytes(), properties, ) - .await - .map_err(|e| BroccoliError::Publish(format!("Failed to publish message: {e}")))?; + .await?; published_messages.push(message.clone()); } @@ -175,10 +168,7 @@ impl Broker for RabbitMQBroker { BroccoliError::Broker(format!("Failed to get connection from pool: {e}")) })?; - let channel = conn - .create_channel() - .await - .map_err(|e| BroccoliError::Broker(format!("Failed to create channel: {e}")))?; + let channel = conn.create_channel().await?; self.setup_queue(&channel, queue_name).await?; @@ -237,10 +227,7 @@ impl Broker for RabbitMQBroker { BroccoliError::Broker(format!("Failed to get connection from pool: {e}")) })?; - let channel = conn - .create_channel() - .await - .map_err(|e| BroccoliError::Broker(format!("Failed to create channel: {e}")))?; + let channel = conn.create_channel().await?; self.setup_queue(&channel, queue_name).await?; let auto_ack = options.is_some_and(|x| x.auto_ack.unwrap_or(false)); @@ -255,12 +242,10 @@ impl Broker for RabbitMQBroker { }, FieldTable::default(), ) - .await - .map_err(|e| BroccoliError::Consume(format!("Failed to create consumer: {e}")))?; + .await?; if let Some(delivery) = consumer.next().await { - let delivery = delivery - .map_err(|e| BroccoliError::Consume(format!("Failed to receive delivery: {e}")))?; + let delivery = delivery?; let task_id = delivery .properties @@ -326,10 +311,7 @@ impl Broker for RabbitMQBroker { channel .basic_ack(delivery_tag, BasicAckOptions::default()) - .await - .map_err(|e| { - BroccoliError::Acknowledge(format!("Failed to acknowledge message: {e}")) - })?; + .await?; Ok(()) } @@ -364,8 +346,7 @@ impl Broker for RabbitMQBroker { channel .basic_reject(delivery_tag, BasicRejectOptions::default()) - .await - .map_err(|e| BroccoliError::Cancel(format!("Failed to cancel message: {e}")))?; + .await?; if message.attempts < 3 { self.publish(queue_name, None, &[message], None).await?; } diff --git a/broccoli-queue/src/brokers/rabbitmq/management.rs b/broccoli-queue/src/brokers/rabbitmq/management.rs new file mode 100644 index 0000000..774f7b7 --- /dev/null +++ b/broccoli-queue/src/brokers/rabbitmq/management.rs @@ -0,0 +1,110 @@ +use lapin::{ + options::{BasicAckOptions, BasicGetOptions, BasicPublishOptions, QueueDeclareOptions}, + types::FieldTable, + BasicProperties, +}; + +use crate::{ + brokers::management::{BrokerWithManagement, QueueManagement, QueueStatus, QueueType}, + error::BroccoliError, +}; + +use super::RabbitMQBroker; + +#[async_trait::async_trait] +impl QueueManagement for RabbitMQBroker { + async fn retry_queue( + &self, + queue_name: &str, + _disambiguator: Option, + source_type: QueueType, + ) -> Result { + let pool = self.ensure_pool()?; + let conn = pool + .get() + .await + .map_err(|e| BroccoliError::Consume(format!("Failed to consume message: {e:?}")))?; + let channel = conn.create_channel().await?; + + let source_queue = match source_type { + QueueType::Failed => format!("{queue_name}_failed"), + QueueType::Processing => format!("{queue_name}_processing"), + QueueType::Main => { + return Err(BroccoliError::InvalidOperation( + "Cannot retry from ingestion queue".into(), + )) + } + }; + + let mut count = 0; + while let Some(delivery) = channel + .basic_get(&source_queue, BasicGetOptions::default()) + .await? + { + channel + .basic_publish( + "broccoli", + queue_name, + BasicPublishOptions::default(), + &delivery.data, + BasicProperties::default(), + ) + .await?; + + channel + .basic_ack(delivery.delivery_tag, BasicAckOptions::default()) + .await?; + count += 1; + } + + Ok(count) + } + + async fn get_queue_size( + &self, + queue_name: &str, + queue_type: QueueType, + ) -> Result { + let pool = self.ensure_pool()?; + let conn = pool + .get() + .await + .map_err(|e| BroccoliError::Consume(format!("Failed to consume message: {e:?}")))?; + let channel = conn.create_channel().await?; + + let queue = match queue_type { + QueueType::Failed => format!("{queue_name}_failed"), + QueueType::Processing => format!("{queue_name}_processing"), + QueueType::Main => queue_name.to_string(), + }; + + let queue_info = channel + .queue_declare( + &queue, + QueueDeclareOptions::default(), + FieldTable::default(), + ) + .await?; + Ok(queue_info.message_count() as usize) + } + + async fn get_queue_status(&self) -> Result, BroccoliError> { + let pool = self.ensure_pool()?; + let conn = pool + .get() + .await + .map_err(|e| BroccoliError::Consume(format!("Failed to consume message: {e:?}")))?; + let channel = conn.create_channel().await?; + + // List queues through management API or channel operations + // This is a simplified version - in practice you'd want to use the RabbitMQ Management API + let mut statuses = Vec::new(); + + // Implementation note: RabbitMQ doesn't provide memory usage through regular AMQP + // You would need to use the HTTP Management API to get this information + + Ok(statuses) + } +} + +impl BrokerWithManagement for RabbitMQBroker {} diff --git a/src/brokers/rabbitmq/mod.rs b/broccoli-queue/src/brokers/rabbitmq/mod.rs similarity index 53% rename from src/brokers/rabbitmq/mod.rs rename to broccoli-queue/src/brokers/rabbitmq/mod.rs index 2f34aa3..2f82dc3 100644 --- a/src/brokers/rabbitmq/mod.rs +++ b/broccoli-queue/src/brokers/rabbitmq/mod.rs @@ -1,4 +1,5 @@ mod broker; +#[cfg(feature = "management")] +mod management; mod utils; - pub use broker::RabbitMQBroker; diff --git a/broccoli-queue/src/brokers/rabbitmq/utils.rs b/broccoli-queue/src/brokers/rabbitmq/utils.rs new file mode 100644 index 0000000..7e2ab07 --- /dev/null +++ b/broccoli-queue/src/brokers/rabbitmq/utils.rs @@ -0,0 +1,128 @@ +use lapin::{ + options::{QueueBindOptions, QueueDeclareOptions}, + types::{AMQPValue, FieldTable}, + Channel, Queue, +}; + +use crate::{brokers::broker::BrokerConfig, error::BroccoliError}; + +use super::{broker::RabbitPool, RabbitMQBroker}; + +impl RabbitMQBroker { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn new_with_config(config: BrokerConfig) -> Self { + Self { + pool: None, + connected: false, + config: Some(config), + consume_channels: Default::default(), + } + } + + pub(crate) fn ensure_pool(&self) -> Result<&RabbitPool, BroccoliError> { + if !self.connected { + return Err(BroccoliError::Broker( + "RabbitMQ broker not connected".to_string(), + )); + } + self.pool.as_ref().ok_or_else(|| { + BroccoliError::Broker("RabbitMQ connection pool not initialized".to_string()) + }) + } + + pub(crate) async fn setup_exchange( + &self, + channel: &Channel, + exchange_name: &str, + ) -> Result<(), BroccoliError> { + #[allow(unused_mut)] + let mut args = FieldTable::default(); + let exchange_kind = if self + .config + .as_ref() + .is_some_and(|c| c.enable_scheduling.unwrap_or(false)) + { + args.insert( + "x-delayed-type".into(), + AMQPValue::LongString("direct".into()), + ); + lapin::ExchangeKind::Custom("x-delayed-message".into()) + } else { + lapin::ExchangeKind::Direct + }; + + channel + .exchange_declare( + exchange_name, + exchange_kind, + lapin::options::ExchangeDeclareOptions::default(), + args.clone(), + ) + .await + .map_err(|e| BroccoliError::Broker(format!("Failed to declare exchange: {e:?}")))?; + + Ok(()) + } + + pub(crate) async fn setup_queue( + &self, + channel: &Channel, + queue_name: &str, + ) -> Result { + let mut args = FieldTable::default(); + if !queue_name.contains("failed") { + args.insert( + "x-dead-letter-exchange".into(), + AMQPValue::LongString("".into()), + ); + args.insert( + "x-dead-letter-routing-key".into(), + AMQPValue::LongString(format!("{queue_name}_failed").into()), + ); + } + + args.insert("x-max-priority".into(), AMQPValue::LongInt(5)); + + let queue = channel + .queue_declare( + queue_name, + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + args, + ) + .await + .map_err(|e| BroccoliError::Broker(format!("Failed to declare queue: {e:?}")))?; + + channel + .queue_bind( + queue_name, + "broccoli", + queue_name, // Using queue name as routing key + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .map_err(|e| BroccoliError::Broker(format!("Failed to bind queue: {e}")))?; + + // Setup DLX for failed messages + let failed_queue = format!("{queue_name}_failed"); + channel + .queue_declare( + &failed_queue, + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .map_err(|e| BroccoliError::Broker(format!("Failed to declare failed queue: {e:?}")))?; + + Ok(queue) + } +} diff --git a/src/brokers/redis/broker.rs b/broccoli-queue/src/brokers/redis/broker.rs similarity index 100% rename from src/brokers/redis/broker.rs rename to broccoli-queue/src/brokers/redis/broker.rs diff --git a/broccoli-queue/src/brokers/redis/management.rs b/broccoli-queue/src/brokers/redis/management.rs new file mode 100644 index 0000000..046e895 --- /dev/null +++ b/broccoli-queue/src/brokers/redis/management.rs @@ -0,0 +1,174 @@ +use std::collections::HashMap; + +use redis::AsyncCommands; + +use crate::{ + brokers::{ + broker::Broker, + management::{BrokerWithManagement, QueueManagement, QueueStatus, QueueType}, + }, + error::BroccoliError, +}; + +use super::{utils::OptionalInternalBrokerMessage, RedisBroker}; + +#[async_trait::async_trait] +impl QueueManagement for RedisBroker { + async fn retry_queue( + &self, + queue_name: &str, + disambiguator: Option, + source_type: QueueType, + ) -> Result { + let mut redis = self.get_redis_connection().await?; + let source_queue = if let Some(ref disambiguator) = disambiguator { + match source_type { + QueueType::Failed => format!("{queue_name}_{disambiguator}_failed"), + QueueType::Processing => format!("{queue_name}_{disambiguator}_processing"), + QueueType::Main => { + return Err(BroccoliError::InvalidOperation( + "Cannot retry from ingestion queue".into(), + )) + } + } + } else { + match source_type { + QueueType::Failed => format!("{queue_name}_failed"), + QueueType::Processing => format!("{queue_name}_processing"), + QueueType::Main => { + return Err(BroccoliError::InvalidOperation( + "Cannot retry from ingestion queue".into(), + )) + } + } + }; + + let count = redis.llen(&source_queue).await?; + + if count == 0 { + return Ok(0); + } + + let mut messages = vec![]; + for _ in 0..count { + let task_id: Option = redis.lpop(&source_queue, None).await?; + let message: OptionalInternalBrokerMessage = redis + .hgetall(&task_id) + .await + .map_err(|e| BroccoliError::Consume(format!("Failed to consume message: {e:?}")))?; + if let Some(mut message) = message.0 { + message.attempts = 0; + messages.push(message); + } + } + + self.publish(queue_name, disambiguator, &messages, None) + .await?; + + Ok(count) + } + + async fn get_queue_size( + &self, + queue_name: &str, + queue_type: QueueType, + ) -> Result { + let mut redis = self.get_redis_connection().await?; + let queue = match queue_type { + QueueType::Failed => format!("{queue_name}_failed"), + QueueType::Processing => format!("{queue_name}_processing"), + QueueType::Main => queue_name.to_string(), + }; + + match queue_type { + QueueType::Main => redis.zcard(&queue).await.map_err(std::convert::Into::into), + _ => redis.llen(&queue).await.map_err(std::convert::Into::into), + } + } + + async fn get_queue_status( + &self, + queue_name: Option<&str>, + ) -> Result, BroccoliError> { + let mut redis = self.get_redis_connection().await?; + let mut keys: Vec = if let Some(queue_name) = queue_name { + redis.keys(&format!("{}*", queue_name)).await? + } else { + redis.keys("*").await? + }; + + keys.sort(); + + let mut queues: HashMap = HashMap::new(); + let grouped_keys = keys.iter().fold(queues, |mut acc, key| { + let (queue_name, queue_type) = + if key.ends_with("_failed") || key.ends_with("_processing") { + let queue_name = key + .trim_end_matches("_failed") + .trim_end_matches("_processing"); + let queue_type = if key.ends_with("_failed") { + QueueType::Failed + } else { + QueueType::Processing + }; + (queue_name, queue_type) + } else if redis.key_type::<&str, String>(&key).await? == *"zset" { + let queue_name = key; + let queue_type = QueueType::Main; + (queue_name, queue_type) + } else { + return acc; + }; + + let status = acc.entry(queue_name.to_string()).or_insert(QueueStatus { + name: queue_name.to_string(), + queue_type, + size: 0, + }); + + match queue_type { + QueueType::Main => { + let size: usize = redis.zcard(&queue_name).await.unwrap_or(0); + status.size = size; + } + _ => { + let size: usize = redis.llen(&key).await.unwrap_or(0); + match queue_type { + QueueType::Failed => status.failed = size, + QueueType::Processing => status.processing = size, + _ => {} + } + } + } + + acc + }); + + for key in keys { + let (size, queue_type) = if key.ends_with("_failed") || key.ends_with("_processing") { + let size: usize = redis.llen(&key).await?; + let queue_type = if key.ends_with("_failed") { + QueueType::Failed + } else { + QueueType::Processing + }; + (size, queue_type) + } else if redis.key_type::<&str, String>(&key).await? == *"zset" { + let size: usize = redis.zcard(&key).await?; + (size, QueueType::Main) + } else { + continue; + }; + + QueueStatus { + name: key, + queue_type, + size, + } + } + + Ok(statuses) + } +} + +impl BrokerWithManagement for RedisBroker {} diff --git a/broccoli-queue/src/brokers/redis/mod.rs b/broccoli-queue/src/brokers/redis/mod.rs new file mode 100644 index 0000000..2a0ea74 --- /dev/null +++ b/broccoli-queue/src/brokers/redis/mod.rs @@ -0,0 +1,9 @@ +/// Contains the Redis Broker implementation +mod broker; +#[cfg(feature = "management")] +/// Contains the management interface for the Redis Broker +mod management; +/// Utility functions for the Redis Broker +mod utils; + +pub use broker::RedisBroker; diff --git a/broccoli-queue/src/brokers/redis/utils.rs b/broccoli-queue/src/brokers/redis/utils.rs new file mode 100644 index 0000000..f9e875a --- /dev/null +++ b/broccoli-queue/src/brokers/redis/utils.rs @@ -0,0 +1,274 @@ +//! Utility functions for Redis broker implementation. +//! +//! This module provides helper methods for managing Redis message operations, +//! including message parsing and manipulation of message metadata. + +use std::collections::HashMap; + +use super::broker::{RedisBroker, RedisPool}; +use crate::{ + brokers::broker::{BrokerConfig, InternalBrokerMessage, MetadataTypes}, + error::BroccoliError, + queue::ConsumeOptions, +}; +use redis::{aio::MultiplexedConnection, FromRedisValue}; + +impl RedisBroker { + /// Creates a new `RedisBroker` instance with default configuration. + #[must_use] + pub const fn new() -> Self { + Self { + redis_pool: None, + broker_url: String::new(), + config: None, + } + } + + /// Creates a new `RedisBroker` instance with the specified configuration. + /// + /// # Arguments + /// * `config` - The broker configuration to use + #[must_use] + pub const fn new_with_config(config: BrokerConfig) -> Self { + Self { + redis_pool: None, + broker_url: String::new(), + config: Some(config), + } + } + + pub(crate) fn ensure_pool(&self) -> Result { + match &self.redis_pool { + Some(pool) => Ok(pool + .try_read() + .map_err(|_| { + BroccoliError::Broker("Failed to acquire read lock on redis pool".to_string()) + })? + .clone()), + None => Err(BroccoliError::Broker( + "Redis pool not initialized".to_string(), + )), + } + } + + pub(crate) async fn get_task_id( + &self, + queue_name: &str, + redis_connection: &mut MultiplexedConnection, + options: Option, + ) -> Result, BroccoliError> { + let popped_message: Option = if options + .as_ref() + .is_some_and(|x| x.fairness.unwrap_or(false)) + { + let script = redis::Script::new( + r#" + local current_time = tonumber(ARGV[1]) + local queue_to_process = redis.call('LPOP', KEYS[1]) + if not queue_to_process then + return nil + end + + local popped_message = redis.call('ZPOPMIN', string.format("%s_%s_queue", KEYS[2], queue_to_process), 1) + if #popped_message == 0 then + return nil + end + + local message = popped_message[1] + local score = tonumber(popped_message[2]) + + if score > (5.0 * current_time) then + redis.call('ZADD', string.format("%s_%s_queue", KEYS[2], queue_to_process), score, message) + redis.call('RPUSH', KEYS[1], queue_to_process) + return nil + end + + local does_subqueue_exist = redis.call('EXISTS', string.format("%s_%s_queue", KEYS[2], queue_to_process)) == 1 + if does_subqueue_exist then + redis.call('RPUSH', KEYS[1], queue_to_process) + else + redis.call('SREM', KEYS[3], queue_to_process) + end + + if ARGV[2] == "false" then + local processing_queue_name = string.format("%s_%s_processing", KEYS[2], queue_to_process) + redis.call('LPUSH', processing_queue_name, message) + end + + return message + "#, + ); + + script + .arg(time::OffsetDateTime::now_utc().unix_timestamp_nanos() as f64) + .arg( + options + .is_some_and(|x| x.auto_ack.unwrap_or(false)) + .to_string(), + ) + .key(format!("{queue_name}_fairness_round_robin")) + .key(queue_name) + .key(format!("{queue_name}_fairness_set")) + .invoke_async(redis_connection) + .await? + } else { + let script = redis::Script::new( + r#" + local current_time = tonumber(ARGV[1]) + local popped_message = redis.call('ZPOPMIN', KEYS[1], 1) + if #popped_message == 0 then + return nil + end + + local message = popped_message[1] + local score = tonumber(popped_message[2]) + + if score > (5.0 * current_time) then + redis.call('ZADD', KEYS[1], score, message) + return nil + end + + if ARGV[2] == "false" then + redis.call('LPUSH', KEYS[2], message) + end + return message + "#, + ); + + script + .arg(time::OffsetDateTime::now_utc().unix_timestamp_nanos() as f64) + .arg( + options + .is_some_and(|x| x.auto_ack.unwrap_or(false)) + .to_string(), + ) + .key(queue_name) + .key(format!("{queue_name}_processing")) + .invoke_async(redis_connection) + .await? + }; + + Ok(popped_message) + } + + /// Retrieves a Redis connection from the pool, retrying with exponential backoff if necessary. + /// + /// # Arguments + /// * `redis_pool` - A reference to the Redis connection pool. + /// + /// # Returns + /// A `Result` containing a `RedisConnection` on success, or a `BroccoliError` on failure. + pub(crate) async fn get_redis_connection( + &self, + ) -> Result { + let mut redis_conn_sleep = std::time::Duration::from_secs(1); + + let redis_pool = self.ensure_pool()?; + + #[allow(unused_assignments)] + let mut opt_redis_connection = None; + + loop { + let borrowed_redis_connection = match redis_pool.get().await { + Ok(redis_connection) => Some(redis_connection), + Err(err) => { + let redis_manager = bb8_redis::RedisConnectionManager::new( + self.broker_url.clone(), + ) + .map_err(|e| { + BroccoliError::Broker(format!("Failed to create redis manager: {e:?}")) + })?; + + let redis_pool = bb8_redis::bb8::Pool::builder() + .max_size( + self.config + .as_ref() + .map_or(10, |config| config.pool_connections.unwrap_or(10)) + .into(), + ) + .connection_timeout(std::time::Duration::from_secs(2)) + .build(redis_manager) + .await + .map_err(|e| { + BroccoliError::Broker(format!("Failed to create redis pool: {e:?}")) + })?; + + { + let mut pool_write = self + .redis_pool + .as_ref() + .ok_or_else(|| { + BroccoliError::Broker("Redis pool not initialized".to_string()) + })? + .write() + .map_err(|_| { + BroccoliError::Broker( + "Failed to acquire write lock on redis pool".to_string(), + ) + })?; + *pool_write = redis_pool; + } + BroccoliError::Broker(format!("Failed to get redis connection: {err:?}")); + None + } + }; + + if borrowed_redis_connection.is_some() { + opt_redis_connection = borrowed_redis_connection; + break; + } + + tokio::time::sleep(redis_conn_sleep).await; + redis_conn_sleep = + std::cmp::min(redis_conn_sleep * 2, std::time::Duration::from_secs(300)); + } + + let redis_connection = opt_redis_connection + .ok_or_else(|| BroccoliError::Broker("Failed to get redis connection".to_string()))?; + + Ok(redis_connection.clone()) + } +} + +pub struct OptionalInternalBrokerMessage(pub Option); + +impl FromRedisValue for OptionalInternalBrokerMessage { + fn from_redis_value(v: &redis::Value) -> redis::RedisResult { + let map: std::collections::HashMap = redis::from_redis_value(v)?; + if map.is_empty() { + return Ok(Self(None)); + } + + let task_id = map.get("task_id").ok_or_else(|| { + redis::RedisError::from((redis::ErrorKind::TypeError, "Missing field: task_id")) + })?; + + let payload = map.get("payload").ok_or_else(|| { + redis::RedisError::from((redis::ErrorKind::TypeError, "Missing field: payload")) + })?; + + let attempts = map.get("attempts").ok_or_else(|| { + redis::RedisError::from((redis::ErrorKind::TypeError, "Missing field: attempts")) + })?; + + let priority = map.get("priority").ok_or_else(|| { + redis::RedisError::from((redis::ErrorKind::TypeError, "Missing field: priority")) + })?; + + let disambiguator = map.get("disambiguator"); + + let mut metadata = HashMap::new(); + metadata.insert( + "priority".to_string(), + MetadataTypes::String(priority.to_string()), + ); + + Ok(Self(Some(InternalBrokerMessage { + task_id: task_id.to_string(), + payload: payload.to_string(), + attempts: attempts.parse().unwrap_or_default(), + disambiguator: disambiguator.cloned(), + metadata: Some(metadata), + }))) + } +} diff --git a/src/error.rs b/broccoli-queue/src/error.rs similarity index 87% rename from src/error.rs rename to broccoli-queue/src/error.rs index 07f5363..7bf2997 100644 --- a/src/error.rs +++ b/broccoli-queue/src/error.rs @@ -72,6 +72,13 @@ pub enum BroccoliError { #[error("Redis error: {0}")] Redis(#[from] redis::RedisError), + /// Represents RabbitMQ-specific errors. + /// + /// This variant wraps the underlying AMQP error. + #[cfg(feature = "rabbitmq")] + #[error("RabbitMQ error: {0}")] + RabbitMQ(#[from] lapin::Error), + /// Represents errors that occur during job processing. /// /// This variant can wrap any error that implements the Error trait and is Send + Sync. @@ -91,4 +98,11 @@ pub enum BroccoliError { /// - Getting the position of a message for `RabbitMQ` #[error("Feature not implemented")] NotImplemented, + + /// Represents errors that occur when an operation is invalid. + /// + /// # Examples + /// - Retrying messages from the ingestion queue + #[error("Invalid operation: {0}")] + InvalidOperation(String), } diff --git a/src/lib.rs b/broccoli-queue/src/lib.rs similarity index 100% rename from src/lib.rs rename to broccoli-queue/src/lib.rs diff --git a/src/queue.rs b/broccoli-queue/src/queue.rs similarity index 93% rename from src/queue.rs rename to broccoli-queue/src/queue.rs index 7143be6..6fd11c0 100644 --- a/src/queue.rs +++ b/broccoli-queue/src/queue.rs @@ -5,6 +5,8 @@ use time::Duration; use time::OffsetDateTime; +use crate::brokers::management::BrokerWithManagement; +use crate::brokers::management::QueueManagement; use crate::{ brokers::{ broker::{Broker, BrokerConfig, BrokerMessage, InternalBrokerMessage}, @@ -344,7 +346,10 @@ impl PublishOptionsBuilder { /// as well as processing messages with custom handlers. pub struct BroccoliQueue { /// The underlying message broker implementation + #[cfg(not(feature = "management"))] broker: Arc>, + #[cfg(feature = "management")] + broker: Arc>, } impl Clone for BroccoliQueue { @@ -915,4 +920,64 @@ impl BroccoliQueue { } } } + + /// Retrieves the status of the specified queue. + /// + /// # Arguments + /// * `queue_name` - The name of the queue. + /// + /// # Returns + /// A `Result` containing the status of the queue, or a `BroccoliError` on failure. + /// + /// # Errors + /// If the queue status fails to be retrieved, a `BroccoliError` will be returned. + #[cfg(feature = "management")] + pub async fn queue_status(&self, queue_name: &str) -> Result { + self.broker + .get_queue_status(queue_name) + .await + .map_err(|e| BroccoliError::QueueStatus(format!("Failed to get queue status: {e:?}"))) + } + + /// Retrieves the status of the specified queue. + /// + /// # Arguments + /// * `queue_name` - The name of the queue. + /// * `disambiguator` - The disambiguator for the queue. + /// * `source_type` - The type of queue to retry from. + /// + /// # Returns + /// A `Result` containing the number of messages retried, or a `BroccoliError` on failure. + /// + /// # Errors + /// If the messages fail to be retried, a `BroccoliError` will be returned. + #[cfg(feature = "management")] + pub async fn retry_queue( + &self, + queue_name: &str, + disambiguator: Option, + source_type: QueueType, + ) -> Result { + self.broker + .retry_queue(queue_name, disambiguator, source_type) + .await + .map_err(|e| BroccoliError::Retry(format!("Failed to retry queue: {e:?}"))) + } + + /// Moves all messages from the failed queue back to the main queue for retrying. + /// + /// # Arguments + /// * `queue_name` - The name of the queue. + /// + /// # Returns + /// A `Result` containing the number of messages retried, or a `BroccoliError` on failure. + /// + /// # Errors + /// If the messages fail to be retried, a `BroccoliError` will be returned. + pub async fn retry_failed_queue(&self, queue_name: &str) -> Result { + self.broker + .retry_failed_queue(queue_name) + .await + .map_err(|e| BroccoliError::Retry(format!("Failed to retry failed queue: {e:?}"))) + } } diff --git a/tests/common/mod.rs b/broccoli-queue/tests/common/mod.rs similarity index 100% rename from tests/common/mod.rs rename to broccoli-queue/tests/common/mod.rs diff --git a/broccoli-queue/tests/edge_cases.rs b/broccoli-queue/tests/edge_cases.rs new file mode 100644 index 0000000..748d1bf --- /dev/null +++ b/broccoli-queue/tests/edge_cases.rs @@ -0,0 +1,507 @@ +use broccoli_queue::queue::ConsumeOptions; +use broccoli_queue::queue::ConsumeOptionsBuilder; +use broccoli_queue::queue::PublishOptions; +#[cfg(feature = "redis")] +use redis::AsyncCommands; +use serde::{Deserialize, Serialize}; +use time::Duration; + +mod common; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +struct TestMessage { + id: String, + content: String, +} + +#[tokio::test] +async fn test_invalid_broker_url() { + let result = common::setup_queue_with_url("invalid://localhost:6379").await; + assert!(result.is_err()); +} + +#[tokio::test] +async fn test_empty_payload() { + let queue = common::setup_queue().await; + + let test_topic = "test_empty_topic"; + + let empty_message = TestMessage { + id: "".to_string(), + content: "".to_string(), + }; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + + #[cfg(not(feature = "test-fairness"))] + let result = queue.publish(test_topic, None, &empty_message, None).await; + #[cfg(feature = "test-fairness")] + let result = queue + .publish( + test_topic, + Some(String::from("job-1")), + &empty_message, + None, + ) + .await; + assert!(result.is_ok()); + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + let consumed = queue + .consume::(test_topic, Some(consume_options)) + .await + .unwrap(); + assert_eq!(consumed.payload, empty_message); + + #[cfg(feature = "redis")] + { + // Verify queue state after consuming empty message + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + let queue_len: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!(queue_len, 0, "Queue should be empty after consuming"); + } +} + +#[tokio::test] +async fn test_very_large_payload() { + let queue = common::setup_queue().await; + + let test_topic = "test_large_topic"; + + let large_content = "x".repeat(1024 * 1024); // 1MB of data + let large_message = TestMessage { + id: "large".to_string(), + content: large_content.clone(), + }; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + + #[cfg(not(feature = "test-fairness"))] + let result = queue.publish(test_topic, None, &large_message, None).await; + #[cfg(feature = "test-fairness")] + let result = queue + .publish( + test_topic, + Some(String::from("job-1")), + &large_message, + None, + ) + .await; + assert!(result.is_ok()); + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + let consumed = queue + .consume::(test_topic, Some(consume_options)) + .await + .unwrap(); + assert_eq!(consumed.payload.content.len(), large_content.len()); + + #[cfg(feature = "redis")] + { + let stored_payload: String = redis + .hget(consumed.task_id.to_string(), "payload") + .await + .unwrap(); + assert_eq!( + stored_payload.len(), + serde_json::to_string(&large_message).unwrap().len() + ); + } +} + +#[tokio::test] +async fn test_concurrent_consume() { + let queue = common::setup_queue().await; + + let test_topic = "test_concurrent_topic"; + + // Redis client for verification + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + + // Publish multiple messages + let messages: Vec<_> = (0..10) + .map(|i| TestMessage { + id: i.to_string(), + content: format!("content {}", i), + }) + .collect(); + + #[cfg(not(feature = "test-fairness"))] + queue + .publish_batch(test_topic, None, messages, None) + .await + .unwrap(); + + #[cfg(feature = "test-fairness")] + queue + .publish_batch(test_topic, Some(String::from("job-1")), messages, None) + .await + .unwrap(); + + // Consume concurrently + let mut handles = vec![]; + for _ in 0..5 { + let queue_clone = queue.clone(); + let topic = test_topic.to_string(); + handles.push(tokio::spawn(async move { + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + let msg = queue_clone + .consume::(&topic, Some(consume_options)) + .await + .unwrap(); + queue_clone + .acknowledge(test_topic, msg.clone()) + .await + .unwrap(); + msg + })); + } + + let results = futures::future::join_all(handles).await; + let consumed_messages: Vec<_> = results.into_iter().map(|r| r.unwrap().payload).collect(); + + assert_eq!(consumed_messages.len(), 5); + // Ensure no duplicate messages were consumed + let unique_ids: std::collections::HashSet<_> = + consumed_messages.iter().map(|m| m.id.clone()).collect(); + assert_eq!(unique_ids.len(), 5); + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Verify remaining message count + let remaining: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!(remaining, 5, "Should have 5 messages remaining"); + + #[cfg(not(feature = "test-fairness"))] + let processing_queue_name = format!("{}_processing", test_topic); + #[cfg(feature = "test-fairness")] + let processing_queue_name = format!("{}_job-1_processing", test_topic); + + // Verify processing queue + let processing: usize = redis.llen(processing_queue_name).await.unwrap(); + assert_eq!( + processing, 0, + "Processing queue should be empty after acknowledgments" + ); + + // Add verification for each consumed message + for consumed_msg in &consumed_messages { + let task_exists: bool = redis + .hexists(consumed_msg.id.clone(), "payload") + .await + .unwrap(); + assert!( + !task_exists, + "Message should be cleaned up after acknowledgment" + ); + } + + // Verify fairness is maintained during concurrent consumption + if unique_ids.len() != 5 { + let mut id_counts = std::collections::HashMap::new(); + for msg in consumed_messages { + *id_counts.entry(msg.id).or_insert(0) += 1; + } + for (id, count) in id_counts { + assert!(count <= 1, "Message {} was consumed {} times", id, count); + } + } + } +} + +#[tokio::test] +async fn test_zero_ttl() { + let queue = common::setup_queue().await; + + let test_topic = "test_zero_ttl_topic"; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + + let message = TestMessage { + id: "zero_ttl".to_string(), + content: "expires immediately".to_string(), + }; + + let options = PublishOptions::builder().ttl(Duration::seconds(0)).build(); + + #[cfg(not(feature = "test-fairness"))] + queue + .publish(test_topic, None, &message, Some(options)) + .await + .unwrap(); + #[cfg(feature = "test-fairness")] + queue + .publish( + test_topic, + Some(String::from("job-1")), + &message, + Some(options), + ) + .await + .unwrap(); + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // Message should not be available + let result = queue + .try_consume::(test_topic, Some(consume_options)) + .await + .unwrap(); + assert!(result.is_none()); + + #[cfg(feature = "redis")] + { + // Verify message was deleted due to TTL + let exists: bool = redis.exists(&message.id).await.unwrap(); + assert!(!exists, "Message should be deleted due to TTL"); + + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Verify queue is empty + let queue_len: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!(queue_len, 0, "Queue should be empty"); + + // Verify immediate TTL expiration + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let exists: bool = redis.exists(&message.id).await.unwrap(); + assert!(!exists, "Message should be deleted due to zero TTL"); + + // Verify message not in queue + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + let queue_len: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!(queue_len, 0, "Queue should be empty with zero TTL message"); + } +} + +#[tokio::test] +async fn test_message_ordering() { + let queue = common::setup_queue().await; + + let test_topic = "test_ordering_topic"; + + // Redis client for verification + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + + // Publish messages with different delays + let messages = vec![ + ( + TestMessage { + id: "1".to_string(), + content: "first".to_string(), + }, + Some( + PublishOptions::builder() + .delay(Duration::seconds(2)) + .build(), + ), + ), + ( + TestMessage { + id: "2".to_string(), + content: "second".to_string(), + }, + Some( + PublishOptions::builder() + .delay(Duration::seconds(1)) + .build(), + ), + ), + ( + TestMessage { + id: "3".to_string(), + content: "third".to_string(), + }, + None, + ), + ]; + + for (msg, opt) in messages { + #[cfg(not(feature = "test-fairness"))] + queue.publish(test_topic, None, &msg, opt).await.unwrap(); + #[cfg(feature = "test-fairness")] + queue + .publish(test_topic, Some(String::from("job-1")), &msg, opt) + .await + .unwrap(); + } + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // Consume messages + let third = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .unwrap(); + queue.acknowledge(test_topic, third.clone()).await.unwrap(); + assert_eq!(third.payload.id, "3"); + + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + let second = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .unwrap(); + queue.acknowledge(test_topic, second.clone()).await.unwrap(); + let first = queue + .consume::(test_topic, Some(consume_options)) + .await + .unwrap(); + queue.acknowledge(test_topic, first.clone()).await.unwrap(); + + assert_eq!(second.payload.id, "2"); + assert_eq!(first.payload.id, "1"); + + // Verify Redis state after test + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Check queue is empty + let len: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!(len, 0, "Queue should be empty after consuming all messages"); + + #[cfg(not(feature = "test-fairness"))] + let processing_queue_name = format!("{}_processing", test_topic); + #[cfg(feature = "test-fairness")] + let processing_queue_name = format!("{}_job-1_processing", test_topic); + + // Check processing queue is empty + let proc_len: usize = redis.llen(processing_queue_name).await.unwrap(); + assert_eq!(proc_len, 0, "Processing queue should be empty"); + + // Verify message order in Redis sorted set + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + let scores: Vec<(String, f64)> = redis + .zrangebyscore_withscores(queue_name, "-inf", "+inf") + .await + .unwrap(); + + // Verify scores are ordered correctly (delayed messages have higher scores) + for i in 0..scores.len().saturating_sub(1) { + assert!( + scores[i].1 <= scores[i + 1].1, + "Messages should be ordered by score" + ); + } + + // Verify processing state after consumption + #[cfg(not(feature = "test-fairness"))] + let processing_queue = format!("{}_processing", test_topic); + #[cfg(feature = "test-fairness")] + let processing_queue = format!("{}_job-1_processing", test_topic); + + let proc_len: usize = redis.llen(&processing_queue).await.unwrap(); + assert_eq!( + proc_len, 0, + "Processing queue should be empty after acknowledgments" + ); + } +} + +#[tokio::test] +#[cfg(feature = "redis")] +async fn test_redis_specific_queue_structure() { + use std::collections::HashMap; + + let queue = common::setup_queue().await; + let test_topic = "test_redis_structure"; + let mut redis = common::get_redis_client().await; + + // Test message + let message = TestMessage { + id: "struct_test".to_string(), + content: "test content".to_string(), + }; + + // Publish message + let broker_message = queue + .publish(test_topic, None, &message, None) + .await + .unwrap(); + + // Verify queue structure + let queue_type: String = redis.key_type(test_topic).await.unwrap(); + assert_eq!(queue_type, "zset", "Main queue should be a sorted set"); + + // Verify message metadata + let metadata: HashMap = redis + .hgetall(broker_message.task_id.to_string()) + .await + .unwrap(); + assert!( + metadata.contains_key("task_id"), + "Message should have task_id" + ); + assert!( + metadata.contains_key("payload"), + "Message should have payload" + ); + assert!( + metadata.contains_key("attempts"), + "Message should have attempts counter" + ); + + // Consume message + let consumed = queue + .consume::(test_topic, None) + .await + .unwrap(); + + // Verify processing queue + let proc_type: String = redis + .key_type(format!("{}_processing", test_topic)) + .await + .unwrap(); + assert_eq!(proc_type, "list", "Processing queue should be a list"); + + // Acknowledge and verify cleanup + queue.acknowledge(test_topic, consumed).await.unwrap(); + let exists: bool = redis.exists(&message.id).await.unwrap(); + assert!(!exists, "Message should be cleaned up after acknowledgment"); +} diff --git a/broccoli-queue/tests/fairness.rs b/broccoli-queue/tests/fairness.rs new file mode 100644 index 0000000..79ac9e6 --- /dev/null +++ b/broccoli-queue/tests/fairness.rs @@ -0,0 +1,382 @@ +use broccoli_queue::queue::ConsumeOptionsBuilder; +#[cfg(all(feature = "redis", feature = "test-fairness"))] +use broccoli_queue::queue::PublishOptions; +#[cfg(all(feature = "redis", feature = "test-fairness"))] +use redis::AsyncCommands; +#[cfg(all(feature = "redis", feature = "test-fairness"))] +use serde::{Deserialize, Serialize}; +#[cfg(all(feature = "redis", feature = "test-fairness"))] +use time::Duration; + +mod common; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[cfg(all(feature = "redis", feature = "test-fairness"))] +struct TestMessage { + id: String, + content: String, +} + +#[tokio::test] +#[cfg(all(feature = "redis", feature = "test-fairness"))] +async fn test_fairness_round_robin() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_fairness_topic"; + + // Publish messages from different jobs + let messages = vec![ + ("job-1", "message 1 from job 1"), + ("job-1", "message 2 from job 1"), + ("job-2", "message 1 from job 2"), + ("job-2", "message 2 from job 2"), + ("job-3", "message 1 from job 3"), + ("job-3", "message 2 from job 3"), + ]; + + for (job_id, content) in messages { + let message = TestMessage { + id: job_id.to_string(), + content: content.to_string(), + }; + queue + .publish(test_topic, Some(String::from(job_id)), &message, None) + .await + .expect("Failed to publish message"); + } + + #[cfg(feature = "redis")] + { + // Verify fairness data structures + let set_exists: bool = redis + .exists(format!("{}_fairness_set", test_topic)) + .await + .unwrap(); + assert!(set_exists, "Fairness set should exist"); + + let round_robin_exists: bool = redis + .exists(format!("{}_fairness_round_robin", test_topic)) + .await + .unwrap(); + assert!(round_robin_exists, "Round robin list should exist"); + + // Verify job queues + for job_id in ["job-1", "job-2", "job-3"] { + let queue_exists: bool = redis + .exists(format!("{}_{}_queue", test_topic, job_id)) + .await + .unwrap(); + assert!(queue_exists, "Job queue for {} should exist", job_id); + } + } + + // Consume messages - they should come in round-robin order + let mut consumed_messages = Vec::new(); + for _ in 0..6 { + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + let msg = queue + .consume::(test_topic, Some(consume_options)) + .await + .expect("Failed to consume message"); + consumed_messages.push((msg.payload.id.clone(), msg.payload.content.clone())); + queue + .acknowledge(test_topic, msg) + .await + .expect("Failed to acknowledge message"); + } + + // Verify round-robin order (one from each job before repeating) + assert_eq!(consumed_messages[0].0, "job-1"); + assert_eq!(consumed_messages[1].0, "job-2"); + assert_eq!(consumed_messages[2].0, "job-3"); + assert_eq!(consumed_messages[3].0, "job-1"); + assert_eq!(consumed_messages[4].0, "job-2"); + assert_eq!(consumed_messages[5].0, "job-3"); +} + +#[tokio::test] +#[cfg(all(feature = "redis", feature = "test-fairness"))] +async fn test_fairness_with_priorities() { + let queue = common::setup_queue().await; + let test_topic = "test_fairness_priority_topic"; + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + + // Publish messages with different priorities from different jobs + let messages = vec![ + ("job-1", 5, "low priority from job 1"), + ("job-2", 1, "high priority from job 2"), + ("job-1", 3, "medium priority from job 1"), + ("job-2", 3, "medium priority from job 2"), + ]; + + for (job_id, priority, content) in messages { + let message = TestMessage { + id: job_id.to_string(), + content: content.to_string(), + }; + let options = PublishOptions::builder().priority(priority).build(); + let published = queue + .publish( + test_topic, + Some(String::from(job_id)), + &message, + Some(options), + ) + .await + .expect("Failed to publish message"); + + #[cfg(feature = "redis")] + { + // Verify message metadata includes priority + let priority_stored: String = redis + .hget(published.task_id.to_string(), "priority") + .await + .unwrap(); + assert_eq!( + priority_stored, + priority.to_string(), + "Priority should be stored correctly" + ); + } + } + + // Consume messages - they should respect both fairness and priority + let mut consumed_messages = Vec::new(); + for _ in 0..4 { + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + let msg = queue + .consume::(test_topic, Some(consume_options)) + .await + .expect("Failed to consume message"); + consumed_messages.push((msg.payload.id.clone(), msg.payload.content.clone())); + queue + .acknowledge(test_topic, msg) + .await + .expect("Failed to acknowledge message"); + } + + #[cfg(feature = "redis")] + { + // Verify queues are empty after consumption + for job_id in ["job-1", "job-2"] { + let queue_name = format!("{}_{}_queue", test_topic, job_id); + let queue_len: usize = redis.zcard(&queue_name).await.unwrap(); + assert_eq!(queue_len, 0, "Queue should be empty after consumption"); + + let processing_queue = format!("{}_{}_processing", test_topic, job_id); + let proc_len: usize = redis.llen(&processing_queue).await.unwrap(); + assert_eq!(proc_len, 0, "Processing queue should be empty"); + } + } + + // Verify that high priority messages come first but still alternate between jobs + assert!( + consumed_messages[0].1.contains("high priority from job 2") + || consumed_messages[1].1.contains("high priority from job 2") + ); + assert!( + consumed_messages[2].1.contains("medium priority") + && consumed_messages[3].1.contains("low priority") + || consumed_messages[2].1.contains("low priority") + && consumed_messages[3].1.contains("medium priority") + ); +} + +#[tokio::test] +#[cfg(all(feature = "redis", feature = "test-fairness"))] +async fn test_fairness_with_delayed_messages() { + let queue = common::setup_queue().await; + let test_topic = "test_fairness_delay_topic"; + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + + // Publish immediate and delayed messages from different jobs + let immediate_msg = TestMessage { + id: "job-1".to_string(), + content: "immediate from job 1".to_string(), + }; + queue + .publish( + test_topic, + Some(String::from("job-1")), + &immediate_msg, + None, + ) + .await + .expect("Failed to publish immediate message"); + + let delayed_msg = TestMessage { + id: "job-2".to_string(), + content: "delayed from job 2".to_string(), + }; + let options = PublishOptions::builder() + .delay(Duration::seconds(2)) + .build(); + queue + .publish( + test_topic, + Some(String::from("job-2")), + &delayed_msg, + Some(options), + ) + .await + .expect("Failed to publish delayed message"); + + #[cfg(feature = "redis")] + { + // Verify initial state + let queue_name = format!("{}_job-1_queue", test_topic); + let queue_len: usize = redis.zcard(&queue_name).await.unwrap(); + assert_eq!(queue_len, 1, "Queue should have one message"); + + let queue_name_2 = format!("{}_job-2_queue", test_topic); + let queue_len_2: usize = redis.zcard(&queue_name_2).await.unwrap(); + assert_eq!(queue_len_2, 1, "Queue should have one message"); + + // Verify message scores (delayed message should have higher score) + let scores_1: Vec<(String, f64)> = redis + .zrangebyscore_withscores(&queue_name, "-inf", "+inf") + .await + .unwrap(); + let scores_2: Vec<(String, f64)> = redis + .zrangebyscore_withscores(&queue_name_2, "-inf", "+inf") + .await + .unwrap(); + + assert!( + scores_2[0].1 > scores_1[0].1, + "Delayed message should have higher score" + ); + } + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // Consume immediate message + let first = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to consume first message"); + assert_eq!(first.payload.content, "immediate from job 1"); + queue + .acknowledge(test_topic, first) + .await + .expect("Failed to acknowledge first message"); + + // Wait for delayed message + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + + // Consume delayed message + let second = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to consume second message"); + assert_eq!(second.payload.content, "delayed from job 2"); + queue + .acknowledge(test_topic, second) + .await + .expect("Failed to acknowledge second message"); + + #[cfg(feature = "redis")] + { + // Verify final state + for job_id in ["job-1", "job-2"] { + let queue_name = format!("{}_{}_queue", test_topic, job_id); + let queue_len: usize = redis.zcard(&queue_name).await.unwrap(); + assert_eq!(queue_len, 0, "Queue should be empty after consumption"); + + let processing_queue = format!("{}_{}_processing", test_topic, job_id); + let proc_len: usize = redis.llen(&processing_queue).await.unwrap(); + assert_eq!(proc_len, 0, "Processing queue should be empty"); + } + } +} + +#[tokio::test] +#[cfg(all(feature = "redis", feature = "test-fairness"))] +async fn test_fairness_with_retries() { + let queue = common::setup_queue().await; + let test_topic = "test_fairness_retry_topic"; + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + + // Publish messages from different jobs + let messages = vec![ + ("job-1", "message from job 1"), + ("job-2", "message from job 2"), + ]; + + for (job_id, content) in messages { + let message = TestMessage { + id: job_id.to_string(), + content: content.to_string(), + }; + queue + .publish(test_topic, Some(String::from(job_id)), &message, None) + .await + .expect("Failed to publish message"); + } + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // Consume and reject messages + for _ in 0..2 { + for _ in 0..3 { + let msg = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to consume message"); + queue + .reject(test_topic, msg) + .await + .expect("Failed to reject message"); + } + } + + // Verify messages are in failed queue + let result = queue + .try_consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to try consume"); + assert!(result.is_none()); + + #[cfg(feature = "redis")] + { + for job_id in ["job-1", "job-2"] { + // Verify messages moved to failed queues + let failed_queue = format!("{}_{}_failed", test_topic, job_id); + let failed_len: usize = redis.llen(&failed_queue).await.unwrap(); + assert_eq!( + failed_len, 1, + "Failed queue for {} should have one message", + job_id + ); + + // Verify original queues are empty + let queue_name = format!("{}_{}_queue", test_topic, job_id); + let queue_len: usize = redis.zcard(&queue_name).await.unwrap(); + assert_eq!(queue_len, 0, "Queue should be empty after retries"); + + // Verify processing queues are empty + let processing_queue = format!("{}_{}_processing", test_topic, job_id); + let proc_len: usize = redis.llen(&processing_queue).await.unwrap(); + assert_eq!(proc_len, 0, "Processing queue should be empty"); + } + } +} diff --git a/broccoli-queue/tests/happy_path.rs b/broccoli-queue/tests/happy_path.rs new file mode 100644 index 0000000..bc4a15e --- /dev/null +++ b/broccoli-queue/tests/happy_path.rs @@ -0,0 +1,710 @@ +use broccoli_queue::queue::{ConsumeOptions, ConsumeOptionsBuilder, PublishOptions}; +#[cfg(feature = "redis")] +use redis::AsyncCommands; +use serde::{Deserialize, Serialize}; +use time::Duration; + +mod common; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +struct TestMessage { + id: String, + content: String, +} + +#[tokio::test] +async fn test_publish_and_consume() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_publish_topic"; + + // Test message + let message = TestMessage { + id: "1".to_string(), + content: "test content".to_string(), + }; + + // Publish message + #[cfg(not(feature = "test-fairness"))] + let published = queue + .publish(test_topic, None, &message, None) + .await + .expect("Failed to publish message"); + #[cfg(feature = "test-fairness")] + let published = queue + .publish(test_topic, Some(String::from("job-1")), &message, None) + .await + .expect("Failed to publish message"); + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // Consume message + let consumed = queue + .consume::(test_topic, Some(consume_options)) + .await + .expect("Failed to consume message"); + + assert_eq!(published.payload, consumed.payload); + assert_eq!(published.task_id, consumed.task_id); + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let processing_queue_name = format!("{}_processing", test_topic); + #[cfg(feature = "test-fairness")] + let processing_queue_name = format!("{}_job-1_processing", test_topic); + + // Verify message is in the processing queue + let processing: usize = redis.llen(processing_queue_name).await.unwrap(); + assert_eq!(processing, 1, "Message should be in processing queue"); + + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + queue + .acknowledge(&queue_name, consumed) + .await + .expect("Failed to acknowledge message"); + + // After acknowledge, verify cleanup + let exists: bool = redis.exists(published.task_id.to_string()).await.unwrap(); + assert!(!exists, "Message should be cleaned up after acknowledge"); + } +} + +#[tokio::test] +async fn test_batch_publish_and_consume() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_batch_topic"; + + // Test messages + let messages = vec![ + TestMessage { + id: "1".to_string(), + content: "content 1".to_string(), + }, + TestMessage { + id: "2".to_string(), + content: "content 2".to_string(), + }, + ]; + + // Publish batch + #[cfg(not(feature = "test-fairness"))] + let published = queue + .publish_batch(test_topic, None, messages.clone(), None) + .await + .expect("Failed to publish batch"); + #[cfg(feature = "test-fairness")] + let published = queue + .publish_batch( + test_topic, + Some(String::from("job-1")), + messages.clone(), + None, + ) + .await + .expect("Failed to publish batch"); + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // Consume messages + let consumed = queue + .consume_batch::(test_topic, 2, Duration::seconds(5), Some(consume_options)) + .await + .expect("Failed to consume batch"); + + assert_eq!(published.len(), consumed.len()); + assert_eq!(published[0].payload, consumed[0].payload); + assert_eq!(published[1].payload, consumed[1].payload); + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Verify queue size after consuming + let remaining: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!( + remaining, 0, + "Queue should be empty after consuming all messages" + ); + } +} + +#[tokio::test] +async fn test_delayed_message() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_delayed_topic"; + + let message = TestMessage { + id: "delayed".to_string(), + content: "delayed content".to_string(), + }; + + // Publish with delay + let options = PublishOptions::builder() + .delay(time::Duration::seconds(2)) + .build(); + + #[cfg(not(feature = "test-fairness"))] + queue + .publish(test_topic, None, &message, Some(options)) + .await + .expect("Failed to publish delayed message"); + #[cfg(feature = "test-fairness")] + queue + .publish( + test_topic, + Some(String::from("job-1")), + &message, + Some(options), + ) + .await + .expect("Failed to publish delayed message"); + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + // Try immediate consume (should be None) + let immediate_result = queue + .try_consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to try_consume"); + assert!(immediate_result.is_none()); + + // Wait for delay + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Verify delayed message score + let scores: Vec<(String, f64)> = redis + .zrangebyscore_withscores(queue_name, "-inf", "+inf") + .await + .unwrap(); + assert!(!scores.is_empty(), "Delayed message should be in queue"); + let now = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as f64; + assert!(scores[0].1 > now, "Message score should be in future"); + } + + // Now consume (should get message) + let delayed_result = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to consume delayed message"); + + assert_eq!(message.content, delayed_result.payload.content); +} + +#[tokio::test] +async fn test_scheduled_message() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_scheduled_topic"; + + let message = TestMessage { + id: "scheduled".to_string(), + content: "scheduled content".to_string(), + }; + + // Schedule for 2 seconds in the future + let schedule_time = time::OffsetDateTime::now_utc() + time::Duration::seconds(2); + let options = PublishOptions::builder().schedule_at(schedule_time).build(); + + #[cfg(not(feature = "test-fairness"))] + let published = queue + .publish(test_topic, None, &message, Some(options)) + .await + .expect("Failed to publish scheduled message"); + #[cfg(feature = "test-fairness")] + let published = queue + .publish( + test_topic, + Some(String::from("job-1")), + &message, + Some(options), + ) + .await + .expect("Failed to publish scheduled message"); + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // Try immediate consume (should be None) + let immediate_result = queue + .try_consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to try_consume"); + + assert!(immediate_result.is_none()); + + // Wait for schedule time + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + + // Now consume (should get message) + let scheduled_result = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to consume scheduled message"); + + assert_eq!(published.payload.content, scheduled_result.payload.content); + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Verify queue is empty + let remaining: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!(remaining, 0, "Queue should be empty after consuming"); + } +} + +#[tokio::test] +async fn test_message_retry() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_retry_topic"; + + let message = TestMessage { + id: "retry".to_string(), + content: "retry content".to_string(), + }; + + // Publish message + #[cfg(not(feature = "test-fairness"))] + let published = queue + .publish(test_topic, None, &message, None) + .await + .expect("Failed to publish message"); + #[cfg(feature = "test-fairness")] + let published = queue + .publish(test_topic, Some(String::from("job-1")), &message, None) + .await + .expect("Failed to publish message"); + + // Simulate failed processing 3 times + for _ in 0..3 { + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + let consumed = queue + .consume::(test_topic, Some(consume_options)) + .await + .expect("Failed to consume message"); + + // Reject the message + queue + .reject(test_topic, consumed) + .await + .expect("Failed to reject message"); + } + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // // Try to consume again - should be in failed queue + let result = queue + .try_consume::(test_topic, Some(consume_options)) + .await + .unwrap(); + assert!(result.is_none()); + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let failed_queue_name = format!("{}_failed", test_topic); + #[cfg(feature = "test-fairness")] + let failed_queue_name = format!("{}_job-1_failed", test_topic); + + // Verify message in failed queue + let failed_len: usize = redis.llen(failed_queue_name).await.unwrap(); + assert_eq!(failed_len, 1, "Message should be in failed queue"); + + // Verify attempts counter + let attempts: String = redis + .hget(published.task_id.to_string(), "attempts") + .await + .unwrap(); + assert_eq!(attempts, "2", "Attempts counter should be 2"); + } +} + +#[tokio::test] +async fn test_message_acknowledgment() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_ack_topic"; + + let message = TestMessage { + id: "ack".to_string(), + content: "ack content".to_string(), + }; + + // Publish message + #[cfg(not(feature = "test-fairness"))] + queue + .publish(test_topic, None, &message, None) + .await + .expect("Failed to publish message"); + #[cfg(feature = "test-fairness")] + queue + .publish(test_topic, Some(String::from("job-1")), &message, None) + .await + .expect("Failed to publish message"); + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + // Consume and acknowledge + let consumed = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to consume message"); + + queue + .acknowledge(test_topic, consumed) + .await + .expect("Failed to acknowledge message"); + + // Try to consume again - should be none + let result = queue + .try_consume::(test_topic, Some(consume_options.clone())) + .await + .unwrap(); + assert!(result.is_none()); + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Verify queue is empty + let remaining: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!(remaining, 0, "Queue should be empty after acknowledgment"); + + // Verify processing queue is empty + #[cfg(not(feature = "test-fairness"))] + let processing_queue = format!("{}_processing", test_topic); + #[cfg(feature = "test-fairness")] + let processing_queue = format!("{}_job-1_processing", test_topic); + + let processing: usize = redis.llen(processing_queue).await.unwrap(); + assert_eq!(processing, 0, "Processing queue should be empty"); + } +} + +#[tokio::test] +async fn test_message_auto_ack() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_auto_ack_topic"; + + let message = TestMessage { + id: "ack".to_string(), + content: "ack content".to_string(), + }; + + // Publish message + #[cfg(not(feature = "test-fairness"))] + queue + .publish(test_topic, None, &message, None) + .await + .expect("Failed to publish message"); + #[cfg(feature = "test-fairness")] + queue + .publish(test_topic, Some(String::from("job-1")), &message, None) + .await + .expect("Failed to publish message"); + + #[cfg(not(feature = "test-fairness"))] + let options = ConsumeOptionsBuilder::new().auto_ack(true).build(); + #[cfg(feature = "test-fairness")] + let options = ConsumeOptionsBuilder::new() + .fairness(true) + .auto_ack(true) + .build(); + // Consume and auto-ack + queue + .consume::(test_topic, Some(options.clone())) + .await + .expect("Failed to consume message"); + // Try to consume again - should be none + let result = queue + .try_consume::(test_topic, Some(options)) + .await + .unwrap(); + assert!(result.is_none()); + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Verify queue is empty + let remaining: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!(remaining, 0, "Queue should be empty after auto-ack"); + + // Verify processing queue doesn't exist (auto-ack skips processing queue) + #[cfg(not(feature = "test-fairness"))] + let processing_queue = format!("{}_processing", test_topic); + #[cfg(feature = "test-fairness")] + let processing_queue = format!("{}_job-1_processing", test_topic); + + let processing: usize = redis.llen(processing_queue).await.unwrap(); + assert_eq!(processing, 0, "Processing queue should be empty"); + } +} + +#[tokio::test] +async fn test_message_cancellation() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_cancel_topic"; + + let message = TestMessage { + id: "cancel".to_string(), + content: "cancel content".to_string(), + }; + + // Publish message + #[cfg(not(feature = "test-fairness"))] + let published = queue + .publish(test_topic, None, &message, None) + .await + .expect("Failed to publish message"); + #[cfg(feature = "test-fairness")] + let published = queue + .publish(test_topic, Some(String::from("job-1")), &message, None) + .await + .expect("Failed to publish message"); + // Cancel the message + let result = queue + .cancel(test_topic, published.task_id.to_string()) + .await; + + match result { + Ok(()) => (), + Err(e) if e.to_string().contains("NotImplemented") => { + println!("This feature is not implemented for this broker"); + return; + } + Err(e) => { + panic!("Failed to get message position: {:?}", e); + } + }; + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // Try to consume - should be none + let result = queue + .try_consume::(test_topic, Some(consume_options)) + .await + .unwrap(); + assert!(result.is_none()); + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Verify message removed from queue + let remaining: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!(remaining, 0, "Queue should be empty after cancellation"); + + // Verify message metadata cleaned up + let exists: bool = redis.exists(published.task_id.to_string()).await.unwrap(); + assert!(!exists, "Message should be cleaned up after cancellation"); + } +} + +#[tokio::test] +async fn test_message_priority() { + let queue = common::setup_queue().await; + + #[cfg(feature = "redis")] + let mut redis = common::get_redis_client().await; + let test_topic = "test_priority_topic"; + + // Create messages with different priorities + let messages = [ + TestMessage { + id: "1".to_string(), + content: "low priority".to_string(), + }, + TestMessage { + id: "2".to_string(), + content: "high priority".to_string(), + }, + TestMessage { + id: "3".to_string(), + content: "medium priority".to_string(), + }, + ]; + + // Publish messages with different priorities + let options_low = PublishOptions::builder().priority(5).build(); + let options_high = PublishOptions::builder().priority(1).build(); + let options_medium = PublishOptions::builder().priority(3).build(); + + #[cfg(not(feature = "test-fairness"))] + queue + .publish(test_topic, None, &messages[0], Some(options_low)) + .await + .expect("Failed to publish low priority message"); + #[cfg(feature = "test-fairness")] + queue + .publish( + test_topic, + Some(String::from("job-1")), + &messages[0], + Some(options_low), + ) + .await + .expect("Failed to publish low priority message"); + + #[cfg(not(feature = "test-fairness"))] + queue + .publish(test_topic, None, &messages[1], Some(options_high)) + .await + .expect("Failed to publish high priority message"); + #[cfg(feature = "test-fairness")] + queue + .publish( + test_topic, + Some(String::from("job-1")), + &messages[1], + Some(options_high), + ) + .await + .expect("Failed to publish high priority message"); + + #[cfg(not(feature = "test-fairness"))] + queue + .publish(test_topic, None, &messages[2], Some(options_medium)) + .await + .expect("Failed to publish medium priority message"); + #[cfg(feature = "test-fairness")] + queue + .publish( + test_topic, + Some(String::from("job-1")), + &messages[2], + Some(options_medium), + ) + .await + .expect("Failed to publish medium priority message"); + + #[cfg(not(feature = "test-fairness"))] + let consume_options = ConsumeOptions::default(); + #[cfg(feature = "test-fairness")] + let consume_options = ConsumeOptionsBuilder::new().fairness(true).build(); + + // Consume messages - they should come in priority order (high to low) + let first = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to consume first message"); + queue + .acknowledge(test_topic, first.clone()) + .await + .expect("Failed to acknowledge first message"); + let second = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to consume second message"); + queue + .acknowledge(test_topic, second.clone()) + .await + .expect("Failed to acknowledge second message"); + let third = queue + .consume::(test_topic, Some(consume_options.clone())) + .await + .expect("Failed to consume third message"); + queue + .acknowledge(test_topic, third.clone()) + .await + .expect("Failed to acknowledge third message"); + + // Verify priority ordering + assert_eq!(first.payload.content, "high priority"); + assert_eq!(second.payload.content, "medium priority"); + assert_eq!(third.payload.content, "low priority"); + + #[cfg(feature = "redis")] + { + #[cfg(not(feature = "test-fairness"))] + let queue_name = test_topic; + #[cfg(feature = "test-fairness")] + let queue_name = format!("{}_job-1_queue", test_topic); + + // Verify queue is empty + let remaining: usize = redis.zcard(queue_name).await.unwrap(); + assert_eq!( + remaining, 0, + "Queue should be empty after consuming all messages" + ); + + // Verify all messages cleaned up + let exists_first: bool = redis.exists(first.task_id.to_string()).await.unwrap(); + let exists_second: bool = redis.exists(second.task_id.to_string()).await.unwrap(); + let exists_third: bool = redis.exists(third.task_id.to_string()).await.unwrap(); + assert!( + !exists_first && !exists_second && !exists_third, + "All messages should be cleaned up" + ); + } +} diff --git a/errors.txt b/errors.txt deleted file mode 100644 index e69de29..0000000 diff --git a/src/brokers/mod.rs b/src/brokers/mod.rs index 428e7b9..8b13789 100644 --- a/src/brokers/mod.rs +++ b/src/brokers/mod.rs @@ -1,10 +1 @@ -/// Contains the generic interfaces for brokers -pub mod broker; -/// Contains functions to connect to a broker -pub(crate) mod connect; -/// Contains the `RabbitMQ` broker implementation -#[cfg(feature = "rabbitmq")] -pub mod rabbitmq; -/// Contains the Redis broker implementation -#[cfg(feature = "redis")] -pub mod redis; + diff --git a/src/brokers/rabbitmq/utils.rs b/src/brokers/rabbitmq/utils.rs index 7e2ab07..86b87fc 100644 --- a/src/brokers/rabbitmq/utils.rs +++ b/src/brokers/rabbitmq/utils.rs @@ -1,128 +1,60 @@ -use lapin::{ - options::{QueueBindOptions, QueueDeclareOptions}, - types::{AMQPValue, FieldTable}, - Channel, Queue, -}; - -use crate::{brokers::broker::BrokerConfig, error::BroccoliError}; - -use super::{broker::RabbitPool, RabbitMQBroker}; - -impl RabbitMQBroker { - pub(crate) fn new() -> Self { - Self::default() - } +#[async_trait::async_trait] +impl QueueManagement for RabbitMQBroker { + async fn retry_queue(&self, queue_name: &str, source_type: QueueType) -> Result { + let pool = self.ensure_pool().await?; + let conn = pool.get().await?; + let channel = conn.create_channel().await?; + + let source_queue = match source_type { + QueueType::Failed => format!("{}_failed", queue_name), + QueueType::Processing => format!("{}_processing", queue_name), + QueueType::Ingestion => return Err(BroccoliError::InvalidOperation("Cannot retry from ingestion queue".into())), + }; - pub(crate) fn new_with_config(config: BrokerConfig) -> Self { - Self { - pool: None, - connected: false, - config: Some(config), - consume_channels: Default::default(), + let mut count = 0; + while let Some(delivery) = channel.basic_get(&source_queue, BasicGetOptions::default()).await? { + channel.basic_publish( + "broccoli", + queue_name, + BasicPublishOptions::default(), + delivery.data, + BasicProperties::default() + ).await?; + + channel.basic_ack(delivery.delivery_tag, BasicAckOptions::default()).await?; + count += 1; } - } - pub(crate) fn ensure_pool(&self) -> Result<&RabbitPool, BroccoliError> { - if !self.connected { - return Err(BroccoliError::Broker( - "RabbitMQ broker not connected".to_string(), - )); - } - self.pool.as_ref().ok_or_else(|| { - BroccoliError::Broker("RabbitMQ connection pool not initialized".to_string()) - }) + Ok(count) } - pub(crate) async fn setup_exchange( - &self, - channel: &Channel, - exchange_name: &str, - ) -> Result<(), BroccoliError> { - #[allow(unused_mut)] - let mut args = FieldTable::default(); - let exchange_kind = if self - .config - .as_ref() - .is_some_and(|c| c.enable_scheduling.unwrap_or(false)) - { - args.insert( - "x-delayed-type".into(), - AMQPValue::LongString("direct".into()), - ); - lapin::ExchangeKind::Custom("x-delayed-message".into()) - } else { - lapin::ExchangeKind::Direct - }; + async fn get_queue_size(&self, queue_name: &str, queue_type: QueueType) -> Result { + let pool = self.ensure_pool().await?; + let conn = pool.get().await?; + let channel = conn.create_channel().await?; - channel - .exchange_declare( - exchange_name, - exchange_kind, - lapin::options::ExchangeDeclareOptions::default(), - args.clone(), - ) - .await - .map_err(|e| BroccoliError::Broker(format!("Failed to declare exchange: {e:?}")))?; + let queue = match queue_type { + QueueType::Failed => format!("{}_failed", queue_name), + QueueType::Processing => format!("{}_processing", queue_name), + QueueType::Ingestion => queue_name.to_string(), + }; - Ok(()) + let queue_info = channel.queue_declare(&queue, QueueDeclareOptions::default(), FieldTable::default()).await?; + Ok(queue_info.message_count() as usize) } - pub(crate) async fn setup_queue( - &self, - channel: &Channel, - queue_name: &str, - ) -> Result { - let mut args = FieldTable::default(); - if !queue_name.contains("failed") { - args.insert( - "x-dead-letter-exchange".into(), - AMQPValue::LongString("".into()), - ); - args.insert( - "x-dead-letter-routing-key".into(), - AMQPValue::LongString(format!("{queue_name}_failed").into()), - ); - } - - args.insert("x-max-priority".into(), AMQPValue::LongInt(5)); - - let queue = channel - .queue_declare( - queue_name, - QueueDeclareOptions { - durable: true, - ..Default::default() - }, - args, - ) - .await - .map_err(|e| BroccoliError::Broker(format!("Failed to declare queue: {e:?}")))?; - - channel - .queue_bind( - queue_name, - "broccoli", - queue_name, // Using queue name as routing key - QueueBindOptions::default(), - FieldTable::default(), - ) - .await - .map_err(|e| BroccoliError::Broker(format!("Failed to bind queue: {e}")))?; - - // Setup DLX for failed messages - let failed_queue = format!("{queue_name}_failed"); - channel - .queue_declare( - &failed_queue, - QueueDeclareOptions { - durable: true, - ..Default::default() - }, - FieldTable::default(), - ) - .await - .map_err(|e| BroccoliError::Broker(format!("Failed to declare failed queue: {e:?}")))?; - - Ok(queue) + async fn get_queue_status(&self) -> Result, BroccoliError> { + let pool = self.ensure_pool().await?; + let conn = pool.get().await?; + let channel = conn.create_channel().await?; + + // List queues through management API or channel operations + // This is a simplified version - in practice you'd want to use the RabbitMQ Management API + let mut statuses = Vec::new(); + + // Implementation note: RabbitMQ doesn't provide memory usage through regular AMQP + // You would need to use the HTTP Management API to get this information + + Ok(statuses) } } diff --git a/src/brokers/redis/mod.rs b/src/brokers/redis/mod.rs deleted file mode 100644 index 26406e3..0000000 --- a/src/brokers/redis/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -/// Contains the Redis Broker implementation -pub mod broker; -/// Utility functions for the Redis Broker -pub(crate) mod utils; - -pub use broker::RedisBroker; diff --git a/src/brokers/redis/utils.rs b/src/brokers/redis/utils.rs index f9e875a..6d487af 100644 --- a/src/brokers/redis/utils.rs +++ b/src/brokers/redis/utils.rs @@ -1,274 +1,78 @@ -//! Utility functions for Redis broker implementation. -//! -//! This module provides helper methods for managing Redis message operations, -//! including message parsing and manipulation of message metadata. - -use std::collections::HashMap; - -use super::broker::{RedisBroker, RedisPool}; -use crate::{ - brokers::broker::{BrokerConfig, InternalBrokerMessage, MetadataTypes}, - error::BroccoliError, - queue::ConsumeOptions, -}; -use redis::{aio::MultiplexedConnection, FromRedisValue}; +// ...existing code... + +#[async_trait::async_trait] +impl QueueManagement for RedisBroker { + async fn retry_queue(&self, queue_name: &str, source_type: QueueType) -> Result { + let mut redis = self.get_redis_connection().await?; + let source_queue = match source_type { + QueueType::Failed => format!("{}_failed", queue_name), + QueueType::Processing => format!("{}_processing", queue_name), + QueueType::Ingestion => return Err(BroccoliError::InvalidOperation("Cannot retry from ingestion queue".into())), + }; -impl RedisBroker { - /// Creates a new `RedisBroker` instance with default configuration. - #[must_use] - pub const fn new() -> Self { - Self { - redis_pool: None, - broker_url: String::new(), - config: None, + let count: usize = redis.llen(&source_queue).await.map_err(|e| BroccoliError::Broker(e.to_string()))?; + if count == 0 { + return Ok(0); } - } - /// Creates a new `RedisBroker` instance with the specified configuration. - /// - /// # Arguments - /// * `config` - The broker configuration to use - #[must_use] - pub const fn new_with_config(config: BrokerConfig) -> Self { - Self { - redis_pool: None, - broker_url: String::new(), - config: Some(config), + for _ in 0..count { + let message: Option = redis.lpop(&source_queue, None).await?; + if let Some(msg) = message { + redis.zadd( + queue_name, + msg, + time::OffsetDateTime::now_utc().unix_timestamp_nanos() as f64, + ).await?; + } } - } - pub(crate) fn ensure_pool(&self) -> Result { - match &self.redis_pool { - Some(pool) => Ok(pool - .try_read() - .map_err(|_| { - BroccoliError::Broker("Failed to acquire read lock on redis pool".to_string()) - })? - .clone()), - None => Err(BroccoliError::Broker( - "Redis pool not initialized".to_string(), - )), - } + Ok(count) } - pub(crate) async fn get_task_id( - &self, - queue_name: &str, - redis_connection: &mut MultiplexedConnection, - options: Option, - ) -> Result, BroccoliError> { - let popped_message: Option = if options - .as_ref() - .is_some_and(|x| x.fairness.unwrap_or(false)) - { - let script = redis::Script::new( - r#" - local current_time = tonumber(ARGV[1]) - local queue_to_process = redis.call('LPOP', KEYS[1]) - if not queue_to_process then - return nil - end - - local popped_message = redis.call('ZPOPMIN', string.format("%s_%s_queue", KEYS[2], queue_to_process), 1) - if #popped_message == 0 then - return nil - end - - local message = popped_message[1] - local score = tonumber(popped_message[2]) - - if score > (5.0 * current_time) then - redis.call('ZADD', string.format("%s_%s_queue", KEYS[2], queue_to_process), score, message) - redis.call('RPUSH', KEYS[1], queue_to_process) - return nil - end - - local does_subqueue_exist = redis.call('EXISTS', string.format("%s_%s_queue", KEYS[2], queue_to_process)) == 1 - if does_subqueue_exist then - redis.call('RPUSH', KEYS[1], queue_to_process) - else - redis.call('SREM', KEYS[3], queue_to_process) - end - - if ARGV[2] == "false" then - local processing_queue_name = string.format("%s_%s_processing", KEYS[2], queue_to_process) - redis.call('LPUSH', processing_queue_name, message) - end - - return message - "#, - ); - - script - .arg(time::OffsetDateTime::now_utc().unix_timestamp_nanos() as f64) - .arg( - options - .is_some_and(|x| x.auto_ack.unwrap_or(false)) - .to_string(), - ) - .key(format!("{queue_name}_fairness_round_robin")) - .key(queue_name) - .key(format!("{queue_name}_fairness_set")) - .invoke_async(redis_connection) - .await? - } else { - let script = redis::Script::new( - r#" - local current_time = tonumber(ARGV[1]) - local popped_message = redis.call('ZPOPMIN', KEYS[1], 1) - if #popped_message == 0 then - return nil - end - - local message = popped_message[1] - local score = tonumber(popped_message[2]) - - if score > (5.0 * current_time) then - redis.call('ZADD', KEYS[1], score, message) - return nil - end - - if ARGV[2] == "false" then - redis.call('LPUSH', KEYS[2], message) - end - return message - "#, - ); - - script - .arg(time::OffsetDateTime::now_utc().unix_timestamp_nanos() as f64) - .arg( - options - .is_some_and(|x| x.auto_ack.unwrap_or(false)) - .to_string(), - ) - .key(queue_name) - .key(format!("{queue_name}_processing")) - .invoke_async(redis_connection) - .await? + async fn get_queue_size(&self, queue_name: &str, queue_type: QueueType) -> Result { + let mut redis = self.get_redis_connection().await?; + let queue = match queue_type { + QueueType::Failed => format!("{}_failed", queue_name), + QueueType::Processing => format!("{}_processing", queue_name), + QueueType::Ingestion => queue_name.to_string(), }; - Ok(popped_message) - } - - /// Retrieves a Redis connection from the pool, retrying with exponential backoff if necessary. - /// - /// # Arguments - /// * `redis_pool` - A reference to the Redis connection pool. - /// - /// # Returns - /// A `Result` containing a `RedisConnection` on success, or a `BroccoliError` on failure. - pub(crate) async fn get_redis_connection( - &self, - ) -> Result { - let mut redis_conn_sleep = std::time::Duration::from_secs(1); - - let redis_pool = self.ensure_pool()?; - - #[allow(unused_assignments)] - let mut opt_redis_connection = None; - - loop { - let borrowed_redis_connection = match redis_pool.get().await { - Ok(redis_connection) => Some(redis_connection), - Err(err) => { - let redis_manager = bb8_redis::RedisConnectionManager::new( - self.broker_url.clone(), - ) - .map_err(|e| { - BroccoliError::Broker(format!("Failed to create redis manager: {e:?}")) - })?; - - let redis_pool = bb8_redis::bb8::Pool::builder() - .max_size( - self.config - .as_ref() - .map_or(10, |config| config.pool_connections.unwrap_or(10)) - .into(), - ) - .connection_timeout(std::time::Duration::from_secs(2)) - .build(redis_manager) - .await - .map_err(|e| { - BroccoliError::Broker(format!("Failed to create redis pool: {e:?}")) - })?; - - { - let mut pool_write = self - .redis_pool - .as_ref() - .ok_or_else(|| { - BroccoliError::Broker("Redis pool not initialized".to_string()) - })? - .write() - .map_err(|_| { - BroccoliError::Broker( - "Failed to acquire write lock on redis pool".to_string(), - ) - })?; - *pool_write = redis_pool; - } - BroccoliError::Broker(format!("Failed to get redis connection: {err:?}")); - None - } - }; - - if borrowed_redis_connection.is_some() { - opt_redis_connection = borrowed_redis_connection; - break; - } - - tokio::time::sleep(redis_conn_sleep).await; - redis_conn_sleep = - std::cmp::min(redis_conn_sleep * 2, std::time::Duration::from_secs(300)); + match queue_type { + QueueType::Ingestion => redis.zcard(&queue).await.map_err(|e| e.into()), + _ => redis.llen(&queue).await.map_err(|e| e.into()), } - - let redis_connection = opt_redis_connection - .ok_or_else(|| BroccoliError::Broker("Failed to get redis connection".to_string()))?; - - Ok(redis_connection.clone()) } -} -pub struct OptionalInternalBrokerMessage(pub Option); + async fn get_queue_status(&self) -> Result, BroccoliError> { + let mut redis = self.get_redis_connection().await?; + let mut keys: Vec = redis.keys("*").await?; + keys.sort(); + + let mut statuses = Vec::new(); + for key in keys { + let (size, queue_type) = if key.ends_with("_failed") || key.ends_with("_processing") { + let size: usize = redis.llen(&key).await?; + let queue_type = if key.ends_with("_failed") { + QueueType::Failed + } else { + QueueType::Processing + }; + (size, queue_type) + } else { + let size: usize = redis.zcard(&key).await?; + (size, QueueType::Ingestion) + }; -impl FromRedisValue for OptionalInternalBrokerMessage { - fn from_redis_value(v: &redis::Value) -> redis::RedisResult { - let map: std::collections::HashMap = redis::from_redis_value(v)?; - if map.is_empty() { - return Ok(Self(None)); + let memory: i64 = redis.memory_usage(&key, None).await?; + + statuses.push(QueueStatus { + name: key, + queue_type, + size, + memory_usage: memory as u64, + }); } - let task_id = map.get("task_id").ok_or_else(|| { - redis::RedisError::from((redis::ErrorKind::TypeError, "Missing field: task_id")) - })?; - - let payload = map.get("payload").ok_or_else(|| { - redis::RedisError::from((redis::ErrorKind::TypeError, "Missing field: payload")) - })?; - - let attempts = map.get("attempts").ok_or_else(|| { - redis::RedisError::from((redis::ErrorKind::TypeError, "Missing field: attempts")) - })?; - - let priority = map.get("priority").ok_or_else(|| { - redis::RedisError::from((redis::ErrorKind::TypeError, "Missing field: priority")) - })?; - - let disambiguator = map.get("disambiguator"); - - let mut metadata = HashMap::new(); - metadata.insert( - "priority".to_string(), - MetadataTypes::String(priority.to_string()), - ); - - Ok(Self(Some(InternalBrokerMessage { - task_id: task_id.to_string(), - payload: payload.to_string(), - attempts: attempts.parse().unwrap_or_default(), - disambiguator: disambiguator.cloned(), - metadata: Some(metadata), - }))) + Ok(statuses) } } From 058b234c6e46bdb7a7ca1b89332c3206e0d9c5f8 Mon Sep 17 00:00:00 2001 From: densumesh Date: Mon, 24 Feb 2025 23:42:44 -0700 Subject: [PATCH 2/2] feature: added management api for redis --- .github/workflows/integ_tests.yaml | 4 +- .vscode/settings.json | 9 +- Cargo.lock | 203 ++++++++++--- broccoli-queue/Cargo.toml | 3 +- broccoli-queue/examples/consumer.rs | 2 +- broccoli-queue/src/brokers/management.rs | 25 +- .../src/brokers/rabbitmq/management.rs | 46 +-- broccoli-queue/src/brokers/redis/broker.rs | 2 +- .../src/brokers/redis/management.rs | 286 +++++++++++------- broccoli-queue/src/error.rs | 8 + broccoli-queue/src/queue.rs | 31 +- broccoli-queue/tests/edge_cases.rs | 3 +- src/brokers/mod.rs | 1 - src/brokers/rabbitmq/utils.rs | 60 ---- src/brokers/redis/utils.rs | 78 ----- 15 files changed, 387 insertions(+), 374 deletions(-) delete mode 100644 src/brokers/mod.rs delete mode 100644 src/brokers/rabbitmq/utils.rs delete mode 100644 src/brokers/redis/utils.rs diff --git a/.github/workflows/integ_tests.yaml b/.github/workflows/integ_tests.yaml index 80e02a5..3fbd3c8 100644 --- a/.github/workflows/integ_tests.yaml +++ b/.github/workflows/integ_tests.yaml @@ -37,7 +37,7 @@ jobs: restore-keys: ${{ runner.os }}-cargo- - name: Make test script executable - run: chmod +x ./run-tests.sh + run: chmod +x ./broccoli-queue/run-tests.sh - name: Run tests - run: ./run-tests.sh \ No newline at end of file + run: ./broccoli-queue/run-tests.sh \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 29a1270..d1e339e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,4 @@ { - "rust-analyzer.showUnlinkedFileNotification": false, - "rust-analyzer.cargo.features": [ - "redis", - "rabbitmq", - "test-fairness", - "management" - ] + "rust-analyzer.showUnlinkedFileNotification": true, + "rust-analyzer.cargo.features": "all" } diff --git a/Cargo.lock b/Cargo.lock index 3ac9310..2444c67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,9 +158,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" [[package]] name = "arc-swap" @@ -180,7 +180,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror", + "thiserror 2.0.11", "time", ] @@ -450,14 +450,11 @@ dependencies = [ "anyhow", "broccoli_queue", "clap", - "colored", - "env_logger", - "humansize", - "log", + "dotenv", + "prettytable", "redis", "serde_json", - "tabwriter", - "thiserror", + "thiserror 2.0.11", "tokio", ] @@ -472,6 +469,7 @@ dependencies = [ "dashmap", "deadpool", "deadpool-lapin", + "derive_more", "env_logger", "futures", "lapin", @@ -481,7 +479,7 @@ dependencies = [ "serde", "serde_json", "sha256", - "thiserror", + "thiserror 2.0.11", "time", "tokio", "uuid", @@ -645,16 +643,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" -[[package]] -name = "colored" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" -dependencies = [ - "lazy_static", - "windows-sys 0.59.0", -] - [[package]] name = "combine" version = "4.6.7" @@ -794,6 +782,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +dependencies = [ + "memchr", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -892,6 +901,27 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + [[package]] name = "des" version = "0.8.1" @@ -912,6 +942,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -929,12 +980,24 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "env_filter" version = "0.1.3" @@ -1250,15 +1313,6 @@ dependencies = [ "digest", ] -[[package]] -name = "humansize" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" -dependencies = [ - "libm", -] - [[package]] name = "humantime" version = "2.1.0" @@ -1543,10 +1597,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] -name = "libm" -version = "0.2.11" +name = "libredox" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.8.0", + "libc", +] [[package]] name = "linux-raw-sys" @@ -1722,7 +1780,7 @@ dependencies = [ "rc2", "sha1", "sha2", - "thiserror", + "thiserror 2.0.11", "x509-parser", ] @@ -1919,6 +1977,20 @@ dependencies = [ "zerocopy 0.7.35", ] +[[package]] +name = "prettytable" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46480520d1b77c9a3482d39939fcf96831537a250ec62d4fd8fbdf8e0302e781" +dependencies = [ + "csv", + "encode_unicode", + "is-terminal", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -2046,6 +2118,17 @@ dependencies = [ "bitflags 2.8.0", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.15", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "regex" version = "1.11.1" @@ -2458,15 +2541,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tabwriter" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce91f2f0ec87dff7e6bcbbeb267439aa1188703003c6055193c821487400432" -dependencies = [ - "unicode-width", -] - [[package]] name = "tcp-stream" version = "0.28.0" @@ -2479,13 +2553,44 @@ dependencies = [ "rustls-pemfile", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.11", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2643,9 +2748,15 @@ checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -3030,7 +3141,7 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror", + "thiserror 2.0.11", "time", ] diff --git a/broccoli-queue/Cargo.toml b/broccoli-queue/Cargo.toml index dc4d2ac..86778a6 100644 --- a/broccoli-queue/Cargo.toml +++ b/broccoli-queue/Cargo.toml @@ -32,6 +32,7 @@ redis = { version = "0.27.5", features = [ "aio", ], optional = true } dashmap = "6.1.0" +derive_more = { version = "2.0.1", features = ["display"], optional = true } [dev-dependencies] chrono = { version = "0.4.39", features = ["serde"] } @@ -52,7 +53,7 @@ rabbitmq = ["dep:lapin", "dep:deadpool", "dep:deadpool-lapin"] test-fairness = [] # Allows for access to the management API for the queue. -management = [] +management = ["dep:derive_more"] [[bench]] name = "queue_benchmark" diff --git a/broccoli-queue/examples/consumer.rs b/broccoli-queue/examples/consumer.rs index c5f5d97..8e9aff8 100644 --- a/broccoli-queue/examples/consumer.rs +++ b/broccoli-queue/examples/consumer.rs @@ -23,7 +23,7 @@ async fn process_job(job: JobPayload) -> Result<(), BroccoliError> { // Simulate some work tokio::time::sleep(Duration::from_secs(1)).await; - Ok(()) + Err(BroccoliError::Consume("Failed to process job".into())) } async fn success_handler(msg: JobPayload) -> Result<(), BroccoliError> { diff --git a/broccoli-queue/src/brokers/management.rs b/broccoli-queue/src/brokers/management.rs index 2acba56..5782abd 100644 --- a/broccoli-queue/src/brokers/management.rs +++ b/broccoli-queue/src/brokers/management.rs @@ -1,3 +1,5 @@ +use derive_more::Display; + use crate::error::BroccoliError; use super::broker::Broker; @@ -5,37 +7,36 @@ use super::broker::Broker; #[async_trait::async_trait] /// Trait for managing queues. pub trait QueueManagement { - /// Retries all messages in the queue. + /// Retries messages in the queue. async fn retry_queue( &self, - queue_name: &str, - disambiguator: Option, + queue_name: String, source_type: QueueType, ) -> Result; - /// Gets the size of the queue. - async fn get_queue_size( - &self, - queue_name: &str, - queue_type: QueueType, - ) -> Result; /// Gets the status of specific or all queues. If `queue_name` is `None`, returns the status of all queues. async fn get_queue_status( &self, - queue_name: Option<&str>, + queue_name: Option, ) -> Result, BroccoliError>; } pub(crate) trait BrokerWithManagement: Broker + QueueManagement {} -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, Display)] /// Enum representing the type of queue. pub enum QueueType { /// Failed queue. + #[display("failed")] Failed, /// Processing queue. + #[display("processing")] Processing, /// Main queue. + #[display("main")] Main, + /// Fairness queue. + #[display("fairness")] + Fairness, } #[derive(Debug, Clone)] @@ -51,4 +52,6 @@ pub struct QueueStatus { pub processing: usize, /// Number of messages that failed to be processed. pub failed: usize, + /// If the queue is a fairness queue, the number of disambiguators. + pub disambiguator_count: Option, } diff --git a/broccoli-queue/src/brokers/rabbitmq/management.rs b/broccoli-queue/src/brokers/rabbitmq/management.rs index 774f7b7..78e35ad 100644 --- a/broccoli-queue/src/brokers/rabbitmq/management.rs +++ b/broccoli-queue/src/brokers/rabbitmq/management.rs @@ -1,6 +1,5 @@ use lapin::{ - options::{BasicAckOptions, BasicGetOptions, BasicPublishOptions, QueueDeclareOptions}, - types::FieldTable, + options::{BasicAckOptions, BasicGetOptions, BasicPublishOptions}, BasicProperties, }; @@ -15,8 +14,7 @@ use super::RabbitMQBroker; impl QueueManagement for RabbitMQBroker { async fn retry_queue( &self, - queue_name: &str, - _disambiguator: Option, + queue_name: String, source_type: QueueType, ) -> Result { let pool = self.ensure_pool()?; @@ -34,6 +32,11 @@ impl QueueManagement for RabbitMQBroker { "Cannot retry from ingestion queue".into(), )) } + QueueType::Fairness => { + return Err(BroccoliError::InvalidOperation( + "Cannot retry from fairness queue".into(), + )) + } }; let mut count = 0; @@ -44,7 +47,7 @@ impl QueueManagement for RabbitMQBroker { channel .basic_publish( "broccoli", - queue_name, + &queue_name, BasicPublishOptions::default(), &delivery.data, BasicProperties::default(), @@ -60,35 +63,10 @@ impl QueueManagement for RabbitMQBroker { Ok(count) } - async fn get_queue_size( + async fn get_queue_status( &self, - queue_name: &str, - queue_type: QueueType, - ) -> Result { - let pool = self.ensure_pool()?; - let conn = pool - .get() - .await - .map_err(|e| BroccoliError::Consume(format!("Failed to consume message: {e:?}")))?; - let channel = conn.create_channel().await?; - - let queue = match queue_type { - QueueType::Failed => format!("{queue_name}_failed"), - QueueType::Processing => format!("{queue_name}_processing"), - QueueType::Main => queue_name.to_string(), - }; - - let queue_info = channel - .queue_declare( - &queue, - QueueDeclareOptions::default(), - FieldTable::default(), - ) - .await?; - Ok(queue_info.message_count() as usize) - } - - async fn get_queue_status(&self) -> Result, BroccoliError> { + queue_name: Option, + ) -> Result, BroccoliError> { let pool = self.ensure_pool()?; let conn = pool .get() @@ -98,7 +76,7 @@ impl QueueManagement for RabbitMQBroker { // List queues through management API or channel operations // This is a simplified version - in practice you'd want to use the RabbitMQ Management API - let mut statuses = Vec::new(); + let statuses = Vec::new(); // Implementation note: RabbitMQ doesn't provide memory usage through regular AMQP // You would need to use the HTTP Management API to get this information diff --git a/broccoli-queue/src/brokers/redis/broker.rs b/broccoli-queue/src/brokers/redis/broker.rs index a8e5cba..83aaebc 100644 --- a/broccoli-queue/src/brokers/redis/broker.rs +++ b/broccoli-queue/src/brokers/redis/broker.rs @@ -10,7 +10,7 @@ use crate::{ use super::utils::OptionalInternalBrokerMessage; -pub(crate) type RedisPool = bb8_redis::bb8::Pool; +pub type RedisPool = bb8_redis::bb8::Pool; #[derive(Default)] /// A message broker implementation for Redis. diff --git a/broccoli-queue/src/brokers/redis/management.rs b/broccoli-queue/src/brokers/redis/management.rs index 046e895..196a07a 100644 --- a/broccoli-queue/src/brokers/redis/management.rs +++ b/broccoli-queue/src/brokers/redis/management.rs @@ -1,98 +1,119 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use redis::AsyncCommands; use crate::{ - brokers::{ - broker::Broker, - management::{BrokerWithManagement, QueueManagement, QueueStatus, QueueType}, - }, + brokers::management::{BrokerWithManagement, QueueManagement, QueueStatus, QueueType}, error::BroccoliError, }; -use super::{utils::OptionalInternalBrokerMessage, RedisBroker}; +use super::RedisBroker; #[async_trait::async_trait] impl QueueManagement for RedisBroker { async fn retry_queue( &self, - queue_name: &str, - disambiguator: Option, + queue_name: String, source_type: QueueType, ) -> Result { let mut redis = self.get_redis_connection().await?; - let source_queue = if let Some(ref disambiguator) = disambiguator { - match source_type { - QueueType::Failed => format!("{queue_name}_{disambiguator}_failed"), - QueueType::Processing => format!("{queue_name}_{disambiguator}_processing"), - QueueType::Main => { - return Err(BroccoliError::InvalidOperation( - "Cannot retry from ingestion queue".into(), - )) + + if source_type == QueueType::Main { + return Err(BroccoliError::InvalidOperation( + "Cannot retry from main queue".into(), + )); + } + + let fairness_pattern = format!("{queue_name}*_{}", source_type.to_string().to_lowercase()); + let is_fairness_queue = !redis + .keys::<&String, Vec>(&fairness_pattern) + .await? + .is_empty(); + + let mut total_retried = 0; + + if is_fairness_queue { + let failed_pattern = + format!("{}*_{}", queue_name, source_type.to_string().to_lowercase()); + let failed_queues: Vec = redis.keys(&failed_pattern).await?; + + for failed_queue in failed_queues { + println!("Retrying from queue: {failed_queue}"); + let parts: Vec<&str> = failed_queue.split('_').collect(); + if parts.len() < 3 { + continue; } - } - } else { - match source_type { - QueueType::Failed => format!("{queue_name}_failed"), - QueueType::Processing => format!("{queue_name}_processing"), - QueueType::Main => { - return Err(BroccoliError::InvalidOperation( - "Cannot retry from ingestion queue".into(), - )) + + let disambiguator = parts[1]; + + let target_queue = format!("{queue_name}_{disambiguator}_queue"); + + let count = redis.llen(&failed_queue).await?; + if count == 0 { + continue; + } + + for _ in 0..count { + let task_id: Option = redis.lpop(&failed_queue, None).await?; + if let Some(task_id) = task_id { + redis + .zadd::<&str, i64, &str, ()>( + &target_queue, + &task_id, + time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64, + ) + .await?; + } } - } - }; - let count = redis.llen(&source_queue).await?; + let fairness_set = format!("{queue_name}_fairness_set"); + redis + .sadd::<&str, &str, ()>(&fairness_set, disambiguator) + .await?; - if count == 0 { - return Ok(0); - } + let fairness_round_robin = format!("{queue_name}_fairness_round_robin"); + redis + .rpush::<&str, &str, ()>(&fairness_round_robin, disambiguator) + .await?; - let mut messages = vec![]; - for _ in 0..count { - let task_id: Option = redis.lpop(&source_queue, None).await?; - let message: OptionalInternalBrokerMessage = redis - .hgetall(&task_id) - .await - .map_err(|e| BroccoliError::Consume(format!("Failed to consume message: {e:?}")))?; - if let Some(mut message) = message.0 { - message.attempts = 0; - messages.push(message); + total_retried += count as usize; } - } - self.publish(queue_name, disambiguator, &messages, None) - .await?; + Ok(total_retried) + } else { + let source_queue = format!("{}_{}", queue_name, source_type.to_string().to_lowercase()); - Ok(count) - } + let count = redis.llen(&source_queue).await?; - async fn get_queue_size( - &self, - queue_name: &str, - queue_type: QueueType, - ) -> Result { - let mut redis = self.get_redis_connection().await?; - let queue = match queue_type { - QueueType::Failed => format!("{queue_name}_failed"), - QueueType::Processing => format!("{queue_name}_processing"), - QueueType::Main => queue_name.to_string(), - }; + if count == 0 { + return Ok(0); + } + + for _ in 0..count { + let task_id: Option = redis.lpop(&source_queue, None).await?; + if let Some(task_id) = task_id { + redis + .zadd::<&str, f64, &str, ()>( + &queue_name, + &task_id, + time::OffsetDateTime::now_utc().unix_timestamp_nanos() as f64, + ) + .await?; + println!("Retrying task: {task_id}"); + } + } - match queue_type { - QueueType::Main => redis.zcard(&queue).await.map_err(std::convert::Into::into), - _ => redis.llen(&queue).await.map_err(std::convert::Into::into), + Ok(count as usize) } } async fn get_queue_status( &self, - queue_name: Option<&str>, + queue_name: Option, ) -> Result, BroccoliError> { let mut redis = self.get_redis_connection().await?; let mut keys: Vec = if let Some(queue_name) = queue_name { - redis.keys(&format!("{}*", queue_name)).await? + redis.keys(format!("{queue_name}*")).await? } else { redis.keys("*").await? }; @@ -100,51 +121,53 @@ impl QueueManagement for RedisBroker { keys.sort(); let mut queues: HashMap = HashMap::new(); - let grouped_keys = keys.iter().fold(queues, |mut acc, key| { - let (queue_name, queue_type) = - if key.ends_with("_failed") || key.ends_with("_processing") { - let queue_name = key - .trim_end_matches("_failed") - .trim_end_matches("_processing"); - let queue_type = if key.ends_with("_failed") { - QueueType::Failed - } else { - QueueType::Processing - }; - (queue_name, queue_type) - } else if redis.key_type::<&str, String>(&key).await? == *"zset" { - let queue_name = key; - let queue_type = QueueType::Main; - (queue_name, queue_type) - } else { - return acc; - }; - - let status = acc.entry(queue_name.to_string()).or_insert(QueueStatus { - name: queue_name.to_string(), - queue_type, - size: 0, - }); - - match queue_type { - QueueType::Main => { - let size: usize = redis.zcard(&queue_name).await.unwrap_or(0); - status.size = size; - } - _ => { - let size: usize = redis.llen(&key).await.unwrap_or(0); - match queue_type { - QueueType::Failed => status.failed = size, - QueueType::Processing => status.processing = size, - _ => {} + let mut fairness_queues: HashMap> = HashMap::new(); + + // First pass: identify fairness queues and collect disambiguators + for key in &keys { + // Check if it's a fairness queue by looking for pattern {base_name}_{disambiguator}_queue + if key.contains("_queue") { + let parts: Vec<&str> = key.split('_').collect(); + if parts.len() >= 3 && parts.last() == Some(&"queue") { + // This is potentially a fairness queue + // Extract the base name (everything before the disambiguator) + let pos = key.find('_').unwrap_or(0); + if pos > 0 { + let base_name = &key[0..pos]; + + // Extract the disambiguator (between base_name and "_queue") + let disambiguator = key[pos + 1..key.len() - 6].to_string(); // -6 to remove "_queue" + + // Add this disambiguator to the set for this base queue + fairness_queues + .entry(base_name.to_string()) + .or_default() + .insert(disambiguator); } } } + } - acc - }); - + // Second pass: Process all queues for key in keys { + let is_fairness_subqueue = |k: &str| -> Option<(String, String)> { + for base_name in fairness_queues.keys() { + if k.starts_with(base_name) + && (k.ends_with("_failed") + || k.ends_with("_processing") + || k.ends_with("_queue")) + { + let remaining = &k[base_name.len() + 1..]; // +1 for the underscore + let parts: Vec<&str> = remaining.split('_').collect(); + if parts.len() >= 2 { + let disambiguator = parts[0..parts.len() - 1].join("_"); + return Some((base_name.clone(), disambiguator)); + } + } + } + None + }; + let (size, queue_type) = if key.ends_with("_failed") || key.ends_with("_processing") { let size: usize = redis.llen(&key).await?; let queue_type = if key.ends_with("_failed") { @@ -160,14 +183,63 @@ impl QueueManagement for RedisBroker { continue; }; - QueueStatus { - name: key, - queue_type, - size, + // Check if this is a subqueue of a fairness queue + if let Some((base_name, _)) = is_fairness_subqueue(&key) { + // This is a fairness subqueue, add its stats to the base fairness queue + let status = queues.entry(base_name.clone()).or_insert(QueueStatus { + name: base_name.clone(), + queue_type: QueueType::Fairness, + size: 0, + failed: 0, + processing: 0, + disambiguator_count: fairness_queues + .get(&base_name) + .map_or(Some(0), |set| Some(set.len())), + }); + + match queue_type { + QueueType::Main => status.size += size, + QueueType::Failed => status.failed += size, + QueueType::Processing => status.processing += size, + QueueType::Fairness => {} + } + } else { + // Regular queue processing + let queue_name = if queue_type == QueueType::Main { + key.to_string() + } else { + key.trim_end_matches("_failed") + .trim_end_matches("_processing") + .to_string() + }; + + let status = queues.entry(queue_name.clone()).or_insert(QueueStatus { + name: queue_name, + queue_type: QueueType::Main, + size: 0, + failed: 0, + processing: 0, + disambiguator_count: None, + }); + + match queue_type { + QueueType::Main => status.size = size, + QueueType::Failed => status.failed = size, + QueueType::Processing => status.processing = size, + QueueType::Fairness => {} + } + } + } + + // Add fairness queue type for identified fairness queues + for (base_name, disambiguators) in fairness_queues { + if let Some(status) = queues.get_mut(&base_name) { + status.queue_type = QueueType::Fairness; // You'll need to add this variant to QueueType + status.disambiguator_count = Some(disambiguators.len()); } } - Ok(statuses) + Ok(queues.into_values().collect()) } } diff --git a/broccoli-queue/src/error.rs b/broccoli-queue/src/error.rs index 7bf2997..02d66dd 100644 --- a/broccoli-queue/src/error.rs +++ b/broccoli-queue/src/error.rs @@ -85,6 +85,14 @@ pub enum BroccoliError { #[error("Job error: {0}")] Job(String), + /// Represents errors that occur during checking queue status. + #[error("Queue status error: {0}")] + QueueStatus(String), + + /// Represents errors that occur when retrying a queue + #[error("Retry Error: {0}")] + Retry(String), + /// Represents connection timeout errors. /// /// # Arguments diff --git a/broccoli-queue/src/queue.rs b/broccoli-queue/src/queue.rs index 6fd11c0..0fe9278 100644 --- a/broccoli-queue/src/queue.rs +++ b/broccoli-queue/src/queue.rs @@ -5,8 +5,8 @@ use time::Duration; use time::OffsetDateTime; -use crate::brokers::management::BrokerWithManagement; -use crate::brokers::management::QueueManagement; +#[cfg(feature = "management")] +use crate::brokers::management::{BrokerWithManagement, QueueStatus, QueueType}; use crate::{ brokers::{ broker::{Broker, BrokerConfig, BrokerMessage, InternalBrokerMessage}, @@ -932,7 +932,10 @@ impl BroccoliQueue { /// # Errors /// If the queue status fails to be retrieved, a `BroccoliError` will be returned. #[cfg(feature = "management")] - pub async fn queue_status(&self, queue_name: &str) -> Result { + pub async fn queue_status( + &self, + queue_name: Option, + ) -> Result, BroccoliError> { self.broker .get_queue_status(queue_name) .await @@ -954,30 +957,12 @@ impl BroccoliQueue { #[cfg(feature = "management")] pub async fn retry_queue( &self, - queue_name: &str, - disambiguator: Option, + queue_name: String, source_type: QueueType, ) -> Result { self.broker - .retry_queue(queue_name, disambiguator, source_type) + .retry_queue(queue_name, source_type) .await .map_err(|e| BroccoliError::Retry(format!("Failed to retry queue: {e:?}"))) } - - /// Moves all messages from the failed queue back to the main queue for retrying. - /// - /// # Arguments - /// * `queue_name` - The name of the queue. - /// - /// # Returns - /// A `Result` containing the number of messages retried, or a `BroccoliError` on failure. - /// - /// # Errors - /// If the messages fail to be retried, a `BroccoliError` will be returned. - pub async fn retry_failed_queue(&self, queue_name: &str) -> Result { - self.broker - .retry_failed_queue(queue_name) - .await - .map_err(|e| BroccoliError::Retry(format!("Failed to retry failed queue: {e:?}"))) - } } diff --git a/broccoli-queue/tests/edge_cases.rs b/broccoli-queue/tests/edge_cases.rs index 748d1bf..ccb5e29 100644 --- a/broccoli-queue/tests/edge_cases.rs +++ b/broccoli-queue/tests/edge_cases.rs @@ -1,6 +1,5 @@ -use broccoli_queue::queue::ConsumeOptions; use broccoli_queue::queue::ConsumeOptionsBuilder; -use broccoli_queue::queue::PublishOptions; +use broccoli_queue::queue::{ConsumeOptions, PublishOptions}; #[cfg(feature = "redis")] use redis::AsyncCommands; use serde::{Deserialize, Serialize}; diff --git a/src/brokers/mod.rs b/src/brokers/mod.rs deleted file mode 100644 index 8b13789..0000000 --- a/src/brokers/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/brokers/rabbitmq/utils.rs b/src/brokers/rabbitmq/utils.rs deleted file mode 100644 index 86b87fc..0000000 --- a/src/brokers/rabbitmq/utils.rs +++ /dev/null @@ -1,60 +0,0 @@ -#[async_trait::async_trait] -impl QueueManagement for RabbitMQBroker { - async fn retry_queue(&self, queue_name: &str, source_type: QueueType) -> Result { - let pool = self.ensure_pool().await?; - let conn = pool.get().await?; - let channel = conn.create_channel().await?; - - let source_queue = match source_type { - QueueType::Failed => format!("{}_failed", queue_name), - QueueType::Processing => format!("{}_processing", queue_name), - QueueType::Ingestion => return Err(BroccoliError::InvalidOperation("Cannot retry from ingestion queue".into())), - }; - - let mut count = 0; - while let Some(delivery) = channel.basic_get(&source_queue, BasicGetOptions::default()).await? { - channel.basic_publish( - "broccoli", - queue_name, - BasicPublishOptions::default(), - delivery.data, - BasicProperties::default() - ).await?; - - channel.basic_ack(delivery.delivery_tag, BasicAckOptions::default()).await?; - count += 1; - } - - Ok(count) - } - - async fn get_queue_size(&self, queue_name: &str, queue_type: QueueType) -> Result { - let pool = self.ensure_pool().await?; - let conn = pool.get().await?; - let channel = conn.create_channel().await?; - - let queue = match queue_type { - QueueType::Failed => format!("{}_failed", queue_name), - QueueType::Processing => format!("{}_processing", queue_name), - QueueType::Ingestion => queue_name.to_string(), - }; - - let queue_info = channel.queue_declare(&queue, QueueDeclareOptions::default(), FieldTable::default()).await?; - Ok(queue_info.message_count() as usize) - } - - async fn get_queue_status(&self) -> Result, BroccoliError> { - let pool = self.ensure_pool().await?; - let conn = pool.get().await?; - let channel = conn.create_channel().await?; - - // List queues through management API or channel operations - // This is a simplified version - in practice you'd want to use the RabbitMQ Management API - let mut statuses = Vec::new(); - - // Implementation note: RabbitMQ doesn't provide memory usage through regular AMQP - // You would need to use the HTTP Management API to get this information - - Ok(statuses) - } -} diff --git a/src/brokers/redis/utils.rs b/src/brokers/redis/utils.rs deleted file mode 100644 index 6d487af..0000000 --- a/src/brokers/redis/utils.rs +++ /dev/null @@ -1,78 +0,0 @@ -// ...existing code... - -#[async_trait::async_trait] -impl QueueManagement for RedisBroker { - async fn retry_queue(&self, queue_name: &str, source_type: QueueType) -> Result { - let mut redis = self.get_redis_connection().await?; - let source_queue = match source_type { - QueueType::Failed => format!("{}_failed", queue_name), - QueueType::Processing => format!("{}_processing", queue_name), - QueueType::Ingestion => return Err(BroccoliError::InvalidOperation("Cannot retry from ingestion queue".into())), - }; - - let count: usize = redis.llen(&source_queue).await.map_err(|e| BroccoliError::Broker(e.to_string()))?; - if count == 0 { - return Ok(0); - } - - for _ in 0..count { - let message: Option = redis.lpop(&source_queue, None).await?; - if let Some(msg) = message { - redis.zadd( - queue_name, - msg, - time::OffsetDateTime::now_utc().unix_timestamp_nanos() as f64, - ).await?; - } - } - - Ok(count) - } - - async fn get_queue_size(&self, queue_name: &str, queue_type: QueueType) -> Result { - let mut redis = self.get_redis_connection().await?; - let queue = match queue_type { - QueueType::Failed => format!("{}_failed", queue_name), - QueueType::Processing => format!("{}_processing", queue_name), - QueueType::Ingestion => queue_name.to_string(), - }; - - match queue_type { - QueueType::Ingestion => redis.zcard(&queue).await.map_err(|e| e.into()), - _ => redis.llen(&queue).await.map_err(|e| e.into()), - } - } - - async fn get_queue_status(&self) -> Result, BroccoliError> { - let mut redis = self.get_redis_connection().await?; - let mut keys: Vec = redis.keys("*").await?; - keys.sort(); - - let mut statuses = Vec::new(); - for key in keys { - let (size, queue_type) = if key.ends_with("_failed") || key.ends_with("_processing") { - let size: usize = redis.llen(&key).await?; - let queue_type = if key.ends_with("_failed") { - QueueType::Failed - } else { - QueueType::Processing - }; - (size, queue_type) - } else { - let size: usize = redis.zcard(&key).await?; - (size, QueueType::Ingestion) - }; - - let memory: i64 = redis.memory_usage(&key, None).await?; - - statuses.push(QueueStatus { - name: key, - queue_type, - size, - memory_usage: memory as u64, - }); - } - - Ok(statuses) - } -}