From 005817f5328777fa39fb2a8b92c067e6650271ea Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 3 May 2023 16:09:36 +0200 Subject: [PATCH] feat: [#91] added image proxy with cache --- .gitignore | 2 +- Cargo.lock | 341 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 3 + src/app.rs | 3 + src/cache/cache.rs | 197 +++++++++++++++++++++ src/cache/image/manager.rs | 200 ++++++++++++++++++++++ src/cache/image/mod.rs | 1 + src/cache/mod.rs | 2 + src/common.rs | 4 + src/config.rs | 18 ++ src/errors.rs | 5 + src/lib.rs | 1 + src/routes/mod.rs | 2 + src/routes/proxy.rs | 85 +++++++++ src/tracker.rs | 2 +- 15 files changed, 863 insertions(+), 3 deletions(-) create mode 100644 src/cache/cache.rs create mode 100644 src/cache/image/manager.rs create mode 100644 src/cache/image/mod.rs create mode 100644 src/cache/mod.rs create mode 100644 src/routes/proxy.rs diff --git a/.gitignore b/.gitignore index eb90c276..c7d2c3f0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ /storage/ /target /uploads/ - +/.idea/ diff --git a/Cargo.lock b/Cargo.lock index 6caf1c77..fb3ebdd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -309,6 +309,24 @@ dependencies = [ "password-hash", ] +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + [[package]] name = "async-trait" version = "0.1.68" @@ -415,6 +433,12 @@ version = "3.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +[[package]] +name = "bytemuck" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" + [[package]] name = "byteorder" version = "1.4.3" @@ -693,6 +717,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "data-url" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a30bfce702bcfa94e906ef82421f2c0e61c076ad76030c16ee5d2e9a32fe193" +dependencies = [ + "matches", +] + [[package]] name = "der" version = "0.5.1" @@ -704,6 +737,17 @@ dependencies = [ "pem-rfc7468", ] +[[package]] +name = "derive-new" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -807,6 +851,15 @@ dependencies = [ "instant", ] +[[package]] +name = "fdeflate" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d329bdeac514ee06249dabc27877490f17f5d371ec693360768b838e19f3ae10" +dependencies = [ + "simd-adler32", +] + [[package]] name = "fern" version = "0.6.2" @@ -835,9 +888,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.6.2", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" + [[package]] name = "flume" version = "0.10.14" @@ -856,6 +915,17 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fontdb" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b07f5c05414a0d8caba4c17eef8dc8b5c8955fc7c68d324191c7a56d3f3449" +dependencies = [ + "log", + "memmap2", + "ttf-parser", +] + [[package]] name = "foreign-types" version = "0.3.2" @@ -1283,6 +1353,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" + [[package]] name = "js-sys" version = "0.3.61" @@ -1317,6 +1393,15 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "kurbo" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a53776d271cfb873b17c618af0298445c88afc52837f3e948fa3fafd131f449" +dependencies = [ + "arrayvec 0.7.2", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -1449,12 +1534,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matches" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" + [[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "mime" version = "0.3.17" @@ -1486,6 +1586,16 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", + "simd-adler32", +] + [[package]] name = "mio" version = "0.8.6" @@ -1812,6 +1922,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "pico-args" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468" + [[package]] name = "pin-project" version = "1.0.12" @@ -1872,6 +1988,19 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "png" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaeebc51f9e7d2c150d3f3bfeb667f2aa985db5ef1e3d212847bdedb488beeaa" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.7.1", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1932,6 +2061,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rctree" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae028b272a6e99d9f8260ceefa3caa09300a8d6c8d2b2001316474bc52122e9" + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2005,6 +2140,30 @@ dependencies = [ "winreg", ] +[[package]] +name = "resvg" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "256cc9203115db152290219f35f3362e729301b59e2a391fb2721fe3fa155352" +dependencies = [ + "jpeg-decoder", + "log", + "pico-args", + "png", + "rgb", + "tiny-skia", + "usvg", +] + +[[package]] +name = "rgb" +version = "0.8.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.16.20" @@ -2031,6 +2190,15 @@ dependencies = [ "serde", ] +[[package]] +name = "roxmltree" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" +dependencies = [ + "xmlparser", +] + [[package]] name = "rsa" version = "0.6.1" @@ -2115,12 +2283,37 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustybuzz" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44561062e583c4873162861261f16fd1d85fe927c4904d71329a4fe43dc355ef" +dependencies = [ + "bitflags", + "bytemuck", + "smallvec", + "ttf-parser", + "unicode-bidi-mirroring", + "unicode-ccc", + "unicode-general-category", + "unicode-script", +] + [[package]] name = "ryu" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "safe_arch" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ff3d6d9696af502cc3110dacce942840fb06ff4514cad92236ecc455f2ce05" +dependencies = [ + "bytemuck", +] + [[package]] name = "sailfish" version = "0.6.1" @@ -2341,6 +2534,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238abfbb77c1915110ad968465608b68e869e0772622c9656714e73e5a1a522f" + [[package]] name = "simple_asn1" version = "0.6.2" @@ -2353,6 +2552,21 @@ dependencies = [ "time", ] +[[package]] +name = "simplecss" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a11be7c62927d9427e9f40f3444d5499d868648e2edbc4e2116de69e7ec0e89d" +dependencies = [ + "log", +] + +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "slab" version = "0.4.8" @@ -2527,6 +2741,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "svgtypes" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22975e8a2bac6a76bb54f898a6b18764633b00e780330f0b689f65afb3975564" +dependencies = [ + "siphasher", +] + [[package]] name = "syn" version = "1.0.109" @@ -2580,6 +2803,24 @@ dependencies = [ "colored", ] +[[package]] +name = "text-to-png" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b77c9daf0c55b10ef445266dbf0d58705c80496526de2c00643459958d956663" +dependencies = [ + "derive-new", + "fontdb", + "lazy_static", + "png", + "resvg", + "siphasher", + "thiserror", + "tiny-skia", + "usvg", + "xml-rs", +] + [[package]] name = "thiserror" version = "1.0.40" @@ -2627,6 +2868,20 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-skia" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d049bfef0eaa2521e75d9ffb5ce86ad54480932ae19b85f78bec6f52c4d30d78" +dependencies = [ + "arrayref", + "arrayvec 0.5.2", + "bytemuck", + "cfg-if", + "png", + "safe_arch", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2769,11 +3024,13 @@ dependencies = [ "actix-web", "argon2", "async-trait", + "bytes", "chrono", "config", "derive_more", "fern", "futures", + "indexmap", "jsonwebtoken", "lettre", "log", @@ -2792,6 +3049,7 @@ dependencies = [ "sqlx", "tempfile", "text-colorizer", + "text-to-png", "tokio", "toml 0.7.3", "urlencoding", @@ -2831,6 +3089,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "ttf-parser" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ae2f58a822f08abdaf668897e96a5656fe72f5a9ce66422423e8849384872e6" + [[package]] name = "typenum" version = "1.16.0" @@ -2858,6 +3122,24 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +[[package]] +name = "unicode-bidi-mirroring" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d12260fb92d52f9008be7e4bca09f584780eb2266dc8fecc6a192bec561694" + +[[package]] +name = "unicode-ccc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2520efa644f8268dce4dcd3050eaa7fc044fca03961e9998ac7e2e92b77cf1" + +[[package]] +name = "unicode-general-category" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07547e3ee45e28326cc23faac56d44f58f16ab23e413db526debce3b0bfd2742" + [[package]] name = "unicode-ident" version = "1.0.8" @@ -2873,12 +3155,24 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-script" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d817255e1bed6dfd4ca47258685d14d2bdcfbc64fdc9e3819bd5848057b8ecc" + [[package]] name = "unicode-segmentation" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +[[package]] +name = "unicode-vo" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d386ff53b415b7fe27b50bb44679e2cc4660272694b7b6f3326d8480823a94" + [[package]] name = "unicode-width" version = "0.1.10" @@ -2914,6 +3208,33 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +[[package]] +name = "usvg" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f472f6f5d41d3eaef059bc893dcd2382eefcdda3e04ebe0b2860c56b538e491e" +dependencies = [ + "base64 0.13.1", + "data-url", + "flate2", + "float-cmp", + "fontdb", + "kurbo", + "log", + "pico-args", + "rctree", + "roxmltree", + "rustybuzz", + "simplecss", + "siphasher", + "svgtypes", + "ttf-parser", + "unicode-bidi", + "unicode-script", + "unicode-vo", + "xmlwriter", +] + [[package]] name = "uuid" version = "1.3.1" @@ -3252,6 +3573,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "xml-rs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" + +[[package]] +name = "xmlparser" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" + +[[package]] +name = "xmlwriter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9" + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index 99486eba..276efc77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,9 @@ pbkdf2 = { version = "0.12", features = ["simple"] } text-colorizer = "1.0.0" log = "0.4" fern = "0.6" +bytes = "1.4.0" +text-to-png = "0.2.0" +indexmap = "1.9.3" [dev-dependencies] rand = "0.8" diff --git a/src/app.rs b/src/app.rs index c371936a..270d0589 100644 --- a/src/app.rs +++ b/src/app.rs @@ -8,6 +8,7 @@ use log::info; use crate::auth::AuthorizationService; use crate::bootstrap::logging; +use crate::cache::image::manager::ImageCacheService; use crate::common::AppData; use crate::config::Configuration; use crate::databases::database::connect_database; @@ -45,6 +46,7 @@ pub async fn run(configuration: Configuration) -> Running { let auth = Arc::new(AuthorizationService::new(cfg.clone(), database.clone())); let tracker_service = Arc::new(TrackerService::new(cfg.clone(), database.clone())); let mailer_service = Arc::new(MailerService::new(cfg.clone()).await); + let image_cache_service = Arc::new(ImageCacheService::new(cfg.clone()).await); // Build app container @@ -54,6 +56,7 @@ pub async fn run(configuration: Configuration) -> Running { auth.clone(), tracker_service.clone(), mailer_service, + image_cache_service )); // Start repeating task to import tracker torrent data and updating diff --git a/src/cache/cache.rs b/src/cache/cache.rs new file mode 100644 index 00000000..50f31a77 --- /dev/null +++ b/src/cache/cache.rs @@ -0,0 +1,197 @@ +use bytes::Bytes; +use indexmap::IndexMap; + +#[derive(Debug)] +pub enum Error { + EntrySizeLimitExceedsTotalCapacity, + BytesExceedEntrySizeLimit, + CacheCapacityIsTooSmall, +} + +#[derive(Debug, Clone)] +pub struct BytesCacheEntry { + pub bytes: Bytes, +} + +// Individual entry destined for the byte cache. +impl BytesCacheEntry { + pub fn new(bytes: Bytes) -> Self { + Self { bytes } + } +} + +pub struct BytesCache { + bytes_table: IndexMap, + total_capacity: usize, + entry_size_limit: usize, +} + +impl BytesCache { + pub fn new() -> Self { + Self { + bytes_table: IndexMap::new(), + total_capacity: 0, + entry_size_limit: 0, + } + } + + // With a total capacity in bytes. + pub fn with_capacity(capacity: usize) -> Self { + let mut new = Self::new(); + + new.total_capacity = capacity; + + new + } + + // With a limit for individual entry sizes. + pub fn with_entry_size_limit(entry_size_limit: usize) -> Self { + let mut new = Self::new(); + + new.entry_size_limit = entry_size_limit; + + new + } + + // With both a total capacity limit and an individual entry size limit. + pub fn with_capacity_and_entry_size_limit(capacity: usize, entry_size_limit: usize) -> Result { + if entry_size_limit > capacity { + return Err(Error::EntrySizeLimitExceedsTotalCapacity); + } + + let mut new = Self::new(); + + new.total_capacity = capacity; + new.entry_size_limit = entry_size_limit; + + Ok(new) + } + + pub async fn get(&self, key: &str) -> Option { + self.bytes_table.get(key).cloned() + } + + // Return the amount of entries in the map. + pub async fn len(&self) -> usize { + self.bytes_table.len() + } + + // Size of all the entry bytes combined. + pub fn total_size(&self) -> usize { + let mut size: usize = 0; + + for (_, entry) in self.bytes_table.iter() { + size += entry.bytes.len(); + } + + size + } + + // Insert bytes using key. + // TODO: Freed space might need to be reserved. Hold and pass write lock between functions? + // For TO DO above: semaphore: Arc, might be a solution. + pub async fn set(&mut self, key: String, bytes: Bytes) -> Result, Error> { + if bytes.len() > self.entry_size_limit { + return Err(Error::BytesExceedEntrySizeLimit); + } + + // Remove the old entry so that a new entry will be added as last in the queue. + let _ = self.bytes_table.shift_remove(&key); + + let bytes_cache_entry = BytesCacheEntry::new(bytes); + + self.free_size(bytes_cache_entry.bytes.len())?; + + Ok(self.bytes_table.insert(key, bytes_cache_entry)) + } + + // Free space. Size amount in bytes. + fn free_size(&mut self, size: usize) -> Result<(), Error> { + // Size may not exceed the total capacity of the bytes cache. + if size > self.total_capacity { + return Err(Error::CacheCapacityIsTooSmall); + } + + let cache_size = self.total_size(); + let size_to_be_freed = size.saturating_sub(self.total_capacity - cache_size); + let mut size_freed: usize = 0; + + while size_freed < size_to_be_freed { + let oldest_entry = self + .pop() + .expect("bytes cache has no more entries, yet there isn't enough space."); + + size_freed += oldest_entry.bytes.len(); + } + + Ok(()) + } + + // Remove and return the oldest entry. + pub fn pop(&mut self) -> Option { + self.bytes_table.shift_remove_index(0).map(|(_, entry)| entry) + } +} + +impl Default for BytesCache { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + + use crate::cache::cache::BytesCache; + + #[tokio::test] + async fn set_bytes_cache_with_capacity_and_entry_size_limit_should_succeed() { + let mut bytes_cache = BytesCache::with_capacity_and_entry_size_limit(6, 6).unwrap(); + let bytes: Bytes = Bytes::from("abcdef"); + + assert!(bytes_cache.set("1".to_string(), bytes).await.is_ok()) + } + + #[tokio::test] + async fn given_a_bytes_cache_with_a_capacity_and_entry_size_limit_it_should_allow_adding_new_entries_if_the_limit_is_not_exceeded() { + + let bytes: Bytes = Bytes::from("abcdef"); + + let mut bytes_cache = BytesCache::with_capacity_and_entry_size_limit(bytes.len() * 2, bytes.len()).unwrap(); + + // Add first entry (6 bytes) + assert!(bytes_cache.set("key1".to_string(), bytes.clone()).await.is_ok()); + + // Add second entry (6 bytes) + assert!(bytes_cache.set("key2".to_string(), bytes).await.is_ok()); + + // Both entries were added because we did not reach the limit + assert_eq!(bytes_cache.len().await, 2) + } + + #[tokio::test] + async fn given_a_bytes_cache_with_a_capacity_and_entry_size_limit_it_should_not_allow_adding_new_entries_if_the_capacity_is_exceeded() { + + let bytes: Bytes = Bytes::from("abcdef"); + + let mut bytes_cache = BytesCache::with_capacity_and_entry_size_limit(bytes.len() * 2 - 1, bytes.len()).unwrap(); + + // Add first entry (6 bytes) + assert!(bytes_cache.set("key1".to_string(), bytes.clone()).await.is_ok()); + + // Add second entry (6 bytes) + assert!(bytes_cache.set("key2".to_string(), bytes).await.is_ok()); + + // Only one entry is in the cache, because otherwise the total capacity would have been exceeded + assert_eq!(bytes_cache.len().await, 1) + } + + #[tokio::test] + async fn set_bytes_cache_with_capacity_and_entry_size_limit_should_fail() { + let mut bytes_cache = BytesCache::with_capacity_and_entry_size_limit(6, 5).unwrap(); + let bytes: Bytes = Bytes::from("abcdef"); + + assert!(bytes_cache.set("1".to_string(), bytes).await.is_err()) + } +} diff --git a/src/cache/image/manager.rs b/src/cache/image/manager.rs new file mode 100644 index 00000000..8cc96cec --- /dev/null +++ b/src/cache/image/manager.rs @@ -0,0 +1,200 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use bytes::Bytes; +use tokio::sync::RwLock; + +use crate::cache::cache::BytesCache; +use crate::config::Configuration; +use crate::models::user::UserCompact; + +pub enum Error { + UrlIsUnreachable, + UrlIsNotAnImage, + ImageTooBig, + UserQuotaMet, + Unauthenticated, +} + +type UserQuotas = HashMap; + +pub fn now_in_secs() -> u64 { + match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) { + Ok(n) => n.as_secs(), + Err(_) => panic!("SystemTime before UNIX EPOCH!"), + } +} + +#[derive(Clone)] +pub struct ImageCacheQuota { + pub user_id: i64, + pub usage: usize, + pub max_usage: usize, + pub date_start_secs: u64, + pub period_secs: u64, +} + +impl ImageCacheQuota { + pub fn new(user_id: i64, max_usage: usize, period_secs: u64) -> Self { + Self { + user_id, + usage: 0, + max_usage, + date_start_secs: now_in_secs(), + period_secs, + } + } + + pub fn add_usage(&mut self, amount: usize) -> Result<(), ()> { + // Check if quota needs to be reset. + if now_in_secs() - self.date_start_secs > self.period_secs { + self.reset(); + } + + if self.is_reached() { + return Err(()); + } + + self.usage = self.usage.saturating_add(amount); + + Ok(()) + } + + pub fn reset(&mut self) { + self.usage = 0; + self.date_start_secs = now_in_secs(); + } + + pub fn is_reached(&self) -> bool { + self.usage >= self.max_usage + } +} + +pub struct ImageCacheService { + image_cache: RwLock, + user_quotas: RwLock, + reqwest_client: reqwest::Client, + cfg: Arc, +} + +impl ImageCacheService { + pub async fn new(cfg: Arc) -> Self { + let settings = cfg.settings.read().await; + + let image_cache = + BytesCache::with_capacity_and_entry_size_limit(settings.image_cache.capacity, settings.image_cache.entry_size_limit) + .expect("Could not create image cache."); + + let reqwest_client = reqwest::Client::builder() + .timeout(Duration::from_millis(settings.image_cache.max_request_timeout_ms)) + .build() + .unwrap(); + + drop(settings); + + Self { + image_cache: RwLock::new(image_cache), + user_quotas: RwLock::new(HashMap::new()), + reqwest_client, + cfg, + } + } + + /// Get an image from the url and insert it into the cache if it isn't cached already. + /// Unauthenticated users can only get already cached images. + pub async fn get_image_by_url(&self, url: &str, opt_user: Option) -> Result { + if let Some(entry) = self.image_cache.read().await.get(url).await { + return Ok(entry.bytes); + } + + if opt_user.is_none() { + return Err(Error::Unauthenticated); + } + + let user = opt_user.unwrap(); + + self.check_user_quota(&user).await?; + + let image_bytes = self.get_image_from_url_as_bytes(url).await?; + + self.check_image_size(&image_bytes).await?; + + // These two functions could be executed after returning the image to the client, + // but than we would need a dedicated task or thread that executes these functions. + // This can be problematic if a task is spawned after every user request. + // Since these functions execute very fast, I don't see a reason to further optimize this. + // For now. + self.update_image_cache(url, &image_bytes).await?; + + self.update_user_quota(&user, image_bytes.len()).await?; + + Ok(image_bytes) + } + + async fn get_image_from_url_as_bytes(&self, url: &str) -> Result { + let res = self.reqwest_client.clone() + .get(url) + .send() + .await + .map_err(|_| Error::UrlIsUnreachable)?; + + if let Some(content_type) = res.headers().get("Content-Type") { + if content_type != "image/jpeg" && content_type != "image/png" { + return Err(Error::UrlIsNotAnImage); + } + } else { + return Err(Error::UrlIsNotAnImage); + } + + res.bytes().await.map_err(|_| Error::UrlIsNotAnImage) + } + + async fn check_user_quota(&self, user: &UserCompact) -> Result<(), Error> { + if let Some(quota) = self.user_quotas.read().await.get(&user.user_id) { + if quota.is_reached() { + return Err(Error::UserQuotaMet); + } + } + + Ok(()) + } + + async fn check_image_size(&self, image_bytes: &Bytes) -> Result<(), Error> { + let settings = self.cfg.settings.read().await; + + if image_bytes.len() > settings.image_cache.entry_size_limit { + return Err(Error::ImageTooBig); + } + + Ok(()) + } + + async fn update_image_cache(&self, url: &str, image_bytes: &Bytes) -> Result<(), Error> { + if self.image_cache.write().await.set(url.to_string(), image_bytes.clone()).await.is_err() { + return Err(Error::ImageTooBig); + } + + Ok(()) + } + + async fn update_user_quota(&self, user: &UserCompact, amount: usize) -> Result<(), Error> { + let settings = self.cfg.settings.read().await; + + let mut quota = self.user_quotas.read().await + .get(&user.user_id) + .cloned() + .unwrap_or(ImageCacheQuota::new( + user.user_id, + settings.image_cache.user_quota_bytes, + settings.image_cache.user_quota_period_seconds, + )); + + let _ = quota.add_usage(amount); + + let _ = self.user_quotas.write().await.insert(user.user_id, quota); + + Ok(()) + } + +} diff --git a/src/cache/image/mod.rs b/src/cache/image/mod.rs new file mode 100644 index 00000000..ff8de9eb --- /dev/null +++ b/src/cache/image/mod.rs @@ -0,0 +1 @@ +pub mod manager; diff --git a/src/cache/mod.rs b/src/cache/mod.rs new file mode 100644 index 00000000..3afdefbc --- /dev/null +++ b/src/cache/mod.rs @@ -0,0 +1,2 @@ +pub mod cache; +pub mod image; diff --git a/src/common.rs b/src/common.rs index 9bd43dd9..3432383b 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use crate::auth::AuthorizationService; +use crate::cache::image::manager::ImageCacheService; use crate::config::Configuration; use crate::databases::database::Database; use crate::mailer::MailerService; @@ -16,6 +17,7 @@ pub struct AppData { pub auth: Arc, pub tracker: Arc, pub mailer: Arc, + pub image_cache_manager: Arc, } impl AppData { @@ -25,6 +27,7 @@ impl AppData { auth: Arc, tracker: Arc, mailer: Arc, + image_cache_manager: Arc, ) -> AppData { AppData { cfg, @@ -32,6 +35,7 @@ impl AppData { auth, tracker, mailer, + image_cache_manager, } } } diff --git a/src/config.rs b/src/config.rs index ea11f554..ddccea13 100644 --- a/src/config.rs +++ b/src/config.rs @@ -70,6 +70,15 @@ pub struct Mail { pub port: u16, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ImageCache { + pub max_request_timeout_ms: u64, + pub capacity: usize, + pub entry_size_limit: usize, + pub user_quota_period_seconds: u64, + pub user_quota_bytes: usize +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TorrustConfig { pub website: Website, @@ -78,6 +87,7 @@ pub struct TorrustConfig { pub auth: Auth, pub database: Database, pub mail: Mail, + pub image_cache: ImageCache } impl TorrustConfig { @@ -116,10 +126,18 @@ impl TorrustConfig { server: "".to_string(), port: 25, }, + image_cache: ImageCache { + max_request_timeout_ms: 1000, + capacity: 128_000_000, + entry_size_limit: 4_000_000, + user_quota_period_seconds: 3600, + user_quota_bytes: 64_000_000 + } } } } + #[derive(Debug)] pub struct Configuration { pub settings: RwLock, diff --git a/src/errors.rs b/src/errors.rs index 24a413e6..571fd9fe 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -114,6 +114,9 @@ pub enum ServiceError { #[display(fmt = "Sorry, we have an error with our tracker connection.")] TrackerOffline, + #[display(fmt = "Could not whitelist torrent.")] + WhitelistingError, + #[display(fmt = "Failed to send verification email.")] FailedToSendVerificationEmail, @@ -172,6 +175,8 @@ impl ResponseError for ServiceError { ServiceError::TrackerOffline => StatusCode::INTERNAL_SERVER_ERROR, + ServiceError::WhitelistingError => StatusCode::INTERNAL_SERVER_ERROR, + ServiceError::CategoryExists => StatusCode::BAD_REQUEST, _ => StatusCode::INTERNAL_SERVER_ERROR, diff --git a/src/lib.rs b/src/lib.rs index 89776482..6db3f410 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ pub mod app; pub mod auth; pub mod bootstrap; +pub mod cache; pub mod common; pub mod config; pub mod console; diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 5761390a..946e776f 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -2,6 +2,7 @@ use actix_web::web; pub mod about; pub mod category; +pub mod proxy; pub mod root; pub mod settings; pub mod torrent; @@ -13,5 +14,6 @@ pub fn init_routes(cfg: &mut web::ServiceConfig) { category::init_routes(cfg); settings::init_routes(cfg); about::init_routes(cfg); + proxy::init_routes(cfg); root::init_routes(cfg); } diff --git a/src/routes/proxy.rs b/src/routes/proxy.rs new file mode 100644 index 00000000..443900df --- /dev/null +++ b/src/routes/proxy.rs @@ -0,0 +1,85 @@ +use std::sync::Once; + +use actix_web::http::StatusCode; +use actix_web::{web, HttpRequest, HttpResponse, Responder}; +use bytes::Bytes; +use text_to_png::TextRenderer; + +use crate::cache::image::manager::Error; +use crate::common::WebAppData; +use crate::errors::ServiceResult; + +static ERROR_IMAGE_LOADER: Once = Once::new(); + +static mut ERROR_IMAGE_URL_IS_UNREACHABLE: Bytes = Bytes::new(); +static mut ERROR_IMAGE_URL_IS_NOT_AN_IMAGE: Bytes = Bytes::new(); +static mut ERROR_IMAGE_TOO_BIG: Bytes = Bytes::new(); +static mut ERROR_IMAGE_USER_QUOTA_MET: Bytes = Bytes::new(); +static mut ERROR_IMAGE_UNAUTHENTICATED: Bytes = Bytes::new(); + +const ERROR_IMG_FONT_SIZE: u8 = 16; +const ERROR_IMG_COLOR: &str = "Red"; + +const ERROR_IMAGE_URL_IS_UNREACHABLE_TEXT: &str = "Could not find image."; +const ERROR_IMAGE_URL_IS_NOT_AN_IMAGE_TEXT: &str = "Invalid image."; +const ERROR_IMAGE_TOO_BIG_TEXT: &str = "Image is too big."; +const ERROR_IMAGE_USER_QUOTA_MET_TEXT: &str = "Image proxy quota met."; +const ERROR_IMAGE_UNAUTHENTICATED_TEXT: &str = "Sign in to see image."; + +pub fn init_routes(cfg: &mut web::ServiceConfig) { + cfg.service(web::scope("/proxy").service(web::resource("/image/{url}").route(web::get().to(get_proxy_image)))); + + load_error_images(); +} + +fn generate_img_from_text(text: &str) -> Bytes { + let renderer = TextRenderer::default(); + + Bytes::from( + renderer + .render_text_to_png_data(text, ERROR_IMG_FONT_SIZE, ERROR_IMG_COLOR) + .unwrap() + .data, + ) +} + +fn load_error_images() { + ERROR_IMAGE_LOADER.call_once(|| unsafe { + ERROR_IMAGE_URL_IS_UNREACHABLE = generate_img_from_text(ERROR_IMAGE_URL_IS_UNREACHABLE_TEXT); + ERROR_IMAGE_URL_IS_NOT_AN_IMAGE = generate_img_from_text(ERROR_IMAGE_URL_IS_NOT_AN_IMAGE_TEXT); + ERROR_IMAGE_TOO_BIG = generate_img_from_text(ERROR_IMAGE_TOO_BIG_TEXT); + ERROR_IMAGE_USER_QUOTA_MET = generate_img_from_text(ERROR_IMAGE_USER_QUOTA_MET_TEXT); + ERROR_IMAGE_UNAUTHENTICATED = generate_img_from_text(ERROR_IMAGE_UNAUTHENTICATED_TEXT); + }); +} + +pub async fn get_proxy_image(req: HttpRequest, app_data: WebAppData, path: web::Path) -> ServiceResult { + // Check for optional user. + let opt_user = app_data.auth.get_user_compact_from_request(&req).await.ok(); + + let encoded_url = path.into_inner(); + let url = urlencoding::decode(&encoded_url).unwrap_or_default(); + + match app_data.image_cache_manager.get_image_by_url(&url, opt_user).await { + Ok(image_bytes) => Ok(HttpResponse::build(StatusCode::OK) + .content_type("image/png") + .append_header(("Cache-Control", "max-age=15552000")) + .body(image_bytes)), + Err(e) => unsafe { + // Handling status codes in the frontend other tan OK is quite a pain. + // Return OK for now. + let (_status_code, error_image_bytes): (StatusCode, Bytes) = match e { + Error::UrlIsUnreachable => (StatusCode::GATEWAY_TIMEOUT, ERROR_IMAGE_URL_IS_UNREACHABLE.clone()), + Error::UrlIsNotAnImage => (StatusCode::BAD_REQUEST, ERROR_IMAGE_URL_IS_NOT_AN_IMAGE.clone()), + Error::ImageTooBig => (StatusCode::BAD_REQUEST, ERROR_IMAGE_TOO_BIG.clone()), + Error::UserQuotaMet => (StatusCode::TOO_MANY_REQUESTS, ERROR_IMAGE_USER_QUOTA_MET.clone()), + Error::Unauthenticated => (StatusCode::UNAUTHORIZED, ERROR_IMAGE_UNAUTHENTICATED.clone()), + }; + + Ok(HttpResponse::build(StatusCode::OK) + .content_type("image/png") + .append_header(("Cache-Control", "no-cache")) + .body(error_image_bytes)) + }, + } +} diff --git a/src/tracker.rs b/src/tracker.rs index cb69bab7..4e547f75 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -65,7 +65,7 @@ impl TrackerService { if response.status().is_success() { Ok(()) } else { - Err(ServiceError::InternalServerError) + Err(ServiceError::WhitelistingError) } }