From 26ccd7428ede3aece4680f7d4c7a5c93e5303252 Mon Sep 17 00:00:00 2001 From: Gaius Date: Thu, 22 Aug 2024 21:40:57 +0800 Subject: [PATCH] feat: optimize calculate file by crc32 Signed-off-by: Gaius --- Cargo.lock | 80 +++++++++++++++---------- dragonfly-client-util/Cargo.toml | 4 +- dragonfly-client-util/src/digest/mod.rs | 30 +++++----- 3 files changed, 66 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66f49c73..abf9ae81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -757,7 +757,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.0", + "windows-targets 0.52.6", ] [[package]] @@ -2402,7 +2402,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.0", + "windows-targets 0.48.5", ] [[package]] @@ -4568,15 +4568,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.9.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand 2.0.1", - "redox_syscall", + "once_cell", "rustix 0.38.31", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5485,7 +5485,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core", - "windows-targets 0.52.0", + "windows-targets 0.52.6", ] [[package]] @@ -5494,7 +5494,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.6", ] [[package]] @@ -5512,7 +5512,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -5532,17 +5541,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -5553,9 +5563,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -5565,9 +5575,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -5577,9 +5587,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -5589,9 +5605,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -5601,9 +5617,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -5613,9 +5629,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -5625,9 +5641,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" diff --git a/dragonfly-client-util/Cargo.toml b/dragonfly-client-util/Cargo.toml index 59888beb..8db91baf 100644 --- a/dragonfly-client-util/Cargo.toml +++ b/dragonfly-client-util/Cargo.toml @@ -28,4 +28,6 @@ openssl.workspace = true blake3.workspace = true crc32fast.workspace = true base16ct.workspace = true -tempfile = "3.9.0" + +[dev-dependencies] +tempfile = "3.12.0" diff --git a/dragonfly-client-util/src/digest/mod.rs b/dragonfly-client-util/src/digest/mod.rs index 20c2d788..8c303fa6 100644 --- a/dragonfly-client-util/src/digest/mod.rs +++ b/dragonfly-client-util/src/digest/mod.rs @@ -134,7 +134,7 @@ pub fn calculate_file_hash(algorithm: Algorithm, path: &Path) -> ClientResult { let mut hasher = crc32fast::Hasher::new(); - let mut buffer = [0; 1024]; // Set buffer size + let mut buffer = [0; 4096]; loop { let count = reader.read(&mut buffer)?; if count == 0 { @@ -142,8 +142,10 @@ pub fn calculate_file_hash(algorithm: Algorithm, path: &Path) -> ClientResult { let mut hasher = blake3::Hasher::new(); @@ -171,7 +173,6 @@ mod tests { use super::*; use std::fs::File; use std::io::Write; - use tempfile::NamedTempFile; #[test] fn test_algorithm_display() { @@ -199,30 +200,29 @@ mod tests { #[test] fn test_calculate_file_hash() { let content = b"test content"; - let temp_file = NamedTempFile::new().expect("Failed to create temp file"); + let temp_file = tempfile::NamedTempFile::new().expect("failed to create temp file"); let path = temp_file.path(); - let mut file = File::create(path).expect("Failed to create file"); - file.write_all(content).expect("Failed to write to file"); + let mut file = File::create(path).expect("failed to create file"); + file.write_all(content).expect("failed to write to file"); let expected_blake3 = "ead3df8af4aece7792496936f83b6b6d191a7f256585ce6b6028db161278017e"; - let expected_sha256 = "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72"; - let expected_sha512 = "0cbf4caef38047bba9a24e621a961484e5d2a92176a859e7eb27df343dd34eb98d538a6c5f4da1ce302ec250b821cc001e46cc97a704988297185a4df7e99602"; - let expected_crc32 = "57f4675d"; - let digest = - calculate_file_hash(Algorithm::Blake3, path).expect("Failed to calculate Blake3 hash"); + calculate_file_hash(Algorithm::Blake3, path).expect("failed to calculate Blake3 hash"); assert_eq!(digest.encoded(), expected_blake3); + let expected_sha256 = "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72"; let digest = - calculate_file_hash(Algorithm::Sha256, path).expect("Failed to calculate Sha256 hash"); + calculate_file_hash(Algorithm::Sha256, path).expect("failed to calculate Sha256 hash"); assert_eq!(digest.encoded(), expected_sha256); + let expected_sha512 = "0cbf4caef38047bba9a24e621a961484e5d2a92176a859e7eb27df343dd34eb98d538a6c5f4da1ce302ec250b821cc001e46cc97a704988297185a4df7e99602"; let digest = - calculate_file_hash(Algorithm::Sha512, path).expect("Failed to calculate Sha512 hash"); + calculate_file_hash(Algorithm::Sha512, path).expect("failed to calculate Sha512 hash"); assert_eq!(digest.encoded(), expected_sha512); + let expected_crc32 = "57f4675d"; let digest = - calculate_file_hash(Algorithm::Crc32, path).expect("Failed to calculate Sha512 hash"); + calculate_file_hash(Algorithm::Crc32, path).expect("failed to calculate Sha512 hash"); assert_eq!(digest.encoded(), expected_crc32); } }