Skip to content

Commit

Permalink
feat: optimize calculate file by crc32 (#685)
Browse files Browse the repository at this point in the history
Signed-off-by: Gaius <[email protected]>
  • Loading branch information
gaius-qi authored Aug 22, 2024
1 parent b656988 commit adc2bd6
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 48 deletions.
80 changes: 48 additions & 32 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion dragonfly-client-util/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,6 @@ openssl.workspace = true
blake3.workspace = true
crc32fast.workspace = true
base16ct.workspace = true
tempfile = "3.9.0"

[dev-dependencies]
tempfile = "3.12.0"
30 changes: 15 additions & 15 deletions dragonfly-client-util/src/digest/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -134,16 +134,18 @@ pub fn calculate_file_hash(algorithm: Algorithm, path: &Path) -> ClientResult<Di
match algorithm {
Algorithm::Crc32 => {
let mut hasher = crc32fast::Hasher::new();
let mut buffer = [0; 1024]; // Set buffer size
let mut buffer = [0; 4096];
loop {
let count = reader.read(&mut buffer)?;
if count == 0 {
break;
}
hasher.update(&buffer[..count]);
}
let checksum = hasher.finalize();
Ok(Digest::new(algorithm, format!("{:x}", checksum)))
Ok(Digest::new(
algorithm,
base16ct::lower::encode_string(&hasher.finalize().to_be_bytes()),
))
}
Algorithm::Blake3 => {
let mut hasher = blake3::Hasher::new();
Expand Down Expand Up @@ -171,7 +173,6 @@ mod tests {
use super::*;
use std::fs::File;
use std::io::Write;
use tempfile::NamedTempFile;

#[test]
fn test_algorithm_display() {
Expand Down Expand Up @@ -199,30 +200,29 @@ mod tests {
#[test]
fn test_calculate_file_hash() {
let content = b"test content";
let temp_file = NamedTempFile::new().expect("Failed to create temp file");
let temp_file = tempfile::NamedTempFile::new().expect("failed to create temp file");
let path = temp_file.path();
let mut file = File::create(path).expect("Failed to create file");
file.write_all(content).expect("Failed to write to file");
let mut file = File::create(path).expect("failed to create file");
file.write_all(content).expect("failed to write to file");

let expected_blake3 = "ead3df8af4aece7792496936f83b6b6d191a7f256585ce6b6028db161278017e";
let expected_sha256 = "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72";
let expected_sha512 = "0cbf4caef38047bba9a24e621a961484e5d2a92176a859e7eb27df343dd34eb98d538a6c5f4da1ce302ec250b821cc001e46cc97a704988297185a4df7e99602";
let expected_crc32 = "57f4675d";

let digest =
calculate_file_hash(Algorithm::Blake3, path).expect("Failed to calculate Blake3 hash");
calculate_file_hash(Algorithm::Blake3, path).expect("failed to calculate Blake3 hash");
assert_eq!(digest.encoded(), expected_blake3);

let expected_sha256 = "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72";
let digest =
calculate_file_hash(Algorithm::Sha256, path).expect("Failed to calculate Sha256 hash");
calculate_file_hash(Algorithm::Sha256, path).expect("failed to calculate Sha256 hash");
assert_eq!(digest.encoded(), expected_sha256);

let expected_sha512 = "0cbf4caef38047bba9a24e621a961484e5d2a92176a859e7eb27df343dd34eb98d538a6c5f4da1ce302ec250b821cc001e46cc97a704988297185a4df7e99602";
let digest =
calculate_file_hash(Algorithm::Sha512, path).expect("Failed to calculate Sha512 hash");
calculate_file_hash(Algorithm::Sha512, path).expect("failed to calculate Sha512 hash");
assert_eq!(digest.encoded(), expected_sha512);

let expected_crc32 = "57f4675d";
let digest =
calculate_file_hash(Algorithm::Crc32, path).expect("Failed to calculate Sha512 hash");
calculate_file_hash(Algorithm::Crc32, path).expect("failed to calculate Sha512 hash");
assert_eq!(digest.encoded(), expected_crc32);
}
}

0 comments on commit adc2bd6

Please sign in to comment.