Fixed the crc32c codec so it uses CRC32C rather than CRC32

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Fixed
+ - **Major bug** Fixed the `crc32c` codec so it uses `CRC32C` rather than `CRC32`
+   - All arrays written prior to this release that use the `crc32c` codec are not correct
  - Fixed the `crc32c` codec reserving more memory than necessary
 
 ## [0.11.4] - 2024-02-05

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zarrs"
-version = "0.11.4"
+version = "0.11.5"
 authors = ["Lachlan Deakin <[email protected]>"]
 edition = "2021"
 rust-version = "1.71"
@@ -17,7 +17,7 @@ default = ["transpose", "blosc", "gzip", "sharding", "crc32c", "zstd", "ndarray"
 bitround = [] # Enable the experimental bitround codec
 blosc = ["dep:blosc-sys"] # Enable the blosc codec
 bz2 = ["dep:bzip2"] # Enable the experimental bz2 codec
-crc32c = ["dep:crc32fast"] # Enable the crc32c checksum codec
+crc32c = ["dep:crc32c"] # Enable the crc32c checksum codec
 gzip = ["dep:flate2"] # Enable the gzip codec
 pcodec = ["dep:pco"] # Enable the experimental pcodec codec
 sharding = [] # Enable the sharding codec
@@ -46,7 +46,7 @@ blosc-sys = { version = "0.3.0", package = "blosc-src", features = ["lz4", "zlib
 bytemuck = { version = "1.14.0", features = ["extern_crate_alloc"] }
 bytes = "1.5.0"
 bzip2 = { version = "0.4.4", optional = true, features = ["static"] }
-crc32fast = { version = "1.3", optional = true }
+crc32c = { version = "0.6.4", optional = true }
 derive_more = "0.99"
 dyn-clone = "1"
 flate2 = { version = "1", optional = true }

src/array/codec/bytes_to_bytes/crc32c.rs

@@ -88,8 +88,7 @@ mod tests {
             .try_into()
             .unwrap();
         println!("checksum {checksum:?}");
-        assert_eq!(checksum, &[74, 207, 235, 48]);
-        // println!("checksum {:?}", checksum);
+        assert_eq!(checksum, &[20, 133, 9, 65]);
     }
 
     #[test]

src/array/codec/bytes_to_bytes/crc32c/crc32c_codec.rs

@@ -54,7 +54,7 @@ impl BytesToBytesCodecTraits for Crc32cCodec {
         mut decoded_value: Vec<u8>,
         _parallel: bool,
     ) -> Result<Vec<u8>, CodecError> {
-        let checksum = crc32fast::hash(&decoded_value).to_le_bytes();
+        let checksum = crc32c::crc32c(&decoded_value).to_le_bytes();
         decoded_value.reserve_exact(checksum.len());
         decoded_value.extend(&checksum);
         Ok(decoded_value)
@@ -68,7 +68,7 @@ impl BytesToBytesCodecTraits for Crc32cCodec {
     ) -> Result<Vec<u8>, CodecError> {
         if encoded_value.len() >= CHECKSUM_SIZE {
             let decoded_value = &encoded_value[..encoded_value.len() - CHECKSUM_SIZE];
-            let checksum = crc32fast::hash(decoded_value).to_le_bytes();
+            let checksum = crc32c::crc32c(decoded_value).to_le_bytes();
             if checksum == encoded_value[encoded_value.len() - CHECKSUM_SIZE..] {
                 encoded_value.resize_with(encoded_value.len() - CHECKSUM_SIZE, Default::default);
                 Ok(encoded_value)

tests/data/sharded_array_write_read.zarr/group/array/zarr.json

@@ -50,15 +50,19 @@
             "configuration": {
               "endian": "little"
             }
-          },
-          {
-            "name": "crc32c",
-            "configuration": {}
           }
-        ]
+        ],
+        "index_location": "end"
       }
     }
   ],
+  "attributes": {
+    "_zarrs": {
+      "description": "This array was created with zarrs",
+      "repository": "https://github.com/LDeakin/zarrs",
+      "version": "0.11.4"
+    }
+  },
   "dimension_names": [
     "y",
     "x"