feat(metadata): add support for metadata with must_understand (ZEP0…

…009) (#148)

See ZEP0009 https://zarr.dev/zeps/draft/ZEP0009.html#extension-definitions

zarrs/src/array/codec/array_to_bytes/codec_chain.rs

@@ -232,7 +232,6 @@ impl CodecChain {
     }
 
     /// Get the array to bytes codec
-    #[allow(clippy::borrowed_box)]
     #[must_use]
     pub fn array_to_bytes_codec(&self) -> &NamedArrayToBytesCodec {
         &self.array_to_bytes

zarrs_metadata/CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- Add support for a `must_understand` field to `MetadataV3` (ZEP0009)
+  - Extensions can now be parsed in more than just the additional fields of array/group metadata (e.g. codecs)
 - Add `CodecMap` and `CodecName` for codec `nam` overriding and aliasing
 - Implement `From<T> for MetadataConfiguration` for all codec configuration enums
 - Implement `Copy` for `ZstdCompressionLevel`

zarrs_metadata/src/v3/metadata.rs

@@ -7,32 +7,47 @@ use thiserror::Error;
 
 /// Metadata with a name and optional configuration.
 ///
-/// Represents most fields in Zarr V3 array metadata (see [`ArrayMetadataV3`](crate::v3::ArrayMetadataV3)), which is structured as JSON with a name and optional configuration, or just a string representing the name.
+/// Represents most fields in Zarr V3 array metadata (see [`ArrayMetadataV3`](crate::v3::ArrayMetadataV3)) which is either:
+/// - a string name / identifier, or
+/// - a JSON object with a required `name` field and optional `configuration` and `must_understand` fields.
 ///
-/// Can be deserialised from a JSON string or name/configuration map.
-/// For example:
+/// `must_understand` is implicitly set to [`true`] if omitted.
+/// See [ZEP0009](https://zarr.dev/zeps/draft/ZEP0009.html) for more information on this field and Zarr V3 extensions.
+///
+/// ### Example Metadata
 /// ```json
 /// "bytes"
 /// ```
-/// or
+///
 /// ```json
 /// {
 ///     "name": "bytes",
 /// }
 /// ```
-/// or
+///
 /// ```json
 /// {
 ///     "name": "bytes",
 ///     "configuration": {
 ///       "endian": "little"
 ///     }
 /// }
+/// ```
 ///
+/// ```json
+/// {
+///     "name": "bytes",
+///     "configuration": {
+///       "endian": "little"
+///     },
+///     "must_understand": False
+/// }
+/// ```
 #[derive(Clone, Eq, PartialEq, Debug)]
 pub struct MetadataV3 {
     name: String,
     configuration: Option<MetadataConfiguration>,
+    must_understand: bool,
 }
 
 /// Configuration metadata.
@@ -86,9 +101,12 @@ impl serde::Serialize for MetadataV3 {
                 s.serialize_entry("name", &self.name)?;
                 s.end()
             } else {
-                let mut s = s.serialize_map(Some(2))?;
+                let mut s = s.serialize_map(Some(if self.must_understand { 2 } else { 3 }))?;
                 s.serialize_entry("name", &self.name)?;
                 s.serialize_entry("configuration", configuration)?;
+                if !self.must_understand {
+                    s.serialize_entry("must_understand", &false)?;
+                }
                 s.end()
             }
         } else {
@@ -97,6 +115,10 @@ impl serde::Serialize for MetadataV3 {
     }
 }
 
+fn default_must_understand() -> bool {
+    true
+}
+
 impl<'de> serde::Deserialize<'de> for MetadataV3 {
     fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
         #[derive(Deserialize)]
@@ -105,6 +127,8 @@ impl<'de> serde::Deserialize<'de> for MetadataV3 {
             name: String,
             #[serde(default)]
             configuration: Option<MetadataConfiguration>,
+            #[serde(default = "default_must_understand")]
+            must_understand: bool,
         }
 
         #[derive(Deserialize)]
@@ -121,10 +145,12 @@ impl<'de> serde::Deserialize<'de> for MetadataV3 {
             MetadataIntermediate::Name(name) => Ok(Self {
                 name,
                 configuration: None,
+                must_understand: true,
             }),
             MetadataIntermediate::NameConfiguration(metadata) => Ok(Self {
                 name: metadata.name,
                 configuration: metadata.configuration,
+                must_understand: metadata.must_understand,
             }),
         }
     }
@@ -137,6 +163,7 @@ impl MetadataV3 {
         Self {
             name: name.into(),
             configuration: None,
+            must_understand: true,
         }
     }
 
@@ -149,9 +176,17 @@ impl MetadataV3 {
         Self {
             name: name.into(),
             configuration: Some(configuration.into()),
+            must_understand: true,
         }
     }
 
+    /// Set the value of the `must_understand` field.
+    #[must_use]
+    pub fn with_must_understand(mut self, must_understand: bool) -> Self {
+        self.must_understand = must_understand;
+        self
+    }
+
     /// Convert a serializable configuration to [`MetadataV3`].
     ///
     /// # Errors
@@ -195,6 +230,14 @@ impl MetadataV3 {
         self.configuration.as_ref()
     }
 
+    /// Return whether the metadata must be understood as indicated by the `must_understand` field.
+    ///
+    /// The `must_understand` field is implicitly `true` if omitted.
+    #[must_use]
+    pub fn must_understand(&self) -> bool {
+        self.must_understand
+    }
+
     /// Returns true if the configuration is none or an empty map.
     #[must_use]
     pub fn configuration_is_none_or_empty(&self) -> bool {
@@ -368,3 +411,53 @@ where
 /// Additional fields in array or group metadata.
 // NOTE: It would be nice if this was just a serde_json::Map, but it only has implementations for `<String, Value>`.
 pub type AdditionalFields = std::collections::BTreeMap<String, AdditionalField>;
+
+#[cfg(test)]
+mod tests {
+    use super::MetadataV3;
+
+    #[test]
+    fn metadata_must_understand_implicit_string() {
+        let metadata = r#""test""#;
+        let metadata: MetadataV3 = serde_json::from_str(&metadata).unwrap();
+        assert!(metadata.name() == "test");
+        assert!(metadata.must_understand());
+    }
+
+    #[test]
+    fn metadata_must_understand_implicit() {
+        let metadata = r#"{
+    "name": "test"
+}"#;
+        let metadata: MetadataV3 = serde_json::from_str(&metadata).unwrap();
+        assert!(metadata.name() == "test");
+        assert!(metadata.must_understand());
+    }
+
+    #[test]
+    fn metadata_must_understand_true() {
+        let metadata = r#"{
+    "name": "test",
+    "must_understand": true
+}"#;
+        let metadata: MetadataV3 = serde_json::from_str(&metadata).unwrap();
+        assert!(metadata.name() == "test");
+        assert!(metadata.must_understand());
+    }
+
+    #[test]
+    fn metadata_must_understand_false() {
+        let metadata = r#"{
+    "name": "test",
+    "must_understand": false
+}"#;
+        let metadata: MetadataV3 = serde_json::from_str(&metadata).unwrap();
+        assert!(metadata.name() == "test");
+        assert!(!metadata.must_understand());
+        assert_ne!(metadata, MetadataV3::new("test"));
+        assert_eq!(
+            metadata,
+            MetadataV3::new("test").with_must_understand(false)
+        );
+    }
+}