Fix ByteString annotation detection when ByteArray is nullable (#1139)

Fixes bug where @ByteString annotation is not detected when ByteArray property is marked nullable. (was an oversight in #898) Added tests to validate fix.
Kotlin · Oct 13, 2020 · 317ff19 · 317ff19
1 parent 3126d4e
commit 317ff19
Show file tree

Hide file tree

Showing 4 changed files with 109 additions and 2 deletions.
diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt
@@ -592,6 +592,10 @@ private fun Iterable<ByteArray>.flatten(): ByteArray {
 }
 
 @OptIn(ExperimentalSerializationApi::class)
-private fun SerialDescriptor.isByteString(index: Int): Boolean =
-    getElementDescriptor(index) == ByteArraySerializer().descriptor &&
+private fun SerialDescriptor.isByteString(index: Int): Boolean {
+    val elementDescriptor = getElementDescriptor(index)
+    val byteArrayDescriptor = ByteArraySerializer().descriptor
+
+    return (elementDescriptor == byteArrayDescriptor || elementDescriptor == byteArrayDescriptor.nullable) &&
         getElementAnnotations(index).find { it is ByteString } != null
+}
diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt
@@ -109,6 +109,36 @@ class CborReaderTest {
         }
     }
 
+    @Test
+    fun testReadByteStringWhenNullable() {
+        /* A1                         # map(1)
+         *    6A                      # text(10)
+         *       62797465537472696E67 # "byteString"
+         *    44                      # bytes(4)
+         *       01020304             # "\x01\x02\x03\x04"
+         */
+        assertEquals(
+            expected = NullableByteString(byteArrayOf(1, 2, 3, 4)),
+            actual = Cbor.decodeFromHexString(
+                deserializer = NullableByteString.serializer(),
+                hex = "a16a62797465537472696e674401020304"
+            )
+        )
+
+        /* A1                         # map(1)
+         *    6A                      # text(10)
+         *       62797465537472696E67 # "byteString"
+         *    F6                      # primitive(22)
+         */
+        assertEquals(
+            expected = NullableByteString(byteString = null),
+            actual = Cbor.decodeFromHexString(
+                deserializer = NullableByteString.serializer(),
+                hex = "a16a62797465537472696e67f6"
+            )
+        )
+    }
+
     /**
      * CBOR hex data represents serialized versions of [TypesUmbrella] (which does **not** have a root property 'a') so
      * decoding to [Simple] (which has the field 'a') is expected to fail.

diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt
@@ -48,4 +48,54 @@ class CbrWriterTest {
             Cbor.encodeToHexString(NumberTypesUmbrella.serializer(), test)
         )
     }
+
+    @Test
+    fun testWriteByteStringWhenNullable() {
+        /* BF                         # map(*)
+         *    6A                      # text(10)
+         *       62797465537472696E67 # "byteString"
+         *    44                      # bytes(4)
+         *       01020304             # "\x01\x02\x03\x04"
+         *    FF                      # primitive(*)
+         */
+        assertEquals(
+            expected = "bf6a62797465537472696e674401020304ff",
+            actual = Cbor.encodeToHexString(
+                serializer = NullableByteString.serializer(),
+                value = NullableByteString(byteString = byteArrayOf(1, 2, 3, 4))
+            )
+        )
+
+        /* BF                         # map(*)
+         *    6A                      # text(10)
+         *       62797465537472696E67 # "byteString"
+         *    40                      # bytes(0)
+         *                            # ""
+         *    FF                      # primitive(*)
+         */
+        assertEquals(
+            expected = "bf6a62797465537472696e6740ff",
+            actual = Cbor.encodeToHexString(
+                serializer = NullableByteString.serializer(),
+                value = NullableByteString(byteString = byteArrayOf())
+            )
+        )
+    }
+
+    @Test
+    fun testWriteNullForNullableByteString() {
+        /* BF                         # map(*)
+         *    6A                      # text(10)
+         *       62797465537472696E67 # "byteString"
+         *    F6                      # primitive(22)
+         *    FF                      # primitive(*)
+         */
+        assertEquals(
+            expected = "bf6a62797465537472696e67f6ff",
+            actual = Cbor.encodeToHexString(
+                serializer = NullableByteString.serializer(),
+                value = NullableByteString(byteString = null)
+            )
+        )
+    }
 }
diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt
@@ -63,3 +63,26 @@ data class NumberTypesUmbrella(
         val boolean: Boolean,
         val char: Char
 )
+
+@Serializable
+data class NullableByteString(
+    @ByteString val byteString: ByteArray?
+) {
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as NullableByteString
+
+        if (byteString != null) {
+            if (other.byteString == null) return false
+            if (!byteString.contentEquals(other.byteString)) return false
+        } else if (other.byteString != null) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        return byteString?.contentHashCode() ?: 0
+    }
+}