Merge pull request #2908 from nlohmann/issue2863

Fix binary subtypes

.github/workflows/windows.yml

@@ -43,7 +43,7 @@ jobs:
       run: cmake -S . -B build -G "Visual Studio 15 2017" -A ${{ matrix.architecture }} -DJSON_BuildTests=On -DCMAKE_CXX_FLAGS="/W4 /WX"
       if: matrix.build_type == 'Release' && matrix.architecture != 'x64'
     - name: cmake
-      run: cmake -S . -B build -G "Visual Studio 15 2017" -A ${{ matrix.architecture }} -DJSON_BuildTests=On -DJSON_FastTests=ON -DCMAKE_CXX_FLAGS="/W4 /WX"
+      run: cmake -S . -B build -G "Visual Studio 15 2017" -A ${{ matrix.architecture }} -DJSON_BuildTests=On -DJSON_FastTests=ON -DCMAKE_EXE_LINKER_FLAGS="/STACK:4000000" -DCMAKE_CXX_FLAGS="/W4 /WX"
       if: matrix.build_type == 'Debug'
     - name: build
       run: cmake --build build --config ${{ matrix.build_type }} --parallel 10
@@ -75,7 +75,7 @@ jobs:
       run: cmake -S . -B build -G "Visual Studio 16 2019" -A ${{ matrix.architecture }} -DJSON_BuildTests=On -DCMAKE_CXX_FLAGS="/W4 /WX"
       if: matrix.build_type == 'Release'
     - name: cmake
-      run: cmake -S . -B build -G "Visual Studio 16 2019" -A ${{ matrix.architecture }} -DJSON_BuildTests=On -DJSON_FastTests=ON -DCMAKE_CXX_FLAGS="/W4 /WX"
+      run: cmake -S . -B build -G "Visual Studio 16 2019" -A ${{ matrix.architecture }} -DJSON_BuildTests=On -DJSON_FastTests=ON -DCMAKE_EXE_LINKER_FLAGS="/STACK:4000000" -DCMAKE_CXX_FLAGS="/W4 /WX"
       if: matrix.build_type == 'Debug'
     - name: build
       run: cmake --build build --config ${{ matrix.build_type }} --parallel 10
@@ -88,7 +88,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: cmake
-      run: cmake -S . -B build -G "Visual Studio 16 2019" -DJSON_BuildTests=On -DCMAKE_CXX_FLAGS="/permissive- /std:c++latest /utf-8 /W4 /WX"
+      run: cmake -S . -B build -G "Visual Studio 16 2019" -DJSON_BuildTests=On -DCMAKE_CXX_FLAGS="/permissive- /std:c++latest /utf-8 /W4 /WX" -DCMAKE_EXE_LINKER_FLAGS="/STACK:4000000"
     - name: build
       run: cmake --build build --config Release --parallel 10
     - name: test

doc/mkdocs/docs/api/basic_json/binary_t.md

@@ -64,4 +64,4 @@ type `#!cpp binary_t*` must be dereferenced.
 
 ## Version history
 
-- Added in version 3.8.0.
+- Added in version 3.8.0. Changed type of subtype to `std::uint64_t` in version 3.9.2.

doc/mkdocs/docs/api/basic_json/cbor_tag_handler_t.md

@@ -4,7 +4,8 @@
 enum class cbor_tag_handler_t
 {
     error,
-    ignore
+    ignore,
+    store
 };
 ```
 
@@ -16,6 +17,9 @@ error
 ignore
 :   ignore tags
 
+store
+:   store tagged values as binary container with subtype (for bytes 0xd8..0xdb)
+
 ## Version history
 
-- Added in version 3.9.0.
+- Added in version 3.9.0. Added value `store` in 3.9.2.

doc/mkdocs/docs/features/binary_formats/cbor.md

@@ -55,6 +55,8 @@ binary          | *size*: 256..65535                         | byte string (2 by
 binary          | *size*: 65536..4294967295                  | byte string (4 bytes follow)       | 0x5A
 binary          | *size*: 4294967296..18446744073709551615   | byte string (8 bytes follow)       | 0x5B
 
+Binary values with subtype are mapped to tagged values (0xD8..0xDB) depending on the subtype, followed by a byte string,
+see "binary" cells in the table above.
 
 !!! success "Complete mapping"
 
@@ -162,7 +164,7 @@ Double-Precision Float | number_float    | 0xFB
 
 !!! warning "Tagged items"
 
-    Tagged items will throw a parse error by default. However, they can be ignored by passing `cbor_tag_handler_t::ignore` to function `from_cbor`.
+    Tagged items will throw a parse error by default. They can be ignored by passing `cbor_tag_handler_t::ignore` to function `from_cbor`. They can be stored by passing `cbor_tag_handler_t::store` to function `from_cbor`.
 
 ??? example
 

doc/mkdocs/docs/features/binary_formats/index.md

@@ -25,7 +25,7 @@ to efficiently encode JSON values to byte vectors and to decode such vectors.
 | Format      | Binary values | Binary subtypes |
 | ----------- | ------------- | --------------- |
 | BSON        | supported     | supported       |
-| CBOR        | supported     | not supported   |
+| CBOR        | supported     | supported       |
 | MessagePack | supported     | supported       |
 | UBJSON      | not supported | not supported   |
 

doc/mkdocs/docs/features/binary_values.md

@@ -9,10 +9,10 @@ JSON itself does not have a binary value. As such, binary values are an extensio
 ```plantuml
 class json::binary_t {
     -- setters --
-    +void set_subtype(std::uint8_t subtype)
+    +void set_subtype(std::uint64_t subtype)
     +void clear_subtype()
     -- getters --
-    +std::uint8_t subtype() const
+    +std::uint64_t subtype() const
     +bool has_subtype() const
 }
 
@@ -68,14 +68,15 @@ j.get_binary().has_subtype();  // returns true
 j.get_binary().size();         // returns 4
 ```
 
-For convencience, binary JSON values can be constructed via `json::binary`:
+For convenience, binary JSON values can be constructed via `json::binary`:
 
 ```cpp
 auto j2 = json::binary({0xCA, 0xFE, 0xBA, 0xBE}, 23);
 auto j3 = json::binary({0xCA, 0xFE, 0xBA, 0xBE});
 
 j2 == j;                        // returns true
 j3.get_binary().has_subtype();  // returns false
+j3.get_binary().subtype();      // returns std::uint64_t(-1) as j3 has no subtype
 ```
 
 
@@ -184,7 +185,7 @@ JSON does not have a binary type, and this library does not introduce a new type
             0xCA 0xFE 0xBA 0xBE            // content
     ```
 
-    Note that the subtype is serialized as tag. However, parsing tagged values yield a parse error unless `json::cbor_tag_handler_t::ignore` is passed to `json::from_cbor`.
+    Note that the subtype is serialized as tag. However, parsing tagged values yield a parse error unless `json::cbor_tag_handler_t::ignore` or `json::cbor_tag_handler_t::store` is passed to `json::from_cbor`.
 
     ```json
     {

include/nlohmann/byte_container_with_subtype.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <cstdint> // uint8_t
+#include <cstdint> // uint8_t, uint64_t
 #include <tuple> // tie
 #include <utility> // move
 
@@ -18,14 +18,16 @@ order to override the binary type.
 @tparam BinaryType container to store bytes (`std::vector<std::uint8_t>` by
                    default)
 
-@since version 3.8.0
+@since version 3.8.0; changed type of subtypes to std::uint64_t in 3.9.2.
 */
 template<typename BinaryType>
 class byte_container_with_subtype : public BinaryType
 {
   public:
     /// the type of the underlying container
     using container_type = BinaryType;
+    /// the type of the subtype
+    using subtype_type = std::uint64_t;
 
     byte_container_with_subtype() noexcept(noexcept(container_type()))
         : container_type()
@@ -39,13 +41,13 @@ class byte_container_with_subtype : public BinaryType
         : container_type(std::move(b))
     {}
 
-    byte_container_with_subtype(const container_type& b, std::uint8_t subtype_) noexcept(noexcept(container_type(b)))
+    byte_container_with_subtype(const container_type& b, subtype_type subtype_) noexcept(noexcept(container_type(b)))
         : container_type(b)
         , m_subtype(subtype_)
         , m_has_subtype(true)
     {}
 
-    byte_container_with_subtype(container_type&& b, std::uint8_t subtype_) noexcept(noexcept(container_type(std::move(b))))
+    byte_container_with_subtype(container_type&& b, subtype_type subtype_) noexcept(noexcept(container_type(std::move(b))))
         : container_type(std::move(b))
         , m_subtype(subtype_)
         , m_has_subtype(true)
@@ -80,7 +82,7 @@ class byte_container_with_subtype : public BinaryType
 
     @since version 3.8.0
     */
-    void set_subtype(std::uint8_t subtype_) noexcept
+    void set_subtype(subtype_type subtype_) noexcept
     {
         m_subtype = subtype_;
         m_has_subtype = true;
@@ -90,7 +92,7 @@ class byte_container_with_subtype : public BinaryType
     @brief return the binary subtype
 
     Returns the numerical subtype of the value if it has a subtype. If it does
-    not have a subtype, this function will return size_t(-1) as a sentinel
+    not have a subtype, this function will return subtype_type(-1) as a sentinel
     value.
 
     @return the numerical subtype of the binary value
@@ -105,11 +107,12 @@ class byte_container_with_subtype : public BinaryType
     @sa see @ref has_subtype() -- returns whether or not the binary value has a
     subtype
 
-    @since version 3.8.0
+    @since version 3.8.0; fixed return value to properly return
+           subtype_type(-1) as documented in version 3.9.2
     */
-    constexpr std::uint8_t subtype() const noexcept
+    constexpr subtype_type subtype() const noexcept
     {
-        return m_subtype;
+        return m_has_subtype ? m_subtype : subtype_type(-1);
     }
 
     /*!
@@ -159,7 +162,7 @@ class byte_container_with_subtype : public BinaryType
     }
 
   private:
-    std::uint8_t m_subtype = 0;
+    subtype_type m_subtype = 0;
     bool m_has_subtype = false;
 };
 

include/nlohmann/detail/hash.hpp

@@ -103,7 +103,7 @@ std::size_t hash(const BasicJsonType& j)
             auto seed = combine(type, j.get_binary().size());
             const auto h = std::hash<bool> {}(j.get_binary().has_subtype());
             seed = combine(seed, h);
-            seed = combine(seed, j.get_binary().subtype());
+            seed = combine(seed, static_cast<std::size_t>(j.get_binary().subtype()));
             for (const auto byte : j.get_binary())
             {
                 seed = combine(seed, std::hash<std::uint8_t> {}(byte));

include/nlohmann/detail/input/binary_reader.hpp

@@ -30,8 +30,9 @@ namespace detail
 /// how to treat CBOR tags
 enum class cbor_tag_handler_t
 {
-    error,  ///< throw a parse_error exception in case of a tag
-    ignore   ///< ignore tags
+    error,   ///< throw a parse_error exception in case of a tag
+    ignore,  ///< ignore tags
+    store    ///< store tags as binary type
 };
 
 /*!
@@ -723,30 +724,31 @@ class binary_reader
 
                     case cbor_tag_handler_t::ignore:
                     {
+                        // ignore binary subtype
                         switch (current)
                         {
                             case 0xD8:
                             {
-                                std::uint8_t len{};
-                                get_number(input_format_t::cbor, len);
+                                std::uint8_t subtype_to_ignore{};
+                                get_number(input_format_t::cbor, subtype_to_ignore);
                                 break;
                             }
                             case 0xD9:
                             {
-                                std::uint16_t len{};
-                                get_number(input_format_t::cbor, len);
+                                std::uint16_t subtype_to_ignore{};
+                                get_number(input_format_t::cbor, subtype_to_ignore);
                                 break;
                             }
                             case 0xDA:
                             {
-                                std::uint32_t len{};
-                                get_number(input_format_t::cbor, len);
+                                std::uint32_t subtype_to_ignore{};
+                                get_number(input_format_t::cbor, subtype_to_ignore);
                                 break;
                             }
                             case 0xDB:
                             {
-                                std::uint64_t len{};
-                                get_number(input_format_t::cbor, len);
+                                std::uint64_t subtype_to_ignore{};
+                                get_number(input_format_t::cbor, subtype_to_ignore);
                                 break;
                             }
                             default:
@@ -755,6 +757,47 @@ class binary_reader
                         return parse_cbor_internal(true, tag_handler);
                     }
 
+                    case cbor_tag_handler_t::store:
+                    {
+                        binary_t b;
+                        // use binary subtype and store in binary container
+                        switch (current)
+                        {
+                            case 0xD8:
+                            {
+                                std::uint8_t subtype{};
+                                get_number(input_format_t::cbor, subtype);
+                                b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));
+                                break;
+                            }
+                            case 0xD9:
+                            {
+                                std::uint16_t subtype{};
+                                get_number(input_format_t::cbor, subtype);
+                                b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));
+                                break;
+                            }
+                            case 0xDA:
+                            {
+                                std::uint32_t subtype{};
+                                get_number(input_format_t::cbor, subtype);
+                                b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));
+                                break;
+                            }
+                            case 0xDB:
+                            {
+                                std::uint64_t subtype{};
+                                get_number(input_format_t::cbor, subtype);
+                                b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));
+                                break;
+                            }
+                            default:
+                                return parse_cbor_internal(true, tag_handler);
+                        }
+                        get();
+                        return get_cbor_binary(b) && sax->binary(b);
+                    }
+
                     default:                 // LCOV_EXCL_LINE
                         JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
                         return false;        // LCOV_EXCL_LINE

include/nlohmann/detail/input/parser.hpp

@@ -23,7 +23,7 @@ namespace detail
 // parser //
 ////////////
 
-enum class parse_event_t : uint8_t
+enum class parse_event_t : std::uint8_t
 {
     /// the parser read `{` and started to process a JSON object
     object_start,

include/nlohmann/detail/output/binary_writer.hpp

@@ -291,8 +291,26 @@ class binary_writer
             {
                 if (j.m_value.binary->has_subtype())
                 {
-                    write_number(static_cast<std::uint8_t>(0xd8));
-                    write_number(j.m_value.binary->subtype());
+                    if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint8_t>::max)())
+                    {
+                        write_number(static_cast<std::uint8_t>(0xd8));
+                        write_number(static_cast<std::uint8_t>(j.m_value.binary->subtype()));
+                    }
+                    else if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint16_t>::max)())
+                    {
+                        write_number(static_cast<std::uint8_t>(0xd9));
+                        write_number(static_cast<std::uint16_t>(j.m_value.binary->subtype()));
+                    }
+                    else if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint32_t>::max)())
+                    {
+                        write_number(static_cast<std::uint8_t>(0xda));
+                        write_number(static_cast<std::uint32_t>(j.m_value.binary->subtype()));
+                    }
+                    else if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint64_t>::max)())
+                    {
+                        write_number(static_cast<std::uint8_t>(0xdb));
+                        write_number(static_cast<std::uint64_t>(j.m_value.binary->subtype()));
+                    }
                 }
 
                 // step 1: write control byte and the binary array size
@@ -1109,7 +1127,7 @@ class binary_writer
         write_bson_entry_header(name, 0x05);
 
         write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size()));
-        write_number(value.has_subtype() ? value.subtype() : std::uint8_t(0x00));
+        write_number(value.has_subtype() ? static_cast<std::uint8_t>(value.subtype()) : std::uint8_t(0x00));
 
         oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());
     }

include/nlohmann/detail/output/serializer.hpp

@@ -389,7 +389,7 @@ class serializer
 
         for (std::size_t i = 0; i < s.size(); ++i)
         {
-            const auto byte = static_cast<uint8_t>(s[i]);
+            const auto byte = static_cast<std::uint8_t>(s[i]);
 
             switch (decode(state, codepoint, byte))
             {
@@ -674,6 +674,7 @@ class serializer
     @tparam NumberType either @a number_integer_t or @a number_unsigned_t
     */
     template < typename NumberType, detail::enable_if_t <
+                   std::is_integral<NumberType>::value ||
                    std::is_same<NumberType, number_unsigned_t>::value ||
                    std::is_same<NumberType, number_integer_t>::value ||
                    std::is_same<NumberType, binary_char_t>::value,
@@ -706,7 +707,7 @@ class serializer
         // use a pointer to fill the buffer
         auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
 
-        const bool is_negative = std::is_same<NumberType, number_integer_t>::value && !(x >= 0); // see issue #755
+        const bool is_negative = std::is_signed<NumberType>::value && !(x >= 0); // see issue #755
         number_unsigned_t abs_value;
 
         unsigned int n_chars{};