Merge pull request #2158 from kozakusek/iox-2157-large-chunk-support

iox-#2157 Replace uint32_t with uint64_t for payload size

doc/design/chunk_header.md

@@ -43,26 +43,26 @@ Framing with terminology
 ```
 class ChunkHeader
 {
-    uint32_t chunkSize;
+    uint32_t userHeaderSize{0U};
     uint8_t chunkHeaderVersion;
     uint8_t reserved{0};
     uint16_t userHeaderId;
     popo::UniquePortId originId; // underlying type = uint64_t
     uint64_t sequenceNumber;
-    uint32_t userHeaderSize{0U};
+    uint64_t chunkSize;
     uint32_t userPayloadSize{0U};
     uint32_t userPayloadAlignment{1U};
     UserPayloadOffset_t userPayloadOffset; // alias to uint32_t
 };
 ```
 
-- **chunkSize** is the size of the whole chunk
+- **userHeaderSize** is the size of the chunk occupied by the user-header
 - **chunkHeaderVersion** is used to detect incompatibilities for record&replay functionality
 - **reserved** is currently not used and set to `0`
 - **userHeaderId** is currently not used and set to `NO_USER_HEADER`
 - **originId** is the unique identifier of the publisher the chunk was sent from
 - **sequenceNumber** is a serial number for the sent chunks
-- **userHeaderSize** is the size of the chunk occupied by the user-header
+- **chunkSize** is the size of the whole chunk
 - **userPayloadSize** is the size of the chunk occupied by the user-payload
 - **userPayloadAlignment** is the alignment of the chunk occupied by the user-payload
 - **userPayloadOffset** is the offset of the user-payload relative to the begin of the chunk

doc/website/release-notes/iceoryx-unreleased.md

@@ -1293,3 +1293,9 @@
     #include "iox/mpmc_lockfree_queue.hpp"
     iox::concurrent::MpmcLockFreeQueue q;
     ```
+
+59. Payload Size for Memory Chunks is now `uin64_t`.
+    Hence the `ChunkHeader` (iceoryx_posh/mepoo/chunk_header.hpp) layout changes
+    and `m_chunkHeaderVersion` is getting increased.
+    Moreover many functions' signatures are also affected by this change.
+

iceoryx_binding_c/include/iceoryx_binding_c/client.h

@@ -89,7 +89,7 @@ void iox_client_deinit(iox_client_t const self);
 ///       for a custom user-payload alignment please use 'iox_client_loan_aligned_request'
 ENUM iox_AllocationResult iox_client_loan_request(iox_client_t const self,
                                                   void** const payload,
-                                                  const uint32_t payloadSize);
+                                                  const uint64_t payloadSize);
 
 /// @brief allocates a request in the shared memory with a custom alignment for the user-payload
 /// @param[in] self handle of the client
@@ -100,7 +100,7 @@ ENUM iox_AllocationResult iox_client_loan_request(iox_client_t const self,
 ///         describes the error
 ENUM iox_AllocationResult iox_client_loan_aligned_request(iox_client_t const self,
                                                           void** const payload,
-                                                          const uint32_t payloadSize,
+                                                          const uint64_t payloadSize,
                                                           const uint32_t payloadAlignment);
 
 

iceoryx_binding_c/include/iceoryx_binding_c/publisher.h

@@ -87,7 +87,7 @@ void iox_pub_deinit(iox_pub_t const self);
 ///       for a custom user-payload alignment please use 'iox_pub_loan_aligned_chunk'
 ENUM iox_AllocationResult iox_pub_loan_chunk(iox_pub_t const self,
                                              void** const userPayload,
-                                             const uint32_t userPayloadSize);
+                                             const uint64_t userPayloadSize);
 
 /// @brief allocates a chunk in the shared memory with a custom alignment for the user-payload
 /// @param[in] self handle of the publisher
@@ -98,7 +98,7 @@ ENUM iox_AllocationResult iox_pub_loan_chunk(iox_pub_t const self,
 ///         describes the error
 ENUM iox_AllocationResult iox_pub_loan_aligned_chunk(iox_pub_t const self,
                                                      void** const userPayload,
-                                                     const uint32_t userPayloadSize,
+                                                     const uint64_t userPayloadSize,
                                                      const uint32_t userPayloadAlignment);
 
 /// @brief allocates a chunk in the shared memory with a section for the user-header and a custom alignment for the
@@ -113,7 +113,7 @@ ENUM iox_AllocationResult iox_pub_loan_aligned_chunk(iox_pub_t const self,
 ///         describes the error
 ENUM iox_AllocationResult iox_pub_loan_aligned_chunk_with_user_header(iox_pub_t const self,
                                                                       void** const userPayload,
-                                                                      const uint32_t userPayloadSize,
+                                                                      const uint64_t userPayloadSize,
                                                                       const uint32_t userPayloadAlignment,
                                                                       const uint32_t userHeaderSize,
                                                                       const uint32_t userHeaderAlignment);

iceoryx_binding_c/include/iceoryx_binding_c/server.h

@@ -103,7 +103,7 @@ void iox_server_release_request(iox_server_t const self, const void* const paylo
 ENUM iox_AllocationResult iox_server_loan_response(iox_server_t const self,
                                                    const void* const requestPayload,
                                                    void** const payload,
-                                                   const uint32_t payloadSize);
+                                                   const uint64_t payloadSize);
 
 /// @brief allocates a response in the shared memory
 /// @param[in] self handle of the server
@@ -116,7 +116,7 @@ ENUM iox_AllocationResult iox_server_loan_response(iox_server_t const self,
 ENUM iox_AllocationResult iox_server_loan_aligned_response(iox_server_t const self,
                                                            const void* const requestPayload,
                                                            void** const payload,
-                                                           const uint32_t payloadSize,
+                                                           const uint64_t payloadSize,
                                                            const uint32_t payloadAlignment);
 
 /// @brief sends a previously loaned response

iceoryx_binding_c/source/c_client.cpp

@@ -92,14 +92,14 @@ void iox_client_deinit(iox_client_t const self)
     delete self;
 }
 
-iox_AllocationResult iox_client_loan_request(iox_client_t const self, void** const payload, const uint32_t payloadSize)
+iox_AllocationResult iox_client_loan_request(iox_client_t const self, void** const payload, const uint64_t payloadSize)
 {
     return iox_client_loan_aligned_request(self, payload, payloadSize, IOX_C_CHUNK_DEFAULT_USER_PAYLOAD_ALIGNMENT);
 }
 
 iox_AllocationResult iox_client_loan_aligned_request(iox_client_t const self,
                                                      void** const payload,
-                                                     const uint32_t payloadSize,
+                                                     const uint64_t payloadSize,
                                                      const uint32_t payloadAlignment)
 {
     IOX_EXPECTS(self != nullptr);

iceoryx_binding_c/source/c_publisher.cpp

@@ -113,7 +113,7 @@ void iox_pub_deinit(iox_pub_t const self)
     delete self;
 }
 
-iox_AllocationResult iox_pub_loan_chunk(iox_pub_t const self, void** const userPayload, const uint32_t userPayloadSize)
+iox_AllocationResult iox_pub_loan_chunk(iox_pub_t const self, void** const userPayload, const uint64_t userPayloadSize)
 {
     return iox_pub_loan_aligned_chunk_with_user_header(self,
                                                        userPayload,
@@ -125,7 +125,7 @@ iox_AllocationResult iox_pub_loan_chunk(iox_pub_t const self, void** const userP
 
 iox_AllocationResult iox_pub_loan_aligned_chunk(iox_pub_t const self,
                                                 void** const userPayload,
-                                                const uint32_t userPayloadSize,
+                                                const uint64_t userPayloadSize,
                                                 const uint32_t userPayloadAlignment)
 {
     return iox_pub_loan_aligned_chunk_with_user_header(self,
@@ -138,7 +138,7 @@ iox_AllocationResult iox_pub_loan_aligned_chunk(iox_pub_t const self,
 
 iox_AllocationResult iox_pub_loan_aligned_chunk_with_user_header(iox_pub_t const self,
                                                                  void** const userPayload,
-                                                                 const uint32_t userPayloadSize,
+                                                                 const uint64_t userPayloadSize,
                                                                  const uint32_t userPayloadAlignment,
                                                                  const uint32_t userHeaderSize,
                                                                  const uint32_t userHeaderAlignment)

iceoryx_binding_c/source/c_server.cpp

@@ -115,7 +115,7 @@ void iox_server_release_request(iox_server_t const self, const void* const paylo
 iox_AllocationResult iox_server_loan_response(iox_server_t const self,
                                               const void* const requestPayload,
                                               void** const payload,
-                                              const uint32_t payloadSize)
+                                              const uint64_t payloadSize)
 {
     return iox_server_loan_aligned_response(
         self, requestPayload, payload, payloadSize, IOX_C_CHUNK_DEFAULT_USER_PAYLOAD_ALIGNMENT);
@@ -124,7 +124,7 @@ iox_AllocationResult iox_server_loan_response(iox_server_t const self,
 iox_AllocationResult iox_server_loan_aligned_response(iox_server_t const self,
                                                       const void* const requestPayload,
                                                       void** const payload,
-                                                      const uint32_t payloadSize,
+                                                      const uint64_t payloadSize,
                                                       const uint32_t payloadAlignment)
 {
     IOX_EXPECTS(self != nullptr);

iceoryx_binding_c/test/moduletests/test_chunk.cpp

@@ -60,7 +60,7 @@ class Chunk_test : public RouDi_GTest
 TEST_F(Chunk_test, GettingChunkHeaderFromNonConstUserPayloadWorks)
 {
     ::testing::Test::RecordProperty("TEST_ID", "a044b28d-ad7e-45ed-a0e2-e431ef1eacf0");
-    constexpr uint32_t USER_PAYLOAD_SIZE(42U);
+    constexpr uint64_t USER_PAYLOAD_SIZE(42U);
     void* userPayload{nullptr};
     ASSERT_EQ(iox_pub_loan_chunk(publisher, &userPayload, USER_PAYLOAD_SIZE), AllocationResult_SUCCESS);
 
@@ -76,7 +76,7 @@ TEST_F(Chunk_test, GettingChunkHeaderFromNonConstUserPayloadWorks)
 TEST_F(Chunk_test, GettingChunkHeaderFromConstUserPayloadWorks)
 {
     ::testing::Test::RecordProperty("TEST_ID", "9f7bb07a-f0dd-4b58-af84-5daec365d9e2");
-    constexpr uint32_t USER_PAYLOAD_SIZE(42U);
+    constexpr uint64_t USER_PAYLOAD_SIZE(42U);
     void* userPayload{nullptr};
     ASSERT_EQ(iox_pub_loan_chunk(publisher, &userPayload, USER_PAYLOAD_SIZE), AllocationResult_SUCCESS);
     const void* constUserPayload = userPayload;
@@ -93,7 +93,7 @@ TEST_F(Chunk_test, GettingChunkHeaderFromConstUserPayloadWorks)
 TEST_F(Chunk_test, UserPayloadChunkHeaderUserPayloadRoundtripWorksForNonConst)
 {
     ::testing::Test::RecordProperty("TEST_ID", "ea220aac-4d7d-41c2-92ea-7f929b824555");
-    constexpr uint32_t USER_PAYLOAD_SIZE(42U);
+    constexpr uint64_t USER_PAYLOAD_SIZE(42U);
     void* userPayload{nullptr};
     ASSERT_EQ(iox_pub_loan_chunk(publisher, &userPayload, USER_PAYLOAD_SIZE), AllocationResult_SUCCESS);
     const void* constUserPayload = userPayload;
@@ -107,7 +107,7 @@ TEST_F(Chunk_test, UserPayloadChunkHeaderUserPayloadRoundtripWorksForNonConst)
 TEST_F(Chunk_test, UserPayloadChunkHeaderUserPayloadRoundtripWorksForConst)
 {
     ::testing::Test::RecordProperty("TEST_ID", "e094616d-6d99-4b7f-a619-dd98ec7d1e44");
-    constexpr uint32_t USER_PAYLOAD_SIZE(42U);
+    constexpr uint64_t USER_PAYLOAD_SIZE(42U);
     void* userPayload{nullptr};
     ASSERT_EQ(iox_pub_loan_chunk(publisher, &userPayload, USER_PAYLOAD_SIZE), AllocationResult_SUCCESS);
 
@@ -120,7 +120,7 @@ TEST_F(Chunk_test, UserPayloadChunkHeaderUserPayloadRoundtripWorksForConst)
 TEST_F(Chunk_test, GettingUserHeaderFromNonConstChunkHeaderWorks)
 {
     ::testing::Test::RecordProperty("TEST_ID", "a0df7284-a377-4c6a-b22b-454d3f7c7b88");
-    constexpr uint32_t USER_PAYLOAD_SIZE(42U);
+    constexpr uint64_t USER_PAYLOAD_SIZE(42U);
     constexpr uint32_t USER_PAYLOAD_ALIGNMENT(64U);
     constexpr uint32_t USER_HEADER_SIZE = 16U;
     constexpr uint32_t USER_HEADER_ALIGNMENT = 8U;
@@ -146,7 +146,7 @@ TEST_F(Chunk_test, GettingUserHeaderFromNonConstChunkHeaderWorks)
 TEST_F(Chunk_test, GettingUserHeaderFromConstChunkHeaderWorks)
 {
     ::testing::Test::RecordProperty("TEST_ID", "77f4a193-7f44-43ce-8bd8-f9916b8d83dd");
-    constexpr uint32_t USER_PAYLOAD_SIZE(42U);
+    constexpr uint64_t USER_PAYLOAD_SIZE(42U);
     constexpr uint32_t USER_PAYLOAD_ALIGNMENT(64U);
     constexpr uint32_t USER_HEADER_SIZE = 16U;
     constexpr uint32_t USER_HEADER_ALIGNMENT = 8U;

iceoryx_binding_c/test/moduletests/test_listener.cpp

@@ -199,7 +199,7 @@ class iox_listener_test : public Test
     vector<iox_user_trigger_t, MAX_NUMBER_OF_EVENTS_PER_LISTENER + 1U> m_userTrigger;
 
     static constexpr uint32_t NUM_CHUNKS_IN_POOL = MAX_CHUNKS_HELD_PER_SUBSCRIBER_SIMULTANEOUSLY + 2U;
-    static constexpr uint32_t CHUNK_SIZE = 128U;
+    static constexpr uint64_t CHUNK_SIZE = 128U;
     static constexpr uint64_t MEMORY_SIZE = 1024U * 1024U * 100U;
     uint8_t m_memory[MEMORY_SIZE];
     BumpAllocator m_memoryAllocator{m_memory, MEMORY_SIZE};
@@ -511,7 +511,7 @@ TIMING_TEST_F(iox_listener_test, SubscriberCallbackIsCalledSampleIsReceived, Rep
                 Eq(iox_ListenerResult::ListenerResult_SUCCESS));
 
     Subscribe(m_subscriber[0U]);
-    constexpr uint32_t USER_PAYLOAD_SIZE{100U};
+    constexpr uint64_t USER_PAYLOAD_SIZE{100U};
 
     auto chunkSettingsResult = ChunkSettings::create(USER_PAYLOAD_SIZE, iox::CHUNK_DEFAULT_USER_PAYLOAD_ALIGNMENT);
     ASSERT_FALSE(chunkSettingsResult.has_error());
@@ -537,7 +537,7 @@ TIMING_TEST_F(iox_listener_test, SubscriberCallbackWithContextDataIsCalledSample
         Eq(iox_ListenerResult::ListenerResult_SUCCESS));
 
     Subscribe(m_subscriber[0U]);
-    constexpr uint32_t USER_PAYLOAD_SIZE{100U};
+    constexpr uint64_t USER_PAYLOAD_SIZE{100U};
 
     auto chunkSettingsResult = ChunkSettings::create(USER_PAYLOAD_SIZE, iox::CHUNK_DEFAULT_USER_PAYLOAD_ALIGNMENT);
     ASSERT_FALSE(chunkSettingsResult.has_error());

iceoryx_binding_c/test/moduletests/test_notification_info.cpp

@@ -84,7 +84,7 @@ class iox_notification_info_test : public Test
 
     iox::mepoo::SharedChunk getChunkFromMemoryManager()
     {
-        constexpr uint32_t USER_PAYLOAD_SIZE{100U};
+        constexpr uint64_t USER_PAYLOAD_SIZE{100U};
 
         auto chunkSettingsResult = ChunkSettings::create(USER_PAYLOAD_SIZE, iox::CHUNK_DEFAULT_USER_PAYLOAD_ALIGNMENT);
         IOX_ENSURES(chunkSettingsResult.has_value());
@@ -105,7 +105,7 @@ class iox_notification_info_test : public Test
     UserTrigger m_userTrigger;
 
     static constexpr uint32_t NUM_CHUNKS_IN_POOL = MAX_CHUNKS_HELD_PER_SUBSCRIBER_SIMULTANEOUSLY + 2;
-    static constexpr uint32_t CHUNK_SIZE = 128U;
+    static constexpr uint64_t CHUNK_SIZE = 128U;
     static constexpr size_t MEMORY_SIZE = 1024 * 1024 * 100;
     uint8_t m_memory[MEMORY_SIZE];
     BumpAllocator m_memoryAllocator{m_memory, MEMORY_SIZE};

iceoryx_binding_c/test/moduletests/test_publisher.cpp

@@ -98,7 +98,7 @@ class iox_pub_test : public Test
     static constexpr size_t MEMORY_SIZE = 1024 * 1024;
     uint8_t m_memory[MEMORY_SIZE];
     static constexpr uint32_t NUM_CHUNKS_IN_POOL = 20;
-    static constexpr uint32_t CHUNK_SIZE = 256;
+    static constexpr uint64_t CHUNK_SIZE = 256;
 
     using ChunkQueueData_t = popo::ChunkQueueData<DefaultChunkQueueConfig, popo::ThreadSafePolicy>;
     ChunkQueueData_t m_chunkQueueData{iox::popo::QueueFullPolicy::DISCARD_OLDEST_DATA,
@@ -284,7 +284,7 @@ TEST_F(iox_pub_test, allocate_chunkFailsWhenHoldingToManyChunksInParallel)
 TEST_F(iox_pub_test, allocate_chunkFailsWhenOutOfChunks)
 {
     ::testing::Test::RecordProperty("TEST_ID", "7563ed4c-a6d5-487d-9c6c-937d8e8c3d1d");
-    constexpr uint32_t USER_PAYLOAD_SIZE{100U};
+    constexpr uint64_t USER_PAYLOAD_SIZE{100U};
 
     auto chunkSettingsResult = ChunkSettings::create(USER_PAYLOAD_SIZE, iox::CHUNK_DEFAULT_USER_PAYLOAD_ALIGNMENT);
     ASSERT_FALSE(chunkSettingsResult.has_error());

iceoryx_binding_c/test/moduletests/test_server.cpp

@@ -66,7 +66,7 @@ class iox_server_test : public Test
         return &*sutPort;
     }
 
-    void receiveRequest(const int64_t requestValue = 0, const uint32_t chunkSize = sizeof(int64_t))
+    void receiveRequest(const int64_t requestValue = 0, const uint64_t chunkSize = sizeof(int64_t))
     {
         auto chunk = memoryManager.getChunk(*iox::mepoo::ChunkSettings::create(
             chunkSize, iox::CHUNK_DEFAULT_USER_PAYLOAD_ALIGNMENT, sizeof(RequestHeader)));

iceoryx_binding_c/test/moduletests/test_subscriber.cpp

@@ -89,7 +89,7 @@ class iox_sub_test : public Test
 
     iox::mepoo::SharedChunk getChunkFromMemoryManager()
     {
-        constexpr uint32_t USER_PAYLOAD_SIZE{100U};
+        constexpr uint64_t USER_PAYLOAD_SIZE{100U};
 
         auto chunkSettingsResult = ChunkSettings::create(USER_PAYLOAD_SIZE, iox::CHUNK_DEFAULT_USER_PAYLOAD_ALIGNMENT);
         IOX_ENSURES(chunkSettingsResult.has_value());
@@ -104,7 +104,7 @@ class iox_sub_test : public Test
     static constexpr size_t MEMORY_SIZE = 1024 * 1024 * 100;
     uint8_t m_memory[MEMORY_SIZE];
     static constexpr uint32_t NUM_CHUNKS_IN_POOL = MAX_CHUNKS_HELD_PER_SUBSCRIBER_SIMULTANEOUSLY + 2U;
-    static constexpr uint32_t CHUNK_SIZE = 128U;
+    static constexpr uint64_t CHUNK_SIZE = 128U;
 
     BumpAllocator m_memoryAllocator{m_memory, MEMORY_SIZE};
     MePooConfig m_mempoolconf;

iceoryx_meta/build_options.cmake

@@ -34,6 +34,7 @@ option(ROUDI_ENVIRONMENT "Build RouDi Environment for testing, is enabled when b
 option(ADDRESS_SANITIZER "Build with address sanitizer" OFF)
 option(THREAD_SANITIZER "Build with thread sanitizer" OFF)
 option(TEST_WITH_ADDITIONAL_USER "Build Test with additional user accounts for testing access control" OFF)
+option(TEST_WITH_HUGE_PAYLOAD "Build Tests which use payload bigger than 2GB" OFF)
 option(TOML_CONFIG "TOML support for RouDi with dynamic configuration" ON)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # "Create compile_commands.json file"
@@ -90,5 +91,6 @@ function(show_config_options)
   message("          ADDRESS_SANITIZER....................: " ${ADDRESS_SANITIZER})
   message("          THREAD_SANITIZER.....................: " ${THREAD_SANITIZER})
   message("          TEST_WITH_ADDITIONAL_USER ...........: " ${TEST_WITH_ADDITIONAL_USER})
+  message("          TEST_WITH_HUGE_PAYLOAD ..............: " ${TEST_WITH_HUGE_PAYLOAD})
   message("          TOML_CONFIG..........................: " ${TOML_CONFIG})
 endfunction()