Merge pull request eclipse-iceoryx#614 from ApexAI/iox-#14-add-payloa…

…d-alignment-and-custom-header-to-memory-manager

Iox #14 add payload alignment and custom header to memory manager

doc/design/chunk_header.md

@@ -12,7 +12,7 @@ Chunks are the transport capsules in iceoryx. They store data from a publisher a
 | Chunk Header      | contains meta information related to the chunk           |
 | Custom Header     | contains custom meta information, e.g. timestamps        |
 | Payload           | the user data                                            |
-| Back-Offset       | offset stored in front of the payload to calculate back to the chunk header |
+| Back-Offset       | offset stored in front of the payload to calculate back to the chunk header (for the most simple case it will overlap with the payload offset stored in the Chunk Header) |
 
 ## Design
 
@@ -62,13 +62,15 @@ For back calculation from the payload pointer to the `ChunkHeader` pointer, the
 |------------------>|--------------------->|
 |                   |                      |
 +===================+======================+==================================+
-|  ChunkHeader  ¦   |       Payload        |  Padding                         |
+|  ChunkHeader  ¦ * |       Payload        |  Padding                         |
 +===================+======================+==================================+
 |                   |                                                         |
 |   payloadOffset   |                                                         |
 |------------------>|                                                         |
 |                                 chunkSize                                   |
 |---------------------------------------------------------------------------->|
+
+*) payloadOffset from ChunkHeader and back-offset are overlapping
 ```
 
 2. No custom header and alignment exceeds the `ChunkHeader` alignment
@@ -115,22 +117,22 @@ Depending on the address of the chunk there is the chance that `ChunkHeader` is
 payloadOffset = sizeof(ChunkHeader);
 ```
 
-2. No custom header and payload alignment exceeds the `ChunkHeader` alignment, which means the payload is either adjacent to the `ChunkHeader` or there is a padding with at least the size of `ChunkHeader` in front of the payload and therefore enough space to store the `back-offset`
+2. No custom header and payload alignment exceeds the `ChunkHeader` alignment, which means the payload is either adjacent to the `ChunkHeader` or there is a padding with at least the size of `ChunkHeader` alignment in front of the payload and therefore enough space to store the `back-offset`
 
 ```
 headerEndAddress = addressof(chunkHeader) + sizeof(chunkHeader);
-payloadAddress = align(headerEndAddress, payloadAlignment);
-payloadOffset = payloadAddress - addressof(chunkHeader);
+alignedPayloadAddress = align(headerEndAddress, payloadAlignment);
+payloadOffset = alignedPayloadAddress - addressof(chunkHeader);
 ```
 
 3. Custom header is used
 
 ```
 headerEndAddress = addressof(chunkHeader) + sizeof(chunkHeader) + sizeof(customHeader);
-potentialBackOffsetAddress = align(headerEndAddress, alignof(payloadOffset));
-potentialPayloadAddress = potentialBackOffsetAddress + sizeof(payloadOffset);
-payloadAddress = align(potentialPayloadAddress, payloadAlignment);
-payloadOffset = payloadAddress - addressof(chunkHeader);
+anticipatedBackOffsetAddress = align(headerEndAddress, alignof(payloadOffset));
+unalignedPayloadAddress = anticipatedBackOffsetAddress + sizeof(payloadOffset);
+alignedPayloadAddress = align(unalignedPayloadAddress, payloadAlignment);
+payloadOffset = alignedPayloadAddress - addressof(chunkHeader);
 ```
 
 #### Required Chunk Size Calculation
@@ -192,8 +194,8 @@ The following formula is used to calculate the required chunk size.
 ```
 headerSize = sizeof(chunkHeader) + sizeof(customHeader)
 prePayloadAlignmentOverhang = align(headerSize, alignof(payloadOffset));
-maxAlignment = max(alignof(payloadOffset), payloadAlignment);
-chunkSize = prePayloadAlignmentOverhang + maxAlignment + payloadSize;
+maxPadding = max(sizeof(payloadOffset), payloadAlignment);
+chunkSize = prePayloadAlignmentOverhang + maxPadding + payloadSize;
 ```
 
 #### Accessing Chunk Header Extension

iceoryx_binding_c/source/c_publisher.cpp

@@ -101,9 +101,9 @@ void iox_pub_deinit(iox_pub_t const self)
 
 iox_AllocationResult iox_pub_loan_chunk(iox_pub_t const self, void** const chunk, const uint32_t payloadSize)
 {
-    auto result = PublisherPortUser(self->m_portData).tryAllocateChunk(payloadSize).and_then([&](ChunkHeader* h) {
-        *chunk = h->payload();
-    });
+    auto result = PublisherPortUser(self->m_portData)
+                      .tryAllocateChunk(payloadSize, CHUNK_DEFAULT_PAYLOAD_ALIGNMENT)
+                      .and_then([&chunk](ChunkHeader* h) { *chunk = h->payload(); });
     if (result.has_error())
     {
         return cpp2c::AllocationResult(result.get_error());
@@ -155,4 +155,3 @@ iox_service_description_t iox_pub_get_service_description(iox_pub_t const self)
 {
     return TranslateServiceDescription(PublisherPortUser(self->m_portData).getCaProServiceDescription());
 }
-

iceoryx_binding_c/test/moduletests/test_event_info.cpp

@@ -75,6 +75,15 @@ class iox_event_info_test : public Test
         m_lastEventCallbackArgument = arg;
     }
 
+    iox::mepoo::SharedChunk getChunkFromMemoryManager()
+    {
+        constexpr uint32_t PAYLOAD_SIZE{100U};
+        return m_memoryManager.getChunk(PAYLOAD_SIZE,
+                                        iox::CHUNK_DEFAULT_PAYLOAD_ALIGNMENT,
+                                        iox::CHUNK_NO_CUSTOM_HEADER_SIZE,
+                                        iox::CHUNK_NO_CUSTOM_HEADER_ALIGNMENT);
+    }
+
 
     static UserTrigger* m_lastEventCallbackArgument;
     ConditionVariableData m_condVar{"myApp"};
@@ -108,7 +117,7 @@ TEST_F(iox_event_info_test, eventInfoHasCorrectId)
 
     auto eventInfoVector = m_waitSet.wait();
 
-    ASSERT_THAT(eventInfoVector.size(), Eq(1));
+    ASSERT_THAT(eventInfoVector.size(), Eq(1U));
     EXPECT_EQ(iox_event_info_get_event_id(eventInfoVector[0]), ARBITRARY_EVENT_ID);
 }
 
@@ -125,10 +134,9 @@ TEST_F(iox_event_info_test, eventOriginIsUserTriggerPointerWhenItsOriginatingFro
 
 TEST_F(iox_event_info_test, eventOriginIsNotUserTriggerPointerWhenItsNotOriginatingFromThem)
 {
-    constexpr uint64_t CHUNK_SIZE = 100U;
     iox_ws_attach_subscriber_event(&m_waitSet, m_subscriberHandle, SubscriberEvent_HAS_DATA, 587U, NULL);
     this->Subscribe(&m_portPtr);
-    m_chunkPusher.push(m_memoryManager.getChunk(CHUNK_SIZE));
+    m_chunkPusher.push(getChunkFromMemoryManager());
 
     auto eventInfoVector = m_waitSet.wait();
 
@@ -137,10 +145,9 @@ TEST_F(iox_event_info_test, eventOriginIsNotUserTriggerPointerWhenItsNotOriginat
 
 TEST_F(iox_event_info_test, eventOriginIsSubscriberPointerWhenItsOriginatingFromThem)
 {
-    constexpr uint64_t CHUNK_SIZE = 100U;
     iox_ws_attach_subscriber_event(&m_waitSet, m_subscriberHandle, SubscriberEvent_HAS_DATA, 587U, NULL);
     this->Subscribe(&m_portPtr);
-    m_chunkPusher.push(m_memoryManager.getChunk(CHUNK_SIZE));
+    m_chunkPusher.push(getChunkFromMemoryManager());
 
     auto eventInfoVector = m_waitSet.wait();
 
@@ -172,10 +179,9 @@ TEST_F(iox_event_info_test, getOriginReturnsPointerToUserTriggerWhenOriginatingF
 
 TEST_F(iox_event_info_test, getOriginReturnsNullptrUserTriggerWhenNotOriginatingFromThem)
 {
-    constexpr uint64_t CHUNK_SIZE = 100U;
     iox_ws_attach_subscriber_event(&m_waitSet, m_subscriberHandle, SubscriberEvent_HAS_DATA, 587U, NULL);
     this->Subscribe(&m_portPtr);
-    m_chunkPusher.push(m_memoryManager.getChunk(CHUNK_SIZE));
+    m_chunkPusher.push(getChunkFromMemoryManager());
 
     auto eventInfoVector = m_waitSet.wait();
 
@@ -185,10 +191,9 @@ TEST_F(iox_event_info_test, getOriginReturnsNullptrUserTriggerWhenNotOriginating
 
 TEST_F(iox_event_info_test, getOriginReturnsPointerToSubscriberWhenOriginatingFromThem)
 {
-    constexpr uint64_t CHUNK_SIZE = 100U;
     iox_ws_attach_subscriber_event(&m_waitSet, m_subscriberHandle, SubscriberEvent_HAS_DATA, 587U, NULL);
     this->Subscribe(&m_portPtr);
-    m_chunkPusher.push(m_memoryManager.getChunk(CHUNK_SIZE));
+    m_chunkPusher.push(getChunkFromMemoryManager());
 
     auto eventInfoVector = m_waitSet.wait();
 

iceoryx_binding_c/test/moduletests/test_listener.cpp

@@ -268,9 +268,12 @@ TIMING_TEST_F(iox_listener_test, SubscriberCallbackIsCalledSampleIsReceived, Rep
                 Eq(iox_ListenerResult::ListenerResult_SUCCESS));
 
     Subscribe(m_subscriber[0U]);
-    m_chunkPusher[0U].push(m_memoryManager.getChunk(100U));
+    constexpr uint32_t PAYLOAD_SIZE{100U};
+    m_chunkPusher[0U].push(m_memoryManager.getChunk(PAYLOAD_SIZE,
+                                                    iox::CHUNK_DEFAULT_PAYLOAD_ALIGNMENT,
+                                                    iox::CHUNK_NO_CUSTOM_HEADER_SIZE,
+                                                    iox::CHUNK_NO_CUSTOM_HEADER_ALIGNMENT));
 
     std::this_thread::sleep_for(TIMEOUT);
     EXPECT_THAT(g_subscriberCallbackArgument, Eq(&m_subscriber[0U]));
 });
-

iceoryx_binding_c/test/moduletests/test_publisher.cpp

@@ -197,7 +197,11 @@ TEST_F(iox_pub_test, allocate_chunkFailsWhenOutOfChunks)
     std::vector<SharedChunk> chunkBucket;
     while (true)
     {
-        auto sharedChunk = m_memoryManager.getChunk(100);
+        constexpr uint32_t PAYLOAD_SIZE{100U};
+        auto sharedChunk = m_memoryManager.getChunk(PAYLOAD_SIZE,
+                                                    iox::CHUNK_DEFAULT_PAYLOAD_ALIGNMENT,
+                                                    iox::CHUNK_NO_CUSTOM_HEADER_SIZE,
+                                                    iox::CHUNK_NO_CUSTOM_HEADER_ALIGNMENT);
         if (sharedChunk)
             chunkBucket.emplace_back(sharedChunk);
         else
@@ -317,4 +321,3 @@ TEST(iox_pub_options_test, publisherInitializationTerminatesIfOptionsAreNotIniti
 
     EXPECT_DEATH({ iox_pub_init(&storage, "a", "b", "c", &options); }, ".*");
 }
-

iceoryx_binding_c/test/moduletests/test_subscriber.cpp

@@ -79,6 +79,15 @@ class iox_sub_test : public Test
         m_triggerCallbackLatestArgument = sub;
     }
 
+    iox::mepoo::SharedChunk getChunkFromMemoryManager()
+    {
+        constexpr uint32_t PAYLOAD_SIZE{100U};
+        return m_memoryManager.getChunk(PAYLOAD_SIZE,
+                                        iox::CHUNK_DEFAULT_PAYLOAD_ALIGNMENT,
+                                        iox::CHUNK_NO_CUSTOM_HEADER_SIZE,
+                                        iox::CHUNK_NO_CUSTOM_HEADER_ALIGNMENT);
+    }
+
     static iox_sub_t m_triggerCallbackLatestArgument;
     static constexpr size_t MEMORY_SIZE = 1024 * 1024 * 100;
     uint8_t m_memory[MEMORY_SIZE];
@@ -166,7 +175,7 @@ TEST_F(iox_sub_test, initialStateNoChunksAvailable)
 TEST_F(iox_sub_test, receiveChunkWhenThereIsOne)
 {
     this->Subscribe(&m_portPtr);
-    m_chunkPusher.push(m_memoryManager.getChunk(100U));
+    m_chunkPusher.push(getChunkFromMemoryManager());
 
     const void* chunk = nullptr;
     EXPECT_EQ(iox_sub_take_chunk(m_sut, &chunk), ChunkReceiveResult_SUCCESS);
@@ -180,7 +189,7 @@ TEST_F(iox_sub_test, receiveChunkWithContent)
         int value;
     };
 
-    auto sharedChunk = m_memoryManager.getChunk(100U);
+    auto sharedChunk = getChunkFromMemoryManager();
     static_cast<data_t*>(sharedChunk.getPayload())->value = 1234;
     m_chunkPusher.push(sharedChunk);
 
@@ -193,7 +202,7 @@ TEST_F(iox_sub_test, receiveChunkWithContent)
 TEST_F(iox_sub_test, chunkHeaderCanBeObtainedFromChunkAfterTake)
 {
     this->Subscribe(&m_portPtr);
-    auto sharedChunk = m_memoryManager.getChunk(100U);
+    auto sharedChunk = getChunkFromMemoryManager();
     m_chunkPusher.push(sharedChunk);
 
     const void* chunk = nullptr;
@@ -211,18 +220,18 @@ TEST_F(iox_sub_test, receiveChunkWhenToManyChunksAreHold)
     const void* chunk = nullptr;
     for (uint64_t i = 0U; i < MAX_CHUNKS_HELD_PER_SUBSCRIBER_SIMULTANEOUSLY + 1U; ++i)
     {
-        m_chunkPusher.push(m_memoryManager.getChunk(100U));
+        m_chunkPusher.push(getChunkFromMemoryManager());
         iox_sub_take_chunk(m_sut, &chunk);
     }
 
-    m_chunkPusher.push(m_memoryManager.getChunk(100U));
+    m_chunkPusher.push(getChunkFromMemoryManager());
     EXPECT_EQ(iox_sub_take_chunk(m_sut, &chunk), ChunkReceiveResult_TOO_MANY_CHUNKS_HELD_IN_PARALLEL);
 }
 
 TEST_F(iox_sub_test, releaseChunkWorks)
 {
     this->Subscribe(&m_portPtr);
-    m_chunkPusher.push(m_memoryManager.getChunk(100U));
+    m_chunkPusher.push(getChunkFromMemoryManager());
 
     const void* chunk = nullptr;
     iox_sub_take_chunk(m_sut, &chunk);
@@ -237,7 +246,7 @@ TEST_F(iox_sub_test, releaseChunkQueuedChunksWorks)
     this->Subscribe(&m_portPtr);
     for (uint64_t i = 0U; i < MAX_CHUNKS_HELD_PER_SUBSCRIBER_SIMULTANEOUSLY; ++i)
     {
-        m_chunkPusher.push(m_memoryManager.getChunk(100U));
+        m_chunkPusher.push(getChunkFromMemoryManager());
     }
 
     EXPECT_THAT(m_memoryManager.getMemPoolInfo(0).m_usedChunks, Eq(MAX_CHUNKS_HELD_PER_SUBSCRIBER_SIMULTANEOUSLY));
@@ -253,7 +262,7 @@ TEST_F(iox_sub_test, initialStateHasNewChunksFalse)
 TEST_F(iox_sub_test, receivingChunkLeadsToHasNewChunksTrue)
 {
     this->Subscribe(&m_portPtr);
-    m_chunkPusher.push(m_memoryManager.getChunk(100U));
+    m_chunkPusher.push(getChunkFromMemoryManager());
 
     EXPECT_TRUE(iox_sub_has_chunks(m_sut));
 }
@@ -268,7 +277,7 @@ TEST_F(iox_sub_test, sendingTooMuchLeadsToLostChunks)
     this->Subscribe(&m_portPtr);
     for (uint64_t i = 0U; i < DefaultChunkQueueConfig::MAX_QUEUE_CAPACITY + 1U; ++i)
     {
-        m_chunkPusher.push(m_memoryManager.getChunk(100U));
+        m_chunkPusher.push(getChunkFromMemoryManager());
     }
 
     EXPECT_TRUE(iox_sub_has_lost_chunks(m_sut));
@@ -303,7 +312,7 @@ TEST_F(iox_sub_test, hasDataTriggersWaitSetWithCorrectEventId)
 {
     iox_ws_attach_subscriber_event(m_waitSet.get(), m_sut, SubscriberEvent_HAS_DATA, 587U, NULL);
     this->Subscribe(&m_portPtr);
-    m_chunkPusher.push(m_memoryManager.getChunk(100U));
+    m_chunkPusher.push(getChunkFromMemoryManager());
 
     auto triggerVector = m_waitSet->wait();
 
@@ -315,7 +324,7 @@ TEST_F(iox_sub_test, hasDataTriggersWaitSetWithCorrectCallback)
 {
     iox_ws_attach_subscriber_event(m_waitSet.get(), m_sut, SubscriberEvent_HAS_DATA, 0U, iox_sub_test::triggerCallback);
     this->Subscribe(&m_portPtr);
-    m_chunkPusher.push(m_memoryManager.getChunk(100U));
+    m_chunkPusher.push(getChunkFromMemoryManager());
 
     auto triggerVector = m_waitSet->wait();
 

iceoryx_dds/test/mocks/chunk_mock_dds.hpp

@@ -49,7 +49,8 @@ class ChunkMockDDS
 
         memset(m_rawMemory, 0xFF, Size);
 
-        m_chunkHeader = new (m_rawMemory) iox::mepoo::ChunkHeader();
+        m_chunkHeader = new (m_rawMemory) iox::mepoo::ChunkHeader(
+            Size, sizeof(T), alignof(T), iox::CHUNK_NO_CUSTOM_HEADER_SIZE, iox::CHUNK_NO_CUSTOM_HEADER_ALIGNMENT);
         m_chunkHeader->payloadSize = sizeof(T);
 
         // Set the value

iceoryx_posh/include/iceoryx_posh/iceoryx_posh_types.hpp

@@ -1,5 +1,5 @@
-// Copyright (c) 2019 by Robert Bosch GmbH. All rights reserved.
-// Copyright (c) 2021 by Apex.AI Inc. All rights reserved.
+// Copyright (c) 2019 - 2020 by Robert Bosch GmbH. All rights reserved.
+// Copyright (c) 2020 - 2021 by Apex.AI Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -122,6 +122,10 @@ constexpr uint32_t MAX_SHM_SEGMENTS = 100U;
 constexpr uint32_t MAX_NUMBER_OF_MEMORY_PROVIDER = 8U;
 constexpr uint32_t MAX_NUMBER_OF_MEMORY_BLOCKS_PER_MEMORY_PROVIDER = 64U;
 
+constexpr uint32_t CHUNK_DEFAULT_PAYLOAD_ALIGNMENT{8U};
+constexpr uint32_t CHUNK_NO_CUSTOM_HEADER_SIZE{0U};
+constexpr uint32_t CHUNK_NO_CUSTOM_HEADER_ALIGNMENT{1U};
+
 // Message Queue
 constexpr uint32_t ROUDI_MAX_MESSAGES = 5U;
 constexpr uint32_t ROUDI_MESSAGE_SIZE = 512U;