Feature/crop mcap (#163)

* Added controls for cropping log file

* WIP: Working on a new MCAP cropper

* Dense commit: Split up MCAP and added LZ4 frame encoder

* Implementing tools to write MCAP back to file.

* Refactored the LZ4 output stream to work with byte buffers

* WIP

* wip

* WIP: More stuff for cropping

* Some doc, started writing up a test using the demo MCAP file.

* Improving the test.

* Making progress on the test.

* WIP: Tracking down chunk writing/reading bug.

* WIP: Got the cropping test to pass. Still needs a few final checks

* Wrapped up the cropping

* MCAP cropping is working.

scs2-session-logger/src/main/java/us/ihmc/scs2/session/mcap/MCAPBufferedChunk.java

@@ -3,13 +3,15 @@
 import gnu.trove.map.hash.TLongObjectHashMap;
 import us.ihmc.commons.thread.ThreadTools;
 import us.ihmc.log.LogTools;
-import us.ihmc.scs2.session.mcap.MCAP.Chunk;
-import us.ihmc.scs2.session.mcap.MCAP.ChunkIndex;
-import us.ihmc.scs2.session.mcap.MCAP.Message;
-import us.ihmc.scs2.session.mcap.MCAP.Opcode;
-import us.ihmc.scs2.session.mcap.MCAP.Record;
-import us.ihmc.scs2.session.mcap.MCAP.Records;
 import us.ihmc.scs2.session.mcap.input.MCAPDataInput;
+import us.ihmc.scs2.session.mcap.specs.MCAP;
+import us.ihmc.scs2.session.mcap.specs.records.Chunk;
+import us.ihmc.scs2.session.mcap.specs.records.ChunkIndex;
+import us.ihmc.scs2.session.mcap.specs.records.Message;
+import us.ihmc.scs2.session.mcap.specs.records.Opcode;
+import us.ihmc.scs2.session.mcap.specs.records.Record;
+import us.ihmc.scs2.session.mcap.specs.records.RecordDataInputBacked;
+import us.ihmc.scs2.session.mcap.specs.records.Records;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -55,7 +57,7 @@ public MCAPBufferedChunk(MCAP mcap, long desiredLogDT)
          {
             Chunk chunk = (Chunk) record.body();
             numberOfChunks++;
-            long chunkSize = chunk.recordsLength();
+            long chunkSize = chunk.recordsCompressedLength();
             minChunkSize = Math.min(minChunkSize, chunkSize);
             maxChunkSize = Math.max(maxChunkSize, chunkSize);
             totalChunkSize += chunkSize;
@@ -325,7 +327,7 @@ private void loadChunkNow() throws IOException
          if (chunkRecords == null)
          {
             ByteBuffer chunkBuffer = mcap.getDataInput().getByteBuffer(chunkIndex.chunkOffset(), (int) chunkIndex.chunkLength(), true);
-            chunkRecords = ((Chunk) new Record(MCAPDataInput.wrap(chunkBuffer), 0).body()).records();
+            chunkRecords = ((Chunk) new RecordDataInputBacked(MCAPDataInput.wrap(chunkBuffer), 0).body()).records();
          }
 
          if (!loadedChunkBundles.contains(this))
@@ -354,15 +356,15 @@ public void loadMessagesNow() throws IOException
 
          try
          {
+            if (bundledMessages == null)
+               bundledMessages = new TLongObjectHashMap<>();
+
             for (Record record : chunkRecords)
             {
                if (record.op() != Opcode.MESSAGE)
                   continue;
 
-               if (bundledMessages == null)
-                  bundledMessages = new TLongObjectHashMap<>();
-
-               Message message = (Message) record.body();
+               Message message = record.body();
                List<Message> messages = bundledMessages.get(round(message.logTime(), desiredLogDT));
                if (messages == null)
                {

scs2-session-logger/src/main/java/us/ihmc/scs2/session/mcap/MCAPConsoleLogManager.java

@@ -1,12 +1,14 @@
 package us.ihmc.scs2.session.mcap;
 
 import us.ihmc.log.LogTools;
-import us.ihmc.scs2.session.mcap.MCAP.Channel;
-import us.ihmc.scs2.session.mcap.MCAP.Message;
-import us.ihmc.scs2.session.mcap.MCAP.Opcode;
-import us.ihmc.scs2.session.mcap.MCAP.Record;
-import us.ihmc.scs2.session.mcap.MCAP.Schema;
 import us.ihmc.scs2.session.mcap.MCAPBufferedChunk.ChunkBundle;
+import us.ihmc.scs2.session.mcap.encoding.CDRDeserializer;
+import us.ihmc.scs2.session.mcap.specs.MCAP;
+import us.ihmc.scs2.session.mcap.specs.records.Channel;
+import us.ihmc.scs2.session.mcap.specs.records.Message;
+import us.ihmc.scs2.session.mcap.specs.records.Record;
+import us.ihmc.scs2.session.mcap.specs.records.Schema;
+import us.ihmc.scs2.session.mcap.specs.records.Opcode;
 import us.ihmc.scs2.simulation.SpyList;
 
 import java.io.IOException;

scs2-session-logger/src/main/java/us/ihmc/scs2/session/mcap/MCAPFrameTransformManager.java

@@ -9,11 +9,15 @@
 import us.ihmc.scs2.definition.yoGraphic.YoGraphicDefinition;
 import us.ihmc.scs2.definition.yoGraphic.YoGraphicDefinitionFactory;
 import us.ihmc.scs2.definition.yoGraphic.YoGraphicGroupDefinition;
-import us.ihmc.scs2.session.mcap.MCAP.Message;
-import us.ihmc.scs2.session.mcap.MCAP.Opcode;
-import us.ihmc.scs2.session.mcap.MCAP.Record;
 import us.ihmc.scs2.session.mcap.MCAPBufferedChunk.ChunkBundle;
 import us.ihmc.scs2.session.mcap.MCAPSchema.MCAPSchemaField;
+import us.ihmc.scs2.session.mcap.encoding.CDRDeserializer;
+import us.ihmc.scs2.session.mcap.specs.MCAP;
+import us.ihmc.scs2.session.mcap.specs.records.Channel;
+import us.ihmc.scs2.session.mcap.specs.records.Message;
+import us.ihmc.scs2.session.mcap.specs.records.Opcode;
+import us.ihmc.scs2.session.mcap.specs.records.Record;
+import us.ihmc.scs2.session.mcap.specs.records.Schema;
 import us.ihmc.yoVariables.euclid.YoPoint3D;
 import us.ihmc.yoVariables.euclid.YoPose3D;
 import us.ihmc.yoVariables.euclid.YoQuaternion;
@@ -63,7 +67,7 @@ public class MCAPFrameTransformManager
    private final Set<String> unattachedRootNames = new LinkedHashSet<>();
 
    private final YoGraphicGroupDefinition yoGraphicGroupDefinition = new YoGraphicGroupDefinition("FoxgloveFrameTransforms");
-   private MCAP.Schema mcapSchema;
+   private Schema mcapSchema;
 
    public MCAPFrameTransformManager(ReferenceFrame inertialFrame)
    {
@@ -72,12 +76,12 @@ public MCAPFrameTransformManager(ReferenceFrame inertialFrame)
 
    public void initialize(MCAP mcap, MCAPBufferedChunk chunkBuffer) throws IOException
    {
-      for (MCAP.Record record : mcap.records())
+      for (Record record : mcap.records())
       {
          if (record.op() != Opcode.SCHEMA)
             continue;
 
-         mcapSchema = (MCAP.Schema) record.body();
+         mcapSchema = (Schema) record.body();
          if (mcapSchema.name().equalsIgnoreCase("foxglove::FrameTransform"))
          {
             if (mcapSchema.encoding().equalsIgnoreCase("ros2msg"))
@@ -121,11 +125,11 @@ else if (mcapSchema.encoding().equalsIgnoreCase("omgidl"))
       }
 
       TIntObjectHashMap<String> channelIdToTopicMap = new TIntObjectHashMap<>();
-      for (MCAP.Record record : mcap.records())
+      for (Record record : mcap.records())
       {
          if (record.op() == Opcode.CHANNEL)
          {
-            MCAP.Channel channel = (MCAP.Channel) record.body();
+            Channel channel = (Channel) record.body();
             if (channel.schemaId() == foxgloveFrameTransformSchema.getId())
             {
                channelIdToTopicMap.put(channel.id(), channel.topic());
@@ -200,7 +204,7 @@ else if (mcapSchema.encoding().equalsIgnoreCase("omgidl"))
       }
    }
 
-   private void processRecord(MCAP.Record record, TIntObjectHashMap<String> channelIdToTopicMap, Map<String, BasicTransformInfo> allTransforms)
+   private void processRecord(Record record, TIntObjectHashMap<String> channelIdToTopicMap, Map<String, BasicTransformInfo> allTransforms)
    {
       Message message = (Message) record.body();
       String topic = channelIdToTopicMap.get(message.channelId());
@@ -377,7 +381,7 @@ public boolean hasMCAPFrameTransforms()
       return foxgloveFrameTransformSchema != null;
    }
 
-   public MCAP.Schema getMCAPSchema()
+   public Schema getMCAPSchema()
    {
       return mcapSchema;
    }