kafka protocol: generate uniform request&response types for all suppo…

…rted versions (#1674)
hstreamdb · Nov 9, 2023 · c5870d9 · c5870d9
1 parent a34159f
commit c5870d9
Show file tree

Hide file tree

Showing 6 changed files with 2,501 additions and 56 deletions.
diff --git a/hstream-kafka/hstream-kafka.cabal b/hstream-kafka/hstream-kafka.cabal
@@ -54,6 +54,7 @@ library kafka-protocol
     Kafka.Protocol.Encoding.Internal
     Kafka.Protocol.Encoding.Parser
     Kafka.Protocol.Message.Struct
+    Kafka.Protocol.Message.Total
 
   hs-source-dirs:     protocol
   build-tool-depends: hpp:hpp >=0.6 && <0.7

diff --git a/hstream-kafka/protocol/Kafka/Protocol/Encoding.hs b/hstream-kafka/protocol/Kafka/Protocol/Encoding.hs
@@ -50,6 +50,8 @@ module Kafka.Protocol.Encoding
     -- ** Misc
   , pattern NonNullKaArray
   , unNonNullKaArray
+  , kaArrayToCompact
+  , kaArrayFromCompact
     -- * Internals
   , Parser
   , runParser
@@ -222,10 +224,16 @@ newtype KaArray a = KaArray
   { unKaArray :: Maybe (Vector a) }
   deriving newtype (Show, Eq, Ord)
 
+instance Functor KaArray where
+  fmap f (KaArray xs) = KaArray $ fmap f <$> xs
+
 newtype CompactKaArray a = CompactKaArray
   { unCompactKaArray :: Maybe (Vector a) }
   deriving newtype (Show, Eq, Ord)
 
+instance Functor CompactKaArray where
+  fmap f (CompactKaArray xs) = CompactKaArray $ fmap f <$> xs
+
 newtype RecordKey = RecordKey { unRecordKey :: Maybe ByteString }
   deriving newtype (Show, Eq, Ord)
 
@@ -585,6 +593,14 @@ unNonNullKaArray :: KaArray a -> Vector a
 unNonNullKaArray =
   fromMaybe (error "non-nullable field was serialized as null") . unKaArray
 
+kaArrayToCompact :: KaArray a -> CompactKaArray a
+kaArrayToCompact = CompactKaArray . unKaArray
+{-# INLINE kaArrayToCompact #-}
+
+kaArrayFromCompact :: CompactKaArray a -> KaArray a
+kaArrayFromCompact = KaArray . unCompactKaArray
+{-# INLINE kaArrayFromCompact #-}
+
 -------------------------------------------------------------------------------
 -- Internals
 

diff --git a/hstream-kafka/protocol/Kafka/Protocol/Message.hs b/hstream-kafka/protocol/Kafka/Protocol/Message.hs
@@ -8,13 +8,15 @@ module Kafka.Protocol.Message
   , getResponseHeader
 
   , module Kafka.Protocol.Message.Struct
+  , module Kafka.Protocol.Message.Total
   ) where
 
 import qualified Data.ByteString.Lazy          as BL
 import           Data.Int
 
 import           Kafka.Protocol.Encoding
 import           Kafka.Protocol.Message.Struct
+import           Kafka.Protocol.Message.Total
 
 data RequestHeader = RequestHeader
   { requestApiKey        :: {-# UNPACK #-} !ApiKey

diff --git a/hstream-kafka/protocol/Kafka/Protocol/Message/Struct.hs b/hstream-kafka/protocol/Kafka/Protocol/Message/Struct.hs
@@ -1,7 +1,7 @@
 -------------------------------------------------------------------------------
 -- Autogenerated by kafka message json schema
 --
--- $ ./script/kafka_gen.py run > hstream-kafka/protocol/Kafka/Protocol/Message/Struct.hs
+-- $ ./script/kafka_gen.py run
 --
 -- DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING