Merge branch 'main' into improve_kafka_configs

hstreamdb · Nov 27, 2023 · c71287e · c71287e
2 parents 62eb491 + 47862b3
commit c71287e
Show file tree

Hide file tree

Showing 13 changed files with 30 additions and 27 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -206,8 +206,7 @@ jobs:
           rm -f ~/data/hstream_tests.tar
           find dist-newstyle/build -type f \( \
             -name "*-test" -o \
-            -name "hstream-server" -o \
-            -name "hstream-kafka-server" \) \
+            -name "hstream-server" \) \
             -exec tar -rvf ~/data/hstream_tests.tar {} \;
 
       - uses: actions/upload-artifact@v3
@@ -364,7 +363,6 @@ jobs:
         run: |
           docker load -i ~/data/new_hstream_image.tar
           docker run -t --rm $NEW_HSTREAM_IMAGE /usr/local/bin/hstream-server +RTS --info
-          docker run -t --rm $NEW_HSTREAM_IMAGE /usr/local/bin/hstream-kafka-server +RTS --info
 
       - name: fetch tests source code
         uses: actions/checkout@v3

diff --git a/.gitmodules b/.gitmodules
@@ -16,9 +16,6 @@
 [submodule "external/hs-grpc"]
 	path = external/hs-grpc
 	url = https://github.com/4eUeP/hs-grpc.git
-[submodule "external/table-layout"]
-	path = external/table-layout
-	url = https://github.com/muesli4/table-layout.git
 [submodule "external/haskell-rocksdb-bindings"]
 	path = external/haskell-rocksdb-bindings
 	url = https://github.com/hstreamdb/haskell-rocksdb-bindings.git
diff --git a/cabal.project b/cabal.project
@@ -2,8 +2,6 @@ packages:
   -- * External
   external/proto3-wire/proto3-wire.cabal
   external/proto3-suite/proto3-suite.cabal
-  -- We need to add this table-layout since the hackage version is too old.
-  external/table-layout/table-layout.cabal
   external/hs-grpc/hs-grpc-server/hs-grpc-server.cabal
   external/gRPC-haskell/grpc-haskell.cabal
   external/gRPC-haskell/core/grpc-haskell-core.cabal
@@ -37,9 +35,6 @@ package proto3-wire
 package proto3-suite
   tests: false
 
-package table-layout
-  tests: false
-
 package hs-grpc-server
   flags: +hsgrpc_use_protosuite
   tests: false
@@ -61,7 +56,7 @@ constraints:
     Z-Data      == 2.0.0.2
   , zoovisitor  == 0.2.6.1
   , criterion  ^>= 1.6
-  , aeson      ^>= 2.0
+  , aeson      ^>= 2.1
   , filepath    >= 1.4.100
 
   -- The following is needed to avoid breaking the build.

diff --git a/cabal.project.hadmin.store b/cabal.project.hadmin.store
@@ -1,8 +1,6 @@
 with-compiler: ghc-8.10.7
 
 packages:
-  -- We need to add this table-layout since the hackage version is too old.
-  external/table-layout/table-layout.cabal
   external/hsthrift/common/util/fb-util.cabal
   external/hsthrift/common/github/fb-stubs.cabal
   external/hsthrift/common/mangle/mangle.cabal

diff --git a/common/base/HStream/Logger.hs b/common/base/HStream/Logger.hs
@@ -1,4 +1,5 @@
 {-# LANGUAGE BangPatterns         #-}
+{-# LANGUAGE CPP                  #-}
 {-# LANGUAGE GADTs                #-}
 {-# LANGUAGE MagicHash            #-}
 {-# LANGUAGE PatternSynonyms      #-}
@@ -60,7 +61,7 @@ import           Control.Exception              (finally)
 import           Control.Monad                  (forever, when)
 import           Data.IORef                     (IORef, atomicWriteIORef,
                                                  newIORef, readIORef)
-import           Foreign.C.Types                (CInt (..))
+import           Foreign.C.Types
 import           GHC.Conc.Sync                  (ThreadId (..), myThreadId)
 import           GHC.Exts                       (ThreadId#)
 import           GHC.Stack
@@ -409,4 +410,13 @@ logBylevel flushLevel level cstack s = do
 
 -------------------------------------------------------------------------------
 
+-- this cannot be capi, as GHC panics.
+#if __GLASGOW_HASKELL__ >= 904
+-- https://gitlab.haskell.org/ghc/ghc/-/merge_requests/6163
+foreign import ccall unsafe "rts_getThreadId" getThreadId :: ThreadId# -> CULLong
+#elif __GLASGOW_HASKELL__ >= 900
+-- https://gitlab.haskell.org/ghc/ghc/-/merge_requests/1254
+foreign import ccall unsafe "rts_getThreadId" getThreadId :: ThreadId# -> CLong
+#else
 foreign import ccall unsafe "rts_getThreadId" getThreadId :: ThreadId# -> CInt
+#endif
diff --git a/common/hstream/hstream-common.cabal b/common/hstream/hstream-common.cabal
@@ -94,7 +94,7 @@ library
     , hsc2hs:hsc2hs
 
   build-depends:
-    , aeson                 >=1.5   && <2.0 || ^>=2.0
+    , aeson                 >=1.5   && <2.0 || ^>=2.1
     , ansi-terminal
     , async
     , attoparsec

diff --git a/external/table-layout b/external/table-layout
diff --git a/hstream-admin/server/hstream-admin-server.cabal b/hstream-admin/server/hstream-admin-server.cabal
@@ -90,8 +90,8 @@ library
     , network               ^>=3.1.2
     , optparse-applicative  ^>=0.16
     , proto3-suite
-    , table-layout          ^>=0.9.1
-    , text                  ^>=1.2.3.0
+    , table-layout          ^>=1.0.0
+    , text
     , time
     , transformers          ^>=0.5.5.0
     , unordered-containers  ^>=0.2.9.0

diff --git a/hstream-admin/store/hstream-admin-store.cabal b/hstream-admin/store/hstream-admin-store.cabal
@@ -104,7 +104,7 @@ library
     , hstream-store
     , optparse-applicative  ^>=0.16
     , STMonadTrans
-    , table-layout          ^>=0.9.1
+    , table-layout          ^>=1.0
     , text                  ^>=1.2.3.0
     , thrift-cpp-channel
     , thrift-lib

diff --git a/hstream-sql/hstream-sql.cabal b/hstream-sql/hstream-sql.cabal
@@ -135,7 +135,7 @@ library
     , rio
     , scientific
     , string-random
-    , text                  ^>=1.2
+    , text
     , time                  >=1.9.1 && <2
     , time-compat
     , unordered-containers

diff --git a/hstream-store/HStream/Store/Internal/LogDevice/LogConfigTypes.hs b/hstream-store/HStream/Store/Internal/LogDevice/LogConfigTypes.hs
@@ -29,6 +29,9 @@ import           GHC.Stack                                      (HasCallStack,
 import qualified Z.Data.CBytes                                  as CBytes
 import           Z.Data.CBytes                                  (CBytes)
 import qualified Z.Foreign                                      as Z
+#if __GLASGOW_HASKELL__ >= 904
+import           GHC.Exts                                       (ArrayArray# (ArrayArray#))
+#endif
 
 import           HStream.Foreign
 import qualified HStream.Store.Exception                        as E

diff --git a/hstream-store/HStream/Store/Internal/LogDevice/Writer.hs b/hstream-store/HStream/Store/Internal/LogDevice/Writer.hs
@@ -33,6 +33,9 @@ import           Z.Data.CBytes                  (CBytes)
 import qualified Z.Data.Vector                  as V
 import           Z.Data.Vector                  (Bytes)
 import qualified Z.Foreign                      as Z
+#if __GLASGOW_HASKELL__ >= 904
+import           GHC.Exts                       (ArrayArray# (ArrayArray#))
+#endif
 
 import qualified HStream.Store.Exception        as E
 import           HStream.Store.Internal.Foreign

diff --git a/hstream-store/hstream-store.cabal b/hstream-store/hstream-store.cabal
@@ -80,18 +80,18 @@ library
     HStream.Store.Stream
 
   build-depends:
-    , base                 >=4.13   && <5
-    , bytestring           >=0.10   && <0.12
+    , base                 >=4.13  && <5
+    , bytestring           >=0.10  && <0.12
     , cache                ^>=0.1
     , containers           ^>=0.6
     , data-default         ^>=0.7
-    , filepath             ^>=1.4.2
-    , ghc-prim             >=0.5    && <1.0
-    , hashable             >=1.3.5  && <1.5
+    , filepath
+    , ghc-prim             >=0.5   && <1.0
+    , hashable             >=1.3.5 && <1.5
     , hstream-common-base
     , primitive            ^>=0.7
     , text
-    , vector               >=0.12   && <0.14
+    , vector               >=0.12  && <0.14
     , vector-algorithms    ^>=0.9
     , Z-Data