f

Signed-off-by: Lloyd-Pottiger <[email protected]>

dbms/src/Storages/DeltaMerge/BitmapFilter/BitmapFilter.cpp

@@ -15,6 +15,7 @@
 #include <Storages/DeltaMerge/BitmapFilter/BitmapFilter.h>
 #include <Storages/DeltaMerge/DeltaMergeHelpers.h>
 #include <Storages/DeltaMerge/Segment.h>
+#include <functional>
 
 namespace DB::DM
 {
@@ -115,10 +116,29 @@ void BitmapFilter::rangeAnd(BitmapFilterPtr & f) const
 {
     if (!all_match)
     {
-        std::transform(filter.cbegin(), filter.cend(), f->filter.cbegin(), f->filter.begin(), [](const bool a, const bool b) { return a && b; });
+        std::transform(filter.cbegin(), filter.cend(), f->filter.cbegin(), f->filter.begin(), std::logical_and<>());
     }
 }
 
+BitmapFilter& BitmapFilter::operator|=(const BitmapFilter& b)
+{
+    if (all_match)
+    {
+        return *this;
+    }
+    else if (b.all_match)
+    {
+        all_match = true;
+        std::fill(filter.begin(), filter.end(), true);
+    }
+    else
+    {
+        std::transform(filter.cbegin(), filter.cend(), b.filter.cbegin(), filter.begin(), std::logical_or<>());
+        runOptimize();
+    }
+    return *this;
+}
+
 void BitmapFilter::runOptimize()
 {
     all_match = std::find(filter.begin(), filter.end(), false) == filter.end();

dbms/src/Storages/DeltaMerge/BitmapFilter/BitmapFilter.h

@@ -35,9 +35,12 @@ class BitmapFilter
     void set(IColumn::Filter & f, UInt32 start, UInt32 limit);
     // If return true, all data is match and do not fill the filter.
     bool get(IColumn::Filter & f, UInt32 start, UInt32 limit) const;
-    // filter[start, limit] & f -> f
+    // filter[start, start + limit) & f -> f
     void rangeAnd(IColumn::Filter & f, UInt32 start, UInt32 limit) const;
+    // filter & f -> f
     void rangeAnd(BitmapFilterPtr & f) const;
+    // filter | f -> filter
+    BitmapFilter& operator|=(const BitmapFilter& b);
 
     void runOptimize();
 

dbms/src/Storages/DeltaMerge/FilterExpressionCache.cpp

@@ -20,7 +20,7 @@
 namespace DB::DM
 {
 
-std::optional<BitmapFilterPtr> FilterExpressionCache::get(const std::string & filter_expression) const
+std::optional<FilterExpressionCache::Value> FilterExpressionCache::get(const Key & filter_expression) const
 {
     std::shared_lock lock(rw_mutex);
 
@@ -32,16 +32,18 @@ std::optional<BitmapFilterPtr> FilterExpressionCache::get(const std::string & fi
     return it->second->second;
 }
 
-void FilterExpressionCache::set(const std::string & filter_expression, const BitmapFilterPtr & result)
+void FilterExpressionCache::set(const Key & filter_expression, const Value & result)
 {
-    if (result->size() == 0)
+    if (result.second->size() == 0)
         return;
     std::unique_lock lock(rw_mutex);
     auto it = map.find(filter_expression);
     if (it != map.end())
     {
         list.splice(list.begin(), list, it->second);
-        it->second->second = result;
+        auto & [use_packs, bitmap_filter] = it->second->second;
+        use_packs |= result.first;
+        *bitmap_filter |= *result.second;
         return;
     }
 

dbms/src/Storages/DeltaMerge/FilterExpressionCache.h

@@ -16,9 +16,9 @@
 
 #include <Storages/DeltaMerge/BitmapFilter/BitmapFilter.h>
 
+#include <boost/dynamic_bitset.hpp>
 #include <list>
 #include <shared_mutex>
-#include <string>
 
 // TODO: make it configurable.
 #define DefaultFilterExpressionCacheCapacity 100
@@ -34,17 +34,20 @@ namespace DB::DM
 class FilterExpressionCache
 {
 public:
+    using Key = std::string;
+    using Value = std::pair<boost::dynamic_bitset<>, BitmapFilterPtr>;
+
     explicit FilterExpressionCache(size_t capacity = DefaultFilterExpressionCacheCapacity)
         : capacity(capacity)
     {}
 
     ~FilterExpressionCache() = default;
 
     // Get the result of filter expression from cache.
-    std::optional<BitmapFilterPtr> get(const std::string & filter_expression) const;
+    std::optional<Value> get(const Key & filter_expression) const;
 
     // Set the result of filter expression to cache.
-    void set(const std::string & filter_expression, const BitmapFilterPtr & result);
+    void set(const Key & filter_expression, const Value & result);
 
     // Clear the cache.
     void clear();
@@ -60,10 +63,11 @@ class FilterExpressionCache
 
     // The list to store the filter expression.
     // The most recently used item is at the front of the list.
-    mutable std::list<std::pair<std::string, BitmapFilterPtr>> list;
+    // filter_expression -> <use_packs, bitmap_filter>
+    mutable std::list<std::pair<Key, Value>> list;
 
     // The map to store the filter expression and its result.
-    std::unordered_map<std::string, std::list<std::pair<std::string, BitmapFilterPtr>>::iterator> map;
+    std::unordered_map<Key, std::list<std::pair<Key, Value>>::iterator> map;
 };
 
 } // namespace DB::DM

dbms/src/Storages/DeltaMerge/Segment.cpp

@@ -2908,11 +2908,11 @@ BlockInputStreamPtr Segment::getBitmapFilterInputStream(const DMContext & dm_con
                 max_version,
                 expected_block_size);
         }
-        LOG_DEBUG(log, "Cache hit for filter expression: {}, stable rows: {}, skipped rows: {}, cache size: {}", filter_expression, cache_result->get()->size(), cache_result->get()->size() - cache_result->get()->count(), filter_expression_cache.size());
+        LOG_DEBUG(log, "Cache hit for filter expression: {}, stable rows: {}, skipped rows: {}, cache size: {}", filter_expression, cache_result->second->size(), cache_result->second->size() - cache_result->second->count(), filter_expression_cache.size());
         // The size of the cache should be equal to the number of rows in the stable layer of the segment
-        RUNTIME_CHECK(cache_result->get()->size() == segment_snap->stable->getDMFilesRows());
+        RUNTIME_CHECK(cache_result->second->size() == segment_snap->stable->getDMFilesRows());
         // cache_result & bitmap_filter[0:stable_rows] = bitmap_filter[0:stable_rows]
-        cache_result->get()->rangeAnd(bitmap_filter);
+        cache_result->second->rangeAnd(bitmap_filter);
         // TODO: call runOptimize twice here, should be optimized
         bitmap_filter->runOptimize();
     }