Skip to content

Commit

Permalink
datastore: HistoryRangeByKeys and DomainRangeAsOf (#2758)
Browse files Browse the repository at this point in the history
extras:
* rename HistoryRangeQuery to HistoryRangeInPeriodQuery
* fix HistoryRangeInPeriodQuery: merge by key instead of join
  • Loading branch information
battlmonstr authored Mar 6, 2025
2 parents 9912e31 + a9bd934 commit c12dd00
Show file tree
Hide file tree
Showing 29 changed files with 1,163 additions and 150 deletions.
137 changes: 137 additions & 0 deletions silkworm/db/datastore/common/ranges/if_view.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
/*
Copyright 2025 The Silkworm Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#pragma once

#include <cstdlib>
#include <iterator>
#include <optional>
#include <ranges>
#include <type_traits>
#include <utility>

#include <silkworm/core/common/assert.hpp>

namespace silkworm::views {

template <std::ranges::input_range Range1, std::ranges::input_range Range2>
class IfView : public std::ranges::view_interface<IfView<Range1, Range2>> {
public:
class Iterator {
public:
using Range1Iterator = std::ranges::iterator_t<Range1>;
using Range1Sentinel = std::ranges::sentinel_t<Range1>;
using Range1ReferenceType = std::iter_reference_t<Range1Iterator>;
using Range2Iterator = std::ranges::iterator_t<Range2>;
using Range2Sentinel = std::ranges::sentinel_t<Range2>;
using Range2ReferenceType = std::iter_reference_t<Range2Iterator>;
using DereferenceType = std::conditional_t<!std::is_reference_v<Range1ReferenceType>, Range1ReferenceType, Range2ReferenceType>;

using value_type = std::iter_value_t<Range1Iterator>;
using iterator_category [[maybe_unused]] = std::input_iterator_tag;
using difference_type = std::iter_difference_t<Range1Iterator>;
using reference = DereferenceType;
using pointer = std::remove_reference_t<reference>*;

Iterator() = default;
Iterator(
std::optional<Range1Iterator> it1,
std::optional<Range1Sentinel> sentinel1,
std::optional<Range2Iterator> it2,
std::optional<Range2Sentinel> sentinel2)
: it1_{std::move(it1)},
sentinel1_{std::move(sentinel1)},
it2_{std::move(it2)},
sentinel2_{std::move(sentinel2)} {}

reference operator*() const {
if (it1_) return **it1_;
if (it2_) return **it2_;
SILKWORM_ASSERT(false);
std::abort();
}

Iterator operator++(int) { return std::exchange(*this, ++Iterator{*this}); }
Iterator& operator++() {
if (it1_) ++(*it1_);
if (it2_) ++(*it2_);
return *this;
}

friend bool operator==(const Iterator& it, const std::default_sentinel_t&) {
return (it.it1_ && (*it.it1_ == *it.sentinel1_)) ||
(it.it2_ && (*it.it2_ == *it.sentinel2_));
}
friend bool operator!=(const Iterator& it, const std::default_sentinel_t& s) {
return !(it == s);
}
friend bool operator==(const std::default_sentinel_t& s, const Iterator& it) {
return it == s;
}
friend bool operator!=(const std::default_sentinel_t& s, const Iterator& it) {
return !(it == s);
}

private:
std::optional<Range1Iterator> it1_;
std::optional<Range1Sentinel> sentinel1_;
std::optional<Range2Iterator> it2_;
std::optional<Range2Sentinel> sentinel2_;
};

static_assert(std::input_iterator<Iterator>);

IfView(bool cond, Range1 range1, Range2 range2)
: cond_{cond},
range1_{std::move(range1)},
range2_{std::move(range2)} {}
IfView() = default;

IfView(IfView&&) = default;
IfView& operator=(IfView&&) noexcept = default;

Iterator begin() {
if (cond_) {
return Iterator{
std::ranges::begin(range1_),
std::ranges::end(range1_),
std::nullopt,
std::nullopt,
};
} else {
return Iterator{
std::nullopt,
std::nullopt,
std::ranges::begin(range2_),
std::ranges::end(range2_),
};
}
}

std::default_sentinel_t end() const { return std::default_sentinel; }

private:
bool cond_{false};
Range1 range1_;
Range2 range2_;
};

template <class Range1, class Range2>
IfView<Range1, Range2> if_view(bool cond, Range1&& v1, Range2&& v2) {
return IfView<Range1, Range2>{cond, std::forward<Range1>(v1), std::forward<Range2>(v2)};
}

} // namespace silkworm::views
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

namespace silkworm::views {

struct MergeUniqueCompareFunc {
struct MergeCompareFunc {
template <typename T>
constexpr std::strong_ordering operator()(const T& lhs, const T& rhs) const noexcept {
return std::compare_strong_order_fallback(lhs, rhs);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,17 @@

#include <silkworm/core/common/assert.hpp>

#include "merge_unique_compare_func.hpp"
#include "merge_compare_func.hpp"

namespace silkworm::views {

template <
std::ranges::input_range Ranges,
std::ranges::input_range Range = std::iter_value_t<std::ranges::iterator_t<Ranges>>,
class Comp = MergeUniqueCompareFunc,
class Proj = std::identity>
class MergeUniqueManyView : public std::ranges::view_interface<MergeUniqueManyView<Range, Ranges, Comp, Proj>> {
class Comp = MergeCompareFunc,
class Proj = std::identity,
bool kUnique = false>
class MergeManyView : public std::ranges::view_interface<MergeManyView<Range, Ranges, Comp, Proj, kUnique>> {
public:
class Iterator {
public:
Expand Down Expand Up @@ -82,22 +83,18 @@ class MergeUniqueManyView : public std::ranges::view_interface<MergeUniqueManyVi

Iterator operator++(int) { return std::exchange(*this, ++Iterator{*this}); }
Iterator& operator++() {
if constexpr (!kUnique) {
next();
return *this;
}

// backup the current key for duplicate detection
auto current_key = std::invoke(proj_, **this);

// first iteration: increment the current iterator once and restore the order
// next iterations: skip duplicate keys and restore the order
do {
size_t current = order_.front();
++iterators_[current];

std::ranges::pop_heap(order_, order_compare_func());
order_.pop_back();

if (!it_ended(current)) {
order_.push_back(current);
std::ranges::push_heap(order_, order_compare_func());
}
next();
} while (
!order_.empty() &&
std::is_eq(std::invoke(*comp_, std::invoke(proj_, *iterators_[order_.front()]), current_key)));
Expand All @@ -119,6 +116,20 @@ class MergeUniqueManyView : public std::ranges::view_interface<MergeUniqueManyVi
}

private:
//! Increment the current iterator once and restore the order
void next() {
size_t current = order_.front();
++iterators_[current];

std::ranges::pop_heap(order_, order_compare_func());
order_.pop_back();

if (!it_ended(current)) {
order_.push_back(current);
std::ranges::push_heap(order_, order_compare_func());
}
}

bool it_ended(size_t i) const {
return iterators_[i] == sentinels_[i];
}
Expand Down Expand Up @@ -152,16 +163,16 @@ class MergeUniqueManyView : public std::ranges::view_interface<MergeUniqueManyVi

static_assert(std::input_iterator<Iterator>);

MergeUniqueManyView(
MergeManyView(
Ranges ranges,
Comp comp, Proj proj)
: ranges_{std::move(ranges)},
comp_{std::move(comp)},
proj_{std::move(proj)} {}
MergeUniqueManyView() = default;
MergeManyView() = default;

MergeUniqueManyView(MergeUniqueManyView&&) = default;
MergeUniqueManyView& operator=(MergeUniqueManyView&&) noexcept = default;
MergeManyView(MergeManyView&&) = default;
MergeManyView& operator=(MergeManyView&&) noexcept = default;

Iterator begin() { return Iterator{ranges_, &comp_, proj_}; }
std::default_sentinel_t end() const { return std::default_sentinel; }
Expand All @@ -175,12 +186,27 @@ class MergeUniqueManyView : public std::ranges::view_interface<MergeUniqueManyVi
template <
class Ranges,
class Range = std::iter_value_t<std::ranges::iterator_t<Ranges>>,
class Comp = MergeUniqueCompareFunc,
class Comp = MergeCompareFunc,
class Proj = std::identity>
MergeManyView<Ranges, Range, Comp, Proj> merge_many(
Ranges&& ranges,
Comp comp = {}, Proj proj = {}) {
return MergeManyView<Ranges, Range, Comp, Proj>{
std::forward<Ranges>(ranges),
std::move(comp),
std::move(proj),
};
}

template <
class Ranges,
class Range = std::iter_value_t<std::ranges::iterator_t<Ranges>>,
class Comp = MergeCompareFunc,
class Proj = std::identity>
MergeUniqueManyView<Ranges, Range, Comp, Proj> merge_unique_many(
MergeManyView<Ranges, Range, Comp, Proj, /* kUnique = */ true> merge_unique_many(
Ranges&& ranges,
Comp comp = {}, Proj proj = {}) {
return MergeUniqueManyView<Ranges, Range, Comp, Proj>{
return MergeManyView<Ranges, Range, Comp, Proj, /* kUnique = */ true>{
std::forward<Ranges>(ranges),
std::move(comp),
std::move(proj),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
limitations under the License.
*/

#include "merge_unique_many_view.hpp"
#include "merge_many_view.hpp"

#include <catch2/catch_test_macros.hpp>

Expand All @@ -23,8 +23,8 @@

namespace silkworm::views {

static_assert(std::ranges::input_range<MergeUniqueManyView<std::vector<std::vector<int>>>>);
static_assert(std::ranges::view<MergeUniqueManyView<std::vector<std::vector<int>>>>);
static_assert(std::ranges::input_range<MergeManyView<std::vector<std::vector<int>>>>);
static_assert(std::ranges::view<MergeManyView<std::vector<std::vector<int>>>>);

template <std::ranges::input_range TRange>
std::vector<TRange> ranges(TRange r1, TRange r2) {
Expand All @@ -37,6 +37,50 @@ std::vector<TRange> ranges(TRange r1, TRange r2) {
// Skip to avoid Windows error C3889: call to object of class type 'std::ranges::_Begin::_Cpo': no matching call operator found
// Unable to reproduce: https://godbolt.org/z/3jd5brKMj
#ifndef _WIN32
TEST_CASE("MergeManyView") {
CHECK(vector_from_range(merge_many(ranges(
silkworm::ranges::owning_view(std::vector<int>{1, 2, 3}),
silkworm::ranges::owning_view(std::vector<int>{2, 3, 4})))) ==
std::vector<int>{1, 2, 2, 3, 3, 4});

CHECK(vector_from_range(merge_many(ranges(
silkworm::ranges::owning_view(std::vector<int>{1, 2, 3, 4, 5}),
silkworm::ranges::owning_view(std::vector<int>{3, 4, 5, 6, 7})))) ==
std::vector<int>{1, 2, 3, 3, 4, 4, 5, 5, 6, 7});

CHECK(vector_from_range(merge_many(ranges(
silkworm::ranges::owning_view(std::vector<int>{1, 2, 3, 4, 5, 5, 5}),
silkworm::ranges::owning_view(std::vector<int>{3, 4, 5, 6, 7})))) ==
std::vector<int>{1, 2, 3, 3, 4, 4, 5, 5, 5, 5, 6, 7});

CHECK(vector_from_range(merge_many(ranges(
silkworm::ranges::owning_view(std::vector<int>{0, 2, 2, 2, 4, 5, 6, 6, 7, 7}),
silkworm::ranges::owning_view(std::vector<int>{0, 0, 1, 2, 3, 5, 5, 6, 8, 9})))) ==
std::vector<int>{0, 0, 0, 1, 2, 2, 2, 2, 3, 4, 5, 5, 5, 6, 6, 6, 7, 7, 8, 9});

using IntPredicate = std::function<bool(int)>;
IntPredicate even = [](int x) { return x % 2 == 0; };
IntPredicate odd = [](int x) { return x % 2 == 1; };
CHECK(vector_from_range(merge_many(ranges(
silkworm::ranges::owning_view(std::vector<int>{1, 2, 3}) | std::views::filter(even),
silkworm::ranges::owning_view(std::vector<int>{2, 3, 4}) | std::views::filter(odd)))) ==
std::vector<int>{2, 3});
CHECK(vector_from_range(merge_many(ranges(
silkworm::ranges::owning_view(std::vector<int>{1, 2, 3}) | std::views::filter(odd),
silkworm::ranges::owning_view(std::vector<int>{2, 3, 4}) | std::views::filter(even)))) ==
std::vector<int>{1, 2, 3, 4});

CHECK(vector_from_range(merge_many(ranges(std::vector<int>{}, std::vector<int>{}))).empty());
CHECK(vector_from_range(merge_many(ranges(silkworm::ranges::owning_view(std::vector<int>{1, 2, 3}), silkworm::ranges::owning_view(std::vector<int>{})))) == std::vector<int>{1, 2, 3});
CHECK(vector_from_range(merge_many(ranges(silkworm::ranges::owning_view(std::vector<int>{}), silkworm::ranges::owning_view(std::vector<int>{2, 3, 4})))) == std::vector<int>{2, 3, 4});

using IntToVectorFunc = std::function<std::vector<int>(int)>;
CHECK(vector_from_range(merge_many(ranges(
silkworm::ranges::owning_view(std::vector<int>{1, 2, 3}) | std::views::transform(IntToVectorFunc{[](int v) { return std::vector<int>{v, v, v}; }}) | std::views::join,
silkworm::ranges::owning_view(std::vector<int>{4, 4, 4}) | std::views::transform(IntToVectorFunc{[](int v) { return std::vector<int>{v}; }}) | std::views::join))) ==
std::vector<int>{1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4});
}

TEST_CASE("MergeUniqueManyView") {
CHECK(vector_from_range(merge_unique_many(ranges(
silkworm::ranges::owning_view(std::vector<int>{1, 2, 3}),
Expand Down
6 changes: 3 additions & 3 deletions silkworm/db/datastore/common/ranges/merge_unique_view.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@

#include <silkworm/core/common/assert.hpp>

#include "merge_unique_compare_func.hpp"
#include "merge_compare_func.hpp"

namespace silkworm::views {

template <
std::ranges::input_range Range1, std::ranges::input_range Range2,
class Comp = MergeUniqueCompareFunc,
class Comp = MergeCompareFunc,
class Proj1 = std::identity, class Proj2 = std::identity>
class MergeUniqueView : public std::ranges::view_interface<MergeUniqueView<Range1, Range2, Comp, Proj1, Proj2>> {
public:
Expand Down Expand Up @@ -176,7 +176,7 @@ class MergeUniqueView : public std::ranges::view_interface<MergeUniqueView<Range

template <
class Range1, class Range2,
class Comp = MergeUniqueCompareFunc,
class Comp = MergeCompareFunc,
class Proj1 = std::identity, class Proj2 = std::identity>
MergeUniqueView<Range1, Range2, Comp, Proj1, Proj2> merge_unique(
Range1&& v1, Range2&& v2,
Expand Down
6 changes: 3 additions & 3 deletions silkworm/db/datastore/common/ranges/unique_view.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ namespace silkworm::views {

template <
std::ranges::input_range TRange,
class Comp = MergeUniqueCompareFunc,
class Comp = MergeCompareFunc,
class Proj = std::identity>
using UniqueView = MergeUniqueView<TRange, std::ranges::empty_view<std::iter_value_t<std::ranges::iterator_t<TRange>>>, Comp, Proj, Proj>;

template <class Comp = MergeUniqueCompareFunc, class Proj = std::identity>
template <class Comp = MergeCompareFunc, class Proj = std::identity>
struct UniqueViewFactory {
template <class TRange>
constexpr UniqueView<TRange, Comp, Proj> operator()(
Expand All @@ -41,7 +41,7 @@ struct UniqueViewFactory {
}
};

template <class Comp = MergeUniqueCompareFunc, class Proj = std::identity>
template <class Comp = MergeCompareFunc, class Proj = std::identity>
inline constexpr UniqueViewFactory<Comp, Proj> unique;

} // namespace silkworm::views
Loading

0 comments on commit c12dd00

Please sign in to comment.