diff --git a/CMakeLists.txt b/CMakeLists.txt index df3d3831e..68580f3d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,7 @@ +cmake_minimum_required(VERSION 3.10..3.13) + project(reindexer) -cmake_minimum_required(VERSION 3.10) + enable_testing() set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/bindings/consts.go b/bindings/consts.go index 9e14918ec..50ff9b1ed 100644 --- a/bindings/consts.go +++ b/bindings/consts.go @@ -2,7 +2,7 @@ package bindings const CInt32Max = int(^uint32(0) >> 1) -const ReindexerVersion = "v3.28.0" +const ReindexerVersion = "v3.28.1" // public go consts from type_consts.h and reindexer_ctypes.h const ( diff --git a/changelog.md b/changelog.md index cb7ae41e0..e7a7400f7 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,18 @@ +# Version 3.28.1 (05.09.2024) +## Core +- [fix] Fixed pagination for `MERGE` fulltext queries. Previously those queries could return duplicates on different pages due to missing ordering guarantees +- [fix] Fixed fields filters serialization for `DISTINCT` aggregations (affects SQL logging only) +- [fix] Temporary disabled default values logic for indexed field from v3.27.0 - this logic may cause stability issues and will be reworked in further releases +- [fix] Add extra check for composites indexes substitution +- [fix] Fix composites substitution after indexes update + +## Reindexer tool +- [fix] Fixed crash in `\quit`-command after previous errors +- [fix] Fixed crash in `git bash` on `Windows` platforms + +## Face +- [fix] Fixed XSS vulnerability in table view + # Version 3.28.0 (16.08.2024) ## Core - [fea] Updated [logging library](https://github.com/gabime/spdlog) to v1.14.1 and [formatting library](https://github.com/fmtlib/fmt) to v11.0.2 diff --git a/cjson/decoder.go b/cjson/decoder.go index 0383a9acc..317942572 100644 --- a/cjson/decoder.go +++ b/cjson/decoder.go @@ -313,6 +313,7 @@ func (dec *Decoder) decodeSlice(pl *payloadIface, rdser *Serializer, v *reflect. var ptr unsafe.Pointer k := v.Kind() + offset := 0 switch k { case reflect.Slice: @@ -331,7 +332,11 @@ func (dec *Decoder) decodeSlice(pl *payloadIface, rdser *Serializer, v *reflect. // offset is 0 // No concatenation for the fixed size arrays default: - panic(fmt.Errorf("can not convert '%s' to 'array'", v.Type().Kind().String())) + if count == 0 { // Allows empty slice for any scalar type (using default value) + return + } else { + panic(fmt.Errorf("can not convert '%s' to 'array'", v.Type().Kind().String())) + } } if subtag != TAG_OBJECT { @@ -583,6 +588,7 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V ctagName := ctag.Name() k := v.Kind() + initialV := v if k == reflect.Ptr { if v.IsNil() { v.Set(reflect.New(v.Type().Elem())) @@ -641,6 +647,7 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V } else { panic(fmt.Errorf("err: intf=%s, name='%s' %s", v.Type().Name(), dec.state.tagsMatcher.tag2name(ctagName), ctag.Dump())) } + initialV = v k = v.Kind() if k == reflect.Ptr { if v.IsNil() { @@ -659,8 +666,12 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V switch ctagType { case TAG_ARRAY: count := int(rdser.GetVarUInt()) - pl.getArray(int(ctagField), *cnt, count, v) - *cnt += count + if k == reflect.Slice || k == reflect.Array || count != 0 { // Allows empty slice for any scalar type (using default value) + pl.getArray(int(ctagField), *cnt, count, v) + *cnt += count + } else { + initialV.Set(reflect.Zero(initialV.Type())) // Set nil to scalar pointers, intialized with empty arrays + } default: pl.getValue(int(ctagField), *cnt, v) (*cnt)++ @@ -670,6 +681,9 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V switch ctagType { case TAG_ARRAY: dec.decodeSlice(pl, rdser, &v, fieldsoutcnt, cctagsPath) + if v.Kind() != reflect.Array && v.Kind() != reflect.Slice && v.Kind() != reflect.Interface { + initialV.Set(reflect.Zero(initialV.Type())) // Set nil to scalar pointers, intialized with empty arrays + } case TAG_OBJECT: for dec.decodeValue(pl, rdser, v, fieldsoutcnt, cctagsPath) { } diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt index c13ea8244..ccca77433 100644 --- a/cpp_src/CMakeLists.txt +++ b/cpp_src/CMakeLists.txt @@ -44,7 +44,7 @@ else() option(LINK_RESOURCES "Link web resources as binary data" ON) endif() -set(REINDEXER_VERSION_DEFAULT "3.28.0") +set(REINDEXER_VERSION_DEFAULT "3.28.1") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo") @@ -291,7 +291,11 @@ else() endif() # Static LevelDB v1.23 is built with -fno-rtti by default. To inherit our logger from leveldb's logger, this file must be built with -fno-rtti to -set_source_files_properties (${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "-fno-rtti") +if(MSVC) + set_source_files_properties (${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "/GR-") +else() + set_source_files_properties (${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "-fno-rtti") +endif() list(APPEND REINDEXER_LIBRARIES reindexer) add_library(${TARGET} STATIC ${HDRS} ${SRCS} ${VENDORS}) @@ -348,6 +352,7 @@ else() GIT_TAG "1.1.8" CMAKE_ARGS -DSNAPPY_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR} -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON ) include_directories(${CMAKE_CURRENT_BINARY_DIR}/include) link_directories(${CMAKE_CURRENT_BINARY_DIR}) @@ -434,9 +439,10 @@ if(NOT LevelDB_LIBRARY OR NOT LevelDB_INCLUDE_DIR OR WITH_TSAN) GIT_TAG "master" CMAKE_ARGS -DLEVELDB_BUILD_TESTS=OFF -DLEVELDB_BUILD_BENCHMARKS=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR} - "-DCMAKE_CXX_FLAGS=-fPIC -I${CMAKE_CURRENT_BINARY_DIR}/include" + "-DCMAKE_CXX_FLAGS=-I${CMAKE_CURRENT_BINARY_DIR}/include" -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR} -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON ) if(NOT SNAPPY_FOUND) add_dependencies(leveldb_lib snappy_lib) diff --git a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc index 0802cf656..085741e3c 100644 --- a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc @@ -56,11 +56,14 @@ Error CommandsExecutor::GetSuggestions(const std::string& input, st template Error CommandsExecutor::Stop() { - GenericCommand cmd([this] { - stop(true); - return Error{}; - }); - auto err = execCommand(cmd); + Error err; + if (GetStatus().running) { + GenericCommand cmd([this] { + stop(true); + return Error{}; + }); + err = execCommand(cmd); + } if (err.ok() && executorThr_.joinable()) { executorThr_.join(); } diff --git a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc index fb435a0de..3e1845f7b 100644 --- a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc @@ -170,10 +170,7 @@ bool CommandsProcessor::Run(const std::string& command) { template Error CommandsProcessor::stop() { - if (executor_.GetStatus().running) { - return executor_.Stop(); - } - return Error(); + return executor_.Stop(); } template class CommandsProcessor; diff --git a/cpp_src/core/ft/ft_fast/selecter.cc b/cpp_src/core/ft/ft_fast/selecter.cc index c02d82ed7..e0bef67a1 100644 --- a/cpp_src/core/ft/ft_fast/selecter.cc +++ b/cpp_src/core/ft/ft_fast/selecter.cc @@ -120,8 +120,8 @@ void Selecter::prepareVariants(std::vector& variants, RV // RX_NO_INLINE just for build test purpose. Do not expect any effect here template template -RX_NO_INLINE MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, - const RdxContext& rdxCtx) { +RX_NO_INLINE MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, + FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) { FtSelectContext ctx; ctx.rawResults.reserve(dsl.size()); // STEP 2: Search dsl terms for each variant @@ -213,11 +213,11 @@ RX_NO_INLINE MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransa const auto maxMergedSize = std::min(size_t(holder_.cfg_->mergeLimit), ctx.totalORVids); if (maxMergedSize < 0xFFFF) { - return mergeResultsBmType(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, std::move(mergeStatuses), - rdxCtx); + return mergeResultsBmType(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, ftSortType, + std::move(mergeStatuses), rdxCtx); } else if (maxMergedSize < 0xFFFFFFFF) { - return mergeResultsBmType(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, std::move(mergeStatuses), - rdxCtx); + return mergeResultsBmType(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, ftSortType, + std::move(mergeStatuses), rdxCtx); } else { assertrx_throw(false); } @@ -227,17 +227,17 @@ RX_NO_INLINE MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransa template template MergeData Selecter::mergeResultsBmType(std::vector&& results, size_t totalORVids, - const std::vector& synonymsBounds, bool inTransaction, + const std::vector& synonymsBounds, bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) { switch (holder_.cfg_->bm25Config.bm25Type) { case FtFastConfig::Bm25Config::Bm25Type::rx: - return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, + return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, ftSortType, std::move(mergeStatuses), rdxCtx); case FtFastConfig::Bm25Config::Bm25Type::classic: - return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, + return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, ftSortType, std::move(mergeStatuses), rdxCtx); case FtFastConfig::Bm25Config::Bm25Type::wordCount: - return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, + return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, ftSortType, std::move(mergeStatuses), rdxCtx); } assertrx_throw(false); @@ -1284,7 +1284,7 @@ bool Selecter::TyposHandler::isWordFitMaxLettPerm(const std::string_view template template MergeData Selecter::mergeResults(std::vector&& rawResults, size_t maxMergedSize, - const std::vector& synonymsBounds, bool inTransaction, + const std::vector& synonymsBounds, bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) { const auto& vdocs = holder_.vdocs_; MergeData merged; @@ -1388,9 +1388,19 @@ MergeData Selecter::mergeResults(std::vector&& rawRes merged.maxRank = m.proc; } } - - boost::sort::pdqsort_branchless(merged.begin(), merged.end(), - [](const MergeInfo& lhs, const MergeInfo& rhs) noexcept { return lhs.proc > rhs.proc; }); + switch (ftSortType) { + case FtSortType::RankOnly: { + boost::sort::pdqsort_branchless(merged.begin(), merged.end(), + [](const MergeInfo& lhs, const MergeInfo& rhs) noexcept { return lhs.proc > rhs.proc; }); + return merged; + } + case FtSortType::RankAndID: { + return merged; + } + case FtSortType::ExternalExpression: + throw Error(errLogic, "FtSortType::ExternalExpression not implemented."); + break; + } return merged; } @@ -1431,13 +1441,14 @@ void Selecter::printVariants(const FtSelectContext& ctx, const TextSearc } template class Selecter; -template MergeData Selecter::Process(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&, +template MergeData Selecter::Process(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&, const RdxContext&); -template MergeData Selecter::Process(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&, +template MergeData Selecter::Process(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&, const RdxContext&); template class Selecter; -template MergeData Selecter::Process(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&, const RdxContext&); -template MergeData Selecter::Process(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&, +template MergeData Selecter::Process(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&, + const RdxContext&); +template MergeData Selecter::Process(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&, const RdxContext&); } // namespace reindexer diff --git a/cpp_src/core/ft/ft_fast/selecter.h b/cpp_src/core/ft/ft_fast/selecter.h index 9cb546d53..0e159f484 100644 --- a/cpp_src/core/ft/ft_fast/selecter.h +++ b/cpp_src/core/ft/ft_fast/selecter.h @@ -1,6 +1,7 @@ #pragma once #include "core/ft/ftdsl.h" #include "core/ft/idrelset.h" +#include "core/selectfunc/ctx/ftctx.h" #include "dataholder.h" namespace reindexer { @@ -53,7 +54,8 @@ class Selecter { }; template - MergeData Process(FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext&); + MergeData Process(FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, + const RdxContext&); private: struct TextSearchResult { @@ -178,7 +180,7 @@ class Selecter { template MergeData mergeResults(std::vector&& rawResults, size_t maxMergedSize, const std::vector& synonymsBounds, - bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext&); + bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext&); template void mergeIteration(TextSearchResults& rawRes, index_t rawResIndex, FtMergeStatuses::Statuses& mergeStatuses, MergeData& merged, @@ -244,7 +246,8 @@ class Selecter { template MergeData mergeResultsBmType(std::vector&& results, size_t totalORVids, const std::vector& synonymsBounds, - bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx); + bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, + const RdxContext& rdxCtx); void debugMergeStep(const char* msg, int vid, float normBm25, float normDist, int finalRank, int prevRank); template diff --git a/cpp_src/core/idset.h b/cpp_src/core/idset.h index af1b7d44b..d6235ac90 100644 --- a/cpp_src/core/idset.h +++ b/cpp_src/core/idset.h @@ -35,6 +35,7 @@ class IdSetPlain : protected base_idset { using base_idset::rend; using base_idset::const_reverse_iterator; using base_idset::const_iterator; + using base_idset::operator[]; enum EditMode { Ordered, // Keep idset ordered, and ready to select (insert is slow O(logN)+O(N)) diff --git a/cpp_src/core/index/index.cc b/cpp_src/core/index/index.cc index c7b8a6151..c2fa3a9a9 100644 --- a/cpp_src/core/index/index.cc +++ b/cpp_src/core/index/index.cc @@ -86,7 +86,7 @@ void Index::dump(S& os, std::string_view step, std::string_view offset) const { payloadType_.Dump(os, step, newOffset); if (IsComposite(type_)) { os << ",\n" << newOffset << "fields: "; - fields_.Dump(os); + fields_.Dump(os, FieldsSet::DumpWithMask::Yes); } os << ",\n" << newOffset << "sortedIdxCount: " << sortedIdxCount_ << ",\n" diff --git a/cpp_src/core/index/index.h b/cpp_src/core/index/index.h index 5b6aac3e5..7e4875c6e 100644 --- a/cpp_src/core/index/index.h +++ b/cpp_src/core/index/index.h @@ -34,7 +34,8 @@ class Index { forceComparator(0), unbuiltSortOrders(0), indexesNotOptimized(0), - inTransaction{0} {} + inTransaction{0}, + ftSortType(0) {} unsigned itemsCountInNamespace; int maxIterations; unsigned distinct : 1; @@ -43,6 +44,7 @@ class Index { unsigned unbuiltSortOrders : 1; unsigned indexesNotOptimized : 1; unsigned inTransaction : 1; + unsigned ftSortType : 2; }; using KeyEntry = reindexer::KeyEntry; using KeyEntryPlain = reindexer::KeyEntry; diff --git a/cpp_src/core/index/indextext/fastindextext.cc b/cpp_src/core/index/indextext/fastindextext.cc index 7e56292de..fdded3b1f 100644 --- a/cpp_src/core/index/indextext/fastindextext.cc +++ b/cpp_src/core/index/indextext/fastindextext.cc @@ -3,8 +3,8 @@ #include "core/ft/filters/kblayout.h" #include "core/ft/filters/synonyms.h" #include "core/ft/filters/translit.h" -#include "core/ft/ft_fast/selecter.h" #include "estl/contexted_locks.h" +#include "sort/pdqsort.hpp" #include "tools/clock.h" #include "tools/logger.h" @@ -124,10 +124,60 @@ IndexMemStat FastIndexText::GetMemStat(const RdxContext& ctx) { ret.idsetCache = this->cache_ft_ ? this->cache_ft_->GetMemStat() : LRUCacheMemStat(); return ret; } +template +MergeData::iterator FastIndexText::unstableRemoveIf(MergeData& md, int minRelevancy, double scalingFactor, size_t& releventDocs, + int& cnt) { + if (md.empty()) { + return md.begin(); + } + auto& holder = *this->holder_; + auto first = md.begin(); + auto last = md.end(); + while (true) { + while (true) { + if (first == last) { + return first; + } + auto& vdoc = holder.vdocs_[first->id]; + if (!vdoc.keyEntry) { + break; + } + first->proc *= scalingFactor; + if (first->proc < minRelevancy) { + break; + } + assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); + cnt += vdoc.keyEntry->Sorted(0).size(); + ++releventDocs; + + ++first; + } + while (true) { + --last; + if (first == last) { + return first; + } + if (!holder.vdocs_[last->id].keyEntry) { + continue; + } + last->proc *= scalingFactor; + if (last->proc >= minRelevancy) { + break; + } + } + auto& vdoc = holder.vdocs_[last->id]; + assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); + cnt += vdoc.keyEntry->Sorted(0).size(); + ++releventDocs; + + *first = std::move(*last); + ++first; + } +} template -IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&& statuses, - FtUseExternStatuses useExternSt, const RdxContext& rdxCtx) { +IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, + FtMergeStatuses&& statuses, FtUseExternStatuses useExternSt, const RdxContext& rdxCtx) { fctx->GetData()->extraWordSymbols_ = this->getConfig()->extraWordSymbols; fctx->GetData()->isWordPositions_ = true; @@ -138,9 +188,11 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTr assertrx_throw(d); Selecter selecter{*d, this->Fields().size(), fctx->NeedArea(), holder_->cfg_->maxAreasInDoc}; if (useExternSt == FtUseExternStatuses::No) { - mergeData = selecter.Process(std::move(dsl), inTransaction, std::move(statuses.statuses), rdxCtx); + mergeData = selecter.Process(std::move(dsl), inTransaction, ftSortType, + std::move(statuses.statuses), rdxCtx); } else { - mergeData = selecter.Process(std::move(dsl), inTransaction, std::move(statuses.statuses), rdxCtx); + mergeData = selecter.Process(std::move(dsl), inTransaction, ftSortType, + std::move(statuses.statuses), rdxCtx); } break; } @@ -149,9 +201,11 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTr assertrx_throw(d); Selecter selecter{*d, this->Fields().size(), fctx->NeedArea(), holder_->cfg_->maxAreasInDoc}; if (useExternSt == FtUseExternStatuses::No) { - mergeData = selecter.Process(std::move(dsl), inTransaction, std::move(statuses.statuses), rdxCtx); + mergeData = selecter.Process(std::move(dsl), inTransaction, ftSortType, + std::move(statuses.statuses), rdxCtx); } else { - mergeData = selecter.Process(std::move(dsl), inTransaction, std::move(statuses.statuses), rdxCtx); + mergeData = selecter.Process(std::move(dsl), inTransaction, ftSortType, + std::move(statuses.statuses), rdxCtx); } break; } @@ -170,19 +224,32 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTr const double scalingFactor = mergeData.maxRank > 255 ? 255.0 / mergeData.maxRank : 1.0; const int minRelevancy = getConfig()->minRelevancy * 100 * scalingFactor; size_t releventDocs = 0; - for (auto& vid : mergeData) { - auto& vdoc = holder.vdocs_[vid.id]; - if (!vdoc.keyEntry) { - continue; - } - vid.proc *= scalingFactor; - if (vid.proc <= minRelevancy) { + switch (ftSortType) { + case FtSortType::RankAndID: { + auto itF = unstableRemoveIf(mergeData, minRelevancy, scalingFactor, releventDocs, cnt); + mergeData.erase(itF, mergeData.end()); break; } + case FtSortType::RankOnly: { + for (auto& vid : mergeData) { + auto& vdoc = holder.vdocs_[vid.id]; + if (!vdoc.keyEntry) { + continue; + } + vid.proc *= scalingFactor; + if (vid.proc <= minRelevancy) { + break; + } - assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); - cnt += vdoc.keyEntry->Sorted(0).size(); - ++releventDocs; + assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); + cnt += vdoc.keyEntry->Sorted(0).size(); + ++releventDocs; + } + break; + } + case FtSortType::ExternalExpression: { + throw Error(errLogic, "FtSortType::ExternalExpression not implemented."); + } } mergedIds->reserve(cnt); @@ -238,6 +305,44 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTr logPrintf(LogInfo, "Relevancy(%d): %s", fctx->Size(), str); } assertrx_throw(mergedIds->size() == fctx->Size()); + if (ftSortType == FtSortType::RankAndID) { + std::vector sortIds; + size_t nItems = mergedIds->size(); + sortIds.reserve(mergedIds->size()); + for (size_t i = 0; i < nItems; i++) { + sortIds.emplace_back(i); + } + std::vector& proc = fctx->GetData()->proc_; + boost::sort::pdqsort(sortIds.begin(), sortIds.end(), [&proc, mergedIds](size_t i1, size_t i2) { + int p1 = proc[i1]; + int p2 = proc[i2]; + if (p1 > p2) { + return true; + } else if (p1 < p2) { + return false; + } else { + return (*mergedIds)[i1] < (*mergedIds)[i2]; + } + }); + + for (size_t i = 0; i < nItems; i++) { + auto vm = (*mergedIds)[i]; + auto vp = proc[i]; + size_t j = i; + while (true) { + size_t k = sortIds[j]; + sortIds[j] = j; + if (k == i) { + break; + } + (*mergedIds)[j] = (*mergedIds)[k]; + proc[j] = proc[k]; + j = k; + } + (*mergedIds)[j] = vm; + proc[j] = vp; + } + } return mergedIds; } template @@ -351,11 +456,12 @@ template template RX_ALWAYS_INLINE void FastIndexText::appendMergedIds(MergeData& mergeData, size_t releventDocs, F&& appender) { auto& holder = *this->holder_; - for (size_t i = 0; i < releventDocs; ++i) { + for (size_t i = 0; i < releventDocs;) { auto& vid = mergeData[i]; auto& vdoc = holder.vdocs_[vid.id]; if (vdoc.keyEntry) { appender(vdoc.keyEntry->Sorted(0).begin(), vdoc.keyEntry->Sorted(0).end(), vid); + i++; } } } diff --git a/cpp_src/core/index/indextext/fastindextext.h b/cpp_src/core/index/indextext/fastindextext.h index 054afa650..8e902d945 100644 --- a/cpp_src/core/index/indextext/fastindextext.h +++ b/cpp_src/core/index/indextext/fastindextext.h @@ -2,6 +2,7 @@ #include "core/ft/config/ftfastconfig.h" #include "core/ft/ft_fast/dataholder.h" +#include "core/ft/ft_fast/selecter.h" #include "indextext.h" namespace reindexer { @@ -31,8 +32,8 @@ class FastIndexText : public IndexText { // Creates uncommited copy return std::make_unique>(*this); } - IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, FtUseExternStatuses, - const RdxContext&) override final; + IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransactionbool, FtSortType ftSortType, FtMergeStatuses&&, + FtUseExternStatuses, const RdxContext&) override final; IndexMemStat GetMemStat(const RdxContext&) override final; Variant Upsert(const Variant& key, IdType id, bool& clearCache) override final; void Delete(const Variant& key, IdType id, StringsHolder&, bool& clearCache) override final; @@ -55,6 +56,8 @@ class FastIndexText : public IndexText { template void appendMergedIds(MergeData& merged, size_t releventDocs, F&& appender); + MergeData::iterator unstableRemoveIf(MergeData& md, const int minRelevancy, double scalingFactor, size_t& releventDocs, int& cnt); + std::unique_ptr holder_; }; diff --git a/cpp_src/core/index/indextext/fuzzyindextext.cc b/cpp_src/core/index/indextext/fuzzyindextext.cc index a3ea56d02..148db2e0c 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.cc +++ b/cpp_src/core/index/indextext/fuzzyindextext.cc @@ -5,9 +5,10 @@ namespace reindexer { template -IdSet::Ptr FuzzyIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, +IdSet::Ptr FuzzyIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, FtMergeStatuses&&, FtUseExternStatuses withExternSt, const RdxContext& rdxCtx) { assertrx_throw(withExternSt == FtUseExternStatuses::No); + (void)ftSortType; (void)withExternSt; auto result = engine_.Search(dsl, inTransaction, rdxCtx); diff --git a/cpp_src/core/index/indextext/fuzzyindextext.h b/cpp_src/core/index/indextext/fuzzyindextext.h index 0c5d90846..d92a0a210 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.h +++ b/cpp_src/core/index/indextext/fuzzyindextext.h @@ -23,7 +23,7 @@ class FuzzyIndexText : public IndexText { abort(); } std::unique_ptr Clone() const override final { return std::make_unique>(*this); } - IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, FtUseExternStatuses, + IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, FtMergeStatuses&&, FtUseExternStatuses, const RdxContext&) override final; Variant Upsert(const Variant& key, IdType id, bool& clearCache) override final { this->isBuilt_ = false; diff --git a/cpp_src/core/index/indextext/indextext.cc b/cpp_src/core/index/indextext/indextext.cc index 6ba1425ed..695456a7e 100644 --- a/cpp_src/core/index/indextext/indextext.cc +++ b/cpp_src/core/index/indextext/indextext.cc @@ -118,7 +118,7 @@ SelectKeyResults IndexText::SelectKey(const VariantArray& keys, CondType cond } } return doSelectKey(keys, needPutCache ? std::optional{std::move(ckey)} : std::nullopt, std::move(mergeStatuses), - FtUseExternStatuses::No, opts.inTransaction, std::move(ftctx), rdxCtx); + FtUseExternStatuses::No, opts.inTransaction, FtSortType(opts.ftSortType), std::move(ftctx), rdxCtx); } template @@ -135,7 +135,7 @@ SelectKeyResults IndexText::resultFromCache(const VariantArray& keys, FtIdSet template SelectKeyResults IndexText::doSelectKey(const VariantArray& keys, const std::optional& ckey, FtMergeStatuses&& mergeStatuses, FtUseExternStatuses useExternSt, bool inTransaction, - FtCtx::Ptr ftctx, const RdxContext& rdxCtx) { + FtSortType ftSortType, FtCtx::Ptr ftctx, const RdxContext& rdxCtx) { if rx_unlikely (cfg_->logLevel >= LogInfo) { logPrintf(LogInfo, "Searching for '%s' in '%s' %s", keys[0].As(), this->payloadType_ ? this->payloadType_->Name() : "", ckey ? "(will cache)" : ""); @@ -145,7 +145,7 @@ SelectKeyResults IndexText::doSelectKey(const VariantArray& keys, const std:: FtDSLQuery dsl(this->ftFields_, this->cfg_->stopWords, this->cfg_->extraWordSymbols); dsl.parse(keys[0].As()); - IdSet::Ptr mergedIds = Select(ftctx, std::move(dsl), inTransaction, std::move(mergeStatuses), useExternSt, rdxCtx); + IdSet::Ptr mergedIds = Select(ftctx, std::move(dsl), inTransaction, ftSortType, std::move(mergeStatuses), useExternSt, rdxCtx); SelectKeyResult res; if (mergedIds) { bool need_put = (useExternSt == FtUseExternStatuses::No) && ckey.has_value(); @@ -189,7 +189,8 @@ SelectKeyResults IndexText::SelectKey(const VariantArray& keys, CondType cond if rx_unlikely (keys.size() < 1 || (condition != CondEq && condition != CondSet)) { throw Error(errParams, "Full text index (%s) support only EQ or SET condition with 1 or 2 parameter", Index::Name()); } - return doSelectKey(keys, std::nullopt, std::move(preselect), FtUseExternStatuses::Yes, opts.inTransaction, prepareFtCtx(ctx), rdxCtx); + return doSelectKey(keys, std::nullopt, std::move(preselect), FtUseExternStatuses::Yes, opts.inTransaction, FtSortType(opts.ftSortType), + prepareFtCtx(ctx), rdxCtx); } template diff --git a/cpp_src/core/index/indextext/indextext.h b/cpp_src/core/index/indextext/indextext.h index 65f54d7b0..52b094da8 100644 --- a/cpp_src/core/index/indextext/indextext.h +++ b/cpp_src/core/index/indextext/indextext.h @@ -33,8 +33,8 @@ class IndexText : public IndexUnordered { SelectKeyResults SelectKey(const VariantArray& keys, CondType, Index::SelectOpts, const BaseFunctionCtx::Ptr&, FtPreselectT&&, const RdxContext&) override; void UpdateSortedIds(const UpdateSortedContext&) override {} - virtual IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, FtUseExternStatuses, - const RdxContext&) = 0; + virtual IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, FtMergeStatuses&&, + FtUseExternStatuses, const RdxContext&) = 0; void SetOpts(const IndexOpts& opts) override; void Commit() override final { // Do nothing @@ -67,7 +67,7 @@ class IndexText : public IndexUnordered { virtual void commitFulltextImpl() = 0; FtCtx::Ptr prepareFtCtx(const BaseFunctionCtx::Ptr&); SelectKeyResults doSelectKey(const VariantArray& keys, const std::optional&, FtMergeStatuses&&, - FtUseExternStatuses useExternSt, bool inTransaction, FtCtx::Ptr, const RdxContext&); + FtUseExternStatuses useExternSt, bool inTransaction, FtSortType ftSortType, FtCtx::Ptr, const RdxContext&); SelectKeyResults resultFromCache(const VariantArray& keys, FtIdSetCache::Iterator&&, FtCtx::Ptr&); void build(const RdxContext& rdxCtx); diff --git a/cpp_src/core/namespace/namespaceimpl.cc b/cpp_src/core/namespace/namespaceimpl.cc index e7257c26c..f32901545 100644 --- a/cpp_src/core/namespace/namespaceimpl.cc +++ b/cpp_src/core/namespace/namespaceimpl.cc @@ -3,7 +3,7 @@ #include #include #include "core/cjson/cjsondecoder.h" -#include "core/cjson/defaultvaluecoder.h" +// #include "core/cjson/defaultvaluecoder.h" #include "core/cjson/jsonbuilder.h" #include "core/cjson/uuid_recoders.h" #include "core/formatters/lsn_fmt.h" @@ -26,6 +26,7 @@ #include "tools/flagguard.h" #include "tools/fsops.h" #include "tools/logger.h" +#include "tools/scope_guard.h" #include "tools/stringstools.h" #include "tools/timetools.h" @@ -356,7 +357,6 @@ class NamespaceImpl::RollBack_recreateCompositeIndexes final : private RollBackB public: RollBack_recreateCompositeIndexes(NamespaceImpl& ns, size_t startIdx, size_t count) : ns_{ns}, startIdx_{startIdx} { indexes_.reserve(count); - std::swap(ns_.indexesToComposites_, indexesToComposites_); } ~RollBack_recreateCompositeIndexes() override { RollBack(); @@ -378,7 +378,6 @@ class NamespaceImpl::RollBack_recreateCompositeIndexes final : private RollBackB for (size_t i = 0, s = indexes_.size(); i < s; ++i) { std::swap(ns_.indexes_[i + startIdx_], indexes_[i]); } - std::swap(ns_.indexesToComposites_, indexesToComposites_); Disable(); } void SaveIndex(std::unique_ptr&& idx) { indexes_.emplace_back(std::move(idx)); } @@ -393,7 +392,6 @@ class NamespaceImpl::RollBack_recreateCompositeIndexes final : private RollBackB private: NamespaceImpl& ns_; std::vector> indexes_; - fast_hash_map> indexesToComposites_; size_t startIdx_; }; @@ -428,10 +426,6 @@ NamespaceImpl::RollBack_recreateCompositeIndexes NamespaceImpl::re auto newIndex{Index::New(indexDef, PayloadType{payloadType_}, FieldsSet{fields}, config_.cacheConfig)}; rollbacker.SaveIndex(std::move(index)); std::swap(index, newIndex); - - for (auto field : fields) { - indexesToComposites_[field].emplace_back(i); - } } } return rollbacker; @@ -570,15 +564,17 @@ NamespaceImpl::RollBack_updateItems NamespaceImpl::updateItems(con } else { recoder = std::make_unique(changedField); } - } else { - const auto& indexToUpdate = indexes_[changedField]; - if (!IsComposite(indexToUpdate->Type()) && !indexToUpdate->Opts().IsSparse()) { - auto tagsNames = pickJsonPath(fld); - if (!tagsNames.empty()) { - recoder = std::make_unique(name_, fld, std::move(tagsNames), changedField); - } - } } + // TODO: This logic must be reenabled after #1353. Now it's potentially unsafe + // else { + // const auto& indexToUpdate = indexes_[changedField]; + // if (!IsComposite(indexToUpdate->Type()) && !indexToUpdate->Opts().IsSparse()) { + // auto tagsNames = pickJsonPath(fld); + // if (!tagsNames.empty()) { + // recoder = std::make_unique(name_, fld, std::move(tagsNames), changedField); + // } + // } + // } } rollbacker.SaveTuple(); @@ -810,6 +806,10 @@ void NamespaceImpl::dropIndex(const IndexDef& index) { throw Error(errParams, errMsg, index.name_); } + // Guard approach is a bit suboptimal, but simpler + const auto compositesMappingGuard = + MakeScopeGuard([this] { indexesToComposites_.clear(); }, [this] { rebuildIndexesToCompositeMapping(); }); + int fieldIdx = itIdxName->second; std::unique_ptr& indexToRemove = indexes_[fieldIdx]; if (!IsComposite(indexToRemove->Type()) && indexToRemove->Opts().IsSparse()) { @@ -848,15 +848,7 @@ void NamespaceImpl::dropIndex(const IndexDef& index) { idx->SetFields(std::move(newFields)); } - const bool isComposite = IsComposite(indexToRemove->Type()); - if (isComposite) { - for (auto& v : indexesToComposites_) { - const auto f = std::find(v.second.begin(), v.second.end(), fieldIdx); - if (f != v.second.end()) { - v.second.erase(f); - } - } - } else if (!indexToRemove->Opts().IsSparse()) { + if (!IsComposite(indexToRemove->Type()) && !indexToRemove->Opts().IsSparse()) { PayloadType oldPlType = payloadType_; payloadType_.Drop(index.name_); tagsMatcher_.UpdatePayloadType(payloadType_); @@ -1183,6 +1175,10 @@ bool NamespaceImpl::addIndex(const IndexDef& indexDef) { indexes_[currentPKIndex->second]->Name()); } + // Guard approach is a bit suboptimal, but simpler + const auto compositesMappingGuard = + MakeScopeGuard([this] { indexesToComposites_.clear(); }, [this] { rebuildIndexesToCompositeMapping(); }); + if (IsComposite(indexDef.Type())) { verifyCompositeIndex(indexDef); addCompositeIndex(indexDef); @@ -1307,10 +1303,6 @@ void NamespaceImpl::addCompositeIndex(const IndexDef& indexDef) { } } - for (auto field : fields) { - indexesToComposites_[field].emplace_back(idxPos); - } - updateSortedIdxCount(); insertIndex_rollbacker.Disable(); } @@ -2717,6 +2709,8 @@ bool NamespaceImpl::loadIndexesFromStorage() { addIndex(indexDef); } catch (const Error& e) { err = e; + } catch (std::exception& e) { + err = Error(errLogic, "Exception: '%s'", e.what()); } } if (!err.ok()) { @@ -3391,4 +3385,20 @@ int64_t NamespaceImpl::correctMaxIterationsIdSetPreResult(int64_t maxIterationsI return res; } +void NamespaceImpl::rebuildIndexesToCompositeMapping() noexcept { + // The only possible exception here is bad_alloc, but required memory footprint is really small + const auto beg = indexes_.firstCompositePos(); + const auto end = beg + indexes_.compositeIndexesSize(); + fast_hash_map> indexesToComposites; + for (auto i = beg; i < end; ++i) { + const auto& index = indexes_[i]; + assertrx(IsComposite(index->Type())); + const auto& fields = index->Fields(); + for (auto field : fields) { + indexesToComposites[field].emplace_back(i); + } + } + indexesToComposites_ = std::move(indexesToComposites); +} + } // namespace reindexer diff --git a/cpp_src/core/namespace/namespaceimpl.h b/cpp_src/core/namespace/namespaceimpl.h index 8e0d22d80..3f0a75824 100644 --- a/cpp_src/core/namespace/namespaceimpl.h +++ b/cpp_src/core/namespace/namespaceimpl.h @@ -431,6 +431,7 @@ class NamespaceImpl final : public intrusive_atomic_rc_base { // NOLINT(*perfor bool SortOrdersBuilt() const noexcept { return optimizationState_.load(std::memory_order_acquire) == OptimizationCompleted; } int64_t correctMaxIterationsIdSetPreResult(int64_t maxIterationsIdSetPreResult) const; + void rebuildIndexesToCompositeMapping() noexcept; IndexesStorage indexes_; fast_hash_map indexesNames_; diff --git a/cpp_src/core/nsselecter/comparator/comparator_indexed.cc b/cpp_src/core/nsselecter/comparator/comparator_indexed.cc index d6b6ace4d..513c81ac8 100644 --- a/cpp_src/core/nsselecter/comparator/comparator_indexed.cc +++ b/cpp_src/core/nsselecter/comparator/comparator_indexed.cc @@ -103,10 +103,6 @@ void initComparator(const reindexer::VariantArray& from, typename reindexer::com template void initComparator(CondType cond, const reindexer::VariantArray& from, reindexer::comparators::DataHolder& to) { using namespace reindexer::comparators; - using SingleType = typename DataHolder::SingleType; - using RangeType = typename DataHolder::RangeType; - using SetPtrType = typename DataHolder::SetPtrType; - using AllSetPtrType = typename DataHolder::AllSetPtrType; using SetWrpType = typename DataHolder::SetWrpType; using AllSetWrpType = typename DataHolder::AllSetWrpType; switch (cond) { @@ -118,17 +114,15 @@ void initComparator(CondType cond, const reindexer::VariantArray& from, reindexe to.value_ = GetValue(cond, from, 0); break; case CondRange: - to.value_.~SingleType(); - new (&to.range_) RangeType{GetValue(cond, from, 0), GetValue(cond, from, 1)}; + to.value_ = GetValue(cond, from, 0); + to.value2_ = GetValue(cond, from, 1); break; case CondSet: - to.value_.~SingleType(); - new (&to.setPtr_) SetPtrType{make_intrusive()}; + to.setPtr_ = make_intrusive(); initComparator(from, *to.setPtr_); break; case CondAllSet: - to.value_.~SingleType(); - new (&to.allSetPtr_) AllSetPtrType{make_intrusive()}; + to.allSetPtr_ = make_intrusive(); initComparator(from, *to.allSetPtr_); break; case CondAny: @@ -143,12 +137,8 @@ void initComparator(CondType cond, const reindexer::VariantArray& from, reindexe void initStringComparator(CondType cond, const reindexer::VariantArray& from, reindexer::comparators::DataHolder& to, const CollateOpts& collate) { using namespace reindexer::comparators; - using SingleType = DataHolder::SingleType; - using RangeType = DataHolder::RangeType; using SetType = DataHolder::SetType; using AllSetType = DataHolder::AllSetType; - using SetPtrType = DataHolder::SetPtrType; - using AllSetPtrType = DataHolder::AllSetPtrType; using SetWrpType = DataHolder::SetWrpType; using AllSetWrpType = DataHolder::AllSetWrpType; switch (cond) { @@ -161,17 +151,15 @@ void initStringComparator(CondType cond, const reindexer::VariantArray& from, re to.value_ = GetValue(cond, from, 0); break; case CondRange: - to.value_.~SingleType(); - new (&to.range_) RangeType{GetValue(cond, from, 0), GetValue(cond, from, 1)}; + to.value_ = GetValue(cond, from, 0); + to.value2_ = GetValue(cond, from, 1); break; case CondSet: - to.value_.~SingleType(); - new (&to.setPtr_) SetPtrType{make_intrusive(SetType{collate})}; + to.setPtr_ = make_intrusive(SetType{collate}); initComparator(from, *to.setPtr_); break; case CondAllSet: - to.value_.~SingleType(); - new (&to.allSetPtr_) AllSetPtrType{make_intrusive(AllSetType{collate, {}})}; + to.allSetPtr_ = make_intrusive(AllSetType{collate, {}}); initComparator(from, *to.allSetPtr_); break; case CondAny: @@ -185,13 +173,8 @@ void initStringComparator(CondType cond, const reindexer::VariantArray& from, re template [[nodiscard]] std::string comparatorCondStr(const V& values) { using namespace std::string_literals; - if constexpr (Cond == CondRange) { - if constexpr (std::is_same_v) { - return fmt::sprintf("RANGE(%s, %s)", values.value1_.valueView_, values.value2_.valueView_); - } else { - return fmt::format("RANGE({}, {})", values.first, values.second); - } - } else if constexpr (Cond == CondSet) { + static_assert(Cond != CondRange, "Incorrect specialization"); + if constexpr (Cond == CondSet) { if (values.empty()) { return "IN []"s; } else { @@ -205,12 +188,25 @@ template } } else if constexpr (Cond == CondEq || Cond == CondLt || Cond == CondLe || Cond == CondGt || Cond == CondGe) { if constexpr (std::is_same_v) { - return fmt::sprintf("%s %s", reindexer::comparators::CondToStr(), values.valueView_); + return fmt::format("{} {}", reindexer::comparators::CondToStr(), values.valueView_); } else { return fmt::format("{} {}", reindexer::comparators::CondToStr(), values); } } else if constexpr (Cond == CondLike && std::is_same_v) { - return fmt::sprintf("LIKE \"%s\"", values.valueView_); + return fmt::format("LIKE \"{}\"", values.valueView_); + } + abort(); +} + +template +[[nodiscard]] std::string comparatorCondStr(const V& value, const V& value2) { + using namespace std::string_literals; + if constexpr (Cond == CondRange) { + if constexpr (std::is_same_v) { + return fmt::format("RANGE({}, {})", value.valueView_, value2.valueView_); + } else { + return fmt::format("RANGE({}, {})", value, value2); + } } abort(); } @@ -229,7 +225,7 @@ template case CondGe: return comparatorCondStr(data.value_); case CondRange: - return comparatorCondStr(data.range_); + return comparatorCondStr(data.value_, data.value2_); case CondSet: assertrx_dbg(data.setPtr_); return comparatorCondStr(*data.setPtr_); @@ -251,27 +247,33 @@ template const typename reindexer::comparators::ValuesHolder::Type& values, const reindexer::PayloadType& payloadType, const reindexer::FieldsSet& fields) { using namespace std::string_literals; - if constexpr (Cond == CondRange) { - return fmt::sprintf("RANGE(%s, %s)", reindexer::Variant{values.first}.As(payloadType, fields), - reindexer::Variant{values.second}.As(payloadType, fields)); - } else if constexpr (Cond == CondSet) { + static_assert(Cond != CondRange, "Incorrect specialization"); + if constexpr (Cond == CondSet) { if (values.empty()) { return "IN []"s; } else { - return fmt::sprintf("IN [%s, ...]", reindexer::Variant{*values.begin()}.As(payloadType, fields)); + return fmt::format("IN [{}, ...]", reindexer::Variant{*values.begin()}.As(payloadType, fields)); } } else if constexpr (Cond == CondAllSet) { if (values.values_.empty()) { return "ALLSET []"s; } else { - return fmt::sprintf("ALLSET [%s, ...]", reindexer::Variant{values.values_.begin()->first}.As(payloadType, fields)); + return fmt::format("ALLSET [{}, ...]", reindexer::Variant{values.values_.begin()->first}.As(payloadType, fields)); } } else if constexpr (Cond == CondEq || Cond == CondLt || Cond == CondLe || Cond == CondGt || Cond == CondGe) { - return fmt::sprintf("%s %s", reindexer::comparators::CondToStr(), - reindexer::Variant{values}.As(payloadType, fields)); + return fmt::format("{} {}", reindexer::comparators::CondToStr(), + reindexer::Variant{values}.As(payloadType, fields)); } } +template +[[nodiscard]] std::string compositeRangeComparatorCondStr(const V& value, const V& value2, const reindexer::PayloadType& payloadType, + const reindexer::FieldsSet& fields) { + using namespace std::string_literals; + return fmt::format("RANGE({}, {})", reindexer::Variant{value}.As(payloadType, fields), + reindexer::Variant{value2}.As(payloadType, fields)); +} + [[nodiscard]] std::string anyComparatorCondStr() { using namespace std::string_literals; return "IS NOT NULL"s; @@ -483,7 +485,7 @@ ComparatorIndexedJsonPathStringDistinct::ComparatorIndexedJsonPathStringDistinct ComparatorIndexedComposite::ComparatorIndexedComposite(const VariantArray& values, const CollateOpts& collate, const FieldsSet& fields, const PayloadType& payloadType, CondType cond) - : collateOpts_{&collate}, fields_{fields}, payloadType_{payloadType} { + : collateOpts_{&collate}, fields_{make_intrusive(fields)}, payloadType_{payloadType} { switch (cond) { case CondEq: case CondLt: @@ -493,19 +495,16 @@ ComparatorIndexedComposite::ComparatorIndexedComposite(const VariantArray& value value_ = GetValue(cond, values, 0); break; case CondRange: - value_.~PayloadValue(); - new (&range_) RangeType{GetValue(cond, values, 0), GetValue(cond, values, 1)}; + value_ = GetValue(cond, values, 0); + value2_ = GetValue(cond, values, 1); break; case CondSet: - value_.~PayloadValue(); - new (&setPtr_) SetPtrType{make_intrusive(SetType{values.size(), reindexer::hash_composite_ref{payloadType, fields}, - reindexer::equal_composite_ref{payloadType, fields}})}; + setPtr_ = make_intrusive(SetType{values.size(), reindexer::hash_composite_ref{payloadType, fields}, + reindexer::equal_composite_ref{payloadType, fields}}); initComparator(values, *setPtr_); break; case CondAllSet: - value_.~PayloadValue(); - new (&allSetPtr_) AllSetPtrType{ - make_intrusive(AllSetType{{reindexer::PayloadType{payloadType}, reindexer::FieldsSet{fields}}, {}})}; + allSetPtr_ = make_intrusive(AllSetType{{reindexer::PayloadType{payloadType}, reindexer::FieldsSet{fields}}, {}}); initComparator(values, *allSetPtr_); break; case CondAny: @@ -520,23 +519,23 @@ ComparatorIndexedComposite::ComparatorIndexedComposite(const VariantArray& value [[nodiscard]] std::string ComparatorIndexedComposite::ConditionStr() const { switch (cond_) { case CondEq: - return compositeComparatorCondStr(value_, payloadType_, fields_); + return compositeComparatorCondStr(value_, payloadType_, *fields_); case CondLt: - return compositeComparatorCondStr(value_, payloadType_, fields_); + return compositeComparatorCondStr(value_, payloadType_, *fields_); case CondLe: - return compositeComparatorCondStr(value_, payloadType_, fields_); + return compositeComparatorCondStr(value_, payloadType_, *fields_); case CondGt: - return compositeComparatorCondStr(value_, payloadType_, fields_); + return compositeComparatorCondStr(value_, payloadType_, *fields_); case CondGe: - return compositeComparatorCondStr(value_, payloadType_, fields_); + return compositeComparatorCondStr(value_, payloadType_, *fields_); case CondRange: - return compositeComparatorCondStr(range_, payloadType_, fields_); + return compositeRangeComparatorCondStr(value_, value2_, payloadType_, *fields_); case CondSet: assertrx_dbg(setPtr_); - return compositeComparatorCondStr(*setPtr_, payloadType_, fields_); + return compositeComparatorCondStr(*setPtr_, payloadType_, *fields_); case CondAllSet: assertrx_dbg(allSetPtr_); - return compositeComparatorCondStr(*allSetPtr_, payloadType_, fields_); + return compositeComparatorCondStr(*allSetPtr_, payloadType_, *fields_); case CondLike: case CondAny: case CondEmpty: diff --git a/cpp_src/core/nsselecter/comparator/comparator_indexed.h b/cpp_src/core/nsselecter/comparator/comparator_indexed.h index 6f3e4e5b4..d88509a35 100644 --- a/cpp_src/core/nsselecter/comparator/comparator_indexed.h +++ b/cpp_src/core/nsselecter/comparator/comparator_indexed.h @@ -55,14 +55,6 @@ struct ValuesHolder { using Type = std::pair; }; -template <> -struct ValuesHolder { - struct Type { - ValuesHolder::Type value1_; - ValuesHolder::Type value2_; - }; -}; - template struct ValuesHolder { using Type = fast_hash_set; @@ -105,7 +97,6 @@ struct ValuesHolder { template struct DataHolder { using SingleType = typename ValuesHolder::Type; - using RangeType = typename ValuesHolder::Type; using SetType = typename ValuesHolder::Type; using SetWrpType = intrusive_rc_wrapper; using SetPtrType = intrusive_ptr; @@ -113,188 +104,16 @@ struct DataHolder { using AllSetWrpType = intrusive_rc_wrapper; using AllSetPtrType = intrusive_ptr; - DataHolder() noexcept : cond_{CondEq}, value_{} {} - DataHolder(DataHolder&& other) noexcept : cond_{other.cond_} { - switch (other.cond_) { - case CondEq: - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - new (&this->value_) SingleType{std::move(other.value_)}; - break; - case CondRange: - new (&this->range_) RangeType{std::move(other.range_)}; - break; - case CondSet: - new (&this->setPtr_) SetPtrType{std::move(other.setPtr_)}; - break; - case CondAllSet: - new (&this->allSetPtr_) AllSetPtrType{std::move(other.allSetPtr_)}; - break; - case CondAny: - case CondEmpty: - case CondDWithin: - new (&this->value_) SingleType{}; - break; - } - } - DataHolder(const DataHolder& other) : cond_{CondAny} { - switch (other.cond_) { - case CondEq: - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - new (&this->value_) SingleType{other.value_}; - break; - case CondRange: - new (&this->range_) RangeType{other.range_}; - break; - case CondSet: - new (&this->setPtr_) SetPtrType{other.setPtr_}; - break; - case CondAllSet: - new (&this->allSetPtr_) AllSetPtrType{other.allSetPtr_}; - break; - case CondAny: - case CondEmpty: - case CondDWithin: - new (&this->value_) SingleType{}; - break; - } - cond_ = other.cond_; - } - DataHolder& operator=(DataHolder&& other) noexcept { - if (this == &other) { - return *this; - } - if (cond_ != other.cond_) { - clear(); - switch (other.cond_) { - case CondEq: - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - new (&this->value_) SingleType{std::move(other.value_)}; - break; - case CondRange: - new (&this->range_) RangeType{std::move(other.range_)}; - break; - case CondSet: - new (&this->setPtr_) SetPtrType{std::move(other.setPtr_)}; - break; - case CondAllSet: - new (&this->allSetPtr_) AllSetPtrType{std::move(other.allSetPtr_)}; - break; - case CondAny: - case CondEmpty: - case CondDWithin: - break; - } - this->cond_ = other.cond_; - } else { - switch (other.cond_) { - case CondEq: - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - this->value_ = std::move(other.value_); - break; - case CondRange: - this->range_ = std::move(other.range_); - break; - case CondSet: - this->setPtr_ = std::move(other.setPtr_); - break; - case CondAllSet: - this->allSetPtr_ = std::move(other.allSetPtr_); - break; - case CondAny: - case CondEmpty: - case CondDWithin: - break; - } - } - return *this; - } - DataHolder& operator=(const DataHolder& other) { - if (this == &other) { - return *this; - } - if (cond_ != other.cond_) { - clear(); - switch (other.cond_) { - case CondEq: - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - new (&this->value_) SingleType{other.value_}; - break; - case CondRange: - new (&this->range_) RangeType{other.range_}; - break; - case CondSet: - new (&this->setPtr_) SetPtrType{other.setPtr_}; - break; - case CondAllSet: - new (&this->allSetPtr_) AllSetPtrType{other.allSetPtr_}; - break; - case CondAny: - case CondEmpty: - case CondDWithin: - break; - } - this->cond_ = other.cond_; - } else { - DataHolder tmp{other}; - *this = std::move(tmp); - } - return *this; - } - ~DataHolder() { clear(); } - void clear() noexcept { - switch (cond_) { - case CondEq: - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - value_.~SingleType(); - break; - case CondRange: - range_.~RangeType(); - break; - case CondSet: - setPtr_.~SetPtrType(); - break; - case CondAllSet: - allSetPtr_.~AllSetPtrType(); - break; - case CondDWithin: - case CondAny: - case CondEmpty: - default: - break; - } - cond_ = CondAny; - } + DataHolder() noexcept : cond_{CondEq} {} + DataHolder(DataHolder&& other) noexcept = default; + DataHolder(const DataHolder& other) = default; + DataHolder& operator=(DataHolder&& other) noexcept = default; + DataHolder& operator=(const DataHolder& other) = default; CondType cond_; - union { - SingleType value_; - RangeType range_; - SetPtrType setPtr_; - AllSetPtrType allSetPtr_; - }; + SingleType value_{}; // Either single value or right range boundry + SingleType value2_{}; // Left range boundry + SetPtrType setPtr_{}; + AllSetPtrType allSetPtr_{}; }; template @@ -315,7 +134,7 @@ class ComparatorIndexedOffsetScalar : private DataHolder { case CondGe: return *ptr >= this->value_; case CondRange: - return this->range_.first <= *ptr && *ptr <= this->range_.second; + return this->value_ <= *ptr && *ptr <= this->value2_; case CondSet: assertrx_dbg(this->setPtr_); return this->setPtr_->find(*ptr) != this->setPtr_->cend(); @@ -357,7 +176,7 @@ class ComparatorIndexedColumnScalar : private DataHolder { case CondGe: return v >= this->value_; case CondRange: - return this->range_.first <= v && v <= this->range_.second; + return this->value_ <= v && v <= this->value2_; case CondSet: assertrx_dbg(this->setPtr_); return this->setPtr_->find(v) != this->setPtr_->cend(); @@ -399,7 +218,7 @@ class ComparatorIndexedOffsetScalarDistinct : private DataHolder { case CondGe: return value >= this->value_ && distinct_.Compare(value); case CondRange: - return this->range_.first <= value && value <= this->range_.second && distinct_.Compare(value); + return this->value_ <= value && value <= this->value2_ && distinct_.Compare(value); case CondSet: assertrx_dbg(this->setPtr_); return this->setPtr_->find(value) != this->setPtr_->cend() && distinct_.Compare(value); @@ -445,7 +264,7 @@ class ComparatorIndexedColumnScalarDistinct : private DataHolder { case CondGe: return value >= this->value_ && distinct_.Compare(value); case CondRange: - return this->range_.first <= value && value <= this->range_.second && distinct_.Compare(value); + return this->value_ <= value && value <= this->value2_ && distinct_.Compare(value); case CondSet: assertrx_dbg(this->setPtr_); return this->setPtr_->find(value) != this->setPtr_->cend() && distinct_.Compare(value); @@ -510,7 +329,7 @@ class ComparatorIndexedOffsetArray : private DataHolder { } continue; case CondRange: - if (this->range_.first <= *ptr && *ptr <= this->range_.second) { + if (this->value_ <= *ptr && *ptr <= this->value2_) { return true; } continue; @@ -589,7 +408,7 @@ class ComparatorIndexedOffsetArrayDistinct : private DataHolder { } continue; case CondRange: - if (this->range_.first <= *ptr && *ptr <= this->range_.second && distinct_.Compare(*ptr)) { + if (this->value_ <= *ptr && *ptr <= this->value2_ && distinct_.Compare(*ptr)) { return true; } continue; @@ -684,7 +503,7 @@ class ComparatorIndexedJsonPath : private DataHolder { continue; case CondRange: { const auto v = value.As(); - if (this->range_.first <= v && v <= this->range_.second) { + if (this->value_ <= v && v <= this->value2_) { return true; } } @@ -774,7 +593,7 @@ class ComparatorIndexedJsonPathDistinct : private DataHolder { continue; case CondRange: { const auto v = value; - if (this->range_.first <= v && v <= this->range_.second && distinct_.Compare(value)) { + if (this->value_ <= v && v <= this->value2_ && distinct_.Compare(value)) { return true; } } @@ -839,8 +658,8 @@ class ComparatorIndexedOffsetScalarString : private DataHolder { assertrx_dbg(this->setPtr_); return setPtr_->find(value) != setPtr_->cend(); case CondRange: - return (collateCompare(value, range_.value1_.valueView_, *collateOpts_) & ComparationResult::Ge) && - (collateCompare(value, range_.value2_.valueView_, *collateOpts_) & ComparationResult::Le); + return (collateCompare(value, value_.valueView_, *collateOpts_) & ComparationResult::Ge) && + (collateCompare(value, value2_.valueView_, *collateOpts_) & ComparationResult::Le); case CondAllSet: assertrx_dbg(this->allSetPtr_); return allSetPtr_->values_.size() == 1 && allSetPtr_->values_.find(value) != allSetPtr_->values_.cend(); @@ -884,8 +703,8 @@ class ComparatorIndexedColumnScalarString : private DataHolder { assertrx_dbg(this->setPtr_); return setPtr_->find(value) != setPtr_->cend(); case CondRange: - return (collateCompare(value, range_.value1_.valueView_, *collateOpts_) & ComparationResult::Ge) && - (collateCompare(value, range_.value2_.valueView_, *collateOpts_) & ComparationResult::Le); + return (collateCompare(value, value_.valueView_, *collateOpts_) & ComparationResult::Ge) && + (collateCompare(value, value2_.valueView_, *collateOpts_) & ComparationResult::Le); case CondAllSet: assertrx_dbg(this->allSetPtr_); return allSetPtr_->values_.size() == 1 && allSetPtr_->values_.find(value) != allSetPtr_->values_.cend(); @@ -928,9 +747,8 @@ class ComparatorIndexedOffsetScalarStringDistinct : private DataHoldersetPtr_); return setPtr_->find(value) != setPtr_->cend() && distinct_.Compare(value); case CondRange: - return (collateCompare(value, range_.value1_.valueView_, *collateOpts_) & ComparationResult::Ge) && - (collateCompare(value, range_.value2_.valueView_, *collateOpts_) & ComparationResult::Le) && - distinct_.Compare(value); + return (collateCompare(value, value_.valueView_, *collateOpts_) & ComparationResult::Ge) && + (collateCompare(value, value2_.valueView_, *collateOpts_) & ComparationResult::Le) && distinct_.Compare(value); case CondAllSet: assertrx_dbg(this->allSetPtr_); return allSetPtr_->values_.size() == 1 && allSetPtr_->values_.find(value) != allSetPtr_->values_.cend() && @@ -977,9 +795,8 @@ class ComparatorIndexedColumnScalarStringDistinct : private DataHoldersetPtr_); return setPtr_->find(value) != setPtr_->cend() && distinct_.Compare(value); case CondRange: - return (collateCompare(value, range_.value1_.valueView_, *collateOpts_) & ComparationResult::Ge) && - (collateCompare(value, range_.value2_.valueView_, *collateOpts_) & ComparationResult::Le) && - distinct_.Compare(value); + return (collateCompare(value, value_.valueView_, *collateOpts_) & ComparationResult::Ge) && + (collateCompare(value, value2_.valueView_, *collateOpts_) & ComparationResult::Le) && distinct_.Compare(value); case CondAllSet: assertrx_dbg(this->allSetPtr_); return allSetPtr_->values_.size() == 1 && allSetPtr_->values_.find(value) != allSetPtr_->values_.cend() && @@ -1033,8 +850,8 @@ class ComparatorIndexedOffsetArrayString : private DataHolder { } continue; case CondRange: - if ((collateCompare(value, range_.value1_.valueView_, *collateOpts_) & ComparationResult::Ge) && - (collateCompare(value, range_.value2_.valueView_, *collateOpts_) & ComparationResult::Le)) { + if ((collateCompare(value, value_.valueView_, *collateOpts_) & ComparationResult::Ge) && + (collateCompare(value, value2_.valueView_, *collateOpts_) & ComparationResult::Le)) { return true; } continue; @@ -1118,9 +935,8 @@ class ComparatorIndexedOffsetArrayStringDistinct : private DataHolder { } break; case CondRange: - if ((collateCompare(value, range_.value1_.valueView_, *collateOpts_) & ComparationResult::Ge) && - (collateCompare(value, range_.value2_.valueView_, *collateOpts_) & ComparationResult::Le)) { + if ((collateCompare(value, value_.valueView_, *collateOpts_) & ComparationResult::Ge) && + (collateCompare(value, value2_.valueView_, *collateOpts_) & ComparationResult::Le)) { return true; } break; @@ -1303,9 +1119,8 @@ class ComparatorIndexedJsonPathStringDistinct : private DataHolder { } break; case CondRange: - if ((collateCompare(value, range_.value1_.valueView_, *collateOpts_) & ComparationResult::Ge) && - (collateCompare(value, range_.value2_.valueView_, *collateOpts_) & ComparationResult::Le) && - distinct_.Compare(value)) { + if ((collateCompare(value, value_.valueView_, *collateOpts_) & ComparationResult::Ge) && + (collateCompare(value, value2_.valueView_, *collateOpts_) & ComparationResult::Le) && distinct_.Compare(value)) { return true; } break; @@ -1391,26 +1206,26 @@ class ComparatorIndexedComposite : private DataHolder { return setPtr_->find(item) != setPtr_->cend(); case CondRange: { ConstPayload pv{payloadType_, item}; - return (pv.Compare(range_.first, fields_, *collateOpts_) & ComparationResult::Ge) && - (pv.Compare(range_.second, fields_, *collateOpts_) & ComparationResult::Le); + return (pv.Compare(value_, *fields_, *collateOpts_) & ComparationResult::Ge) && + (pv.Compare(value2_, *fields_, *collateOpts_) & ComparationResult::Le); } case CondAllSet: assertrx_dbg(this->allSetPtr_); return allSetPtr_->values_.size() == 1 && allSetPtr_->values_.find(item) != allSetPtr_->values_.end(); case CondEq: - return ConstPayload{payloadType_, item}.Compare(value_, fields_, *collateOpts_) == + return ConstPayload{payloadType_, item}.Compare(value_, *fields_, *collateOpts_) == ComparationResult::Eq; case CondLt: - return ConstPayload{payloadType_, item}.Compare(value_, fields_, *collateOpts_) == + return ConstPayload{payloadType_, item}.Compare(value_, *fields_, *collateOpts_) == ComparationResult::Lt; case CondLe: - return ConstPayload{payloadType_, item}.Compare(value_, fields_, *collateOpts_) & + return ConstPayload{payloadType_, item}.Compare(value_, *fields_, *collateOpts_) & ComparationResult::Le; case CondGt: - return ConstPayload{payloadType_, item}.Compare(value_, fields_, *collateOpts_) == + return ConstPayload{payloadType_, item}.Compare(value_, *fields_, *collateOpts_) == ComparationResult::Gt; case CondGe: - return ConstPayload{payloadType_, item}.Compare(value_, fields_, *collateOpts_) & + return ConstPayload{payloadType_, item}.Compare(value_, *fields_, *collateOpts_) & ComparationResult::Ge; case CondAny: case CondEmpty: @@ -1426,8 +1241,11 @@ class ComparatorIndexedComposite : private DataHolder { void ClearDistinctValues() const noexcept {} private: + // Using pointer for cheap copying and ExpressionTree size reduction + using FieldsSetWrp = intrusive_rc_wrapper; + const CollateOpts* collateOpts_; - FieldsSet fields_; + intrusive_ptr fields_; PayloadType payloadType_; }; @@ -1478,7 +1296,7 @@ class ComparatorIndexedOffsetArrayDWithinDistinct { void ClearDistinctValues() noexcept { distinct_.ClearValues(); } private: - ComparatorIndexedDistinct distinct_; + ComparatorIndexedDistinct> distinct_; Point point_; double distance_; size_t offset_; @@ -1533,7 +1351,7 @@ class ComparatorIndexedJsonPathDWithinDistinct { void ClearDistinctValues() noexcept { distinct_.ClearValues(); } private: - ComparatorIndexedDistinct distinct_; + ComparatorIndexedDistinct> distinct_; PayloadType payloadType_; TagsPath tagsPath_; VariantArray buffer_; @@ -1638,7 +1456,10 @@ class ComparatorIndexedOffsetArrayAnyDistinct { void ClearDistinctValues() noexcept { distinct_.ClearValues(); } private: - ComparatorIndexedDistinct distinct_; + using ComparatoristincType = std::conditional_t, ComparatorIndexedDistinct>, + ComparatorIndexedDistinct>; + + ComparatoristincType distinct_; size_t offset_; }; diff --git a/cpp_src/core/nsselecter/comparator/comparator_indexed_distinct.h b/cpp_src/core/nsselecter/comparator/comparator_indexed_distinct.h index 615443cb9..5a0fd6ae9 100644 --- a/cpp_src/core/nsselecter/comparator/comparator_indexed_distinct.h +++ b/cpp_src/core/nsselecter/comparator/comparator_indexed_distinct.h @@ -2,15 +2,14 @@ #include #include -#include #include +#include "estl/fast_hash_set.h" #include "tools/stringstools.h" -#include "vendor/hopscotch/hopscotch_sc_set.h" namespace reindexer { -template +template > class ComparatorIndexedDistinct { public: ComparatorIndexedDistinct() : values_{make_intrusive()} {} @@ -28,7 +27,6 @@ class ComparatorIndexedDistinct { } private: - using SetType = std::unordered_set; using SetWrpType = intrusive_rc_wrapper; using SetPtrType = intrusive_ptr; diff --git a/cpp_src/core/nsselecter/comparator/comparator_not_indexed.cc b/cpp_src/core/nsselecter/comparator/comparator_not_indexed.cc index ce24c7ddc..1f9350ed1 100644 --- a/cpp_src/core/nsselecter/comparator/comparator_not_indexed.cc +++ b/cpp_src/core/nsselecter/comparator/comparator_not_indexed.cc @@ -76,12 +76,13 @@ template } // namespace comparators [[nodiscard]] std::string ComparatorNotIndexed::ConditionStr() const { - return std::visit([&](const auto& impl) { return impl.ConditionStr(); }, impl_); + assertrx_dbg(dynamic_cast(impl_.get())); + return std::visit([&](const auto& impl) { return impl.ConditionStr(); }, *static_cast(impl_.get())); } -comparators::ComparatorNotIndexedVariant ComparatorNotIndexed::createImpl(CondType cond, const VariantArray& values, - const PayloadType& payloadType, const TagsPath& fieldPath, - bool distinct) { +ComparatorNotIndexed::ImplVariantType ComparatorNotIndexed::createImpl(CondType cond, const VariantArray& values, + const PayloadType& payloadType, const TagsPath& fieldPath, + bool distinct) { using namespace comparators; if (distinct) { switch (cond) { diff --git a/cpp_src/core/nsselecter/comparator/comparator_not_indexed.h b/cpp_src/core/nsselecter/comparator/comparator_not_indexed.h index 149fc9e1d..097dff18f 100644 --- a/cpp_src/core/nsselecter/comparator/comparator_not_indexed.h +++ b/cpp_src/core/nsselecter/comparator/comparator_not_indexed.h @@ -1,6 +1,5 @@ #pragma once -#include #include "comparator_not_indexed_distinct.h" #include "const.h" #include "core/keyvalue/geometry.h" @@ -19,6 +18,11 @@ template class ComparatorNotIndexedImplBase { protected: ComparatorNotIndexedImplBase(const VariantArray&); + ComparatorNotIndexedImplBase(const ComparatorNotIndexedImplBase&) = delete; + ComparatorNotIndexedImplBase& operator=(const ComparatorNotIndexedImplBase&) = delete; + ComparatorNotIndexedImplBase(ComparatorNotIndexedImplBase&&) = default; + ComparatorNotIndexedImplBase& operator=(ComparatorNotIndexedImplBase&&) = default; + [[nodiscard]] std::string ConditionStr() const; [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const Variant& v) const { if constexpr (Cond == CondEq) { @@ -44,6 +48,11 @@ template <> class ComparatorNotIndexedImplBase { protected: ComparatorNotIndexedImplBase(const VariantArray&); + ComparatorNotIndexedImplBase(const ComparatorNotIndexedImplBase&) = delete; + ComparatorNotIndexedImplBase& operator=(const ComparatorNotIndexedImplBase&) = delete; + ComparatorNotIndexedImplBase(ComparatorNotIndexedImplBase&&) = default; + ComparatorNotIndexedImplBase& operator=(ComparatorNotIndexedImplBase&&) = default; + [[nodiscard]] std::string ConditionStr() const; [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const Variant& v) const { return (v.RelaxCompare(value1_) & ComparationResult::Ge) && @@ -62,6 +71,11 @@ class ComparatorNotIndexedImplBase { values_.insert(v); } } + ComparatorNotIndexedImplBase(const ComparatorNotIndexedImplBase&) = delete; + ComparatorNotIndexedImplBase& operator=(const ComparatorNotIndexedImplBase&) = delete; + ComparatorNotIndexedImplBase(ComparatorNotIndexedImplBase&&) = default; + ComparatorNotIndexedImplBase& operator=(ComparatorNotIndexedImplBase&&) = default; + [[nodiscard]] std::string ConditionStr() const; [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const Variant& v) const { return values_.find(v) != values_.cend(); } @@ -75,6 +89,11 @@ template <> class ComparatorNotIndexedImplBase { protected: ComparatorNotIndexedImplBase(const VariantArray&); + ComparatorNotIndexedImplBase(const ComparatorNotIndexedImplBase&) = delete; + ComparatorNotIndexedImplBase& operator=(const ComparatorNotIndexedImplBase&) = delete; + ComparatorNotIndexedImplBase(ComparatorNotIndexedImplBase&&) = default; + ComparatorNotIndexedImplBase& operator=(ComparatorNotIndexedImplBase&&) = default; + [[nodiscard]] std::string ConditionStr() const; [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const Variant& v) const { if (!v.Type().Is()) { @@ -98,6 +117,11 @@ class ComparatorNotIndexedImpl : private ComparatorNotIndexedImplBa public: ComparatorNotIndexedImpl(const VariantArray& values, const PayloadType& payloadType, const TagsPath& fieldPath) : Base{values}, payloadType_{payloadType}, fieldPath_{fieldPath} {} + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType /*rowId*/) { ConstPayload{payloadType_, item}.GetByJsonPath(fieldPath_, buffer_, KeyValueType::Undefined{}); for (const Variant& v : buffer_) { @@ -125,6 +149,11 @@ class ComparatorNotIndexedImpl : private ComparatorNotIndexedImplBas public: ComparatorNotIndexedImpl(const VariantArray& values, const PayloadType& payloadType, const TagsPath& fieldPath) : Base{values}, distinct_{}, payloadType_{payloadType}, fieldPath_{fieldPath} {} + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType /*rowId*/) { ConstPayload{payloadType_, item}.GetByJsonPath(fieldPath_, buffer_, KeyValueType::Undefined{}); for (const Variant& v : buffer_) { @@ -156,6 +185,11 @@ class ComparatorNotIndexedImpl { public: ComparatorNotIndexedImpl(const PayloadType& payloadType, const TagsPath& fieldPath) : payloadType_{payloadType}, fieldPath_{fieldPath} {} + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] std::string ConditionStr() const; [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType /*rowId*/) { ConstPayload{payloadType_, item}.GetByJsonPath(fieldPath_, buffer_, KeyValueType::Undefined{}); @@ -181,6 +215,11 @@ class ComparatorNotIndexedImpl { public: ComparatorNotIndexedImpl(const PayloadType& payloadType, const TagsPath& fieldPath) : payloadType_{payloadType}, fieldPath_{fieldPath} {} + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] std::string ConditionStr() const; [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType /*rowId*/) { ConstPayload{payloadType_, item}.GetByJsonPath(fieldPath_, buffer_, KeyValueType::Undefined{}); @@ -212,6 +251,11 @@ class ComparatorNotIndexedImpl { public: ComparatorNotIndexedImpl(const PayloadType& payloadType, const TagsPath& fieldPath) : payloadType_{payloadType}, fieldPath_{fieldPath} {} + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] std::string ConditionStr() const; [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType /*rowId*/) { ConstPayload{payloadType_, item}.GetByJsonPath(fieldPath_, buffer_, KeyValueType::Undefined{}); @@ -238,6 +282,11 @@ class ComparatorNotIndexedImpl : private ComparatorNotIndexedIm public: ComparatorNotIndexedImpl(const PayloadType& payloadType, const TagsPath& fieldPath) : Base{payloadType, fieldPath} {} + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] bool IsDistinct() const noexcept { return true; } using Base::ClearDistinctValues; using Base::ExcludeDistinctValues; @@ -249,6 +298,11 @@ template <> class ComparatorNotIndexedImpl { public: ComparatorNotIndexedImpl(const VariantArray& values, const PayloadType& payloadType, const TagsPath& fieldPath); + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] std::string ConditionStr() const; [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType /*rowId*/) { ConstPayload{payloadType_, item}.GetByJsonPath(fieldPath_, buffer_, KeyValueType::Undefined{}); @@ -276,6 +330,11 @@ class ComparatorNotIndexedImpl : private ComparatorNotIndexed public: ComparatorNotIndexedImpl(const VariantArray& values, const PayloadType& payloadType, const TagsPath& fieldPath) : Base{values, payloadType, fieldPath} {} + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType /*rowId*/) { ConstPayload{payloadType_, item}.GetByJsonPath(fieldPath_, buffer_, KeyValueType::Undefined{}); if (buffer_.size() != 2 || !buffer_[0].Type().Is() || !buffer_[0].Type().Is()) { @@ -311,6 +370,11 @@ class ComparatorNotIndexedImpl { ++i; } } + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] std::string ConditionStr() const; [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType /*rowId*/) { allSetValues_.clear(); @@ -337,7 +401,7 @@ class ComparatorNotIndexedImpl { MultiHashMap::indexesCount, RelaxedHasher, RelaxedComparator> values_; - std::unordered_set allSetValues_; + fast_hash_set allSetValues_; }; template <> @@ -347,6 +411,11 @@ class ComparatorNotIndexedImpl : private ComparatorNotIndexedI public: ComparatorNotIndexedImpl(const VariantArray& values, const PayloadType& payloadType, const TagsPath& fieldPath) : Base{values, payloadType, fieldPath} {} + ComparatorNotIndexedImpl(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl& operator=(const ComparatorNotIndexedImpl&) = delete; + ComparatorNotIndexedImpl(ComparatorNotIndexedImpl&&) = default; + ComparatorNotIndexedImpl& operator=(ComparatorNotIndexedImpl&&) = default; + [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType /*rowId*/) { allSetValues_.clear(); ConstPayload{payloadType_, item}.GetByJsonPath(fieldPath_, buffer_, KeyValueType::Undefined{}); @@ -384,13 +453,17 @@ using ComparatorNotIndexedVariant = std::variant< ComparatorNotIndexedImpl, ComparatorNotIndexedImpl, ComparatorNotIndexedImpl, ComparatorNotIndexedImpl, ComparatorNotIndexedImpl, ComparatorNotIndexedImpl, ComparatorNotIndexedImpl, ComparatorNotIndexedImpl, ComparatorNotIndexedImpl>; + +using ComparatorNotIndexedVariantWrp = intrusive_rc_wrapper; + } // namespace comparators class ComparatorNotIndexed { public: ComparatorNotIndexed(std::string_view fieldName, CondType cond, const VariantArray& values, const PayloadType& payloadType, const TagsPath& fieldPath, bool distinct) - : impl_{createImpl(cond, values, payloadType, fieldPath, distinct)}, fieldName_{fieldName} {} + : impl_{make_intrusive(createImpl(cond, values, payloadType, fieldPath, distinct))}, + fieldName_{fieldName} {} [[nodiscard]] const std::string& Name() const& noexcept { return fieldName_; } auto Name() const&& = delete; [[nodiscard]] std::string ConditionStr() const; @@ -405,78 +478,78 @@ class ComparatorNotIndexed { [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const PayloadValue& item, IdType rowId) { static_assert(std::variant_size_v == 24); bool res; - switch (impl_.index()) { + switch (impl_->index()) { case 0: - res = std::get_if<0>(&impl_)->Compare(item, rowId); + res = std::get_if<0>(impl_.get())->Compare(item, rowId); break; case 1: - res = std::get_if<1>(&impl_)->Compare(item, rowId); + res = std::get_if<1>(impl_.get())->Compare(item, rowId); break; case 2: - res = std::get_if<2>(&impl_)->Compare(item, rowId); + res = std::get_if<2>(impl_.get())->Compare(item, rowId); break; case 3: - res = std::get_if<3>(&impl_)->Compare(item, rowId); + res = std::get_if<3>(impl_.get())->Compare(item, rowId); break; case 4: - res = std::get_if<4>(&impl_)->Compare(item, rowId); + res = std::get_if<4>(impl_.get())->Compare(item, rowId); break; case 5: - res = std::get_if<5>(&impl_)->Compare(item, rowId); + res = std::get_if<5>(impl_.get())->Compare(item, rowId); break; case 6: - res = std::get_if<6>(&impl_)->Compare(item, rowId); + res = std::get_if<6>(impl_.get())->Compare(item, rowId); break; case 7: - res = std::get_if<7>(&impl_)->Compare(item, rowId); + res = std::get_if<7>(impl_.get())->Compare(item, rowId); break; case 8: - res = std::get_if<8>(&impl_)->Compare(item, rowId); + res = std::get_if<8>(impl_.get())->Compare(item, rowId); break; case 9: - res = std::get_if<9>(&impl_)->Compare(item, rowId); + res = std::get_if<9>(impl_.get())->Compare(item, rowId); break; case 10: - res = std::get_if<10>(&impl_)->Compare(item, rowId); + res = std::get_if<10>(impl_.get())->Compare(item, rowId); break; case 11: - res = std::get_if<11>(&impl_)->Compare(item, rowId); + res = std::get_if<11>(impl_.get())->Compare(item, rowId); break; case 12: - res = std::get_if<12>(&impl_)->Compare(item, rowId); + res = std::get_if<12>(impl_.get())->Compare(item, rowId); break; case 13: - res = std::get_if<13>(&impl_)->Compare(item, rowId); + res = std::get_if<13>(impl_.get())->Compare(item, rowId); break; case 14: - res = std::get_if<14>(&impl_)->Compare(item, rowId); + res = std::get_if<14>(impl_.get())->Compare(item, rowId); break; case 15: - res = std::get_if<15>(&impl_)->Compare(item, rowId); + res = std::get_if<15>(impl_.get())->Compare(item, rowId); break; case 16: - res = std::get_if<16>(&impl_)->Compare(item, rowId); + res = std::get_if<16>(impl_.get())->Compare(item, rowId); break; case 17: - res = std::get_if<17>(&impl_)->Compare(item, rowId); + res = std::get_if<17>(impl_.get())->Compare(item, rowId); break; case 18: - res = std::get_if<18>(&impl_)->Compare(item, rowId); + res = std::get_if<18>(impl_.get())->Compare(item, rowId); break; case 19: - res = std::get_if<19>(&impl_)->Compare(item, rowId); + res = std::get_if<19>(impl_.get())->Compare(item, rowId); break; case 20: - res = std::get_if<20>(&impl_)->Compare(item, rowId); + res = std::get_if<20>(impl_.get())->Compare(item, rowId); break; case 21: - res = std::get_if<21>(&impl_)->Compare(item, rowId); + res = std::get_if<21>(impl_.get())->Compare(item, rowId); break; case 22: - res = std::get_if<22>(&impl_)->Compare(item, rowId); + res = std::get_if<22>(impl_.get())->Compare(item, rowId); break; case 23: - res = std::get_if<23>(&impl_)->Compare(item, rowId); + res = std::get_if<23>(impl_.get())->Compare(item, rowId); break; default: abort(); @@ -486,122 +559,124 @@ class ComparatorNotIndexed { } void ClearDistinctValues() noexcept { static_assert(std::variant_size_v == 24); - switch (impl_.index()) { + switch (impl_->index()) { case 0: - return std::get_if<0>(&impl_)->ClearDistinctValues(); + return std::get_if<0>(impl_.get())->ClearDistinctValues(); case 1: - return std::get_if<1>(&impl_)->ClearDistinctValues(); + return std::get_if<1>(impl_.get())->ClearDistinctValues(); case 2: - return std::get_if<2>(&impl_)->ClearDistinctValues(); + return std::get_if<2>(impl_.get())->ClearDistinctValues(); case 3: - return std::get_if<3>(&impl_)->ClearDistinctValues(); + return std::get_if<3>(impl_.get())->ClearDistinctValues(); case 4: - return std::get_if<4>(&impl_)->ClearDistinctValues(); + return std::get_if<4>(impl_.get())->ClearDistinctValues(); case 5: - return std::get_if<5>(&impl_)->ClearDistinctValues(); + return std::get_if<5>(impl_.get())->ClearDistinctValues(); case 6: - return std::get_if<6>(&impl_)->ClearDistinctValues(); + return std::get_if<6>(impl_.get())->ClearDistinctValues(); case 7: - return std::get_if<7>(&impl_)->ClearDistinctValues(); + return std::get_if<7>(impl_.get())->ClearDistinctValues(); case 8: - return std::get_if<8>(&impl_)->ClearDistinctValues(); + return std::get_if<8>(impl_.get())->ClearDistinctValues(); case 9: - return std::get_if<9>(&impl_)->ClearDistinctValues(); + return std::get_if<9>(impl_.get())->ClearDistinctValues(); case 10: - return std::get_if<10>(&impl_)->ClearDistinctValues(); + return std::get_if<10>(impl_.get())->ClearDistinctValues(); case 11: - return std::get_if<11>(&impl_)->ClearDistinctValues(); + return std::get_if<11>(impl_.get())->ClearDistinctValues(); case 12: - return std::get_if<12>(&impl_)->ClearDistinctValues(); + return std::get_if<12>(impl_.get())->ClearDistinctValues(); case 13: - return std::get_if<13>(&impl_)->ClearDistinctValues(); + return std::get_if<13>(impl_.get())->ClearDistinctValues(); case 14: - return std::get_if<14>(&impl_)->ClearDistinctValues(); + return std::get_if<14>(impl_.get())->ClearDistinctValues(); case 15: - return std::get_if<15>(&impl_)->ClearDistinctValues(); + return std::get_if<15>(impl_.get())->ClearDistinctValues(); case 16: - return std::get_if<16>(&impl_)->ClearDistinctValues(); + return std::get_if<16>(impl_.get())->ClearDistinctValues(); case 17: - return std::get_if<17>(&impl_)->ClearDistinctValues(); + return std::get_if<17>(impl_.get())->ClearDistinctValues(); case 18: - return std::get_if<18>(&impl_)->ClearDistinctValues(); + return std::get_if<18>(impl_.get())->ClearDistinctValues(); case 19: - return std::get_if<19>(&impl_)->ClearDistinctValues(); + return std::get_if<19>(impl_.get())->ClearDistinctValues(); case 20: - return std::get_if<20>(&impl_)->ClearDistinctValues(); + return std::get_if<20>(impl_.get())->ClearDistinctValues(); case 21: - return std::get_if<21>(&impl_)->ClearDistinctValues(); + return std::get_if<21>(impl_.get())->ClearDistinctValues(); case 22: - return std::get_if<22>(&impl_)->ClearDistinctValues(); + return std::get_if<22>(impl_.get())->ClearDistinctValues(); case 23: - return std::get_if<23>(&impl_)->ClearDistinctValues(); + return std::get_if<23>(impl_.get())->ClearDistinctValues(); default: abort(); } } void ExcludeDistinctValues(const PayloadValue& item, IdType rowId) { static_assert(std::variant_size_v == 24); - switch (impl_.index()) { + switch (impl_->index()) { case 0: - return std::get_if<0>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<0>(impl_.get())->ExcludeDistinctValues(item, rowId); case 1: - return std::get_if<1>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<1>(impl_.get())->ExcludeDistinctValues(item, rowId); case 2: - return std::get_if<2>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<2>(impl_.get())->ExcludeDistinctValues(item, rowId); case 3: - return std::get_if<3>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<3>(impl_.get())->ExcludeDistinctValues(item, rowId); case 4: - return std::get_if<4>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<4>(impl_.get())->ExcludeDistinctValues(item, rowId); case 5: - return std::get_if<5>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<5>(impl_.get())->ExcludeDistinctValues(item, rowId); case 6: - return std::get_if<6>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<6>(impl_.get())->ExcludeDistinctValues(item, rowId); case 7: - return std::get_if<7>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<7>(impl_.get())->ExcludeDistinctValues(item, rowId); case 8: - return std::get_if<8>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<8>(impl_.get())->ExcludeDistinctValues(item, rowId); case 9: - return std::get_if<9>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<9>(impl_.get())->ExcludeDistinctValues(item, rowId); case 10: - return std::get_if<10>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<10>(impl_.get())->ExcludeDistinctValues(item, rowId); case 11: - return std::get_if<11>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<11>(impl_.get())->ExcludeDistinctValues(item, rowId); case 12: - return std::get_if<12>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<12>(impl_.get())->ExcludeDistinctValues(item, rowId); case 13: - return std::get_if<13>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<13>(impl_.get())->ExcludeDistinctValues(item, rowId); case 14: - return std::get_if<14>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<14>(impl_.get())->ExcludeDistinctValues(item, rowId); case 15: - return std::get_if<15>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<15>(impl_.get())->ExcludeDistinctValues(item, rowId); case 16: - return std::get_if<16>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<16>(impl_.get())->ExcludeDistinctValues(item, rowId); case 17: - return std::get_if<17>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<17>(impl_.get())->ExcludeDistinctValues(item, rowId); case 18: - return std::get_if<18>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<18>(impl_.get())->ExcludeDistinctValues(item, rowId); case 19: - return std::get_if<19>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<19>(impl_.get())->ExcludeDistinctValues(item, rowId); case 20: - return std::get_if<20>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<20>(impl_.get())->ExcludeDistinctValues(item, rowId); case 21: - return std::get_if<21>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<21>(impl_.get())->ExcludeDistinctValues(item, rowId); case 22: - return std::get_if<22>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<22>(impl_.get())->ExcludeDistinctValues(item, rowId); case 23: - return std::get_if<23>(&impl_)->ExcludeDistinctValues(item, rowId); + return std::get_if<23>(impl_.get())->ExcludeDistinctValues(item, rowId); default: abort(); } } [[nodiscard]] bool IsDistinct() const noexcept { - return std::visit([](auto& impl) { return impl.IsDistinct(); }, impl_); + assertrx_dbg(dynamic_cast(impl_.get())); + return std::visit([](auto& impl) { return impl.IsDistinct(); }, *static_cast(impl_.get())); } private: - static comparators::ComparatorNotIndexedVariant createImpl(CondType, const VariantArray& values, const PayloadType&, const TagsPath&, - bool distinct); - comparators::ComparatorNotIndexedVariant impl_; + using ImplVariantType = comparators::ComparatorNotIndexedVariant; + static ImplVariantType createImpl(CondType, const VariantArray& values, const PayloadType&, const TagsPath&, bool distinct); + // Using pointer to reduce ExpressionTree Node size + intrusive_ptr impl_; int matchedCount_{0}; bool isNotOperation_{false}; std::string fieldName_; diff --git a/cpp_src/core/nsselecter/comparator/comparator_not_indexed_distinct.h b/cpp_src/core/nsselecter/comparator/comparator_not_indexed_distinct.h index e41f790f4..fa00b3095 100644 --- a/cpp_src/core/nsselecter/comparator/comparator_not_indexed_distinct.h +++ b/cpp_src/core/nsselecter/comparator/comparator_not_indexed_distinct.h @@ -1,33 +1,26 @@ #pragma once -#include -#include #include "core/keyvalue/variant.h" +#include "estl/fast_hash_set.h" namespace reindexer { class ComparatorNotIndexedDistinct { public: - ComparatorNotIndexedDistinct() : values_{make_intrusive()} {} - [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const Variant& v) const { - assertrx_dbg(values_); - return values_->find(v) == values_->cend(); - } - void ExcludeValues(Variant&& v) { - assertrx_dbg(values_); - values_->emplace(std::move(v)); - } - void ClearValues() noexcept { - assertrx_dbg(values_); - values_->clear(); - } + ComparatorNotIndexedDistinct() = default; + ComparatorNotIndexedDistinct(const ComparatorNotIndexedDistinct&) = delete; + ComparatorNotIndexedDistinct& operator=(const ComparatorNotIndexedDistinct&) = delete; + ComparatorNotIndexedDistinct(ComparatorNotIndexedDistinct&&) = default; + ComparatorNotIndexedDistinct& operator=(ComparatorNotIndexedDistinct&&) = default; + + [[nodiscard]] RX_ALWAYS_INLINE bool Compare(const Variant& v) const { return values_.find(v) == values_.cend(); } + void ExcludeValues(Variant&& v) { values_.emplace(std::move(v)); } + void ClearValues() noexcept { values_.clear(); } private: - using SetType = std::unordered_set; - using SetWrpType = intrusive_rc_wrapper; - using SetPtrType = intrusive_ptr; + using SetType = fast_hash_set; - SetPtrType values_; + SetType values_; }; } // namespace reindexer diff --git a/cpp_src/core/nsselecter/comparator/equalposition_comparator.cc b/cpp_src/core/nsselecter/comparator/equalposition_comparator.cc index e280835c2..4cd3e775e 100644 --- a/cpp_src/core/nsselecter/comparator/equalposition_comparator.cc +++ b/cpp_src/core/nsselecter/comparator/equalposition_comparator.cc @@ -6,18 +6,19 @@ namespace reindexer { -void EqualPositionComparator::BindField(const std::string& name, int field, const VariantArray& values, CondType cond, - const CollateOpts& collate) { +void EqualPositionComparatorImpl::BindField(const std::string& name, int field, const VariantArray& values, CondType cond, + const CollateOpts& collate) { bindField(name, field, values, cond, collate); } -void EqualPositionComparator::BindField(const std::string& name, const FieldsPath& fieldPath, const VariantArray& values, CondType cond) { +void EqualPositionComparatorImpl::BindField(const std::string& name, const FieldsPath& fieldPath, const VariantArray& values, + CondType cond) { bindField(name, fieldPath, values, cond, CollateOpts{}); } template -void EqualPositionComparator::bindField(const std::string& name, F field, const VariantArray& values, CondType cond, - const CollateOpts& collate) { +void EqualPositionComparatorImpl::bindField(const std::string& name, F field, const VariantArray& values, CondType cond, + const CollateOpts& collate) { fields_.push_back(field); Context& ctx = ctx_.emplace_back(collate); @@ -33,7 +34,7 @@ void EqualPositionComparator::bindField(const std::string& name, F field, const name_ += ' ' + name; } -bool EqualPositionComparator::Compare(const PayloadValue& pv, IdType /*rowId*/) { +bool EqualPositionComparatorImpl::Compare(const PayloadValue& pv, IdType /*rowId*/) { ConstPayload pl(payloadType_, pv); size_t len = INT_MAX; @@ -72,7 +73,7 @@ bool EqualPositionComparator::Compare(const PayloadValue& pv, IdType /*rowId*/) return false; } -bool EqualPositionComparator::compareField(size_t field, const Variant& v) { +bool EqualPositionComparatorImpl::compareField(size_t field, const Variant& v) { return v.Type().EvaluateOneOf( [&](KeyValueType::Bool) { return ctx_[field].cmpBool.Compare(ctx_[field].cond, static_cast(v)); }, [&](KeyValueType::Int) { return ctx_[field].cmpInt.Compare(ctx_[field].cond, static_cast(v)); }, diff --git a/cpp_src/core/nsselecter/comparator/equalposition_comparator.h b/cpp_src/core/nsselecter/comparator/equalposition_comparator.h index 83e33ea54..786f86db8 100644 --- a/cpp_src/core/nsselecter/comparator/equalposition_comparator.h +++ b/cpp_src/core/nsselecter/comparator/equalposition_comparator.h @@ -7,9 +7,13 @@ namespace reindexer { -class EqualPositionComparator { +class EqualPositionComparatorImpl : public intrusive_rc_base { public: - EqualPositionComparator(const PayloadType& payloadType) : payloadType_{payloadType}, name_{"EqualPositions"} {} + EqualPositionComparatorImpl(const PayloadType& payloadType) : payloadType_{payloadType}, name_{"EqualPositions"} {} + EqualPositionComparatorImpl(const EqualPositionComparatorImpl&) = delete; + EqualPositionComparatorImpl(EqualPositionComparatorImpl&&) = delete; + EqualPositionComparatorImpl& operator=(const EqualPositionComparatorImpl&) = delete; + EqualPositionComparatorImpl& operator=(EqualPositionComparatorImpl&&) = delete; void BindField(const std::string& name, int field, const VariantArray&, CondType, const CollateOpts&); void BindField(const std::string& name, const FieldsPath&, const VariantArray&, CondType); @@ -38,12 +42,12 @@ class EqualPositionComparator { struct Context { Context(const CollateOpts& collate) : cmpString{collate} {} CondType cond; - EqualPositionComparatorImpl cmpBool; - EqualPositionComparatorImpl cmpInt; - EqualPositionComparatorImpl cmpInt64; - EqualPositionComparatorImpl cmpDouble; - EqualPositionComparatorImpl cmpString; - EqualPositionComparatorImpl cmpUuid; + EqualPositionComparatorTypeImpl cmpBool; + EqualPositionComparatorTypeImpl cmpInt; + EqualPositionComparatorTypeImpl cmpInt64; + EqualPositionComparatorTypeImpl cmpDouble; + EqualPositionComparatorTypeImpl cmpString; + EqualPositionComparatorTypeImpl cmpUuid; }; std::vector ctx_; @@ -53,4 +57,30 @@ class EqualPositionComparator { int matchedCount_{0}; }; +class EqualPositionComparator { +public: + EqualPositionComparator(const PayloadType& payloadType) : impl_{make_intrusive(payloadType)} {} + + void BindField(const std::string& name, int field, const VariantArray& values, CondType cond, const CollateOpts& opts) { + return impl_->BindField(name, field, values, cond, opts); + } + void BindField(const std::string& name, const FieldsPath& fields, const VariantArray& values, CondType cond) { + return impl_->BindField(name, fields, values, cond); + } + bool Compare(const PayloadValue& pv, IdType id) { return impl_->Compare(pv, id); } + bool IsBinded() noexcept { return impl_->IsBinded(); } + [[nodiscard]] int GetMatchedCount() const noexcept { return impl_->GetMatchedCount(); } + [[nodiscard]] int FieldsCount() const noexcept { return impl_->FieldsCount(); } + [[nodiscard]] const std::string& Name() const& noexcept { return impl_->Name(); } + [[nodiscard]] const std::string& Dump() const& noexcept { return impl_->Name(); } + [[nodiscard]] double Cost(int expectedIterations) const noexcept { return impl_->Cost(expectedIterations); } + + auto Name() const&& = delete; + auto Dump() const&& = delete; + +private: + // Using pointer to reduce ExpressionTree Node size + intrusive_ptr impl_; +}; + } // namespace reindexer diff --git a/cpp_src/core/nsselecter/comparator/equalposition_comparator_impl.h b/cpp_src/core/nsselecter/comparator/equalposition_comparator_impl.h index ea5818181..e22ba6756 100644 --- a/cpp_src/core/nsselecter/comparator/equalposition_comparator_impl.h +++ b/cpp_src/core/nsselecter/comparator/equalposition_comparator_impl.h @@ -1,11 +1,10 @@ #pragma once #include -#include #include "core/index/string_map.h" #include "core/keyvalue/geometry.h" #include "core/keyvalue/p_string.h" -#include "estl/intrusive_ptr.h" +#include "estl/fast_hash_set.h" #include "estl/one_of.h" #include "tools/string_regexp_functions.h" #include "vendor/hopscotch/hopscotch_sc_set.h" @@ -13,19 +12,23 @@ namespace reindexer { template -class EqualPositionComparatorImpl { - using ValuesSet = intrusive_atomic_rc_wrapper>; - using AllSetValuesSet = intrusive_atomic_rc_wrapper>; +class EqualPositionComparatorTypeImpl { + using ValuesSet = fast_hash_set; + using AllSetValuesSet = fast_hash_set; public: - EqualPositionComparatorImpl() noexcept = default; + EqualPositionComparatorTypeImpl() noexcept = default; + EqualPositionComparatorTypeImpl(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl& operator=(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl(EqualPositionComparatorTypeImpl&&) = default; + EqualPositionComparatorTypeImpl& operator=(EqualPositionComparatorTypeImpl&&) = default; void SetValues(CondType cond, const VariantArray& values) { if (cond == CondSet) { - valuesS_.reset(new ValuesSet{}); + valuesS_ = std::make_unique(); } else if (cond == CondAllSet) { - valuesS_.reset(new ValuesSet{}); - allSetValuesS_.reset(new AllSetValuesSet{}); + valuesS_ = std::make_unique(); + allSetValuesS_ = std::make_unique(); } for (Variant key : values) { @@ -85,8 +88,8 @@ class EqualPositionComparatorImpl { } h_vector values_; - intrusive_ptr valuesS_; - intrusive_ptr allSetValuesS_; + std::unique_ptr valuesS_; + std::unique_ptr allSetValuesS_; private: KeyValueType type() { @@ -115,19 +118,23 @@ class EqualPositionComparatorImpl { }; template <> -class EqualPositionComparatorImpl { - using ValuesSet = intrusive_atomic_rc_wrapper>; - using AllSetValuesSet = intrusive_atomic_rc_wrapper>; +class EqualPositionComparatorTypeImpl { + using ValuesSet = fast_hash_set; + using AllSetValuesSet = fast_hash_set; public: - EqualPositionComparatorImpl() noexcept = default; + EqualPositionComparatorTypeImpl() noexcept = default; + EqualPositionComparatorTypeImpl(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl& operator=(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl(EqualPositionComparatorTypeImpl&&) = default; + EqualPositionComparatorTypeImpl& operator=(EqualPositionComparatorTypeImpl&&) = default; void SetValues(CondType cond, const VariantArray& values) { if (cond == CondSet) { - valuesS_.reset(new ValuesSet{}); + valuesS_ = std::make_unique(); } else if (cond == CondAllSet) { - valuesS_.reset(new ValuesSet{}); - allSetValuesS_.reset(new AllSetValuesSet{}); + valuesS_ = std::make_unique(); + allSetValuesS_ = std::make_unique(); } for (const Variant& key : values) { @@ -190,8 +197,8 @@ class EqualPositionComparatorImpl { } h_vector values_; - intrusive_ptr valuesS_; - intrusive_ptr allSetValuesS_; + std::unique_ptr valuesS_; + std::unique_ptr allSetValuesS_; private: void addValue(CondType cond, Uuid value) { @@ -204,16 +211,20 @@ class EqualPositionComparatorImpl { }; template <> -class EqualPositionComparatorImpl { +class EqualPositionComparatorTypeImpl { public: - EqualPositionComparatorImpl(const CollateOpts& collate) : collate_{collate} {} + EqualPositionComparatorTypeImpl(const CollateOpts& collate) : collate_{collate} {} + EqualPositionComparatorTypeImpl(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl& operator=(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl(EqualPositionComparatorTypeImpl&&) = default; + EqualPositionComparatorTypeImpl& operator=(EqualPositionComparatorTypeImpl&&) = default; void SetValues(CondType cond, const VariantArray& values) { if (cond == CondSet) { - valuesS_ = make_intrusive>(collate_); + valuesS_ = std::make_unique(collate_); } else if (cond == CondAllSet) { - valuesS_ = make_intrusive>(collate_); - allSetValuesS_ = make_intrusive>>(); + valuesS_ = std::make_unique(collate_); + allSetValuesS_ = std::make_unique>(); } for (Variant key : values) { @@ -279,8 +290,8 @@ class EqualPositionComparatorImpl { less_key_string(opts)) {} }; - intrusive_ptr> valuesS_; - intrusive_ptr>> allSetValuesS_; + std::unique_ptr valuesS_; + std::unique_ptr> allSetValuesS_; private: void addValue(CondType cond, const key_string& value) { @@ -297,15 +308,23 @@ class EqualPositionComparatorImpl { }; template <> -class EqualPositionComparatorImpl { +class EqualPositionComparatorTypeImpl { public: - EqualPositionComparatorImpl() = delete; + EqualPositionComparatorTypeImpl() = delete; + EqualPositionComparatorTypeImpl(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl& operator=(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl(EqualPositionComparatorTypeImpl&&) = default; + EqualPositionComparatorTypeImpl& operator=(EqualPositionComparatorTypeImpl&&) = default; }; template <> -class EqualPositionComparatorImpl { +class EqualPositionComparatorTypeImpl { public: - EqualPositionComparatorImpl() noexcept = default; + EqualPositionComparatorTypeImpl() noexcept = default; + EqualPositionComparatorTypeImpl(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl& operator=(const EqualPositionComparatorTypeImpl&) = delete; + EqualPositionComparatorTypeImpl(EqualPositionComparatorTypeImpl&&) = default; + EqualPositionComparatorTypeImpl& operator=(EqualPositionComparatorTypeImpl&&) = default; void SetValues(const VariantArray& values) { if (values.size() != 2) { diff --git a/cpp_src/core/nsselecter/comparator/fieldscomparator.cc b/cpp_src/core/nsselecter/comparator/fieldscomparator.cc index 6b84de8c0..54edeba82 100644 --- a/cpp_src/core/nsselecter/comparator/fieldscomparator.cc +++ b/cpp_src/core/nsselecter/comparator/fieldscomparator.cc @@ -68,7 +68,7 @@ class ArrayAdapter { namespace reindexer { -FieldsComparator::FieldsComparator(std::string_view lField, CondType cond, std::string_view rField, PayloadType plType) +FieldsComparatorImpl::FieldsComparatorImpl(std::string_view lField, CondType cond, std::string_view rField, PayloadType plType) : condition_{cond}, payloadType_{std::move(plType)} { switch (condition_) { case CondEq: @@ -92,7 +92,7 @@ FieldsComparator::FieldsComparator(std::string_view lField, CondType cond, std:: } template -bool FieldsComparator::compare(const LArr& lhs, const RArr& rhs) { +bool FieldsComparatorImpl::compare(const LArr& lhs, const RArr& rhs) { static constexpr bool needCompareTypes{std::is_same_v || std::is_same_v}; switch (condition_) { case CondRange: @@ -214,7 +214,7 @@ bool FieldsComparator::compare(const LArr& lhs, const RArr& rhs) { } } -bool FieldsComparator::compare(const PayloadValue& item, const Context& ctx) { +bool FieldsComparatorImpl::compare(const PayloadValue& item, const Context& ctx) { bool result; if (ctx.lCtx_.fields_.getTagsPathsLength() > 0) { VariantArray lhs; @@ -262,7 +262,7 @@ bool FieldsComparator::compare(const PayloadValue& item, const Context& ctx) { return result; } -void FieldsComparator::validateTypes(KeyValueType lType, KeyValueType rType) const { +void FieldsComparatorImpl::validateTypes(KeyValueType lType, KeyValueType rType) const { if (lType.IsSame(rType) || lType.Is() || rType.Is()) { return; } diff --git a/cpp_src/core/nsselecter/comparator/fieldscomparator.h b/cpp_src/core/nsselecter/comparator/fieldscomparator.h index 41a6fbd8f..d785b48c9 100644 --- a/cpp_src/core/nsselecter/comparator/fieldscomparator.h +++ b/cpp_src/core/nsselecter/comparator/fieldscomparator.h @@ -10,9 +10,14 @@ namespace reindexer { -class FieldsComparator { +class FieldsComparatorImpl : public intrusive_rc_base { public: - FieldsComparator(std::string_view lField, CondType cond, std::string_view rField, PayloadType plType); + FieldsComparatorImpl(std::string_view lField, CondType cond, std::string_view rField, PayloadType plType); + FieldsComparatorImpl(const FieldsComparatorImpl&) = delete; + FieldsComparatorImpl(FieldsComparatorImpl&&) = delete; + FieldsComparatorImpl& operator=(const FieldsComparatorImpl&) = delete; + FieldsComparatorImpl& operator=(FieldsComparatorImpl&&) = delete; + bool Compare(const PayloadValue& item, IdType /*rowId*/) { if (ctx_.size() > 1) { for (const auto& c : ctx_) { @@ -155,4 +160,26 @@ class FieldsComparator { bool leftFieldSet = false; }; +class FieldsComparator { +public: + FieldsComparator(std::string_view lField, CondType cond, std::string_view rField, PayloadType plType) + : impl_{make_intrusive(lField, cond, rField, plType)} {} + bool Compare(const PayloadValue& item, IdType rowId) { return impl_->Compare(item, rowId); } + double Cost(int expectedIterations) const noexcept { return impl_->Cost(expectedIterations); } + const std::string& Name() const& noexcept { return impl_->Name(); } + const std::string& Name() const&& = delete; + const std::string& Dump() const& noexcept { return impl_->Dump(); } + const std::string& Dump() const&& = delete; + int GetMatchedCount() const noexcept { return impl_->GetMatchedCount(); } + void SetLeftField(const FieldsSet& fields) { return impl_->SetLeftField(fields); } + void SetRightField(const FieldsSet& fields) { return impl_->SetRightField(fields); } + void SetLeftField(const FieldsSet& fset, KeyValueType type, bool isArray) { return impl_->SetLeftField(fset, type, isArray); } + void SetRightField(const FieldsSet& fset, KeyValueType type, bool isArray) { return impl_->SetRightField(fset, type, isArray); } + void SetCollateOpts(const CollateOpts& cOpts) { impl_->SetCollateOpts(cOpts); } + +private: + // Using pointer to reduce ExpressionTree Node size + intrusive_ptr impl_; +}; + } // namespace reindexer diff --git a/cpp_src/core/nsselecter/nsselecter.cc b/cpp_src/core/nsselecter/nsselecter.cc index 984c4a78a..827bc19f6 100644 --- a/cpp_src/core/nsselecter/nsselecter.cc +++ b/cpp_src/core/nsselecter/nsselecter.cc @@ -191,7 +191,11 @@ void NsSelecter::operator()(QueryResults& result, SelectCtxWithJoinPreSelectType()); isIndexSparse = index->Opts().IsSparse(); Index::SelectOpts opts; + opts.ftSortType = ftSortType; opts.itemsCountInNamespace = ns.itemsCount(); if (!ns.SortOrdersBuilt()) { opts.disableIdSetCache = 1; @@ -512,15 +513,15 @@ std::vector SelectIteratorContainer::pr return result; } -void SelectIteratorContainer::PrepareIteratorsForSelectLoop(QueryPreprocessor& qPreproc, unsigned sortId, bool isFt, +void SelectIteratorContainer::PrepareIteratorsForSelectLoop(QueryPreprocessor& qPreproc, unsigned sortId, bool isFt, FtSortType ftSortType, const NamespaceImpl& ns, SelectFunction::Ptr& selectFnc, FtCtx::Ptr& ftCtx, const RdxContext& rdxCtx) { - prepareIteratorsForSelectLoop(qPreproc, 0, qPreproc.Size(), sortId, isFt, ns, selectFnc, ftCtx, rdxCtx); + prepareIteratorsForSelectLoop(qPreproc, 0, qPreproc.Size(), sortId, isFt, ftSortType, ns, selectFnc, ftCtx, rdxCtx); } bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor& qPreproc, size_t begin, size_t end, unsigned sortId, - bool isQueryFt, const NamespaceImpl& ns, SelectFunction::Ptr& selectFnc, - FtCtx::Ptr& ftCtx, const RdxContext& rdxCtx) { + bool isQueryFt, FtSortType ftSortType, const NamespaceImpl& ns, + SelectFunction::Ptr& selectFnc, FtCtx::Ptr& ftCtx, const RdxContext& rdxCtx) { const auto& queries = qPreproc.GetQueryEntries(); auto equalPositions = prepareEqualPositions(queries, begin, end); bool sortIndexFound = false; @@ -534,7 +535,7 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor& q [&](const QueryEntriesBracket&) { OpenBracket(op); const bool contFT = - prepareIteratorsForSelectLoop(qPreproc, i + 1, next, sortId, isQueryFt, ns, selectFnc, ftCtx, rdxCtx); + prepareIteratorsForSelectLoop(qPreproc, i + 1, next, sortId, isQueryFt, ftSortType, ns, selectFnc, ftCtx, rdxCtx); if (contFT && (op == OpOr || (next < end && queries.GetOperation(next) == OpOr))) { throw Error(errLogic, "OR operation is not allowed with bracket containing fulltext index"); } @@ -560,8 +561,8 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor& q } sortIndexFound = true; } - selectResults = processQueryEntry(qe, enableSortIndexOptimize, ns, sortId, isQueryFt, selectFnc, isIndexFt, - isIndexSparse, ftCtx, qPreproc, rdxCtx); + selectResults = processQueryEntry(qe, enableSortIndexOptimize, ns, sortId, isQueryFt, ftSortType, selectFnc, + isIndexFt, isIndexSparse, ftCtx, qPreproc, rdxCtx); } else { auto strictMode = ns.config_.strictMode; if (ctx_) { @@ -670,11 +671,14 @@ bool SelectIteratorContainer::checkIfSatisfyAllConditions(iterator begin, iterat }, [&] RX_PRE_LMBD_ALWAYS_INLINE(SelectIterator & sit) RX_POST_LMBD_ALWAYS_INLINE { return checkIfSatisfyCondition(sit, &lastFinish, rowId); }, - [&] RX_PRE_LMBD_ALWAYS_INLINE(JoinSelectIterator & jit) - RX_POST_LMBD_ALWAYS_INLINE { return checkIfSatisfyCondition(jit, pv, properRowId, match); }, + [&] /*RX_PRE_LMBD_ALWAYS_INLINE*/ (JoinSelectIterator & jit) /*RX_POST_LMBD_ALWAYS_INLINE*/ { + return checkIfSatisfyCondition(jit, pv, properRowId, match); + }, Restricted>{}( - [&pv, properRowId] RX_PRE_LMBD_ALWAYS_INLINE(auto& c) RX_POST_LMBD_ALWAYS_INLINE { return c.Compare(pv, properRowId); }), + [&pv, properRowId] /*RX_PRE_LMBD_ALWAYS_INLINE*/ (auto& c) /*RX_POST_LMBD_ALWAYS_INLINE*/ { + return c.Compare(pv, properRowId); + }), [] RX_PRE_LMBD_ALWAYS_INLINE(AlwaysTrue&) RX_POST_LMBD_ALWAYS_INLINE noexcept { return true; }); if (op == OpOr) { result |= lastResult; diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.h b/cpp_src/core/nsselecter/selectiteratorcontainer.h index 4f23b5446..5ec4cc240 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.h +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.h @@ -52,8 +52,8 @@ class SelectIteratorContainer void CheckFirstQuery(); // Let iterators choose most effecive algorith void SetExpectMaxIterations(int expectedIterations); - void PrepareIteratorsForSelectLoop(QueryPreprocessor&, unsigned sortId, bool isFt, const NamespaceImpl&, SelectFunction::Ptr&, - FtCtx::Ptr&, const RdxContext&); + void PrepareIteratorsForSelectLoop(QueryPreprocessor&, unsigned sortId, bool isFt, FtSortType ftSortType, const NamespaceImpl&, + SelectFunction::Ptr&, FtCtx::Ptr&, const RdxContext&); template bool Process(PayloadValue&, bool* finish, IdType* rowId, IdType, bool match); @@ -92,8 +92,8 @@ class SelectIteratorContainer } private: - bool prepareIteratorsForSelectLoop(QueryPreprocessor&, size_t begin, size_t end, unsigned sortId, bool isFt, const NamespaceImpl&, - SelectFunction::Ptr&, FtCtx::Ptr&, const RdxContext&); + bool prepareIteratorsForSelectLoop(QueryPreprocessor&, size_t begin, size_t end, unsigned sortId, bool isFt, FtSortType ftSortType, + const NamespaceImpl&, SelectFunction::Ptr&, FtCtx::Ptr&, const RdxContext&); void sortByCost(span indexes, span costs, unsigned from, unsigned to, int expectedIterations); double fullCost(span indexes, unsigned i, unsigned from, unsigned to, int expectedIterations) const noexcept; double cost(span indexes, unsigned cur, int expectedIterations) const noexcept; @@ -117,8 +117,8 @@ class SelectIteratorContainer SelectKeyResults processQueryEntry(const QueryEntry& qe, const NamespaceImpl& ns, StrictMode strictMode); SelectKeyResults processQueryEntry(const QueryEntry& qe, bool enableSortIndexOptimize, const NamespaceImpl& ns, unsigned sortId, - bool isQueryFt, SelectFunction::Ptr& selectFnc, bool& isIndexFt, bool& isIndexSparse, FtCtx::Ptr&, - QueryPreprocessor& qPreproc, const RdxContext&); + bool isQueryFt, FtSortType ftSortType, SelectFunction::Ptr& selectFnc, bool& isIndexFt, + bool& isIndexSparse, FtCtx::Ptr&, QueryPreprocessor& qPreproc, const RdxContext&); template void processField(FieldsComparator&, const QueryField&, const NamespaceImpl&) const; void processJoinEntry(const JoinQueryEntry&, OpType); diff --git a/cpp_src/core/nsselecter/substitutionhelpers.h b/cpp_src/core/nsselecter/substitutionhelpers.h index 3c89f197b..e605fb678 100644 --- a/cpp_src/core/nsselecter/substitutionhelpers.h +++ b/cpp_src/core/nsselecter/substitutionhelpers.h @@ -2,6 +2,7 @@ #include "core/index/index.h" #include "core/namespace/namespaceimpl.h" +#include "tools/logger.h" namespace reindexer { @@ -36,11 +37,32 @@ class CompositeSearcher { void Add(int field, const std::vector& composites, unsigned entry) { assertrx_throw(entry < std::numeric_limits::max()); + const auto compositesBeg = ns_.indexes_.firstCompositePos(); + const auto compositesEnd = compositesBeg + ns_.indexes_.compositeIndexesSize(); for (auto composite : composites) { - const auto idxType = ns_.indexes_[composite]->Type(); + if rx_unlikely (composite < compositesBeg || composite >= compositesEnd) { + // TODO: this may be removed later (somewhere around v3.31/v3.32) after some extra investigations (relates to #1830) + logFmt(LogError, + ": Unexpected composite index identifier during substitution attempt: {}. Composites range is " + "[{}, {});\n(field: {}; {})", + composite, compositesBeg, compositesEnd, field, ns_.payloadType_.Field(field).ToString()); + assertrx_dbg(false); + continue; + } + auto compositePtr = ns_.indexes_[composite].get(); + const auto idxType = compositePtr->Type(); if (idxType != IndexCompositeBTree && idxType != IndexCompositeHash) { continue; } + if (auto& idxFields = compositePtr->Fields(); !idxFields.contains(field)) { + // TODO: this may be removed later (somewhere around v3.31/v3.32) after some extra investigations (relates to #1830) + logFmt(LogError, + ": Unexpected field {} in composite index {}:{} during substitution attempt. Actual composite " + "fields: {}", + field, composite, compositePtr->Name(), idxFields.ToString(FieldsSet::DumpWithMask::No)); + assertrx_dbg(false); + continue; + } bool found = false; for (auto& d : d_) { if (d.idx == composite) { diff --git a/cpp_src/core/payload/fieldsset.cc b/cpp_src/core/payload/fieldsset.cc index 6f929ab16..3a2dfeee7 100644 --- a/cpp_src/core/payload/fieldsset.cc +++ b/cpp_src/core/payload/fieldsset.cc @@ -13,6 +13,12 @@ FieldsSet::FieldsSet(const TagsMatcher& tagsMatcher, const h_vector - void Dump(T& os) const { + void Dump(T& os, DumpWithMask withMask) const { const DumpFieldsPath fieldsPathDumper{os}; os << "{["; for (auto b = begin(), it = b, e = end(); it != e; ++it) { @@ -228,7 +230,11 @@ class FieldsSet : protected base_fields_set { } os << *it; } - os << "], mask: " << mask_ << ", tagsPaths: ["; + os << "], "; + if (withMask == DumpWithMask::Yes) { + os << "mask: " << mask_ << ", "; + } + os << "tagsPaths: ["; for (auto b = tagsPaths_.cbegin(), it = b, e = tagsPaths_.cend(); it != e; ++it) { if (it != b) { os << ", "; @@ -245,6 +251,7 @@ class FieldsSet : protected base_fields_set { } os << "]}"; } + std::string ToString(DumpWithMask withMask) const; private: template diff --git a/cpp_src/core/payload/payloadfieldtype.cc b/cpp_src/core/payload/payloadfieldtype.cc index 36d83f02f..a66ce588f 100644 --- a/cpp_src/core/payload/payloadfieldtype.cc +++ b/cpp_src/core/payload/payloadfieldtype.cc @@ -1,4 +1,5 @@ #include "payloadfieldtype.h" +#include #include "core/keyvalue/p_string.h" #include "core/keyvalue/uuid.h" #include "estl/one_of.h" @@ -39,4 +40,17 @@ size_t PayloadFieldType::Alignof() const noexcept { }); } +std::string PayloadFieldType::ToString() const { + std::stringstream ss; + ss << "{ type: " << type_.Name() << ", name: " << name_ << ", jsonpaths: ["; + for (auto jit = jsonPaths_.cbegin(); jit != jsonPaths_.cend(); ++jit) { + if (jit != jsonPaths_.cbegin()) { + ss << ", "; + } + ss << *jit; + } + ss << "] }"; + return ss.str(); +} + } // namespace reindexer diff --git a/cpp_src/core/payload/payloadfieldtype.h b/cpp_src/core/payload/payloadfieldtype.h index b4cb5588c..438b3563b 100644 --- a/cpp_src/core/payload/payloadfieldtype.h +++ b/cpp_src/core/payload/payloadfieldtype.h @@ -24,6 +24,7 @@ class PayloadFieldType { const std::vector& JsonPaths() const& noexcept { return jsonPaths_; } const std::vector& JsonPaths() && = delete; void AddJsonPath(const std::string& jsonPath) { jsonPaths_.push_back(jsonPath); } + std::string ToString() const; private: KeyValueType type_; diff --git a/cpp_src/core/query/query.cc b/cpp_src/core/query/query.cc index 90f61590a..7fa15673e 100644 --- a/cpp_src/core/query/query.cc +++ b/cpp_src/core/query/query.cc @@ -168,6 +168,13 @@ void Query::Join(JoinedQuery&& jq) & { adoptNested(joinQueries_.back()); } +void Query::checkSetObjectValue(const Variant& value) const { + if (!value.Type().Is()) { + throw Error(errLogic, "Unexpected variant type in SetObject: %s. Expecting KeyValueType::String with JSON-content", + value.Type().Name()); + } +} + VariantArray Query::deserializeValues(Serializer& ser, CondType cond) { VariantArray values; auto cnt = ser.GetVarUint(); diff --git a/cpp_src/core/query/query.h b/cpp_src/core/query/query.h index 12b6ed234..ca459bd63 100644 --- a/cpp_src/core/query/query.h +++ b/cpp_src/core/query/query.h @@ -400,11 +400,8 @@ class Query { /// @param hasExpressions - true: value has expressions in it template >* = nullptr> Query& SetObject(Str&& field, VariantArray value, bool hasExpressions = false) & { - for (auto& it : value) { - if (!it.Type().Is()) { - throw Error(errLogic, "Unexpected variant type in SetObject: %s. Expecting KeyValueType::String with JSON-content", - it.Type().Name()); - } + for (const auto& it : value) { + checkSetObjectValue(it); } updateFields_.emplace_back(std::forward(field), std::move(value), FieldModeSetJson, hasExpressions); return *this; @@ -687,7 +684,8 @@ class Query { throw Error(errConflict, kAggregationWithSelectFieldsMsgError); } selectFilter_.insert(selectFilter_.begin(), l.begin(), l.end()); - selectFilter_.erase(std::remove_if(selectFilter_.begin(), selectFilter_.end(), [](const auto& s) { return s == "*"sv; }), + selectFilter_.erase( + std::remove_if(selectFilter_.begin(), selectFilter_.end(), [](const auto& s) { return s == "*"sv || s.empty(); }), selectFilter_.end()); return *this; } @@ -968,6 +966,7 @@ class Query { using OnHelper = OnHelperTempl; using OnHelperR = OnHelperTempl; + void checkSetObjectValue(const Variant& value) const; void deserialize(Serializer& ser, bool& hasJoinConditions); VariantArray deserializeValues(Serializer&, CondType); void checkSubQueryNoData() const; diff --git a/cpp_src/core/query/queryentry.cc b/cpp_src/core/query/queryentry.cc index 6e2d8fcb6..73feb9b4a 100644 --- a/cpp_src/core/query/queryentry.cc +++ b/cpp_src/core/query/queryentry.cc @@ -270,6 +270,12 @@ std::string BetweenFieldsQueryEntry::Dump() const { return std::string{ser.Slice()}; } +void BetweenFieldsQueryEntry::checkCondition(CondType cond) const { + if (cond == CondAny || cond == CondEmpty || cond == CondDWithin) { + throw Error{errLogic, "Condition '%s' is inapplicable between two fields", CondTypeToStr(cond)}; + } +} + void QueryEntries::serialize(CondType cond, const VariantArray& values, WrSerializer& ser) { ser.PutVarUint(cond); if (cond == CondDWithin) { @@ -910,6 +916,11 @@ std::string SubQueryFieldEntry::Dump(const std::vector& subQueries) const return ss.str(); } +void SubQueryFieldEntry::checkCondition(CondType cond) const { + if (cond == CondAny || cond == CondEmpty) { + throw Error{errQueryExec, "Condition %s with field and subquery", cond == CondAny ? "Any" : "Empty"}; + } +} template void QueryEntries::dump(size_t level, const_iterator begin, const_iterator end, const std::vector& joinedSelectors, const std::vector& subQueries, WrSerializer& ser) { diff --git a/cpp_src/core/query/queryentry.h b/cpp_src/core/query/queryentry.h index e8c7969fe..202b69f3d 100644 --- a/cpp_src/core/query/queryentry.h +++ b/cpp_src/core/query/queryentry.h @@ -20,8 +20,8 @@ using ConstPayload = PayloadIface; class TagsMatcher; struct JoinQueryEntry { - JoinQueryEntry(size_t joinIdx) noexcept : joinIndex{joinIdx} {} - size_t joinIndex; + explicit JoinQueryEntry(size_t joinIdx) noexcept : joinIndex{joinIdx} {} + size_t joinIndex{std::numeric_limits::max()}; bool operator==(const JoinQueryEntry& other) const noexcept { return joinIndex == other.joinIndex; } bool operator!=(const JoinQueryEntry& other) const noexcept { return !operator==(other); } @@ -38,8 +38,6 @@ class QueryField { template explicit QueryField(Str&& fieldName) noexcept : fieldName_{std::forward(fieldName)} {} - QueryField(std::string&& fieldName, int idxNo, FieldsSet fields, KeyValueType fieldType, - std::vector&& compositeFieldsTypes); QueryField(QueryField&&) noexcept = default; QueryField(const QueryField&) = default; QueryField& operator=(QueryField&&) noexcept = default; @@ -160,9 +158,7 @@ class QueryEntry : private QueryField { [[nodiscard]] std::string Dump() const; [[nodiscard]] std::string DumpBrief() const; - [[nodiscard]] bool IsInjectedFromMain() const noexcept { return injectedFrom_ == InjectedFromMain; } [[nodiscard]] bool IsInjectedFrom(size_t joinedQueryNo) const noexcept { return injectedFrom_ == joinedQueryNo; } - void InjectedFrom(size_t joinedQueryNo) noexcept { injectedFrom_ = joinedQueryNo; } auto Values() const&& = delete; auto FieldData() const&& = delete; @@ -172,7 +168,7 @@ class QueryEntry : private QueryField { void verifyNotIgnoringEmptyValues() const { VerifyQueryEntryValues(condition_, values_); } VariantArray values_; - CondType condition_; + CondType condition_{CondAny}; bool distinct_{false}; size_t injectedFrom_{NotInjected}; }; @@ -182,9 +178,7 @@ class BetweenFieldsQueryEntry { template BetweenFieldsQueryEntry(StrL&& fstIdx, CondType cond, StrR&& sndIdx) : leftField_{std::forward(fstIdx)}, rightField_{std::forward(sndIdx)}, condition_{cond} { - if (condition_ == CondAny || condition_ == CondEmpty || condition_ == CondDWithin) { - throw Error{errLogic, "Condition '%s' is inapplicable between two fields", CondTypeToStr(condition_)}; - } + checkCondition(cond); } [[nodiscard]] bool operator==(const BetweenFieldsQueryEntry&) const noexcept; @@ -224,9 +218,11 @@ class BetweenFieldsQueryEntry { auto RightFieldData() const&& = delete; private: + void checkCondition(CondType cond) const; + QueryField leftField_; QueryField rightField_; - CondType condition_; + CondType condition_{CondAny}; }; struct AlwaysFalse {}; @@ -264,9 +260,9 @@ class SubQueryEntry { auto Values() const&& = delete; private: - CondType condition_; + CondType condition_{CondAny}; // index of Query in Query::subQueries_ - size_t queryIndex_; + size_t queryIndex_{std::numeric_limits::max()}; VariantArray values_; }; @@ -274,9 +270,7 @@ class SubQueryFieldEntry { public: template SubQueryFieldEntry(Str&& field, CondType cond, size_t qIdx) : field_{std::forward(field)}, condition_{cond}, queryIndex_{qIdx} { - if (cond == CondAny || cond == CondEmpty) { - throw Error{errQueryExec, "Condition %s with field and subquery", cond == CondAny ? "Any" : "Empty"}; - } + checkCondition(cond); } [[nodiscard]] const std::string& FieldName() const& noexcept { return field_; } [[nodiscard]] std::string&& FieldName() && noexcept { return std::move(field_); } @@ -291,10 +285,12 @@ class SubQueryFieldEntry { auto FieldName() const&& = delete; private: + void checkCondition(CondType cond) const; + std::string field_; - CondType condition_; + CondType condition_{CondAny}; // index of Query in Query::subQueries_ - size_t queryIndex_; + size_t queryIndex_{std::numeric_limits::max()}; }; class UpdateEntry { @@ -319,8 +315,8 @@ class UpdateEntry { private: std::string column_; VariantArray values_; - FieldModifyMode mode_ = FieldModeSet; - bool isExpression_ = false; + FieldModifyMode mode_{FieldModeSet}; + bool isExpression_{false}; }; class QueryJoinEntry { @@ -379,11 +375,11 @@ class QueryJoinEntry { private: QueryField leftField_; QueryField rightField_; - const OpType op_; - const CondType condition_; - const bool reverseNamespacesOrder_; ///< controls SQL encoding order - ///< false: mainNs.index Condition joinNs.joinIndex - ///< true: joinNs.joinIndex Invert(Condition) mainNs.index + const OpType op_{OpOr}; + const CondType condition_{CondAny}; + const bool reverseNamespacesOrder_{false}; ///< controls SQL encoding order + ///< false: mainNs.index Condition joinNs.joinIndex + ///< true: joinNs.joinIndex Invert(Condition) mainNs.index }; enum class InjectionDirection : bool { IntoMain, FromMain }; @@ -393,7 +389,7 @@ class QueryEntries : public ExpressionTree { using Base = ExpressionTree; - QueryEntries(Base&& b) : Base{std::move(b)} {} + explicit QueryEntries(Base&& b) : Base{std::move(b)} {} public: QueryEntries() = default; @@ -402,7 +398,6 @@ class QueryEntries : public ExpressionTree& subQueries) const { serialize(cbegin(), cend(), ser, subQueries); } bool CheckIfSatisfyConditions(const ConstPayload& pl) const { return checkIfSatisfyConditions(cbegin(), cend(), pl); } static bool CheckIfSatisfyCondition(const VariantArray& lValues, CondType, const VariantArray& rValues); @@ -422,7 +417,6 @@ class QueryEntries : public ExpressionTree& subQueries); static void serialize(CondType, const VariantArray& values, WrSerializer&); static bool checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload&); @@ -473,11 +467,11 @@ class AggregateEntry { void SetOffset(unsigned); private: - AggType type_; + AggType type_{AggUnknown}; h_vector fields_; SortingEntries sortingEntries_; - unsigned limit_ = QueryEntry::kDefaultLimit; - unsigned offset_ = QueryEntry::kDefaultOffset; + unsigned limit_{QueryEntry::kDefaultLimit}; + unsigned offset_{QueryEntry::kDefaultOffset}; }; } // namespace reindexer diff --git a/cpp_src/core/query/sql/sqlencoder.cc b/cpp_src/core/query/sql/sqlencoder.cc index aabbbf98b..84210ef4f 100644 --- a/cpp_src/core/query/sql/sqlencoder.cc +++ b/cpp_src/core/query/sql/sqlencoder.cc @@ -203,9 +203,6 @@ WrSerializer& SQLEncoder::GetSQL(WrSerializer& ser, bool stripArgs) const { } } else { for (const auto& filter : query_.SelectFilters()) { - if (filter == distinctIndex) { - continue; - } if (needComma) { ser << ", "; } else { diff --git a/cpp_src/core/selectfunc/ctx/ftctx.h b/cpp_src/core/selectfunc/ctx/ftctx.h index 2031cb8d5..636ccc42a 100644 --- a/cpp_src/core/selectfunc/ctx/ftctx.h +++ b/cpp_src/core/selectfunc/ctx/ftctx.h @@ -8,6 +8,8 @@ namespace reindexer { +enum FtSortType { RankOnly, RankAndID, ExternalExpression }; + class FtCtx : public BaseFunctionCtx { public: typedef intrusive_ptr Ptr; diff --git a/cpp_src/debug/backtrace.h b/cpp_src/debug/backtrace.h index 6d367a6ab..d55d9bd38 100644 --- a/cpp_src/debug/backtrace.h +++ b/cpp_src/debug/backtrace.h @@ -2,6 +2,7 @@ #include #include +#include namespace reindexer { namespace debug { diff --git a/cpp_src/estl/charset.h b/cpp_src/estl/charset.h index b10cf80ef..43037bfc5 100644 --- a/cpp_src/estl/charset.h +++ b/cpp_src/estl/charset.h @@ -37,7 +37,7 @@ class Charset { WordT set_[kWordsCount] = {0}; }; -static_assert(std::numeric_limits::max() - std::numeric_limits::min() + 1 == Charset::max_values_count(), +static_assert(size_t(std::numeric_limits::max() - std::numeric_limits::min() + 1) == Charset::max_values_count(), "Expecting max uint8_t range of [0, 255] for the simplicity"); } // namespace reindexer::estl diff --git a/cpp_src/estl/intrusive_ptr.h b/cpp_src/estl/intrusive_ptr.h index 6c5a0bb38..512390be9 100644 --- a/cpp_src/estl/intrusive_ptr.h +++ b/cpp_src/estl/intrusive_ptr.h @@ -188,11 +188,11 @@ template class intrusive_atomic_rc_wrapper : public T { public: template - intrusive_atomic_rc_wrapper(Args&&... args) : T(std::forward(args)...), refcount(0) {} + intrusive_atomic_rc_wrapper(Args&&... args) : T(std::forward(args)...) {} intrusive_atomic_rc_wrapper& operator=(const intrusive_atomic_rc_wrapper&) = delete; protected: - std::atomic refcount; + std::atomic refcount{0}; friend void intrusive_ptr_add_ref<>(intrusive_atomic_rc_wrapper* x) noexcept; friend void intrusive_ptr_release<>(intrusive_atomic_rc_wrapper* x) noexcept; @@ -225,11 +225,11 @@ template class intrusive_rc_wrapper : public T { public: template - intrusive_rc_wrapper(Args&&... args) : T(std::forward(args)...), refcount(0) {} + intrusive_rc_wrapper(Args&&... args) : T(std::forward(args)...) {} intrusive_rc_wrapper& operator=(const intrusive_rc_wrapper&) = delete; protected: - int refcount; + int refcount{0}; friend void intrusive_ptr_add_ref<>(intrusive_rc_wrapper* x) noexcept; friend void intrusive_ptr_release<>(intrusive_rc_wrapper* x) noexcept; @@ -238,12 +238,11 @@ class intrusive_rc_wrapper : public T { class intrusive_atomic_rc_base { public: - intrusive_atomic_rc_base() noexcept : refcount(0) {} intrusive_atomic_rc_base& operator=(const intrusive_atomic_rc_base&) = delete; virtual ~intrusive_atomic_rc_base() = default; protected: - std::atomic refcount; + std::atomic refcount{0}; friend void intrusive_ptr_add_ref(intrusive_atomic_rc_base* x) noexcept; friend void intrusive_ptr_release(intrusive_atomic_rc_base* x) noexcept; @@ -267,6 +266,33 @@ inline bool intrusive_ptr_is_unique(intrusive_atomic_rc_base* x) noexcept { return !x || (x->refcount.load(std::memory_order_acquire) == 1); } +class intrusive_rc_base { +public: + intrusive_rc_base& operator=(const intrusive_rc_base&) = delete; + virtual ~intrusive_rc_base() = default; + +protected: + int refcount{0}; + + friend void intrusive_ptr_add_ref(intrusive_rc_base* x) noexcept; + friend void intrusive_ptr_release(intrusive_rc_base* x) noexcept; + friend bool intrusive_ptr_is_unique(intrusive_rc_base* x) noexcept; +}; + +inline void intrusive_ptr_add_ref(intrusive_rc_base* x) noexcept { + if (x) { + ++x->refcount; + } +} + +inline void intrusive_ptr_release(intrusive_rc_base* x) noexcept { + if (x && --x->refcount == 0) { + delete x; + } +} + +inline bool intrusive_ptr_is_unique(intrusive_rc_base* x) noexcept { return !x || (x->refcount == 1); } + template intrusive_ptr make_intrusive(Args&&... args) { return intrusive_ptr(new T(std::forward(args)...)); diff --git a/cpp_src/gtests/tests/fixtures/queries_api.h b/cpp_src/gtests/tests/fixtures/queries_api.h index 4c01e758f..bbca249b6 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.h +++ b/cpp_src/gtests/tests/fixtures/queries_api.h @@ -236,7 +236,8 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { template void ExecuteAndVerifyWithSql(Q&& query) { ExecuteAndVerify(query); - Query queryFromSql = Query::FromSQL(query.GetSQL()); + Query queryFromSql = Query::FromSQL(query.GetSQL()).Strict(query.GetStrictMode()).Debug(query.GetDebugLevel()); + ASSERT_EQ(query, queryFromSql); ExecuteAndVerify(std::move(queryFromSql)); } @@ -256,7 +257,8 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { template void ExecuteAndVerifyWithSql(Q&& query, QueryResults& qr) { ExecuteAndVerify(query, qr); - Query queryFromSql = Query::FromSQL(query.GetSQL()); + Query queryFromSql = Query::FromSQL(query.GetSQL()).Strict(query.GetStrictMode()).Debug(query.GetDebugLevel()); + ASSERT_EQ(query, queryFromSql); qr.Clear(); ExecuteAndVerify(std::move(queryFromSql), qr); } @@ -740,6 +742,12 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .Where(kFieldNameGenre, CondEq, randomGenre) .Distinct(distinct.c_str()) .Sort(kFieldNameYear, true)); + + ExecuteAndVerifyWithSql(Query(default_namespace) + .Select({distinct.c_str()}) + .Distinct(distinct.c_str()) + .Where(kFieldNameGenre, CondEq, randomGenre) + .Sort(kFieldNameYear, true)); } } diff --git a/cpp_src/gtests/tests/unit/grpcclient_test.cc b/cpp_src/gtests/tests/unit/grpcclient_test.cc index 7c0e5edec..f61870ea2 100644 --- a/cpp_src/gtests/tests/unit/grpcclient_test.cc +++ b/cpp_src/gtests/tests/unit/grpcclient_test.cc @@ -75,7 +75,7 @@ TEST_F(GrpcClientApi, SelectJSON) { for (auto field : object) { name = std::string_view(field.key); const auto& fieldValue(field.value); - if (name == "id") { + if (name == "id" || name == "age") { ASSERT_TRUE(fieldValue.getTag() == gason::JSON_NUMBER); } else if (name == "joined_test_namespace2") { ASSERT_TRUE(fieldValue.getTag() == gason::JSON_ARRAY); diff --git a/cpp_src/gtests/tests/unit/index_tuple_test.cc b/cpp_src/gtests/tests/unit/index_tuple_test.cc index f6ca60b6b..ce7729fe2 100644 --- a/cpp_src/gtests/tests/unit/index_tuple_test.cc +++ b/cpp_src/gtests/tests/unit/index_tuple_test.cc @@ -250,7 +250,7 @@ class IndexTupleTest : public ReindexerApi { } }; -TEST_F(IndexTupleTest, ScalarTest) { +TEST_F(IndexTupleTest, DISABLED_ScalarTest) { static const std::string ns = "testNSScalar"; const auto storage = CreateEmptyNamespace(ns); @@ -265,7 +265,7 @@ TEST_F(IndexTupleTest, ScalarTest) { ValidateReloadState(rt.reindexer, ns, R"({"id":%d,"text":"","int":0})", "reload ns (ScalarTest)", storage); } -TEST_F(IndexTupleTest, ScalarNestedTest) { +TEST_F(IndexTupleTest, DISABLED_ScalarNestedTest) { static const std::string ns = "testNSNested"; const auto storage = CreateEmptyNamespace(ns); @@ -343,7 +343,8 @@ TEST_F(IndexTupleTest, NestedUpdateTest) { "reload ns (NestedUpdateTest)", storage); } -TEST_F(IndexTupleTest, ArrayTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_ArrayTest) { static const std::string ns = "testNSArray"; const auto storage = CreateEmptyNamespace(ns); @@ -379,7 +380,8 @@ TEST_F(IndexTupleTest, ArrayTest) { "reload ns (ArrayTest)", storage); } -TEST_F(IndexTupleTest, ArrayNestedTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_ArrayNestedTest) { static const std::string ns = "testNSArrayObj"; const auto storage = CreateNamespace(ns); @@ -453,7 +455,8 @@ TEST_F(IndexTupleTest, ArrayInToArrayTest) { "reload ns (ArrayInToArrayTest)", storage); } -TEST_F(IndexTupleTest, NestedOrderingTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_NestedOrderingTest) { static const std::string ns = "testNSNestedOrdering"; const auto storage = CreateEmptyNamespace(ns); @@ -471,7 +474,8 @@ TEST_F(IndexTupleTest, NestedOrderingTest) { "reload ns (NestedDiffOrderingTest)", storage); } -TEST_F(IndexTupleTest, NullTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_NullTest) { static const std::string ns = "testNSNull"; const auto storage = CreateEmptyNamespace(ns); @@ -497,7 +501,8 @@ TEST_F(IndexTupleTest, NullTest) { "null values test", storage); } -TEST_F(IndexTupleTest, FailTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_FailTest) { static const std::string ns = "testNSFail"; const auto storage = CreateEmptyNamespace(ns); @@ -508,7 +513,8 @@ TEST_F(IndexTupleTest, FailTest) { ValidateReloadState(rt.reindexer, ns, R"({"id":%d,"obj":{"nest":0},"idx":false})", "reload ns (FailTest)", storage); } -TEST_F(IndexTupleTest, NestedArrayTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_NestedArrayTest) { static const std::string ns = "testNSNestedArray"; const auto storage = CreateArrayNamespace(ns); diff --git a/cpp_src/server/CMakeLists.txt b/cpp_src/server/CMakeLists.txt index 1b5565c4f..fb6179926 100644 --- a/cpp_src/server/CMakeLists.txt +++ b/cpp_src/server/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.10) project(reindexer_server_library) set (SWAGGER_VERSION "2.x") -set (GH_FACE_VERSION "3.28.0") +set (GH_FACE_VERSION "3.28.1") set (GH_FACE_TAG "v${GH_FACE_VERSION}") set (TARGET reindexer_server_library) set (SERVER_LIB_DIR ${PROJECT_BINARY_DIR} PARENT_SCOPE) diff --git a/cpp_src/server/contrib/server.md b/cpp_src/server/contrib/server.md index 69eb5e8a4..b7f205c7b 100644 --- a/cpp_src/server/contrib/server.md +++ b/cpp_src/server/contrib/server.md @@ -133,7 +133,7 @@ Reindexer is compact, fast and it does not have heavy dependencies. ### Version information -*Version* : 3.28.0 +*Version* : 3.28.1 ### License information diff --git a/cpp_src/server/contrib/server.yml b/cpp_src/server/contrib/server.yml index a8f29bd02..2b7660bf1 100644 --- a/cpp_src/server/contrib/server.yml +++ b/cpp_src/server/contrib/server.yml @@ -4,7 +4,7 @@ info: **Reindexer** is an embeddable, in-memory, document-oriented database with a high-level Query builder interface. Reindexer's goal is to provide fast search with complex queries. Reindexer is compact, fast and it does not have heavy dependencies. - version: "3.28.0" + version: "3.28.1" title: "Reindexer REST API" license: name: "Apache 2.0" diff --git a/cpp_src/tools/scope_guard.h b/cpp_src/tools/scope_guard.h new file mode 100644 index 000000000..e5631c9bc --- /dev/null +++ b/cpp_src/tools/scope_guard.h @@ -0,0 +1,20 @@ +#pragma once + +namespace reindexer { + +template +class ScopeGuard { +public: + ScopeGuard(F1&& onConstruct, F2&& onDestruct) noexcept : onDestruct_(std::move(onDestruct)) { onConstruct(); } + ~ScopeGuard() { onDestruct_(); } + +private: + F2 onDestruct_; +}; + +template +ScopeGuard MakeScopeGuard(F1&& onConstruct, F2&& onDestruct) noexcept { + return ScopeGuard(std::move(onConstruct), std::move(onDestruct)); +} + +} // namespace reindexer diff --git a/cpp_src/tools/string_regexp_functions.h b/cpp_src/tools/string_regexp_functions.h index 5bd7951e2..06cebe92c 100644 --- a/cpp_src/tools/string_regexp_functions.h +++ b/cpp_src/tools/string_regexp_functions.h @@ -1,7 +1,7 @@ #pragma once +#include #include -#include namespace reindexer { diff --git a/cpp_src/vendor/libbacktrace/dwarf.c b/cpp_src/vendor/libbacktrace/dwarf.c index 86f29e4fc..b760353f4 100644 --- a/cpp_src/vendor/libbacktrace/dwarf.c +++ b/cpp_src/vendor/libbacktrace/dwarf.c @@ -1,5 +1,7 @@ +#ifndef _MSC_VER #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" +#endif #ifdef __linux__ /* dwarf.c -- Get file/line information from DWARF for backtraces. @@ -3419,4 +3421,6 @@ int backtrace_dwarf_add(struct backtrace_state *state, uintptr_t base_address, c int ___dwarf_c_dummy_suppress_warning; #endif +#ifndef _MSC_VER #pragma GCC diagnostic pop +#endif diff --git a/test/composite_indexes_test.go b/test/composite_indexes_test.go index bd2c2ba58..48bfa2a99 100644 --- a/test/composite_indexes_test.go +++ b/test/composite_indexes_test.go @@ -909,6 +909,82 @@ func TestCompositeIndexesSubstitution(t *testing.T) { }, }, "") }) + + t.Run("substitution after other indexes drop", func(t *testing.T) { + checkSubstitution := func() { + it := DB.Query(ns). + Where("first1", reindexer.EQ, item.First1). + Where("third", reindexer.EQ, item.Third). + Explain().Exec(t) + require.NoError(t, it.Error()) + defer it.Close() + require.Equal(t, it.Count(), 1) + explainRes, err := it.GetExplainResults() + require.NoError(t, err) + require.NotNil(t, explainRes) + + printExplainRes(explainRes) + checkExplain(t, explainRes.Selectors, []expectedExplain{ + { + Field: "first1+third", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, + }, + }, "") + } + + err := DB.DropIndex(ns, "second1+second2") + require.NoError(t, err) + // Check substitution right after composite deletion + checkSubstitution() + + err = DB.DropIndex(ns, "second1") + require.NoError(t, err) + err = DB.DropIndex(ns, "second2") + require.NoError(t, err) + // Check substitution after other indexes deletion + checkSubstitution() + }) + + t.Run("no substitution after current index drop", func(t *testing.T) { + err := DB.DropIndex(ns, "first1+third") + require.NoError(t, err) + it := DB.Query(ns). + Where("first1", reindexer.EQ, item.First1). + Where("third", reindexer.EQ, item.Third). + Explain().Exec(t) + require.NoError(t, it.Error()) + defer it.Close() + require.Equal(t, it.Count(), 1) + explainRes, err := it.GetExplainResults() + require.NoError(t, err) + require.NotNil(t, explainRes) + + printExplainRes(explainRes) + checkExplain(t, explainRes.Selectors, []expectedExplain{ + { + Field: "-scan", + Method: "scan", + Matched: 1, + }, + { + Field: "first1", + FieldType: "indexed", + Method: "scan", + Matched: 1, + Comparators: 1, + }, + { + Field: "third", + FieldType: "indexed", + Method: "scan", + Matched: 1, + Comparators: 1, + }, + }, "") + }) } func TestCompositeIndexesBestSubstitution(t *testing.T) { diff --git a/test/encdec_test.go b/test/encdec_test.go index 628afa0f3..09f71e274 100644 --- a/test/encdec_test.go +++ b/test/encdec_test.go @@ -330,6 +330,8 @@ func TestEncDec(t *testing.T) { } func TestSingleElemToSlice(t *testing.T) { + t.Parallel() + ns := "test_single_elem_slice" item := SingleElemSliceItem{ ID: 1, @@ -359,6 +361,8 @@ func TestSingleElemToSlice(t *testing.T) { } func TestSlicesConcatenation(t *testing.T) { + t.Parallel() + ns := "test_slices_concatenation" item := SlicesConcatenationItem{ ID: 1,