From 5e1867c830af99df1940f521e781b6ca3f07e263 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Thu, 16 Sep 2021 15:23:53 +0800 Subject: [PATCH 01/38] stash --- src/storage/exec/IndexEdgeScanNode.h | 10 + src/storage/exec/IndexGathorNode.h | 19 ++ src/storage/exec/IndexNode.h | 26 ++ src/storage/exec/IndexProjectionNode.h | 8 + src/storage/exec/IndexScanNode2.cpp | 27 ++ src/storage/exec/IndexScanNode2.h | 332 +++++++++++++++++++ src/storage/exec/IndexVertexScanNode.cpp | 32 ++ src/storage/exec/IndexVertexScanNode.h | 108 ++++++ src/storage/index/LookupBaseProcessor2-inl.h | 8 + src/storage/index/LookupBaseProcessor2.h | 29 ++ src/storage/index/LookupProcessor2.cpp | 14 + src/storage/index/LookupProcessor2.h | 37 +++ 12 files changed, 650 insertions(+) create mode 100644 src/storage/exec/IndexEdgeScanNode.h create mode 100644 src/storage/exec/IndexGathorNode.h create mode 100644 src/storage/exec/IndexNode.h create mode 100644 src/storage/exec/IndexProjectionNode.h create mode 100644 src/storage/exec/IndexScanNode2.cpp create mode 100644 src/storage/exec/IndexScanNode2.h create mode 100644 src/storage/exec/IndexVertexScanNode.cpp create mode 100644 src/storage/exec/IndexVertexScanNode.h create mode 100644 src/storage/index/LookupBaseProcessor2-inl.h create mode 100644 src/storage/index/LookupBaseProcessor2.h create mode 100644 src/storage/index/LookupProcessor2.cpp create mode 100644 src/storage/index/LookupProcessor2.h diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h new file mode 100644 index 00000000000..473e4bd4c7e --- /dev/null +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -0,0 +1,10 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once + +namespace nebula { +namespace storage {} // namespace storage +} // namespace nebula diff --git a/src/storage/exec/IndexGathorNode.h b/src/storage/exec/IndexGathorNode.h new file mode 100644 index 00000000000..a36ba3e1c1e --- /dev/null +++ b/src/storage/exec/IndexGathorNode.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once +#include "storage/exec/RelNode.h" + +namespace nebula { +namespace storage { +class IndexGathorNode : public IterateNode { + public: + IndexGathorNode(); + + private: +}; +} // namespace storage + +} // namespace nebula diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h new file mode 100644 index 00000000000..d8f57f5cf71 --- /dev/null +++ b/src/storage/exec/IndexNode.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once +#include "common/base/StatusOr.h" +#include "common/datatypes/DataSet.h" +#include "interface/gen-cpp2/common_types.h" +#include "storage/CommonUtils.h" +namespace nebula { +namespace storage { +class IndexNode { + public: + explicit IndexNode(RuntimeContext* context); + virtual void init() = 0; + virtual nebula::cpp2::ErrorCode execute(PartitionID partId) = 0; + virtual StatusOr next() = 0; + virtual bool vailed() = 0; + + protected: + RuntimeContext* context_; + GraphSpaceID spaceId_; +}; +} // namespace storage +} // namespace nebula diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h new file mode 100644 index 00000000000..dc28011b425 --- /dev/null +++ b/src/storage/exec/IndexProjectionNode.h @@ -0,0 +1,8 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +namespace nebula { +namespace storage {} +} // namespace nebula diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp new file mode 100644 index 00000000000..d97fa377b13 --- /dev/null +++ b/src/storage/exec/IndexScanNode2.cpp @@ -0,0 +1,27 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#include "storage/exec/IndexScanNode2.h" + +namespace nebula { +namespace storage { +// Define of Path +// End of Path + +// Define of RangePath +// End of RangePath + +// Define of PrefixPath + +// End of PrefixPath + +// Define of PathBuilder + +// End of PathBuilder + +// Define of IndexScan +// End of IndexScan +} // namespace storage +} // namespace nebula diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h new file mode 100644 index 00000000000..67426369633 --- /dev/null +++ b/src/storage/exec/IndexScanNode2.h @@ -0,0 +1,332 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/DataSet.h" +#include "common/utils/IndexKeyUtils.h" +#include "folly/Optional.h" +#include "interface/gen-cpp2/meta_types.h" +#include "interface/gen-cpp2/storage_types.h" +#include "storage/CommonUtils.h" +#include "storage/exec/IndexNode.h" +namespace nebula { +namespace storage { +template +using Map = folly::F14FastMap; + +class Path; +class IndexScanNode : public IndexNode { + public: + IndexScanNode(RuntimeContext* context, + IndexID indexId, + const std::vector& columnHists) + : IndexNode(context), indexId_(indexId), columnHists_(columnHists) {} + nebula::cpp2::ErrorCode execute(PartitionID partId) override { + auto ret = resetIter(partId); + return ret; + } + StatusOr next(bool& hasNext) { + hasNext = true; + while (iter_ && iter_->valid()) { + if (!checkTTL()) { + continue; + } + auto q = path_->qualified(iter_->key()); + if (q == Path::Qualified::INCOMPATIBLE) { + continue; + } + if (q == Path::Qualified::COMPATIBLE) { + return decodeFromIndex(iter_->key()); + } + std::pair kv; + auto ret = getBaseData(iter_->key(), kv); + if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + continue; + } + Map rowData = decodeFromBase(kv.first, kv.second); + q = path_->qualified(rowData); + CHECK(q != Path::Qualified::UNCERTAIN); + if (q == Path::Qualified::INCOMPATIBLE) { + continue; + } + if (q == Path::Qualified::COMPATIBLE) { + Row row; + for (auto& col : requiredColumns_) { + row.emplace_back(std::move(rowData.at(col))); + } + } + } + hasNext = false; + return Status::OK(); + } + + protected: + virtual Row decodeFromIndex(folly::StringPiece key) = 0; + virtual nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, + std::pair& kv) = 0; + virtual Map decodeFromBase(const std::string& key, + const std::string& value) = 0; + virtual const meta::SchemaProviderIf* getSchema() = 0; + std::pair> ttlProps_; + std::unique_ptr>> ttlProps_; + bool checkTTL() { + if (iter_->val().empty() || ttlProps_.first == false) { + return true; + } + auto v = IndexKeyUtils::parseIndexTTL(iter_->val()); + if (CommonUtils::checkDataExpiredForTTL( + getSchema(), std::move(v), ttlProps_.second.second, ttlProps_.second.first)) { + return false; + } + return true; + } + nebula::cpp2::ErrorCode resetIter(PartitionID partId) { + path_->resetPart(partId); + nebula::cpp2::ErrorCode ret; + if (path_->isRange()) { + auto rangePath = dynamic_cast(path_.get()); + ret = kvstore_->range( + spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter_); + } else { + auto prefixPath = dynamic_cast(path_.get()); + ret = kvstore_->prefix(spaceId_, partId, prefixPath->getPrefixKey(), &iter_); + } + return ret; + } + const IndexID indexId_; + const std::vector& columnHists_; + std::unique_ptr path_; + std::unique_ptr iter_; + nebula::kvstore::KVStore* kvstore_; + std::vector requiredColumns_; +}; +/** + * Path + * + * Path表示一个Index查询范围（range或prefix）。 + */ +class Path { + public: + using ColumnTypeDef = ::nebula::meta::cpp2::ColumnTypeDef; + Path(std::shared_ptr index, + std::shared_ptr schema, + const std::vector& hints) + : index_(index), schema_(schema), hints_(hints) {} + enum class Qualified : int16_t { INCOMPATIBLE = 0, UNCERTAIN = 1, COMPATIBLE = 2 }; + using QualifiedFunction = std::function; + virtual Qualified qualified(const folly::StringPiece& key) { + Qualified ret = Qualified::COMPATIBLE; + for (auto& func : QFList_) { + ret = std::min(ret, func(key.toString())); + } + return ret; + } + virtual Qualified qualified(const Map& rowData) = 0; + virtual ~Path() = default; + virtual void resetPart(PartitionID partId) = 0; + virtual bool isRange() { return dynamic_cast(this) != nullptr; } + + protected: + std::string encodeValue(const Value& value, const ColumnTypeDef& colDef, std::string& key) { + std::string val; + if (value.type() == Value::Type::STRING) { + val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); + if (val.back() != '\0') { + QFList_.clear(); + QFList_.emplace_back([](const std::string& key) { return Qualified::UNCERTAIN; }); + } + } else { + val = IndexKeyUtils::encodeValue(value); + } + key.append(val); + return val; + } + std::vector QFList_; + std::shared_ptr index_; + std::shared_ptr schema_; + const std::vector& hints_; + std::string startKey_, endKey_; + bool eqToStart_, eqToEnd_; +}; +class PrefixPath : public Path { + public: + PrefixPath(std::shared_ptr index, + std::shared_ptr schema, + const std::vector& hints) + : Path(index, schema, hints) { + buildKey(); + } + const std::string& getPrefixKey(); + Qualified qualified(const Map& rowData) override { + for (auto& hint : hints_) { + if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { + return Qualified::INCOMPATIBLE; + } + } + return Qualified::COMPATIBLE; + } + void resetPart(PartitionID partId) override { + std::string p = IndexKeyUtils::indexPrefix(partId); + prefix_.replace(0, p.size(), p); + } + + private: + std::string prefix_; + void buildKey() { + std::string common; + common.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); + auto fieldIter = index_->get_fields().begin(); + for (size_t i = 0; i < hints_.size(); i++, fieldIter++) { + auto& hint = hints_[i]; + CHECK(fieldIter->get_name() == hint.get_column_name()); + auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); + CHECK(type == Value::Type::STRING && !fieldIter->get_type().type_length_ref().has_value()); + encodeValue(hint.get_begin_value(), fieldIter->get_type(), common); + } + prefix_ = std::move(common); + } +}; +class RangePath : public Path { + public: + RangePath(std::shared_ptr index, + std::shared_ptr schema, + const std::vector& hints) + : Path(index, schema, hints) { + buildKey(); + } + void resetPart(PartitionID partId) override { + std::string p = IndexKeyUtils::indexPrefix(partId); + startKey_.replace(0, p.size(), p); + endKey_.replace(0, p.size(), p); + } + + Qualified qualified(const Map& rowData) override { + for (size_t i = 0; i < hints_.size() - 1; i++) { + auto& hint = hints_[i]; + if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { + return Qualified::INCOMPATIBLE; + } + } + auto& hint = hints_.back(); + // TODO(hs.zhang): IDL添加开闭区间信息 + if (hint.begin_value_ref().has_value()) { + bool ret = hint.get_begin_value() < rowData.at(hint.get_column_name()); + if (!ret) { + return Qualified::INCOMPATIBLE; + } + } + if (hint.end_value_ref().has_value()) { + bool ret = hint.get_end_value() > rowData.at(hint.get_column_name()); + if (!ret) { + return Qualified::INCOMPATIBLE; + } + } + return Qualified::COMPATIBLE; + } + inline bool eqToStart() { return eqToStart_; } + inline bool eqToEnd() { return eqToEnd_; } + inline const std::string& getStartKey() { return startKey_; } + inline const std::string& getEndKey() { return endKey_; } + + private: + void buildKey() { + std::string common; + common.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); + auto fieldIter = index_->get_fields().begin(); + for (size_t i = 0; i < hints_.size() - 1; i++, fieldIter++) { + auto& hint = hints_[i]; + CHECK(fieldIter->get_name() == hint.get_column_name()); + auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); + CHECK(type == Value::Type::STRING && !fieldIter->get_type().type_length_ref().has_value()); + encodeValue(hint.get_begin_value(), fieldIter->get_type(), common); + } + auto& hint = hints_.back(); + startKey_ = common; + endKey_ = std::move(common); + if (hint.begin_value_ref().has_value()) { + encodeBeginValue(hint.get_begin_value(), fieldIter->get_type(), startKey_); + } + if (hint.end_value_ref().has_value()) { + encodeEndValue(hint.get_end_value(), fieldIter->get_type(), endKey_); + } else { + endKey_.append(startKey_.size() - endKey_.size(), '0xFF'); + } + } + + std::string encodeBeginValue(const Value& value, const ColumnTypeDef& colDef, std::string& key) { + std::string val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); + if (value.type() == Value::Type::STRING && val.back() != '\0') { + QFList_.emplace_back([&startKey = this->startKey_, + startPos = key.size(), + length = val.size()](const std::string& key) { + int ret = memcmp(startKey.data() + startPos, key.data() + startPos, length); + CHECK_LE(ret, 0); + return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; + }); + } else { + QFList_.emplace_back([&startKey = this->startKey_, + &allowEq = this->eqToStart_, + startPos = key.size(), + length = val.size()](const std::string& key) { + int ret = memcmp(startKey.data() + startPos, key.data() + startPos, length); + CHECK_LE(ret, 0); + return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; + }); + } + key += val; + return val; + } + std::string encodeEndValue(const Value& value, const ColumnTypeDef& colDef, std::string& key) { + std::string val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); + if (value.type() == Value::Type::STRING && val.back() != '\0') { + QFList_.emplace_back([&endKey = this->endKey_, startPos = key.size(), length = val.size()]( + const std::string& key) { + int ret = memcmp(endKey.data() + startPos, key.data() + startPos, length); + CHECK_GE(ret, 0); + return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; + }); + } else { + QFList_.emplace_back([&endKey = this->endKey_, + &allowEq = this->eqToEnd_, + startPos = key.size(), + length = val.size()](const std::string& key) { + int ret = memcmp(endKey.data() + startPos, key.data() + startPos, length); + CHECK_GE(ret, 0); + return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; + }); + } + key += val; + return val; + } +}; +class PathBuilder { + public: + PathBuilder(std::shared_ptr index, + std::shared_ptr schema, + const std::vector& hints); + std::unique_ptr release() { + std::unique_ptr ret; + if (hints_.back().get_scan_type() == cpp2::ScanType::RANGE) { + ret.reset(new RangePath(index_, schema_, hints_)); + } else { + ret.reset(new PrefixPath(index_, schema_, hints_)); + } + return std::move(ret); + } + + private: + std::shared_ptr index_; + std::shared_ptr schema_; + const std::vector& hints_; +}; + +/* define inline functions */ +} // namespace storage + +} // namespace nebula diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp new file mode 100644 index 00000000000..3bb258f08ab --- /dev/null +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -0,0 +1,32 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#include "storage/exec/IndexVertexScanNode.h" + +namespace nebula { +namespace storage { + +IndexVertexScanNode::IndexVertexScanNode(RuntimeContext* context, + IndexID indexId, + const std::vector& clolumnHint) + : IndexScanNode(context, indexId, clolumnHint) {} +// nebula::cpp2::ErrorCode IndexVertexScanNode::execute(PartitionID partId, const nullptr_t&) { +// std::unique_ptr iter; +// if (constrain_->isRange()) { +// auto start_key = constraint_->getStartKey(); +// auto end_key = constrain_->getEndkey(); +// iter.reset(context_->env()->kvstore_->range()); +// } else { +// auto prefix = constrain_->getPrefix(); +// iter.reset(context_->env()->kvstore_->prefix()); +// } +// return nebula::cpp2::ErrorCode::SUCCEEDED; +// } +// bool IndexVertexScanNode::next(Row& row){ + +// }; + +} // namespace storage +} // namespace nebula diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h new file mode 100644 index 00000000000..6c769fe82cf --- /dev/null +++ b/src/storage/exec/IndexVertexScanNode.h @@ -0,0 +1,108 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once + +#include "common/base/Base.h" +#include "storage/exec/IndexScanNode2.h" +#include "storage/exec/RelNode.h" +#include "storage/exec/StorageIterator.h" +namespace nebula { +namespace storage { +class IndexVertexScanNode final : public IndexScanNode { + public: + static StatusOr make(RuntimeContext* context, + IndexID indexId, + const std::vector& columnHint) { + IndexVertexScanNode node(context, indexId, columnHint); + auto env = context->env(); + auto spaceId = context->spaceId(); + auto indexMgr = env->indexMan_; + auto schemaMgr = env->schemaMan_; + auto index = indexMgr->getTagIndex(spaceId, indexId).value(); + auto tagSchema = schemaMgr->getTagSchema(spaceId, index->get_schema_id()).value(); + node.index_ = index; + node.tag_ = tagSchema; + return StatusOr(std::move(node)); + } + + private: + IndexVertexScanNode(RuntimeContext* context, + IndexID indexId, + const std::vector& clolumnHint) + : IndexScanNode(context, indexId, clolumnHint) {} + nebula::cpp2::ErrorCode init() override {} + nebula::cpp2::ErrorCode execute(PartitionID partId) override { + auto ret = resetIter(partId); + return ret; + } + + nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, + std::pair& kv) override { + kv.first = NebulaKeyUtils::vertexKey(context_->vIdLen(), + partId_, + key.subpiece(key.size() - context_->vIdLen()).toString(), + context_->tagId_); + return context_->env()->kvstore_->get(context_->spaceId(), partId_, kv.first, &kv.second); + } + Row decodeFromIndex(folly::StringPiece key) override { + std::vector values(requiredColumns_.size()); + Map colPosMap; + for (size_t i = 0; i < requiredColumns_.size(); i++) { + colPosMap[requiredColumns_[i]] = i; + } + if (colPosMap.count(kVid)) { + auto vId = IndexKeyUtils::getIndexVertexID(context_->vIdLen(), key); + if (context_->isIntId()) { + values[colPosMap[kVid]] = Value(*reinterpret_cast(vId.data())); + } else { + values[colPosMap[kVid]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + } + } + if (colPosMap.count(kTag)) { + values[colPosMap[kTag]] = Value(context_->tagId_); + } + size_t offset = 0; + for (auto& field : index_->get_fields()) { + } + index_->get_fields(); + for (auto& col : requiredColumns_) { + switch (QueryUtils::toReturnColType(col)) { + case QueryUtils::ReturnColType::kVid: { + auto vId = IndexKeyUtils::getIndexVertexID(context_->vIdLen(), key); + if (context_->isIntId()) { + values.emplace_back(*reinterpret_cast(vId.data())); + } else { + values.emplace_back(vId.subpiece(0, vId.find_first_of('\0')).toString()); + } + break; + } + case QueryUtils::ReturnColType::kTag: { + values.emplace_back(context_->tagId_); + break; + } + case QueryUtils::ReturnColType::kOther: { + auto v = IndexKeyUtils::getValueFromIndexKey( + context_->vIdLen(), key, col, fields_, false, hasNullableCol_); + values.emplace_back(std::move(v)); + break; + } + default: + LOG(FATAL) << "Unexpect column " << col << " in IndexVertexScanNode"; + } + } + return Row(std::move(values)); + } + Map decodeFromBase(const std::string& key, + const std::string& value) override {} + const meta::SchemaProviderIf* getSchema() override { return tag_.get(); } + + private: + std::shared_ptr index_; + std::shared_ptr tag_; + std::unique_ptr iter_; +}; +} // namespace storage +} // namespace nebula diff --git a/src/storage/index/LookupBaseProcessor2-inl.h b/src/storage/index/LookupBaseProcessor2-inl.h new file mode 100644 index 00000000000..dc28011b425 --- /dev/null +++ b/src/storage/index/LookupBaseProcessor2-inl.h @@ -0,0 +1,8 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +namespace nebula { +namespace storage {} +} // namespace nebula diff --git a/src/storage/index/LookupBaseProcessor2.h b/src/storage/index/LookupBaseProcessor2.h new file mode 100644 index 00000000000..e019c42e9a8 --- /dev/null +++ b/src/storage/index/LookupBaseProcessor2.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once +#include "storage/BaseProcessor.h" +namespace nebula { +namespace storage { +template +class LookupBaseProcessor : public BaseProcessor { + public: + virtual ~LookupBaseProcessor() = default; + virtual void process(const REQ& req) = 0; + + protected: + LookupBaseProcessor(StorageEnv* env, + const ProcessorCounters* counters, + folly::Executor* executor = nullptr) {} + + virtual void onProcessFinished() = 0; + + StatusOr> buildPlan(); + nebula::DataSet resultDataSet_; + std::vector partResults_; +}; +} // namespace storage +} // namespace nebula +#include "storage/index/LookupBaseProcessor-inl.h" diff --git a/src/storage/index/LookupProcessor2.cpp b/src/storage/index/LookupProcessor2.cpp new file mode 100644 index 00000000000..a5cf7537b2e --- /dev/null +++ b/src/storage/index/LookupProcessor2.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#include "storage/index/LookupProcessor2.h" + +namespace nebula { +namespace storage { + +void LookupProcessor::process(const cpp2::LookupIndexRequest& req) {} + +} // namespace storage +} // namespace nebula diff --git a/src/storage/index/LookupProcessor2.h b/src/storage/index/LookupProcessor2.h new file mode 100644 index 00000000000..6f700b7dfc7 --- /dev/null +++ b/src/storage/index/LookupProcessor2.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once +#include "common/base/Base.h" +#include "interface/gen-cpp2/storage_types.h" +#include "storage/BaseProcessor.h" +namespace nebula { +namespace storage { +extern ProcessorCounters kLookupCounters; +using IndexFilterItem = + std::unordered_map, Expression*>>; + +class LookupProcessor : public BaseProcessor { + public: + static LookupProcessor* instance(StorageEnv* env, + const ProcessorCounters* counters = &kLookupCounters, + folly::Executor* executor = nullptr); + void process(const cpp2::LookupIndexRequest& req); + + private: + LookupProcessor(StorageEnv* env, const ProcessorCounters* counters, folly::Executor* executor); + void onProcessFinished(); + + void runInSingleThread(const cpp2::LookupIndexRequest& req); + void runInMultipleThread(const cpp2::LookupIndexRequest& req); + folly::Future> runInExecutor( + IndexFilterItem* filterItem, nebula::DataSet* result, PartitionID partId); + + void doProcess(const cpp2::LookupIndexRequest& req); + folly::Executor* executor_{nullptr}; +}; +} // namespace storage + +} // namespace nebula From 90e92c303fda1a3e971b0f423f4cc88fe9089571 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Fri, 17 Sep 2021 17:10:03 +0800 Subject: [PATCH 02/38] commit IndexEdge/VertexScanNode --- src/storage/exec/IndexEdgeScanNode.cpp | 97 ++++++++ src/storage/exec/IndexEdgeScanNode.h | 29 ++- src/storage/exec/IndexNode.h | 9 +- src/storage/exec/IndexScanNode2.cpp | 290 ++++++++++++++++++++++- src/storage/exec/IndexScanNode2.h | 282 ++++------------------ src/storage/exec/IndexVertexScanNode.cpp | 15 -- src/storage/exec/IndexVertexScanNode.h | 60 +++-- 7 files changed, 481 insertions(+), 301 deletions(-) create mode 100644 src/storage/exec/IndexEdgeScanNode.cpp diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp new file mode 100644 index 00000000000..6f744415d0f --- /dev/null +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -0,0 +1,97 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "storage/exec/IndexEdgeScanNode.h" +namespace nebula { +namespace storage { +IndexNode::ErrorOr IndexEdgeScanNode::make( + RuntimeContext* context, + IndexID indexId, + const std::vector& columnHint) { + IndexEdgeScanNode node(context, indexId, columnHint); + auto env = context->env(); + auto spaceId = context->spaceId(); + auto indexMgr = env->indexMan_; + auto schemaMgr = env->schemaMan_; + auto index = indexMgr->getTagIndex(spaceId, indexId).value(); + auto edgeSchema = schemaMgr->getEdgeSchema(spaceId, index->get_schema_id().get_tag_id()); + node.index_ = index; + node.edge_ = edgeSchema; + return ErrorOr(std::move(node)); +} +Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { + std::vector values(requiredColumns_.size()); + Map colPosMap; + for (size_t i = 0; i < requiredColumns_.size(); i++) { + colPosMap[requiredColumns_[i]] = i; + } + if (colPosMap.count(kSrc)) { + auto vId = IndexKeyUtils::getIndexSrcId(context_->vIdLen(), key); + if (context_->isIntId()) { + values[colPosMap[kSrc]] = Value(*reinterpret_cast(vId.data())); + } else { + values[colPosMap[kSrc]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + } + } + if (colPosMap.count(kDst)) { + auto vId = IndexKeyUtils::getIndexSrcId(context_->vIdLen(), key); + if (context_->isIntId()) { + values[colPosMap[kSrc]] = Value(*reinterpret_cast(vId.data())); + } else { + values[colPosMap[kSrc]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + } + } + if (colPosMap.count(kRank)) { + auto rank = IndexKeyUtils::getIndexRank(context_->vIdLen(), key); + } + key.subtract(context_->vIdLen() * 2 + sizeof(EdgeType)); + decodePropFromIndex(key, colPosMap, values); + return Row(std::move(values)); +} +nebula::cpp2::ErrorCode IndexEdgeScanNode::getBaseData(folly::StringPiece key, + std::pair& kv) { + auto vIdLen = context_->vIdLen(); + kv.first = NebulaKeyUtils::edgeKey(vIdLen, + partId_, + IndexKeyUtils::getIndexSrcId(vIdLen, key).str(), + context_->edgeType_, + IndexKeyUtils::getIndexRank(vIdLen, key), + IndexKeyUtils::getIndexDstId(vIdLen, key).str()); + return kvstore_->get(context_->spaceId(), partId_, kv.first, &kv.second); +} +Map IndexEdgeScanNode::decodeFromBase(const std::string& key, + const std::string& value) { + Map values; + auto reader = RowReaderWrapper::getRowReader(edge_.get(), value); + for (auto& col : requiredColumns_) { + switch (QueryUtils::toReturnColType(col)) { + case QueryUtils::ReturnColType::kSrc: { + auto vId = NebulaKeyUtils::getSrcId(context_->vIdLen(), key); + if (context_->isIntId()) { + values[col] = Value(*reinterpret_cast(vId.data())); + } else { + values[col] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + } + } break; + case QueryUtils::ReturnColType::kDst: { + values[col] = Value(context_->tagId_); + } break; + case QueryUtils::ReturnColType::kOther: { + auto retVal = QueryUtils::readValue(reader.get(), col, edge_->field(col)); + if (!retVal.ok()) { + LOG(FATAL) << "Bad value for field" << col; + } + values[col] = std::move(retVal.value()); + } break; + default: + LOG(FATAL) << "Unexpect column name:" << col; + } + } + return values; +} +const meta::SchemaProviderIf* IndexEdgeScanNode::getSchema() { return edge_.get(); } +} // namespace storage +} // namespace nebula diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index 473e4bd4c7e..9d569c38262 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -4,7 +4,32 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once - +#include "common/utils/NebulaKeyUtils.h" +#include "folly/Likely.h" +#include "storage/exec/IndexScanNode2.h" +#include "storage/exec/QueryUtils.h" namespace nebula { -namespace storage {} // namespace storage +namespace storage { + +class IndexEdgeScanNode : public IndexScanNode { + public: + static ErrorOr make(RuntimeContext* context, + IndexID indexId, + const std::vector& columnHint); + + private: + IndexEdgeScanNode(RuntimeContext* context, + IndexID indexId, + const std::vector& columnHint) + : IndexScanNode(context, indexId, columnHint) {} + Row decodeFromIndex(folly::StringPiece key) override; + nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, + std::pair& kv) override; + Map decodeFromBase(const std::string& key, const std::string& value) override; + const meta::SchemaProviderIf* getSchema() override; + + private: + std::shared_ptr edge_; +}; +} // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index d8f57f5cf71..bb01297ae4a 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -4,19 +4,20 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once -#include "common/base/StatusOr.h" +#include "common/base/ErrorOr.h" #include "common/datatypes/DataSet.h" #include "interface/gen-cpp2/common_types.h" #include "storage/CommonUtils.h" namespace nebula { namespace storage { +using ErrorCode = ::nebula::cpp2::ErrorCode; class IndexNode { public: + template + using ErrorOr = ::nebula::ErrorOr; explicit IndexNode(RuntimeContext* context); - virtual void init() = 0; virtual nebula::cpp2::ErrorCode execute(PartitionID partId) = 0; - virtual StatusOr next() = 0; - virtual bool vailed() = 0; + virtual ErrorOr next(bool& hasNext) = 0; protected: RuntimeContext* context_; diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index d97fa377b13..7d434254b2a 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -8,20 +8,298 @@ namespace nebula { namespace storage { // Define of Path +std::unique_ptr Path::make(std::shared_ptr index, + std::shared_ptr schema, + const std::vector& hints) { + std::unique_ptr ret; + if (hints.back().get_scan_type() == cpp2::ScanType::RANGE) { + ret.reset(new RangePath(index, schema, hints)); + } else { + ret.reset(new PrefixPath(index, schema, hints)); + } + return std::move(ret); +} +Path::Qualified Path::qualified(const folly::StringPiece& key) { + Qualified ret = Qualified::COMPATIBLE; + for (auto& func : QFList_) { + ret = std::min(ret, func(key.toString())); + } + return ret; +} +std::string Path::encodeValue(const Value& value, const ColumnTypeDef& colDef, std::string& key) { + std::string val; + if (value.type() == Value::Type::STRING) { + val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); + if (val.back() != '\0') { + QFList_.clear(); + QFList_.emplace_back([](const std::string& key) { return Qualified::UNCERTAIN; }); + } + } else { + val = IndexKeyUtils::encodeValue(value); + } + key.append(val); + return val; +} // End of Path // Define of RangePath +RangePath::RangePath(std::shared_ptr index, + std::shared_ptr schema, + const std::vector& hints) + : Path(index, schema, hints) { + buildKey(); +} +void RangePath::resetPart(PartitionID partId) { + std::string p = IndexKeyUtils::indexPrefix(partId); + startKey_.replace(0, p.size(), p); + endKey_.replace(0, p.size(), p); +} +Path::Qualified RangePath::qualified(const Map& rowData) { + for (size_t i = 0; i < hints_.size() - 1; i++) { + auto& hint = hints_[i]; + if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { + return Qualified::INCOMPATIBLE; + } + } + auto& hint = hints_.back(); + // TODO(hs.zhang): IDL添加开闭区间信息 + if (hint.begin_value_ref().has_value()) { + bool ret = hint.get_begin_value() < rowData.at(hint.get_column_name()); + if (!ret) { + return Qualified::INCOMPATIBLE; + } + } + if (hint.end_value_ref().has_value()) { + bool ret = hint.get_end_value() > rowData.at(hint.get_column_name()); + if (!ret) { + return Qualified::INCOMPATIBLE; + } + } + return Qualified::COMPATIBLE; +} +void RangePath::buildKey() { + std::string common; + common.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); + auto fieldIter = index_->get_fields().begin(); + for (size_t i = 0; i < hints_.size() - 1; i++, fieldIter++) { + auto& hint = hints_[i]; + CHECK(fieldIter->get_name() == hint.get_column_name()); + auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); + CHECK(type == Value::Type::STRING && !fieldIter->get_type().type_length_ref().has_value()); + encodeValue(hint.get_begin_value(), fieldIter->get_type(), common); + } + auto& hint = hints_.back(); + startKey_ = common; + endKey_ = std::move(common); + if (hint.begin_value_ref().has_value()) { + encodeBeginValue(hint.get_begin_value(), fieldIter->get_type(), startKey_); + } + if (hint.end_value_ref().has_value()) { + encodeEndValue(hint.get_end_value(), fieldIter->get_type(), endKey_); + } else { + endKey_.append(startKey_.size() - endKey_.size(), '0xFF'); + } +} +std::string RangePath::encodeBeginValue(const Value& value, + const ColumnTypeDef& colDef, + std::string& key) { + std::string val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); + if (value.type() == Value::Type::STRING && val.back() != '\0') { + QFList_.emplace_back([&startKey = this->startKey_, startPos = key.size(), length = val.size()]( + const std::string& key) { + int ret = memcmp(startKey.data() + startPos, key.data() + startPos, length); + CHECK_LE(ret, 0); + return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; + }); + } else { + QFList_.emplace_back([&startKey = this->startKey_, + &allowEq = this->eqToStart_, + startPos = key.size(), + length = val.size()](const std::string& key) { + int ret = memcmp(startKey.data() + startPos, key.data() + startPos, length); + CHECK_LE(ret, 0); + return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; + }); + } + key += val; + return val; +} +std::string RangePath::encodeEndValue(const Value& value, + const ColumnTypeDef& colDef, + std::string& key) { + std::string val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); + if (value.type() == Value::Type::STRING && val.back() != '\0') { + QFList_.emplace_back([&endKey = this->endKey_, startPos = key.size(), length = val.size()]( + const std::string& key) { + int ret = memcmp(endKey.data() + startPos, key.data() + startPos, length); + CHECK_GE(ret, 0); + return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; + }); + } else { + QFList_.emplace_back([&endKey = this->endKey_, + &allowEq = this->eqToEnd_, + startPos = key.size(), + length = val.size()](const std::string& key) { + int ret = memcmp(endKey.data() + startPos, key.data() + startPos, length); + CHECK_GE(ret, 0); + return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; + }); + } + key += val; + return val; +} + // End of RangePath // Define of PrefixPath - +PrefixPath::PrefixPath(std::shared_ptr index, + std::shared_ptr schema, + const std::vector& hints) + : Path(index, schema, hints) { + buildKey(); +} +Path::Qualified PrefixPath::qualified(const Map& rowData) { + for (auto& hint : hints_) { + if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { + return Qualified::INCOMPATIBLE; + } + } + return Qualified::COMPATIBLE; +} +void PrefixPath::resetPart(PartitionID partId) { + std::string p = IndexKeyUtils::indexPrefix(partId); + prefix_.replace(0, p.size(), p); +} +void PrefixPath::buildKey() { + std::string common; + common.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); + auto fieldIter = index_->get_fields().begin(); + for (size_t i = 0; i < hints_.size(); i++, fieldIter++) { + auto& hint = hints_[i]; + CHECK(fieldIter->get_name() == hint.get_column_name()); + auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); + CHECK(type == Value::Type::STRING && !fieldIter->get_type().type_length_ref().has_value()); + encodeValue(hint.get_begin_value(), fieldIter->get_type(), common); + } + prefix_ = std::move(common); +} // End of PrefixPath - -// Define of PathBuilder - -// End of PathBuilder - // Define of IndexScan +nebula::cpp2::ErrorCode IndexScanNode::execute(PartitionID partId) { + auto ret = resetIter(partId); + return ret; +} +IndexNode::ErrorOr IndexScanNode::next(bool& hasNext) { + hasNext = true; + while (iter_ && iter_->valid()) { + if (!checkTTL()) { + continue; + } + auto q = path_->qualified(iter_->key()); + if (q == Path::Qualified::INCOMPATIBLE) { + continue; + } + if (q == Path::Qualified::COMPATIBLE) { + return decodeFromIndex(iter_->key()); + } + std::pair kv; + auto ret = getBaseData(iter_->key(), kv); + if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + continue; + } + Map rowData = decodeFromBase(kv.first, kv.second); + q = path_->qualified(rowData); + CHECK(q != Path::Qualified::UNCERTAIN); + if (q == Path::Qualified::INCOMPATIBLE) { + continue; + } + if (q == Path::Qualified::COMPATIBLE) { + Row row; + for (auto& col : requiredColumns_) { + row.emplace_back(std::move(rowData.at(col))); + } + } + } + hasNext = false; + return ErrorOr(Row()); +} +bool IndexScanNode::checkTTL() { + if (iter_->val().empty() || ttlProps_.first == false) { + return true; + } + auto v = IndexKeyUtils::parseIndexTTL(iter_->val()); + if (CommonUtils::checkDataExpiredForTTL( + getSchema(), std::move(v), ttlProps_.second.second, ttlProps_.second.first)) { + return false; + } + return true; +} +nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { + path_->resetPart(partId); + nebula::cpp2::ErrorCode ret; + if (path_->isRange()) { + auto rangePath = dynamic_cast(path_.get()); + ret = + kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter_); + } else { + auto prefixPath = dynamic_cast(path_.get()); + ret = kvstore_->prefix(spaceId_, partId, prefixPath->getPrefixKey(), &iter_); + } + return ret; +} +void IndexScanNode::decodePropFromIndex(folly::StringPiece key, + const Map& colPosMap, + std::vector& values) { + size_t offset = sizeof(PartitionID) + sizeof(IndexID); + size_t len = 0; + std::bitset<16> nullableBit; + int8_t nullableColPosit = 15; + if (indexNullable_) { + auto bitOffset = key.size() - context_->vIdLen() - sizeof(uint16_t); + auto v = *reinterpret_cast(key.data() + bitOffset); + nullableBit = v; + } + for (auto& field : index_->get_fields()) { + int len = 0; + auto type = IndexKeyUtils::toValueType(field.type.get_type()); + switch (type) { + case Value::Type::BOOL: + len = sizeof(bool); + break; + case Value::Type::INT: + len = sizeof(int64_t); + break; + case Value::Type::FLOAT: + len = sizeof(double); + break; + case Value::Type::STRING: + len = *field.type.get_type_length(); + break; + case Value::Type::TIME: + len = sizeof(int8_t) * 3 + sizeof(int32_t); + break; + case Value::Type::DATE: + len = sizeof(int8_t) * 2 + sizeof(int16_t); + break; + case Value::Type::DATETIME: + len = sizeof(int32_t) + sizeof(int16_t) + sizeof(int8_t) * 5; + break; + default: + LOG(FATAL) << "Unexpect value type:" << int(field.type.get_type()); + } + if (colPosMap.count(field.get_name())) { + if (indexNullable_ && nullableBit.test(nullableColPosit)) { + values[colPosMap.at(field.get_name())] = Value(NullType::__NULL__); + } else { + values[colPosMap.at(field.get_name())] = + IndexKeyUtils::decodeValue(key.subpiece(offset, len), type); + } + } + offset += len; + nullableColPosit -= 1; + } +} + // End of IndexScan } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index 67426369633..ec80785fe01 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -10,7 +10,6 @@ #include "common/base/Base.h" #include "common/datatypes/DataSet.h" #include "common/utils/IndexKeyUtils.h" -#include "folly/Optional.h" #include "interface/gen-cpp2/meta_types.h" #include "interface/gen-cpp2/storage_types.h" #include "storage/CommonUtils.h" @@ -21,90 +20,40 @@ template using Map = folly::F14FastMap; class Path; +class RangePath; +class PrefixPath; class IndexScanNode : public IndexNode { public: IndexScanNode(RuntimeContext* context, IndexID indexId, const std::vector& columnHists) : IndexNode(context), indexId_(indexId), columnHists_(columnHists) {} - nebula::cpp2::ErrorCode execute(PartitionID partId) override { - auto ret = resetIter(partId); - return ret; - } - StatusOr next(bool& hasNext) { - hasNext = true; - while (iter_ && iter_->valid()) { - if (!checkTTL()) { - continue; - } - auto q = path_->qualified(iter_->key()); - if (q == Path::Qualified::INCOMPATIBLE) { - continue; - } - if (q == Path::Qualified::COMPATIBLE) { - return decodeFromIndex(iter_->key()); - } - std::pair kv; - auto ret = getBaseData(iter_->key(), kv); - if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - continue; - } - Map rowData = decodeFromBase(kv.first, kv.second); - q = path_->qualified(rowData); - CHECK(q != Path::Qualified::UNCERTAIN); - if (q == Path::Qualified::INCOMPATIBLE) { - continue; - } - if (q == Path::Qualified::COMPATIBLE) { - Row row; - for (auto& col : requiredColumns_) { - row.emplace_back(std::move(rowData.at(col))); - } - } - } - hasNext = false; - return Status::OK(); - } + nebula::cpp2::ErrorCode execute(PartitionID partId) final; + ErrorOr next(bool& hasNext) final; protected: + void decodePropFromIndex(folly::StringPiece key, + const Map& colPosMap, + std::vector& values); virtual Row decodeFromIndex(folly::StringPiece key) = 0; virtual nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, std::pair& kv) = 0; virtual Map decodeFromBase(const std::string& key, const std::string& value) = 0; virtual const meta::SchemaProviderIf* getSchema() = 0; - std::pair> ttlProps_; std::unique_ptr>> ttlProps_; - bool checkTTL() { - if (iter_->val().empty() || ttlProps_.first == false) { - return true; - } - auto v = IndexKeyUtils::parseIndexTTL(iter_->val()); - if (CommonUtils::checkDataExpiredForTTL( - getSchema(), std::move(v), ttlProps_.second.second, ttlProps_.second.first)) { - return false; - } - return true; - } - nebula::cpp2::ErrorCode resetIter(PartitionID partId) { - path_->resetPart(partId); - nebula::cpp2::ErrorCode ret; - if (path_->isRange()) { - auto rangePath = dynamic_cast(path_.get()); - ret = kvstore_->range( - spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter_); - } else { - auto prefixPath = dynamic_cast(path_.get()); - ret = kvstore_->prefix(spaceId_, partId, prefixPath->getPrefixKey(), &iter_); - } - return ret; - } + bool checkTTL(); + nebula::cpp2::ErrorCode resetIter(PartitionID partId); + PartitionID partId_; const IndexID indexId_; + std::shared_ptr index_; + bool indexNullable_ = false; const std::vector& columnHists_; std::unique_ptr path_; std::unique_ptr iter_; nebula::kvstore::KVStore* kvstore_; std::vector requiredColumns_; + std::pair> ttlProps_; }; /** * Path @@ -113,217 +62,68 @@ class IndexScanNode : public IndexNode { */ class Path { public: + enum class Qualified : int16_t { INCOMPATIBLE = 0, UNCERTAIN = 1, COMPATIBLE = 2 }; + using QualifiedFunction = std::function; using ColumnTypeDef = ::nebula::meta::cpp2::ColumnTypeDef; Path(std::shared_ptr index, std::shared_ptr schema, const std::vector& hints) : index_(index), schema_(schema), hints_(hints) {} - enum class Qualified : int16_t { INCOMPATIBLE = 0, UNCERTAIN = 1, COMPATIBLE = 2 }; - using QualifiedFunction = std::function; - virtual Qualified qualified(const folly::StringPiece& key) { - Qualified ret = Qualified::COMPATIBLE; - for (auto& func : QFList_) { - ret = std::min(ret, func(key.toString())); - } - return ret; - } - virtual Qualified qualified(const Map& rowData) = 0; virtual ~Path() = default; + + static std::unique_ptr make( + std::shared_ptr index, + std::shared_ptr schema, + const std::vector& hints); + virtual Qualified qualified(const folly::StringPiece& key); + virtual bool isRange() { return false; } + + virtual Qualified qualified(const Map& rowData) = 0; virtual void resetPart(PartitionID partId) = 0; - virtual bool isRange() { return dynamic_cast(this) != nullptr; } protected: - std::string encodeValue(const Value& value, const ColumnTypeDef& colDef, std::string& key) { - std::string val; - if (value.type() == Value::Type::STRING) { - val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); - if (val.back() != '\0') { - QFList_.clear(); - QFList_.emplace_back([](const std::string& key) { return Qualified::UNCERTAIN; }); - } - } else { - val = IndexKeyUtils::encodeValue(value); - } - key.append(val); - return val; - } + std::string encodeValue(const Value& value, const ColumnTypeDef& colDef, std::string& key); std::vector QFList_; std::shared_ptr index_; std::shared_ptr schema_; const std::vector& hints_; - std::string startKey_, endKey_; - bool eqToStart_, eqToEnd_; }; class PrefixPath : public Path { public: PrefixPath(std::shared_ptr index, std::shared_ptr schema, - const std::vector& hints) - : Path(index, schema, hints) { - buildKey(); - } + const std::vector& hints); + // Override + Qualified qualified(const Map& rowData) override; + void resetPart(PartitionID partId) override; + const std::string& getPrefixKey(); - Qualified qualified(const Map& rowData) override { - for (auto& hint : hints_) { - if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { - return Qualified::INCOMPATIBLE; - } - } - return Qualified::COMPATIBLE; - } - void resetPart(PartitionID partId) override { - std::string p = IndexKeyUtils::indexPrefix(partId); - prefix_.replace(0, p.size(), p); - } private: std::string prefix_; - void buildKey() { - std::string common; - common.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); - auto fieldIter = index_->get_fields().begin(); - for (size_t i = 0; i < hints_.size(); i++, fieldIter++) { - auto& hint = hints_[i]; - CHECK(fieldIter->get_name() == hint.get_column_name()); - auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); - CHECK(type == Value::Type::STRING && !fieldIter->get_type().type_length_ref().has_value()); - encodeValue(hint.get_begin_value(), fieldIter->get_type(), common); - } - prefix_ = std::move(common); - } + void buildKey(); }; class RangePath : public Path { public: RangePath(std::shared_ptr index, std::shared_ptr schema, - const std::vector& hints) - : Path(index, schema, hints) { - buildKey(); - } - void resetPart(PartitionID partId) override { - std::string p = IndexKeyUtils::indexPrefix(partId); - startKey_.replace(0, p.size(), p); - endKey_.replace(0, p.size(), p); - } + const std::vector& hints); + // Override + Qualified qualified(const Map& rowData) override; + void resetPart(PartitionID partId) override; - Qualified qualified(const Map& rowData) override { - for (size_t i = 0; i < hints_.size() - 1; i++) { - auto& hint = hints_[i]; - if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { - return Qualified::INCOMPATIBLE; - } - } - auto& hint = hints_.back(); - // TODO(hs.zhang): IDL添加开闭区间信息 - if (hint.begin_value_ref().has_value()) { - bool ret = hint.get_begin_value() < rowData.at(hint.get_column_name()); - if (!ret) { - return Qualified::INCOMPATIBLE; - } - } - if (hint.end_value_ref().has_value()) { - bool ret = hint.get_end_value() > rowData.at(hint.get_column_name()); - if (!ret) { - return Qualified::INCOMPATIBLE; - } - } - return Qualified::COMPATIBLE; - } inline bool eqToStart() { return eqToStart_; } inline bool eqToEnd() { return eqToEnd_; } inline const std::string& getStartKey() { return startKey_; } inline const std::string& getEndKey() { return endKey_; } + virtual bool isRange() { return true; } private: - void buildKey() { - std::string common; - common.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); - auto fieldIter = index_->get_fields().begin(); - for (size_t i = 0; i < hints_.size() - 1; i++, fieldIter++) { - auto& hint = hints_[i]; - CHECK(fieldIter->get_name() == hint.get_column_name()); - auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); - CHECK(type == Value::Type::STRING && !fieldIter->get_type().type_length_ref().has_value()); - encodeValue(hint.get_begin_value(), fieldIter->get_type(), common); - } - auto& hint = hints_.back(); - startKey_ = common; - endKey_ = std::move(common); - if (hint.begin_value_ref().has_value()) { - encodeBeginValue(hint.get_begin_value(), fieldIter->get_type(), startKey_); - } - if (hint.end_value_ref().has_value()) { - encodeEndValue(hint.get_end_value(), fieldIter->get_type(), endKey_); - } else { - endKey_.append(startKey_.size() - endKey_.size(), '0xFF'); - } - } - - std::string encodeBeginValue(const Value& value, const ColumnTypeDef& colDef, std::string& key) { - std::string val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); - if (value.type() == Value::Type::STRING && val.back() != '\0') { - QFList_.emplace_back([&startKey = this->startKey_, - startPos = key.size(), - length = val.size()](const std::string& key) { - int ret = memcmp(startKey.data() + startPos, key.data() + startPos, length); - CHECK_LE(ret, 0); - return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; - }); - } else { - QFList_.emplace_back([&startKey = this->startKey_, - &allowEq = this->eqToStart_, - startPos = key.size(), - length = val.size()](const std::string& key) { - int ret = memcmp(startKey.data() + startPos, key.data() + startPos, length); - CHECK_LE(ret, 0); - return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; - }); - } - key += val; - return val; - } - std::string encodeEndValue(const Value& value, const ColumnTypeDef& colDef, std::string& key) { - std::string val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); - if (value.type() == Value::Type::STRING && val.back() != '\0') { - QFList_.emplace_back([&endKey = this->endKey_, startPos = key.size(), length = val.size()]( - const std::string& key) { - int ret = memcmp(endKey.data() + startPos, key.data() + startPos, length); - CHECK_GE(ret, 0); - return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; - }); - } else { - QFList_.emplace_back([&endKey = this->endKey_, - &allowEq = this->eqToEnd_, - startPos = key.size(), - length = val.size()](const std::string& key) { - int ret = memcmp(endKey.data() + startPos, key.data() + startPos, length); - CHECK_GE(ret, 0); - return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; - }); - } - key += val; - return val; - } -}; -class PathBuilder { - public: - PathBuilder(std::shared_ptr index, - std::shared_ptr schema, - const std::vector& hints); - std::unique_ptr release() { - std::unique_ptr ret; - if (hints_.back().get_scan_type() == cpp2::ScanType::RANGE) { - ret.reset(new RangePath(index_, schema_, hints_)); - } else { - ret.reset(new PrefixPath(index_, schema_, hints_)); - } - return std::move(ret); - } - - private: - std::shared_ptr index_; - std::shared_ptr schema_; - const std::vector& hints_; + std::string startKey_, endKey_; + bool eqToStart_, eqToEnd_; + void buildKey(); + std::string encodeBeginValue(const Value& value, const ColumnTypeDef& colDef, std::string& key); + std::string encodeEndValue(const Value& value, const ColumnTypeDef& colDef, std::string& key); }; /* define inline functions */ diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index 3bb258f08ab..9821181dd47 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -12,21 +12,6 @@ IndexVertexScanNode::IndexVertexScanNode(RuntimeContext* context, IndexID indexId, const std::vector& clolumnHint) : IndexScanNode(context, indexId, clolumnHint) {} -// nebula::cpp2::ErrorCode IndexVertexScanNode::execute(PartitionID partId, const nullptr_t&) { -// std::unique_ptr iter; -// if (constrain_->isRange()) { -// auto start_key = constraint_->getStartKey(); -// auto end_key = constrain_->getEndkey(); -// iter.reset(context_->env()->kvstore_->range()); -// } else { -// auto prefix = constrain_->getPrefix(); -// iter.reset(context_->env()->kvstore_->prefix()); -// } -// return nebula::cpp2::ErrorCode::SUCCEEDED; -// } -// bool IndexVertexScanNode::next(Row& row){ - -// }; } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index 6c769fe82cf..81215261f76 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -13,19 +13,19 @@ namespace nebula { namespace storage { class IndexVertexScanNode final : public IndexScanNode { public: - static StatusOr make(RuntimeContext* context, - IndexID indexId, - const std::vector& columnHint) { + static ErrorOr make(RuntimeContext* context, + IndexID indexId, + const std::vector& columnHint) { IndexVertexScanNode node(context, indexId, columnHint); auto env = context->env(); auto spaceId = context->spaceId(); auto indexMgr = env->indexMan_; auto schemaMgr = env->schemaMan_; auto index = indexMgr->getTagIndex(spaceId, indexId).value(); - auto tagSchema = schemaMgr->getTagSchema(spaceId, index->get_schema_id()).value(); + auto tagSchema = schemaMgr->getTagSchema(spaceId, index->get_schema_id().get_tag_id()); node.index_ = index; node.tag_ = tagSchema; - return StatusOr(std::move(node)); + return ErrorOr(std::move(node)); } private: @@ -33,11 +33,6 @@ class IndexVertexScanNode final : public IndexScanNode { IndexID indexId, const std::vector& clolumnHint) : IndexScanNode(context, indexId, clolumnHint) {} - nebula::cpp2::ErrorCode init() override {} - nebula::cpp2::ErrorCode execute(PartitionID partId) override { - auto ret = resetIter(partId); - return ret; - } nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, std::pair& kv) override { @@ -64,45 +59,44 @@ class IndexVertexScanNode final : public IndexScanNode { if (colPosMap.count(kTag)) { values[colPosMap[kTag]] = Value(context_->tagId_); } - size_t offset = 0; - for (auto& field : index_->get_fields()) { - } - index_->get_fields(); + key.subtract(context_->vIdLen()); + decodePropFromIndex(key, colPosMap, values); + return Row(std::move(values)); + } + Map decodeFromBase(const std::string& key, + const std::string& value) override { + Map values; + auto reader = RowReaderWrapper::getRowReader(tag_.get(), value); for (auto& col : requiredColumns_) { switch (QueryUtils::toReturnColType(col)) { case QueryUtils::ReturnColType::kVid: { - auto vId = IndexKeyUtils::getIndexVertexID(context_->vIdLen(), key); + auto vId = NebulaKeyUtils::getVertexId(context_->vIdLen(), key); if (context_->isIntId()) { - values.emplace_back(*reinterpret_cast(vId.data())); + values[col] = Value(*reinterpret_cast(vId.data())); } else { - values.emplace_back(vId.subpiece(0, vId.find_first_of('\0')).toString()); + values[col] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } - break; - } + } break; case QueryUtils::ReturnColType::kTag: { - values.emplace_back(context_->tagId_); - break; - } + values[col] = Value(context_->tagId_); + } break; case QueryUtils::ReturnColType::kOther: { - auto v = IndexKeyUtils::getValueFromIndexKey( - context_->vIdLen(), key, col, fields_, false, hasNullableCol_); - values.emplace_back(std::move(v)); - break; - } + auto retVal = QueryUtils::readValue(reader.get(), col, tag_->field(col)); + if (!retVal.ok()) { + LOG(FATAL) << "Bad value for field" << col; + } + values[col] = std::move(retVal.value()); + } break; default: - LOG(FATAL) << "Unexpect column " << col << " in IndexVertexScanNode"; + LOG(FATAL) << "Unexpect column name:" << col; } } - return Row(std::move(values)); + return values; } - Map decodeFromBase(const std::string& key, - const std::string& value) override {} const meta::SchemaProviderIf* getSchema() override { return tag_.get(); } private: - std::shared_ptr index_; std::shared_ptr tag_; - std::unique_ptr iter_; }; } // namespace storage } // namespace nebula From f8bf50c8e9a1cd7f66cc530bbb1a5688eed9e208 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Sat, 18 Sep 2021 14:26:43 +0800 Subject: [PATCH 03/38] commit IndexSelectionNode --- src/storage/exec/IndexNode.h | 4 ++ src/storage/exec/IndexProjectionNode.h | 10 ++- src/storage/exec/IndexScanNode2.h | 2 - src/storage/exec/IndexSelectionNode.cpp | 49 +++++++++++++++ src/storage/exec/IndexSelectionNode.h | 84 +++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 src/storage/exec/IndexSelectionNode.cpp create mode 100644 src/storage/exec/IndexSelectionNode.h diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index bb01297ae4a..7395fbca798 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -6,11 +6,14 @@ #pragma once #include "common/base/ErrorOr.h" #include "common/datatypes/DataSet.h" +#include "folly/container/F14Map.h" #include "interface/gen-cpp2/common_types.h" #include "storage/CommonUtils.h" namespace nebula { namespace storage { using ErrorCode = ::nebula::cpp2::ErrorCode; +template +using Map = folly::F14FastMap; class IndexNode { public: template @@ -22,6 +25,7 @@ class IndexNode { protected: RuntimeContext* context_; GraphSpaceID spaceId_; + std::vector children_; }; } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index dc28011b425..3b8df016586 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -3,6 +3,14 @@ * This source code is licensed under Apache 2.0 License, * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ +#pragma once + +#include "storage/exec/IndexNode.h" namespace nebula { -namespace storage {} +namespace storage { +class IndexProjectionNode : public IndexNode { + public: + private: +}; +} // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index ec80785fe01..f3233e0d2d8 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -16,8 +16,6 @@ #include "storage/exec/IndexNode.h" namespace nebula { namespace storage { -template -using Map = folly::F14FastMap; class Path; class RangePath; diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp new file mode 100644 index 00000000000..068442d890d --- /dev/null +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -0,0 +1,49 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#include "storage/exec/IndexSelectionNode.h" +namespace nebula { +namespace storage { +IndexSelectionNode::IndexSelectionNode(RuntimeContext* context, + std::unique_ptr expr, + const std::vector& inputCols) + : IndexNode(context), expr_(std::move(expr)) { + for (size_t i = 0; i < inputCols.size(); i++) { + colPos_[inputCols[i]] = i; + } +} +IndexNode::ErrorOr IndexSelectionNode::next(bool& hasNext) { + DCHECK_EQ(children_.size(), 1); + auto& child = *children_[0]; + do { + auto result = child.next(hasNext); + if (!hasNext || !nebula::ok(result)) { + return result; + } + if (filter(nebula::value(result))) { + return result; + } + } while (true); + return ErrorOr(Row()); +} +Value IndexSelectionNode::ExprContext::getEdgeProp(const std::string& edgeType, + const std::string& prop) const { + DCHECK(row_ != nullptr); + auto iter = colPos_.find(prop); + DCHECK(iter != colPos_.end()); + DCHECK(iter->second < row_->size()); + return row_[iter->second]; +} +Value IndexSelectionNode::ExprContext::getTagProp(const std::string& tag, + const std::string& prop) const { + DCHECK(row_ != nullptr); + auto iter = colPos_.find(prop); + DCHECK(iter != colPos_.end()); + DCHECK(iter->second < row_->size()); + return row_[iter->second]; +} +} // namespace storage + +} // namespace nebula diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h new file mode 100644 index 00000000000..2350666a0fc --- /dev/null +++ b/src/storage/exec/IndexSelectionNode.h @@ -0,0 +1,84 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once + +#include "common/context/ExpressionContext.h" +#include "common/expression/Expression.h" +#include "folly/container/F14Map.h" +#include "storage/exec/IndexNode.h" +namespace nebula { +namespace storage { +class IndexSelectionNode : public IndexNode { + public: + IndexSelectionNode(RuntimeContext* context, + std::unique_ptr expr, + const std::vector& inputCols); + nebula::cpp2::ErrorCode execute(PartitionID partId) override; + ErrorOr next(bool& hasNext) override; + + private: + inline bool filter(const Row& row) { + ctx_->setRow(row); + auto& result = expr_->eval(*ctx_); + return result.type() == Value::Type::BOOL ? result.getBool() : false; + } + Map colPos_; + std::unique_ptr expr_; + class ExprContext : public ExpressionContext { + public: + explicit ExprContext(const Map& colPos) : colPos_(colPos) {} + void setRow(const Row& row) { row_ = &row; } + Value getEdgeProp(const std::string& edgeType, const std::string& prop) const override; + Value getTagProp(const std::string& tag, const std::string& prop) const override; + // override + const Value& getVar(const std::string& var) const override { + LOG(FATAL) << "Unexpect"; + return Value(); + } + const Value& getVersionedVar(const std::string& var, int64_t version) const override { + LOG(FATAL) << "Unexpect"; + return Value(); + } + const Value& getVarProp(const std::string& var, const std::string& prop) const override { + LOG(FATAL) << "Unexpect"; + return Value(); + } + + Value getSrcProp(const std::string& tag, const std::string& prop) const override { + LOG(FATAL) << "Unexpect"; + return Value(); + } + const Value& getDstProp(const std::string& tag, const std::string& prop) const override { + LOG(FATAL) << "Unexpect"; + return Value(); + } + const Value& getInputProp(const std::string& prop) const override { + LOG(FATAL) << "Unexpect"; + return Value(); + } + Value getVertex() const override { + LOG(FATAL) << "Unexpect"; + return Value(); + } + Value getEdge() const override { + LOG(FATAL) << "Unexpect"; + return Value(); + } + Value getColumn(int32_t index) const override { + LOG(FATAL) << "Unexpect"; + return Value(); + } + void setVar(const std::string& var, Value val) override { LOG(FATAL) << "Unexpect"; } + + private: + const Map& colPos_; + const Row* row_; + }; + std::unique_ptr ctx_; +}; +} // namespace storage + +} // namespace nebula From a7c93d2c02316899cdf764d7b06e784df0740fdd Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Sat, 18 Sep 2021 16:45:32 +0800 Subject: [PATCH 04/38] commit Projection/Dedup Node --- src/storage/exec/IndexDedupNode.cpp | 27 ++++++++++ src/storage/exec/IndexDedupNode.h | 64 ++++++++++++++++++++++++ src/storage/exec/IndexGathorNode.h | 19 ------- src/storage/exec/IndexProjectionNode.cpp | 36 +++++++++++++ src/storage/exec/IndexProjectionNode.h | 23 +++++++++ 5 files changed, 150 insertions(+), 19 deletions(-) create mode 100644 src/storage/exec/IndexDedupNode.cpp create mode 100644 src/storage/exec/IndexDedupNode.h delete mode 100644 src/storage/exec/IndexGathorNode.h create mode 100644 src/storage/exec/IndexProjectionNode.cpp diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp new file mode 100644 index 00000000000..f8bb0c9d57d --- /dev/null +++ b/src/storage/exec/IndexDedupNode.cpp @@ -0,0 +1,27 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#include "storage/exec/IndexDedupNode.h" + +namespace nebula { +namespace storage { + +IndexDedupNode::IndexDedupNode(RuntimeContext* context, + const std::vector& dedupColumn, + const std::vector& inputColumn) + : IndexNode(context) { + Map m; + for (size_t i = 0; i < inputColumn.size(); i++) { + m[inputColumn[i]] = i; + } + for (auto& col : dedupColumn) { + DCHECK(m.count(col)); + dedupPos_.push_back(m.at(col)); + } +} + +} // namespace storage + +} // namespace nebula diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h new file mode 100644 index 00000000000..368f4ca155a --- /dev/null +++ b/src/storage/exec/IndexDedupNode.h @@ -0,0 +1,64 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once +#include "common/datatypes/DataSet.h" +#include "folly/container/F14Set.h" +#include "storage/exec/IndexNode.h" +namespace nebula { +namespace storage { +class IndexDedupNode : public IndexNode { + public: + IndexDedupNode(RuntimeContext* context, + const std::vector& dedupColumn, + const std::vector& inputColumn); + nebula::cpp2::ErrorCode execute(PartitionID partId) = 0; + ErrorOr next(bool& hasNext) override { + DCHECK_EQ(children_.size(), 1); + auto& child = *children_[0]; + do { + auto result = child.next(hasNext); + if (!hasNext || !nebula::ok(result)) { + return result; + } + if (dedup(::nebula::value(result))) { + return result; + } + } while (true); + } + + private: + bool dedup(const Row& row) { + auto result = dedupSet_.emplace(row); + return result.second; + } + class RowWrapper { + public: + RowWrapper(const Row& row, const std::vector& posList) { + for (auto p : posList) { + values_.emplace_back(row[p]); + } + } + const List& values() const { return values_; } + + private: + List values_; + }; + struct Hasher { + size_t operator()(const RowWrapper& wrapper) const { + return std::hash()(wrapper.values()); + } + }; + struct Equal { + bool operator()(const RowWrapper& a, const RowWrapper& b) const { + return a.values() == b.values(); + } + }; + std::vector dedupPos_; + folly::F14FastSet dedupSet_; +}; +} // namespace storage + +} // namespace nebula diff --git a/src/storage/exec/IndexGathorNode.h b/src/storage/exec/IndexGathorNode.h deleted file mode 100644 index a36ba3e1c1e..00000000000 --- a/src/storage/exec/IndexGathorNode.h +++ /dev/null @@ -1,19 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ -#pragma once -#include "storage/exec/RelNode.h" - -namespace nebula { -namespace storage { -class IndexGathorNode : public IterateNode { - public: - IndexGathorNode(); - - private: -}; -} // namespace storage - -} // namespace nebula diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp new file mode 100644 index 00000000000..e250cccec59 --- /dev/null +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -0,0 +1,36 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#include "storage/exec/IndexProjectionNode.h" +namespace nebula { +namespace storage { +IndexProjectionNode::IndexProjectionNode(RuntimeContext* context, + const std::vector& requiredColumns, + const std::vector& inputColumns) + : IndexNode(context), requiredColumns_(requiredColumns) { + for (size_t i = 0; i < inputColumns.size(); i++) { + colPos_[inputColumns[i]] = i; + } +} +IndexNode::ErrorOr IndexProjectionNode::next(bool& hasNext) { + DCHECK_EQ(children_.size(), 1); + auto& child = *children_[0]; + auto result = child.next(hasNext); + if (::nebula::ok(result) && hasNext) { + return ErrorOr(project(::nebula::value(std::move(result)))); + } else { + return ErrorOr(Row()); + } +} +Row IndexProjectionNode::project(Row&& row) { + Row ret; + for (auto& col : requiredColumns_) { + ret.emplace_back(std::move(row[colPos_[col]])); + } + return ret; +} + +} // namespace storage +} // namespace nebula diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index 3b8df016586..fc079c07949 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -10,7 +10,30 @@ namespace nebula { namespace storage { class IndexProjectionNode : public IndexNode { public: + IndexProjectionNode(RuntimeContext* context, + const std::vector& requiredColumns, + const std::vector& inputColumns); + nebula::cpp2::ErrorCode execute(PartitionID partId) = 0; + ErrorOr next(bool& hasNext) override { + DCHECK_EQ(children_.size(), 1); + auto& child = *children_[0]; + auto result = child.next(hasNext); + if (::nebula::ok(result) && hasNext) { + return ErrorOr(project(::nebula::value(std::move(result)))); + } else { + return ErrorOr(Row()); + } + } + private: + Row project(Row&& row) { + Row ret; + for (auto& col : requiredColumns_) { + ret.emplace_back(std::move(row[colPos_[col]])); + } + } + std::vector requiredColumns_; + Map colPos_; }; } // namespace storage } // namespace nebula From f43ec57944b02a897a1cc3c1d55d38be8be8a40a Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Sat, 18 Sep 2021 17:03:40 +0800 Subject: [PATCH 05/38] commit IndexLimitNode --- src/storage/exec/IndexLimitNode.h | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/storage/exec/IndexLimitNode.h diff --git a/src/storage/exec/IndexLimitNode.h b/src/storage/exec/IndexLimitNode.h new file mode 100644 index 00000000000..71859653084 --- /dev/null +++ b/src/storage/exec/IndexLimitNode.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once +#include "folly/Likely.h" +#include "storage/exec/IndexNode.h" +namespace nebula { +namespace storage { +class IndexLimitNode : public IndexNode { + public: + IndexLimitNode(RuntimeContext* context, uint64_t offset, uint64_t limit) + : IndexNode(context), offset_(offset), limit_(limit) {} + IndexLimitNode(RuntimeContext* context, uint64_t limit) : IndexLimitNode(context, 0, limit) {} + nebula::cpp2::ErrorCode execute(PartitionID partId) override; + ErrorOr next(bool& hasNext) override { + DCHECK_EQ(children_.size(), 1); + auto& child = *children_[0]; + while (UNLIKELY(currentOffset_ < offset_)) { + auto result = child.next(hasNext); + if (!::nebula::ok(result) || !hasNext) { + return result; + } + currentOffset_++; + } + if (currentOffset_ < offset_ + limit_) { + return child.next(hasNext); + } else { + hasNext = false; + return ErrorOr(Row()); + } + } + + private: + const uint64_t offset_, limit_; + uint64_t currentOffset_ = 0; +}; +} // namespace storage + +} // namespace nebula From 3fb1792a44769e837a4c28f66956f48beb4ce219 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Fri, 24 Sep 2021 14:33:23 +0800 Subject: [PATCH 06/38] stash --- src/storage/exec/IndexDedupNode.cpp | 10 +- src/storage/exec/IndexDedupNode.h | 27 +++- src/storage/exec/IndexEdgeScanNode.h | 7 +- src/storage/exec/IndexLimitNode.cpp | 35 ++++++ src/storage/exec/IndexLimitNode.h | 25 +--- src/storage/exec/IndexNode.cpp | 22 ++++ src/storage/exec/IndexNode.h | 55 ++++++++- src/storage/exec/IndexProjectionNode.h | 29 ++++- src/storage/exec/IndexScanNode2.cpp | 4 +- src/storage/exec/IndexScanNode2.h | 16 ++- src/storage/exec/IndexSelectionNode.cpp | 20 ++- src/storage/exec/IndexSelectionNode.h | 96 +++++++++++--- src/storage/exec/IndexVertexScanNode.cpp | 72 ++++++++++- src/storage/exec/IndexVertexScanNode.h | 83 +------------ src/storage/index/LookupProcessor2.cpp | 151 ++++++++++++++++++++++- src/storage/index/LookupProcessor2.h | 23 ++-- 16 files changed, 513 insertions(+), 162 deletions(-) create mode 100644 src/storage/exec/IndexLimitNode.cpp create mode 100644 src/storage/exec/IndexNode.cpp diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index f8bb0c9d57d..92780b3d018 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -4,14 +4,10 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/exec/IndexDedupNode.h" - namespace nebula { namespace storage { - -IndexDedupNode::IndexDedupNode(RuntimeContext* context, - const std::vector& dedupColumn, - const std::vector& inputColumn) - : IndexNode(context) { +IndexDedupNode::IndexDedupNode(RuntimeContext* context, const std::vector& dedupColumn) + : IndexNode(context, "IndexDedupNode"), dedupColumns_(dedupColumn) { Map m; for (size_t i = 0; i < inputColumn.size(); i++) { m[inputColumn[i]] = i; @@ -21,7 +17,5 @@ IndexDedupNode::IndexDedupNode(RuntimeContext* context, dedupPos_.push_back(m.at(col)); } } - } // namespace storage - } // namespace nebula diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index 368f4ca155a..2ef82e0dcf4 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -11,11 +11,25 @@ namespace nebula { namespace storage { class IndexDedupNode : public IndexNode { public: - IndexDedupNode(RuntimeContext* context, - const std::vector& dedupColumn, - const std::vector& inputColumn); - nebula::cpp2::ErrorCode execute(PartitionID partId) = 0; - ErrorOr next(bool& hasNext) override { + IndexDedupNode(RuntimeContext* context, const std::vector& dedupColumn); + ::nebula::cpp2::ErrorCode init(InitContext& ctx) override { + for (auto& col : dedupColumns_) { + ctx.requiredColumns.insert(col); + } + // The return Row format of each child node must be the same + InitContext childCtx = ctx; + for (auto& child : children_) { + child->init(childCtx); + childCtx = ctx; + } + ctx = childCtx; + for (auto& col : dedupColumns_) { + dedupPos_.push_back(ctx.retColMap[col]); + } + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + + ErrorOr doNext(bool& hasNext) override { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; do { @@ -31,7 +45,7 @@ class IndexDedupNode : public IndexNode { private: bool dedup(const Row& row) { - auto result = dedupSet_.emplace(row); + auto result = dedupSet_.emplace(row, dedupPos_); return result.second; } class RowWrapper { @@ -56,6 +70,7 @@ class IndexDedupNode : public IndexNode { return a.values() == b.values(); } }; + std::vector dedupColumns_; std::vector dedupPos_; folly::F14FastSet dedupSet_; }; diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index 9d569c38262..be5c86101c3 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -16,19 +16,18 @@ class IndexEdgeScanNode : public IndexScanNode { static ErrorOr make(RuntimeContext* context, IndexID indexId, const std::vector& columnHint); - - private: IndexEdgeScanNode(RuntimeContext* context, IndexID indexId, const std::vector& columnHint) - : IndexScanNode(context, indexId, columnHint) {} + : IndexScanNode(context, "IndexEdgeScanNode", indexId, columnHint) {} + + private: Row decodeFromIndex(folly::StringPiece key) override; nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, std::pair& kv) override; Map decodeFromBase(const std::string& key, const std::string& value) override; const meta::SchemaProviderIf* getSchema() override; - private: std::shared_ptr edge_; }; } // namespace storage diff --git a/src/storage/exec/IndexLimitNode.cpp b/src/storage/exec/IndexLimitNode.cpp new file mode 100644 index 00000000000..3c6eec1eec4 --- /dev/null +++ b/src/storage/exec/IndexLimitNode.cpp @@ -0,0 +1,35 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#include "storage/exec/IndexLimitNode.h" +namespace nebula { +namespace storage { +IndexLimitNode::IndexLimitNode(RuntimeContext* context, uint64_t offset, uint64_t limit) + : IndexNode(context, "IndexLimitNode"), offset_(offset), limit_(limit) {} +IndexLimitNode::IndexLimitNode(RuntimeContext* context, uint64_t limit) + : IndexLimitNode(context, 0, limit) {} +nebula::cpp2::ErrorCode IndexLimitNode::doExecute(PartitionID partId) override { + currentOffset_ = 0; + return children_[0]->execute(partId); +} +IndexNode::ErrorOr IndexLimitNode::doNext(bool& hasNext) override { + DCHECK_EQ(children_.size(), 1); + auto& child = *children_[0]; + while (UNLIKELY(currentOffset_ < offset_)) { + auto result = child.next(hasNext); + if (!::nebula::ok(result) || !hasNext) { + return result; + } + currentOffset_++; + } + if (currentOffset_ < offset_ + limit_) { + return child.next(hasNext); + } else { + hasNext = false; + return ErrorOr(Row()); + } +} +} // namespace storage +} // namespace nebula diff --git a/src/storage/exec/IndexLimitNode.h b/src/storage/exec/IndexLimitNode.h index 71859653084..ae24483961b 100644 --- a/src/storage/exec/IndexLimitNode.h +++ b/src/storage/exec/IndexLimitNode.h @@ -10,29 +10,12 @@ namespace nebula { namespace storage { class IndexLimitNode : public IndexNode { public: - IndexLimitNode(RuntimeContext* context, uint64_t offset, uint64_t limit) - : IndexNode(context), offset_(offset), limit_(limit) {} - IndexLimitNode(RuntimeContext* context, uint64_t limit) : IndexLimitNode(context, 0, limit) {} - nebula::cpp2::ErrorCode execute(PartitionID partId) override; - ErrorOr next(bool& hasNext) override { - DCHECK_EQ(children_.size(), 1); - auto& child = *children_[0]; - while (UNLIKELY(currentOffset_ < offset_)) { - auto result = child.next(hasNext); - if (!::nebula::ok(result) || !hasNext) { - return result; - } - currentOffset_++; - } - if (currentOffset_ < offset_ + limit_) { - return child.next(hasNext); - } else { - hasNext = false; - return ErrorOr(Row()); - } - } + IndexLimitNode(RuntimeContext* context, uint64_t offset, uint64_t limit); + IndexLimitNode(RuntimeContext* context, uint64_t limit); private: + nebula::cpp2::ErrorCode doExecute(PartitionID partId) override; + ErrorOr doNext(bool& hasNext) override; const uint64_t offset_, limit_; uint64_t currentOffset_ = 0; }; diff --git a/src/storage/exec/IndexNode.cpp b/src/storage/exec/IndexNode.cpp new file mode 100644 index 00000000000..aa506541ed9 --- /dev/null +++ b/src/storage/exec/IndexNode.cpp @@ -0,0 +1,22 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "storage/exec/IndexNode.h" + +#include "folly/Likely.h" +namespace nebula { +namespace storage { +nebula::cpp2::ErrorCode IndexNode::doExecute(PartitionID partId) { + for (auto& child : children_) { + auto ret = child->execute(partId); + if (UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + return ret; + } + } + return ::nebula::cpp2::ErrorCode::SUCCEEDED; +} +} // namespace storage +} // namespace nebula diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index 7395fbca798..47eea5bd81a 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -6,6 +6,8 @@ #pragma once #include "common/base/ErrorOr.h" #include "common/datatypes/DataSet.h" +#include "common/time/Duration.h" +#include "folly/AtomicLinkedList.h" #include "folly/container/F14Map.h" #include "interface/gen-cpp2/common_types.h" #include "storage/CommonUtils.h" @@ -14,18 +16,63 @@ namespace storage { using ErrorCode = ::nebula::cpp2::ErrorCode; template using Map = folly::F14FastMap; +template +using Set = folly::F14FastSet; +struct InitContext { + Set requiredColumns; + std::vector returnColumns; + Map retColMap; +}; class IndexNode { public: template using ErrorOr = ::nebula::ErrorOr; - explicit IndexNode(RuntimeContext* context); - virtual nebula::cpp2::ErrorCode execute(PartitionID partId) = 0; - virtual ErrorOr next(bool& hasNext) = 0; + explicit IndexNode(RuntimeContext* context, const std::string& name); + virtual ::nebula::cpp2::ErrorCode init(InitContext& initCtx) { + DCHECK_EQ(children_.size(), 1); + return children_[0]->init(initCtx); + } + + inline nebula::cpp2::ErrorCode execute(PartitionID partId); + + inline ErrorOr next(bool& hasNext); + + void addChild(std::unique_ptr child) { children_.emplace_back(std::move(child)); } + virtual std::unique_ptr copy() = 0; + const std::vector>& children(); protected: + virtual ErrorOr doNext(bool& hasNext) = 0; + void beforeNext(); + void afterNext(); + virtual nebula::cpp2::ErrorCode doExecute(PartitionID partId); + void beforeExecute(); + void afterExecute(); + IndexNode(const IndexNode& node); + RuntimeContext* context_; GraphSpaceID spaceId_; - std::vector children_; + std::vector> children_; + std::string name_; + time::Duration duration_; }; + +/* Defination of inline function */ +inline IndexNode::ErrorOr IndexNode::next(bool& hasNext) { + beforeNext(); + auto ret = doNext(hasNext); + afterNext(); + return std::move(ret); +} +inline void IndexNode::beforeNext() { duration_.resume(); } +inline void IndexNode::afterNext() { duration_.pause(); } +inline nebula::cpp2::ErrorCode IndexNode::execute(PartitionID partId) { + beforeExecute(); + auto ret = doExecute(partId); + afterExecute(); + return std::move(ret); +} +inline void IndexNode::beforeExecute() { duration_.resume(); } +inline void IndexNode::afterExecute() { duration_.pause(); } } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index fc079c07949..c13c7a5f0e0 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -5,16 +5,35 @@ */ #pragma once +#include "folly/Likely.h" #include "storage/exec/IndexNode.h" namespace nebula { namespace storage { class IndexProjectionNode : public IndexNode { public: - IndexProjectionNode(RuntimeContext* context, - const std::vector& requiredColumns, - const std::vector& inputColumns); - nebula::cpp2::ErrorCode execute(PartitionID partId) = 0; - ErrorOr next(bool& hasNext) override { + IndexProjectionNode(RuntimeContext* context, const std::vector& requiredColumns); + nebula::cpp2::ErrorCode init(InitContext& ctx) override { + DCHECK_EQ(children_.size(), 1); + for (auto& col : requiredColumns_) { + ctx.requiredColumns.insert(col); + } + auto ret = children_[0]->init(ctx); + if (UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + return ret; + } + for (auto& col : requiredColumns_) { + auto iter = ctx.retColMap.find(col); + DCHECK_NE(iter, ctx.retColMap.end()); + colPos_[col] = iter->second; + } + ctx.returnColumns = requiredColumns_; + ctx.retColMap.clear(); + for (size_t i = 0; i < ctx.returnColumns.size(); i++) { + ctx.retColMap[ctx.returnColumns[i]] = i; + } + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + ErrorOr doNext(bool& hasNext) override { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; auto result = child.next(hasNext); diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index 7d434254b2a..dfa84d0ee5f 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -185,11 +185,11 @@ void PrefixPath::buildKey() { } // End of PrefixPath // Define of IndexScan -nebula::cpp2::ErrorCode IndexScanNode::execute(PartitionID partId) { +nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { auto ret = resetIter(partId); return ret; } -IndexNode::ErrorOr IndexScanNode::next(bool& hasNext) { +IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { hasNext = true; while (iter_ && iter_->valid()) { if (!checkTTL()) { diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index f3233e0d2d8..ce3e3c48695 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -23,13 +23,23 @@ class PrefixPath; class IndexScanNode : public IndexNode { public: IndexScanNode(RuntimeContext* context, + const std::string& name, IndexID indexId, const std::vector& columnHists) - : IndexNode(context), indexId_(indexId), columnHists_(columnHists) {} - nebula::cpp2::ErrorCode execute(PartitionID partId) final; - ErrorOr next(bool& hasNext) final; + : IndexNode(context, name), indexId_(indexId), columnHists_(columnHists) {} + ::nebula::cpp2::ErrorCode init(InitContext& ctx) override { + for (auto& col : ctx.requiredColumns) { + requiredColumns_.push_back(col); + } + ctx.returnColumns = requiredColumns_; + for (size_t i = 0; i < ctx.returnColumns.size(); i++) { + ctx.retColMap[ctx.returnColumns[i]] = i; + } + } protected: + nebula::cpp2::ErrorCode doExecute(PartitionID partId) final; + ErrorOr doNext(bool& hasNext) final; void decodePropFromIndex(folly::StringPiece key, const Map& colPosMap, std::vector& values); diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index 068442d890d..3b82fe5f3b4 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -9,12 +9,28 @@ namespace storage { IndexSelectionNode::IndexSelectionNode(RuntimeContext* context, std::unique_ptr expr, const std::vector& inputCols) - : IndexNode(context), expr_(std::move(expr)) { + : IndexNode(context, "IndexSelectionNode"), expr_(std::move(expr)) { for (size_t i = 0; i < inputCols.size(); i++) { colPos_[inputCols[i]] = i; } } -IndexNode::ErrorOr IndexSelectionNode::next(bool& hasNext) { +nebula::cpp2::ErrorCode IndexSelectionNode::init(InitContext& ctx) { + DCHECK_EQ(children_.size(), 1); + SelectionExprVisitor vis; + expr_->accept(&vis); + for (auto& col : vis.getRequiredColumns()) { + ctx.requiredColumns.insert(col); + } + auto ret = children_[0]->init(ctx); + if (UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + return ret; + } + for (auto& col : vis.getRequiredColumns()) { + colPos_[col] = ctx.retColMap.at(col); + } + return ::nebula::cpp2::ErrorCode::SUCCEEDED; +} +IndexNode::ErrorOr IndexSelectionNode::doNext(bool& hasNext) { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; do { diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 2350666a0fc..031ef63a7aa 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -6,6 +6,7 @@ #pragma once #include "common/context/ExpressionContext.h" +#include "common/expression/ExprVisitor.h" #include "common/expression/Expression.h" #include "folly/container/F14Map.h" #include "storage/exec/IndexNode.h" @@ -13,49 +14,49 @@ namespace nebula { namespace storage { class IndexSelectionNode : public IndexNode { public: - IndexSelectionNode(RuntimeContext* context, + IndexSelectionNode(RuntimeContext *context, std::unique_ptr expr, - const std::vector& inputCols); - nebula::cpp2::ErrorCode execute(PartitionID partId) override; - ErrorOr next(bool& hasNext) override; + const std::vector &inputCols); + nebula::cpp2::ErrorCode init(InitContext &ctx) override; private: - inline bool filter(const Row& row) { + ErrorOr doNext(bool &hasNext) override; + inline bool filter(const Row &row) { ctx_->setRow(row); - auto& result = expr_->eval(*ctx_); + auto &result = expr_->eval(*ctx_); return result.type() == Value::Type::BOOL ? result.getBool() : false; } Map colPos_; std::unique_ptr expr_; class ExprContext : public ExpressionContext { public: - explicit ExprContext(const Map& colPos) : colPos_(colPos) {} - void setRow(const Row& row) { row_ = &row; } - Value getEdgeProp(const std::string& edgeType, const std::string& prop) const override; - Value getTagProp(const std::string& tag, const std::string& prop) const override; + explicit ExprContext(const Map &colPos) : colPos_(colPos) {} + void setRow(const Row &row) { row_ = &row; } + Value getEdgeProp(const std::string &edgeType, const std::string &prop) const override; + Value getTagProp(const std::string &tag, const std::string &prop) const override; // override - const Value& getVar(const std::string& var) const override { + const Value &getVar(const std::string &var) const override { LOG(FATAL) << "Unexpect"; return Value(); } - const Value& getVersionedVar(const std::string& var, int64_t version) const override { + const Value &getVersionedVar(const std::string &var, int64_t version) const override { LOG(FATAL) << "Unexpect"; return Value(); } - const Value& getVarProp(const std::string& var, const std::string& prop) const override { + const Value &getVarProp(const std::string &var, const std::string &prop) const override { LOG(FATAL) << "Unexpect"; return Value(); } - Value getSrcProp(const std::string& tag, const std::string& prop) const override { + Value getSrcProp(const std::string &tag, const std::string &prop) const override { LOG(FATAL) << "Unexpect"; return Value(); } - const Value& getDstProp(const std::string& tag, const std::string& prop) const override { + const Value &getDstProp(const std::string &tag, const std::string &prop) const override { LOG(FATAL) << "Unexpect"; return Value(); } - const Value& getInputProp(const std::string& prop) const override { + const Value &getInputProp(const std::string &prop) const override { LOG(FATAL) << "Unexpect"; return Value(); } @@ -71,14 +72,71 @@ class IndexSelectionNode : public IndexNode { LOG(FATAL) << "Unexpect"; return Value(); } - void setVar(const std::string& var, Value val) override { LOG(FATAL) << "Unexpect"; } + void setVar(const std::string &var, Value val) override { LOG(FATAL) << "Unexpect"; } private: - const Map& colPos_; - const Row* row_; + const Map &colPos_; + const Row *row_; }; std::unique_ptr ctx_; }; + +class SelectionExprVisitor : public ::nebula::ExprVisitor { + public: + void visit(ConstantExpression *expr) override { return; } + void visit(UnaryExpression *expr) override { expr->operand()->accept(this); } + void visit(TypeCastingExpression *expr) override { expr->operand()->accept(this); } + void visit(LabelExpression *expr) override { return; } + void visit(LabelAttributeExpression *expr) override { return; } + void visit(ArithmeticExpression *expr) override { + expr->left()->accept(this); + expr->right()->accept(this); + } + void visit(RelationalExpression *expr) override { + expr->left()->accept(this); + expr->right()->accept(this); + } + void visit(SubscriptExpression *expr) override { + expr->left()->accept(this); + expr->right()->accept(this); + } + void visit(AttributeExpression *expr) override; + void visit(LogicalExpression *expr) override; + void visit(FunctionCallExpression *expr) override; + void visit(AggregateExpression *expr) override; + void visit(UUIDExpression *expr) override; + void visit(VariableExpression *expr) override; + void visit(VersionedVariableExpression *expr) override; + void visit(ListExpression *expr) override; + void visit(SetExpression *expr) override; + void visit(MapExpression *expr) override; + void visit(TagPropertyExpression *expr) override; + void visit(EdgePropertyExpression *expr) override; + void visit(InputPropertyExpression *expr) override; + void visit(VariablePropertyExpression *expr) override; + void visit(DestPropertyExpression *expr) override; + void visit(SourcePropertyExpression *expr) override; + void visit(EdgeSrcIdExpression *expr) override; + void visit(EdgeTypeExpression *expr) override; + void visit(EdgeRankExpression *expr) override; + void visit(EdgeDstIdExpression *expr) override; + void visit(VertexExpression *expr) override; + void visit(EdgeExpression *expr) override; + void visit(CaseExpression *expr) override; + void visit(PathBuildExpression *expr) override; + void visit(ColumnExpression *expr) override; + void visit(PredicateExpression *expr) override; + void visit(ListComprehensionExpression *expr) override; + void visit(ReduceExpression *expr) override; + void visit(SubscriptRangeExpression *expr) override; + const Set &getRequiredColumns(); + ::nebula::cpp2::ErrorCode getCode(); + + private: + Set requiredColumns_; + ::nebula::cpp2::ErrorCode code_; +}; + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index 9821181dd47..4ab03062d33 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -11,7 +11,77 @@ namespace storage { IndexVertexScanNode::IndexVertexScanNode(RuntimeContext* context, IndexID indexId, const std::vector& clolumnHint) - : IndexScanNode(context, indexId, clolumnHint) {} + : IndexScanNode(context, "IndexVertexScanNode", indexId, clolumnHint) {} +::nebula::cpp2::ErrorCode IndexVertexScanNode::init(InitContext& ctx) { + auto env = context_->env(); + auto spaceId = context_->spaceId(); + auto indexMgr = env->indexMan_; + auto schemaMgr = env->schemaMan_; + auto index = indexMgr->getTagIndex(spaceId, indexId_).value(); + auto tagSchema = schemaMgr->getTagSchema(spaceId, index->get_schema_id().get_tag_id()); + this->index_ = index; + this->tag_ = tagSchema; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; +} +nebula::cpp2::ErrorCode IndexVertexScanNode::getBaseData(folly::StringPiece key, + std::pair& kv) { + kv.first = NebulaKeyUtils::vertexKey(context_->vIdLen(), + partId_, + key.subpiece(key.size() - context_->vIdLen()).toString(), + context_->tagId_); + return context_->env()->kvstore_->get(context_->spaceId(), partId_, kv.first, &kv.second); +} +Row IndexVertexScanNode::decodeFromIndex(folly::StringPiece key) { + std::vector values(requiredColumns_.size()); + Map colPosMap; + for (size_t i = 0; i < requiredColumns_.size(); i++) { + colPosMap[requiredColumns_[i]] = i; + } + if (colPosMap.count(kVid)) { + auto vId = IndexKeyUtils::getIndexVertexID(context_->vIdLen(), key); + if (context_->isIntId()) { + values[colPosMap[kVid]] = Value(*reinterpret_cast(vId.data())); + } else { + values[colPosMap[kVid]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + } + } + if (colPosMap.count(kTag)) { + values[colPosMap[kTag]] = Value(context_->tagId_); + } + key.subtract(context_->vIdLen()); + decodePropFromIndex(key, colPosMap, values); + return Row(std::move(values)); +} +Map IndexVertexScanNode::decodeFromBase(const std::string& key, + const std::string& value) { + Map values; + auto reader = RowReaderWrapper::getRowReader(tag_.get(), value); + for (auto& col : requiredColumns_) { + switch (QueryUtils::toReturnColType(col)) { + case QueryUtils::ReturnColType::kVid: { + auto vId = NebulaKeyUtils::getVertexId(context_->vIdLen(), key); + if (context_->isIntId()) { + values[col] = Value(*reinterpret_cast(vId.data())); + } else { + values[col] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + } + } break; + case QueryUtils::ReturnColType::kTag: { + values[col] = Value(context_->tagId_); + } break; + case QueryUtils::ReturnColType::kOther: { + auto retVal = QueryUtils::readValue(reader.get(), col, tag_->field(col)); + if (!retVal.ok()) { + LOG(FATAL) << "Bad value for field" << col; + } + values[col] = std::move(retVal.value()); + } break; + default: + LOG(FATAL) << "Unexpect column name:" << col; + } + } + return values; +} } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index 81215261f76..bd25b651c8c 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -13,89 +13,18 @@ namespace nebula { namespace storage { class IndexVertexScanNode final : public IndexScanNode { public: - static ErrorOr make(RuntimeContext* context, - IndexID indexId, - const std::vector& columnHint) { - IndexVertexScanNode node(context, indexId, columnHint); - auto env = context->env(); - auto spaceId = context->spaceId(); - auto indexMgr = env->indexMan_; - auto schemaMgr = env->schemaMan_; - auto index = indexMgr->getTagIndex(spaceId, indexId).value(); - auto tagSchema = schemaMgr->getTagSchema(spaceId, index->get_schema_id().get_tag_id()); - node.index_ = index; - node.tag_ = tagSchema; - return ErrorOr(std::move(node)); - } - - private: + ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; IndexVertexScanNode(RuntimeContext* context, IndexID indexId, - const std::vector& clolumnHint) - : IndexScanNode(context, indexId, clolumnHint) {} + const std::vector& clolumnHint); + private: nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, - std::pair& kv) override { - kv.first = NebulaKeyUtils::vertexKey(context_->vIdLen(), - partId_, - key.subpiece(key.size() - context_->vIdLen()).toString(), - context_->tagId_); - return context_->env()->kvstore_->get(context_->spaceId(), partId_, kv.first, &kv.second); - } - Row decodeFromIndex(folly::StringPiece key) override { - std::vector values(requiredColumns_.size()); - Map colPosMap; - for (size_t i = 0; i < requiredColumns_.size(); i++) { - colPosMap[requiredColumns_[i]] = i; - } - if (colPosMap.count(kVid)) { - auto vId = IndexKeyUtils::getIndexVertexID(context_->vIdLen(), key); - if (context_->isIntId()) { - values[colPosMap[kVid]] = Value(*reinterpret_cast(vId.data())); - } else { - values[colPosMap[kVid]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); - } - } - if (colPosMap.count(kTag)) { - values[colPosMap[kTag]] = Value(context_->tagId_); - } - key.subtract(context_->vIdLen()); - decodePropFromIndex(key, colPosMap, values); - return Row(std::move(values)); - } - Map decodeFromBase(const std::string& key, - const std::string& value) override { - Map values; - auto reader = RowReaderWrapper::getRowReader(tag_.get(), value); - for (auto& col : requiredColumns_) { - switch (QueryUtils::toReturnColType(col)) { - case QueryUtils::ReturnColType::kVid: { - auto vId = NebulaKeyUtils::getVertexId(context_->vIdLen(), key); - if (context_->isIntId()) { - values[col] = Value(*reinterpret_cast(vId.data())); - } else { - values[col] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); - } - } break; - case QueryUtils::ReturnColType::kTag: { - values[col] = Value(context_->tagId_); - } break; - case QueryUtils::ReturnColType::kOther: { - auto retVal = QueryUtils::readValue(reader.get(), col, tag_->field(col)); - if (!retVal.ok()) { - LOG(FATAL) << "Bad value for field" << col; - } - values[col] = std::move(retVal.value()); - } break; - default: - LOG(FATAL) << "Unexpect column name:" << col; - } - } - return values; - } + std::pair& kv) override; + Row decodeFromIndex(folly::StringPiece key) override; + Map decodeFromBase(const std::string& key, const std::string& value) override; const meta::SchemaProviderIf* getSchema() override { return tag_.get(); } - private: std::shared_ptr tag_; }; } // namespace storage diff --git a/src/storage/index/LookupProcessor2.cpp b/src/storage/index/LookupProcessor2.cpp index a5cf7537b2e..c0b16b2da71 100644 --- a/src/storage/index/LookupProcessor2.cpp +++ b/src/storage/index/LookupProcessor2.cpp @@ -5,10 +5,159 @@ */ #include "storage/index/LookupProcessor2.h" +#include "storage/exec/IndexDedupNode.h" +#include "storage/exec/IndexEdgeScanNode.h" +#include "storage/exec/IndexLimitNode.h" +#include "storage/exec/IndexNode.h" +#include "storage/exec/IndexProjectionNode.h" +#include "storage/exec/IndexSelectionNode.h" +#include "storage/exec/IndexVertexScanNode.h" + namespace nebula { namespace storage { -void LookupProcessor::process(const cpp2::LookupIndexRequest& req) {} +void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { + if (executor_ != nullptr) { + executor_->add([req, this]() { this->doProcess(req); }); + } else { + doProcess(req); + } +} +void LookupProcessor::doProcess(const cpp2::LookupIndexRequest& req) { + prepare(req); + auto plan = buildPlan(req); + InitContext context; + plan->init(context); + if (!FLAGS_query_concurrently) { + runInSingleThread(req.get_parts(), std::move(plan)); + } else { + runInMultipleThread(req.get_parts(), std::move(plan)); + } +} +::nebula::cpp2::ErrorCode LookupProcessor::prepare(const cpp2::LookupIndexRequest& req) {} + +std::unique_ptr LookupProcessor::buildPlan(const cpp2::LookupIndexRequest& req) { + bool is_edge = req.get_indices().get_schema_id().edge_type_ref().has_value(); + std::vector> nodes; + for (auto& ctx : req.get_indices().get_contexts()) { + auto node = buildOneContext(ctx); + } + for (size_t i = 0; i < nodes.size(); i++) { + auto projection = std::make_unique(); + projection->addChild(std::move(nodes[i])); + nodes[i] = std::move(projection); + } + if (nodes.size() > 1) { + auto dedup = std::make_unique(); + for (auto& node : nodes) { + node->addChild(std::move(node)); + } + nodes.clear(); + nodes[0] = std::move(dedup); + } + if (req.limit_ref().has_value()) { + auto limit = req.get_limit(); + auto node = std::make_unique(); + node->addChild(std::move(nodes[0])); + nodes[0] = std::move(node); + } + InitContext ctx; + auto result = nodes[0]->init(ctx); + if (result == ::nebula::cpp2::ErrorCode::SUCCEEDED) { + } else { + } + return std::move(nodes[0]); +} + +std::unique_ptr LookupProcessor::buildOneContext(const cpp2::IndexQueryContext& ctx) { + std::unique_ptr node; + if (context_->isEdge()) { + node = std::make_unique(); + } else { + node = std::make_unique(); + } + if (ctx.filter_ref().has_value()) { + auto filterNode = std::make_unique(context_.get(), nullptr); + } + return std::move(node); +} +void LookupProcessor::runInSingleThread(const std::vector& parts, + std::unique_ptr plan) { + std::vector> datasetList; + std::vector<::nebula::cpp2::ErrorCode> codeList; + for (auto part : parts) { + plan->execute(part); + bool hasNext = true; + ::nebula::cpp2::ErrorCode code; + decltype(datasetList)::value_type dataset; + nebula::DataSet dataset; + do { + auto result = plan->next(hasNext); + if (hasNext && ::nebula::ok(result)) { + dataset.emplace_back(::nebula::value(std::move(result))); + } else { + break; + } + } while (true); + datasetList.emplace_back(std::move(dataset)); + codeList.emplace_back(code); + } + // 处理 leader change +} +void LookupProcessor::runInMultipleThread(const std::vector& parts, + std::unique_ptr plan) { + std::vector> planCopy = reproducePlan(plan.get(), parts.size()); + std::vector>> datasetList(parts.size()); + std::vector<::nebula::cpp2::ErrorCode> codeList(parts.size()); + std::vector> funcs; + std::vector> futures; + for (size_t i = 0; i < parts.size(); i++) { + funcs.emplace_back([this, + &plan = planCopy[i], + &dataset = datasetList[i], + &code = codeList[i], + part = parts[i], + index = i]() { + plan->execute(part); + bool hasNext = true; + do { + auto result = plan->next(hasNext); + if (hasNext && ::nebula::ok(result)) { + dataset.emplace_back(::nebula::value(std::move(result))); + } else { + break; + } + } while (true); + return index; + }); + } + for (size_t i = 0; i < parts.size(); i++) { + futures.emplace_back(folly::via(executor_, std::move(funcs[i]))); + } + folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) { + CHECK(!t.hasException()); + const auto& values = t.value(); + for (auto& value : values) { + // for (size_t i = 0; i < t.size(); i++) { + // // 检查切主 合并数据 + // } + } + }); +} +std::vector> LookupProcessor::reproducePlan(IndexNode* root, + size_t count) { + std::vector> ret(count); + for (size_t i = 0; i < count; i++) { + ret.emplace_back(root->copy()); + } + for (auto& child : root->children()) { + auto childPerPlan = reproducePlan(child.get(), count); + for (size_t i = 0; i < count; i++) { + ret[i]->addChild(std::move(childPerPlan[i])); + } + } + return std::move(ret); +} } // namespace storage } // namespace nebula diff --git a/src/storage/index/LookupProcessor2.h b/src/storage/index/LookupProcessor2.h index 6f700b7dfc7..557ea6398a2 100644 --- a/src/storage/index/LookupProcessor2.h +++ b/src/storage/index/LookupProcessor2.h @@ -5,13 +5,13 @@ */ #pragma once #include "common/base/Base.h" +#include "common/base/ErrorOr.h" #include "interface/gen-cpp2/storage_types.h" #include "storage/BaseProcessor.h" +#include "storage/exec/IndexNode.h" namespace nebula { namespace storage { extern ProcessorCounters kLookupCounters; -using IndexFilterItem = - std::unordered_map, Expression*>>; class LookupProcessor : public BaseProcessor { public: @@ -22,16 +22,21 @@ class LookupProcessor : public BaseProcessor { private: LookupProcessor(StorageEnv* env, const ProcessorCounters* counters, folly::Executor* executor); + void doProcess(const cpp2::LookupIndexRequest& req); void onProcessFinished(); - void runInSingleThread(const cpp2::LookupIndexRequest& req); - void runInMultipleThread(const cpp2::LookupIndexRequest& req); - folly::Future> runInExecutor( - IndexFilterItem* filterItem, nebula::DataSet* result, PartitionID partId); - - void doProcess(const cpp2::LookupIndexRequest& req); + void runInSingleThread(const std::vector& parts, std::unique_ptr plan); + void runInMultipleThread(const std::vector& parts, std::unique_ptr plan); + ::nebula::cpp2::ErrorCode prepare(const cpp2::LookupIndexRequest& req); + std::unique_ptr buildPlan(const cpp2::LookupIndexRequest& req); + std::unique_ptr buildOneContext(const cpp2::IndexQueryContext& ctx); + std::vector> reproducePlan(IndexNode* root, size_t count); folly::Executor* executor_{nullptr}; + folly::Executor* executor_{nullptr}; + std::unique_ptr planContext_; + std::unique_ptr context_; + nebula::DataSet resultDataSet_; + std::vector partResults_; }; } // namespace storage - } // namespace nebula From e738d84b9f00e63b98b6f1f82b6054cfe2920993 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Fri, 24 Sep 2021 15:25:39 +0800 Subject: [PATCH 07/38] stash --- src/storage/exec/IndexDedupNode.cpp | 43 +++++++++++++++--- src/storage/exec/IndexDedupNode.h | 56 +++++++----------------- src/storage/exec/IndexEdgeScanNode.cpp | 18 +++----- src/storage/exec/IndexEdgeScanNode.h | 1 + src/storage/exec/IndexLimitNode.cpp | 4 +- src/storage/exec/IndexProjectionNode.cpp | 28 +++++++++--- src/storage/exec/IndexProjectionNode.h | 40 ++--------------- 7 files changed, 86 insertions(+), 104 deletions(-) diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index 92780b3d018..c4a855e146f 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -7,14 +7,43 @@ namespace nebula { namespace storage { IndexDedupNode::IndexDedupNode(RuntimeContext* context, const std::vector& dedupColumn) - : IndexNode(context, "IndexDedupNode"), dedupColumns_(dedupColumn) { - Map m; - for (size_t i = 0; i < inputColumn.size(); i++) { - m[inputColumn[i]] = i; + : IndexNode(context, "IndexDedupNode"), dedupColumns_(dedupColumn) {} +::nebula::cpp2::ErrorCode IndexDedupNode::init(InitContext& ctx) { + for (auto& col : dedupColumns_) { + ctx.requiredColumns.insert(col); } - for (auto& col : dedupColumn) { - DCHECK(m.count(col)); - dedupPos_.push_back(m.at(col)); + // The return Row format of each child node must be the same + InitContext childCtx = ctx; + for (auto& child : children_) { + child->init(childCtx); + childCtx = ctx; + } + ctx = childCtx; + for (auto& col : dedupColumns_) { + dedupPos_.push_back(ctx.retColMap[col]); + } + return ::nebula::cpp2::ErrorCode::SUCCEEDED; +} +::nebula::cpp2::ErrorCode IndexDedupNode::doExecute(PartitionID partId) { + dedupSet_.clear(); + return IndexNode::doExecute(partId); +} +IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) override { + DCHECK_EQ(children_.size(), 1); + auto& child = *children_[0]; + do { + auto result = child.next(hasNext); + if (!hasNext || !nebula::ok(result)) { + return result; + } + if (dedup(::nebula::value(result))) { + return result; + } + } while (true); +} +IndexDedupNode::RowWrapper::RowWrapper(const Row& row, const std::vector& posList) { + for (auto p : posList) { + values_.emplace_back(row[p]); } } } // namespace storage diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index 2ef82e0dcf4..5a37a64487b 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -12,54 +12,23 @@ namespace storage { class IndexDedupNode : public IndexNode { public: IndexDedupNode(RuntimeContext* context, const std::vector& dedupColumn); - ::nebula::cpp2::ErrorCode init(InitContext& ctx) override { - for (auto& col : dedupColumns_) { - ctx.requiredColumns.insert(col); - } - // The return Row format of each child node must be the same - InitContext childCtx = ctx; - for (auto& child : children_) { - child->init(childCtx); - childCtx = ctx; - } - ctx = childCtx; - for (auto& col : dedupColumns_) { - dedupPos_.push_back(ctx.retColMap[col]); - } - return ::nebula::cpp2::ErrorCode::SUCCEEDED; - } - - ErrorOr doNext(bool& hasNext) override { - DCHECK_EQ(children_.size(), 1); - auto& child = *children_[0]; - do { - auto result = child.next(hasNext); - if (!hasNext || !nebula::ok(result)) { - return result; - } - if (dedup(::nebula::value(result))) { - return result; - } - } while (true); - } + ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; private: - bool dedup(const Row& row) { - auto result = dedupSet_.emplace(row, dedupPos_); - return result.second; - } + inline bool dedup(const Row& row); + ::nebula::cpp2::ErrorCode doExecute(PartitionID partId) override; + ErrorOr doNext(bool& hasNext) override; + + // Define RowWrapper for dedup class RowWrapper { public: - RowWrapper(const Row& row, const std::vector& posList) { - for (auto p : posList) { - values_.emplace_back(row[p]); - } - } - const List& values() const { return values_; } + RowWrapper(const Row& row, const std::vector& posList); + inline const List& values() const { return values_; } private: List values_; }; + // End of RowWrapper struct Hasher { size_t operator()(const RowWrapper& wrapper) const { return std::hash()(wrapper.values()); @@ -74,6 +43,11 @@ class IndexDedupNode : public IndexNode { std::vector dedupPos_; folly::F14FastSet dedupSet_; }; -} // namespace storage +/* Definition of inline function */ +inline bool IndexDedupNode::dedup(const Row& row) { + auto result = dedupSet_.emplace(row, dedupPos_); + return result.second; +} +} // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index 6f744415d0f..a7c6d2829e0 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -7,20 +7,16 @@ #include "storage/exec/IndexEdgeScanNode.h" namespace nebula { namespace storage { -IndexNode::ErrorOr IndexEdgeScanNode::make( - RuntimeContext* context, - IndexID indexId, - const std::vector& columnHint) { - IndexEdgeScanNode node(context, indexId, columnHint); - auto env = context->env(); - auto spaceId = context->spaceId(); +::nebula::cpp2::ErrorCode IndexEdgeScanNode::init(InitContext& ctx) { + auto env = context_->env(); + auto spaceId = context_->spaceId(); auto indexMgr = env->indexMan_; auto schemaMgr = env->schemaMan_; - auto index = indexMgr->getTagIndex(spaceId, indexId).value(); + auto index = indexMgr->getTagIndex(spaceId, indexId_).value(); auto edgeSchema = schemaMgr->getEdgeSchema(spaceId, index->get_schema_id().get_tag_id()); - node.index_ = index; - node.edge_ = edgeSchema; - return ErrorOr(std::move(node)); + this->index_ = index; + this->edge_ = edgeSchema; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; } Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { std::vector values(requiredColumns_.size()); diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index be5c86101c3..dafad9737f9 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -20,6 +20,7 @@ class IndexEdgeScanNode : public IndexScanNode { IndexID indexId, const std::vector& columnHint) : IndexScanNode(context, "IndexEdgeScanNode", indexId, columnHint) {} + ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; private: Row decodeFromIndex(folly::StringPiece key) override; diff --git a/src/storage/exec/IndexLimitNode.cpp b/src/storage/exec/IndexLimitNode.cpp index 3c6eec1eec4..26129850622 100644 --- a/src/storage/exec/IndexLimitNode.cpp +++ b/src/storage/exec/IndexLimitNode.cpp @@ -10,11 +10,11 @@ IndexLimitNode::IndexLimitNode(RuntimeContext* context, uint64_t offset, uint64_ : IndexNode(context, "IndexLimitNode"), offset_(offset), limit_(limit) {} IndexLimitNode::IndexLimitNode(RuntimeContext* context, uint64_t limit) : IndexLimitNode(context, 0, limit) {} -nebula::cpp2::ErrorCode IndexLimitNode::doExecute(PartitionID partId) override { +nebula::cpp2::ErrorCode IndexLimitNode::doExecute(PartitionID partId) { currentOffset_ = 0; return children_[0]->execute(partId); } -IndexNode::ErrorOr IndexLimitNode::doNext(bool& hasNext) override { +IndexNode::ErrorOr IndexLimitNode::doNext(bool& hasNext) { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; while (UNLIKELY(currentOffset_ < offset_)) { diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp index e250cccec59..506f86a5ca9 100644 --- a/src/storage/exec/IndexProjectionNode.cpp +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -7,14 +7,30 @@ namespace nebula { namespace storage { IndexProjectionNode::IndexProjectionNode(RuntimeContext* context, - const std::vector& requiredColumns, - const std::vector& inputColumns) - : IndexNode(context), requiredColumns_(requiredColumns) { - for (size_t i = 0; i < inputColumns.size(); i++) { - colPos_[inputColumns[i]] = i; + const std::vector& requiredColumns) + : IndexNode(context, "IndexProjectionNode"), requiredColumns_(requiredColumns) {} +nebula::cpp2::ErrorCode IndexProjectionNode::init(InitContext& ctx) { + DCHECK_EQ(children_.size(), 1); + for (auto& col : requiredColumns_) { + ctx.requiredColumns.insert(col); + } + auto ret = children_[0]->init(ctx); + if (UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + return ret; + } + for (auto& col : requiredColumns_) { + auto iter = ctx.retColMap.find(col); + DCHECK_NE(iter, ctx.retColMap.end()); + colPos_[col] = iter->second; + } + ctx.returnColumns = requiredColumns_; + ctx.retColMap.clear(); + for (size_t i = 0; i < ctx.returnColumns.size(); i++) { + ctx.retColMap[ctx.returnColumns[i]] = i; } + return ::nebula::cpp2::ErrorCode::SUCCEEDED; } -IndexNode::ErrorOr IndexProjectionNode::next(bool& hasNext) { +IndexNode::ErrorOr IndexProjectionNode::doNext(bool& hasNext) { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; auto result = child.next(hasNext); diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index c13c7a5f0e0..c7b751388bb 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -12,45 +12,11 @@ namespace storage { class IndexProjectionNode : public IndexNode { public: IndexProjectionNode(RuntimeContext* context, const std::vector& requiredColumns); - nebula::cpp2::ErrorCode init(InitContext& ctx) override { - DCHECK_EQ(children_.size(), 1); - for (auto& col : requiredColumns_) { - ctx.requiredColumns.insert(col); - } - auto ret = children_[0]->init(ctx); - if (UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { - return ret; - } - for (auto& col : requiredColumns_) { - auto iter = ctx.retColMap.find(col); - DCHECK_NE(iter, ctx.retColMap.end()); - colPos_[col] = iter->second; - } - ctx.returnColumns = requiredColumns_; - ctx.retColMap.clear(); - for (size_t i = 0; i < ctx.returnColumns.size(); i++) { - ctx.retColMap[ctx.returnColumns[i]] = i; - } - return ::nebula::cpp2::ErrorCode::SUCCEEDED; - } - ErrorOr doNext(bool& hasNext) override { - DCHECK_EQ(children_.size(), 1); - auto& child = *children_[0]; - auto result = child.next(hasNext); - if (::nebula::ok(result) && hasNext) { - return ErrorOr(project(::nebula::value(std::move(result)))); - } else { - return ErrorOr(Row()); - } - } + nebula::cpp2::ErrorCode init(InitContext& ctx) override; private: - Row project(Row&& row) { - Row ret; - for (auto& col : requiredColumns_) { - ret.emplace_back(std::move(row[colPos_[col]])); - } - } + ErrorOr doNext(bool& hasNext) override; + Row project(Row&& row); std::vector requiredColumns_; Map colPos_; }; From 6efb958aa674637b330376ea22386eadb3251d97 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Tue, 28 Sep 2021 16:59:40 +0800 Subject: [PATCH 08/38] stash --- src/storage/CMakeLists.txt | 3 +- src/storage/ExprVisitorBase.cpp | 86 ++++ src/storage/ExprVisitorBase.h | 72 ++++ src/storage/exec/IndexDedupNode.cpp | 9 +- src/storage/exec/IndexDedupNode.h | 2 + src/storage/exec/IndexEdgeScanNode.cpp | 51 ++- src/storage/exec/IndexEdgeScanNode.h | 15 +- src/storage/exec/IndexLimitNode.cpp | 7 + src/storage/exec/IndexLimitNode.h | 2 + src/storage/exec/IndexNode.cpp | 3 + src/storage/exec/IndexNode.h | 5 +- src/storage/exec/IndexProjectionNode.cpp | 5 + src/storage/exec/IndexProjectionNode.h | 2 + src/storage/exec/IndexScanNode.h | 5 +- src/storage/exec/IndexScanNode2.cpp | 12 + src/storage/exec/IndexScanNode2.h | 11 +- src/storage/exec/IndexSelectionNode.cpp | 18 +- src/storage/exec/IndexSelectionNode.h | 114 ++---- src/storage/exec/IndexVertexScanNode.cpp | 52 ++- src/storage/exec/IndexVertexScanNode.h | 19 +- src/storage/index/LookupProcessor2.cpp | 81 ++-- src/storage/index/LookupProcessor2.h | 1 - src/storage/test/IndexTest.cpp | 100 +++++ src/storage/test/IndexTestUtil.h | 485 +++++++++++++++++++++++ 24 files changed, 1014 insertions(+), 146 deletions(-) create mode 100644 src/storage/ExprVisitorBase.cpp create mode 100644 src/storage/ExprVisitorBase.h create mode 100644 src/storage/test/IndexTest.cpp create mode 100644 src/storage/test/IndexTestUtil.h diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index a1b081f3d43..70706693396 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -27,6 +27,7 @@ nebula_add_library( nebula_add_library( graph_storage_service_handler OBJECT GraphStorageServiceHandler.cpp + ExprVisitorBase.cpp context/StorageExpressionContext.cpp mutate/AddVerticesProcessor.cpp mutate/DeleteVerticesProcessor.cpp @@ -39,7 +40,7 @@ nebula_add_library( query/GetPropProcessor.cpp query/ScanVertexProcessor.cpp query/ScanEdgeProcessor.cpp - index/LookupProcessor.cpp + index/LookupProcessor2.cpp ) nebula_add_library( diff --git a/src/storage/ExprVisitorBase.cpp b/src/storage/ExprVisitorBase.cpp new file mode 100644 index 00000000000..c52be9f7803 --- /dev/null +++ b/src/storage/ExprVisitorBase.cpp @@ -0,0 +1,86 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#include "storage/ExprVisitorBase.h" +namespace nebula { +namespace storage { +void ExprVisitorBase::visit(ConstantExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(UnaryExpression *expr) { expr->operand()->accept(this); } +void ExprVisitorBase::visit(TypeCastingExpression *expr) { expr->operand()->accept(this); } +void ExprVisitorBase::visit(LabelExpression *expr) { fatal(expr); } +void ExprVisitorBase::visit(LabelAttributeExpression *expr) { fatal(expr); } +// binary expression +void ExprVisitorBase::visit(ArithmeticExpression *expr) { + expr->left()->accept(this); + expr->right()->accept(this); +} +void ExprVisitorBase::visit(RelationalExpression *expr) { + expr->left()->accept(this); + expr->right()->accept(this); +} +void ExprVisitorBase::visit(SubscriptExpression *expr) { + expr->left()->accept(this); + expr->right()->accept(this); +} +void ExprVisitorBase::visit(AttributeExpression *expr) { fatal(expr); } +void ExprVisitorBase::visit(LogicalExpression *expr) { + for (auto operand : expr->operands()) { + operand->accept(this); + } +} +// function call +void ExprVisitorBase::visit(FunctionCallExpression *expr) { + for (auto arg : expr->args()->args()) { + arg->accept(this); + } +} +void ExprVisitorBase::visit(AggregateExpression *expr) { fatal(expr); } +void ExprVisitorBase::visit(UUIDExpression *expr) { UNUSED(expr); } +// variable expression +void ExprVisitorBase::visit(VariableExpression *expr) { fatal(expr); } +void ExprVisitorBase::visit(VersionedVariableExpression *expr) { fatal(expr); } +// container expression +void ExprVisitorBase::visit(ListExpression *expr) { + for (auto item : expr->items()) { + item->accept(this); + } +} +void ExprVisitorBase::visit(SetExpression *expr) { + for (auto item : expr->items()) { + item->accept(this); + } +} +void ExprVisitorBase::visit(MapExpression *expr) { UNUSED(expr); } +// property Expression +void visit(TagPropertyExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(EdgePropertyExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(InputPropertyExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(VariablePropertyExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(DestPropertyExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(SourcePropertyExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(EdgeSrcIdExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(EdgeTypeExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(EdgeRankExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(EdgeDstIdExpression *expr) { UNUSED(expr); } +// vertex/edge expression +void ExprVisitorBase::visit(VertexExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(EdgeExpression *expr) { UNUSED(expr); } +// case expression +void ExprVisitorBase::visit(CaseExpression *expr) { UNUSED(expr); } +// path build expression +void ExprVisitorBase::visit(PathBuildExpression *expr) { fatal(expr); } +// column expression +void ExprVisitorBase::visit(ColumnExpression *expr) { fatal(expr); } +// predicate expression +void ExprVisitorBase::visit(PredicateExpression *expr) { fatal(expr); } +// list comprehension expression +void ExprVisitorBase::visit(ListComprehensionExpression *expr) { fatal(expr); } +// reduce expression +void ExprVisitorBase::visit(ReduceExpression *expr) { fatal(expr); } +// subscript range expression +void ExprVisitorBase::visit(SubscriptRangeExpression *expr) { fatal(expr); } + +} // namespace storage +} // namespace nebula diff --git a/src/storage/ExprVisitorBase.h b/src/storage/ExprVisitorBase.h new file mode 100644 index 00000000000..d085a902e7a --- /dev/null +++ b/src/storage/ExprVisitorBase.h @@ -0,0 +1,72 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#pragma once +#include "common/expression/ExprVisitor.h" +namespace nebula { +namespace storage { + +class ExprVisitorBase : public ::nebula::ExprVisitor { + public: + void visit(ConstantExpression *expr) override; + void visit(UnaryExpression *expr) override; + void visit(TypeCastingExpression *expr) override; + void visit(LabelExpression *expr) override; + void visit(LabelAttributeExpression *expr) override; + // binary expression + void visit(ArithmeticExpression *expr) override; + void visit(RelationalExpression *expr) override; + void visit(SubscriptExpression *expr) override; + void visit(AttributeExpression *expr) override; + void visit(LogicalExpression *expr) override; + // function call + void visit(FunctionCallExpression *expr) override; + void visit(AggregateExpression *expr) override; + void visit(UUIDExpression *expr) override; + // variable expression + void visit(VariableExpression *expr) override; + void visit(VersionedVariableExpression *expr) override; + // container expression + void visit(ListExpression *expr) override; + void visit(SetExpression *expr) override; + void visit(MapExpression *expr) override; + // property Expression + void visit(TagPropertyExpression *expr) override; + void visit(EdgePropertyExpression *expr) override; + void visit(InputPropertyExpression *expr) override; + void visit(VariablePropertyExpression *expr) override; + void visit(DestPropertyExpression *expr) override; + void visit(SourcePropertyExpression *expr) override; + void visit(EdgeSrcIdExpression *expr) override; + void visit(EdgeTypeExpression *expr) override; + void visit(EdgeRankExpression *expr) override; + void visit(EdgeDstIdExpression *expr) override; + // vertex/edge expression + void visit(VertexExpression *expr) override; + void visit(EdgeExpression *expr) override; + // case expression + void visit(CaseExpression *expr) override; + // path build expression + void visit(PathBuildExpression *expr) override; + // column expression + void visit(ColumnExpression *expr) override; + // predicate expression + void visit(PredicateExpression *expr) override; + // list comprehension expression + void visit(ListComprehensionExpression *expr) override; + // reduce expression + void visit(ReduceExpression *expr) override; + // subscript range expression + void visit(SubscriptRangeExpression *expr) override; + + private: + using ::nebula::ExprVisitor::visit; + inline void fatal(Expression *expr) { + LOG(FATAL) << "Unexpect expression kind " << static_cast(expr->kind()) << " at storage"; + } +}; +} // namespace storage +} // namespace nebula diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index c4a855e146f..2998df083dc 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -6,6 +6,9 @@ #include "storage/exec/IndexDedupNode.h" namespace nebula { namespace storage { +IndexDedupNode::IndexDedupNode(const IndexDedupNode& node) + : IndexNode(node), dedupColumns_(node.dedupColumns_), dedupPos_(node.dedupPos_) {} + IndexDedupNode::IndexDedupNode(RuntimeContext* context, const std::vector& dedupColumn) : IndexNode(context, "IndexDedupNode"), dedupColumns_(dedupColumn) {} ::nebula::cpp2::ErrorCode IndexDedupNode::init(InitContext& ctx) { @@ -28,7 +31,7 @@ ::nebula::cpp2::ErrorCode IndexDedupNode::doExecute(PartitionID partId) { dedupSet_.clear(); return IndexNode::doExecute(partId); } -IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) override { +IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; do { @@ -46,5 +49,9 @@ IndexDedupNode::RowWrapper::RowWrapper(const Row& row, const std::vector values_.emplace_back(row[p]); } } +std::unique_ptr IndexDedupNode::copy() { + return std::make_unique(*this); +} + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index 5a37a64487b..83b539c157e 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -11,8 +11,10 @@ namespace nebula { namespace storage { class IndexDedupNode : public IndexNode { public: + IndexDedupNode(const IndexDedupNode& node); IndexDedupNode(RuntimeContext* context, const std::vector& dedupColumn); ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; + std::unique_ptr copy() override; private: inline bool dedup(const Row& row); diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index a7c6d2829e0..5f6af7c06bf 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -7,16 +7,44 @@ #include "storage/exec/IndexEdgeScanNode.h" namespace nebula { namespace storage { +IndexEdgeScanNode::IndexEdgeScanNode(const IndexEdgeScanNode& node) + : IndexScanNode(node), edge_(node.edge_) { + getIndex = std::function([this]() { + auto env = this->context_->env(); + auto indexMgr = env->indexMan_; + auto index = indexMgr->getTagIndex(this->spaceId_, this->indexId_).value(); + return index; + }); + getEdge = std::function([this]() { + auto env = this->context_->env(); + auto schemaMgr = env->schemaMan_; + auto schema = + schemaMgr->getEdgeSchema(this->spaceId_, this->index_->get_schema_id().get_edge_type()); + return schema; + }); +} +IndexEdgeScanNode::IndexEdgeScanNode(RuntimeContext* context, + IndexID indexId, + const std::vector& columnHint) + : IndexScanNode(context, "IndexEdgeScanNode", indexId, columnHint) { + getIndex = std::function([this]() { + auto env = this->context_->env(); + auto indexMgr = env->indexMan_; + auto index = indexMgr->getTagIndex(this->spaceId_, this->indexId_).value(); + return index; + }); + getEdge = std::function([this]() { + auto env = this->context_->env(); + auto schemaMgr = env->schemaMan_; + auto schema = + schemaMgr->getEdgeSchema(this->spaceId_, this->index_->get_schema_id().get_edge_type()); + return schema; + }); +} ::nebula::cpp2::ErrorCode IndexEdgeScanNode::init(InitContext& ctx) { - auto env = context_->env(); - auto spaceId = context_->spaceId(); - auto indexMgr = env->indexMan_; - auto schemaMgr = env->schemaMan_; - auto index = indexMgr->getTagIndex(spaceId, indexId_).value(); - auto edgeSchema = schemaMgr->getEdgeSchema(spaceId, index->get_schema_id().get_tag_id()); - this->index_ = index; - this->edge_ = edgeSchema; - return ::nebula::cpp2::ErrorCode::SUCCEEDED; + index_ = getIndex().value(); + edge_ = getEdge(); + return IndexScanNode::init(ctx); } Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { std::vector values(requiredColumns_.size()); @@ -89,5 +117,10 @@ Map IndexEdgeScanNode::decodeFromBase(const std::string& key return values; } const meta::SchemaProviderIf* IndexEdgeScanNode::getSchema() { return edge_.get(); } + +std::unique_ptr IndexEdgeScanNode::copy() { + return std::make_unique(*this); +} + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index dafad9737f9..f1bf2ed2f76 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -4,23 +4,22 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once +#include "common/base/Base.h" #include "common/utils/NebulaKeyUtils.h" -#include "folly/Likely.h" #include "storage/exec/IndexScanNode2.h" #include "storage/exec/QueryUtils.h" +#include "storage/exec/StorageIterator.h" namespace nebula { namespace storage { class IndexEdgeScanNode : public IndexScanNode { public: - static ErrorOr make(RuntimeContext* context, - IndexID indexId, - const std::vector& columnHint); + IndexEdgeScanNode(const IndexEdgeScanNode& node); IndexEdgeScanNode(RuntimeContext* context, IndexID indexId, - const std::vector& columnHint) - : IndexScanNode(context, "IndexEdgeScanNode", indexId, columnHint) {} + const std::vector& columnHint); ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; + std::unique_ptr copy() override; private: Row decodeFromIndex(folly::StringPiece key) override; @@ -30,6 +29,10 @@ class IndexEdgeScanNode : public IndexScanNode { const meta::SchemaProviderIf* getSchema() override; std::shared_ptr edge_; + + // Convenient for testing + std::function>()> getIndex; + std::function()> getEdge; }; } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexLimitNode.cpp b/src/storage/exec/IndexLimitNode.cpp index 26129850622..ba8d2abaf16 100644 --- a/src/storage/exec/IndexLimitNode.cpp +++ b/src/storage/exec/IndexLimitNode.cpp @@ -6,6 +6,9 @@ #include "storage/exec/IndexLimitNode.h" namespace nebula { namespace storage { +IndexLimitNode::IndexLimitNode(const IndexLimitNode& node) + : IndexNode(node), offset_(node.offset_), limit_(node.limit_) {} + IndexLimitNode::IndexLimitNode(RuntimeContext* context, uint64_t offset, uint64_t limit) : IndexNode(context, "IndexLimitNode"), offset_(offset), limit_(limit) {} IndexLimitNode::IndexLimitNode(RuntimeContext* context, uint64_t limit) @@ -31,5 +34,9 @@ IndexNode::ErrorOr IndexLimitNode::doNext(bool& hasNext) { return ErrorOr(Row()); } } +std::unique_ptr IndexLimitNode::copy() { + return std::make_unique(*this); +} + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexLimitNode.h b/src/storage/exec/IndexLimitNode.h index ae24483961b..8525207d5d9 100644 --- a/src/storage/exec/IndexLimitNode.h +++ b/src/storage/exec/IndexLimitNode.h @@ -10,8 +10,10 @@ namespace nebula { namespace storage { class IndexLimitNode : public IndexNode { public: + IndexLimitNode(const IndexLimitNode& node); IndexLimitNode(RuntimeContext* context, uint64_t offset, uint64_t limit); IndexLimitNode(RuntimeContext* context, uint64_t limit); + std::unique_ptr copy() override; private: nebula::cpp2::ErrorCode doExecute(PartitionID partId) override; diff --git a/src/storage/exec/IndexNode.cpp b/src/storage/exec/IndexNode.cpp index aa506541ed9..533431ac1c9 100644 --- a/src/storage/exec/IndexNode.cpp +++ b/src/storage/exec/IndexNode.cpp @@ -9,6 +9,9 @@ #include "folly/Likely.h" namespace nebula { namespace storage { +IndexNode::IndexNode(const IndexNode& node) + : context_(node.context_), spaceId_(node.spaceId_), name_(node.name_) {} + nebula::cpp2::ErrorCode IndexNode::doExecute(PartitionID partId) { for (auto& child : children_) { auto ret = child->execute(partId); diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index 47eea5bd81a..e4e51ee5444 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -27,7 +27,9 @@ class IndexNode { public: template using ErrorOr = ::nebula::ErrorOr; + IndexNode(const IndexNode& node); explicit IndexNode(RuntimeContext* context, const std::string& name); + virtual ~IndexNode() = default; virtual ::nebula::cpp2::ErrorCode init(InitContext& initCtx) { DCHECK_EQ(children_.size(), 1); return children_[0]->init(initCtx); @@ -48,7 +50,6 @@ class IndexNode { virtual nebula::cpp2::ErrorCode doExecute(PartitionID partId); void beforeExecute(); void afterExecute(); - IndexNode(const IndexNode& node); RuntimeContext* context_; GraphSpaceID spaceId_; @@ -62,7 +63,7 @@ inline IndexNode::ErrorOr IndexNode::next(bool& hasNext) { beforeNext(); auto ret = doNext(hasNext); afterNext(); - return std::move(ret); + return ret; } inline void IndexNode::beforeNext() { duration_.resume(); } inline void IndexNode::afterNext() { duration_.pause(); } diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp index 506f86a5ca9..df2d480154b 100644 --- a/src/storage/exec/IndexProjectionNode.cpp +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -6,6 +6,8 @@ #include "storage/exec/IndexProjectionNode.h" namespace nebula { namespace storage { +IndexProjectionNode::IndexProjectionNode(const IndexProjectionNode& node) + : IndexNode(node), requiredColumns_(node.requiredColumns_), colPos_(node.colPos_) {} IndexProjectionNode::IndexProjectionNode(RuntimeContext* context, const std::vector& requiredColumns) : IndexNode(context, "IndexProjectionNode"), requiredColumns_(requiredColumns) {} @@ -47,6 +49,9 @@ Row IndexProjectionNode::project(Row&& row) { } return ret; } +std::unique_ptr IndexProjectionNode::copy() { + return std::make_unique(*this); +} } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index c7b751388bb..b252cadfc86 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -11,8 +11,10 @@ namespace nebula { namespace storage { class IndexProjectionNode : public IndexNode { public: + IndexProjectionNode(const IndexProjectionNode& node); IndexProjectionNode(RuntimeContext* context, const std::vector& requiredColumns); nebula::cpp2::ErrorCode init(InitContext& ctx) override; + std::unique_ptr copy() override; private: ErrorOr doNext(bool& hasNext) override; diff --git a/src/storage/exec/IndexScanNode.h b/src/storage/exec/IndexScanNode.h index 5ca22502f91..ddd19a49298 100644 --- a/src/storage/exec/IndexScanNode.h +++ b/src/storage/exec/IndexScanNode.h @@ -51,10 +51,9 @@ class IndexScanNode : public RelNode { } scanPair_ = scanRet.value(); std::unique_ptr iter; - ret = isRangeScan_ ? context_->env()->kvstore_->range( + ret = isRangeScan_ ? kvstore_->range( context_->spaceId(), partId, scanPair_.first, scanPair_.second, &iter) - : context_->env()->kvstore_->prefix( - context_->spaceId(), partId, scanPair_.first, &iter); + : kvstore_->prefix(context_->spaceId(), partId, scanPair_.first, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED && iter && iter->valid()) { context_->isEdge() ? iter_.reset(new EdgeIndexIterator(std::move(iter), context_->vIdLen())) diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index dfa84d0ee5f..5d353aae1b6 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -185,6 +185,18 @@ void PrefixPath::buildKey() { } // End of PrefixPath // Define of IndexScan + +IndexScanNode::IndexScanNode(const IndexScanNode& node) + : IndexNode(node), + partId_(node.partId_), + indexId_(node.indexId_), + index_(node.index_), + indexNullable_(node.indexNullable_), + columnHints_(node.columnHints_), + kvstore_(node.kvstore_), + requiredColumns_(node.requiredColumns_), + ttlProps_(node.ttlProps_) {} + nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { auto ret = resetIter(partId); return ret; diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index ce3e3c48695..f633423464f 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -22,11 +22,12 @@ class RangePath; class PrefixPath; class IndexScanNode : public IndexNode { public: + IndexScanNode(const IndexScanNode& node); IndexScanNode(RuntimeContext* context, const std::string& name, IndexID indexId, - const std::vector& columnHists) - : IndexNode(context, name), indexId_(indexId), columnHists_(columnHists) {} + const std::vector& columnHints) + : IndexNode(context, name), indexId_(indexId), columnHints_(columnHints) {} ::nebula::cpp2::ErrorCode init(InitContext& ctx) override { for (auto& col : ctx.requiredColumns) { requiredColumns_.push_back(col); @@ -35,6 +36,7 @@ class IndexScanNode : public IndexNode { for (size_t i = 0; i < ctx.returnColumns.size(); i++) { ctx.retColMap[ctx.returnColumns[i]] = i; } + return ::nebula::cpp2::ErrorCode::SUCCEEDED; } protected: @@ -49,14 +51,13 @@ class IndexScanNode : public IndexNode { virtual Map decodeFromBase(const std::string& key, const std::string& value) = 0; virtual const meta::SchemaProviderIf* getSchema() = 0; - std::unique_ptr>> ttlProps_; bool checkTTL(); nebula::cpp2::ErrorCode resetIter(PartitionID partId); PartitionID partId_; const IndexID indexId_; std::shared_ptr index_; bool indexNullable_ = false; - const std::vector& columnHists_; + const std::vector& columnHints_; std::unique_ptr path_; std::unique_ptr iter_; nebula::kvstore::KVStore* kvstore_; @@ -83,7 +84,7 @@ class Path { std::shared_ptr index, std::shared_ptr schema, const std::vector& hints); - virtual Qualified qualified(const folly::StringPiece& key); + Qualified qualified(const folly::StringPiece& key); virtual bool isRange() { return false; } virtual Qualified qualified(const Map& rowData) = 0; diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index 3b82fe5f3b4..fd34184681c 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -6,14 +6,13 @@ #include "storage/exec/IndexSelectionNode.h" namespace nebula { namespace storage { -IndexSelectionNode::IndexSelectionNode(RuntimeContext* context, - std::unique_ptr expr, - const std::vector& inputCols) - : IndexNode(context, "IndexSelectionNode"), expr_(std::move(expr)) { - for (size_t i = 0; i < inputCols.size(); i++) { - colPos_[inputCols[i]] = i; - } +IndexSelectionNode::IndexSelectionNode(const IndexSelectionNode& node) + : IndexNode(node), expr_(node.expr_), colPos_(node.colPos_) { + ctx_ = std::make_unique(colPos_); } + +IndexSelectionNode::IndexSelectionNode(RuntimeContext* context, Expression* expr) + : IndexNode(context, "IndexSelectionNode"), expr_(expr) {} nebula::cpp2::ErrorCode IndexSelectionNode::init(InitContext& ctx) { DCHECK_EQ(children_.size(), 1); SelectionExprVisitor vis; @@ -28,6 +27,7 @@ nebula::cpp2::ErrorCode IndexSelectionNode::init(InitContext& ctx) { for (auto& col : vis.getRequiredColumns()) { colPos_[col] = ctx.retColMap.at(col); } + ctx_ = std::make_unique(colPos_); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } IndexNode::ErrorOr IndexSelectionNode::doNext(bool& hasNext) { @@ -44,6 +44,9 @@ IndexNode::ErrorOr IndexSelectionNode::doNext(bool& hasNext) { } while (true); return ErrorOr(Row()); } +std::unique_ptr IndexSelectionNode::copy() { + return std::make_unique(*this); +} Value IndexSelectionNode::ExprContext::getEdgeProp(const std::string& edgeType, const std::string& prop) const { DCHECK(row_ != nullptr); @@ -60,6 +63,7 @@ Value IndexSelectionNode::ExprContext::getTagProp(const std::string& tag, DCHECK(iter->second < row_->size()); return row_[iter->second]; } + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 031ef63a7aa..59e58c36cca 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -6,18 +6,18 @@ #pragma once #include "common/context/ExpressionContext.h" -#include "common/expression/ExprVisitor.h" #include "common/expression/Expression.h" #include "folly/container/F14Map.h" +#include "storage/ExprVisitorBase.h" #include "storage/exec/IndexNode.h" namespace nebula { namespace storage { class IndexSelectionNode : public IndexNode { public: - IndexSelectionNode(RuntimeContext *context, - std::unique_ptr expr, - const std::vector &inputCols); + IndexSelectionNode(const IndexSelectionNode &node); + IndexSelectionNode(RuntimeContext *context, Expression *expr); nebula::cpp2::ErrorCode init(InitContext &ctx) override; + std::unique_ptr copy() override; private: ErrorOr doNext(bool &hasNext) override; @@ -27,7 +27,7 @@ class IndexSelectionNode : public IndexNode { return result.type() == Value::Type::BOOL ? result.getBool() : false; } Map colPos_; - std::unique_ptr expr_; + Expression *expr_; class ExprContext : public ExpressionContext { public: explicit ExprContext(const Map &colPos) : colPos_(colPos) {} @@ -36,103 +36,65 @@ class IndexSelectionNode : public IndexNode { Value getTagProp(const std::string &tag, const std::string &prop) const override; // override const Value &getVar(const std::string &var) const override { - LOG(FATAL) << "Unexpect"; - return Value(); + UNUSED(var); + return fatal(__FILE__, __LINE__); } const Value &getVersionedVar(const std::string &var, int64_t version) const override { - LOG(FATAL) << "Unexpect"; - return Value(); + UNUSED(var), UNUSED(version); + return fatal(__FILE__, __LINE__); } const Value &getVarProp(const std::string &var, const std::string &prop) const override { - LOG(FATAL) << "Unexpect"; - return Value(); + UNUSED(var), UNUSED(prop); + return fatal(__FILE__, __LINE__); } - Value getSrcProp(const std::string &tag, const std::string &prop) const override { - LOG(FATAL) << "Unexpect"; - return Value(); + UNUSED(tag), UNUSED(prop); + return fatal(__FILE__, __LINE__); } const Value &getDstProp(const std::string &tag, const std::string &prop) const override { - LOG(FATAL) << "Unexpect"; - return Value(); + UNUSED(tag), UNUSED(prop); + return fatal(__FILE__, __LINE__); } const Value &getInputProp(const std::string &prop) const override { - LOG(FATAL) << "Unexpect"; - return Value(); - } - Value getVertex() const override { - LOG(FATAL) << "Unexpect"; - return Value(); - } - Value getEdge() const override { - LOG(FATAL) << "Unexpect"; - return Value(); + UNUSED(prop); + return fatal(__FILE__, __LINE__); } + Value getVertex() const override { return fatal(__FILE__, __LINE__); } + Value getEdge() const override { return fatal(__FILE__, __LINE__); } Value getColumn(int32_t index) const override { - LOG(FATAL) << "Unexpect"; - return Value(); + UNUSED(index); + return fatal(__FILE__, __LINE__); + } + void setVar(const std::string &var, Value val) override { + UNUSED(var), UNUSED(val); + fatal(__FILE__, __LINE__); } - void setVar(const std::string &var, Value val) override { LOG(FATAL) << "Unexpect"; } private: const Map &colPos_; const Row *row_; + inline const Value &fatal(const std::string &file, int line) const { + LOG(FATAL) << "Unexpect at " << file << ":" << line; + static Value placeholder; + return placeholder; + } }; std::unique_ptr ctx_; }; -class SelectionExprVisitor : public ::nebula::ExprVisitor { +class SelectionExprVisitor : public ExprVisitorBase { public: - void visit(ConstantExpression *expr) override { return; } - void visit(UnaryExpression *expr) override { expr->operand()->accept(this); } - void visit(TypeCastingExpression *expr) override { expr->operand()->accept(this); } - void visit(LabelExpression *expr) override { return; } - void visit(LabelAttributeExpression *expr) override { return; } - void visit(ArithmeticExpression *expr) override { - expr->left()->accept(this); - expr->right()->accept(this); - } - void visit(RelationalExpression *expr) override { - expr->left()->accept(this); - expr->right()->accept(this); - } - void visit(SubscriptExpression *expr) override { - expr->left()->accept(this); - expr->right()->accept(this); - } - void visit(AttributeExpression *expr) override; - void visit(LogicalExpression *expr) override; - void visit(FunctionCallExpression *expr) override; - void visit(AggregateExpression *expr) override; - void visit(UUIDExpression *expr) override; - void visit(VariableExpression *expr) override; - void visit(VersionedVariableExpression *expr) override; - void visit(ListExpression *expr) override; - void visit(SetExpression *expr) override; - void visit(MapExpression *expr) override; - void visit(TagPropertyExpression *expr) override; - void visit(EdgePropertyExpression *expr) override; - void visit(InputPropertyExpression *expr) override; - void visit(VariablePropertyExpression *expr) override; - void visit(DestPropertyExpression *expr) override; - void visit(SourcePropertyExpression *expr) override; - void visit(EdgeSrcIdExpression *expr) override; - void visit(EdgeTypeExpression *expr) override; - void visit(EdgeRankExpression *expr) override; - void visit(EdgeDstIdExpression *expr) override; - void visit(VertexExpression *expr) override; - void visit(EdgeExpression *expr) override; - void visit(CaseExpression *expr) override; - void visit(PathBuildExpression *expr) override; - void visit(ColumnExpression *expr) override; - void visit(PredicateExpression *expr) override; - void visit(ListComprehensionExpression *expr) override; - void visit(ReduceExpression *expr) override; - void visit(SubscriptRangeExpression *expr) override; + void visit(EdgeSrcIdExpression *expr) override { requiredColumns_.insert(expr->prop()); } + void visit(EdgeTypeExpression *expr) override { requiredColumns_.insert(expr->prop()); } + void visit(EdgeRankExpression *expr) override { requiredColumns_.insert(expr->prop()); } + void visit(EdgeDstIdExpression *expr) override { requiredColumns_.insert(expr->prop()); } + void visit(TagPropertyExpression *expr) { requiredColumns_.insert(expr->prop()); } + void visit(EdgePropertyExpression *expr) override { requiredColumns_.insert(expr->prop()); } const Set &getRequiredColumns(); ::nebula::cpp2::ErrorCode getCode(); private: + using ExprVisitorBase::visit; Set requiredColumns_; ::nebula::cpp2::ErrorCode code_; }; diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index 4ab03062d33..dff1c9ec716 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -5,24 +5,52 @@ */ #include "storage/exec/IndexVertexScanNode.h" +#include "codec/RowReaderWrapper.h" +#include "common/utils/NebulaKeyUtils.h" +#include "storage/exec/QueryUtils.h" namespace nebula { namespace storage { +IndexVertexScanNode::IndexVertexScanNode(const IndexVertexScanNode& node) + : IndexScanNode(node), tag_(node.tag_) { + getIndex = std::function([this]() { + auto env = this->context_->env(); + auto indexMgr = env->indexMan_; + auto index = indexMgr->getTagIndex(this->spaceId_, this->indexId_).value(); + return index; + }); + getTag = std::function([this]() { + auto env = this->context_->env(); + auto schemaMgr = env->schemaMan_; + auto schema = + schemaMgr->getTagSchema(this->spaceId_, this->index_->get_schema_id().get_tag_id()); + return schema; + }); +} + IndexVertexScanNode::IndexVertexScanNode(RuntimeContext* context, IndexID indexId, const std::vector& clolumnHint) - : IndexScanNode(context, "IndexVertexScanNode", indexId, clolumnHint) {} + : IndexScanNode(context, "IndexVertexScanNode", indexId, clolumnHint) { + getIndex = std::function([this]() { + auto env = this->context_->env(); + auto indexMgr = env->indexMan_; + auto index = indexMgr->getTagIndex(this->spaceId_, this->indexId_).value(); + return index; + }); + getTag = std::function([this]() { + auto env = this->context_->env(); + auto schemaMgr = env->schemaMan_; + auto schema = + schemaMgr->getTagSchema(this->spaceId_, this->index_->get_schema_id().get_tag_id()); + return schema; + }); +} ::nebula::cpp2::ErrorCode IndexVertexScanNode::init(InitContext& ctx) { - auto env = context_->env(); - auto spaceId = context_->spaceId(); - auto indexMgr = env->indexMan_; - auto schemaMgr = env->schemaMan_; - auto index = indexMgr->getTagIndex(spaceId, indexId_).value(); - auto tagSchema = schemaMgr->getTagSchema(spaceId, index->get_schema_id().get_tag_id()); - this->index_ = index; - this->tag_ = tagSchema; - return ::nebula::cpp2::ErrorCode::SUCCEEDED; + this->index_ = getIndex().value(); + this->tag_ = getTag(); + return IndexScanNode::init(ctx); } nebula::cpp2::ErrorCode IndexVertexScanNode::getBaseData(folly::StringPiece key, std::pair& kv) { @@ -83,5 +111,9 @@ Map IndexVertexScanNode::decodeFromBase(const std::string& k } return values; } +std::unique_ptr IndexVertexScanNode::copy() { + return std::make_unique(*this); +} + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index bd25b651c8c..1e6b64b9dbd 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -5,18 +5,25 @@ */ #pragma once +#include + +#include +#include + #include "common/base/Base.h" #include "storage/exec/IndexScanNode2.h" -#include "storage/exec/RelNode.h" #include "storage/exec/StorageIterator.h" + namespace nebula { namespace storage { class IndexVertexScanNode final : public IndexScanNode { public: - ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; + IndexVertexScanNode(const IndexVertexScanNode& node); IndexVertexScanNode(RuntimeContext* context, IndexID indexId, const std::vector& clolumnHint); + ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; + std::unique_ptr copy() override; private: nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, @@ -26,6 +33,14 @@ class IndexVertexScanNode final : public IndexScanNode { const meta::SchemaProviderIf* getSchema() override { return tag_.get(); } std::shared_ptr tag_; + + // Convenient for testing + std::function>()> getIndex; + std::function()> getTag; + + FRIEND_TEST(IndexScanTest, VertexBase); + FRIEND_TEST(IndexScanTest, Prefix1); + FRIEND_TEST(IndexScanTest, Prefix2); }; } // namespace storage } // namespace nebula diff --git a/src/storage/index/LookupProcessor2.cpp b/src/storage/index/LookupProcessor2.cpp index c0b16b2da71..c709f5d77db 100644 --- a/src/storage/index/LookupProcessor2.cpp +++ b/src/storage/index/LookupProcessor2.cpp @@ -5,6 +5,7 @@ */ #include "storage/index/LookupProcessor2.h" +#include "folly/Likely.h" #include "storage/exec/IndexDedupNode.h" #include "storage/exec/IndexEdgeScanNode.h" #include "storage/exec/IndexLimitNode.h" @@ -12,7 +13,6 @@ #include "storage/exec/IndexProjectionNode.h" #include "storage/exec/IndexSelectionNode.h" #include "storage/exec/IndexVertexScanNode.h" - namespace nebula { namespace storage { @@ -34,21 +34,38 @@ void LookupProcessor::doProcess(const cpp2::LookupIndexRequest& req) { runInMultipleThread(req.get_parts(), std::move(plan)); } } -::nebula::cpp2::ErrorCode LookupProcessor::prepare(const cpp2::LookupIndexRequest& req) {} +::nebula::cpp2::ErrorCode LookupProcessor::prepare(const cpp2::LookupIndexRequest& req) { + auto retCode = this->getSpaceVidLen(req.get_space_id()); + if (UNLIKELY(retCode != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + return retCode; + } + planContext_ = std::make_unique( + this->env_, req.get_space_id(), this->spaceVidLen_, this->isIntId_, req.common_ref()); + planContext_->isEdge_ = + req.get_indices().get_schema_id().getType() == nebula::cpp2::SchemaID::Type::edge_type; + context_ = std::make_unique(this->planContext_.get()); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; +} std::unique_ptr LookupProcessor::buildPlan(const cpp2::LookupIndexRequest& req) { - bool is_edge = req.get_indices().get_schema_id().edge_type_ref().has_value(); std::vector> nodes; for (auto& ctx : req.get_indices().get_contexts()) { auto node = buildOneContext(ctx); } for (size_t i = 0; i < nodes.size(); i++) { - auto projection = std::make_unique(); + auto projection = + std::make_unique(context_.get(), *req.get_return_columns()); projection->addChild(std::move(nodes[i])); nodes[i] = std::move(projection); } if (nodes.size() > 1) { - auto dedup = std::make_unique(); + std::vector dedupColumn; + if (context_->isEdge()) { + dedupColumn = std::vector{kSrc, kRank, kDst}; + } else { + dedupColumn = std::vector{kVid}; + } + auto dedup = std::make_unique(context_.get(), dedupColumn); for (auto& node : nodes) { node->addChild(std::move(node)); } @@ -56,8 +73,8 @@ std::unique_ptr LookupProcessor::buildPlan(const cpp2::LookupIndexReq nodes[0] = std::move(dedup); } if (req.limit_ref().has_value()) { - auto limit = req.get_limit(); - auto node = std::make_unique(); + auto limit = *req.get_limit(); + auto node = std::make_unique(context_.get(), limit); node->addChild(std::move(nodes[0])); nodes[0] = std::move(node); } @@ -72,14 +89,16 @@ std::unique_ptr LookupProcessor::buildPlan(const cpp2::LookupIndexReq std::unique_ptr LookupProcessor::buildOneContext(const cpp2::IndexQueryContext& ctx) { std::unique_ptr node; if (context_->isEdge()) { - node = std::make_unique(); + node = std::make_unique( + context_.get(), ctx.get_index_id(), ctx.get_column_hints()); } else { - node = std::make_unique(); + node = std::make_unique( + context_.get(), ctx.get_index_id(), ctx.get_column_hints()); } - if (ctx.filter_ref().has_value()) { + if (ctx.filter_ref().is_set()) { auto filterNode = std::make_unique(context_.get(), nullptr); } - return std::move(node); + return node; } void LookupProcessor::runInSingleThread(const std::vector& parts, std::unique_ptr plan) { @@ -90,7 +109,6 @@ void LookupProcessor::runInSingleThread(const std::vector& parts, bool hasNext = true; ::nebula::cpp2::ErrorCode code; decltype(datasetList)::value_type dataset; - nebula::DataSet dataset; do { auto result = plan->next(hasNext); if (hasNext && ::nebula::ok(result)) { @@ -102,12 +120,23 @@ void LookupProcessor::runInSingleThread(const std::vector& parts, datasetList.emplace_back(std::move(dataset)); codeList.emplace_back(code); } - // 处理 leader change + for (size_t i = 0; i < datasetList.size(); i++) { + if (codeList[i] == ::nebula::cpp2::ErrorCode::SUCCEEDED) { + while (!datasetList[i].empty()) { + resultDataSet_.emplace_back(std::move(datasetList[i].front())); + datasetList[i].pop_front(); + } + } else { + handleErrorCode(codeList[i], context_->spaceId(), parts[i]); + } + } + onProcessFinished(); + onFinished(); } void LookupProcessor::runInMultipleThread(const std::vector& parts, std::unique_ptr plan) { std::vector> planCopy = reproducePlan(plan.get(), parts.size()); - std::vector>> datasetList(parts.size()); + std::vector> datasetList(parts.size()); std::vector<::nebula::cpp2::ErrorCode> codeList(parts.size()); std::vector> funcs; std::vector> futures; @@ -134,15 +163,21 @@ void LookupProcessor::runInMultipleThread(const std::vector& parts, for (size_t i = 0; i < parts.size(); i++) { futures.emplace_back(folly::via(executor_, std::move(funcs[i]))); } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) { - CHECK(!t.hasException()); - const auto& values = t.value(); - for (auto& value : values) { - // for (size_t i = 0; i < t.size(); i++) { - // // 检查切主 合并数据 - // } + for (size_t i = 0; i < parts.size(); i++) { + futures[i].result(); + } + for (size_t i = 0; i < datasetList.size(); i++) { + if (codeList[i] == ::nebula::cpp2::ErrorCode::SUCCEEDED) { + while (!datasetList[i].empty()) { + resultDataSet_.emplace_back(std::move(datasetList[i].front())); + datasetList[i].pop_front(); + } + } else { + handleErrorCode(codeList[i], context_->spaceId(), parts[i]); } - }); + } + onProcessFinished(); + onFinished(); } std::vector> LookupProcessor::reproducePlan(IndexNode* root, size_t count) { @@ -156,7 +191,7 @@ std::vector> LookupProcessor::reproducePlan(IndexNode ret[i]->addChild(std::move(childPerPlan[i])); } } - return std::move(ret); + return ret; } } // namespace storage diff --git a/src/storage/index/LookupProcessor2.h b/src/storage/index/LookupProcessor2.h index 557ea6398a2..3c1df8226df 100644 --- a/src/storage/index/LookupProcessor2.h +++ b/src/storage/index/LookupProcessor2.h @@ -32,7 +32,6 @@ class LookupProcessor : public BaseProcessor { std::unique_ptr buildOneContext(const cpp2::IndexQueryContext& ctx); std::vector> reproducePlan(IndexNode* root, size_t count); folly::Executor* executor_{nullptr}; - folly::Executor* executor_{nullptr}; std::unique_ptr planContext_; std::unique_ptr context_; nebula::DataSet resultDataSet_; diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp new file mode 100644 index 00000000000..05c839c3759 --- /dev/null +++ b/src/storage/test/IndexTest.cpp @@ -0,0 +1,100 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include + +#include "kvstore/KVEngine.h" +#include "kvstore/KVIterator.h" +#include "storage/exec/IndexDedupNode.h" +#include "storage/exec/IndexEdgeScanNode.h" +#include "storage/exec/IndexNode.h" +#include "storage/exec/IndexProjectionNode.h" +#include "storage/exec/IndexSelectionNode.h" +#include "storage/exec/IndexVertexScanNode.h" +#include "storage/test/IndexTestUtil.h" +namespace nebula { +namespace storage { +/** + * _KSrc, + * "",123,456,"abc", + * + * + * + * + * + */ + +// class MockKVStore : public ::nebula::kvstore::KVEngine {}; +// class IndexMockNode : public IndexNode { +// public: +// private: +// }; + +// /** +// * +// */ +// class IndexScanTest : public ::testing::Test { +// protected: +// private: +// static std::unique_ptr kvstore_; +// }; +// TEST_F(IndexScanTest, VertexIndexOnlyScan) { +// std::vector rows = R"( +// int | int | int +// 1 | 2 | 3 +// 4 | 5 | 6 +// )"_row; +// auto a = R"( +// Tag:a, +// int +// )"_row; +// auto b = R"()"_edge; +// auto c = R"()"_index() auto node = std::make_unique(nullptr, 0, {}); +// node->kvstore_ = kvstore_; +// node->init(); +// node->execute(0); +// node->next(); +// }; +// TEST_F(IndexScanTest, VertexIndexScan) +// TEST_F(IndexScanTest, EdgeBase){ + +// }; +// TEST_F(IndexScanTest, Prefix1){ + +// }; +// TEST_F(IndexScanTest, Prefix2){ + +// }; +// TEST_F(IndexScanTest, Range1){ + +// }; +// TEST_F(IndexScanTest, Range2){ + +// }; +// TEST_F(IndexScanTest, Range3){ + +// }; +// TEST_F(IndexScanTest, Range4){ + +// }; +// class IndexTest : public ::testing::Test { +// protected: +// private: +// }; +// TEST_F(IndexTest, VertexScan1){ + +// } +// TEST_F(IndexTest, VertexScan2){ + +// } +// TEST_F(IndexTest, VertexScan3){ + +// }; +// TEST_F(IndexTest, VertexScan4){ + +// }; +} // namespace storage +} // namespace nebula diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h new file mode 100644 index 00000000000..fad3f852aea --- /dev/null +++ b/src/storage/test/IndexTestUtil.h @@ -0,0 +1,485 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#pragma once + +#include +#include + +#include "common/datatypes/DataSet.h" +#include "common/meta/NebulaSchemaProvider.h" +#include "folly/String.h" +#include "kvstore/KVIterator.h" +#include "kvstore/KVStore.h" +#include "storage/exec/IndexNode.h" +namespace nebula { +namespace storage { +using ::nebula::kvstore::KVIterator; +class MockKVIterator : public KVIterator { + using KVMap = std::map; + + public: + MockKVIterator(const KVMap& kv_, KVMap::iterator&& iter); + bool valid() const { return iter_ != kv_.end(); } + void next() { iter_++; } + void prev() { iter_--; } + folly::StringPiece key() const { return folly::StringPiece(iter_->first); } + folly::StringPiece val() const { return folly::StringPiece(iter_->second); } + void setValidFunc(const std::function validFunc) { + validFunc_ = validFunc; + } + + private: + const KVMap& kv_; + KVMap::iterator iter_; + std::function validFunc_; +}; +class MockKVStore : public ::nebula::kvstore::KVStore { + private: + GraphSpaceID spaceId_; + std::map kv_; + + public: + // Return bit-OR of StoreCapability values; + uint32_t capability() const override { + assert(false); + return 0; + }; + void stop() {} + ErrorOr partLeader(GraphSpaceID spaceId, PartitionID partID) { + UNUSED(spaceId), UNUSED(partID); + assert(false); + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + // Read a single key + nebula::cpp2::ErrorCode get(GraphSpaceID spaceId, + PartitionID partId, + const std::string& key, + std::string* value, + bool canReadFromFollower = false) override { + UNUSED(canReadFromFollower); + CHECK_EQ(spaceId, spaceId_); + auto iter = kv_.lower_bound(key); + if (iter != kv_.end() && iter->first == key) { + *value = iter->second; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } else { + return ::nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND; + } + } + + // Read multiple keys, if error occurs a cpp2::ErrorCode is returned, + // If key[i] does not exist, the i-th value in return value would be + // Status::KeyNotFound + std::pair> multiGet( + GraphSpaceID spaceId, + PartitionID partId, + const std::vector& keys, + std::vector* values, + bool canReadFromFollower = false) override { + UNUSED(canReadFromFollower); + std::vector status; + nebula::cpp2::ErrorCode ret = nebula::cpp2::ErrorCode::SUCCEEDED; + for (auto& key : keys) { + auto iter = kv_.lower_bound(key); + if (iter != kv_.end() && iter->first == key) { + values->push_back(iter->second); + status.push_back(Status::OK()); + } else { + values->push_back(""); + status.push_back(Status::KeyNotFound()); + ret = nebula::cpp2::ErrorCode::E_PARTIAL_RESULT; + } + } + return {ret, std::move(status)}; + } + + // Get all results in range [start, end) + nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, + PartitionID partId, + const std::string& start, + const std::string& end, + std::unique_ptr* iter, + bool canReadFromFollower = false) override { + CHECK_EQ(spaceId, spaceId_); + auto it = kv_.lower_bound(start); + auto mockIter = std::make_unique(kv_, std::move(it)); + mockIter->setValidFunc([start, end](const decltype(kv_)::iterator& iter) { + if (start <= iter->first && iter->first < end) { + return true; + } else { + return false; + } + }); + (*iter) = std::move(mockIter); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + + nebula::cpp2::ErrorCode prefix(GraphSpaceID spaceId, + PartitionID partId, + const std::string& prefix, + std::unique_ptr* iter, + bool canReadFromFollower = false) override { + UNUSED(canReadFromFollower); + CHECK_EQ(spaceId, spaceId_); + auto it = kv_.lower_bound(prefix); + auto mockIter = std::make_unique(kv_, std::move(it)); + mockIter->setValidFunc([prefix](const decltype(kv_)::iterator& iter) { + if (iter->first.size() < prefix.size()) { + return false; + } + for (size_t i = 0; i < prefix.size(); i++) { + if (prefix[i] != iter->first[i]) { + return false; + } + } + return true; + }); + (*iter) = std::move(mockIter); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + + // Get all results with prefix starting from start + nebula::cpp2::ErrorCode rangeWithPrefix(GraphSpaceID spaceId, + PartitionID partId, + const std::string& start, + const std::string& prefix, + std::unique_ptr* iter, + bool canReadFromFollower = false) override { + UNUSED(canReadFromFollower); + CHECK_EQ(spaceId, spaceId_); + auto it = kv_.lower_bound(start); + auto mockIter = std::make_unique(kv_, std::move(it)); + mockIter->setValidFunc([prefix](const decltype(kv_)::iterator& iter) { + if (iter->first.size() < prefix.size()) { + return false; + } + for (size_t i = 0; i < prefix.size(); i++) { + if (prefix[i] != iter->first[i]) { + return false; + } + } + return true; + }); + (*iter) = std::move(mockIter); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + + nebula::cpp2::ErrorCode sync(GraphSpaceID spaceId, PartitionID partId) { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + + void asyncMultiPut(GraphSpaceID spaceId, + PartitionID partId, + std::vector<::nebula::kvstore::KV>&& keyValues, + ::nebula::kvstore::KVCallback cb) override { + for (size_t i = 0; i < keyValues.size(); i++) { + kv_.emplace(std::move(keyValues[i])); + } + } + + // Asynchronous version of remove methods + void asyncRemove(GraphSpaceID spaceId, + PartitionID partId, + const std::string& key, + ::nebula::kvstore::KVCallback cb) override { + kv_.erase(key); + } + + void asyncMultiRemove(GraphSpaceID spaceId, + PartitionID partId, + std::vector&& keys, + ::nebula::kvstore::KVCallback cb) override { + for (size_t i = 0; i < keys.size(); i++) { + kv_.erase(keys[i]); + } + } + + void asyncRemoveRange(GraphSpaceID spaceId, + PartitionID partId, + const std::string& start, + const std::string& end, + ::nebula::kvstore::KVCallback cb) override { + for (auto iter = kv_.lower_bound(start); iter != kv_.end();) { + if (iter->first < end) { + iter = kv_.erase(iter); + } else { + iter++; + } + } + } + + void asyncAtomicOp(GraphSpaceID spaceId, + PartitionID partId, + raftex::AtomicOp op, + ::nebula::kvstore::KVCallback cb) override { + LOG(FATAL) << "Unexpect"; + } + void asyncAppendBatch(GraphSpaceID spaceId, + PartitionID partId, + std::string&& batch, + ::nebula::kvstore::KVCallback cb) override { + LOG(FATAL) << "Unexpect"; + } + nebula::cpp2::ErrorCode ingest(GraphSpaceID spaceId) override { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + + int32_t allLeader( + std::unordered_map>& leaderIds) override { + LOG(FATAL) << "Unexpect"; + return 0; + } + + ErrorOr> part( + GraphSpaceID spaceId, PartitionID partId) override { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + nebula::cpp2::ErrorCode compact(GraphSpaceID spaceId) override { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + nebula::cpp2::ErrorCode flush(GraphSpaceID spaceId) override { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + + ErrorOr> createCheckpoint( + GraphSpaceID spaceId, const std::string& name) override { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + }; + nebula::cpp2::ErrorCode dropCheckpoint(GraphSpaceID spaceId, const std::string& name) { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + + nebula::cpp2::ErrorCode setWriteBlocking(GraphSpaceID spaceId, bool sign) { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + + ErrorOr> backupTable( + GraphSpaceID spaceId, + const std::string& name, + const std::string& tablePrefix, + std::function filter) { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + // for meta BR + nebula::cpp2::ErrorCode restoreFromFiles(GraphSpaceID spaceId, + const std::vector& files) { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + nebula::cpp2::ErrorCode multiPutWithoutReplicator(GraphSpaceID spaceId, + std::vector<::nebula::kvstore::KV> keyValues) { + LOG(FATAL) << "Unexpect"; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + std::vector getDataRoot() const { + LOG(FATAL) << "Unexpect"; + return {}; + } +}; +class MockIndexNode : public IndexNode { + public: + ::nebula::cpp2::ErrorCode init(InitContext& initCtx) { return initFunc(initCtx); } + std::unique_ptr copy() { LOG(FATAL) << "Unexpect"; } + std::function(bool&)> nextFunc; + std::function<::nebula::cpp2::ErrorCode(PartitionID)> executeFunc; + std::function<::nebula::cpp2::ErrorCode(InitContext& initCtx)> initFunc; + + private: + ErrorOr doNext(bool& hasNext) override { return nextFunc(hasNext); } + ::nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { return executeFunc(partId); }; +}; + +class RowParser { + public: + explicit RowParser(const std::string& str) { + ss = std::stringstream(folly::stripLeftMargin(str)); + ss = std::stringstream(str); + parseHeader(); + parseRow(); + } + void parseHeader() { + std::string line; + std::getline(ss, line); + std::vector types; + folly::split("|", line, types); + for (size_t i = 0; i < types.size(); i++) { + types[i] = folly::stripLeftMargin(types[i]); + } + typeList_ = std::move(types); + } + void parseRow() { + std::string line; + while (std::getline(ss, line)) { + std::vector values; + folly::split("|", line, values); + for (size_t i = 0; i < values.size(); i++) { + values[i] = folly::stripLeftMargin(values[i]); + } + Row row; + for (size_t i = 0; i < values.size(); i++) { + if (values[i] == "") { + row.emplace_back(Value::null()); + } else { + row.emplace_back(transformMap[typeList_[i]](values[i])); + } + } + rowList_.emplace_back(std::move(row)); + } + } + const std::vector& getResult() { return rowList_; } + + private: + std::stringstream ss; + std::vector typeList_; + std::vector rowList_; + std::map> transformMap{ + {"int64", [](const std::string& str) { return Value(std::stoi(str)); }}, + {"string", [](const std::string& str) { return Value(str); }}, + {"double", [](const std::string& str) { return Value(std::stof(str)); }}}; +}; + +/** + * define a schema + * + * format: + * name | type | length | nullable + * example: + * std::string str=R"( + * a | int | | + * b | string | | true + * c | fix_string | 10 | + * )"_schema + */ +class SchemaParser { + public: + explicit SchemaParser(const std::string& str) { + ss = std::stringstream(folly::stripLeftMargin(str)); + parse(); + } + void parse() { + std::string line; + while (std::getline(ss, line)) { + std::vector values; + folly::split("|", line, values); + std::string name = folly::stripLeftMargin(values[0]); + auto type = typeMap[folly::stripLeftMargin(values[1])]; + int length = 0; + { + std::string lenStr = folly::stripLeftMargin(values[2]); + if (lenStr != "") { + length = std::stoi(lenStr); + } + } + bool nullable = false; + { + std::string nullable = folly::stripLeftMargin(values[3]); + if (nullable == "true") { + nullable = true; + } + } + schema->addField(name, type, length, nullable); + } + } + std::shared_ptr<::nebula::meta::NebulaSchemaProvider> getResult() { return schema; } + + private: + std::stringstream ss; + std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema; + std::map typeMap; +}; + +/** + * define index of a schema + * + * format: ${IndexID}: + * example + * std::string str=R"( + * Tag(name,id) + * (i1,1): a,b(10),c + * (i2,2): b(5),c + * )"_tagIndex(schema) + */ +class IndexParser { + public: + using IndexItem = ::nebula::meta::cpp2::IndexItem; + using TagItem = ::nebula::meta::cpp2::TagItem; + using EdgeItem = ::nebula::meta::cpp2::EdgeItem; + explicit IndexParser(const std::string& str) { + ss = std::stringstream(folly::stripLeftMargin(str)); + } + std::vector> operator()(const TagItem& tag) { + std::vector> ret; + std::string line; + while (std::getline(ss, line)) { + auto index = std::make_shared(); + ::nebula::cpp2::SchemaID schemaId; + schemaId.set_tag_id(tag.get_tag_id()); + index->set_schema_id(schemaId); + index->set_schema_name(tag.get_tag_name()); + parseIndex(index, tag.get_schema()); + ret.push_back(index); + } + return ret; + } + std::vector> operator()(const EdgeItem& edge) { + std::vector> ret; + std::string line; + while (std::getline(ss, line)) { + auto index = std::make_shared(); + ::nebula::cpp2::SchemaID schemaId; + schemaId.set_edge_type(edge.get_edge_type()); + index->set_schema_id(schemaId); + index->set_schema_name(edge.get_edge_name()); + parseIndex(index, edge.get_schema()); + ret.push_back(index); + } + return ret; + } + // void parseIndex(std::shared_ptr index,){ + + // } + + // std::vector> operator()() { + // if (isEdge_) { + // schemaID_.set_edge_type(schemaId); + // } else { + // schemaID_.set_tag_id(schemaId); + // } + // std::string line; + // while (std::getline(ss, line)) { + // auto index = std::make_shared<::nebula::meta::cpp2::IndexItem>(); + // index->set_schema_id(schemaID_); + // std::vector values; + // folly::split(":", line, values); + // } + // } + + private: + std::stringstream ss; + + std::shared_ptr<::nebula::meta::cpp2::IndexItem> index_; +}; + +// Definition of UDL +std::vector operator""_row(const char* str, std::size_t) { + auto ret = RowParser(std::string(str)).getResult(); + return ret; +} + +class SchemaParser {} + +} // namespace storage +} // namespace nebula From 9395f0a34e1df4e9f2a5b8b30e982a12d53b6e03 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Wed, 29 Sep 2021 15:55:25 +0800 Subject: [PATCH 09/38] stash --- src/storage/test/IndexTest.cpp | 69 +++++++++--------- src/storage/test/IndexTestUtil.h | 120 +++++++++++++++++++------------ 2 files changed, 109 insertions(+), 80 deletions(-) diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 05c839c3759..b358043c1ce 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -27,37 +27,38 @@ namespace storage { * */ -// class MockKVStore : public ::nebula::kvstore::KVEngine {}; -// class IndexMockNode : public IndexNode { -// public: -// private: -// }; +class MockKVStore : public ::nebula::kvstore::KVEngine {}; +class IndexMockNode : public IndexNode { + public: + private: +}; -// /** -// * -// */ -// class IndexScanTest : public ::testing::Test { -// protected: -// private: -// static std::unique_ptr kvstore_; -// }; -// TEST_F(IndexScanTest, VertexIndexOnlyScan) { -// std::vector rows = R"( -// int | int | int -// 1 | 2 | 3 -// 4 | 5 | 6 -// )"_row; -// auto a = R"( -// Tag:a, -// int -// )"_row; -// auto b = R"()"_edge; -// auto c = R"()"_index() auto node = std::make_unique(nullptr, 0, {}); -// node->kvstore_ = kvstore_; -// node->init(); -// node->execute(0); -// node->next(); -// }; +/** + * + */ +class IndexScanTest : public ::testing::Test { + protected: + private: + static std::unique_ptr kvstore_; +}; +TEST_F(IndexScanTest, VertexIndexOnlyScan) { + std::vector rows = R"( + int | int | int + 1 | 2 | 3 + 4 | 5 | 6 + )"_row; + auto schema = R"( + a | int || + b | int || + c | int || + )"_schema; + auto index = R"( + TAG(a,1) + (i1,2):a + (i2,3):b + )"_index(schema); + std::vector keyValues = encode(rows, schema, index); +} // TEST_F(IndexScanTest, VertexIndexScan) // TEST_F(IndexScanTest, EdgeBase){ @@ -84,12 +85,8 @@ namespace storage { // protected: // private: // }; -// TEST_F(IndexTest, VertexScan1){ - -// } -// TEST_F(IndexTest, VertexScan2){ - -// } +// TEST_F(IndexTest, VertexScan1) {} +// TEST_F(IndexTest, VertexScan2) {} // TEST_F(IndexTest, VertexScan3){ // }; diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index fad3f852aea..12a84a1fe8f 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -8,6 +8,7 @@ #include #include +#include #include "common/datatypes/DataSet.h" #include "common/meta/NebulaSchemaProvider.h" @@ -404,72 +405,97 @@ class SchemaParser { /** * define index of a schema * - * format: ${IndexID}: + * format: + * (Tag|Edge)(name,id) * example * std::string str=R"( * Tag(name,id) * (i1,1): a,b(10),c * (i2,2): b(5),c - * )"_tagIndex(schema) + * )"_index(schema) */ class IndexParser { public: using IndexItem = ::nebula::meta::cpp2::IndexItem; - using TagItem = ::nebula::meta::cpp2::TagItem; - using EdgeItem = ::nebula::meta::cpp2::EdgeItem; + using SchemaProvider = ::nebula::meta::NebulaSchemaProvider; explicit IndexParser(const std::string& str) { ss = std::stringstream(folly::stripLeftMargin(str)); + parseSchema(); } - std::vector> operator()(const TagItem& tag) { - std::vector> ret; + void parseSchema() { + static std::regex pattern(R"((TAG|EDGE)\((.+),(\d+)\))"); + std::smatch match; std::string line; - while (std::getline(ss, line)) { - auto index = std::make_shared(); - ::nebula::cpp2::SchemaID schemaId; - schemaId.set_tag_id(tag.get_tag_id()); - index->set_schema_id(schemaId); - index->set_schema_name(tag.get_tag_name()); - parseIndex(index, tag.get_schema()); - ret.push_back(index); + std::getline(ss, line); + assert(std::regex_match(line, match, pattern)); + std::string name = match.str(2); + int32_t id = std::stoi(match.str(3)); + schemaName_ = name; + if (match.str(1) == "TAG") { + schemaId_.set_tag_id(id); + } else { + schemaId_.set_edge_type(id); } - return ret; } - std::vector> operator()(const EdgeItem& edge) { - std::vector> ret; + std::map> operator()(std::shared_ptr schema) { + schema_ = schema; + std::map> ret; std::string line; while (std::getline(ss, line)) { - auto index = std::make_shared(); - ::nebula::cpp2::SchemaID schemaId; - schemaId.set_edge_type(edge.get_edge_type()); - index->set_schema_id(schemaId); - index->set_schema_name(edge.get_edge_name()); - parseIndex(index, edge.get_schema()); - ret.push_back(index); + auto index = parse(line); + ret[index->get_index_id()] = index; } return ret; } - // void parseIndex(std::shared_ptr index,){ - - // } - - // std::vector> operator()() { - // if (isEdge_) { - // schemaID_.set_edge_type(schemaId); - // } else { - // schemaID_.set_tag_id(schemaId); - // } - // std::string line; - // while (std::getline(ss, line)) { - // auto index = std::make_shared<::nebula::meta::cpp2::IndexItem>(); - // index->set_schema_id(schemaID_); - // std::vector values; - // folly::split(":", line, values); - // } - // } + std::shared_ptr parse(const std::string& line) { + auto ret = std::make_shared(); + ret->set_schema_id(schemaId_); + ret->set_schema_name(schemaName_); + static std::regex pattern(R"(\((.+),(\d+)\):(.+))"); + std::smatch match; + assert(std::regex_match(line, match, pattern)); + ret->set_index_name(folly::stripLeftMargin(match.str(1))); + ret->set_index_id(std::stoi(match.str(2))); + std::string columnStr = match.str(3); + std::vector columns; + folly::split(",", columnStr, columns); + for (size_t i = 0; i < columns.size(); i++) { + columns[i] = folly::stripLeftMargin(columns[i]); + } + std::vector<::nebula::meta::cpp2::ColumnDef> fields; + for (auto& column : columns) { + std::string name; + int length; + std::smatch match; + std::regex pattern(R"((.+)\((\d+)\))"); + if (std::regex_match(column, match, pattern)) { + name = match.str(1); + length = std::stoi(match.str(2)); + } else { + name = column; + length = 0; + } + ::nebula::meta::cpp2::ColumnDef col; + auto field = schema_->field(name); + col.set_name(name); + ::nebula::meta::cpp2::ColumnTypeDef type; + if (length > 0) { + type.set_type_length(length); + } + type.set_type(field->type()); + col.set_type(type); + col.set_nullable(field->nullable()); + fields.emplace_back(std::move(col)); + } + ret->set_fields(fields); + return ret; + } private: std::stringstream ss; - + std::string schemaName_; + ::nebula::cpp2::SchemaID schemaId_; + std::shared_ptr schema_; std::shared_ptr<::nebula::meta::cpp2::IndexItem> index_; }; @@ -478,8 +504,14 @@ std::vector operator""_row(const char* str, std::size_t) { auto ret = RowParser(std::string(str)).getResult(); return ret; } +std::shared_ptr<::nebula::meta::NebulaSchemaProvider> operator"" _schema(const char* str, + std::size_t) { + return SchemaParser(std::string(str)).getResult(); +} -class SchemaParser {} +IndexParser operator"" _index(const char* str, std::size_t) { + return IndexParser(std::string(str)); +} } // namespace storage } // namespace nebula From 1282dcd75737fc894bd4b8dd61b863beca931683 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Mon, 11 Oct 2021 12:23:39 +0800 Subject: [PATCH 10/38] add index int test --- src/interface/storage.thrift | 2 + src/kvstore/KVEngine.h | 6 + src/kvstore/KVStore.h | 9 + src/kvstore/NebulaStore.cpp | 13 +- src/kvstore/NebulaStore.h | 9 + src/kvstore/RocksEngine.cpp | 16 +- src/kvstore/RocksEngine.h | 32 +- src/storage/CMakeLists.txt | 8 + src/storage/ExprVisitorBase.cpp | 2 +- src/storage/GraphStorageServiceHandler.cpp | 2 +- src/storage/exec/IndexEdgeScanNode.cpp | 23 +- src/storage/exec/IndexEdgeScanNode.h | 3 + src/storage/exec/IndexNode.cpp | 5 + src/storage/exec/IndexNode.h | 2 +- src/storage/exec/IndexProjectionNode.cpp | 2 +- src/storage/exec/IndexScanNode.h | 7 +- src/storage/exec/IndexScanNode2.cpp | 200 +++-- src/storage/exec/IndexScanNode2.h | 71 +- src/storage/exec/IndexSelectionNode.cpp | 2 + src/storage/exec/IndexSelectionNode.h | 6 +- src/storage/exec/IndexVertexScanNode.cpp | 5 +- src/storage/exec/IndexVertexScanNode.h | 4 + src/storage/exec/StorageIterator.h | 2 +- ...sor-inl.h => LookupBaseProcessor-inl.h.bk} | 0 ...seProcessor.h => LookupBaseProcessor.h.bk} | 0 src/storage/index/LookupBaseProcessor2-inl.h | 8 - src/storage/index/LookupBaseProcessor2.h | 29 - ...upProcessor.cpp => LookupProcessor.cpp.bk} | 0 ...LookupProcessor.h => LookupProcessor.h.bk} | 0 src/storage/index/LookupProcessor2.cpp | 2 +- src/storage/index/LookupProcessor2.h | 9 +- src/storage/test/CMakeLists.txt | 14 + src/storage/test/IndexTest.cpp | 698 ++++++++++++++++-- src/storage/test/IndexTestUtil.h | 204 +++-- 34 files changed, 1141 insertions(+), 254 deletions(-) rename src/storage/index/{LookupBaseProcessor-inl.h => LookupBaseProcessor-inl.h.bk} (100%) rename src/storage/index/{LookupBaseProcessor.h => LookupBaseProcessor.h.bk} (100%) delete mode 100644 src/storage/index/LookupBaseProcessor2-inl.h delete mode 100644 src/storage/index/LookupBaseProcessor2.h rename src/storage/index/{LookupProcessor.cpp => LookupProcessor.cpp.bk} (100%) rename src/storage/index/{LookupProcessor.h => LookupProcessor.h.bk} (100%) diff --git a/src/interface/storage.thrift b/src/interface/storage.thrift index 2c61e5f3648..6185797fe1d 100644 --- a/src/interface/storage.thrift +++ b/src/interface/storage.thrift @@ -507,6 +507,8 @@ struct IndexColumnHint { 2: ScanType scan_type, 3: common.Value begin_value, 4: common.Value end_value, + 5: bool include_begin = true, + 6: bool include_end = false, } struct IndexQueryContext { diff --git a/src/kvstore/KVEngine.h b/src/kvstore/KVEngine.h index bd485a4333b..9aa7183a307 100644 --- a/src/kvstore/KVEngine.h +++ b/src/kvstore/KVEngine.h @@ -63,6 +63,12 @@ class KVEngine { const std::string& end, std::unique_ptr* iter) = 0; + virtual nebula::cpp2::ErrorCode range(bool includeStart, + const std::string& start, + bool includeEnd, + const std::string& end, + std::unique_ptr* iter) = 0; + // Get all results with 'prefix' str as prefix. virtual nebula::cpp2::ErrorCode prefix(const std::string& prefix, std::unique_ptr* iter) = 0; diff --git a/src/kvstore/KVStore.h b/src/kvstore/KVStore.h index dcef7e1a9f9..aafd88437d4 100644 --- a/src/kvstore/KVStore.h +++ b/src/kvstore/KVStore.h @@ -102,6 +102,15 @@ class KVStore { std::unique_ptr* iter, bool canReadFromFollower = false) = 0; + virtual nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, + PartitionID partId, + bool includeStart, + const std::string& start, + bool includeEnd, + const std::string& end, + std::unique_ptr* iter, + bool canReadFromFollower = false) = 0; + // Since the `range' interface will hold references to its 3rd & 4th // parameter, in `iter', thus the arguments must outlive `iter'. Here we // forbid one to invoke `range' with rvalues, which is the common mistake. diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp index 5ac276adebd..106c0d100ac 100644 --- a/src/kvstore/NebulaStore.cpp +++ b/src/kvstore/NebulaStore.cpp @@ -610,6 +610,16 @@ nebula::cpp2::ErrorCode NebulaStore::range(GraphSpaceID spaceId, const std::string& end, std::unique_ptr* iter, bool canReadFromFollower) { + return range(spaceId, partId, true, start, false, end, iter, canReadFromFollower); +} +nebula::cpp2::ErrorCode NebulaStore::range(GraphSpaceID spaceId, + PartitionID partId, + bool includeStart, + const std::string& start, + bool includeEnd, + const std::string& end, + std::unique_ptr* iter, + bool canReadFromFollower) { auto ret = part(spaceId, partId); if (!ok(ret)) { return error(ret); @@ -618,9 +628,8 @@ nebula::cpp2::ErrorCode NebulaStore::range(GraphSpaceID spaceId, if (!checkLeader(part, canReadFromFollower)) { return nebula::cpp2::ErrorCode::E_LEADER_CHANGED; } - return part->engine()->range(start, end, iter); + return part->engine()->range(includeStart, start, includeEnd, end, iter); } - nebula::cpp2::ErrorCode NebulaStore::prefix(GraphSpaceID spaceId, PartitionID partId, const std::string& prefix, diff --git a/src/kvstore/NebulaStore.h b/src/kvstore/NebulaStore.h index c888a22618c..9889ddc42bc 100644 --- a/src/kvstore/NebulaStore.h +++ b/src/kvstore/NebulaStore.h @@ -129,6 +129,15 @@ class NebulaStore : public KVStore, public Handler { std::unique_ptr* iter, bool canReadFromFollower = false) override; + nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, + PartitionID partId, + bool includeStart, + const std::string& start, + bool includeEnd, + const std::string& end, + std::unique_ptr* iter, + bool canReadFromFollower = false) override; + // Delete the overloading with a rvalue `start' and `end' nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, PartitionID partId, diff --git a/src/kvstore/RocksEngine.cpp b/src/kvstore/RocksEngine.cpp index 73bdb14e104..80de131c4fe 100644 --- a/src/kvstore/RocksEngine.cpp +++ b/src/kvstore/RocksEngine.cpp @@ -201,13 +201,25 @@ std::vector RocksEngine::multiGet(const std::vector& keys, nebula::cpp2::ErrorCode RocksEngine::range(const std::string& start, const std::string& end, std::unique_ptr* storageIter) { + return range(true, start, false, end, storageIter); +} + +nebula::cpp2::ErrorCode RocksEngine::range(bool includeStart, + const std::string& start, + bool includeEnd, + const std::string& end, + std::unique_ptr* storageIter) { rocksdb::ReadOptions options; options.total_order_seek = FLAGS_enable_rocksdb_prefix_filtering; rocksdb::Iterator* iter = db_->NewIterator(options); if (iter) { - iter->Seek(rocksdb::Slice(start)); + auto seekKey = start; + if (!includeStart) { + seekKey += '\0'; + } + iter->Seek(rocksdb::Slice(seekKey)); } - storageIter->reset(new RocksRangeIter(iter, start, end)); + storageIter->reset(new RocksRangeIter(iter, includeStart, start, includeEnd, end)); return nebula::cpp2::ErrorCode::SUCCEEDED; } diff --git a/src/kvstore/RocksEngine.h b/src/kvstore/RocksEngine.h index d94343d84e7..a9448983a5c 100644 --- a/src/kvstore/RocksEngine.h +++ b/src/kvstore/RocksEngine.h @@ -13,6 +13,7 @@ #include #include "common/base/Base.h" +#include "folly/Likely.h" #include "kvstore/KVEngine.h" #include "kvstore/KVIterator.h" #include "kvstore/RocksEngineConfig.h" @@ -22,13 +23,30 @@ namespace kvstore { class RocksRangeIter : public KVIterator { public: - RocksRangeIter(rocksdb::Iterator* iter, rocksdb::Slice start, rocksdb::Slice end) - : iter_(iter), start_(start), end_(end) {} + RocksRangeIter(rocksdb::Iterator* iter, + bool includeStart, + rocksdb::Slice start, + bool includeEnd, + rocksdb::Slice end) + : iter_(iter), + includeStart_(includeStart), + start_(start), + includeEnd_(includeEnd), + end_(end) {} ~RocksRangeIter() = default; bool valid() const override { - return !!iter_ && iter_->Valid() && (iter_->key().compare(end_) < 0); + if (LIKELY(!!iter_ && iter_->Valid())) { + int cmp2end = iter_->key().compare(end_); + if (cmp2end < 0 || (cmp2end == 0 && includeEnd_)) { + return true; + } else { + return false; + } + } else { + return false; + } } void next() override { iter_->Next(); } @@ -45,7 +63,9 @@ class RocksRangeIter : public KVIterator { private: std::unique_ptr iter_; + bool includeStart_; rocksdb::Slice start_; + bool includeEnd_; rocksdb::Slice end_; }; @@ -146,6 +166,12 @@ class RocksEngine : public KVEngine { const std::string& end, std::unique_ptr* iter) override; + nebula::cpp2::ErrorCode range(bool includeStart, + const std::string& start, + bool includeEnd, + const std::string& end, + std::unique_ptr* iter) override; + nebula::cpp2::ErrorCode prefix(const std::string& prefix, std::unique_ptr* iter) override; diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 70706693396..115fd7aa96b 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -41,6 +41,14 @@ nebula_add_library( query/ScanVertexProcessor.cpp query/ScanEdgeProcessor.cpp index/LookupProcessor2.cpp + exec/IndexNode.cpp + exec/IndexDedupNode.cpp + exec/IndexEdgeScanNode.cpp + exec/IndexLimitNode.cpp + exec/IndexProjectionNode.cpp + exec/IndexScanNode2.cpp + exec/IndexSelectionNode.cpp + exec/IndexVertexScanNode.cpp ) nebula_add_library( diff --git a/src/storage/ExprVisitorBase.cpp b/src/storage/ExprVisitorBase.cpp index c52be9f7803..e921af37c24 100644 --- a/src/storage/ExprVisitorBase.cpp +++ b/src/storage/ExprVisitorBase.cpp @@ -54,7 +54,7 @@ void ExprVisitorBase::visit(SetExpression *expr) { } void ExprVisitorBase::visit(MapExpression *expr) { UNUSED(expr); } // property Expression -void visit(TagPropertyExpression *expr) { UNUSED(expr); } +void ExprVisitorBase::visit(TagPropertyExpression *expr) { UNUSED(expr); } void ExprVisitorBase::visit(EdgePropertyExpression *expr) { UNUSED(expr); } void ExprVisitorBase::visit(InputPropertyExpression *expr) { UNUSED(expr); } void ExprVisitorBase::visit(VariablePropertyExpression *expr) { UNUSED(expr); } diff --git a/src/storage/GraphStorageServiceHandler.cpp b/src/storage/GraphStorageServiceHandler.cpp index ec77f713943..1c03dba8f13 100644 --- a/src/storage/GraphStorageServiceHandler.cpp +++ b/src/storage/GraphStorageServiceHandler.cpp @@ -6,7 +6,7 @@ #include "storage/GraphStorageServiceHandler.h" -#include "storage/index/LookupProcessor.h" +#include "storage/index/LookupProcessor2.h" #include "storage/mutate/AddEdgesProcessor.h" #include "storage/mutate/AddVerticesProcessor.h" #include "storage/mutate/DeleteEdgesProcessor.h" diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index 5f6af7c06bf..5754d3bc06b 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -54,22 +54,29 @@ Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { } if (colPosMap.count(kSrc)) { auto vId = IndexKeyUtils::getIndexSrcId(context_->vIdLen(), key); + DVLOG(1) << folly::hexDump(vId.data(), vId.size()); if (context_->isIntId()) { values[colPosMap[kSrc]] = Value(*reinterpret_cast(vId.data())); } else { + DVLOG(2) << vId.subpiece(0, vId.find_first_of('\0')); + DVLOG(2) << vId.subpiece(0, vId.find_first_of('\0')).toString(); + DVLOG(2) << Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); values[colPosMap[kSrc]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } } + DVLOG(1) << values[colPosMap[kSrc]]; if (colPosMap.count(kDst)) { - auto vId = IndexKeyUtils::getIndexSrcId(context_->vIdLen(), key); + auto vId = IndexKeyUtils::getIndexDstId(context_->vIdLen(), key); if (context_->isIntId()) { - values[colPosMap[kSrc]] = Value(*reinterpret_cast(vId.data())); + values[colPosMap[kDst]] = Value(*reinterpret_cast(vId.data())); } else { - values[colPosMap[kSrc]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + values[colPosMap[kDst]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } } + DVLOG(1) << values[colPosMap[kDst]]; if (colPosMap.count(kRank)) { auto rank = IndexKeyUtils::getIndexRank(context_->vIdLen(), key); + values[colPosMap[kRank]] = Value(rank); } key.subtract(context_->vIdLen() * 2 + sizeof(EdgeType)); decodePropFromIndex(key, colPosMap, values); @@ -101,7 +108,15 @@ Map IndexEdgeScanNode::decodeFromBase(const std::string& key } } break; case QueryUtils::ReturnColType::kDst: { - values[col] = Value(context_->tagId_); + auto vId = NebulaKeyUtils::getDstId(context_->vIdLen(), key); + if (context_->isIntId()) { + values[col] = Value(*reinterpret_cast(vId.data())); + } else { + values[col] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + } + } break; + case QueryUtils::ReturnColType::kRank: { + values[col] = Value(NebulaKeyUtils::getRank(context_->vIdLen(), key)); } break; case QueryUtils::ReturnColType::kOther: { auto retVal = QueryUtils::readValue(reader.get(), col, edge_->field(col)); diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index f1bf2ed2f76..9fe15254719 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -33,6 +33,9 @@ class IndexEdgeScanNode : public IndexScanNode { // Convenient for testing std::function>()> getIndex; std::function()> getEdge; + + FRIEND_TEST(IndexScanTest, Edge); + friend class IndexScanTestHelper; }; } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexNode.cpp b/src/storage/exec/IndexNode.cpp index 533431ac1c9..0b44a6424f2 100644 --- a/src/storage/exec/IndexNode.cpp +++ b/src/storage/exec/IndexNode.cpp @@ -9,6 +9,11 @@ #include "folly/Likely.h" namespace nebula { namespace storage { +IndexNode::IndexNode(RuntimeContext* context, const std::string& name) + : context_(context), name_(name) { + spaceId_ = context_->spaceId(); +} + IndexNode::IndexNode(const IndexNode& node) : context_(node.context_), spaceId_(node.spaceId_), name_(node.name_) {} diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index e4e51ee5444..f1c8ac5c9a7 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -41,7 +41,7 @@ class IndexNode { void addChild(std::unique_ptr child) { children_.emplace_back(std::move(child)); } virtual std::unique_ptr copy() = 0; - const std::vector>& children(); + const std::vector>& children() { return children_; } protected: virtual ErrorOr doNext(bool& hasNext) = 0; diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp index df2d480154b..b7379d23e8c 100644 --- a/src/storage/exec/IndexProjectionNode.cpp +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -22,7 +22,7 @@ nebula::cpp2::ErrorCode IndexProjectionNode::init(InitContext& ctx) { } for (auto& col : requiredColumns_) { auto iter = ctx.retColMap.find(col); - DCHECK_NE(iter, ctx.retColMap.end()); + DCHECK(iter != ctx.retColMap.end()); colPos_[col] = iter->second; } ctx.returnColumns = requiredColumns_; diff --git a/src/storage/exec/IndexScanNode.h b/src/storage/exec/IndexScanNode.h index ddd19a49298..4a25f4ba6a1 100644 --- a/src/storage/exec/IndexScanNode.h +++ b/src/storage/exec/IndexScanNode.h @@ -51,9 +51,10 @@ class IndexScanNode : public RelNode { } scanPair_ = scanRet.value(); std::unique_ptr iter; - ret = isRangeScan_ ? kvstore_->range( - context_->spaceId(), partId, scanPair_.first, scanPair_.second, &iter) - : kvstore_->prefix(context_->spaceId(), partId, scanPair_.first, &iter); + ret = isRangeScan_ + ? this->kvstore_->range( + context_->spaceId(), partId, scanPair_.first, scanPair_.second, &iter) + : this->kvstore_->prefix(context_->spaceId(), partId, scanPair_.first, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED && iter && iter->valid()) { context_->isEdge() ? iter_.reset(new EdgeIndexIterator(std::move(iter), context_->vIdLen())) diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index 5d353aae1b6..15d986e885c 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -8,8 +8,22 @@ namespace nebula { namespace storage { // Define of Path -std::unique_ptr Path::make(std::shared_ptr index, - std::shared_ptr schema, +Path::Path(nebula::meta::cpp2::IndexItem* index, + const meta::SchemaProviderIf* schema, + const std::vector& hints) + : index_(index), schema_(schema), hints_(hints) { + bool nullFlag = false; + for (auto field : index->get_fields()) { + bool tmp = field.nullable_ref().value_or(false); + nullable_.push_back(tmp); + nullFlag |= tmp; + } + if (!nullFlag) { + nullable_.clear(); + } +} +std::unique_ptr Path::make(nebula::meta::cpp2::IndexItem* index, + const meta::SchemaProviderIf* schema, const std::vector& hints) { std::unique_ptr ret; if (hints.back().get_scan_type() == cpp2::ScanType::RANGE) { @@ -17,7 +31,7 @@ std::unique_ptr Path::make(std::shared_ptr } else { ret.reset(new PrefixPath(index, schema, hints)); } - return std::move(ret); + return ret; } Path::Qualified Path::qualified(const folly::StringPiece& key) { Qualified ret = Qualified::COMPATIBLE; @@ -26,25 +40,38 @@ Path::Qualified Path::qualified(const folly::StringPiece& key) { } return ret; } -std::string Path::encodeValue(const Value& value, const ColumnTypeDef& colDef, std::string& key) { +std::string Path::encodeValue(const Value& value, + const ColumnTypeDef& colDef, + size_t index, + std::string& key) { std::string val; if (value.type() == Value::Type::STRING) { val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); if (val.back() != '\0') { QFList_.clear(); - QFList_.emplace_back([](const std::string& key) { return Qualified::UNCERTAIN; }); + QFList_.emplace_back([](const std::string&) { return Qualified::UNCERTAIN; }); } } else { val = IndexKeyUtils::encodeValue(value); } + if (!nullable_.empty() && nullable_[index] == true) { + QFList_.emplace_back([isNull = value.isNull(), index](const std::string& k) { + std::bitset<16> nullableBit; + size_t len = k.size(); + auto v = *reinterpret_cast(k.data() + len - 2); + nullableBit = v; + return nullableBit.test(15 - index) ^ isNull ? Qualified::INCOMPATIBLE + : Qualified::COMPATIBLE; + }); + } key.append(val); return val; } // End of Path // Define of RangePath -RangePath::RangePath(std::shared_ptr index, - std::shared_ptr schema, +RangePath::RangePath(nebula::meta::cpp2::IndexItem* index, + const meta::SchemaProviderIf* schema, const std::vector& hints) : Path(index, schema, hints) { buildKey(); @@ -62,15 +89,19 @@ Path::Qualified RangePath::qualified(const Map& rowData) { } } auto& hint = hints_.back(); - // TODO(hs.zhang): IDL添加开闭区间信息 - if (hint.begin_value_ref().has_value()) { - bool ret = hint.get_begin_value() < rowData.at(hint.get_column_name()); + // TODO(hs.zhang): improve performance.Check include or not during build key. + if (hint.begin_value_ref().is_set()) { + bool ret = includeStart_ ? hint.get_begin_value() <= rowData.at(hint.get_column_name()) + : hint.get_begin_value() < rowData.at(hint.get_column_name()); if (!ret) { return Qualified::INCOMPATIBLE; } } - if (hint.end_value_ref().has_value()) { - bool ret = hint.get_end_value() > rowData.at(hint.get_column_name()); + if (hint.end_value_ref().is_set()) { + DVLOG(2) << includeEnd_; + bool ret = includeEnd_ ? hint.get_end_value() >= rowData.at(hint.get_column_name()) + : hint.get_end_value() > rowData.at(hint.get_column_name()); + DVLOG(2) << ret; if (!ret) { return Qualified::INCOMPATIBLE; } @@ -86,65 +117,90 @@ void RangePath::buildKey() { CHECK(fieldIter->get_name() == hint.get_column_name()); auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); CHECK(type == Value::Type::STRING && !fieldIter->get_type().type_length_ref().has_value()); - encodeValue(hint.get_begin_value(), fieldIter->get_type(), common); + encodeValue(hint.get_begin_value(), fieldIter->get_type(), i, common); } auto& hint = hints_.back(); startKey_ = common; endKey_ = std::move(common); - if (hint.begin_value_ref().has_value()) { - encodeBeginValue(hint.get_begin_value(), fieldIter->get_type(), startKey_); + size_t index = hints_.size() - 1; + if (hint.begin_value_ref().is_set()) { + encodeBeginValue(hint.get_begin_value(), fieldIter->get_type(), index, startKey_); + includeStart_ = hint.get_include_begin(); + } else { + includeStart_ = false; } - if (hint.end_value_ref().has_value()) { - encodeEndValue(hint.get_end_value(), fieldIter->get_type(), endKey_); + if (hint.end_value_ref().is_set()) { + encodeEndValue(hint.get_end_value(), fieldIter->get_type(), index, endKey_); + includeEnd_ = hint.get_include_end(); } else { - endKey_.append(startKey_.size() - endKey_.size(), '0xFF'); + endKey_.append(startKey_.size() - endKey_.size(), '\xFF'); + includeEnd_ = true; } } std::string RangePath::encodeBeginValue(const Value& value, const ColumnTypeDef& colDef, + size_t index, std::string& key) { - std::string val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); + std::string val = IndexKeyUtils::encodeValue(value, colDef.type_length_ref().value_or(0)); if (value.type() == Value::Type::STRING && val.back() != '\0') { QFList_.emplace_back([&startKey = this->startKey_, startPos = key.size(), length = val.size()]( - const std::string& key) { - int ret = memcmp(startKey.data() + startPos, key.data() + startPos, length); + const std::string& k) { + int ret = memcmp(startKey.data() + startPos, k.data() + startPos, length); CHECK_LE(ret, 0); return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; }); } else { QFList_.emplace_back([&startKey = this->startKey_, - &allowEq = this->eqToStart_, + &allowEq = this->includeStart_, startPos = key.size(), - length = val.size()](const std::string& key) { - int ret = memcmp(startKey.data() + startPos, key.data() + startPos, length); + length = val.size()](const std::string& k) { + int ret = memcmp(startKey.data() + startPos, k.data() + startPos, length); CHECK_LE(ret, 0); return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); } - key += val; + if (!nullable_.empty() && nullable_[index] == true) { + QFList_.emplace_back([index](const std::string& k) { + std::bitset<16> nullableBit; + size_t len = k.size(); + auto v = *reinterpret_cast(k.data() + len - 2); + nullableBit = v; + return nullableBit.test(15 - index) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; + }); + } return val; } std::string RangePath::encodeEndValue(const Value& value, const ColumnTypeDef& colDef, + size_t index, std::string& key) { - std::string val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); + std::string val = IndexKeyUtils::encodeValue(value, colDef.type_length_ref().value_or(0)); if (value.type() == Value::Type::STRING && val.back() != '\0') { QFList_.emplace_back([&endKey = this->endKey_, startPos = key.size(), length = val.size()]( - const std::string& key) { - int ret = memcmp(endKey.data() + startPos, key.data() + startPos, length); + const std::string& k) { + int ret = memcmp(endKey.data() + startPos, k.data() + startPos, length); CHECK_GE(ret, 0); return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; }); } else { QFList_.emplace_back([&endKey = this->endKey_, - &allowEq = this->eqToEnd_, + &allowEq = this->includeEnd_, startPos = key.size(), - length = val.size()](const std::string& key) { - int ret = memcmp(endKey.data() + startPos, key.data() + startPos, length); + length = val.size()](const std::string& k) { + int ret = memcmp(endKey.data() + startPos, k.data() + startPos, length); CHECK_GE(ret, 0); return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); } + if (!nullable_.empty() && nullable_[index] == true) { + QFList_.emplace_back([index](const std::string& k) { + std::bitset<16> nullableBit; + size_t len = k.size(); + auto v = *reinterpret_cast(k.data() + len - 2); + nullableBit = v; + return nullableBit.test(15 - index) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; + }); + } key += val; return val; } @@ -152,8 +208,8 @@ std::string RangePath::encodeEndValue(const Value& value, // End of RangePath // Define of PrefixPath -PrefixPath::PrefixPath(std::shared_ptr index, - std::shared_ptr schema, +PrefixPath::PrefixPath(nebula::meta::cpp2::IndexItem* index, + const meta::SchemaProviderIf* schema, const std::vector& hints) : Path(index, schema, hints) { buildKey(); @@ -178,8 +234,8 @@ void PrefixPath::buildKey() { auto& hint = hints_[i]; CHECK(fieldIter->get_name() == hint.get_column_name()); auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); - CHECK(type == Value::Type::STRING && !fieldIter->get_type().type_length_ref().has_value()); - encodeValue(hint.get_begin_value(), fieldIter->get_type(), common); + CHECK(type != Value::Type::STRING || !fieldIter->get_type().type_length_ref().has_value()); + encodeValue(hint.get_begin_value(), fieldIter->get_type(), i, common); } prefix_ = std::move(common); } @@ -197,13 +253,41 @@ IndexScanNode::IndexScanNode(const IndexScanNode& node) requiredColumns_(node.requiredColumns_), ttlProps_(node.ttlProps_) {} +::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { + for (auto& col : ctx.requiredColumns) { + requiredColumns_.push_back(col); + } + ctx.returnColumns = requiredColumns_; + for (size_t i = 0; i < ctx.returnColumns.size(); i++) { + ctx.retColMap[ctx.returnColumns[i]] = i; + } + // Analyze whether the scan needs to access base data. + // TODO(hs.zhang): The performance is better to judge based on whether the string is truncated + auto tmp = ctx.requiredColumns; + for (auto& field : index_->get_fields()) { + if (field.get_type().get_type() == PropertyType::STRING) { + continue; + } + tmp.erase(field.get_name()); + } + tmp.erase(kVid); + tmp.erase(kTag); + tmp.erase(kRank); + tmp.erase(kSrc); + tmp.erase(kDst); + needAccessBase_ = !tmp.empty(); + path_ = Path::make(index_.get(), getSchema(), columnHints_); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; +} nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { + partId_ = partId; auto ret = resetIter(partId); return ret; } IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { hasNext = true; - while (iter_ && iter_->valid()) { + for (; iter_ && iter_->valid(); iter_->next()) { + DVLOG(1) << '\n' << folly::hexDump(iter_->key().data(), iter_->key().size()); if (!checkTTL()) { continue; } @@ -211,26 +295,37 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { if (q == Path::Qualified::INCOMPATIBLE) { continue; } - if (q == Path::Qualified::COMPATIBLE) { - return decodeFromIndex(iter_->key()); + bool compatible = q == Path::Qualified::COMPATIBLE; + if (compatible && !needAccessBase_) { + DVLOG(3) << 123; + auto key = iter_->key(); + iter_->next(); + return decodeFromIndex(key); } + DVLOG(3) << 123; std::pair kv; auto ret = getBaseData(iter_->key(), kv); if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + DVLOG(3) << 123; continue; } Map rowData = decodeFromBase(kv.first, kv.second); - q = path_->qualified(rowData); - CHECK(q != Path::Qualified::UNCERTAIN); - if (q == Path::Qualified::INCOMPATIBLE) { - continue; - } - if (q == Path::Qualified::COMPATIBLE) { - Row row; - for (auto& col : requiredColumns_) { - row.emplace_back(std::move(rowData.at(col))); + if (!compatible) { + DVLOG(3) << 123; + q = path_->qualified(rowData); + CHECK(q != Path::Qualified::UNCERTAIN); + if (q == Path::Qualified::INCOMPATIBLE) { + DVLOG(3) << 123; + continue; } } + DVLOG(3) << 123; + Row row; + for (auto& col : requiredColumns_) { + row.emplace_back(std::move(rowData.at(col))); + } + iter_->next(); + return row; } hasNext = false; return ErrorOr(Row()); @@ -246,13 +341,19 @@ bool IndexScanNode::checkTTL() { } return true; } + nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { path_->resetPart(partId); nebula::cpp2::ErrorCode ret; if (path_->isRange()) { auto rangePath = dynamic_cast(path_.get()); - ret = - kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter_); + ret = kvstore_->range(spaceId_, + partId, + rangePath->includeStart(), + rangePath->getStartKey(), + rangePath->includeEnd(), + rangePath->getEndKey(), + &iter_); } else { auto prefixPath = dynamic_cast(path_.get()); ret = kvstore_->prefix(spaceId_, partId, prefixPath->getPrefixKey(), &iter_); @@ -263,7 +364,6 @@ void IndexScanNode::decodePropFromIndex(folly::StringPiece key, const Map& colPosMap, std::vector& values) { size_t offset = sizeof(PartitionID) + sizeof(IndexID); - size_t len = 0; std::bitset<16> nullableBit; int8_t nullableColPosit = 15; if (indexNullable_) { diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index f633423464f..becc4bd4926 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -4,6 +4,8 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once +#include + #include #include @@ -21,6 +23,11 @@ class Path; class RangePath; class PrefixPath; class IndexScanNode : public IndexNode { + FRIEND_TEST(IndexScanTest, Base); + FRIEND_TEST(IndexScanTest, Vertex); + FRIEND_TEST(IndexScanTest, Edge); + friend class IndexScanTestHelper; + public: IndexScanNode(const IndexScanNode& node); IndexScanNode(RuntimeContext* context, @@ -28,16 +35,7 @@ class IndexScanNode : public IndexNode { IndexID indexId, const std::vector& columnHints) : IndexNode(context, name), indexId_(indexId), columnHints_(columnHints) {} - ::nebula::cpp2::ErrorCode init(InitContext& ctx) override { - for (auto& col : ctx.requiredColumns) { - requiredColumns_.push_back(col); - } - ctx.returnColumns = requiredColumns_; - for (size_t i = 0; i < ctx.returnColumns.size(); i++) { - ctx.retColMap[ctx.returnColumns[i]] = i; - } - return ::nebula::cpp2::ErrorCode::SUCCEEDED; - } + ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; protected: nebula::cpp2::ErrorCode doExecute(PartitionID partId) final; @@ -63,6 +61,7 @@ class IndexScanNode : public IndexNode { nebula::kvstore::KVStore* kvstore_; std::vector requiredColumns_; std::pair> ttlProps_; + bool needAccessBase_{false}; }; /** * Path @@ -74,16 +73,14 @@ class Path { enum class Qualified : int16_t { INCOMPATIBLE = 0, UNCERTAIN = 1, COMPATIBLE = 2 }; using QualifiedFunction = std::function; using ColumnTypeDef = ::nebula::meta::cpp2::ColumnTypeDef; - Path(std::shared_ptr index, - std::shared_ptr schema, - const std::vector& hints) - : index_(index), schema_(schema), hints_(hints) {} + Path(nebula::meta::cpp2::IndexItem* index, + const meta::SchemaProviderIf* schema, + const std::vector& hints); virtual ~Path() = default; - static std::unique_ptr make( - std::shared_ptr index, - std::shared_ptr schema, - const std::vector& hints); + static std::unique_ptr make(::nebula::meta::cpp2::IndexItem* index, + const meta::SchemaProviderIf* schema, + const std::vector& hints); Qualified qualified(const folly::StringPiece& key); virtual bool isRange() { return false; } @@ -91,22 +88,26 @@ class Path { virtual void resetPart(PartitionID partId) = 0; protected: - std::string encodeValue(const Value& value, const ColumnTypeDef& colDef, std::string& key); + std::string encodeValue(const Value& value, + const ColumnTypeDef& colDef, + size_t index, + std::string& key); std::vector QFList_; - std::shared_ptr index_; - std::shared_ptr schema_; + ::nebula::meta::cpp2::IndexItem* index_; + const meta::SchemaProviderIf* schema_; const std::vector& hints_; + std::vector nullable_; }; class PrefixPath : public Path { public: - PrefixPath(std::shared_ptr index, - std::shared_ptr schema, + PrefixPath(nebula::meta::cpp2::IndexItem* index, + const meta::SchemaProviderIf* schema, const std::vector& hints); // Override Qualified qualified(const Map& rowData) override; void resetPart(PartitionID partId) override; - const std::string& getPrefixKey(); + const std::string& getPrefixKey() { return prefix_; } private: std::string prefix_; @@ -114,25 +115,33 @@ class PrefixPath : public Path { }; class RangePath : public Path { public: - RangePath(std::shared_ptr index, - std::shared_ptr schema, + RangePath(nebula::meta::cpp2::IndexItem* index, + const meta::SchemaProviderIf* schema, const std::vector& hints); // Override Qualified qualified(const Map& rowData) override; void resetPart(PartitionID partId) override; - inline bool eqToStart() { return eqToStart_; } - inline bool eqToEnd() { return eqToEnd_; } + inline bool includeStart() { return includeStart_; } + inline bool includeEnd() { return includeEnd_; } inline const std::string& getStartKey() { return startKey_; } inline const std::string& getEndKey() { return endKey_; } virtual bool isRange() { return true; } private: std::string startKey_, endKey_; - bool eqToStart_, eqToEnd_; + bool includeStart_ = true; + bool includeEnd_ = false; + void buildKey(); - std::string encodeBeginValue(const Value& value, const ColumnTypeDef& colDef, std::string& key); - std::string encodeEndValue(const Value& value, const ColumnTypeDef& colDef, std::string& key); + std::string encodeBeginValue(const Value& value, + const ColumnTypeDef& colDef, + size_t index, + std::string& key); + std::string encodeEndValue(const Value& value, + const ColumnTypeDef& colDef, + size_t index, + std::string& key); }; /* define inline functions */ diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index fd34184681c..c1725e99721 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -49,6 +49,7 @@ std::unique_ptr IndexSelectionNode::copy() { } Value IndexSelectionNode::ExprContext::getEdgeProp(const std::string& edgeType, const std::string& prop) const { + UNUSED(edgeType); DCHECK(row_ != nullptr); auto iter = colPos_.find(prop); DCHECK(iter != colPos_.end()); @@ -57,6 +58,7 @@ Value IndexSelectionNode::ExprContext::getEdgeProp(const std::string& edgeType, } Value IndexSelectionNode::ExprContext::getTagProp(const std::string& tag, const std::string& prop) const { + UNUSED(tag); DCHECK(row_ != nullptr); auto iter = colPos_.find(prop); DCHECK(iter != colPos_.end()); diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 59e58c36cca..3dfd4c2e4a0 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -26,8 +26,8 @@ class IndexSelectionNode : public IndexNode { auto &result = expr_->eval(*ctx_); return result.type() == Value::Type::BOOL ? result.getBool() : false; } - Map colPos_; Expression *expr_; + Map colPos_; class ExprContext : public ExpressionContext { public: explicit ExprContext(const Map &colPos) : colPos_(colPos) {} @@ -90,8 +90,8 @@ class SelectionExprVisitor : public ExprVisitorBase { void visit(EdgeDstIdExpression *expr) override { requiredColumns_.insert(expr->prop()); } void visit(TagPropertyExpression *expr) { requiredColumns_.insert(expr->prop()); } void visit(EdgePropertyExpression *expr) override { requiredColumns_.insert(expr->prop()); } - const Set &getRequiredColumns(); - ::nebula::cpp2::ErrorCode getCode(); + const Set &getRequiredColumns() { return requiredColumns_; } + ::nebula::cpp2::ErrorCode getCode() { return code_; } private: using ExprVisitorBase::visit; diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index dff1c9ec716..f5fb1891763 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -46,7 +46,6 @@ IndexVertexScanNode::IndexVertexScanNode(RuntimeContext* context, return schema; }); } - ::nebula::cpp2::ErrorCode IndexVertexScanNode::init(InitContext& ctx) { this->index_ = getIndex().value(); this->tag_ = getTag(); @@ -58,7 +57,9 @@ nebula::cpp2::ErrorCode IndexVertexScanNode::getBaseData(folly::StringPiece key, partId_, key.subpiece(key.size() - context_->vIdLen()).toString(), context_->tagId_); - return context_->env()->kvstore_->get(context_->spaceId(), partId_, kv.first, &kv.second); + DVLOG(3) << partId_; + DVLOG(1) << '\n' << folly::hexDump(kv.first.data(), kv.first.size()); + return kvstore_->get(context_->spaceId(), partId_, kv.first, &kv.second); } Row IndexVertexScanNode::decodeFromIndex(folly::StringPiece key) { std::vector values(requiredColumns_.size()); diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index 1e6b64b9dbd..52b74bac758 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -38,9 +38,13 @@ class IndexVertexScanNode final : public IndexScanNode { std::function>()> getIndex; std::function()> getTag; + FRIEND_TEST(IndexScanTest, VertexIndexOnlyScan); FRIEND_TEST(IndexScanTest, VertexBase); FRIEND_TEST(IndexScanTest, Prefix1); FRIEND_TEST(IndexScanTest, Prefix2); + FRIEND_TEST(IndexScanTest, Base); + FRIEND_TEST(IndexScanTest, Vertex); + friend class IndexScanTestHelper; }; } // namespace storage } // namespace nebula diff --git a/src/storage/exec/StorageIterator.h b/src/storage/exec/StorageIterator.h index e5d281f1d20..fefe32e7a76 100644 --- a/src/storage/exec/StorageIterator.h +++ b/src/storage/exec/StorageIterator.h @@ -7,11 +7,11 @@ #ifndef STORAGE_EXEC_STORAGEITERATOR_H_ #define STORAGE_EXEC_STORAGEITERATOR_H_ +#include "codec/RowReaderWrapper.h" #include "common/base/Base.h" #include "kvstore/KVIterator.h" #include "storage/CommonUtils.h" #include "storage/StorageFlags.h" - namespace nebula { namespace storage { diff --git a/src/storage/index/LookupBaseProcessor-inl.h b/src/storage/index/LookupBaseProcessor-inl.h.bk similarity index 100% rename from src/storage/index/LookupBaseProcessor-inl.h rename to src/storage/index/LookupBaseProcessor-inl.h.bk diff --git a/src/storage/index/LookupBaseProcessor.h b/src/storage/index/LookupBaseProcessor.h.bk similarity index 100% rename from src/storage/index/LookupBaseProcessor.h rename to src/storage/index/LookupBaseProcessor.h.bk diff --git a/src/storage/index/LookupBaseProcessor2-inl.h b/src/storage/index/LookupBaseProcessor2-inl.h deleted file mode 100644 index dc28011b425..00000000000 --- a/src/storage/index/LookupBaseProcessor2-inl.h +++ /dev/null @@ -1,8 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ -namespace nebula { -namespace storage {} -} // namespace nebula diff --git a/src/storage/index/LookupBaseProcessor2.h b/src/storage/index/LookupBaseProcessor2.h deleted file mode 100644 index e019c42e9a8..00000000000 --- a/src/storage/index/LookupBaseProcessor2.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ -#pragma once -#include "storage/BaseProcessor.h" -namespace nebula { -namespace storage { -template -class LookupBaseProcessor : public BaseProcessor { - public: - virtual ~LookupBaseProcessor() = default; - virtual void process(const REQ& req) = 0; - - protected: - LookupBaseProcessor(StorageEnv* env, - const ProcessorCounters* counters, - folly::Executor* executor = nullptr) {} - - virtual void onProcessFinished() = 0; - - StatusOr> buildPlan(); - nebula::DataSet resultDataSet_; - std::vector partResults_; -}; -} // namespace storage -} // namespace nebula -#include "storage/index/LookupBaseProcessor-inl.h" diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp.bk similarity index 100% rename from src/storage/index/LookupProcessor.cpp rename to src/storage/index/LookupProcessor.cpp.bk diff --git a/src/storage/index/LookupProcessor.h b/src/storage/index/LookupProcessor.h.bk similarity index 100% rename from src/storage/index/LookupProcessor.h rename to src/storage/index/LookupProcessor.h.bk diff --git a/src/storage/index/LookupProcessor2.cpp b/src/storage/index/LookupProcessor2.cpp index c709f5d77db..c9105f22a93 100644 --- a/src/storage/index/LookupProcessor2.cpp +++ b/src/storage/index/LookupProcessor2.cpp @@ -15,7 +15,7 @@ #include "storage/exec/IndexVertexScanNode.h" namespace nebula { namespace storage { - +ProcessorCounters kLookupCounters; void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { if (executor_ != nullptr) { executor_->add([req, this]() { this->doProcess(req); }); diff --git a/src/storage/index/LookupProcessor2.h b/src/storage/index/LookupProcessor2.h index 3c1df8226df..a329f55adff 100644 --- a/src/storage/index/LookupProcessor2.h +++ b/src/storage/index/LookupProcessor2.h @@ -17,13 +17,16 @@ class LookupProcessor : public BaseProcessor { public: static LookupProcessor* instance(StorageEnv* env, const ProcessorCounters* counters = &kLookupCounters, - folly::Executor* executor = nullptr); + folly::Executor* executor = nullptr) { + return new LookupProcessor(env, counters, executor); + } void process(const cpp2::LookupIndexRequest& req); private: - LookupProcessor(StorageEnv* env, const ProcessorCounters* counters, folly::Executor* executor); + LookupProcessor(StorageEnv* env, const ProcessorCounters* counters, folly::Executor* executor) + : BaseProcessor(env, counters), executor_(executor) {} void doProcess(const cpp2::LookupIndexRequest& req); - void onProcessFinished(); + void onProcessFinished() {} void runInSingleThread(const std::vector& parts, std::unique_ptr plan); void runInMultipleThread(const std::vector& parts, std::unique_ptr plan); diff --git a/src/storage/test/CMakeLists.txt b/src/storage/test/CMakeLists.txt index 33dee520e63..5c19a5be1ff 100644 --- a/src/storage/test/CMakeLists.txt +++ b/src/storage/test/CMakeLists.txt @@ -706,6 +706,20 @@ nebula_add_test( gtest ) +nebula_add_test( + NAME + index_test + SOURCES + IndexTest.cpp + OBJECTS + ${storage_test_deps} + LIBRARIES + ${ROCKSDB_LIBRARIES} + ${THRIFT_LIBRARIES} + ${PROXYGEN_LIBRARIES} + gtest +) + nebula_add_executable( NAME chain_update_edge_test diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index b358043c1ce..14afeac4e14 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -6,6 +6,9 @@ #include +#include "codec/RowReaderWrapper.h" +#include "codec/RowWriterV2.h" +#include "common/utils/NebulaKeyUtils.h" #include "kvstore/KVEngine.h" #include "kvstore/KVIterator.h" #include "storage/exec/IndexDedupNode.h" @@ -17,81 +20,660 @@ #include "storage/test/IndexTestUtil.h" namespace nebula { namespace storage { +namespace { +int schemaVer = 2; +} /** - * _KSrc, - * "",123,456,"abc", - * - * + * IndexScanTest * + * Test: + * 1. Vertex/Edge + * 2. back to table or not + * 3. different value type + * a. int/float/bool/fix_string/time/date/datetime + * b. compound index + * 4. range/prefix + * a. prefix(equal) + * b. range with begin is include/exclude/-INF + * c. range with end id include/exclude/+INF + * 5. nullable + * Case: + * ┌──────────┬─────────────────────────────────────────┐ + * │ section1 │ name │ case │ description │ NOTICE │ + * Base | Base | + * | Vertex| IndexOnly + * | | BackToTable + * | Edge | indexOnly + * | BackToTable + * Value Type | Int | PrefixWithTruncate + * | Float | PrefixWithoutTruncate + * | Bool | INCLUDE_INF + * | String | INCLUDE_EXCLUDE + * | Time | EXCLUDE_INF + * | Date | EXCLUDE_INCLUDE + * | DateTime| INF_INCLUDE + * | Compound| INF_EXCLUDE + * | Nullable| * + * ┌┬┐├┼┤└┴┘┌─┐│┼│└─┘ * */ - -class MockKVStore : public ::nebula::kvstore::KVEngine {}; -class IndexMockNode : public IndexNode { - public: - private: +struct IndexScanTestHelper { + void setKVStore(IndexScanNode* node, kvstore::KVStore* store) { node->kvstore_ = store; } + void setIndex(IndexVertexScanNode* node, std::shared_ptr<::nebula::meta::cpp2::IndexItem> index) { + node->getIndex = [index]() { return index; }; + } + void setIndex(IndexEdgeScanNode* node, std::shared_ptr<::nebula::meta::cpp2::IndexItem> index) { + node->getIndex = [index]() { return index; }; + } + void setTag(IndexVertexScanNode* node, + std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema) { + node->getTag = [schema]() { return schema; }; + } + void setEdge(IndexEdgeScanNode* node, + std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema) { + node->getEdge = [schema]() { return schema; }; + } }; - -/** - * - */ class IndexScanTest : public ::testing::Test { protected: - private: - static std::unique_ptr kvstore_; + using Schema = ::nebula::meta::NebulaSchemaProvider; + using IndexItem = ::nebula::meta::cpp2::IndexItem; + using ColumnHint = ::nebula::storage::cpp2::IndexColumnHint; + static ColumnHint makeColumnHint(const std::string& name, const Value& value) { + ColumnHint hint; + hint.set_column_name(name); + hint.set_begin_value(value); + hint.set_scan_type(cpp2::ScanType::PREFIX); + return hint; + } + template + static ColumnHint makeColumnHint(const std::string& name, const Value& begin, const Value& end) { + ColumnHint hint; + hint.set_column_name(name); + hint.set_scan_type(cpp2::ScanType::RANGE); + hint.set_begin_value(begin); + hint.set_end_value(end); + hint.set_include_begin(includeBegin); + hint.set_include_end(includeEnd); + return hint; + } + template + static ColumnHint makeBeginColumnHint(const std::string& name, const Value& begin) { + ColumnHint hint; + hint.set_column_name(name); + hint.set_scan_type(cpp2::ScanType::RANGE); + hint.set_begin_value(begin); + hint.set_include_begin(include); + return hint; + } + template + static ColumnHint makeEndColumnHint(const std::string& name, const Value& end) { + ColumnHint hint; + hint.set_column_name(name); + hint.set_scan_type(cpp2::ScanType::RANGE); + hint.set_end_value(end); + hint.set_include_end(include); + return hint; + } + static std::vector> encodeTag( + const std::vector& rows, + TagID tagId, + std::shared_ptr schema, + std::vector> indices) { + std::vector> ret(indices.size() + 1); + for (size_t i = 0; i < rows.size(); i++) { + auto key = NebulaKeyUtils::vertexKey(8, 0, std::to_string(i), tagId); + RowWriterV2 writer(schema.get()); + for (size_t j = 0; j < rows[i].size(); j++) { + writer.setValue(j, rows[i][j]); + } + writer.finish(); + auto value = writer.moveEncodedStr(); + assert(ret[0].insert({key, value}).second); + RowReaderWrapper reader(schema.get(), folly::StringPiece(value), schemaVer); + for (size_t j = 0; j < indices.size(); j++) { + auto& index = indices[j]; + auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index->get_fields()).value(); + auto indexKey = IndexKeyUtils::vertexIndexKey( + 8, 0, index->get_index_id(), std::to_string(i), std::move(indexValue)); + assert(ret[j + 1].insert({indexKey, ""}).second); + } + } + return ret; + } + static std::vector> encodeEdge( + const std::vector& rows, + EdgeType edgeType, + std::shared_ptr schema, + std::vector> indices) { + std::vector> ret(indices.size() + 1); + for (size_t i = 0; i < rows.size(); i++) { + auto key = NebulaKeyUtils::edgeKey(8, 0, std::to_string(i), edgeType, i, std::to_string(i)); + DVLOG(1) << '\n' << folly::hexDump(key.data(), key.size()); + RowWriterV2 writer(schema.get()); + for (size_t j = 0; j < rows[i].size(); j++) { + writer.setValue(j, rows[i][j]); + } + writer.finish(); + auto value = writer.moveEncodedStr(); + assert(ret[0].insert({key, value}).second); + RowReaderWrapper reader(schema.get(), folly::StringPiece(value), schemaVer); + for (size_t j = 0; j < indices.size(); j++) { + auto& index = indices[j]; + auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index->get_fields()).value(); + auto indexKey = IndexKeyUtils::edgeIndexKey(8, + 0, + index->get_index_id(), + std::to_string(i), + i, + std::to_string(i), + std::move(indexValue)); + DVLOG(1) << '\n' << folly::hexDump(indexKey.data(), indexKey.size()); + assert(ret[j + 1].insert({indexKey, ""}).second); + } + } + return ret; + } + static PlanContext* getPlanContext() { + static std::unique_ptr ctx = std::make_unique(nullptr, 0, 8, false); + return ctx.get(); + } + static std::unique_ptr makeContext(TagID tagId, EdgeType edgeType) { + auto ctx = std::make_unique(getPlanContext()); + ctx->tagId_ = tagId; + ctx->edgeType_ = edgeType; + return ctx; + } }; -TEST_F(IndexScanTest, VertexIndexOnlyScan) { - std::vector rows = R"( +TEST_F(IndexScanTest, Base) { + auto rows = R"( + int | int + 1 | 2 + 1 | 3 + )"_row; + auto schema = R"( + a | int ||false + b | int ||false + )"_schema; + auto indices = R"( + TAG(t,1) + (i1,2):a + (i2,3):b + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + for (auto& iter : kv) { + for (auto& item : iter) { + kvstore->put(item.first, item.second); + } + } + { // Case 1 + std::vector columnHints{ + makeColumnHint("a", Value(1)) // a=1 + }; + IndexID indexId = 2; + auto context = makeContext(1, 0); + auto scanNode = std::make_unique(context.get(), indexId, columnHints); + scanNode->kvstore_ = kvstore.get(); + scanNode->getIndex = [index = indices[0]]() { return index; }; + scanNode->getTag = [schema]() { return schema; }; + InitContext initCtx; + initCtx.requiredColumns = {kVid, "a"}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + auto expect = R"( + string | int + 0 | 1 + 1 | 1 + )"_row; + std::vector colOrder = {kVid, "a"}; + ASSERT_EQ(result.size(), expect.size()); + for (size_t i = 0; i < result.size(); i++) { + ASSERT_EQ(result[i].size(), expect[i].size()); + for (size_t j = 0; j < expect[i].size(); j++) { + ASSERT_EQ(expect[i][j], result[i][initCtx.retColMap[colOrder[j]]]); + } + } + } // End of Case 1 + { // Case 2 + std::vector columnHints{ + makeColumnHint("b", Value(3)) // b=3 + }; + IndexID indexId = 3; + auto context = makeContext(1, 0); + auto scanNode = std::make_unique(context.get(), indexId, columnHints); + DVLOG(1) << kvstore.get(); + scanNode->kvstore_ = kvstore.get(); + scanNode->getIndex = [index = indices[1]]() { return index; }; + scanNode->getTag = [schema]() { return schema; }; + InitContext initCtx; + initCtx.requiredColumns = {kVid, "b"}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + auto expect = R"( + string | int + 1 | 3 + )"_row; + std::vector colOrder = {kVid, "b"}; + ASSERT_EQ(result.size(), expect.size()); + for (size_t i = 0; i < result.size(); i++) { + ASSERT_EQ(result[i].size(), expect[i].size()); + for (size_t j = 0; j < expect[i].size(); j++) { + ASSERT_EQ(expect[i][j], result[i][initCtx.retColMap[colOrder[j]]]); + } + } + } // End of Case 2 +} +TEST_F(IndexScanTest, Vertex) { + auto rows = R"( + int | int + 1 | 2 + 1 | 3 + )"_row; + auto schema = R"( + a | int | | false + b | int | | false + )"_schema; + auto indices = R"( + TAG(t,1) + (i1,2):a + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + std::vector columnHints{ + makeColumnHint("a", Value(1)) // a=1 + }; + IndexID indexId = 0; + auto context = makeContext(1, 0); + { // Case 1: IndexOnly + // Only put index key-values into kvstore + for (auto& item : kv[1]) { + kvstore->put(item.first, item.second); + } + auto scanNode = std::make_unique(context.get(), indexId, columnHints); + scanNode->kvstore_ = kvstore.get(); + scanNode->getIndex = [index = indices[0]]() { return index; }; + scanNode->getTag = [schema]() { return schema; }; + InitContext initCtx; + initCtx.requiredColumns = {kVid, "a"}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + auto expect = R"( + string | int + 0 | 1 + 1 | 1 + )"_row; + std::vector colOrder = {kVid, "a"}; + ASSERT_EQ(result.size(), expect.size()); + for (size_t i = 0; i < result.size(); i++) { + ASSERT_EQ(result[i].size(), expect[i].size()); + for (size_t j = 0; j < expect[i].size(); j++) { + ASSERT_EQ(expect[i][j], result[i][initCtx.retColMap[colOrder[j]]]); + } + } + } // End of Case 1 + { // Case 2: Access base data + // Put base data key-values into kvstore + for (auto& item : kv[0]) { + kvstore->put(item.first, item.second); + } + auto scanNode = std::make_unique(context.get(), indexId, columnHints); + scanNode->kvstore_ = kvstore.get(); + scanNode->getIndex = [index = indices[0]]() { return index; }; + scanNode->getTag = [schema]() { return schema; }; + InitContext initCtx; + initCtx.requiredColumns = {kVid, "b"}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + auto expect = R"( + string | int + 0 | 2 + 1 | 3 + )"_row; + std::vector colOrder = {kVid, "b"}; + ASSERT_EQ(result.size(), expect.size()); + for (size_t i = 0; i < result.size(); i++) { + ASSERT_EQ(result[i].size(), expect[i].size()); + VLOG(1) << result[i]; + for (size_t j = 0; j < expect[i].size(); j++) { + ASSERT_EQ(expect[i][j], result[i][initCtx.retColMap[colOrder[j]]]); + } + } + } // End of Case 2 +} +TEST_F(IndexScanTest, Edge) { + auto rows = R"( int | int | int - 1 | 2 | 3 - 4 | 5 | 6 + 5 | 2 | 1 + 10 | 3 | 2 + 20 | 3 | 3 )"_row; auto schema = R"( - a | int || - b | int || - c | int || + a | int | | false + b | int | | false + c | int | | false )"_schema; - auto index = R"( - TAG(a,1) + auto indices = R"( + EDGE(e,1) + (i1,2):b,c + )"_index(schema); + auto kv = encodeEdge(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + std::vector columnHints{ + makeColumnHint("b", Value(3)), // b=3 + }; + IndexID indexId = 0; + auto context = makeContext(0, 1); + { // Case 1: IndexOnly + for (auto& item : kv[1]) { + kvstore->put(item.first, item.second); + } + auto scanNode = std::make_unique(context.get(), indexId, columnHints); + scanNode->kvstore_ = kvstore.get(); + scanNode->getIndex = [index = indices[0]]() { return index; }; + scanNode->getEdge = [schema]() { return schema; }; + InitContext initCtx; + initCtx.requiredColumns = {kSrc, kRank, kDst, "c"}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + auto expect = R"( + string | int | string | int + 1 | 1 | 1 | 2 + 2 | 2 | 2 | 3 + )"_row; + std::vector colOrder = {kSrc, kRank, kDst, "c"}; + ASSERT_EQ(result.size(), expect.size()); + for (size_t i = 0; i < result.size(); i++) { + ASSERT_EQ(result[i].size(), expect[i].size()); + DVLOG(1) << result[i]; + for (size_t j = 0; j < expect[i].size(); j++) { + EXPECT_EQ(expect[i][j], result[i][initCtx.retColMap[colOrder[j]]]); + } + } + } // End of Case 1 + { // Case 2: Access base data + for (auto& item : kv[0]) { + kvstore->put(item.first, item.second); + } + auto scanNode = std::make_unique(context.get(), indexId, columnHints); + scanNode->kvstore_ = kvstore.get(); + scanNode->kvstore_ = kvstore.get(); + scanNode->getIndex = [index = indices[0]]() { return index; }; + scanNode->getEdge = [schema]() { return schema; }; + InitContext initCtx; + initCtx.requiredColumns = {kSrc, kRank, kDst, "a"}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + auto expect = R"( + string | int | string | int + 1 | 1 | 1 | 10 + 2 | 2 | 2 | 20 + )"_row; + std::vector colOrder = {kSrc, kRank, kDst, "a"}; + ASSERT_EQ(result.size(), expect.size()); + for (size_t i = 0; i < result.size(); i++) { + ASSERT_EQ(result[i].size(), expect[i].size()); + DVLOG(1) << result[i]; + for (size_t j = 0; j < expect[i].size(); j++) { + EXPECT_EQ(expect[i][j], result[i][initCtx.retColMap[colOrder[j]]]); + } + } + } +} +TEST_F(IndexScanTest, Int) { + auto rows = R"( + int | int | int + 1 | -1 | -10 + 2 | 1 | -9223372036854775808 + 3 | 0 | -1 + 4 | 9223372036854775807 | 0 + 5 | -9223372036854775808 | 9223372036854775807 + 6 | | 0 + )"_row; + auto schema = R"( + a | int | | false + b | int | | true + c | int | | false + )"_schema; + auto indices = R"( + TAG(t,1) (i1,2):a (i2,3):b + (i3,4):c )"_index(schema); - std::vector keyValues = encode(rows, schema, index); + auto kv = encodeTag(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + for (auto& iter : kv) { + for (auto& item : iter) { + kvstore->put(item.first, item.second); + } + } + auto check = [&](std::shared_ptr index, + const std::vector& columnHints, + const std::vector& expect, + const std::string& case_) { + DVLOG(2) << "Start case " << case_; + auto context = makeContext(1, 0); + auto scanNode = std::make_unique(context.get(), 0, columnHints); + IndexScanTestHelper helper; + helper.setKVStore(scanNode.get(), kvstore.get()); + helper.setIndex(scanNode.get(), index); + helper.setTag(scanNode.get(), schema); + InitContext initCtx; + initCtx.requiredColumns = {kVid}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + EXPECT_EQ(result, expect) << "Fail at case " << case_; + }; + auto expect = [](auto... vidList) { + std::vector ret; + std::vector value; + (value.push_back(std::to_string(vidList)), ...); + for (auto& v : value) { + Row row; + row.emplace_back(v); + ret.emplace_back(std::move(row)); + } + return ret; + }; + const int64_t MAX = 0x7fffffffffffffff; + const int64_t MIN = -MAX - 1; + { // Case 1: prefix + std::vector columnHints = {makeColumnHint("a", 1)}; // a=1; + check(indices[0], columnHints, expect(0), "case1.1"); // + columnHints = {makeColumnHint("b", 0x7fffffffffffffff)}; // b=MAX + check(indices[1], columnHints, expect(3), "case1.2"); // + columnHints = {makeColumnHint("b", -0x7fffffffffffffff - 1)}; // b=MIN + check(indices[1], columnHints, expect(4), "case1.3"); // + columnHints = {makeColumnHint("c", 0)}; // c=0 + check(indices[2], columnHints, expect(3, 5), "case1.4"); // + } // End of Case 1 + { // Case 2: [x, INF) + std::vector columnHints = {makeBeginColumnHint("a", -1)}; // Case2.1: a >= -1 + check(indices[0], columnHints, expect(0, 1, 2, 3, 4, 5), "case2.1"); // + columnHints = {makeBeginColumnHint("a", 4)}; // Case2.2: a>=4 + check(indices[0], columnHints, expect(3, 4, 5), "case2.2"); // + columnHints = {makeBeginColumnHint("a", 7)}; // Case2.3: a>=7 + check(indices[0], columnHints, {}, "case2.3"); // + columnHints = {makeBeginColumnHint("b", -0x7fffffffffffffff - 1)}; // Case2.4: b>=INT_MIN + check(indices[1], columnHints, expect(4, 0, 2, 1, 3), "case2.4"); // + columnHints = {makeBeginColumnHint("b", 0x7fffffffffffffff)}; // Case2.5: b>=INT_MAX + check(indices[1], columnHints, expect(3), "case2.5"); // + columnHints = {makeBeginColumnHint("b", 0)}; // Case2.6: b>=0 + check(indices[1], columnHints, expect(2, 1, 3), "case2.6"); // + columnHints = {makeBeginColumnHint("c", -0x7fffffffffffffff - 1)}; // Case2.7: c>=INT_MIN + check(indices[2], columnHints, expect(1, 0, 2, 3, 5, 4), "case2.7"); // + columnHints = {makeBeginColumnHint("c", 0x7fffffffffffffff)}; // Case2.8: c>=INT_MAX + check(indices[2], columnHints, expect(4), "case2.8"); // + columnHints = {makeBeginColumnHint("c", 0)}; // Case2.9: c>=0 + check(indices[2], columnHints, expect(3, 5, 4), "case2.9"); // + } // End of Case 2 + { // Case 3: [x, y) + std::vector columnHints; // + columnHints = {makeColumnHint("a", -1, 10)}; // Case3.1: a >= -1 + check(indices[0], columnHints, expect(0, 1, 2, 3, 4, 5), "case3.1"); + columnHints = {makeColumnHint("a", -100, 4)}; + check(indices[0], columnHints, expect(0, 1, 2), "case3.2"); + columnHints = {makeColumnHint("a", 4, 100)}; + check(indices[0], columnHints, expect(3, 4, 5), "case3.3"); + columnHints = {makeColumnHint("a", 2, 5)}; + check(indices[0], columnHints, expect(1, 2, 3), "case3.4"); + columnHints = {makeColumnHint("a", -100, 0)}; + check(indices[0], columnHints, {}, "case3.5"); + columnHints = {makeColumnHint("a", 10, 100)}; + check(indices[0], columnHints, {}, "case3.6"); + columnHints = {makeColumnHint("b", -0x7fffffffffffffff - 1, 0x7fffffffffffffff)}; + check(indices[1], columnHints, expect(4, 0, 2, 1), "case3.7"); + columnHints = {makeColumnHint("c", -0x7fffffffffffffff - 1, 0x7fffffffffffffff)}; + check(indices[2], columnHints, expect(1, 0, 2, 3, 5), "case3.8"); + } // End of Case 3 + { // Case 4: (x, INF) + std::vector columnHints; + columnHints = {makeBeginColumnHint("a", 3)}; + check(indices[0], columnHints, expect(3, 4, 5), "case4.1"); + columnHints = {makeBeginColumnHint("b", MIN)}; + check(indices[1], columnHints, expect(0, 2, 1, 3), "case4.2"); + columnHints = {makeBeginColumnHint("b", MAX)}; + check(indices[1], columnHints, {}, "case4.3"); + columnHints = {makeBeginColumnHint("c", MIN)}; + check(indices[2], columnHints, expect(0, 2, 3, 5, 4), "case4.4"); + columnHints = {makeBeginColumnHint("c", MAX - 1)}; + check(indices[2], columnHints, expect(4), "case4.4"); + } // End of Case 4 + { // Case 5: (x, y] + std::vector columnHints; + columnHints = {makeColumnHint("a", 1, 6)}; + check(indices[0], columnHints, expect(1, 2, 3, 4, 5), "case5.1"); + columnHints = {makeColumnHint("a", 0, 3)}; + check(indices[0], columnHints, expect(0, 1, 2), "case5.2"); + columnHints = {makeColumnHint("b", MIN, MIN)}; + check(indices[1], columnHints, {}, "case5.3"); + columnHints = {makeColumnHint("b", MAX, MAX)}; + check(indices[1], columnHints, {}, "case5.4"); + columnHints = {makeColumnHint("b", 0, MAX)}; + check(indices[1], columnHints, expect(1, 3), "case5.5"); + columnHints = {makeColumnHint("c", -1, MAX)}; + check(indices[2], columnHints, expect(3, 5, 4), "case5.6"); + } // End of Case 5 + { // Case 6: (-INF, y] + std::vector columnHints; + columnHints = {makeEndColumnHint("a", 4)}; + check(indices[0], columnHints, expect(0, 1, 2, 3), "case6.1"); + columnHints = {makeEndColumnHint("a", 1)}; + check(indices[0], columnHints, expect(0), "case6.2"); + columnHints = {makeEndColumnHint("b", MIN)}; + check(indices[1], columnHints, expect(4), "case6.3"); + columnHints = {makeEndColumnHint("b", MAX)}; + check(indices[1], columnHints, expect(4, 0, 2, 1, 3), "casae6.4"); + columnHints = {makeEndColumnHint("c", MIN)}; + check(indices[2], columnHints, expect(1), "case6.5"); + columnHints = {makeEndColumnHint("c", MAX)}; + check(indices[2], columnHints, expect(1, 0, 2, 3, 5, 4), "case6.6"); + } // End of Case 6 + { // Case 7: (-INF, y) + std::vector columnHints; + columnHints = {makeEndColumnHint("a", 4)}; + check(indices[0], columnHints, expect(0, 1, 2), "case7.1"); + columnHints = {makeEndColumnHint("a", 1)}; + check(indices[0], columnHints, {}, "case7.2"); + columnHints = {makeEndColumnHint("b", MIN)}; + check(indices[1], columnHints, {}, "case7.3"); + columnHints = {makeEndColumnHint("b", MAX)}; + check(indices[1], columnHints, expect(4, 0, 2, 1), "casae7.4"); + columnHints = {makeEndColumnHint("c", MIN)}; + check(indices[2], columnHints, {}, "case7.5"); + columnHints = {makeEndColumnHint("c", MAX)}; + check(indices[2], columnHints, expect(1, 0, 2, 3, 5), "case7.6"); + } // End of Case 7 } -// TEST_F(IndexScanTest, VertexIndexScan) -// TEST_F(IndexScanTest, EdgeBase){ - -// }; -// TEST_F(IndexScanTest, Prefix1){ +TEST_F(IndexScanTest, Float) {} +TEST_F(IndexScanTest, Bool) {} +TEST_F(IndexScanTest, String1) {} +TEST_F(IndexScanTest, String2) {} +TEST_F(IndexScanTest, String3) {} +TEST_F(IndexScanTest, String4) {} +TEST_F(IndexScanTest, Time) {} +TEST_F(IndexScanTest, Date) {} +TEST_F(IndexScanTest, DateTime) {} +TEST_F(IndexScanTest, Compound) {} +TEST_F(IndexScanTest, Nullable) {} -// }; -// TEST_F(IndexScanTest, Prefix2){ - -// }; -// TEST_F(IndexScanTest, Range1){ - -// }; -// TEST_F(IndexScanTest, Range2){ - -// }; -// TEST_F(IndexScanTest, Range3){ - -// }; -// TEST_F(IndexScanTest, Range4){ - -// }; -// class IndexTest : public ::testing::Test { -// protected: -// private: -// }; -// TEST_F(IndexTest, VertexScan1) {} -// TEST_F(IndexTest, VertexScan2) {} -// TEST_F(IndexTest, VertexScan3){ - -// }; -// TEST_F(IndexTest, VertexScan4){ - -// }; } // namespace storage } // namespace nebula +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + return RUN_ALL_TESTS(); +} diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index 12a84a1fe8f..f8ec7ff276e 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -23,9 +24,16 @@ class MockKVIterator : public KVIterator { using KVMap = std::map; public: - MockKVIterator(const KVMap& kv_, KVMap::iterator&& iter); - bool valid() const { return iter_ != kv_.end(); } - void next() { iter_++; } + MockKVIterator(const KVMap& kv, KVMap::iterator&& iter) : kv_(kv), iter_(std::move(iter)) {} + bool valid() const { return iter_ != kv_.end() && validFunc_(iter_); } + void next() { + iter_++; + if (iter_ != kv_.end()) { + DVLOG(2) << '\n' << folly::hexDump(iter_->first.data(), iter_->first.size()); + } else { + DVLOG(2) << "InValid!!!"; + } + } void prev() { iter_--; } folly::StringPiece key() const { return folly::StringPiece(iter_->first); } folly::StringPiece val() const { return folly::StringPiece(iter_->second); } @@ -40,17 +48,19 @@ class MockKVIterator : public KVIterator { }; class MockKVStore : public ::nebula::kvstore::KVStore { private: - GraphSpaceID spaceId_; + GraphSpaceID spaceId_{0}; std::map kv_; public: + MockKVStore() {} // Return bit-OR of StoreCapability values; uint32_t capability() const override { assert(false); return 0; }; - void stop() {} - ErrorOr partLeader(GraphSpaceID spaceId, PartitionID partID) { + void stop() override {} + ErrorOr partLeader(GraphSpaceID spaceId, + PartitionID partID) override { UNUSED(spaceId), UNUSED(partID); assert(false); return nebula::cpp2::ErrorCode::SUCCEEDED; @@ -62,6 +72,7 @@ class MockKVStore : public ::nebula::kvstore::KVStore { std::string* value, bool canReadFromFollower = false) override { UNUSED(canReadFromFollower); + UNUSED(partId); CHECK_EQ(spaceId, spaceId_); auto iter = kv_.lower_bound(key); if (iter != kv_.end() && iter->first == key) { @@ -82,6 +93,8 @@ class MockKVStore : public ::nebula::kvstore::KVStore { std::vector* values, bool canReadFromFollower = false) override { UNUSED(canReadFromFollower); + UNUSED(spaceId); + UNUSED(partId); std::vector status; nebula::cpp2::ErrorCode ret = nebula::cpp2::ErrorCode::SUCCEEDED; for (auto& key : keys) { @@ -105,35 +118,69 @@ class MockKVStore : public ::nebula::kvstore::KVStore { const std::string& end, std::unique_ptr* iter, bool canReadFromFollower = false) override { + return range(spaceId, partId, true, start, false, end, iter, canReadFromFollower); + } + nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, + PartitionID partId, + bool includeStart, + const std::string& start, + bool includeEnd, + const std::string& end, + std::unique_ptr* iter, + bool canReadFromFollower = false) override { + UNUSED(spaceId); + UNUSED(partId); + UNUSED(canReadFromFollower); CHECK_EQ(spaceId, spaceId_); - auto it = kv_.lower_bound(start); - auto mockIter = std::make_unique(kv_, std::move(it)); - mockIter->setValidFunc([start, end](const decltype(kv_)::iterator& iter) { - if (start <= iter->first && iter->first < end) { - return true; - } else { - return false; - } + std::unique_ptr mockIter; + if (!includeStart) { + mockIter = std::make_unique(kv_, kv_.upper_bound(start)); + } else { + mockIter = std::make_unique(kv_, kv_.lower_bound(start)); + } + mockIter->setValidFunc([includeEnd, end](const decltype(kv_)::iterator& it) { + DVLOG(2) << includeEnd; + size_t len = end.size(); + int ret = memcmp(it->first.data(), end.data(), len); + DVLOG(2) << ret; + return includeEnd ? ret <= 0 : ret < 0; }); (*iter) = std::move(mockIter); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } - + virtual nebula::cpp2::ErrorCode prefix(GraphSpaceID spaceId, + PartitionID partId, + std::string&& prefix, + std::unique_ptr* iter, + bool canReadFromFollower = false) = delete; + virtual nebula::cpp2::ErrorCode rangeWithPrefix(GraphSpaceID spaceId, + PartitionID partId, + std::string&& start, + std::string&& prefix, + std::unique_ptr* iter, + bool canReadFromFollower = false) = delete; + virtual nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, + PartitionID partId, + std::string&& start, + std::string&& end, + std::unique_ptr* iter, + bool canReadFromFollower = false) = delete; nebula::cpp2::ErrorCode prefix(GraphSpaceID spaceId, PartitionID partId, const std::string& prefix, std::unique_ptr* iter, bool canReadFromFollower = false) override { UNUSED(canReadFromFollower); + UNUSED(spaceId); + UNUSED(partId); CHECK_EQ(spaceId, spaceId_); - auto it = kv_.lower_bound(prefix); - auto mockIter = std::make_unique(kv_, std::move(it)); - mockIter->setValidFunc([prefix](const decltype(kv_)::iterator& iter) { - if (iter->first.size() < prefix.size()) { + auto mockIter = std::make_unique(kv_, kv_.lower_bound(prefix)); + mockIter->setValidFunc([prefix](const decltype(kv_)::iterator& it) { + if (it->first.size() < prefix.size()) { return false; } for (size_t i = 0; i < prefix.size(); i++) { - if (prefix[i] != iter->first[i]) { + if (prefix[i] != it->first[i]) { return false; } } @@ -151,15 +198,16 @@ class MockKVStore : public ::nebula::kvstore::KVStore { std::unique_ptr* iter, bool canReadFromFollower = false) override { UNUSED(canReadFromFollower); + UNUSED(spaceId); + UNUSED(partId); CHECK_EQ(spaceId, spaceId_); - auto it = kv_.lower_bound(start); - auto mockIter = std::make_unique(kv_, std::move(it)); - mockIter->setValidFunc([prefix](const decltype(kv_)::iterator& iter) { - if (iter->first.size() < prefix.size()) { + auto mockIter = std::make_unique(kv_, kv_.lower_bound(start)); + mockIter->setValidFunc([prefix](const decltype(kv_)::iterator& it) { + if (it->first.size() < prefix.size()) { return false; } for (size_t i = 0; i < prefix.size(); i++) { - if (prefix[i] != iter->first[i]) { + if (prefix[i] != it->first[i]) { return false; } } @@ -170,6 +218,8 @@ class MockKVStore : public ::nebula::kvstore::KVStore { } nebula::cpp2::ErrorCode sync(GraphSpaceID spaceId, PartitionID partId) { + UNUSED(spaceId); + UNUSED(partId); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -178,6 +228,9 @@ class MockKVStore : public ::nebula::kvstore::KVStore { PartitionID partId, std::vector<::nebula::kvstore::KV>&& keyValues, ::nebula::kvstore::KVCallback cb) override { + UNUSED(spaceId); + UNUSED(partId); + UNUSED(cb); for (size_t i = 0; i < keyValues.size(); i++) { kv_.emplace(std::move(keyValues[i])); } @@ -188,6 +241,9 @@ class MockKVStore : public ::nebula::kvstore::KVStore { PartitionID partId, const std::string& key, ::nebula::kvstore::KVCallback cb) override { + UNUSED(spaceId); + UNUSED(partId); + UNUSED(cb); kv_.erase(key); } @@ -195,6 +251,9 @@ class MockKVStore : public ::nebula::kvstore::KVStore { PartitionID partId, std::vector&& keys, ::nebula::kvstore::KVCallback cb) override { + UNUSED(spaceId); + UNUSED(partId); + UNUSED(cb); for (size_t i = 0; i < keys.size(); i++) { kv_.erase(keys[i]); } @@ -205,6 +264,9 @@ class MockKVStore : public ::nebula::kvstore::KVStore { const std::string& start, const std::string& end, ::nebula::kvstore::KVCallback cb) override { + UNUSED(spaceId); + UNUSED(partId); + UNUSED(cb); for (auto iter = kv_.lower_bound(start); iter != kv_.end();) { if (iter->first < end) { iter = kv_.erase(iter); @@ -218,50 +280,70 @@ class MockKVStore : public ::nebula::kvstore::KVStore { PartitionID partId, raftex::AtomicOp op, ::nebula::kvstore::KVCallback cb) override { + UNUSED(spaceId); + UNUSED(partId); + UNUSED(cb); + UNUSED(op); LOG(FATAL) << "Unexpect"; } void asyncAppendBatch(GraphSpaceID spaceId, PartitionID partId, std::string&& batch, ::nebula::kvstore::KVCallback cb) override { - LOG(FATAL) << "Unexpect"; + UNUSED(spaceId); + UNUSED(partId); + UNUSED(cb); + LOG(FATAL) << "Unexpect " << batch; } nebula::cpp2::ErrorCode ingest(GraphSpaceID spaceId) override { + UNUSED(spaceId); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } int32_t allLeader( std::unordered_map>& leaderIds) override { + UNUSED(leaderIds); + LOG(FATAL) << "Unexpect"; return 0; } ErrorOr> part( GraphSpaceID spaceId, PartitionID partId) override { + UNUSED(spaceId); + UNUSED(partId); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode compact(GraphSpaceID spaceId) override { + UNUSED(spaceId); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode flush(GraphSpaceID spaceId) override { + UNUSED(spaceId); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } ErrorOr> createCheckpoint( GraphSpaceID spaceId, const std::string& name) override { + UNUSED(spaceId); + UNUSED(name); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; nebula::cpp2::ErrorCode dropCheckpoint(GraphSpaceID spaceId, const std::string& name) { + UNUSED(spaceId); + UNUSED(name); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode setWriteBlocking(GraphSpaceID spaceId, bool sign) { + UNUSED(spaceId); + UNUSED(sign); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -271,17 +353,25 @@ class MockKVStore : public ::nebula::kvstore::KVStore { const std::string& name, const std::string& tablePrefix, std::function filter) { + UNUSED(spaceId); + UNUSED(name); + UNUSED(tablePrefix); + UNUSED(filter); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } // for meta BR nebula::cpp2::ErrorCode restoreFromFiles(GraphSpaceID spaceId, const std::vector& files) { + UNUSED(spaceId); + UNUSED(files); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode multiPutWithoutReplicator(GraphSpaceID spaceId, std::vector<::nebula::kvstore::KV> keyValues) { + UNUSED(spaceId); + UNUSED(keyValues); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -289,6 +379,19 @@ class MockKVStore : public ::nebula::kvstore::KVStore { LOG(FATAL) << "Unexpect"; return {}; } + + ErrorOr getProperty(GraphSpaceID spaceId, + const std::string& property) override { + UNUSED(spaceId); + UNUSED(property); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + } + void put(const std::string& key, const std::string& value) { kv_[key] = value; } + + private: + using ::nebula::kvstore::KVStore::prefix; + using ::nebula::kvstore::KVStore::range; + using ::nebula::kvstore::KVStore::rangeWithPrefix; }; class MockIndexNode : public IndexNode { public: @@ -306,8 +409,7 @@ class MockIndexNode : public IndexNode { class RowParser { public: explicit RowParser(const std::string& str) { - ss = std::stringstream(folly::stripLeftMargin(str)); - ss = std::stringstream(str); + ss = std::stringstream(folly::trimWhitespace(folly::StringPiece(str)).toString()); parseHeader(); parseRow(); } @@ -317,7 +419,7 @@ class RowParser { std::vector types; folly::split("|", line, types); for (size_t i = 0; i < types.size(); i++) { - types[i] = folly::stripLeftMargin(types[i]); + types[i] = folly::trimWhitespace(folly::StringPiece(types[i])).toString(); } typeList_ = std::move(types); } @@ -327,7 +429,7 @@ class RowParser { std::vector values; folly::split("|", line, values); for (size_t i = 0; i < values.size(); i++) { - values[i] = folly::stripLeftMargin(values[i]); + values[i] = folly::trimWhitespace(folly::StringPiece(values[i])).toString(); } Row row; for (size_t i = 0; i < values.size(); i++) { @@ -347,7 +449,7 @@ class RowParser { std::vector typeList_; std::vector rowList_; std::map> transformMap{ - {"int64", [](const std::string& str) { return Value(std::stoi(str)); }}, + {"int", [](const std::string& str) { return Value(std::stol(str)); }}, {"string", [](const std::string& str) { return Value(str); }}, {"double", [](const std::string& str) { return Value(std::stof(str)); }}}; }; @@ -367,7 +469,8 @@ class RowParser { class SchemaParser { public: explicit SchemaParser(const std::string& str) { - ss = std::stringstream(folly::stripLeftMargin(str)); + schema = std::make_shared<::nebula::meta::NebulaSchemaProvider>(2); + ss = std::stringstream(folly::trimWhitespace(folly::StringPiece(str)).toString()); parse(); } void parse() { @@ -375,19 +478,19 @@ class SchemaParser { while (std::getline(ss, line)) { std::vector values; folly::split("|", line, values); - std::string name = folly::stripLeftMargin(values[0]); - auto type = typeMap[folly::stripLeftMargin(values[1])]; + std::string name = folly::trimWhitespace(folly::StringPiece(values[0])).toString(); + auto type = typeMap[folly::trimWhitespace(folly::StringPiece(values[1])).toString()]; int length = 0; { - std::string lenStr = folly::stripLeftMargin(values[2]); + std::string lenStr = folly::trimWhitespace(folly::StringPiece(values[2])).toString(); if (lenStr != "") { length = std::stoi(lenStr); } } bool nullable = false; { - std::string nullable = folly::stripLeftMargin(values[3]); - if (nullable == "true") { + std::string nullableStr = folly::trimWhitespace(folly::StringPiece(values[3])).toString(); + if (nullableStr == "true") { nullable = true; } } @@ -399,7 +502,8 @@ class SchemaParser { private: std::stringstream ss; std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema; - std::map typeMap; + std::map typeMap{ + {"int", ::nebula::meta::cpp2::PropertyType::INT64}}; }; /** @@ -419,7 +523,7 @@ class IndexParser { using IndexItem = ::nebula::meta::cpp2::IndexItem; using SchemaProvider = ::nebula::meta::NebulaSchemaProvider; explicit IndexParser(const std::string& str) { - ss = std::stringstream(folly::stripLeftMargin(str)); + ss = std::stringstream(folly::trimWhitespace(folly::StringPiece(str)).toString()); parseSchema(); } void parseSchema() { @@ -437,13 +541,13 @@ class IndexParser { schemaId_.set_edge_type(id); } } - std::map> operator()(std::shared_ptr schema) { + std::vector> operator()(std::shared_ptr schema) { schema_ = schema; - std::map> ret; + std::vector> ret; std::string line; while (std::getline(ss, line)) { - auto index = parse(line); - ret[index->get_index_id()] = index; + auto index = parse(folly::trimWhitespace(folly::StringPiece(line)).toString()); + ret.push_back(index); } return ret; } @@ -454,23 +558,23 @@ class IndexParser { static std::regex pattern(R"(\((.+),(\d+)\):(.+))"); std::smatch match; assert(std::regex_match(line, match, pattern)); - ret->set_index_name(folly::stripLeftMargin(match.str(1))); + ret->set_index_name(folly::trimWhitespace(folly::StringPiece(match.str(1)).toString())); ret->set_index_id(std::stoi(match.str(2))); std::string columnStr = match.str(3); std::vector columns; folly::split(",", columnStr, columns); for (size_t i = 0; i < columns.size(); i++) { - columns[i] = folly::stripLeftMargin(columns[i]); + columns[i] = folly::trimWhitespace(folly::StringPiece(columns[i])).toString(); } std::vector<::nebula::meta::cpp2::ColumnDef> fields; for (auto& column : columns) { std::string name; int length; - std::smatch match; - std::regex pattern(R"((.+)\((\d+)\))"); - if (std::regex_match(column, match, pattern)) { - name = match.str(1); - length = std::stoi(match.str(2)); + std::smatch m; + std::regex p(R"((.+)\((\d+)\))"); + if (std::regex_match(column, m, p)) { + name = m.str(1); + length = std::stoi(m.str(2)); } else { name = column; length = 0; From 97d5cdc5417471deb97b989a4b39e5e3e289510e Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Wed, 13 Oct 2021 15:41:24 +0800 Subject: [PATCH 11/38] add double/string1 test --- src/common/utils/IndexKeyUtils.h | 33 +- src/storage/exec/IndexScanNode2.cpp | 68 ++- src/storage/exec/IndexScanNode2.h | 2 + src/storage/test/IndexTest.cpp | 703 +++++++++++++++++++++++----- src/storage/test/IndexTestUtil.h | 30 +- 5 files changed, 701 insertions(+), 135 deletions(-) diff --git a/src/common/utils/IndexKeyUtils.h b/src/common/utils/IndexKeyUtils.h index 23d699abce8..82027a9d6f5 100644 --- a/src/common/utils/IndexKeyUtils.h +++ b/src/common/utils/IndexKeyUtils.h @@ -7,12 +7,13 @@ #ifndef COMMON_UTILS_INDEXKEYUTILS_H_ #define COMMON_UTILS_INDEXKEYUTILS_H_ +#include + #include "codec/RowReader.h" #include "common/base/Base.h" #include "common/base/StatusOr.h" #include "common/utils/Types.h" #include "interface/gen-cpp2/meta_types.h" - namespace nebula { using PropertyType = nebula::meta::cpp2::PropertyType; @@ -197,6 +198,26 @@ class IndexKeyUtils final { */ static std::string encodeDouble(double v) { + if (std::isnan(v)) { + return std::string(sizeof(double), '\xFF'); + } else if (v >= 0) { + auto val = folly::Endian::big(v); + auto* c = reinterpret_cast(&val); + c[0] |= 0x80; + std::string raw; + raw.reserve(sizeof(double)); + raw.append(c, sizeof(double)); + return raw; + } else { + int64_t* x = reinterpret_cast(&v); + *x = ~(*x); + auto val = folly::Endian::big(v); + auto* c = reinterpret_cast(&val); + std::string raw; + raw.reserve(sizeof(double)); + raw.append(c, sizeof(double)); + return raw; + } if (v < 0) { /** * TODO : now, the -(std::numeric_limits::min()) @@ -218,6 +239,16 @@ class IndexKeyUtils final { } static double decodeDouble(const folly::StringPiece& raw) { + { + int64_t val = *reinterpret_cast(raw.data()); + val = folly::Endian::big(val); + if (val < 0) { + val &= 0x7fffffffffffffff; + } else { + val = ~val; + } + return *reinterpret_cast(&val); + } char* v = const_cast(raw.data()); v[0] ^= 0x80; auto val = *reinterpret_cast(v); diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index 15d986e885c..542739db6a7 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -17,7 +17,15 @@ Path::Path(nebula::meta::cpp2::IndexItem* index, bool tmp = field.nullable_ref().value_or(false); nullable_.push_back(tmp); nullFlag |= tmp; + // TODO: improve performance of compute nullable offset in index_key + auto type = IndexKeyUtils::toValueType(field.get_type().get_type()); + auto tmpStr = IndexKeyUtils::encodeNullValue(type, field.get_type().get_type_length()); + index_nullable_offset_ += tmpStr.size(); } + for (auto x : nullable_) { + DVLOG(1) << x; + } + DVLOG(1) << nullFlag; if (!nullFlag) { nullable_.clear(); } @@ -55,14 +63,16 @@ std::string Path::encodeValue(const Value& value, val = IndexKeyUtils::encodeValue(value); } if (!nullable_.empty() && nullable_[index] == true) { - QFList_.emplace_back([isNull = value.isNull(), index](const std::string& k) { - std::bitset<16> nullableBit; - size_t len = k.size(); - auto v = *reinterpret_cast(k.data() + len - 2); - nullableBit = v; - return nullableBit.test(15 - index) ^ isNull ? Qualified::INCOMPATIBLE - : Qualified::COMPATIBLE; - }); + QFList_.emplace_back( + [isNull = value.isNull(), index, offset = index_nullable_offset_](const std::string& k) { + std::bitset<16> nullableBit; + auto v = *reinterpret_cast(k.data() + offset); + nullableBit = v; + DVLOG(3) << isNull; + DVLOG(3) << nullableBit.test(15 - index); + return nullableBit.test(15 - index) ^ isNull ? Qualified::INCOMPATIBLE + : Qualified::COMPATIBLE; + }); } key.append(val); return val; @@ -155,19 +165,30 @@ std::string RangePath::encodeBeginValue(const Value& value, startPos = key.size(), length = val.size()](const std::string& k) { int ret = memcmp(startKey.data() + startPos, k.data() + startPos, length); + DVLOG(3) << '\n' << folly::hexDump(startKey.data() + startPos, length); + DVLOG(3) << '\n' << folly::hexDump(k.data() + startPos, length); CHECK_LE(ret, 0); return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); } - if (!nullable_.empty() && nullable_[index] == true) { - QFList_.emplace_back([index](const std::string& k) { + if (value.isFloat()) { + QFList_.emplace_back([startPos = key.size(), length = val.size()](const std::string& k) { + std::string s(length, '\xFF'); + return memcmp(k.data() + startPos, s.data(), length) == 0 ? Qualified::INCOMPATIBLE + : Qualified::COMPATIBLE; + }); + } else if (!nullable_.empty() && nullable_[index] == true) { + QFList_.emplace_back([index, offset = index_nullable_offset_](const std::string& k) { + DVLOG(1) << "check null"; std::bitset<16> nullableBit; - size_t len = k.size(); - auto v = *reinterpret_cast(k.data() + len - 2); + auto v = *reinterpret_cast(k.data() + offset); nullableBit = v; + DVLOG(1) << nullableBit; + DVLOG(1) << nullableBit.test(15 - index); return nullableBit.test(15 - index) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); } + key += val; return val; } std::string RangePath::encodeEndValue(const Value& value, @@ -192,11 +213,16 @@ std::string RangePath::encodeEndValue(const Value& value, return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); } - if (!nullable_.empty() && nullable_[index] == true) { - QFList_.emplace_back([index](const std::string& k) { + if (value.isFloat()) { + QFList_.emplace_back([startPos = key.size(), length = val.size()](const std::string& k) { + std::string s(length, '\xFF'); + return memcmp(k.data() + startPos, s.data(), length) == 0 ? Qualified::INCOMPATIBLE + : Qualified::COMPATIBLE; + }); + } else if (!nullable_.empty() && nullable_[index] == true) { + QFList_.emplace_back([index, offset = index_nullable_offset_](const std::string& k) { std::bitset<16> nullableBit; - size_t len = k.size(); - auto v = *reinterpret_cast(k.data() + len - 2); + auto v = *reinterpret_cast(k.data() + offset); nullableBit = v; return nullableBit.test(15 - index) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); @@ -234,7 +260,7 @@ void PrefixPath::buildKey() { auto& hint = hints_[i]; CHECK(fieldIter->get_name() == hint.get_column_name()); auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); - CHECK(type != Value::Type::STRING || !fieldIter->get_type().type_length_ref().has_value()); + CHECK(type != Value::Type::STRING || fieldIter->get_type().type_length_ref().has_value()); encodeValue(hint.get_begin_value(), fieldIter->get_type(), i, common); } prefix_ = std::move(common); @@ -265,7 +291,7 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { // TODO(hs.zhang): The performance is better to judge based on whether the string is truncated auto tmp = ctx.requiredColumns; for (auto& field : index_->get_fields()) { - if (field.get_type().get_type() == PropertyType::STRING) { + if (field.get_type().get_type() == PropertyType::FIXED_STRING) { continue; } tmp.erase(field.get_name()); @@ -306,7 +332,11 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { std::pair kv; auto ret = getBaseData(iter_->key(), kv); if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - DVLOG(3) << 123; + if (LIKELY(!fatalOnBaseNotFound_)) { + LOG(WARNING) << "base data not found"; + } else { + LOG(FATAL) << "base data not found"; + } continue; } Map rowData = decodeFromBase(kv.first, kv.second); diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index becc4bd4926..e0d8508e401 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -62,6 +62,7 @@ class IndexScanNode : public IndexNode { std::vector requiredColumns_; std::pair> ttlProps_; bool needAccessBase_{false}; + bool fatalOnBaseNotFound_{false}; }; /** * Path @@ -97,6 +98,7 @@ class Path { const meta::SchemaProviderIf* schema_; const std::vector& hints_; std::vector nullable_; + int64_t index_nullable_offset_{8}; }; class PrefixPath : public Path { public: diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 14afeac4e14..f8174a83268 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -6,6 +6,8 @@ #include +#include + #include "codec/RowReaderWrapper.h" #include "codec/RowWriterV2.h" #include "common/utils/NebulaKeyUtils.h" @@ -74,6 +76,7 @@ struct IndexScanTestHelper { std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema) { node->getEdge = [schema]() { return schema; }; } + void setFatal(IndexScanNode* node, bool value) { node->fatalOnBaseNotFound_ = value; } }; class IndexScanTest : public ::testing::Test { protected: @@ -135,6 +138,7 @@ class IndexScanTest : public ::testing::Test { for (size_t j = 0; j < indices.size(); j++) { auto& index = indices[j]; auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index->get_fields()).value(); + DVLOG(1) << folly::hexDump(indexValue.data(), indexValue.size()); auto indexKey = IndexKeyUtils::vertexIndexKey( 8, 0, index->get_index_id(), std::to_string(i), std::move(indexValue)); assert(ret[j + 1].insert({indexKey, ""}).second); @@ -513,7 +517,7 @@ TEST_F(IndexScanTest, Int) { const std::vector& columnHints, const std::vector& expect, const std::string& case_) { - DVLOG(2) << "Start case " << case_; + DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), 0, columnHints); IndexScanTestHelper helper; @@ -549,120 +553,603 @@ TEST_F(IndexScanTest, Int) { }; const int64_t MAX = 0x7fffffffffffffff; const int64_t MIN = -MAX - 1; - { // Case 1: prefix - std::vector columnHints = {makeColumnHint("a", 1)}; // a=1; - check(indices[0], columnHints, expect(0), "case1.1"); // - columnHints = {makeColumnHint("b", 0x7fffffffffffffff)}; // b=MAX - check(indices[1], columnHints, expect(3), "case1.2"); // - columnHints = {makeColumnHint("b", -0x7fffffffffffffff - 1)}; // b=MIN - check(indices[1], columnHints, expect(4), "case1.3"); // - columnHints = {makeColumnHint("c", 0)}; // c=0 - check(indices[2], columnHints, expect(3, 5), "case1.4"); // - } // End of Case 1 - { // Case 2: [x, INF) + /* Case 1: Prefix */ + { + std::vector columnHints = {makeColumnHint("a", 1)}; // a=1; + check(indices[0], columnHints, expect(0), "case1.1"); // + columnHints = {makeColumnHint("b", MAX)}; // b=MAX + check(indices[1], columnHints, expect(3), "case1.2"); // + columnHints = {makeColumnHint("b", MIN)}; // b=MIN + check(indices[1], columnHints, expect(4), "case1.3"); // + columnHints = {makeColumnHint("c", 0)}; // c=0 + check(indices[2], columnHints, expect(3, 5), "case1.4"); // + } // End of Case 1 + /* Case 2: [x, INF) */ + { std::vector columnHints = {makeBeginColumnHint("a", -1)}; // Case2.1: a >= -1 check(indices[0], columnHints, expect(0, 1, 2, 3, 4, 5), "case2.1"); // columnHints = {makeBeginColumnHint("a", 4)}; // Case2.2: a>=4 check(indices[0], columnHints, expect(3, 4, 5), "case2.2"); // columnHints = {makeBeginColumnHint("a", 7)}; // Case2.3: a>=7 check(indices[0], columnHints, {}, "case2.3"); // - columnHints = {makeBeginColumnHint("b", -0x7fffffffffffffff - 1)}; // Case2.4: b>=INT_MIN - check(indices[1], columnHints, expect(4, 0, 2, 1, 3), "case2.4"); // - columnHints = {makeBeginColumnHint("b", 0x7fffffffffffffff)}; // Case2.5: b>=INT_MAX - check(indices[1], columnHints, expect(3), "case2.5"); // - columnHints = {makeBeginColumnHint("b", 0)}; // Case2.6: b>=0 - check(indices[1], columnHints, expect(2, 1, 3), "case2.6"); // - columnHints = {makeBeginColumnHint("c", -0x7fffffffffffffff - 1)}; // Case2.7: c>=INT_MIN - check(indices[2], columnHints, expect(1, 0, 2, 3, 5, 4), "case2.7"); // - columnHints = {makeBeginColumnHint("c", 0x7fffffffffffffff)}; // Case2.8: c>=INT_MAX - check(indices[2], columnHints, expect(4), "case2.8"); // - columnHints = {makeBeginColumnHint("c", 0)}; // Case2.9: c>=0 - check(indices[2], columnHints, expect(3, 5, 4), "case2.9"); // - } // End of Case 2 - { // Case 3: [x, y) - std::vector columnHints; // - columnHints = {makeColumnHint("a", -1, 10)}; // Case3.1: a >= -1 - check(indices[0], columnHints, expect(0, 1, 2, 3, 4, 5), "case3.1"); - columnHints = {makeColumnHint("a", -100, 4)}; - check(indices[0], columnHints, expect(0, 1, 2), "case3.2"); - columnHints = {makeColumnHint("a", 4, 100)}; - check(indices[0], columnHints, expect(3, 4, 5), "case3.3"); - columnHints = {makeColumnHint("a", 2, 5)}; - check(indices[0], columnHints, expect(1, 2, 3), "case3.4"); - columnHints = {makeColumnHint("a", -100, 0)}; - check(indices[0], columnHints, {}, "case3.5"); - columnHints = {makeColumnHint("a", 10, 100)}; - check(indices[0], columnHints, {}, "case3.6"); - columnHints = {makeColumnHint("b", -0x7fffffffffffffff - 1, 0x7fffffffffffffff)}; - check(indices[1], columnHints, expect(4, 0, 2, 1), "case3.7"); - columnHints = {makeColumnHint("c", -0x7fffffffffffffff - 1, 0x7fffffffffffffff)}; - check(indices[2], columnHints, expect(1, 0, 2, 3, 5), "case3.8"); - } // End of Case 3 - { // Case 4: (x, INF) - std::vector columnHints; - columnHints = {makeBeginColumnHint("a", 3)}; - check(indices[0], columnHints, expect(3, 4, 5), "case4.1"); - columnHints = {makeBeginColumnHint("b", MIN)}; - check(indices[1], columnHints, expect(0, 2, 1, 3), "case4.2"); - columnHints = {makeBeginColumnHint("b", MAX)}; - check(indices[1], columnHints, {}, "case4.3"); - columnHints = {makeBeginColumnHint("c", MIN)}; - check(indices[2], columnHints, expect(0, 2, 3, 5, 4), "case4.4"); - columnHints = {makeBeginColumnHint("c", MAX - 1)}; - check(indices[2], columnHints, expect(4), "case4.4"); - } // End of Case 4 - { // Case 5: (x, y] - std::vector columnHints; - columnHints = {makeColumnHint("a", 1, 6)}; - check(indices[0], columnHints, expect(1, 2, 3, 4, 5), "case5.1"); - columnHints = {makeColumnHint("a", 0, 3)}; - check(indices[0], columnHints, expect(0, 1, 2), "case5.2"); - columnHints = {makeColumnHint("b", MIN, MIN)}; - check(indices[1], columnHints, {}, "case5.3"); - columnHints = {makeColumnHint("b", MAX, MAX)}; - check(indices[1], columnHints, {}, "case5.4"); - columnHints = {makeColumnHint("b", 0, MAX)}; - check(indices[1], columnHints, expect(1, 3), "case5.5"); - columnHints = {makeColumnHint("c", -1, MAX)}; - check(indices[2], columnHints, expect(3, 5, 4), "case5.6"); - } // End of Case 5 - { // Case 6: (-INF, y] - std::vector columnHints; - columnHints = {makeEndColumnHint("a", 4)}; - check(indices[0], columnHints, expect(0, 1, 2, 3), "case6.1"); - columnHints = {makeEndColumnHint("a", 1)}; - check(indices[0], columnHints, expect(0), "case6.2"); - columnHints = {makeEndColumnHint("b", MIN)}; - check(indices[1], columnHints, expect(4), "case6.3"); - columnHints = {makeEndColumnHint("b", MAX)}; - check(indices[1], columnHints, expect(4, 0, 2, 1, 3), "casae6.4"); - columnHints = {makeEndColumnHint("c", MIN)}; - check(indices[2], columnHints, expect(1), "case6.5"); - columnHints = {makeEndColumnHint("c", MAX)}; - check(indices[2], columnHints, expect(1, 0, 2, 3, 5, 4), "case6.6"); - } // End of Case 6 - { // Case 7: (-INF, y) - std::vector columnHints; - columnHints = {makeEndColumnHint("a", 4)}; - check(indices[0], columnHints, expect(0, 1, 2), "case7.1"); - columnHints = {makeEndColumnHint("a", 1)}; - check(indices[0], columnHints, {}, "case7.2"); - columnHints = {makeEndColumnHint("b", MIN)}; - check(indices[1], columnHints, {}, "case7.3"); - columnHints = {makeEndColumnHint("b", MAX)}; - check(indices[1], columnHints, expect(4, 0, 2, 1), "casae7.4"); - columnHints = {makeEndColumnHint("c", MIN)}; - check(indices[2], columnHints, {}, "case7.5"); - columnHints = {makeEndColumnHint("c", MAX)}; - check(indices[2], columnHints, expect(1, 0, 2, 3, 5), "case7.6"); - } // End of Case 7 + columnHints = {makeBeginColumnHint("b", MIN)}; // Case2.4: b>=INT_MIN + check(indices[1], columnHints, expect(4, 0, 2, 1, 3), "case2.4"); // + columnHints = {makeBeginColumnHint("b", MAX)}; // Case2.5: b>=INT_MAX + check(indices[1], columnHints, expect(3), "case2.5"); // + columnHints = {makeBeginColumnHint("b", 0)}; // Case2.6: b>=0 + check(indices[1], columnHints, expect(2, 1, 3), "case2.6"); // + columnHints = {makeBeginColumnHint("c", MIN)}; // Case2.7: c>=INT_MIN + check(indices[2], columnHints, expect(1, 0, 2, 3, 5, 4), "case2.7"); // + columnHints = {makeBeginColumnHint("c", MAX)}; // Case2.8: c>=INT_MAX + check(indices[2], columnHints, expect(4), "case2.8"); // + columnHints = {makeBeginColumnHint("c", 0)}; // Case2.9: c>=0 + check(indices[2], columnHints, expect(3, 5, 4), "case2.9"); // + } // End of Case 2 + /* Case 3: [x, y) */ + { + std::vector columnHints; // + columnHints = {makeColumnHint("a", -1, 10)}; // Case3.1: -1<=a<10 + check(indices[0], columnHints, expect(0, 1, 2, 3, 4, 5), "case3.1"); // + columnHints = {makeColumnHint("a", -100, 4)}; // Case3.2: -100<=a<4 + check(indices[0], columnHints, expect(0, 1, 2), "case3.2"); // + columnHints = {makeColumnHint("a", 4, 100)}; // Case3.3: 4<=a<100 + check(indices[0], columnHints, expect(3, 4, 5), "case3.3"); // + columnHints = {makeColumnHint("a", 2, 5)}; // Case3.4: 2<=a<5 + check(indices[0], columnHints, expect(1, 2, 3), "case3.4"); // + columnHints = {makeColumnHint("a", -100, 0)}; // Case3.5: -100<=a<0 + check(indices[0], columnHints, {}, "case3.5"); // + columnHints = {makeColumnHint("a", 10, 100)}; // Case3.6: 10<=a<100 + check(indices[0], columnHints, {}, "case3.6"); // + columnHints = {makeColumnHint("b", MIN, MAX)}; // Case3.7: MIN<=b("c", MIN, MAX)}; // Case3.8: MIN<=c columnHints; // + columnHints = {makeBeginColumnHint("a", 3)}; // Case 4.1: a>3 + check(indices[0], columnHints, expect(3, 4, 5), "case4.1"); // + columnHints = {makeBeginColumnHint("b", MIN)}; // Case 4.2: b>MIN + check(indices[1], columnHints, expect(0, 2, 1, 3), "case4.2"); // + columnHints = {makeBeginColumnHint("b", MAX)}; // Case4.3: b>MAX + check(indices[1], columnHints, {}, "case4.3"); // + columnHints = {makeBeginColumnHint("c", MIN)}; // Case4.4: c>MIN + check(indices[2], columnHints, expect(0, 2, 3, 5, 4), "case4.4"); // + columnHints = {makeBeginColumnHint("c", MAX - 1)}; // Case4.5: c>MAX-1 + check(indices[2], columnHints, expect(4), "case4.4"); // + } // End of Case 4 + /* Case 5: (x, y] */ + { + std::vector columnHints; // + columnHints = {makeColumnHint("a", 1, 6)}; // Case5.1: 1("a", 0, 3)}; // Case5.2: 0("b", MIN, MIN)}; // Case5.3: MIN("b", MAX, MAX)}; // Case5.4: MAX("b", 0, MAX)}; // Case5.5: 0("c", -1, MAX)}; // Case5.6: -1 columnHints; // + columnHints = {makeEndColumnHint("a", 4)}; // Case6.1: a<=4 + check(indices[0], columnHints, expect(0, 1, 2, 3), "case6.1"); // + columnHints = {makeEndColumnHint("a", 1)}; // Case6.2: a<=1 + check(indices[0], columnHints, expect(0), "case6.2"); // + columnHints = {makeEndColumnHint("b", MIN)}; // Case6.3: b<=MIN + check(indices[1], columnHints, expect(4), "case6.3"); // + columnHints = {makeEndColumnHint("b", MAX)}; // Case6.4: b<=MAX + check(indices[1], columnHints, expect(4, 0, 2, 1, 3), "casae6.4"); // + columnHints = {makeEndColumnHint("c", MIN)}; // Case6.5: c<=MIN + check(indices[2], columnHints, expect(1), "case6.5"); // + columnHints = {makeEndColumnHint("c", MAX)}; // Case6.6: c<=MAX + check(indices[2], columnHints, expect(1, 0, 2, 3, 5, 4), "case6.6"); // + } // End of Case 6 + /* Case 7: (-INF, y) */ + { + std::vector columnHints; // + columnHints = {makeEndColumnHint("a", 4)}; // Case7.1: a<4 + check(indices[0], columnHints, expect(0, 1, 2), "case7.1"); // + columnHints = {makeEndColumnHint("a", 1)}; // Case7.2: a<1 + check(indices[0], columnHints, {}, "case7.2"); // + columnHints = {makeEndColumnHint("b", MIN)}; // Case7.3: b("b", MAX)}; // Case7.4: b("c", MIN)}; // Case7.5: c("c", MAX)}; // Case7.6: c::infinity(); + auto rows = R"( +float | float | float | int +-100.0 | 0.0 | | 0 +-20.0 | -0.0 | | 1 +-5.0 | | 1.7976931348623157e+308 | 2 +-0.0 | <-INF> | 1.7976931348623157e+308 | 3 +0.0 | | | 4 +1.234e10 | <-NaN> | | 5 +5.0 | 4.9406564584124654e-324 | <-INF> | 6 +20.0 | -4.9406564584124654e-324 | <-INF> | 7 +100.0 | 2.2250738585072009e-308 | | 8 +1.2345e10 | -2.2250738585072009e-308 | | 9 +-7e-10 | 2.2250738585072014e-308 | <-NaN> | 10 +7e10 | -2.2250738585072014e-308 | <-NaN> | 11 +-7e10 | 1.7976931348623157e+308 | -0.0 | 12 +7e-10 | -1.7976931348623157e+308 | 0.0 | 13 + )"_row; + auto schema = R"( + a | double | | false + b | double | | false + c | double | | true + )"_schema; + auto indices = R"( + EDGE(e,1) + (i1,2):a + (i2,3):b + (i3,4):c + )"_index(schema); + auto kv = encodeEdge(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + for (auto& iter : kv) { + for (auto& item : iter) { + kvstore->put(item.first, item.second); + } + } + auto check = [&](std::shared_ptr index, + const std::vector& columnHints, + const std::vector& expect, + const std::string& case_) { + DVLOG(1) << "Start case " << case_; + auto context = makeContext(0, 1); + auto scanNode = std::make_unique(context.get(), 0, columnHints); + IndexScanTestHelper helper; + helper.setKVStore(scanNode.get(), kvstore.get()); + helper.setIndex(scanNode.get(), index); + helper.setEdge(scanNode.get(), schema); + InitContext initCtx; + initCtx.requiredColumns = {kSrc}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + EXPECT_EQ(result, expect) << "Fail at case " << case_; + DVLOG(1) << "End case " << case_; + }; + auto expect = [](auto... vidList) { + std::vector ret; + std::vector value; + (value.push_back(std::to_string(vidList)), ...); + for (auto& v : value) { + Row row; + row.emplace_back(v); + ret.emplace_back(std::move(row)); + } + return ret; + }; + /* Case 1: prefix */ { + auto hint = [](const char* name, double value) { + return std::vector{makeColumnHint(name, value)}; + }; + check(indices[0], hint("a", 1000.0), {}, "case1.1"); // Case1.1: a=1000.0 + check(indices[0], hint("a", 0.0), expect(3, 4), "case1.2"); // Case1.2: a=0.0 + check(indices[1], hint("b", MAX_NV), expect(12), "case1.3"); // Case1.3: b=MAX_NV + check(indices[1], hint("b", MIN_NV), expect(10), "case1.4"); // Case1.4: b=MIN_NV + check(indices[1], hint("b", MAX_SV), expect(8), "case1.5"); // Case1.5: b=MAX_SV + check(indices[1], hint("b", MIN_SV), expect(6), "case1.6"); // Case1.6: b=MIN_SV + check(indices[1], hint("b", -MAX_NV), expect(13), "case1.7"); // Case1.7: b=-MAX_NV + check(indices[1], hint("b", -MIN_NV), expect(11), "case1.8"); // Case1.8: b=-MIN_NV + check(indices[1], hint("b", -MAX_SV), expect(9), "case1.9"); // Case1.9: b=-MAX_SV + check(indices[1], hint("b", -MIN_SV), expect(7), "case1.10"); // Case1.10 b=-MIN_SV + check(indices[1], hint("b", 0.0), expect(0, 1), "case1.11"); // Case1.11: b=0.0 + check(indices[1], hint("b", -0.0), expect(0, 1), "case1.12"); // Case1.12: b=-0.0 + check(indices[1], hint("b", INF), expect(2), "case1.13"); // Case1.13: b= + check(indices[1], hint("b", -INF), expect(3), "case1.14"); // Case1.14: b=<-INF> + check(indices[2], hint("c", INF), expect(4, 5), "case1.15"); // Case1.15: c= + } // End of Case 1 + // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 + auto aOrder = expect(12, 0, 1, 2, 10, 3, 4, 13, 6, 7, 8, 5, 9, 11); + auto bOrder = expect(3, 13, 11, 9, 7, 0, 1, 6, 8, 10, 12, 2); + auto cOrder = expect(6, 7, 12, 13, 2, 3, 4, 5); + /* Case 2: [x, INF) */ { + auto hint = [](const char* name, double value) { + return std::vector{makeBeginColumnHint(name, value)}; + }; + auto slice = [](decltype(aOrder) all, size_t start) { + return decltype(all){all.begin() + start, all.end()}; + }; + check(indices[0], hint("a", -100.0), slice(aOrder, 1), "case2.1"); // Case 2.1: a>=-100 + check(indices[0], hint("a", 0.0), slice(aOrder, 5), "case2.2"); // Case 2.2: a>=0.0 + // Case 2.3~2.14: a>={each of $val} + std::vector val{ + -INF, -MAX_NV, -MIN_NV, -MAX_SV, -MIN_SV, -0.0, 0.0, MIN_SV, MAX_SV, MIN_NV, MAX_NV, INF}; + for (size_t i = 0; i < val.size(); i++) { + std::string case_ = fmt::format("case2.{}", i + 3); + auto offset = i; + if (val[i] == 0 && val[i - 1] == 0) { + offset--; + } + check(indices[1], hint("b", val[i]), slice(bOrder, offset), case_); + } + check(indices[2], hint("c", -INF), slice(cOrder, 0), "case2.15"); + check(indices[2], hint("c", 0.0), slice(cOrder, 2), "case2.16"); + check(indices[2], hint("c", MAX_NV), slice(cOrder, 4), "case2.17"); + check(indices[2], hint("c", INF), slice(cOrder, 6), "case2.18"); + } + /* Case 3: [x, y)*/ { + auto hint = [](const char* name, double left, double right) { + return std::vector{makeColumnHint(name, left, right)}; + }; + auto slice = [](decltype(aOrder) all, size_t start, size_t end) { + return decltype(all){all.begin() + start, all.begin() + std::min(end, all.size())}; + }; + check( + indices[0], hint("a", -100.0, -0.0), slice(aOrder, 1, 5), "case3.1"); // Case3.1:-100<=a<0 + check(indices[0], hint("a", 10, 1e9), slice(aOrder, 9, 11), "case3.2"); + check(indices[0], hint("a", 1, 2), {}, "case3.3"); + check(indices[0], hint("a", -INF, INF), aOrder, "case3.4"); + check(indices[0], hint("a", -INF, 0), slice(aOrder, 0, 5), "case3.5"); + check(indices[0], hint("a", 0, INF), slice(aOrder, 5, 14), "case3.6"); + // Case 3.7~3.18: b<{each of $val} + std::vector val{ + -INF, -MAX_NV, -MIN_NV, -MAX_SV, -MIN_SV, -0.0, 0.0, MIN_SV, MAX_SV, MIN_NV, MAX_NV, INF}; + for (size_t i = 0; i < val.size(); i++) { + std::string case_ = fmt::format("case3.{}", i + 7); + auto offset = i; + if (val[i] == 0 && val[i - 1] == 0) { + offset--; + } + check(indices[1], hint("b", -INF, val[i]), slice(bOrder, 0, offset), case_); + } + check(indices[2], hint("c", -INF, INF), slice(cOrder, 0, 6), "case3.19"); + } + /* Case 4: (x, INF)*/ { + auto hint = [](const char* name, double value) { + return std::vector{makeBeginColumnHint(name, value)}; + }; + auto slice = [](decltype(aOrder) all, size_t start) { + return decltype(all){all.begin() + start, all.end()}; + }; + check(indices[0], hint("a", 100), slice(aOrder, 11), "case4.1"); + check(indices[1], hint("b", INF), {}, "case4.2"); + int64_t x = *reinterpret_cast(&INF); + x--; + double y = *reinterpret_cast(&x); + check(indices[1], hint("b", y), slice(bOrder, 11), "case4.3"); + check(indices[2], hint("c", INF), {}, "case4.4"); + check(indices[2], hint("c", y), slice(cOrder, 6), "case4.5"); + } /* Case 5: (x, y]*/ + { + auto hint = [](const char* name, double left, double right) { + return std::vector{makeColumnHint(name, left, right)}; + }; + auto slice = [](decltype(aOrder) all, size_t start, size_t end) { + return decltype(all){all.begin() + start, all.begin() + end}; + }; + check( + indices[0], hint("a", -100.0, -0.0), slice(aOrder, 2, 7), "case5.1"); // Case3.1:-100<=a<0 + check(indices[0], hint("a", 10, 1e9), slice(aOrder, 9, 11), "case5.2"); + check(indices[0], hint("a", 1, 2), {}, "case5.3"); + check(indices[0], hint("a", -INF, INF), aOrder, "case5.4"); + check(indices[0], hint("a", -INF, 0), slice(aOrder, 0, 7), "case5.5"); + check(indices[0], hint("a", 0, INF), slice(aOrder, 7, 14), "case5.6"); + // Case 5.7~5.18: b>{each of $val} + std::vector val{ + -INF, -MAX_NV, -MIN_NV, -MAX_SV, -MIN_SV, -0.0, 0.0, MIN_SV, MAX_SV, MIN_NV, MAX_NV, INF}; + for (size_t i = 0; i < val.size(); i++) { + std::string case_ = fmt::format("case5.{}", i + 7); + auto offset = i + 1; + if (val[i] == 0 && val[i + 1] == 0) { + offset++; + } + check(indices[1], hint("b", val[i], INF), slice(bOrder, offset, bOrder.size()), case_); + } + check(indices[2], hint("c", -INF, INF), slice(cOrder, 2, 8), "case5.19"); + } /* Case 6: (-INF, y]*/ + { + auto hint = [](const char* name, double value) { + return std::vector{makeEndColumnHint(name, value)}; + }; + auto slice = [](decltype(aOrder) all, size_t end) { + return decltype(all){all.begin(), all.begin() + end}; + }; + check(indices[0], hint("a", 0), slice(aOrder, 7), "case6.1"); + check(indices[0], hint("a", -0.0), slice(aOrder, 7), "case6.2"); + check(indices[0], hint("a", -100.0), slice(aOrder, 2), "case6.3"); + // Case 6.4~6.15 + std::vector val{ + -INF, -MAX_NV, -MIN_NV, -MAX_SV, -MIN_SV, -0.0, 0.0, MIN_SV, MAX_SV, MIN_NV, MAX_NV, INF}; + for (size_t i = 0; i < val.size(); i++) { + std::string case_ = fmt::format("case6.{}", i + 3); + auto offset = i + 1; + if (val[i] == 0 && val[i + 1] == 0) { + offset++; + } + check(indices[1], hint("b", val[i]), slice(bOrder, offset), case_); + } + check(indices[2], hint("c", INF), cOrder, "case6.16"); + } + /* Case 7: (-INF, y)*/ { + auto hint = [](const char* name, double value) { + return std::vector{makeEndColumnHint(name, value)}; + }; + auto slice = [](decltype(aOrder) all, size_t end) { + return decltype(all){all.begin(), all.begin() + end}; + }; + check(indices[0], hint("a", 100), slice(aOrder, 10), "case7.1"); + check(indices[1], hint("b", -INF), {}, "case7.2"); + int64_t x = *reinterpret_cast(&INF); + x--; + double y = *reinterpret_cast(&x); + check(indices[1], hint("b", -y), slice(bOrder, 1), "case7.3"); + check(indices[2], hint("c", -INF), {}, "case7.4"); + check(indices[2], hint("c", -y), slice(cOrder, 2), "case7.5"); + } +} +TEST_F(IndexScanTest, Bool) { + auto rows = R"( + bool | bool + true | true + true | false + false | + false | false + true | + )"_row; + auto schema = R"( + a | bool | | + b | bool | | true + )"_schema; + auto indices = R"( + TAG(t,2) + (i1,2):a + (i2,3):b + )"_index(schema); + auto kv = encodeTag(rows, 2, schema, indices); + auto kvstore = std::make_unique(); + for (auto& iter : kv) { + for (auto& item : iter) { + kvstore->put(item.first, item.second); + } + } + auto check = [&](std::shared_ptr index, + const std::vector& columnHints, + const std::vector& expect, + const std::string& case_) { + DVLOG(1) << "Start case " << case_; + auto context = makeContext(1, 0); + auto scanNode = std::make_unique(context.get(), 0, columnHints); + IndexScanTestHelper helper; + helper.setKVStore(scanNode.get(), kvstore.get()); + helper.setIndex(scanNode.get(), index); + helper.setTag(scanNode.get(), schema); + InitContext initCtx; + initCtx.requiredColumns = {kVid}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + EXPECT_EQ(result, expect) << "Fail at case " << case_; + DVLOG(1) << "End case " << case_; + }; + auto expect = [](auto... vidList) { + std::vector ret; + std::vector value; + (value.push_back(std::to_string(vidList)), ...); + for (auto& v : value) { + Row row; + row.emplace_back(v); + ret.emplace_back(std::move(row)); + } + return ret; + }; + /* Case 1: Prefix */ { + check(indices[0], {makeColumnHint("a", true)}, expect(0, 1, 4), "case1.1"); + check(indices[0], {makeColumnHint("a", false)}, expect(2, 3), "case1.2"); + check(indices[1], {makeColumnHint("b", true)}, expect(0), "case1.3"); + check(indices[1], {makeColumnHint("b", false)}, expect(1, 3), "case1.4"); + } + /* Case 2: [x,INF) */ { + check(indices[0], {makeBeginColumnHint("a", false)}, expect(2, 3, 0, 1, 4), "case2.1"); + check(indices[0], {makeBeginColumnHint("a", true)}, expect(0, 1, 4), "case2.2"); + check(indices[1], {makeBeginColumnHint("b", true)}, expect(0), "case2.3"); + } +} +TEST_F(IndexScanTest, String1) { + /** + * data and query both without truncate + * That means ScanNode only access Index Key-Values + */ + auto rows = + " string | string | string | int \n" + " 123456789 | abcdefghi | \xFF\xFF\xFF\xFF\xFF\xFF\xFF | 0 \n" + " 123456789 | | | 1 \n" + " 12345678 | | \x01 | 2 \n" + " 123456788 | \xFF\xFF | | 3 \n" + " 12345678: | aacd | \xFF\xFF\xFF\xFF\xFF\xFF\xFE | 4 \n" + " a1234 | accd | \x00\x01 | 5 \n" + " | | | 6 \n" + ""_row; + auto schema = R"( + a | string | 10 | false + b | string | 10 | true + c | string | 10 | true + )"_schema; + auto indices = R"( + TAG(t,1) + (ia,2): a(10) + (ib,3): b(10) + (ic,4): c(10) + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + for (size_t i = 0; i < kv.size(); i++) { + for (auto& item : kv[i]) { + kvstore->put(item.first, item.second); + } + } + auto check = [&](std::shared_ptr index, + const std::vector& columnHints, + const std::vector& acquiredColumns, + const std::vector& expect, + const std::string& case_) { + DVLOG(1) << "Start case " << case_; + auto context = makeContext(1, 0); + auto scanNode = std::make_unique(context.get(), 0, columnHints); + IndexScanTestHelper helper; + helper.setKVStore(scanNode.get(), kvstore.get()); + helper.setIndex(scanNode.get(), index); + helper.setTag(scanNode.get(), schema); + helper.setFatal(scanNode.get(), true); + InitContext initCtx; + initCtx.requiredColumns.insert(acquiredColumns.begin(), acquiredColumns.end()); + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + std::vector result2(result.size()); + for (size_t j = 0; j < acquiredColumns.size(); j++) { + int p = initCtx.retColMap[acquiredColumns[j]]; + for (size_t i = 0; i < result.size(); i++) { + result2[i].emplace_back(result[i][p]); + } + } + result = result2; + EXPECT_EQ(result, expect) << "Fail at case " << case_; + }; + auto expect = [&rows](const std::vector& vidList, const std::vector& columns) { + std::vector ret; + for (size_t i = 0; i < vidList.size(); i++) { + Row row; + row.emplace_back(Value(std::to_string(vidList[i]))); + for (size_t j = 0; j < columns.size(); j++) { + row.emplace_back(rows[vidList[i]][columns[j]]); + } + ret.emplace_back(std::move(row)); + } + return ret; + }; + /* Case 1: prefix */ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeColumnHint(name, value)}; + }; + check(indices[0], hint("a", "123456789"), {kVid, "a"}, expect({0, 1}, {0}), "case1.1"); + check(indices[0], hint("a", "12345678"), {kVid, "a"}, expect({2}, {0}), "case1.2"); + check(indices[0], hint("a", ""), {kVid, "a"}, expect({6}, {0}), "case1.3"); + check(indices[1], hint("b", "\xFF\xFF"), {kVid, "b"}, expect({3}, {1}), "case1.4"); + check(indices[1], hint("b", ""), {kVid, "b"}, expect({6}, {1}), "case1.5"); + auto columnHint = hint("c", "\xFF\xFF\xFF\xFF\xFF\xFF\xFE"); + check(indices[2], columnHint, {kVid, "c"}, expect({4}, {2}), "case1.6"); + } + // 0 1 2 3 4 5 6 7 + std::vector a = {6, 2, 3, 0, 1, 4, 5}; + std::vector b = {6, 4, 0, 5, 3}; + std::vector c = {1, 3, 5, 2, 4, 0}; + auto slice = [](decltype(a) all, int begin) { + return decltype(all){all.begin() + begin, all.end()}; + }; + /* Case 2: [x, INF)*/ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeBeginColumnHint(name, value)}; + }; + check(indices[0], hint("a", "12345678"), {kVid, "a"}, expect(slice(a, 1), {0}), "case2.1"); + check(indices[0], hint("a", "123456780"), {kVid, "a"}, expect(slice(a, 2), {0}), "case2.2"); + check(indices[0], hint("a", ""), {kVid, "a"}, expect(a, {0}), "case2.3"); + check(indices[1], hint("b", ""), {kVid, "b"}, expect(b, {1}), "case2.4"); + check(indices[1], hint("b", "abc"), {kVid, "b"}, expect(slice(b, 2), {1}), "case2.5"); + check(indices[1], hint("b", "aac"), {kVid, "b"}, expect(slice(b, 1), {1}), "case2.6"); + check(indices[1], hint("b", "aacd\x01"), {kVid, "b"}, expect(slice(b, 2), {1}), "case2.7"); + check(indices[1], hint("b", "\xFF\xFF"), {kVid, "b"}, expect(slice(b, 4), {1}), "case2.8"); + check(indices[1], hint("b", "\xFF\xFF\x01"), {kVid, "b"}, {}, "case2.9"); + check(indices[2], hint("c", ""), {kVid, "c"}, expect(c, {2}), "case2.10"); + check(indices[2], hint("c", "\x01"), {kVid, "c"}, expect(slice(c, 3), {2}), "case2.11"); + check(indices[2], + hint("c", "\xFF\xFF\xFF\xFF\xFF\xFF\xFF"), + {kVid, "c"}, + expect(slice(c, 5), {2}), + "case2.12"); + } /* Case 3: (x,y) */ + // { + + // } /* Case 4: (INF,y]*/ {} +} +TEST_F(IndexScanTest, String2) { + /** + * data with truncate + * query without truncate + */ +} +TEST_F(IndexScanTest, String3) { + /** + * data without truncate + * query with truncate + */ +} +TEST_F(IndexScanTest, String4) { + /** + * data with truncate + * query with truncate + */ +} +TEST_F(IndexScanTest, String5) { + /** + * mix + */ } -TEST_F(IndexScanTest, Float) {} -TEST_F(IndexScanTest, Bool) {} -TEST_F(IndexScanTest, String1) {} -TEST_F(IndexScanTest, String2) {} -TEST_F(IndexScanTest, String3) {} -TEST_F(IndexScanTest, String4) {} TEST_F(IndexScanTest, Time) {} TEST_F(IndexScanTest, Date) {} TEST_F(IndexScanTest, DateTime) {} diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index f8ec7ff276e..c841b8520bb 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -5,14 +5,16 @@ */ #pragma once - #include #include +#include #include #include +#include #include "common/datatypes/DataSet.h" #include "common/meta/NebulaSchemaProvider.h" +#include "folly/Conv.h" #include "folly/String.h" #include "kvstore/KVIterator.h" #include "kvstore/KVStore.h" @@ -435,6 +437,14 @@ class RowParser { for (size_t i = 0; i < values.size(); i++) { if (values[i] == "") { row.emplace_back(Value::null()); + } else if (values[i] == "") { + row.emplace_back(std::numeric_limits::infinity()); + } else if (values[i] == "<-INF>") { + row.emplace_back(-std::numeric_limits::infinity()); + } else if (values[i] == "") { + row.emplace_back(std::numeric_limits::quiet_NaN()); + } else if (values[i] == "<-NaN>") { + row.emplace_back(-std::numeric_limits::quiet_NaN()); } else { row.emplace_back(transformMap[typeList_[i]](values[i])); } @@ -451,7 +461,8 @@ class RowParser { std::map> transformMap{ {"int", [](const std::string& str) { return Value(std::stol(str)); }}, {"string", [](const std::string& str) { return Value(str); }}, - {"double", [](const std::string& str) { return Value(std::stof(str)); }}}; + {"float", [](const std::string& str) { return Value(folly::to(str)); }}, + {"bool", [](const std::string& str) { return Value(str == "true" ? true : false); }}}; }; /** @@ -463,7 +474,7 @@ class RowParser { * std::string str=R"( * a | int | | * b | string | | true - * c | fix_string | 10 | + * c | double | 10 | * )"_schema */ class SchemaParser { @@ -503,7 +514,10 @@ class SchemaParser { std::stringstream ss; std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema; std::map typeMap{ - {"int", ::nebula::meta::cpp2::PropertyType::INT64}}; + {"int", ::nebula::meta::cpp2::PropertyType::INT64}, + {"double", ::nebula::meta::cpp2::PropertyType::DOUBLE}, + {"string", ::nebula::meta::cpp2::PropertyType::STRING}, + {"bool", ::nebula::meta::cpp2::PropertyType::BOOL}}; }; /** @@ -585,8 +599,10 @@ class IndexParser { ::nebula::meta::cpp2::ColumnTypeDef type; if (length > 0) { type.set_type_length(length); + type.set_type(::nebula::meta::cpp2::PropertyType::FIXED_STRING); + } else { + type.set_type(field->type()); } - type.set_type(field->type()); col.set_type(type); col.set_nullable(field->nullable()); fields.emplace_back(std::move(col)); @@ -604,8 +620,8 @@ class IndexParser { }; // Definition of UDL -std::vector operator""_row(const char* str, std::size_t) { - auto ret = RowParser(std::string(str)).getResult(); +std::vector operator""_row(const char* str, std::size_t len) { + auto ret = RowParser(std::string(str, len)).getResult(); return ret; } std::shared_ptr<::nebula::meta::NebulaSchemaProvider> operator"" _schema(const char* str, From 7d4b275261a07a828b9b5e94ddcf81d04a8a0b0e Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Thu, 14 Oct 2021 15:18:07 +0800 Subject: [PATCH 12/38] add string2/string3 test --- src/storage/exec/IndexSelectionNode.h | 2 +- src/storage/test/IndexTest.cpp | 404 ++++++++++++++++++++++++-- 2 files changed, 384 insertions(+), 22 deletions(-) diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 3dfd4c2e4a0..53ef1f7afc9 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -59,7 +59,7 @@ class IndexSelectionNode : public IndexNode { UNUSED(prop); return fatal(__FILE__, __LINE__); } - Value getVertex() const override { return fatal(__FILE__, __LINE__); } + Value getVertex(const std::string &) const override { return fatal(__FILE__, __LINE__); } Value getEdge() const override { return fatal(__FILE__, __LINE__); } Value getColumn(int32_t index) const override { UNUSED(index); diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index f8174a83268..33f4ad2d23f 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -7,6 +7,7 @@ #include #include +#include #include "codec/RowReaderWrapper.h" #include "codec/RowWriterV2.h" @@ -24,7 +25,8 @@ namespace nebula { namespace storage { namespace { int schemaVer = 2; -} +using std::string_literals::operator""s; +} // namespace /** * IndexScanTest * @@ -1016,6 +1018,11 @@ TEST_F(IndexScanTest, String1) { " a1234 | accd | \x00\x01 | 5 \n" " | | | 6 \n" ""_row; + // 0 1 2 3 4 5 6 7 + std::vector a = {6, 2, 3, 0, 1, 4, 5}; + std::vector b = {6, 4, 0, 5, 3}; + std::vector c = {1, 3, 5, 2, 4, 0}; + auto schema = R"( a | string | 10 | false b | string | 10 | true @@ -1095,17 +1102,14 @@ TEST_F(IndexScanTest, String1) { auto columnHint = hint("c", "\xFF\xFF\xFF\xFF\xFF\xFF\xFE"); check(indices[2], columnHint, {kVid, "c"}, expect({4}, {2}), "case1.6"); } - // 0 1 2 3 4 5 6 7 - std::vector a = {6, 2, 3, 0, 1, 4, 5}; - std::vector b = {6, 4, 0, 5, 3}; - std::vector c = {1, 3, 5, 2, 4, 0}; - auto slice = [](decltype(a) all, int begin) { - return decltype(all){all.begin() + begin, all.end()}; - }; + /* Case 2: [x, INF)*/ { auto hint = [](const char* name, const std::string& value) { return std::vector{makeBeginColumnHint(name, value)}; }; + auto slice = [](decltype(a) all, int begin) { + return decltype(all){all.begin() + begin, all.end()}; + }; check(indices[0], hint("a", "12345678"), {kVid, "a"}, expect(slice(a, 1), {0}), "case2.1"); check(indices[0], hint("a", "123456780"), {kVid, "a"}, expect(slice(a, 2), {0}), "case2.2"); check(indices[0], hint("a", ""), {kVid, "a"}, expect(a, {0}), "case2.3"); @@ -1122,39 +1126,397 @@ TEST_F(IndexScanTest, String1) { {kVid, "c"}, expect(slice(c, 5), {2}), "case2.12"); - } /* Case 3: (x,y) */ - // { - - // } /* Case 4: (INF,y]*/ {} + } + /* Case 3: (x,y) */ { + auto hint = [](const char* name, const std::string& begin, const std::string& end) { + return std::vector{makeColumnHint(name, begin, end)}; + }; + auto slice = [](decltype(a) all, int begin, int end) { + return decltype(all){all.begin() + begin, all.begin() + end}; + }; + auto columnHint = hint("a", "12345678", "123456789"); + check(indices[0], columnHint, {kVid, "a"}, expect(slice(a, 2, 3), {0}), "case3.1"); + check(indices[0], hint("a", "", "123456"), {kVid, "a"}, {}, "case3.2"); + check(indices[1], hint("b", "", "\xFF"), {kVid, "b"}, expect(slice(b, 1, 4), {1}), "case3.3"); + columnHint = hint("b", "aaccd", "\xFF\xFF"); + check(indices[1], columnHint, {kVid, "b"}, expect(slice(b, 1, 4), {1}), "case3.4"); + columnHint = hint("b", "\xFF", "\xFF\xFF\x01"); + check(indices[1], columnHint, {kVid, "b"}, expect(slice(b, 4, 5), {1}), "case3.5"); + check(indices[2], hint("c", "", "\x01"), {kVid, "c"}, expect(slice(c, 2, 3), {2}), "case3.6"); + columnHint = hint("c", "\x00\x00\x01"s, "\x01\x01"); + check(indices[2], columnHint, {kVid, "c"}, expect(slice(c, 2, 4), {2}), "case3.7"); + columnHint = hint("c", "\x00\x01"s, "\x01\x01"); + check(indices[2], columnHint, {kVid, "c"}, expect(slice(c, 3, 4), {2}), "case3.8"); + } + /* Case 4: (INF,y]*/ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeEndColumnHint(name, value)}; + }; + auto slice = [](decltype(a) all, int end) { + return decltype(all){all.begin(), all.begin() + end}; + }; + check(indices[0], hint("a", "123456789"), {kVid, "a"}, expect(slice(a, 5), {0}), "case4.1"); + check(indices[0], hint("a", ""), {kVid, "a"}, expect(slice(a, 1), {0}), "case4.2"); + check(indices[0], hint("a", "\xFF"), {kVid, "a"}, expect(slice(a, 7), {0}), "case4.3"); + check(indices[1], hint("b", "\xFF\xFF"), {kVid, "b"}, expect(slice(b, 5), {1}), "case4.4"); + check(indices[1], hint("b", "\xFF\xFE"), {kVid, "b"}, expect(slice(b, 4), {1}), "case4.5"); + check(indices[2], hint("c", "\x00\x00\x01"s), {kVid, "c"}, expect(slice(c, 2), {2}), "case4.6"); + check(indices[2], hint("c", "\x00\x01"s), {kVid, "c"}, expect(slice(c, 3), {2}), "case4.7"); + check(indices[2], hint("c", "\x01"), {kVid, "c"}, expect(slice(c, 4), {2}), "case4.8"); + auto columnHint = hint("c", "\xFF\xFF\xFF\xFF\xFF\xFF\xFF"); + check(indices[2], columnHint, {kVid, "c"}, expect(c, {2}), "case4.9"); + } } TEST_F(IndexScanTest, String2) { /** * data with truncate * query without truncate + * That means ScanNode need to access base data only when require indexed column */ + auto rows = + " string | string | string | int \n" + " 123456 | ABCDE2 | | 0 \n" + " 1234567 | ABCDE1 | \xFF\xFF\xFF\xFF\xFF | 1 \n" + " 1234567 | ABCDE | \xFF\xFF\xFF\xFF\xFF\x00\x01 | 2 \n" + " 123457 | ABCDF | \xFF\xFF\xFF\xFF\xFF | 3 \n" + ""_row; + auto schema = R"( + c1 | string | | false + c2 | string | | true + c3 | string | | true + )"_schema; + auto indices = R"( + TAG(t,1) + (i1,2):c1(5) + (i2,3):c2(5) + (i3,4):c3(5) + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + for (size_t i = 0; i < kv.size(); i++) { + for (auto& item : kv[i]) { + kvstore->put(item.first, item.second); + } + } + auto check = [&](std::shared_ptr index, + const std::vector& columnHints, + const std::vector& acquiredColumns, + const std::vector& expect, + const std::string& case_) { + DVLOG(1) << "Start case " << case_; + auto context = makeContext(1, 0); + auto scanNode = std::make_unique(context.get(), 0, columnHints); + IndexScanTestHelper helper; + helper.setKVStore(scanNode.get(), kvstore.get()); + helper.setIndex(scanNode.get(), index); + helper.setTag(scanNode.get(), schema); + helper.setFatal(scanNode.get(), true); + InitContext initCtx; + initCtx.requiredColumns.insert(acquiredColumns.begin(), acquiredColumns.end()); + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + std::vector result2(result.size()); + for (size_t j = 0; j < acquiredColumns.size(); j++) { + int p = initCtx.retColMap[acquiredColumns[j]]; + for (size_t i = 0; i < result.size(); i++) { + result2[i].emplace_back(result[i][p]); + } + } + result = result2; + EXPECT_EQ(result, expect) << "Fail at case " << case_; + }; + auto expect = [&rows](const std::vector& vidList, const std::vector& columns) { + std::vector ret; + for (size_t i = 0; i < vidList.size(); i++) { + Row row; + row.emplace_back(Value(std::to_string(vidList[i]))); + for (size_t j = 0; j < columns.size(); j++) { + row.emplace_back(rows[vidList[i]][columns[j]]); + } + ret.emplace_back(std::move(row)); + } + return ret; + }; + /* Case 1: Prefix */ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeColumnHint(name, value)}; + }; + check(indices[0], hint("c1", "1234"), {kVid, "c1"}, {}, "case1.1"); + check(indices[0], hint("c1", "12345"), {kVid, "c1"}, {}, "case1.2"); + check(indices[1], hint("c2", "ABCDE"), {kVid, "c2"}, expect({2}, {1}), "case1.3"); + check(indices[2], + hint("c3", "\xFF\xFF\xFF\xFF\xFF"), + {kVid, "c3"}, + expect({1, 3}, {2}), + "case1.4"); + } + /* Case 2: (x, INF)*/ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeBeginColumnHint(name, value)}; + }; + check(indices[0], hint("c1", "12345"), {kVid, "c1"}, expect({0, 1, 2, 3}, {0}), "case2.1"); + check(indices[1], hint("c2", "ABCDE"), {kVid, "c2"}, expect({0, 1, 3}, {1}), "case2.2"); + check( + indices[2], hint("c3", "\xFF\xFF\xFF\xFF\xFF"), {kVid, "c3"}, expect({2}, {2}), "case2.3"); + } + /* Case 3: [x, y] */ { + auto hint = [](const char* name, const std::string& begin, const std::string& end) { + return std::vector{makeColumnHint(name, begin, end)}; + }; + auto columnHint = hint("c1", "12345", "12346"); + check(indices[0], columnHint, {kVid, "c1"}, expect({0, 1, 2, 3}, {0}), "case3.1"); + columnHint = hint("c1", "12345", "12345"); + check(indices[0], columnHint, {kVid, "c1"}, {}, "case3.2"); + columnHint = hint("c2", "ABCDE", "ABCDF"); + check(indices[1], columnHint, {kVid, "c2"}, expect({0, 1, 2, 3}, {1}), "case3.3"); + columnHint = hint("c2", "ABCDE", "ABCDE"); + check(indices[1], columnHint, {kVid, "c2"}, expect({2}, {1}), "case3.4"); + columnHint = hint("c3", "\xFF\xFF\xFF\xFF\xFF", "\xFF\xFF\xFF\xFF\xFF"); + check(indices[2], columnHint, {kVid, "c3"}, expect({1, 3}, {2}), "case3.5"); + } + /* Case 4: (INF,y)*/ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeEndColumnHint(name, value)}; + }; + auto columnHint = hint("c1", "12345"); + check(indices[0], columnHint, {kVid, "c1"}, {}, "case4.1"); + columnHint = hint("c2", "ABCDE"); + check(indices[1], columnHint, {kVid, "c2"}, {}, "case4.2"); + columnHint = hint("c2", "ABCDF"); + check(indices[1], columnHint, {kVid, "c2"}, expect({0, 1, 2}, {1}), "case4.3"); + columnHint = hint("c3", " \xFF\xFF\xFF\xFF\xFF"); + check(indices[2], columnHint, {kVid, "c3"}, {}, "case4.4"); + } } TEST_F(IndexScanTest, String3) { /** * data without truncate * query with truncate + * That means ScanNode only access Index Key-Values */ + auto rows = + " string | string | string | int \n" + " abcde | 98765 | | 0 \n" + " abcda | 12345 | \xFF\xFF\xFF\xFF\xFF | 1 \n" + " abcda | 98766 | | 2 \n" + " | | | 3 \n" + ""_row; + auto schema = R"( + a | string | | false + b | string | | true + c | string | | true + )"_schema; + auto indices = R"( + TAG(t,0) + (ia,1): a(6) + (ib,2): b(6) + (ic,3): c(6) + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + for (size_t i = 0; i < kv.size(); i++) { + for (auto& item : kv[i]) { + kvstore->put(item.first, item.second); + } + } + auto check = [&](std::shared_ptr index, + const std::vector& columnHints, + const std::vector& acquiredColumns, + const std::vector& expect, + const std::string& case_) { + DVLOG(1) << "Start case " << case_; + auto context = makeContext(1, 0); + auto scanNode = std::make_unique(context.get(), 0, columnHints); + IndexScanTestHelper helper; + helper.setKVStore(scanNode.get(), kvstore.get()); + helper.setIndex(scanNode.get(), index); + helper.setTag(scanNode.get(), schema); + helper.setFatal(scanNode.get(), true); + InitContext initCtx; + initCtx.requiredColumns.insert(acquiredColumns.begin(), acquiredColumns.end()); + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + std::vector result2(result.size()); + for (size_t j = 0; j < acquiredColumns.size(); j++) { + int p = initCtx.retColMap[acquiredColumns[j]]; + for (size_t i = 0; i < result.size(); i++) { + result2[i].emplace_back(result[i][p]); + } + } + result = result2; + EXPECT_EQ(result, expect) << "Fail at case " << case_; + }; + auto expect = [&rows](const std::vector& vidList, const std::vector& columns) { + std::vector ret; + for (size_t i = 0; i < vidList.size(); i++) { + Row row; + row.emplace_back(Value(std::to_string(vidList[i]))); + for (size_t j = 0; j < columns.size(); j++) { + row.emplace_back(rows[vidList[i]][columns[j]]); + } + ret.emplace_back(std::move(row)); + } + return ret; + }; + /* Case 1: Prefix */ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeColumnHint(name, value)}; + }; + check(indices[0], hint("a", "abcde "), {kVid, "a"}, {}, "case1.1"); + check(indices[2], hint("c", "\xFF\xFF\xFF\xFF\xFF\xFF"), {kVid, "c"}, {}, "case1.2"); + } + /* Case 2: [x, INF)*/ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeBeginColumnHint(name, value)}; + }; + check(indices[0], hint("a", "abcdef"), {kVid, "a"}, {}, "case2.1"); + check(indices[0], hint("a", "abcda "), {kVid, "a"}, expect({0}, {0}), "case2.2"); + check(indices[1], hint("b", "987654 "), {kVid, "b"}, expect({2}, {1}), "case2.3"); + check(indices[2], hint("c", "\xFF\xFF\xFF\xFF\xFF\xFF"), {kVid, "c"}, {}, "case2.4"); + } + /* Case 3: (x, y]*/ { + auto hint = [](const char* name, const std::string& begin, const std::string& end) { + return std::vector{makeColumnHint(name, begin, end)}; + }; + auto columnHint = hint("a", "abcda ", "abcde "); + check(indices[0], columnHint, {kVid, "a"}, expect({0}, {0}), "case3.1"); + columnHint = hint("b", "98765 ", "98766 "); + check(indices[1], columnHint, {kVid, "b"}, expect({2}, {1}), "case3.2"); + columnHint = hint("c", "\xFF\xFF\xFF\xFF\xFE ", "\xFF\xFF\xFF\xFF\xFF "); + check(indices[2], columnHint, {kVid, "c"}, expect({1}, {2}), "case3.3"); + } + /* Case 4: (INF,y)*/ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeEndColumnHint(name, value)}; + }; + check(indices[0], hint("a", "abcde "), {kVid, "a"}, expect({3, 1, 2, 0}, {0}), "case4.1"); + check(indices[1], hint("b", "98764 "), {kVid, "b"}, expect({1}, {1}), "case4.2"); + check(indices[2], + hint("c", "\xFF\xFF\xFF\xFF\xFF "), + {kVid, "c"}, + expect({2, 3, 1}, {2}), + "case4.3"); + } } TEST_F(IndexScanTest, String4) { /** * data with truncate * query with truncate + * That means ScanNode always need to access base data. */ + auto rows = + " string | string | int \n" + " abcde1 | 987654 | 0 \n" + " abcdd | 98765 | 1 \n" + " abcdf | 12345 | 2 \n" + " abcde | \xFF\xFF\xFF\xFF\xFF\xFF | 3 \n" + " abcde12 | | 4 \n" + " abcde123 | \xFF\xFF\xFF\xFF\xFF | 5 \n" + " abcde1234 | \xFF\xFF\xFF\xFF\xFF\xFF\x01| 6 \n" + " abcde1234 | | 7 \n" + ""_row; + auto schema = R"( + a | string | | false + b | string | | true + c | string | | true + )"_schema; + auto indices = R"( + TAG(t,0) + (ia,1): a(5) + (ib,2): b(5) + (ic,3): c(5) + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + for (size_t i = 0; i < kv.size(); i++) { + for (auto& item : kv[i]) { + kvstore->put(item.first, item.second); + } + } + auto check = [&](std::shared_ptr index, + const std::vector& columnHints, + const std::vector& acquiredColumns, + const std::vector& expect, + const std::string& case_) { + DVLOG(1) << "Start case " << case_; + auto context = makeContext(1, 0); + auto scanNode = std::make_unique(context.get(), 0, columnHints); + IndexScanTestHelper helper; + helper.setKVStore(scanNode.get(), kvstore.get()); + helper.setIndex(scanNode.get(), index); + helper.setTag(scanNode.get(), schema); + helper.setFatal(scanNode.get(), true); + InitContext initCtx; + initCtx.requiredColumns.insert(acquiredColumns.begin(), acquiredColumns.end()); + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + std::vector result2(result.size()); + for (size_t j = 0; j < acquiredColumns.size(); j++) { + int p = initCtx.retColMap[acquiredColumns[j]]; + for (size_t i = 0; i < result.size(); i++) { + result2[i].emplace_back(result[i][p]); + } + } + result = result2; + EXPECT_EQ(result, expect) << "Fail at case " << case_; + }; + auto expect = [&rows](const std::vector& vidList, const std::vector& columns) { + std::vector ret; + for (size_t i = 0; i < vidList.size(); i++) { + Row row; + row.emplace_back(Value(std::to_string(vidList[i]))); + for (size_t j = 0; j < columns.size(); j++) { + row.emplace_back(rows[vidList[i]][columns[j]]); + } + ret.emplace_back(std::move(row)); + } + return ret; + }; } -TEST_F(IndexScanTest, String5) { - /** - * mix - */ -} -TEST_F(IndexScanTest, Time) {} -TEST_F(IndexScanTest, Date) {} -TEST_F(IndexScanTest, DateTime) {} -TEST_F(IndexScanTest, Compound) {} TEST_F(IndexScanTest, Nullable) {} +TEST_F(IndexScanTest, Time) { + // TODO(hs.zhang): add unittest +} +TEST_F(IndexScanTest, Date) { + // TODO(hs.zhang): add unittest +} +TEST_F(IndexScanTest, DateTime) { + // TODO(hs.zhang): add unittest +} +TEST_F(IndexScanTest, Compound) { + // TODO(hs.zhang): add unittest +} +TEST_F(IndexScanTest, TTL) {} } // namespace storage } // namespace nebula From a1ef35abc92d0ce7fd3ab233a5b1296a1797cb72 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Mon, 18 Oct 2021 17:12:27 +0800 Subject: [PATCH 13/38] finish index_scan_node unittest --- src/kvstore/KVEngine.h | 6 - src/kvstore/KVStore.h | 9 - src/kvstore/NebulaStore.cpp | 13 +- src/kvstore/NebulaStore.h | 9 - src/kvstore/RocksEngine.cpp | 16 +- src/kvstore/RocksEngine.h | 32 +--- src/storage/exec/IndexScanNode2.cpp | 262 +++++++++++++++---------- src/storage/exec/IndexScanNode2.h | 29 ++- src/storage/test/IndexTest.cpp | 283 +++++++++++++++++++++++++++- src/storage/test/IndexTestUtil.h | 27 +-- 10 files changed, 472 insertions(+), 214 deletions(-) diff --git a/src/kvstore/KVEngine.h b/src/kvstore/KVEngine.h index 9aa7183a307..bd485a4333b 100644 --- a/src/kvstore/KVEngine.h +++ b/src/kvstore/KVEngine.h @@ -63,12 +63,6 @@ class KVEngine { const std::string& end, std::unique_ptr* iter) = 0; - virtual nebula::cpp2::ErrorCode range(bool includeStart, - const std::string& start, - bool includeEnd, - const std::string& end, - std::unique_ptr* iter) = 0; - // Get all results with 'prefix' str as prefix. virtual nebula::cpp2::ErrorCode prefix(const std::string& prefix, std::unique_ptr* iter) = 0; diff --git a/src/kvstore/KVStore.h b/src/kvstore/KVStore.h index aafd88437d4..dcef7e1a9f9 100644 --- a/src/kvstore/KVStore.h +++ b/src/kvstore/KVStore.h @@ -102,15 +102,6 @@ class KVStore { std::unique_ptr* iter, bool canReadFromFollower = false) = 0; - virtual nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, - PartitionID partId, - bool includeStart, - const std::string& start, - bool includeEnd, - const std::string& end, - std::unique_ptr* iter, - bool canReadFromFollower = false) = 0; - // Since the `range' interface will hold references to its 3rd & 4th // parameter, in `iter', thus the arguments must outlive `iter'. Here we // forbid one to invoke `range' with rvalues, which is the common mistake. diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp index 106c0d100ac..5ac276adebd 100644 --- a/src/kvstore/NebulaStore.cpp +++ b/src/kvstore/NebulaStore.cpp @@ -610,16 +610,6 @@ nebula::cpp2::ErrorCode NebulaStore::range(GraphSpaceID spaceId, const std::string& end, std::unique_ptr* iter, bool canReadFromFollower) { - return range(spaceId, partId, true, start, false, end, iter, canReadFromFollower); -} -nebula::cpp2::ErrorCode NebulaStore::range(GraphSpaceID spaceId, - PartitionID partId, - bool includeStart, - const std::string& start, - bool includeEnd, - const std::string& end, - std::unique_ptr* iter, - bool canReadFromFollower) { auto ret = part(spaceId, partId); if (!ok(ret)) { return error(ret); @@ -628,8 +618,9 @@ nebula::cpp2::ErrorCode NebulaStore::range(GraphSpaceID spaceId, if (!checkLeader(part, canReadFromFollower)) { return nebula::cpp2::ErrorCode::E_LEADER_CHANGED; } - return part->engine()->range(includeStart, start, includeEnd, end, iter); + return part->engine()->range(start, end, iter); } + nebula::cpp2::ErrorCode NebulaStore::prefix(GraphSpaceID spaceId, PartitionID partId, const std::string& prefix, diff --git a/src/kvstore/NebulaStore.h b/src/kvstore/NebulaStore.h index 9889ddc42bc..c888a22618c 100644 --- a/src/kvstore/NebulaStore.h +++ b/src/kvstore/NebulaStore.h @@ -129,15 +129,6 @@ class NebulaStore : public KVStore, public Handler { std::unique_ptr* iter, bool canReadFromFollower = false) override; - nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, - PartitionID partId, - bool includeStart, - const std::string& start, - bool includeEnd, - const std::string& end, - std::unique_ptr* iter, - bool canReadFromFollower = false) override; - // Delete the overloading with a rvalue `start' and `end' nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, PartitionID partId, diff --git a/src/kvstore/RocksEngine.cpp b/src/kvstore/RocksEngine.cpp index 80de131c4fe..73bdb14e104 100644 --- a/src/kvstore/RocksEngine.cpp +++ b/src/kvstore/RocksEngine.cpp @@ -201,25 +201,13 @@ std::vector RocksEngine::multiGet(const std::vector& keys, nebula::cpp2::ErrorCode RocksEngine::range(const std::string& start, const std::string& end, std::unique_ptr* storageIter) { - return range(true, start, false, end, storageIter); -} - -nebula::cpp2::ErrorCode RocksEngine::range(bool includeStart, - const std::string& start, - bool includeEnd, - const std::string& end, - std::unique_ptr* storageIter) { rocksdb::ReadOptions options; options.total_order_seek = FLAGS_enable_rocksdb_prefix_filtering; rocksdb::Iterator* iter = db_->NewIterator(options); if (iter) { - auto seekKey = start; - if (!includeStart) { - seekKey += '\0'; - } - iter->Seek(rocksdb::Slice(seekKey)); + iter->Seek(rocksdb::Slice(start)); } - storageIter->reset(new RocksRangeIter(iter, includeStart, start, includeEnd, end)); + storageIter->reset(new RocksRangeIter(iter, start, end)); return nebula::cpp2::ErrorCode::SUCCEEDED; } diff --git a/src/kvstore/RocksEngine.h b/src/kvstore/RocksEngine.h index a9448983a5c..d94343d84e7 100644 --- a/src/kvstore/RocksEngine.h +++ b/src/kvstore/RocksEngine.h @@ -13,7 +13,6 @@ #include #include "common/base/Base.h" -#include "folly/Likely.h" #include "kvstore/KVEngine.h" #include "kvstore/KVIterator.h" #include "kvstore/RocksEngineConfig.h" @@ -23,30 +22,13 @@ namespace kvstore { class RocksRangeIter : public KVIterator { public: - RocksRangeIter(rocksdb::Iterator* iter, - bool includeStart, - rocksdb::Slice start, - bool includeEnd, - rocksdb::Slice end) - : iter_(iter), - includeStart_(includeStart), - start_(start), - includeEnd_(includeEnd), - end_(end) {} + RocksRangeIter(rocksdb::Iterator* iter, rocksdb::Slice start, rocksdb::Slice end) + : iter_(iter), start_(start), end_(end) {} ~RocksRangeIter() = default; bool valid() const override { - if (LIKELY(!!iter_ && iter_->Valid())) { - int cmp2end = iter_->key().compare(end_); - if (cmp2end < 0 || (cmp2end == 0 && includeEnd_)) { - return true; - } else { - return false; - } - } else { - return false; - } + return !!iter_ && iter_->Valid() && (iter_->key().compare(end_) < 0); } void next() override { iter_->Next(); } @@ -63,9 +45,7 @@ class RocksRangeIter : public KVIterator { private: std::unique_ptr iter_; - bool includeStart_; rocksdb::Slice start_; - bool includeEnd_; rocksdb::Slice end_; }; @@ -166,12 +146,6 @@ class RocksEngine : public KVEngine { const std::string& end, std::unique_ptr* iter) override; - nebula::cpp2::ErrorCode range(bool includeStart, - const std::string& start, - bool includeEnd, - const std::string& end, - std::unique_ptr* iter) override; - nebula::cpp2::ErrorCode prefix(const std::string& prefix, std::unique_ptr* iter) override; diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index 542739db6a7..f74e9ff6a2a 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -10,7 +10,8 @@ namespace storage { // Define of Path Path::Path(nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, - const std::vector& hints) + const std::vector& hints, + int64_t vidLen) : index_(index), schema_(schema), hints_(hints) { bool nullFlag = false; for (auto field : index->get_fields()) { @@ -21,6 +22,7 @@ Path::Path(nebula::meta::cpp2::IndexItem* index, auto type = IndexKeyUtils::toValueType(field.get_type().get_type()); auto tmpStr = IndexKeyUtils::encodeNullValue(type, field.get_type().get_type_length()); index_nullable_offset_ += tmpStr.size(); + totalKeyLength_ += tmpStr.size(); } for (auto x : nullable_) { DVLOG(1) << x; @@ -28,16 +30,26 @@ Path::Path(nebula::meta::cpp2::IndexItem* index, DVLOG(1) << nullFlag; if (!nullFlag) { nullable_.clear(); + } else { + totalKeyLength_ += 2; + } + if (index_->get_schema_id().tag_id_ref().has_value()) { + totalKeyLength_ += vidLen; + suffixLength_ = vidLen; + } else { + totalKeyLength_ += vidLen * 2 + sizeof(EdgeRanking); + suffixLength_ = vidLen * 2 + sizeof(EdgeRanking); } } std::unique_ptr Path::make(nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, - const std::vector& hints) { + const std::vector& hints, + int64_t vidLen) { std::unique_ptr ret; if (hints.back().get_scan_type() == cpp2::ScanType::RANGE) { - ret.reset(new RangePath(index, schema, hints)); + ret.reset(new RangePath(index, schema, hints, vidLen)); } else { - ret.reset(new PrefixPath(index, schema, hints)); + ret.reset(new PrefixPath(index, schema, hints, vidLen)); } return ret; } @@ -53,26 +65,32 @@ std::string Path::encodeValue(const Value& value, size_t index, std::string& key) { std::string val; + bool isNull = false; if (value.type() == Value::Type::STRING) { val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); if (val.back() != '\0') { QFList_.clear(); QFList_.emplace_back([](const std::string&) { return Qualified::UNCERTAIN; }); } + } else if (value.type() == Value::Type::NULLVALUE) { + auto vtype = IndexKeyUtils::toValueType(colDef.get_type()); + val = IndexKeyUtils::encodeNullValue(vtype, colDef.get_type_length()); + isNull = true; } else { val = IndexKeyUtils::encodeValue(value); } if (!nullable_.empty() && nullable_[index] == true) { - QFList_.emplace_back( - [isNull = value.isNull(), index, offset = index_nullable_offset_](const std::string& k) { - std::bitset<16> nullableBit; - auto v = *reinterpret_cast(k.data() + offset); - nullableBit = v; - DVLOG(3) << isNull; - DVLOG(3) << nullableBit.test(15 - index); - return nullableBit.test(15 - index) ^ isNull ? Qualified::INCOMPATIBLE - : Qualified::COMPATIBLE; - }); + QFList_.emplace_back([isNull, index, offset = index_nullable_offset_](const std::string& k) { + std::bitset<16> nullableBit; + auto v = *reinterpret_cast(k.data() + offset); + nullableBit = v; + DVLOG(3) << isNull; + DVLOG(3) << nullableBit.test(15 - index); + return nullableBit.test(15 - index) ^ isNull ? Qualified::INCOMPATIBLE + : Qualified::COMPATIBLE; + }); + } else if (isNull) { + QFList_.emplace_back([](const std::string&) { return Qualified::INCOMPATIBLE; }); } key.append(val); return val; @@ -82,8 +100,9 @@ std::string Path::encodeValue(const Value& value, // Define of RangePath RangePath::RangePath(nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, - const std::vector& hints) - : Path(index, schema, hints) { + const std::vector& hints, + int64_t vidLen) + : Path(index, schema, hints, vidLen) { buildKey(); } void RangePath::resetPart(PartitionID partId) { @@ -130,104 +149,144 @@ void RangePath::buildKey() { encodeValue(hint.get_begin_value(), fieldIter->get_type(), i, common); } auto& hint = hints_.back(); - startKey_ = common; - endKey_ = std::move(common); size_t index = hints_.size() - 1; - if (hint.begin_value_ref().is_set()) { - encodeBeginValue(hint.get_begin_value(), fieldIter->get_type(), index, startKey_); - includeStart_ = hint.get_include_begin(); - } else { - includeStart_ = false; + auto [a, b] = encodeRange(hint, fieldIter->get_type(), index, common.size()); + startKey_ = common + a; + endKey_ = common + b; + if (!hint.end_value_ref().is_set()) { + endKey_.append(totalKeyLength_ - endKey_.size() + 1, '\xFF'); } + DVLOG(2) << "start key:\n" << folly::hexDump(startKey_.data(), startKey_.size()); + DVLOG(2) << "end key:\n" << folly::hexDump(endKey_.data(), endKey_.size()); +} +std::tuple RangePath::encodeRange( + const cpp2::IndexColumnHint& hint, + const nebula::meta::cpp2::ColumnTypeDef& colTypeDef, + size_t colIndex, + size_t offset) { + std::string startKey, endKey; + bool needCheckNullable = !nullable_.empty() && nullable_[colIndex]; if (hint.end_value_ref().is_set()) { - encodeEndValue(hint.get_end_value(), fieldIter->get_type(), index, endKey_); includeEnd_ = hint.get_include_end(); - } else { - endKey_.append(startKey_.size() - endKey_.size(), '\xFF'); - includeEnd_ = true; + auto tmp = encodeEndValue(hint.get_end_value(), colTypeDef, endKey, offset); + if (memcmp(tmp.data(), std::string(tmp.size(), '\xFF').data(), tmp.size() != 0)) { + needCheckNullable &= false; + } } -} -std::string RangePath::encodeBeginValue(const Value& value, - const ColumnTypeDef& colDef, - size_t index, - std::string& key) { - std::string val = IndexKeyUtils::encodeValue(value, colDef.type_length_ref().value_or(0)); - if (value.type() == Value::Type::STRING && val.back() != '\0') { - QFList_.emplace_back([&startKey = this->startKey_, startPos = key.size(), length = val.size()]( - const std::string& k) { - int ret = memcmp(startKey.data() + startPos, k.data() + startPos, length); - CHECK_LE(ret, 0); - return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; - }); - } else { - QFList_.emplace_back([&startKey = this->startKey_, - &allowEq = this->includeStart_, - startPos = key.size(), - length = val.size()](const std::string& k) { - int ret = memcmp(startKey.data() + startPos, k.data() + startPos, length); - DVLOG(3) << '\n' << folly::hexDump(startKey.data() + startPos, length); - DVLOG(3) << '\n' << folly::hexDump(k.data() + startPos, length); - CHECK_LE(ret, 0); - return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; - }); + if (hint.begin_value_ref().is_set()) { + includeStart_ = hint.get_include_begin(); + encodeBeginValue(hint.get_begin_value(), colTypeDef, startKey, offset); } - if (value.isFloat()) { - QFList_.emplace_back([startPos = key.size(), length = val.size()](const std::string& k) { - std::string s(length, '\xFF'); - return memcmp(k.data() + startPos, s.data(), length) == 0 ? Qualified::INCOMPATIBLE - : Qualified::COMPATIBLE; - }); - } else if (!nullable_.empty() && nullable_[index] == true) { - QFList_.emplace_back([index, offset = index_nullable_offset_](const std::string& k) { + if (UNLIKELY(needCheckNullable)) { + QFList_.emplace_back([colIndex, offset = index_nullable_offset_](const std::string& k) { DVLOG(1) << "check null"; std::bitset<16> nullableBit; auto v = *reinterpret_cast(k.data() + offset); nullableBit = v; DVLOG(1) << nullableBit; - DVLOG(1) << nullableBit.test(15 - index); - return nullableBit.test(15 - index) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; + DVLOG(1) << nullableBit.test(15 - colIndex); + return nullableBit.test(15 - colIndex) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; + }); + } + return {startKey, endKey}; +} +std::string RangePath::encodeBeginValue(const Value& value, + const ColumnTypeDef& colDef, + std::string& key, + size_t offset) { + std::string val; + bool greater = !includeStart_; + CHECK_NE(value.type(), Value::Type::NULLVALUE); + if (value.type() == Value::Type::STRING) { + bool truncated = false; + val = encodeString(value, *colDef.get_type_length(), truncated); + greater &= !truncated; + if (UNLIKELY(truncated)) { + QFList_.emplace_back([&startKey = this->startKey_, + startPos = key.size(), + length = val.size()](const std::string& k) { + int ret = memcmp(startKey.data() + startPos, k.data() + startPos, length); + CHECK_LE(ret, 0); + return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; + }); + } + } else if (value.type() == Value::Type::FLOAT) { + bool isNaN = false; + val = encodeFloat(value, isNaN); + greater |= isNaN; + // TODO(hs.zhang): Optimize the logic of judging NaN + QFList_.emplace_back([startPos = offset, length = val.size()](const std::string& k) { + int ret = memcmp("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", k.data() + startPos, length); + return ret == 0 ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); + } else { + val = IndexKeyUtils::encodeValue(value); + } + if (greater) { + val.append(suffixLength_ + 1, '\xFF'); } key += val; return val; } std::string RangePath::encodeEndValue(const Value& value, const ColumnTypeDef& colDef, - size_t index, - std::string& key) { - std::string val = IndexKeyUtils::encodeValue(value, colDef.type_length_ref().value_or(0)); - if (value.type() == Value::Type::STRING && val.back() != '\0') { - QFList_.emplace_back([&endKey = this->endKey_, startPos = key.size(), length = val.size()]( - const std::string& k) { - int ret = memcmp(endKey.data() + startPos, k.data() + startPos, length); - CHECK_GE(ret, 0); - return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; - }); + std::string& key, + size_t offset) { + CHECK_NE(value.type(), Value::Type::NULLVALUE); + std::string val; + bool greater = includeEnd_; + if (value.type() == Value::Type::STRING) { + bool truncated = false; + val = encodeString(value, *colDef.get_type_length(), truncated); + greater |= truncated; + if (UNLIKELY(truncated)) { + QFList_.emplace_back([&endKey = this->endKey_, startPos = key.size(), length = val.size()]( + const std::string& k) { + int ret = memcmp(endKey.data() + startPos, k.data() + startPos, length); + CHECK_LE(ret, 0); + return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; + }); + } + } else if (value.type() == Value::Type::FLOAT) { + bool isNaN = false; + val = encodeFloat(value, isNaN); + greater |= isNaN; + if (UNLIKELY(isNaN)) { + QFList_.emplace_back([startPos = offset, length = val.size()](const std::string& k) { + DLOG(INFO) << "check NaN:" << startPos << "\t" << length; + DLOG(INFO) << '\n' << folly::hexDump(k.data(), k.size()); + int ret = memcmp("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", k.data() + startPos, length); + return ret == 0 ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; + }); + } } else { - QFList_.emplace_back([&endKey = this->endKey_, - &allowEq = this->includeEnd_, - startPos = key.size(), - length = val.size()](const std::string& k) { - int ret = memcmp(endKey.data() + startPos, k.data() + startPos, length); - CHECK_GE(ret, 0); - return (ret == 0 && !allowEq) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; - }); + val = IndexKeyUtils::encodeValue(value); } - if (value.isFloat()) { - QFList_.emplace_back([startPos = key.size(), length = val.size()](const std::string& k) { - std::string s(length, '\xFF'); - return memcmp(k.data() + startPos, s.data(), length) == 0 ? Qualified::INCOMPATIBLE - : Qualified::COMPATIBLE; - }); - } else if (!nullable_.empty() && nullable_[index] == true) { - QFList_.emplace_back([index, offset = index_nullable_offset_](const std::string& k) { - std::bitset<16> nullableBit; - auto v = *reinterpret_cast(k.data() + offset); - nullableBit = v; - return nullableBit.test(15 - index) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; - }); + DVLOG(2) << includeEnd_; + DVLOG(2) << greater; + if (greater) { + val.append(suffixLength_ + 1, '\xFF'); } key += val; + DVLOG(2) << '\n' << folly::hexDump(key.data(), key.size()); + return val; +} +inline std::string RangePath::encodeString(const Value& value, size_t len, bool& truncated) { + std::string val = IndexKeyUtils::encodeValue(value); + if (val.size() < len) { + val.append(len - val.size(), '\x00'); + } else { + val = val.substr(0, len); + truncated = true; + } + return val; +} +std::string RangePath::encodeFloat(const Value& value, bool& isNaN) { + std::string val = IndexKeyUtils::encodeValue(value); + // check NaN + if (UNLIKELY(memcmp(val.data(), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", val.size()) == 0)) { + isNaN = true; + } return val; } @@ -236,8 +295,9 @@ std::string RangePath::encodeEndValue(const Value& value, // Define of PrefixPath PrefixPath::PrefixPath(nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, - const std::vector& hints) - : Path(index, schema, hints) { + const std::vector& hints, + int64_t vidLen) + : Path(index, schema, hints, vidLen) { buildKey(); } Path::Qualified PrefixPath::qualified(const Map& rowData) { @@ -302,7 +362,7 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { tmp.erase(kSrc); tmp.erase(kDst); needAccessBase_ = !tmp.empty(); - path_ = Path::make(index_.get(), getSchema(), columnHints_); + path_ = Path::make(index_.get(), getSchema(), columnHints_, context_->vIdLen()); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { @@ -377,15 +437,13 @@ nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { nebula::cpp2::ErrorCode ret; if (path_->isRange()) { auto rangePath = dynamic_cast(path_.get()); - ret = kvstore_->range(spaceId_, - partId, - rangePath->includeStart(), - rangePath->getStartKey(), - rangePath->includeEnd(), - rangePath->getEndKey(), - &iter_); + ret = + kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter_); } else { auto prefixPath = dynamic_cast(path_.get()); + DVLOG(1) << '\n' + << folly::hexDump(prefixPath->getPrefixKey().data(), + prefixPath->getPrefixKey().size()); ret = kvstore_->prefix(spaceId_, partId, prefixPath->getPrefixKey(), &iter_); } return ret; diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index e0d8508e401..a8a88faf972 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -76,12 +76,14 @@ class Path { using ColumnTypeDef = ::nebula::meta::cpp2::ColumnTypeDef; Path(nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, - const std::vector& hints); + const std::vector& hints, + int64_t vidLen); virtual ~Path() = default; static std::unique_ptr make(::nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, - const std::vector& hints); + const std::vector& hints, + int64_t vidLen); Qualified qualified(const folly::StringPiece& key); virtual bool isRange() { return false; } @@ -99,12 +101,15 @@ class Path { const std::vector& hints_; std::vector nullable_; int64_t index_nullable_offset_{8}; + int64_t totalKeyLength_{8}; + int64_t suffixLength_; }; class PrefixPath : public Path { public: PrefixPath(nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, - const std::vector& hints); + const std::vector& hints, + int64_t vidLen); // Override Qualified qualified(const Map& rowData) override; void resetPart(PartitionID partId) override; @@ -119,7 +124,8 @@ class RangePath : public Path { public: RangePath(nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, - const std::vector& hints); + const std::vector& hints, + int64_t vidLen); // Override Qualified qualified(const Map& rowData) override; void resetPart(PartitionID partId) override; @@ -136,14 +142,21 @@ class RangePath : public Path { bool includeEnd_ = false; void buildKey(); + std::tuple encodeRange( + const cpp2::IndexColumnHint& hint, + const nebula::meta::cpp2::ColumnTypeDef& colTypeDef, + size_t colIndex, + size_t offset); + inline std::string encodeString(const Value& value, size_t len, bool& truncated); + inline std::string encodeFloat(const Value& value, bool& isNaN); std::string encodeBeginValue(const Value& value, const ColumnTypeDef& colDef, - size_t index, - std::string& key); + std::string& key, + size_t offset); std::string encodeEndValue(const Value& value, const ColumnTypeDef& colDef, - size_t index, - std::string& key); + std::string& key, + size_t offset); }; /* define inline functions */ diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 33f4ad2d23f..d8ce210036c 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -11,6 +11,10 @@ #include "codec/RowReaderWrapper.h" #include "codec/RowWriterV2.h" +#include "common/base/ObjectPool.h" +#include "common/expression/ConstantExpression.h" +#include "common/expression/PropertyExpression.h" +#include "common/expression/RelationalExpression.h" #include "common/utils/NebulaKeyUtils.h" #include "kvstore/KVEngine.h" #include "kvstore/KVIterator.h" @@ -1438,13 +1442,11 @@ TEST_F(IndexScanTest, String4) { auto schema = R"( a | string | | false b | string | | true - c | string | | true )"_schema; auto indices = R"( TAG(t,0) (ia,1): a(5) (ib,2): b(5) - (ic,3): c(5) )"_index(schema); auto kv = encodeTag(rows, 1, schema, indices); auto kvstore = std::make_unique(); @@ -1502,8 +1504,207 @@ TEST_F(IndexScanTest, String4) { } return ret; }; + /* Case 1: Prefix */ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeColumnHint(name, value)}; + }; + check(indices[0], hint("a", "abcde"), {kVid, "a"}, expect({3}, {0}), "case1.1"); + check(indices[0], hint("a", "abcde1234"), {kVid, "a"}, expect({6, 7}, {0}), "case1.2"); + check(indices[0], hint("a", "abcde2"), {kVid, "a"}, {}, "case1.3"); + check(indices[1], hint("b", "\xFF\xFF\xFF\xFF\xFF"), {kVid, "b"}, expect({5}, {1}), "case1.4"); + } + /* Case 2: (x, INF) */ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeBeginColumnHint(name, value)}; + }; + check(indices[0], hint("a", "abcde"), {kVid, "a"}, expect({0, 4, 5, 6, 7, 2}, {0}), "case2.1"); + check(indices[0], hint("a", "abcde12"), {kVid, "a"}, expect({5, 6, 7, 2}, {0}), "case2.2"); + check(indices[0], hint("a", "abcde12345"), {kVid, "a"}, expect({2}, {0}), "case2.3"); + check( + indices[0], hint("a", "abcdd"), {kVid, "a"}, expect({0, 3, 4, 5, 6, 7, 2}, {0}), "case2.4"); + auto columnHint = hint("b", "\xFF\xFF\xFF\xFF\xFF"); + check(indices[1], columnHint, {kVid, "b"}, expect({3, 6}, {1}), "case2.5"); + columnHint = hint("b", "\xFF\xFF\xFF\xFF\xFF\x01"); + check(indices[1], columnHint, {kVid, "b"}, expect({3, 6}, {1}), "case2.6"); + columnHint = hint("b", "\xFF\xFF\xFF\xFF\xFF\xFF"); + check(indices[1], columnHint, {kVid, "b"}, expect({6}, {1}), "case2.7"); + } + /* Case 3: [x,y) */ { + auto hint = [](const char* name, const std::string& begin, const std::string& end) { + return std::vector{makeColumnHint(name, begin, end)}; + }; + auto columnHint = hint("a", "abcdd123", "abcde1234"); + check(indices[0], columnHint, {kVid, "a"}, expect({0, 3, 4, 5}, {0}), "case3.1"); + columnHint = hint("a", "abcde1", "abcdf"); + check(indices[0], columnHint, {kVid, "a"}, expect({0, 4, 5, 6, 7}, {0}), "case3.2"); + columnHint = hint("a", "abcde12345", "abcde123456"); + check(indices[0], columnHint, {kVid, "a"}, {}, "case3.3"); + columnHint = hint("a", "abcde1234", "abcde12345"); + check(indices[0], columnHint, {kVid, "a"}, expect({6, 7}, {0}), "case3.4"); + columnHint = hint("b", "\xFF\xFF\xFF\xFF\xFF", "\xFF\xFF\xFF\xFF\xFF\x00\x01"s); + check(indices[1], columnHint, {kVid, "b"}, expect({5}, {1}), "case3.5"); + columnHint = hint("b", "\xFF\xFF\xFF\xFF\xFF\x01", "\xFF\xFF\xFF\xFF\xFF\xFF\xFF"); + check(indices[1], columnHint, {kVid, "b"}, expect({3, 6}, {1}), "case3.6"); + } + /* Case 4: (INF,y] */ { + auto hint = [](const char* name, const std::string& value) { + return std::vector{makeEndColumnHint(name, value)}; + }; + check(indices[0], hint("a", "abcde123"), {kVid, "a"}, expect({1, 0, 3, 4, 5}, {0}), "case4.1"); + check(indices[0], hint("a", "abcde"), {kVid, "a"}, expect({1, 3}, {0}), "case4.2"); + check(indices[0], + hint("a", "abcde1234"), + {kVid, "a"}, + expect({1, 0, 3, 4, 5, 6, 7}, {0}), + "case4.3"); + check(indices[1], + hint("b", "\xFF\xFF\xFF\xFF\xFF"), + {kVid, "b"}, + expect({2, 0, 1, 5}, {1}), + "case4.4"); + check(indices[1], + hint("b", "\xFF\xFF\xFF\xFF\xFF\xFF"), + {kVid, "b"}, + expect({2, 0, 1, 3, 5}, {1}), + "case4.5"); + } +} +TEST_F(IndexScanTest, Nullable) { + std::shared_ptr schema; + auto kvstore = std::make_unique(); + auto check = [&](std::shared_ptr index, + const std::vector& columnHints, + const std::vector& expect, + const std::string& case_) { + DVLOG(1) << "Start case " << case_; + auto context = makeContext(1, 0); + auto scanNode = std::make_unique(context.get(), 0, columnHints); + IndexScanTestHelper helper; + helper.setKVStore(scanNode.get(), kvstore.get()); + helper.setIndex(scanNode.get(), index); + helper.setTag(scanNode.get(), schema); + helper.setFatal(scanNode.get(), true); + InitContext initCtx; + initCtx.requiredColumns = {kVid}; + scanNode->init(initCtx); + scanNode->execute(0); + bool hasNext = false; + std::vector result; + while (true) { + auto res = scanNode->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + EXPECT_EQ(result, expect) << "Fail at case " << case_; + }; + auto hint = [](const std::string& name) { + return std::vector{makeColumnHint(name, Value::kNullValue)}; + }; + auto expect = [](auto... vidList) { + std::vector ret; + std::vector value; + (value.push_back(std::to_string(vidList)), ...); + for (auto& v : value) { + Row row; + row.emplace_back(v); + ret.emplace_back(std::move(row)); + } + return ret; + }; + /* Case 1: Int*/ { + auto rows = R"( + int | int + 0 | 0 + 9223372036854775807 | + 9223372036854775807 | + -9223372036854775807 | 9223372036854775807 + )"_row; + schema = R"( + a | int | | false + b | int | | true + )"_schema; + auto indices = R"( + TAG(t,1) + (ia,2):a + (ib,3):b + (iba,4):b,a + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + kvstore = std::make_unique(); + for (auto& iter : kv) { + for (auto& item : iter) { + kvstore->put(item.first, item.second); + } + } + check(indices[0], hint("a"), {}, "case1.1"); + check(indices[1], hint("b"), expect(1, 2), "case1.2"); + check(indices[2], hint("b"), expect(1, 2), "case1.3"); + } + /* Case 2: Float */ { + auto rows = R"( + float | float + 1.7976931348623157e+308 | + 0 | + | + | <-NaN> + )"_row; + schema = R"( + a | double | | false + b | double | | true + )"_schema; + auto indices = R"( + TAG(t,1) + (ia,2):a + (ib,3):b + (iba,4):b,a + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + kvstore = std::make_unique(); + for (auto& iter : kv) { + for (auto& item : iter) { + kvstore->put(item.first, item.second); + } + } + check(indices[0], hint("a"), {}, "case2.1"); + check(indices[1], hint("b"), expect(0, 2), "case2.2"); + check(indices[2], hint("b"), expect(0, 2), "case2.3"); + } + /* Case 3: String */ { + auto rows = R"( + string | string + \xFF\xFF\xFF | + 123 | 456 + \xFF\xFF\x01 | \xFF\xFF\xFF + \xFF\xFF\x01 | + )"_row; + schema = R"( + a | string | | false + b | string | | true + )"_schema; + auto indices = R"( + TAG(t,1) + (ia,2):a(3) + (ib,3):b(3) + (iba,4):b(3),a(3) + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + kvstore = std::make_unique(); + for (auto& iter : kv) { + for (auto& item : iter) { + kvstore->put(item.first, item.second); + } + } + check(indices[0], hint("a"), {}, "case3.1"); + check(indices[1], hint("b"), expect(0, 3), "case3.2"); + check(indices[2], hint("b"), expect(0, 3), "case3.3"); + } +} +TEST_F(IndexScanTest, TTL) { + // TODO(hs.zhang): add unittest } -TEST_F(IndexScanTest, Nullable) {} TEST_F(IndexScanTest, Time) { // TODO(hs.zhang): add unittest } @@ -1516,8 +1717,82 @@ TEST_F(IndexScanTest, DateTime) { TEST_F(IndexScanTest, Compound) { // TODO(hs.zhang): add unittest } -TEST_F(IndexScanTest, TTL) {} +class IndexTest : public ::testing::Test { + protected: + static PlanContext* getPlanContext() { + static std::unique_ptr ctx = std::make_unique(nullptr, 0, 8, false); + return ctx.get(); + } + static std::unique_ptr makeContext() { + auto ctx = std::make_unique(getPlanContext()); + ctx->tagId_ = 0; + ctx->edgeType_ = 0; + return ctx; + } + static std::vector collectResult(IndexNode* node) { + std::vector result; + InitContext initCtx; + bool hasNext = false; + node->init(initCtx); + while (true) { + auto res = node->next(hasNext); + ASSERT(::nebula::ok(res)); + if (!hasNext) { + break; + } + result.emplace_back(::nebula::value(std::move(res))); + } + return result; + } + static std::vector pick(const std::vector& rows, const std::vector& indices) { + std::vector ret; + for (auto i : indices) { + ret.push_back(rows[i]); + } + return ret; + } + ::nebula::ObjectPool pool; +}; + +TEST_F(IndexTest, Selection) { + const auto rows = R"( + int | int + 1 | 2 + | + 2 | 10 + 2 | 10 + )"_row; + size_t currentOffset = 0; + auto ctx = makeContext(); + auto expr = RelationalExpression::makeLE(&pool, + TagPropertyExpression::make(&pool, "", "a"), + ConstantExpression::make(&pool, Value(5))); + + auto selection = std::make_unique(ctx.get(), expr); + auto mockChild = std::make_unique(ctx.get()); + mockChild->executeFunc = [&rows](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + mockChild->nextFunc = [&rows, ¤tOffset](bool& hasNext) -> IndexNode::ErrorOr { + if (currentOffset < rows.size()) { + hasNext = true; + return rows[currentOffset++]; + } else { + hasNext = false; + return {}; + } + }; + mockChild->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { + ASSERT(initCtx.requiredColumns.find("a") != initCtx.requiredColumns.end()); + initCtx.returnColumns = {"a", "b"}; + initCtx.retColMap = {{"a", 0}, {"b", 1}}; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + }; + selection->addChild(std::move(mockChild)); + ASSERT_EQ(collectResult(selection.get()), pick(rows, {2, 3})); +} +TEST_F(IndexTest, Projection) {} +TEST_F(IndexTest, Limit) {} +TEST_F(IndexTest, Dedup) {} } // namespace storage } // namespace nebula int main(int argc, char** argv) { diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index c841b8520bb..ecd5139596a 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -120,33 +120,13 @@ class MockKVStore : public ::nebula::kvstore::KVStore { const std::string& end, std::unique_ptr* iter, bool canReadFromFollower = false) override { - return range(spaceId, partId, true, start, false, end, iter, canReadFromFollower); - } - nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, - PartitionID partId, - bool includeStart, - const std::string& start, - bool includeEnd, - const std::string& end, - std::unique_ptr* iter, - bool canReadFromFollower = false) override { UNUSED(spaceId); UNUSED(partId); UNUSED(canReadFromFollower); CHECK_EQ(spaceId, spaceId_); std::unique_ptr mockIter; - if (!includeStart) { - mockIter = std::make_unique(kv_, kv_.upper_bound(start)); - } else { - mockIter = std::make_unique(kv_, kv_.lower_bound(start)); - } - mockIter->setValidFunc([includeEnd, end](const decltype(kv_)::iterator& it) { - DVLOG(2) << includeEnd; - size_t len = end.size(); - int ret = memcmp(it->first.data(), end.data(), len); - DVLOG(2) << ret; - return includeEnd ? ret <= 0 : ret < 0; - }); + mockIter = std::make_unique(kv_, kv_.lower_bound(start)); + mockIter->setValidFunc([end](const decltype(kv_)::iterator& it) { return it->first < end; }); (*iter) = std::move(mockIter); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -397,6 +377,7 @@ class MockKVStore : public ::nebula::kvstore::KVStore { }; class MockIndexNode : public IndexNode { public: + explicit MockIndexNode(RuntimeContext* context) : IndexNode(context, "MockIndexNode") {} ::nebula::cpp2::ErrorCode init(InitContext& initCtx) { return initFunc(initCtx); } std::unique_ptr copy() { LOG(FATAL) << "Unexpect"; } std::function(bool&)> nextFunc; @@ -406,6 +387,8 @@ class MockIndexNode : public IndexNode { private: ErrorOr doNext(bool& hasNext) override { return nextFunc(hasNext); } ::nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { return executeFunc(partId); }; + int offset_; + int total_; }; class RowParser { From 8942749ae71ba8fab49a7af6ef468bea164313a3 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Tue, 19 Oct 2021 10:55:04 +0800 Subject: [PATCH 14/38] add index node test --- src/storage/exec/IndexDedupNode.cpp | 35 ++++-- src/storage/exec/IndexDedupNode.h | 1 + src/storage/exec/IndexLimitNode.cpp | 1 + src/storage/exec/IndexSelectionNode.cpp | 7 +- src/storage/exec/IndexSelectionNode.h | 1 + src/storage/test/IndexTest.cpp | 143 +++++++++++++++++++++++- 6 files changed, 170 insertions(+), 18 deletions(-) diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index 2998df083dc..b714c8890f0 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -28,21 +28,36 @@ ::nebula::cpp2::ErrorCode IndexDedupNode::init(InitContext& ctx) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; } ::nebula::cpp2::ErrorCode IndexDedupNode::doExecute(PartitionID partId) { + currentChild_ = 0; dedupSet_.clear(); return IndexNode::doExecute(partId); } IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) { - DCHECK_EQ(children_.size(), 1); - auto& child = *children_[0]; - do { - auto result = child.next(hasNext); - if (!hasNext || !nebula::ok(result)) { - return result; + Row ret; + hasNext = false; + while (currentChild_ < children_.size()) { + auto& child = *children_[currentChild_]; + do { + auto result = child.next(hasNext); + if (!nebula::ok(result)) { + return result; + } + if (!hasNext) { + break; + } + if (dedup(::nebula::value(result))) { + ret = ::nebula::value(std::move(result)); + hasNext = true; + break; + } + } while (true); + if (!hasNext) { + currentChild_++; + } else { + break; } - if (dedup(::nebula::value(result))) { - return result; - } - } while (true); + } + return ret; } IndexDedupNode::RowWrapper::RowWrapper(const Row& row, const std::vector& posList) { for (auto p : posList) { diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index 83b539c157e..63d664b9ae0 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -44,6 +44,7 @@ class IndexDedupNode : public IndexNode { std::vector dedupColumns_; std::vector dedupPos_; folly::F14FastSet dedupSet_; + size_t currentChild_ = 0; }; /* Definition of inline function */ diff --git a/src/storage/exec/IndexLimitNode.cpp b/src/storage/exec/IndexLimitNode.cpp index ba8d2abaf16..0cf7a5d9970 100644 --- a/src/storage/exec/IndexLimitNode.cpp +++ b/src/storage/exec/IndexLimitNode.cpp @@ -28,6 +28,7 @@ IndexNode::ErrorOr IndexLimitNode::doNext(bool& hasNext) { currentOffset_++; } if (currentOffset_ < offset_ + limit_) { + currentOffset_++; return child.next(hasNext); } else { hasNext = false; diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index c1725e99721..0e10908d33a 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -38,6 +38,7 @@ IndexNode::ErrorOr IndexSelectionNode::doNext(bool& hasNext) { if (!hasNext || !nebula::ok(result)) { return result; } + DVLOG(1) << nebula::value(result); if (filter(nebula::value(result))) { return result; } @@ -54,7 +55,8 @@ Value IndexSelectionNode::ExprContext::getEdgeProp(const std::string& edgeType, auto iter = colPos_.find(prop); DCHECK(iter != colPos_.end()); DCHECK(iter->second < row_->size()); - return row_[iter->second]; + DLOG(INFO) << (*row_)[iter->second]; + return (*row_)[iter->second]; } Value IndexSelectionNode::ExprContext::getTagProp(const std::string& tag, const std::string& prop) const { @@ -63,7 +65,8 @@ Value IndexSelectionNode::ExprContext::getTagProp(const std::string& tag, auto iter = colPos_.find(prop); DCHECK(iter != colPos_.end()); DCHECK(iter->second < row_->size()); - return row_[iter->second]; + DLOG(INFO) << (*row_)[iter->second]; + return (*row_)[iter->second]; } } // namespace storage diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 53ef1f7afc9..eaa21f38406 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -22,6 +22,7 @@ class IndexSelectionNode : public IndexNode { private: ErrorOr doNext(bool &hasNext) override; inline bool filter(const Row &row) { + DLOG(INFO) << row; ctx_->setRow(row); auto &result = expr_->eval(*ctx_); return result.type() == Value::Type::BOOL ? result.getBool() : false; diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index d8ce210036c..3cce6aae2b7 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -20,6 +20,7 @@ #include "kvstore/KVIterator.h" #include "storage/exec/IndexDedupNode.h" #include "storage/exec/IndexEdgeScanNode.h" +#include "storage/exec/IndexLimitNode.h" #include "storage/exec/IndexNode.h" #include "storage/exec/IndexProjectionNode.h" #include "storage/exec/IndexSelectionNode.h" @@ -1760,12 +1761,12 @@ TEST_F(IndexTest, Selection) { int | int 1 | 2 | - 2 | 10 - 2 | 10 + 8 | 10 + 8 | 10 )"_row; size_t currentOffset = 0; auto ctx = makeContext(); - auto expr = RelationalExpression::makeLE(&pool, + auto expr = RelationalExpression::makeGE(&pool, TagPropertyExpression::make(&pool, "", "a"), ConstantExpression::make(&pool, Value(5))); @@ -1790,9 +1791,139 @@ TEST_F(IndexTest, Selection) { selection->addChild(std::move(mockChild)); ASSERT_EQ(collectResult(selection.get()), pick(rows, {2, 3})); } -TEST_F(IndexTest, Projection) {} -TEST_F(IndexTest, Limit) {} -TEST_F(IndexTest, Dedup) {} +TEST_F(IndexTest, Projection) { + const auto rows = R"( + int | int | int + 1 | 2 | 3 + 4 | 5 | 6 + 7 | 8 |9 + )"_row; + size_t currentOffset = 0; + auto ctx = makeContext(); + auto projection = + std::make_unique(ctx.get(), std::vector{"c", "a", "b"}); + auto mockChild = std::make_unique(ctx.get()); + mockChild->executeFunc = [&rows](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + mockChild->nextFunc = [&rows, ¤tOffset](bool& hasNext) -> IndexNode::ErrorOr { + if (currentOffset < rows.size()) { + hasNext = true; + return rows[currentOffset++]; + } else { + hasNext = false; + return {}; + } + }; + mockChild->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { + ASSERT(initCtx.requiredColumns.find("a") != initCtx.requiredColumns.end()); + ASSERT(initCtx.requiredColumns.find("b") != initCtx.requiredColumns.end()); + ASSERT(initCtx.requiredColumns.find("c") != initCtx.requiredColumns.end()); + initCtx.returnColumns = {"a", "b", "c"}; + initCtx.retColMap = {{"a", 0}, {"b", 1}, {"c", 2}}; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + }; + projection->addChild(std::move(mockChild)); + auto expect = R"( + int | int | int + 3 | 1 | 2 + 6 | 4 | 5 + 9 | 7 | 8 + )"_row; + ASSERT_EQ(collectResult(projection.get()), expect); +} +TEST_F(IndexTest, Limit) { + auto genRows = [](int start, int end) { + std::vector ret; + for (int i = start; i < end; i++) { + Row row; + row.emplace_back(Value(i)); + row.emplace_back(Value(i * i)); + row.emplace_back(Value(i * i * i)); + ret.emplace_back(std::move(row)); + } + return ret; + }; + auto rows = genRows(0, 1000); + size_t currentOffset = 0; + auto ctx = makeContext(); + auto limit = std::make_unique(ctx.get(), 10); + auto mockChild = std::make_unique(ctx.get()); + mockChild->executeFunc = [&rows](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + mockChild->nextFunc = [&rows, ¤tOffset](bool& hasNext) -> IndexNode::ErrorOr { + if (currentOffset < rows.size()) { + hasNext = true; + return rows[currentOffset++]; + } else { + hasNext = false; + return {}; + } + }; + mockChild->initFunc = [](InitContext&) -> ::nebula::cpp2::ErrorCode { + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + }; + limit->addChild(std::move(mockChild)); + ASSERT_EQ(collectResult(limit.get()), genRows(0, 10)); +} +TEST_F(IndexTest, Dedup) { + auto rows1 = R"( + int | int + 1 | 2 + 1 | 3 + 2 | 2 + )"_row; + auto rows2 = R"( + int | int + 1 | 4 + 2 | 3 + 1 | 5 + 3 | 6 + )"_row; + size_t offset1 = 0, offset2 = 0; + auto ctx = makeContext(); + auto dedup = std::make_unique(ctx.get(), std::vector{"a"}); + auto child1 = std::make_unique(ctx.get()); + child1->executeFunc = [&rows1](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + child1->nextFunc = [&rows1, &offset1](bool& hasNext) -> IndexNode::ErrorOr { + if (offset1 < rows1.size()) { + hasNext = true; + return rows1[offset1++]; + } else { + hasNext = false; + return {}; + } + }; + child1->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { + ASSERT(initCtx.requiredColumns.find("a") != initCtx.requiredColumns.end()); + initCtx.returnColumns = {"a", "b"}; + initCtx.retColMap = {{"a", 0}, {"b", 1}}; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + }; + auto child2 = std::make_unique(ctx.get()); + child2->executeFunc = [&rows2](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + child2->nextFunc = [&rows2, &offset2](bool& hasNext) -> IndexNode::ErrorOr { + if (offset2 < rows2.size()) { + hasNext = true; + return rows2[offset2++]; + } else { + hasNext = false; + return {}; + } + }; + child2->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { + ASSERT(initCtx.requiredColumns.find("a") != initCtx.requiredColumns.end()); + initCtx.returnColumns = {"a", "b"}; + initCtx.retColMap = {{"a", 0}, {"b", 1}}; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + }; + dedup->addChild(std::move(child1)); + dedup->addChild(std::move(child2)); + auto expect = R"( + int | int + 1 | 2 + 2 | 2 + 3 | 6 + )"_row; + ASSERT_EQ(collectResult(dedup.get()), expect); +} } // namespace storage } // namespace nebula int main(int argc, char** argv) { From 25d71730992c37c1f805ed585ec645b202ae69d6 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Wed, 20 Oct 2021 18:33:09 +0800 Subject: [PATCH 15/38] pass lookupindex test --- src/storage/exec/IndexEdgeScanNode.cpp | 15 ++- src/storage/exec/IndexEdgeScanNode.h | 3 +- src/storage/exec/IndexNode.h | 1 + src/storage/exec/IndexScanNode2.cpp | 116 ++++++++++++++++--- src/storage/exec/IndexScanNode2.h | 7 +- src/storage/exec/IndexVertexScanNode.cpp | 6 +- src/storage/exec/IndexVertexScanNode.h | 3 +- src/storage/index/LookupProcessor2.cpp | 139 +++++++++++++++-------- src/storage/index/LookupProcessor2.h | 4 +- src/storage/test/IndexTest.cpp | 33 +++--- src/storage/test/LookupIndexTest.cpp | 6 +- 11 files changed, 242 insertions(+), 91 deletions(-) diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index 5754d3bc06b..c31a9cf2b70 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -25,12 +25,13 @@ IndexEdgeScanNode::IndexEdgeScanNode(const IndexEdgeScanNode& node) } IndexEdgeScanNode::IndexEdgeScanNode(RuntimeContext* context, IndexID indexId, - const std::vector& columnHint) - : IndexScanNode(context, "IndexEdgeScanNode", indexId, columnHint) { + const std::vector& columnHint, + ::nebula::kvstore::KVStore* kvstore) + : IndexScanNode(context, "IndexEdgeScanNode", indexId, columnHint, kvstore) { getIndex = std::function([this]() { auto env = this->context_->env(); auto indexMgr = env->indexMan_; - auto index = indexMgr->getTagIndex(this->spaceId_, this->indexId_).value(); + auto index = indexMgr->getEdgeIndex(this->spaceId_, this->indexId_).value(); return index; }); getEdge = std::function([this]() { @@ -78,6 +79,9 @@ Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { auto rank = IndexKeyUtils::getIndexRank(context_->vIdLen(), key); values[colPosMap[kRank]] = Value(rank); } + if (colPosMap.count(kType)) { + values[colPosMap[kType]] = Value(context_->edgeType_); + } key.subtract(context_->vIdLen() * 2 + sizeof(EdgeType)); decodePropFromIndex(key, colPosMap, values); return Row(std::move(values)); @@ -99,6 +103,9 @@ Map IndexEdgeScanNode::decodeFromBase(const std::string& key auto reader = RowReaderWrapper::getRowReader(edge_.get(), value); for (auto& col : requiredColumns_) { switch (QueryUtils::toReturnColType(col)) { + case QueryUtils::ReturnColType::kType: { + values[col] = Value(context_->edgeType_); + } break; case QueryUtils::ReturnColType::kSrc: { auto vId = NebulaKeyUtils::getSrcId(context_->vIdLen(), key); if (context_->isIntId()) { @@ -128,6 +135,8 @@ Map IndexEdgeScanNode::decodeFromBase(const std::string& key default: LOG(FATAL) << "Unexpect column name:" << col; } + DLOG(INFO) << col; + DLOG(INFO) << values[col]; } return values; } diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index 9fe15254719..3f243bdc97c 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -17,7 +17,8 @@ class IndexEdgeScanNode : public IndexScanNode { IndexEdgeScanNode(const IndexEdgeScanNode& node); IndexEdgeScanNode(RuntimeContext* context, IndexID indexId, - const std::vector& columnHint); + const std::vector& columnHint, + ::nebula::kvstore::KVStore* kvstore); ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; std::unique_ptr copy() override; diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index f1c8ac5c9a7..43c6b143cf5 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -42,6 +42,7 @@ class IndexNode { void addChild(std::unique_ptr child) { children_.emplace_back(std::move(child)); } virtual std::unique_ptr copy() = 0; const std::vector>& children() { return children_; } + const std::string& name() { return name_; } protected: virtual ErrorOr doNext(bool& hasNext) = 0; diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index f74e9ff6a2a..91745ff27e6 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -46,18 +46,23 @@ std::unique_ptr Path::make(nebula::meta::cpp2::IndexItem* index, const std::vector& hints, int64_t vidLen) { std::unique_ptr ret; - if (hints.back().get_scan_type() == cpp2::ScanType::RANGE) { - ret.reset(new RangePath(index, schema, hints, vidLen)); - } else { + if (hints.empty() || hints.back().get_scan_type() == cpp2::ScanType::PREFIX) { ret.reset(new PrefixPath(index, schema, hints, vidLen)); + + } else { + ret.reset(new RangePath(index, schema, hints, vidLen)); } return ret; } Path::Qualified Path::qualified(const folly::StringPiece& key) { + DLOG(INFO) << "x"; + Qualified ret = Qualified::COMPATIBLE; for (auto& func : QFList_) { ret = std::min(ret, func(key.toString())); } + DLOG(INFO) << "x"; + return ret; } std::string Path::encodeValue(const Value& value, @@ -106,13 +111,29 @@ RangePath::RangePath(nebula::meta::cpp2::IndexItem* index, buildKey(); } void RangePath::resetPart(PartitionID partId) { + DLOG(INFO) << hints_.size(); std::string p = IndexKeyUtils::indexPrefix(partId); - startKey_.replace(0, p.size(), p); - endKey_.replace(0, p.size(), p); + DLOG(INFO) << hints_.size(); + DLOG(INFO) << folly::hexDump(&hints_, sizeof(hints_)); + // DVLOG(3) << '\n' << folly::hexDump(startKey_.data(), startKey_.size()); + DLOG(INFO) << folly::hexDump(&hints_, sizeof(hints_)); + DLOG(INFO) << folly::hexDump(&hints_, sizeof(hints_)); + + DLOG(INFO) << hints_.size(); + startKey_ = startKey_.replace(0, p.size(), p); + DLOG(INFO) << hints_.size(); + // DVLOG(3) << '\n' << folly::hexDump(startKey_.data(), startKey_.size()); + DVLOG(3) << '\n' << folly::hexDump(endKey_.data(), endKey_.size()); + DLOG(INFO) << hints_.size(); + endKey_ = endKey_.replace(0, p.size(), p); + DLOG(INFO) << hints_.size(); + // DVLOG(3) << '\n' << folly::hexDump(endKey_.data(), endKey_.size()); } Path::Qualified RangePath::qualified(const Map& rowData) { + DLOG(INFO) << hints_.size(); for (size_t i = 0; i < hints_.size() - 1; i++) { auto& hint = hints_[i]; + DLOG(INFO) << ::apache::thrift::SimpleJSONSerializer::serialize(hint); if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { return Qualified::INCOMPATIBLE; } @@ -120,6 +141,7 @@ Path::Qualified RangePath::qualified(const Map& rowData) { auto& hint = hints_.back(); // TODO(hs.zhang): improve performance.Check include or not during build key. if (hint.begin_value_ref().is_set()) { + DLOG(INFO) << hint.get_column_name(); bool ret = includeStart_ ? hint.get_begin_value() <= rowData.at(hint.get_column_name()) : hint.get_begin_value() < rowData.at(hint.get_column_name()); if (!ret) { @@ -128,6 +150,7 @@ Path::Qualified RangePath::qualified(const Map& rowData) { } if (hint.end_value_ref().is_set()) { DVLOG(2) << includeEnd_; + DLOG(INFO) << hint.get_column_name(); bool ret = includeEnd_ ? hint.get_end_value() >= rowData.at(hint.get_column_name()) : hint.get_end_value() > rowData.at(hint.get_column_name()); DVLOG(2) << ret; @@ -145,7 +168,7 @@ void RangePath::buildKey() { auto& hint = hints_[i]; CHECK(fieldIter->get_name() == hint.get_column_name()); auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); - CHECK(type == Value::Type::STRING && !fieldIter->get_type().type_length_ref().has_value()); + CHECK(type != Value::Type::STRING || fieldIter->get_type().type_length_ref().has_value()); encodeValue(hint.get_begin_value(), fieldIter->get_type(), i, common); } auto& hint = hints_.back(); @@ -302,6 +325,7 @@ PrefixPath::PrefixPath(nebula::meta::cpp2::IndexItem* index, } Path::Qualified PrefixPath::qualified(const Map& rowData) { for (auto& hint : hints_) { + DLOG(INFO) << hint.get_column_name(); if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { return Qualified::INCOMPATIBLE; } @@ -310,7 +334,7 @@ Path::Qualified PrefixPath::qualified(const Map& rowData) { } void PrefixPath::resetPart(PartitionID partId) { std::string p = IndexKeyUtils::indexPrefix(partId); - prefix_.replace(0, p.size(), p); + prefix_ = prefix_.replace(0, p.size(), p); } void PrefixPath::buildKey() { std::string common; @@ -337,11 +361,37 @@ IndexScanNode::IndexScanNode(const IndexScanNode& node) columnHints_(node.columnHints_), kvstore_(node.kvstore_), requiredColumns_(node.requiredColumns_), - ttlProps_(node.ttlProps_) {} + ttlProps_(node.ttlProps_), + needAccessBase_(node.needAccessBase_) { + if (node.path_->isRange()) { + path_ = std::make_unique(*dynamic_cast(node.path_.get())); + } else { + path_ = std::make_unique(*dynamic_cast(node.path_.get())); + } +} ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { + DLOG(INFO) << columnHints_.size(); + CHECK(requiredColumns_.empty()); + for (auto& hint : columnHints_) { + ctx.requiredColumns.insert(hint.get_column_name()); + } + DLOG(INFO) << "vvvvvv"; + for (auto& col : requiredColumns_) { + DLOG(INFO) << col; + } + DLOG(INFO) << "aaaaaaaaaaa"; + for (auto& x : ctx.requiredColumns) { + DLOG(INFO) << x.size() << '\t' << x; + } + DLOG(INFO) << "bbbbbbbbbbb"; for (auto& col : ctx.requiredColumns) { requiredColumns_.push_back(col); + DLOG(INFO) << col; + } + DLOG(INFO) << "xxxxxxxxxxxxxxxxxxxx"; + for (auto& col : requiredColumns_) { + DLOG(INFO) << col; } ctx.returnColumns = requiredColumns_; for (size_t i = 0; i < ctx.returnColumns.size(); i++) { @@ -361,33 +411,48 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { tmp.erase(kRank); tmp.erase(kSrc); tmp.erase(kDst); + tmp.erase(kType); needAccessBase_ = !tmp.empty(); + DLOG(INFO) << needAccessBase_; + DLOG(INFO) << tmp.size(); path_ = Path::make(index_.get(), getSchema(), columnHints_, context_->vIdLen()); + DLOG(INFO) << columnHints_.size(); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { + DLOG(INFO) << columnHints_.size(); partId_ = partId; auto ret = resetIter(partId); + DLOG(INFO) << columnHints_.size(); return ret; } IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { + DVLOG(3) << "x"; + DVLOG(3) << columnHints_.size(); hasNext = true; for (; iter_ && iter_->valid(); iter_->next()) { DVLOG(1) << '\n' << folly::hexDump(iter_->key().data(), iter_->key().size()); if (!checkTTL()) { continue; } + DVLOG(3) << "x"; auto q = path_->qualified(iter_->key()); if (q == Path::Qualified::INCOMPATIBLE) { continue; } + DVLOG(3) << "x"; bool compatible = q == Path::Qualified::COMPATIBLE; + DLOG(INFO) << compatible; + DLOG(INFO) << needAccessBase_; if (compatible && !needAccessBase_) { DVLOG(3) << 123; - auto key = iter_->key(); + auto key = iter_->key().toString(); iter_->next(); - return decodeFromIndex(key); + auto ret = decodeFromIndex(key); + DVLOG(3) << ret; + return ret; } + DVLOG(3) << "x"; DVLOG(3) << 123; std::pair kv; auto ret = getBaseData(iter_->key(), kv); @@ -399,22 +464,31 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { } continue; } + DVLOG(3) << "x"; + Map rowData = decodeFromBase(kv.first, kv.second); + DVLOG(3) << "x"; + for (auto& iter : rowData) { + DVLOG(3) << iter.first << ":" << iter.second; + } if (!compatible) { - DVLOG(3) << 123; + DVLOG(3) << "x"; + DVLOG(3) << path_.get(); q = path_->qualified(rowData); CHECK(q != Path::Qualified::UNCERTAIN); if (q == Path::Qualified::INCOMPATIBLE) { - DVLOG(3) << 123; continue; } } DVLOG(3) << 123; Row row; for (auto& col : requiredColumns_) { + DVLOG(3) << col; + DVLOG(3) << rowData.at(col); row.emplace_back(std::move(rowData.at(col))); } iter_->next(); + DVLOG(3) << row; return row; } hasNext = false; @@ -434,17 +508,31 @@ bool IndexScanNode::checkTTL() { nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { path_->resetPart(partId); + DLOG(INFO) << columnHints_.size(); + DLOG(INFO) << path_.get(); nebula::cpp2::ErrorCode ret; if (path_->isRange()) { auto rangePath = dynamic_cast(path_.get()); - ret = - kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter_); + DLOG(INFO) << rangePath; + DLOG(INFO) << columnHints_.size(); + DVLOG(1) << '\n' + << folly::hexDump(rangePath->getStartKey().data(), rangePath->getStartKey().size()); + DLOG(INFO) << columnHints_.size(); + DVLOG(1) << '\n' + << folly::hexDump(rangePath->getEndKey().data(), rangePath->getEndKey().size()); + DLOG(INFO) << columnHints_.size(); + std::unique_ptr<::nebula::kvstore::KVIterator> iter; + kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter); + DLOG(INFO) << columnHints_.size(); + iter_ = std::move(iter); + DLOG(INFO) << columnHints_.size(); } else { auto prefixPath = dynamic_cast(path_.get()); DVLOG(1) << '\n' << folly::hexDump(prefixPath->getPrefixKey().data(), prefixPath->getPrefixKey().size()); ret = kvstore_->prefix(spaceId_, partId, prefixPath->getPrefixKey(), &iter_); + DLOG(INFO) << columnHints_.size(); } return ret; } diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index a8a88faf972..9ba084d2234 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -33,8 +33,9 @@ class IndexScanNode : public IndexNode { IndexScanNode(RuntimeContext* context, const std::string& name, IndexID indexId, - const std::vector& columnHints) - : IndexNode(context, name), indexId_(indexId), columnHints_(columnHints) {} + const std::vector& columnHints, + ::nebula::kvstore::KVStore* kvstore) + : IndexNode(context, name), indexId_(indexId), columnHints_(columnHints), kvstore_(kvstore) {} ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; protected: @@ -98,7 +99,7 @@ class Path { std::vector QFList_; ::nebula::meta::cpp2::IndexItem* index_; const meta::SchemaProviderIf* schema_; - const std::vector& hints_; + const std::vector hints_; std::vector nullable_; int64_t index_nullable_offset_{8}; int64_t totalKeyLength_{8}; diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index f5fb1891763..7017fae697e 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -30,8 +30,9 @@ IndexVertexScanNode::IndexVertexScanNode(const IndexVertexScanNode& node) IndexVertexScanNode::IndexVertexScanNode(RuntimeContext* context, IndexID indexId, - const std::vector& clolumnHint) - : IndexScanNode(context, "IndexVertexScanNode", indexId, clolumnHint) { + const std::vector& clolumnHint, + ::nebula::kvstore::KVStore* kvstore) + : IndexScanNode(context, "IndexVertexScanNode", indexId, clolumnHint, kvstore) { getIndex = std::function([this]() { auto env = this->context_->env(); auto indexMgr = env->indexMan_; @@ -87,6 +88,7 @@ Map IndexVertexScanNode::decodeFromBase(const std::string& k Map values; auto reader = RowReaderWrapper::getRowReader(tag_.get(), value); for (auto& col : requiredColumns_) { + DLOG(INFO) << col; switch (QueryUtils::toReturnColType(col)) { case QueryUtils::ReturnColType::kVid: { auto vId = NebulaKeyUtils::getVertexId(context_->vIdLen(), key); diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index 52b74bac758..951326a4661 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -21,7 +21,8 @@ class IndexVertexScanNode final : public IndexScanNode { IndexVertexScanNode(const IndexVertexScanNode& node); IndexVertexScanNode(RuntimeContext* context, IndexID indexId, - const std::vector& clolumnHint); + const std::vector& clolumnHint, + ::nebula::kvstore::KVStore* kvstore); ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; std::unique_ptr copy() override; diff --git a/src/storage/index/LookupProcessor2.cpp b/src/storage/index/LookupProcessor2.cpp index c9105f22a93..df082998ab0 100644 --- a/src/storage/index/LookupProcessor2.cpp +++ b/src/storage/index/LookupProcessor2.cpp @@ -5,7 +5,13 @@ */ #include "storage/index/LookupProcessor2.h" +#include +#include + #include "folly/Likely.h" +#include "interface/gen-cpp2/common_types.tcc" +#include "interface/gen-cpp2/meta_types.tcc" +#include "interface/gen-cpp2/storage_types.tcc" #include "storage/exec/IndexDedupNode.h" #include "storage/exec/IndexEdgeScanNode.h" #include "storage/exec/IndexLimitNode.h" @@ -17,6 +23,7 @@ namespace nebula { namespace storage { ProcessorCounters kLookupCounters; void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { + DLOG(INFO) << ::apache::thrift::SimpleJSONSerializer::serialize(req); if (executor_ != nullptr) { executor_->add([req, this]() { this->doProcess(req); }); } else { @@ -26,8 +33,6 @@ void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { void LookupProcessor::doProcess(const cpp2::LookupIndexRequest& req) { prepare(req); auto plan = buildPlan(req); - InitContext context; - plan->init(context); if (!FLAGS_query_concurrently) { runInSingleThread(req.get_parts(), std::move(plan)); } else { @@ -44,6 +49,21 @@ ::nebula::cpp2::ErrorCode LookupProcessor::prepare(const cpp2::LookupIndexReques planContext_->isEdge_ = req.get_indices().get_schema_id().getType() == nebula::cpp2::SchemaID::Type::edge_type; context_ = std::make_unique(this->planContext_.get()); + std::string schemaName; + if (planContext_->isEdge_) { + auto edgeType = req.get_indices().get_schema_id().get_edge_type(); + schemaName = env_->schemaMan_->toEdgeName(req.get_space_id(), edgeType).value(); + context_->edgeType_ = edgeType; + } else { + auto tagId = req.get_indices().get_schema_id().get_tag_id(); + schemaName = env_->schemaMan_->toTagName(req.get_space_id(), tagId).value(); + context_->tagId_ = tagId; + } + std::vector colNames; + for (auto& col : *req.get_return_columns()) { + colNames.emplace_back(schemaName + "." + col); + } + resultDataSet_ = ::nebula::DataSet(colNames); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -51,6 +71,7 @@ std::unique_ptr LookupProcessor::buildPlan(const cpp2::LookupIndexReq std::vector> nodes; for (auto& ctx : req.get_indices().get_contexts()) { auto node = buildOneContext(ctx); + nodes.emplace_back(std::move(node)); } for (size_t i = 0; i < nodes.size(); i++) { auto projection = @@ -67,7 +88,7 @@ std::unique_ptr LookupProcessor::buildPlan(const cpp2::LookupIndexReq } auto dedup = std::make_unique(context_.get(), dedupColumn); for (auto& node : nodes) { - node->addChild(std::move(node)); + dedup->addChild(std::move(node)); } nodes.clear(); nodes[0] = std::move(dedup); @@ -88,30 +109,48 @@ std::unique_ptr LookupProcessor::buildPlan(const cpp2::LookupIndexReq std::unique_ptr LookupProcessor::buildOneContext(const cpp2::IndexQueryContext& ctx) { std::unique_ptr node; + DLOG(INFO) << ctx.get_column_hints().size(); + DLOG(INFO) << &ctx.get_column_hints(); + DLOG(INFO) << ::apache::thrift::SimpleJSONSerializer::serialize(ctx); if (context_->isEdge()) { node = std::make_unique( - context_.get(), ctx.get_index_id(), ctx.get_column_hints()); + context_.get(), ctx.get_index_id(), ctx.get_column_hints(), context_->env()->kvstore_); } else { node = std::make_unique( - context_.get(), ctx.get_index_id(), ctx.get_column_hints()); + context_.get(), ctx.get_index_id(), ctx.get_column_hints(), context_->env()->kvstore_); } - if (ctx.filter_ref().is_set()) { - auto filterNode = std::make_unique(context_.get(), nullptr); + if (ctx.filter_ref().is_set() && !ctx.get_filter().empty()) { + auto expr = Expression::decode(context_->objPool(), *ctx.filter_ref()); + auto filterNode = std::make_unique(context_.get(), expr); + filterNode->addChild(std::move(node)); + node = std::move(filterNode); } return node; } +void printPlan(IndexNode* node, int tab = 0) { + DLOG(INFO) << std::string(tab, '\t') << node->name() << "(" << node << ")"; + for (auto& child : node->children()) { + printPlan(child.get(), tab + 1); + } +} void LookupProcessor::runInSingleThread(const std::vector& parts, std::unique_ptr plan) { + printPlan(plan.get()); std::vector> datasetList; std::vector<::nebula::cpp2::ErrorCode> codeList; for (auto part : parts) { + DLOG(INFO) << "execute part:" << part; plan->execute(part); bool hasNext = true; - ::nebula::cpp2::ErrorCode code; + ::nebula::cpp2::ErrorCode code = ::nebula::cpp2::ErrorCode::SUCCEEDED; decltype(datasetList)::value_type dataset; do { auto result = plan->next(hasNext); - if (hasNext && ::nebula::ok(result)) { + if (!::nebula::ok(result)) { + code = ::nebula::error(result); + break; + } + if (hasNext) { dataset.emplace_back(::nebula::value(std::move(result))); } else { break; @@ -127,63 +166,67 @@ void LookupProcessor::runInSingleThread(const std::vector& parts, datasetList[i].pop_front(); } } else { + DLOG(INFO) << int(codeList[i]); handleErrorCode(codeList[i], context_->spaceId(), parts[i]); } } onProcessFinished(); onFinished(); } + void LookupProcessor::runInMultipleThread(const std::vector& parts, std::unique_ptr plan) { std::vector> planCopy = reproducePlan(plan.get(), parts.size()); - std::vector> datasetList(parts.size()); - std::vector<::nebula::cpp2::ErrorCode> codeList(parts.size()); - std::vector> funcs; - std::vector> futures; + using ReturnType = std::tuple>; + std::vector> futures; for (size_t i = 0; i < parts.size(); i++) { - funcs.emplace_back([this, - &plan = planCopy[i], - &dataset = datasetList[i], - &code = codeList[i], - part = parts[i], - index = i]() { - plan->execute(part); - bool hasNext = true; - do { - auto result = plan->next(hasNext); - if (hasNext && ::nebula::ok(result)) { - dataset.emplace_back(::nebula::value(std::move(result))); - } else { - break; + futures.emplace_back(folly::via( + executor_, [this, plan = std::move(planCopy[i]), part = parts[i]]() -> ReturnType { + ::nebula::cpp2::ErrorCode code = ::nebula::cpp2::ErrorCode::SUCCEEDED; + std::deque dataset; + plan->execute(part); + bool hasNext = true; + do { + auto result = plan->next(hasNext); + if (!::nebula::ok(result)) { + code = ::nebula::error(result); + break; + } + if (hasNext) { + dataset.emplace_back(::nebula::value(std::move(result))); + } else { + break; + } + } while (true); + return {part, code, dataset}; + })); + } + DLOG(INFO) << "xxxxxxxxxxxxxxxxxxxxxxx"; + folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) { + CHECK(!t.hasException()); + const auto& tries = t.value(); + for (size_t j = 0; j < tries.size(); j++) { + CHECK(!tries[j].hasException()); + auto& [partId, code, dataset] = tries[j].value(); + if (code == ::nebula::cpp2::ErrorCode::SUCCEEDED) { + for (auto& row : dataset) { + resultDataSet_.emplace_back(std::move(row)); } - } while (true); - return index; - }); - } - for (size_t i = 0; i < parts.size(); i++) { - futures.emplace_back(folly::via(executor_, std::move(funcs[i]))); - } - for (size_t i = 0; i < parts.size(); i++) { - futures[i].result(); - } - for (size_t i = 0; i < datasetList.size(); i++) { - if (codeList[i] == ::nebula::cpp2::ErrorCode::SUCCEEDED) { - while (!datasetList[i].empty()) { - resultDataSet_.emplace_back(std::move(datasetList[i].front())); - datasetList[i].pop_front(); + } else { + handleErrorCode(code, context_->spaceId(), partId); } - } else { - handleErrorCode(codeList[i], context_->spaceId(), parts[i]); } - } - onProcessFinished(); - onFinished(); + DLOG(INFO) << "finish"; + this->onProcessFinished(); + this->onFinished(); + }); } std::vector> LookupProcessor::reproducePlan(IndexNode* root, size_t count) { std::vector> ret(count); for (size_t i = 0; i < count; i++) { - ret.emplace_back(root->copy()); + ret[i] = root->copy(); + DLOG(INFO) << ret[i].get(); } for (auto& child : root->children()) { auto childPerPlan = reproducePlan(child.get(), count); diff --git a/src/storage/index/LookupProcessor2.h b/src/storage/index/LookupProcessor2.h index a329f55adff..ed22562ffee 100644 --- a/src/storage/index/LookupProcessor2.h +++ b/src/storage/index/LookupProcessor2.h @@ -26,7 +26,9 @@ class LookupProcessor : public BaseProcessor { LookupProcessor(StorageEnv* env, const ProcessorCounters* counters, folly::Executor* executor) : BaseProcessor(env, counters), executor_(executor) {} void doProcess(const cpp2::LookupIndexRequest& req); - void onProcessFinished() {} + void onProcessFinished() { + BaseProcessor::resp_.set_data(std::move(resultDataSet_)); + } void runInSingleThread(const std::vector& parts, std::unique_ptr plan); void runInMultipleThread(const std::vector& parts, std::unique_ptr plan); diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 3cce6aae2b7..50f710a4bca 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -68,7 +68,6 @@ using std::string_literals::operator""s; * */ struct IndexScanTestHelper { - void setKVStore(IndexScanNode* node, kvstore::KVStore* store) { node->kvstore_ = store; } void setIndex(IndexVertexScanNode* node, std::shared_ptr<::nebula::meta::cpp2::IndexItem> index) { node->getIndex = [index]() { return index; }; } @@ -526,9 +525,9 @@ TEST_F(IndexScanTest, Int) { const std::string& case_) { DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); - auto scanNode = std::make_unique(context.get(), 0, columnHints); + auto scanNode = + std::make_unique(context.get(), 0, columnHints, kvstore.get()); IndexScanTestHelper helper; - helper.setKVStore(scanNode.get(), kvstore.get()); helper.setIndex(scanNode.get(), index); helper.setTag(scanNode.get(), schema); InitContext initCtx; @@ -740,9 +739,9 @@ float | float | float | int const std::string& case_) { DVLOG(1) << "Start case " << case_; auto context = makeContext(0, 1); - auto scanNode = std::make_unique(context.get(), 0, columnHints); + auto scanNode = + std::make_unique(context.get(), 0, columnHints, kvstore.get()); IndexScanTestHelper helper; - helper.setKVStore(scanNode.get(), kvstore.get()); helper.setIndex(scanNode.get(), index); helper.setEdge(scanNode.get(), schema); InitContext initCtx; @@ -963,9 +962,9 @@ TEST_F(IndexScanTest, Bool) { const std::string& case_) { DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); - auto scanNode = std::make_unique(context.get(), 0, columnHints); + auto scanNode = + std::make_unique(context.get(), 0, columnHints, kvstore.get()); IndexScanTestHelper helper; - helper.setKVStore(scanNode.get(), kvstore.get()); helper.setIndex(scanNode.get(), index); helper.setTag(scanNode.get(), schema); InitContext initCtx; @@ -1053,9 +1052,9 @@ TEST_F(IndexScanTest, String1) { const std::string& case_) { DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); - auto scanNode = std::make_unique(context.get(), 0, columnHints); + auto scanNode = + std::make_unique(context.get(), 0, columnHints, kvstore.get()); IndexScanTestHelper helper; - helper.setKVStore(scanNode.get(), kvstore.get()); helper.setIndex(scanNode.get(), index); helper.setTag(scanNode.get(), schema); helper.setFatal(scanNode.get(), true); @@ -1210,9 +1209,9 @@ TEST_F(IndexScanTest, String2) { const std::string& case_) { DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); - auto scanNode = std::make_unique(context.get(), 0, columnHints); + auto scanNode = + std::make_unique(context.get(), 0, columnHints, kvstore.get()); IndexScanTestHelper helper; - helper.setKVStore(scanNode.get(), kvstore.get()); helper.setIndex(scanNode.get(), index); helper.setTag(scanNode.get(), schema); helper.setFatal(scanNode.get(), true); @@ -1341,9 +1340,9 @@ TEST_F(IndexScanTest, String3) { const std::string& case_) { DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); - auto scanNode = std::make_unique(context.get(), 0, columnHints); + auto scanNode = + std::make_unique(context.get(), 0, columnHints, kvstore.get()); IndexScanTestHelper helper; - helper.setKVStore(scanNode.get(), kvstore.get()); helper.setIndex(scanNode.get(), index); helper.setTag(scanNode.get(), schema); helper.setFatal(scanNode.get(), true); @@ -1463,9 +1462,9 @@ TEST_F(IndexScanTest, String4) { const std::string& case_) { DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); - auto scanNode = std::make_unique(context.get(), 0, columnHints); + auto scanNode = + std::make_unique(context.get(), 0, columnHints, kvstore.get()); IndexScanTestHelper helper; - helper.setKVStore(scanNode.get(), kvstore.get()); helper.setIndex(scanNode.get(), index); helper.setTag(scanNode.get(), schema); helper.setFatal(scanNode.get(), true); @@ -1579,9 +1578,9 @@ TEST_F(IndexScanTest, Nullable) { const std::string& case_) { DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); - auto scanNode = std::make_unique(context.get(), 0, columnHints); + auto scanNode = + std::make_unique(context.get(), 0, columnHints, kvstore.get()); IndexScanTestHelper helper; - helper.setKVStore(scanNode.get(), kvstore.get()); helper.setIndex(scanNode.get(), index); helper.setTag(scanNode.get(), schema); helper.setFatal(scanNode.get(), true); diff --git a/src/storage/test/LookupIndexTest.cpp b/src/storage/test/LookupIndexTest.cpp index c9d35dc892e..6fbaf565ed1 100644 --- a/src/storage/test/LookupIndexTest.cpp +++ b/src/storage/test/LookupIndexTest.cpp @@ -10,6 +10,10 @@ #include "codec/RowWriterV2.h" #include "codec/test/RowWriterV1.h" #include "common/base/Base.h" +#include "common/expression/ConstantExpression.h" +#include "common/expression/LogicalExpression.h" +#include "common/expression/PropertyExpression.h" +#include "common/expression/RelationalExpression.h" #include "common/fs/TempDir.h" #include "common/utils/IndexKeyUtils.h" #include "interface/gen-cpp2/common_types.h" @@ -18,7 +22,7 @@ #include "mock/AdHocSchemaManager.h" #include "mock/MockCluster.h" #include "mock/MockData.h" -#include "storage/index/LookupProcessor.h" +#include "storage/index/LookupProcessor2.h" #include "storage/mutate/AddVerticesProcessor.h" #include "storage/test/QueryTestUtils.h" From 48076d0795db1f3b59113ee13fe5bfca1ab182aa Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Thu, 21 Oct 2021 10:50:18 +0800 Subject: [PATCH 16/38] pass ttl test --- src/storage/exec/IndexEdgeNode.h | 108 ------- src/storage/exec/IndexFilterNode.h | 171 ------------ src/storage/exec/IndexOutputNode.h | 357 ------------------------ src/storage/exec/IndexScanNode.h | 187 ------------- src/storage/exec/IndexScanNode2.cpp | 1 + src/storage/exec/IndexVertexNode.h | 100 ------- src/storage/test/IndexScanLimitTest.cpp | 2 +- src/storage/test/IndexWithTTLTest.cpp | 2 +- src/storage/test/IndexWriteTest.cpp | 2 +- 9 files changed, 4 insertions(+), 926 deletions(-) delete mode 100644 src/storage/exec/IndexEdgeNode.h delete mode 100644 src/storage/exec/IndexFilterNode.h delete mode 100644 src/storage/exec/IndexOutputNode.h delete mode 100644 src/storage/exec/IndexScanNode.h delete mode 100644 src/storage/exec/IndexVertexNode.h diff --git a/src/storage/exec/IndexEdgeNode.h b/src/storage/exec/IndexEdgeNode.h deleted file mode 100644 index f9daabb2aef..00000000000 --- a/src/storage/exec/IndexEdgeNode.h +++ /dev/null @@ -1,108 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ -#ifndef STORAGE_EXEC_INDEXEDGENODE_H_ -#define STORAGE_EXEC_INDEXEDGENODE_H_ - -#include "common/base/Base.h" -#include "storage/exec/IndexScanNode.h" -#include "storage/exec/RelNode.h" - -namespace nebula { -namespace storage { - -template -class IndexEdgeNode final : public RelNode { - public: - using RelNode::doExecute; - - IndexEdgeNode(RuntimeContext* context, - IndexScanNode* indexScanNode, - const std::vector>& schemas, - const std::string& schemaName, - int64_t limit = -1) - : context_(context), - indexScanNode_(indexScanNode), - schemas_(schemas), - schemaName_(schemaName), - limit_(limit) { - RelNode::name_ = "IndexEdgeNode"; - } - - nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { - auto ret = RelNode::doExecute(partId); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - return ret; - } - - auto ttlProp = CommonUtils::ttlProps(context_->edgeSchema_); - - data_.clear(); - std::vector edges; - auto* iter = static_cast(indexScanNode_->iterator()); - while (iter && iter->valid()) { - if (context_->isPlanKilled()) { - return nebula::cpp2::ErrorCode::E_PLAN_IS_KILLED; - } - if (!iter->val().empty() && ttlProp.first) { - auto v = IndexKeyUtils::parseIndexTTL(iter->val()); - if (CommonUtils::checkDataExpiredForTTL( - context_->edgeSchema_, std::move(v), ttlProp.second.second, ttlProp.second.first)) { - iter->next(); - continue; - } - } - storage::cpp2::EdgeKey edge; - edge.set_src(iter->srcId()); - edge.set_edge_type(context_->edgeType_); - edge.set_ranking(iter->ranking()); - edge.set_dst(iter->dstId()); - edges.emplace_back(std::move(edge)); - iter->next(); - } - int64_t count = 0; - for (const auto& edge : edges) { - auto key = NebulaKeyUtils::edgeKey(context_->vIdLen(), - partId, - (*edge.src_ref()).getStr(), - context_->edgeType_, - edge.get_ranking(), - (*edge.dst_ref()).getStr()); - std::string val; - ret = context_->env()->kvstore_->get(context_->spaceId(), partId, key, &val); - if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { - data_.emplace_back(std::move(key), std::move(val)); - } else if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - continue; - } else { - return ret; - } - if (limit_ > 0 && ++count >= limit_) { - break; - } - } - return nebula::cpp2::ErrorCode::SUCCEEDED; - } - - std::vector moveData() { return std::move(data_); } - - const std::vector>& getSchemas() { - return schemas_; - } - - const std::string& getSchemaName() { return schemaName_; } - - private: - RuntimeContext* context_; - IndexScanNode* indexScanNode_; - const std::vector>& schemas_; - const std::string& schemaName_; - int64_t limit_; - std::vector data_; -}; - -} // namespace storage -} // namespace nebula -#endif // STORAGE_EXEC_INDEXEDGENODE_H_ diff --git a/src/storage/exec/IndexFilterNode.h b/src/storage/exec/IndexFilterNode.h deleted file mode 100644 index 23d9620a4fe..00000000000 --- a/src/storage/exec/IndexFilterNode.h +++ /dev/null @@ -1,171 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ -#ifndef STORAGE_EXEC_INDEXFILTERNODE_H_ -#define STORAGE_EXEC_INDEXFILTERNODE_H_ - -#include "common/base/Base.h" -#include "common/context/ExpressionContext.h" -#include "common/expression/Expression.h" -#include "storage/exec/IndexEdgeNode.h" -#include "storage/exec/IndexScanNode.h" -#include "storage/exec/IndexVertexNode.h" -#include "storage/exec/RelNode.h" - -namespace nebula { -namespace storage { - -template -class IndexFilterNode final : public RelNode { - public: - using RelNode::doExecute; - - // evalExprByIndex_ is true, all fileds in filter is in index. No need to read - // data anymore. - IndexFilterNode(RuntimeContext* context, - IndexScanNode* indexScanNode, - StorageExpressionContext* exprCtx, - Expression* exp, - bool isEdge, - int64_t limit = -1) - : context_(context), - indexScanNode_(indexScanNode), - exprCtx_(exprCtx), - filterExp_(exp), - isEdge_(isEdge), - limit_(limit) { - evalExprByIndex_ = true; - RelNode::name_ = "IndexFilterNode"; - } - - // evalExprByIndex_ is false, some fileds in filter is out of index, which - // need to read data. - IndexFilterNode(RuntimeContext* context, - IndexEdgeNode* indexEdgeNode, - StorageExpressionContext* exprCtx, - Expression* exp, - int64_t limit = -1) - : context_(context), - indexEdgeNode_(indexEdgeNode), - exprCtx_(exprCtx), - filterExp_(exp), - limit_(limit) { - evalExprByIndex_ = false; - isEdge_ = true; - } - - // evalExprByIndex_ is false, some fileds in filter is out of index, which - // need to read data. - IndexFilterNode(RuntimeContext* context, - IndexVertexNode* indexVertexNode, - StorageExpressionContext* exprCtx, - Expression* exp, - int64_t limit = -1) - : context_(context), - indexVertexNode_(indexVertexNode), - exprCtx_(exprCtx), - filterExp_(exp), - limit_(limit) { - evalExprByIndex_ = false; - isEdge_ = false; - } - - nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { - data_.clear(); - auto ret = RelNode::doExecute(partId); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - return ret; - } - std::vector data; - if (evalExprByIndex_) { - data = indexScanNode_->moveData(); - } else if (isEdge_) { - data = indexEdgeNode_->moveData(); - } else { - data = indexVertexNode_->moveData(); - } - int64_t count = 0; - for (const auto& k : data) { - if (context_->isPlanKilled()) { - return nebula::cpp2::ErrorCode::E_PLAN_IS_KILLED; - } - if (evalExprByIndex_) { - if (check(k.first)) { - data_.emplace_back(k.first, k.second); - count++; - } - } else { - const auto& schemas = - isEdge_ ? indexEdgeNode_->getSchemas() : indexVertexNode_->getSchemas(); - auto reader = RowReaderWrapper::getRowReader(schemas, k.second); - if (!reader) { - continue; - } - if (check(reader.get(), k.first)) { - data_.emplace_back(k.first, k.second); - count++; - } - } - if (limit_ > 0 && count >= limit_) { - break; - } - } - return nebula::cpp2::ErrorCode::SUCCEEDED; - } - - std::vector moveData() { return std::move(data_); } - - const std::vector>& getSchemas() { - return isEdge_ ? indexEdgeNode_->getSchemas() : indexVertexNode_->getSchemas(); - } - - bool hasNullableCol() const { return exprCtx_->hasNullableCol(); } - - const std::vector& indexCols() const { return exprCtx_->indexCols(); } - - private: - bool check(const std::string& raw) { - if (filterExp_ != nullptr) { - exprCtx_->reset(raw); - auto result = filterExp_->eval(*exprCtx_); - if (result.type() == Value::Type::BOOL) { - return result.getBool(); - } else { - return false; - } - } - return false; - } - - bool check(RowReader* reader, const std::string& raw) { - if (filterExp_ != nullptr) { - exprCtx_->reset(reader, raw); - auto result = filterExp_->eval(*exprCtx_); - if (result.type() == Value::Type::BOOL) { - return result.getBool(); - } else { - return false; - } - } - return false; - } - - private: - RuntimeContext* context_; - IndexScanNode* indexScanNode_{nullptr}; - IndexEdgeNode* indexEdgeNode_{nullptr}; - IndexVertexNode* indexVertexNode_{nullptr}; - StorageExpressionContext* exprCtx_; - Expression* filterExp_; - bool isEdge_; - bool evalExprByIndex_; - int64_t limit_; - std::vector data_{}; -}; - -} // namespace storage -} // namespace nebula - -#endif // STORAGE_EXEC_INDEXFILTERNODE_H_ diff --git a/src/storage/exec/IndexOutputNode.h b/src/storage/exec/IndexOutputNode.h deleted file mode 100644 index c9ed8433ceb..00000000000 --- a/src/storage/exec/IndexOutputNode.h +++ /dev/null @@ -1,357 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ -#ifndef STORAGE_EXEC_INDEXOUTPUTNODE_H_ -#define STORAGE_EXEC_INDEXOUTPUTNODE_H_ - -#include "common/base/Base.h" -#include "storage/exec/IndexEdgeNode.h" -#include "storage/exec/IndexFilterNode.h" -#include "storage/exec/IndexScanNode.h" -#include "storage/exec/IndexVertexNode.h" -#include "storage/exec/RelNode.h" - -namespace nebula { -namespace storage { - -template -class IndexOutputNode final : public RelNode { - public: - using RelNode::doExecute; - - enum class IndexResultType : int8_t { - kEdgeFromIndexScan, - kEdgeFromIndexFilter, - kEdgeFromDataScan, - kEdgeFromDataFilter, - kVertexFromIndexScan, - kVertexFromIndexFilter, - kVertexFromDataScan, - kVertexFromDataFilter, - }; - - IndexOutputNode(nebula::DataSet* result, - RuntimeContext* context, - IndexScanNode* indexScanNode, - bool hasNullableCol, - const std::vector& fields) - : result_(result), - context_(context), - indexScanNode_(indexScanNode), - hasNullableCol_(hasNullableCol), - fields_(fields) { - type_ = context_->isEdge() ? IndexResultType::kEdgeFromIndexScan - : IndexResultType::kVertexFromIndexScan; - RelNode::name_ = "IndexOpuputNode"; - } - - IndexOutputNode(nebula::DataSet* result, RuntimeContext* context, IndexEdgeNode* indexEdgeNode) - : result_(result), context_(context), indexEdgeNode_(indexEdgeNode) { - type_ = IndexResultType::kEdgeFromDataScan; - RelNode::name_ = "IndexOpuputNode"; - } - - IndexOutputNode(nebula::DataSet* result, - RuntimeContext* context, - IndexVertexNode* indexVertexNode) - : result_(result), context_(context), indexVertexNode_(indexVertexNode) { - type_ = IndexResultType::kVertexFromDataScan; - RelNode::name_ = "IndexOpuputNode"; - } - - IndexOutputNode(nebula::DataSet* result, - RuntimeContext* context, - IndexFilterNode* indexFilterNode, - bool indexFilter = false) - : result_(result), context_(context), indexFilterNode_(indexFilterNode) { - hasNullableCol_ = indexFilterNode->hasNullableCol(); - fields_ = indexFilterNode_->indexCols(); - if (indexFilter) { - type_ = context_->isEdge() ? IndexResultType::kEdgeFromIndexFilter - : IndexResultType::kVertexFromIndexFilter; - } else { - type_ = context_->isEdge() ? IndexResultType::kEdgeFromDataFilter - : IndexResultType::kVertexFromDataFilter; - } - RelNode::name_ = "IndexOpuputNode"; - } - - nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { - auto ret = RelNode::doExecute(partId); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - return ret; - } - - switch (type_) { - case IndexResultType::kEdgeFromIndexScan: { - ret = collectResult(indexScanNode_->moveData()); - break; - } - case IndexResultType::kEdgeFromIndexFilter: { - ret = collectResult(indexFilterNode_->moveData()); - break; - } - case IndexResultType::kEdgeFromDataScan: { - ret = collectResult(indexEdgeNode_->moveData()); - break; - } - case IndexResultType::kEdgeFromDataFilter: { - ret = collectResult(indexFilterNode_->moveData()); - break; - } - case IndexResultType::kVertexFromIndexScan: { - ret = collectResult(indexScanNode_->moveData()); - break; - } - case IndexResultType::kVertexFromIndexFilter: { - ret = collectResult(indexFilterNode_->moveData()); - break; - } - case IndexResultType::kVertexFromDataScan: { - ret = collectResult(indexVertexNode_->moveData()); - break; - } - case IndexResultType::kVertexFromDataFilter: { - ret = collectResult(indexFilterNode_->moveData()); - break; - } - } - return ret; - } - - private: - nebula::cpp2::ErrorCode collectResult(const std::vector& data) { - if (context_->isPlanKilled()) { - return nebula::cpp2::ErrorCode::E_PLAN_IS_KILLED; - } - auto ret = nebula::cpp2::ErrorCode::SUCCEEDED; - switch (type_) { - case IndexResultType::kEdgeFromIndexScan: - case IndexResultType::kEdgeFromIndexFilter: { - ret = edgeRowsFromIndex(data); - break; - } - case IndexResultType::kEdgeFromDataScan: - case IndexResultType::kEdgeFromDataFilter: { - ret = edgeRowsFromData(data); - break; - } - case IndexResultType::kVertexFromIndexScan: - case IndexResultType::kVertexFromIndexFilter: { - ret = vertexRowsFromIndex(data); - break; - } - case IndexResultType::kVertexFromDataScan: - case IndexResultType::kVertexFromDataFilter: { - ret = vertexRowsFromData(data); - break; - } - } - return ret; - } - - nebula::cpp2::ErrorCode vertexRowsFromData(const std::vector& data) { - const auto& schemas = type_ == IndexResultType::kVertexFromDataScan - ? indexVertexNode_->getSchemas() - : indexFilterNode_->getSchemas(); - if (schemas.empty()) { - return nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND; - } - for (const auto& val : data) { - Row row; - auto reader = RowReaderWrapper::getRowReader(schemas, val.second); - if (!reader) { - VLOG(1) << "Can't get tag reader"; - return nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND; - } - for (const auto& col : result_->colNames) { - auto ret = addIndexValue(row, reader.get(), val, col, schemas.back().get()); - if (!ret.ok()) { - return nebula::cpp2::ErrorCode::E_INVALID_DATA; - } - } - result_->rows.emplace_back(std::move(row)); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; - } - - nebula::cpp2::ErrorCode vertexRowsFromIndex(const std::vector& data) { - for (const auto& val : data) { - Row row; - for (const auto& col : result_->colNames) { - auto ret = addIndexValue(row, val, col); - if (!ret.ok()) { - return nebula::cpp2::ErrorCode::E_INVALID_DATA; - } - } - result_->rows.emplace_back(std::move(row)); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; - } - - nebula::cpp2::ErrorCode edgeRowsFromData(const std::vector& data) { - const auto& schemas = type_ == IndexResultType::kEdgeFromDataScan - ? indexEdgeNode_->getSchemas() - : indexFilterNode_->getSchemas(); - if (schemas.empty()) { - return nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND; - } - for (const auto& val : data) { - Row row; - auto reader = RowReaderWrapper::getRowReader(schemas, val.second); - if (!reader) { - VLOG(1) << "Can't get tag reader"; - return nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND; - } - for (const auto& col : result_->colNames) { - auto ret = addIndexValue(row, reader.get(), val, col, schemas.back().get()); - if (!ret.ok()) { - return nebula::cpp2::ErrorCode::E_INVALID_DATA; - } - } - result_->rows.emplace_back(std::move(row)); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; - } - - nebula::cpp2::ErrorCode edgeRowsFromIndex(const std::vector& data) { - for (const auto& val : data) { - Row row; - for (const auto& col : result_->colNames) { - auto ret = addIndexValue(row, val, col); - if (!ret.ok()) { - return nebula::cpp2::ErrorCode::E_INVALID_DATA; - } - } - result_->rows.emplace_back(std::move(row)); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; - } - - // Add the value by data val - Status addIndexValue(Row& row, - RowReader* reader, - const kvstore::KV& data, - const std::string& col, - const meta::NebulaSchemaProvider* schema) { - switch (QueryUtils::toReturnColType(col)) { - case QueryUtils::ReturnColType::kVid: { - auto vId = NebulaKeyUtils::getVertexId(context_->vIdLen(), data.first); - if (context_->isIntId()) { - row.emplace_back(*reinterpret_cast(vId.data())); - } else { - row.emplace_back(vId.subpiece(0, vId.find_first_of('\0')).toString()); - } - break; - } - case QueryUtils::ReturnColType::kTag: { - row.emplace_back(NebulaKeyUtils::getTagId(context_->vIdLen(), data.first)); - break; - } - case QueryUtils::ReturnColType::kSrc: { - auto src = NebulaKeyUtils::getSrcId(context_->vIdLen(), data.first); - if (context_->isIntId()) { - row.emplace_back(*reinterpret_cast(src.data())); - } else { - row.emplace_back(src.subpiece(0, src.find_first_of('\0')).toString()); - } - break; - } - case QueryUtils::ReturnColType::kType: { - row.emplace_back(NebulaKeyUtils::getEdgeType(context_->vIdLen(), data.first)); - break; - } - case QueryUtils::ReturnColType::kRank: { - row.emplace_back(NebulaKeyUtils::getRank(context_->vIdLen(), data.first)); - break; - } - case QueryUtils::ReturnColType::kDst: { - auto dst = NebulaKeyUtils::getDstId(context_->vIdLen(), data.first); - if (context_->isIntId()) { - row.emplace_back(*reinterpret_cast(dst.data())); - } else { - row.emplace_back(dst.subpiece(0, dst.find_first_of('\0')).toString()); - } - break; - } - default: { - auto retVal = QueryUtils::readValue(reader, col, schema); - if (!retVal.ok()) { - VLOG(3) << "Bad value for field : " << col; - return retVal.status(); - } - row.emplace_back(std::move(retVal.value())); - } - } - return Status::OK(); - } - - // Add the value by index key - Status addIndexValue(Row& row, const kvstore::KV& data, const std::string& col) { - switch (QueryUtils::toReturnColType(col)) { - case QueryUtils::ReturnColType::kVid: { - auto vId = IndexKeyUtils::getIndexVertexID(context_->vIdLen(), data.first); - if (context_->isIntId()) { - row.emplace_back(*reinterpret_cast(vId.data())); - } else { - row.emplace_back(vId.subpiece(0, vId.find_first_of('\0')).toString()); - } - break; - } - case QueryUtils::ReturnColType::kTag: { - row.emplace_back(context_->tagId_); - break; - } - case QueryUtils::ReturnColType::kSrc: { - auto src = IndexKeyUtils::getIndexSrcId(context_->vIdLen(), data.first); - if (context_->isIntId()) { - row.emplace_back(*reinterpret_cast(src.data())); - } else { - row.emplace_back(src.subpiece(0, src.find_first_of('\0')).toString()); - } - break; - } - case QueryUtils::ReturnColType::kType: { - row.emplace_back(context_->edgeType_); - break; - } - case QueryUtils::ReturnColType::kRank: { - row.emplace_back(IndexKeyUtils::getIndexRank(context_->vIdLen(), data.first)); - break; - } - case QueryUtils::ReturnColType::kDst: { - auto dst = IndexKeyUtils::getIndexDstId(context_->vIdLen(), data.first); - if (context_->isIntId()) { - row.emplace_back(*reinterpret_cast(dst.data())); - } else { - row.emplace_back(dst.subpiece(0, dst.find_first_of('\0')).toString()); - } - break; - } - default: { - auto v = IndexKeyUtils::getValueFromIndexKey( - context_->vIdLen(), data.first, col, fields_, context_->isEdge(), hasNullableCol_); - row.emplace_back(std::move(v)); - } - } - return Status::OK(); - } - - private: - nebula::DataSet* result_; - RuntimeContext* context_; - IndexResultType type_; - IndexScanNode* indexScanNode_{nullptr}; - IndexEdgeNode* indexEdgeNode_{nullptr}; - IndexVertexNode* indexVertexNode_{nullptr}; - IndexFilterNode* indexFilterNode_{nullptr}; - bool hasNullableCol_{}; - std::vector fields_; -}; - -} // namespace storage -} // namespace nebula - -#endif // STORAGE_EXEC_INDEXOUTPUTNODE_H_ diff --git a/src/storage/exec/IndexScanNode.h b/src/storage/exec/IndexScanNode.h deleted file mode 100644 index 4a25f4ba6a1..00000000000 --- a/src/storage/exec/IndexScanNode.h +++ /dev/null @@ -1,187 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#ifndef STORAGE_EXEC_INDEXSCANNODE_H_ -#define STORAGE_EXEC_INDEXSCANNODE_H_ - -#include "common/base/Base.h" -#include "storage/exec/RelNode.h" -#include "storage/exec/StorageIterator.h" - -namespace nebula { -namespace storage { -template -class IndexScanNode : public RelNode { - public: - using RelNode::doExecute; - - IndexScanNode(RuntimeContext* context, - IndexID indexId, - std::vector columnHints, - int64_t limit = -1) - : context_(context), indexId_(indexId), columnHints_(std::move(columnHints)), limit_(limit) { - /** - * columnHints's elements are {scanType = PREFIX|RANGE; beginStr; endStr}, - * {scanType = PREFIX|RANGE; beginStr; - * endStr},... if the scanType is RANGE, means the index scan is range scan. - * if all scanType are PREFIX, means the index scan is prefix scan. - * there should be only one RANGE hnit, and it must be the last one. - */ - for (size_t i = 0; i < columnHints_.size(); i++) { - if (columnHints_[i].get_scan_type() == cpp2::ScanType::RANGE) { - isRangeScan_ = true; - CHECK_EQ(columnHints_.size() - 1, i); - break; - } - } - RelNode::name_ = "IndexScanNode"; - } - - nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { - auto ret = RelNode::doExecute(partId); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - return ret; - } - auto scanRet = scanStr(partId); - if (!scanRet.ok()) { - return nebula::cpp2::ErrorCode::E_INVALID_FIELD_VALUE; - } - scanPair_ = scanRet.value(); - std::unique_ptr iter; - ret = isRangeScan_ - ? this->kvstore_->range( - context_->spaceId(), partId, scanPair_.first, scanPair_.second, &iter) - : this->kvstore_->prefix(context_->spaceId(), partId, scanPair_.first, &iter); - if (ret == nebula::cpp2::ErrorCode::SUCCEEDED && iter && iter->valid()) { - context_->isEdge() - ? iter_.reset(new EdgeIndexIterator(std::move(iter), context_->vIdLen())) - : iter_.reset(new VertexIndexIterator(std::move(iter), context_->vIdLen())); - } else { - iter_.reset(); - return ret; - } - return nebula::cpp2::ErrorCode::SUCCEEDED; - } - - IndexIterator* iterator() { return iter_.get(); } - - std::vector moveData() { - auto* sh = context_->isEdge() ? context_->edgeSchema_ : context_->tagSchema_; - auto ttlProp = CommonUtils::ttlProps(sh); - data_.clear(); - int64_t count = 0; - while (!!iter_ && iter_->valid()) { - if (context_->isPlanKilled()) { - return {}; - } - if (!iter_->val().empty() && ttlProp.first) { - auto v = IndexKeyUtils::parseIndexTTL(iter_->val()); - if (CommonUtils::checkDataExpiredForTTL( - sh, std::move(v), ttlProp.second.second, ttlProp.second.first)) { - iter_->next(); - continue; - } - } - data_.emplace_back(iter_->key(), ""); - if (limit_ > 0 && ++count >= limit_) { - break; - } - iter_->next(); - } - return std::move(data_); - } - - private: - StatusOr> scanStr(PartitionID partId) { - auto iRet = context_->isEdge() - ? context_->env()->indexMan_->getEdgeIndex(context_->spaceId(), indexId_) - : context_->env()->indexMan_->getTagIndex(context_->spaceId(), indexId_); - if (!iRet.ok()) { - return Status::IndexNotFound(); - } - if (isRangeScan_) { - return getRangeStr(partId, iRet.value()->get_fields()); - } else { - return getPrefixStr(partId, iRet.value()->get_fields()); - } - } - - StatusOr> getPrefixStr( - PartitionID partId, const std::vector<::nebula::meta::cpp2::ColumnDef>& fields) { - std::string prefix; - prefix.append(IndexKeyUtils::indexPrefix(partId, indexId_)); - for (auto& col : columnHints_) { - auto iter = std::find_if(fields.begin(), fields.end(), [col](const auto& field) { - return col.get_column_name() == field.get_name(); - }); - if (iter == fields.end()) { - VLOG(3) << "Field " << col.get_column_name() << " not found "; - return Status::Error("Field not found"); - } - auto type = IndexKeyUtils::toValueType(iter->type.type); - if (type == Value::Type::STRING && !iter->type.type_length_ref().has_value()) { - return Status::Error("String property index has not set prefix length."); - } - prefix.append(encodeValue(*col.begin_value_ref(), type, iter->type.get_type_length())); - } - return std::make_pair(prefix, ""); - } - - StatusOr> getRangeStr( - PartitionID partId, const std::vector<::nebula::meta::cpp2::ColumnDef>& fields) { - std::string start, end; - start.append(IndexKeyUtils::indexPrefix(partId, indexId_)); - end.append(IndexKeyUtils::indexPrefix(partId, indexId_)); - for (auto& col : columnHints_) { - auto iter = std::find_if(fields.begin(), fields.end(), [col](const auto& field) { - return col.get_column_name() == field.get_name(); - }); - if (iter == fields.end()) { - VLOG(3) << "Field " << col.get_column_name() << " not found "; - return Status::Error("Field not found"); - } - auto type = IndexKeyUtils::toValueType(iter->get_type().get_type()); - if (type == Value::Type::STRING && !iter->get_type().type_length_ref().has_value()) { - return Status::Error("String property index has not set prefix length."); - } - if (col.get_scan_type() == cpp2::ScanType::PREFIX) { - start.append(encodeValue(*col.begin_value_ref(), type, iter->type.get_type_length())); - end.append(encodeValue(*col.begin_value_ref(), type, iter->type.get_type_length())); - } else { - start.append(encodeValue(*col.begin_value_ref(), type, iter->type.get_type_length())); - end.append(encodeValue(*col.end_value_ref(), type, iter->type.get_type_length())); - } - } - return std::make_pair(start, end); - } - - // precondition: if type is STRING, strLen must be valid - std::string encodeValue(const Value& val, Value::Type type, const int16_t* strLen) { - if (val.isNull()) { - return IndexKeyUtils::encodeNullValue(type, strLen); - } - if (type == Value::Type::STRING) { - return IndexKeyUtils::encodeValue(val, *strLen); - } else { - return IndexKeyUtils::encodeValue(val); - } - } - - private: - RuntimeContext* context_; - IndexID indexId_; - bool isRangeScan_{false}; - std::unique_ptr iter_; - std::pair scanPair_; - std::vector columnHints_; - int64_t limit_; - std::vector data_; -}; - -} // namespace storage -} // namespace nebula - -#endif // STORAGE_EXEC_INDEXSCANNODE_H_ diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index 91745ff27e6..5e4fd35c86c 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -371,6 +371,7 @@ IndexScanNode::IndexScanNode(const IndexScanNode& node) } ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { + ttlProps_ = CommonUtils::ttlProps(getSchema()); DLOG(INFO) << columnHints_.size(); CHECK(requiredColumns_.empty()); for (auto& hint : columnHints_) { diff --git a/src/storage/exec/IndexVertexNode.h b/src/storage/exec/IndexVertexNode.h deleted file mode 100644 index 7f0eff653db..00000000000 --- a/src/storage/exec/IndexVertexNode.h +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ -#ifndef STORAGE_EXEC_INDEXVERTEXNODE_H_ -#define STORAGE_EXEC_INDEXVERTEXNODE_H_ - -#include "common/base/Base.h" -#include "storage/exec/IndexScanNode.h" -#include "storage/exec/RelNode.h" - -namespace nebula { -namespace storage { - -template -class IndexVertexNode final : public RelNode { - public: - using RelNode::doExecute; - - IndexVertexNode(RuntimeContext* context, - IndexScanNode* indexScanNode, - const std::vector>& schemas, - const std::string& schemaName, - int64_t limit = -1) - : context_(context), - indexScanNode_(indexScanNode), - schemas_(schemas), - schemaName_(schemaName), - limit_(limit) { - RelNode::name_ = "IndexVertexNode"; - } - - nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { - auto ret = RelNode::doExecute(partId); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - return ret; - } - - auto ttlProp = CommonUtils::ttlProps(context_->tagSchema_); - - data_.clear(); - std::vector vids; - auto* iter = static_cast(indexScanNode_->iterator()); - - while (iter && iter->valid()) { - if (context_->isPlanKilled()) { - return nebula::cpp2::ErrorCode::E_PLAN_IS_KILLED; - } - if (!iter->val().empty() && ttlProp.first) { - auto v = IndexKeyUtils::parseIndexTTL(iter->val()); - if (CommonUtils::checkDataExpiredForTTL( - context_->tagSchema_, std::move(v), ttlProp.second.second, ttlProp.second.first)) { - iter->next(); - continue; - } - } - vids.emplace_back(iter->vId()); - iter->next(); - } - int64_t count = 0; - for (const auto& vId : vids) { - VLOG(1) << "partId " << partId << ", vId " << vId << ", tagId " << context_->tagId_; - auto key = NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId, context_->tagId_); - std::string val; - ret = context_->env()->kvstore_->get(context_->spaceId(), partId, key, &val); - if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { - data_.emplace_back(std::move(key), std::move(val)); - } else if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { - continue; - } else { - return ret; - } - if (limit_ > 0 && ++count >= limit_) { - break; - } - } - return nebula::cpp2::ErrorCode::SUCCEEDED; - } - - std::vector moveData() { return std::move(data_); } - - const std::vector>& getSchemas() { - return schemas_; - } - - const std::string& getSchemaName() { return schemaName_; } - - private: - RuntimeContext* context_; - IndexScanNode* indexScanNode_; - const std::vector>& schemas_; - const std::string& schemaName_; - int64_t limit_; - std::vector data_; -}; - -} // namespace storage -} // namespace nebula -#endif // STORAGE_EXEC_INDEXVERTEXNODE_H_ diff --git a/src/storage/test/IndexScanLimitTest.cpp b/src/storage/test/IndexScanLimitTest.cpp index a73e0186a80..e28f3e78b92 100644 --- a/src/storage/test/IndexScanLimitTest.cpp +++ b/src/storage/test/IndexScanLimitTest.cpp @@ -14,7 +14,7 @@ #include "mock/AdHocIndexManager.h" #include "mock/AdHocSchemaManager.h" #include "mock/MockCluster.h" -#include "storage/index/LookupProcessor.h" +#include "storage/index/LookupProcessor2.h" #include "storage/query/GetNeighborsProcessor.h" #include "storage/test/TestUtils.h" diff --git a/src/storage/test/IndexWithTTLTest.cpp b/src/storage/test/IndexWithTTLTest.cpp index 00f9d62d587..733bd8514fd 100644 --- a/src/storage/test/IndexWithTTLTest.cpp +++ b/src/storage/test/IndexWithTTLTest.cpp @@ -19,7 +19,7 @@ #include "storage/admin/AdminTaskManager.h" #include "storage/admin/RebuildEdgeIndexTask.h" #include "storage/admin/RebuildTagIndexTask.h" -#include "storage/index/LookupProcessor.h" +#include "storage/index/LookupProcessor2.h" #include "storage/mutate/AddEdgesProcessor.h" #include "storage/mutate/AddVerticesProcessor.h" #include "storage/mutate/UpdateEdgeProcessor.h" diff --git a/src/storage/test/IndexWriteTest.cpp b/src/storage/test/IndexWriteTest.cpp index 7a8dc0bd85b..ba22ab5e6b6 100644 --- a/src/storage/test/IndexWriteTest.cpp +++ b/src/storage/test/IndexWriteTest.cpp @@ -16,7 +16,7 @@ #include "mock/AdHocSchemaManager.h" #include "mock/MockCluster.h" #include "mock/MockData.h" -#include "storage/index/LookupProcessor.h" +#include "storage/index/LookupProcessor2.h" #include "storage/mutate/AddEdgesProcessor.h" #include "storage/mutate/AddVerticesProcessor.h" #include "storage/mutate/DeleteEdgesProcessor.h" From 45006be5adaac7838cd9cf4f82d225f9f166538a Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Thu, 21 Oct 2021 17:15:44 +0800 Subject: [PATCH 17/38] pass all unittest --- src/common/utils/test/CMakeLists.txt | 3 ++ src/daemons/CMakeLists.txt | 2 ++ src/kvstore/test/CMakeLists.txt | 1 + src/storage/CommonUtils.h | 3 ++ src/storage/exec/IndexDedupNode.cpp | 3 ++ src/storage/exec/IndexDedupNode.h | 1 + src/storage/exec/IndexEdgeScanNode.cpp | 2 +- src/storage/exec/IndexLimitNode.cpp | 7 ++++ src/storage/exec/IndexLimitNode.h | 1 + src/storage/exec/IndexNode.h | 41 ++++++++++++++++++--- src/storage/exec/IndexProjectionNode.cpp | 3 ++ src/storage/exec/IndexProjectionNode.h | 1 + src/storage/exec/IndexScanNode2.cpp | 45 +++++++++++++----------- src/storage/exec/IndexScanNode2.h | 5 +++ src/storage/exec/IndexSelectionNode.cpp | 3 ++ src/storage/exec/IndexSelectionNode.h | 1 + src/storage/exec/IndexVertexScanNode.cpp | 2 +- src/storage/index/LookupProcessor2.cpp | 31 ++++++++++++++++ src/storage/index/LookupProcessor2.h | 2 +- src/storage/test/DeleteTagsTest.cpp | 2 +- src/storage/test/IndexScanTest.cpp | 3 +- src/storage/test/IndexTest.cpp | 44 +++++++++++------------ src/storage/test/IndexTestUtil.h | 1 + src/storage/test/KillQueryTest.cpp | 2 +- src/tools/db-dump/CMakeLists.txt | 1 + src/tools/meta-dump/CMakeLists.txt | 1 + 26 files changed, 157 insertions(+), 54 deletions(-) diff --git a/src/common/utils/test/CMakeLists.txt b/src/common/utils/test/CMakeLists.txt index d02df8575dc..fc68f5711a6 100644 --- a/src/common/utils/test/CMakeLists.txt +++ b/src/common/utils/test/CMakeLists.txt @@ -11,6 +11,7 @@ nebula_add_test( $ $ $ + $ $ LIBRARIES gtest @@ -30,6 +31,7 @@ nebula_add_test( $ $ $ + $ $ LIBRARIES gtest @@ -49,6 +51,7 @@ nebula_add_test( $ $ $ + $ $ LIBRARIES gtest diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 565c2717f54..cd8f332327a 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -45,6 +45,7 @@ set(storage_meta_deps $ $ $ + $ $ ) @@ -128,6 +129,7 @@ nebula_add_executable( $ $ $ + $ $ ${common_deps} LIBRARIES diff --git a/src/kvstore/test/CMakeLists.txt b/src/kvstore/test/CMakeLists.txt index aa039b23854..4162921d13a 100644 --- a/src/kvstore/test/CMakeLists.txt +++ b/src/kvstore/test/CMakeLists.txt @@ -33,6 +33,7 @@ set(KVSTORE_TEST_LIBS $ $ $ + $ $ ) diff --git a/src/storage/CommonUtils.h b/src/storage/CommonUtils.h index a5de1487d26..2bb115fd79b 100644 --- a/src/storage/CommonUtils.h +++ b/src/storage/CommonUtils.h @@ -201,6 +201,9 @@ struct RuntimeContext { ObjectPool* objPool() { return &planContext_->objPool_; } bool isPlanKilled() { + if (env() == nullptr) { + return false; + } return env()->metaClient_ && env()->metaClient_->checkIsPlanKilled(planContext_->sessionId_, planContext_->planId_); } diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index b714c8890f0..a768d0e0eee 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -67,6 +67,9 @@ IndexDedupNode::RowWrapper::RowWrapper(const Row& row, const std::vector std::unique_ptr IndexDedupNode::copy() { return std::make_unique(*this); } +std::string IndexDedupNode::identify() { + return fmt::format("{}(dedup=[{}])", name_, folly::join(',', dedupColumns_)); +} } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index 63d664b9ae0..6e53fbc910c 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -15,6 +15,7 @@ class IndexDedupNode : public IndexNode { IndexDedupNode(RuntimeContext* context, const std::vector& dedupColumn); ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; std::unique_ptr copy() override; + std::string identify() override; private: inline bool dedup(const Row& row); diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index c31a9cf2b70..1b4d0acd89c 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -101,7 +101,7 @@ Map IndexEdgeScanNode::decodeFromBase(const std::string& key const std::string& value) { Map values; auto reader = RowReaderWrapper::getRowReader(edge_.get(), value); - for (auto& col : requiredColumns_) { + for (auto& col : requiredAndHintColumns_) { switch (QueryUtils::toReturnColType(col)) { case QueryUtils::ReturnColType::kType: { values[col] = Value(context_->edgeType_); diff --git a/src/storage/exec/IndexLimitNode.cpp b/src/storage/exec/IndexLimitNode.cpp index 0cf7a5d9970..afe5e661d19 100644 --- a/src/storage/exec/IndexLimitNode.cpp +++ b/src/storage/exec/IndexLimitNode.cpp @@ -38,6 +38,13 @@ IndexNode::ErrorOr IndexLimitNode::doNext(bool& hasNext) { std::unique_ptr IndexLimitNode::copy() { return std::make_unique(*this); } +std::string IndexLimitNode::identify() { + if (offset_ > 0) { + return fmt::format("{}(offset={}, limit={})", name_, offset_, limit_); + } else { + return fmt::format("{}(limit={})", name_, limit_); + } +} } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexLimitNode.h b/src/storage/exec/IndexLimitNode.h index 8525207d5d9..64b0087e43f 100644 --- a/src/storage/exec/IndexLimitNode.h +++ b/src/storage/exec/IndexLimitNode.h @@ -14,6 +14,7 @@ class IndexLimitNode : public IndexNode { IndexLimitNode(RuntimeContext* context, uint64_t offset, uint64_t limit); IndexLimitNode(RuntimeContext* context, uint64_t limit); std::unique_ptr copy() override; + std::string identify() override; private: nebula::cpp2::ErrorCode doExecute(PartitionID partId) override; diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index 43c6b143cf5..acef0649394 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -44,6 +44,11 @@ class IndexNode { const std::vector>& children() { return children_; } const std::string& name() { return name_; } + void enableProfileDetail(); + + virtual std::string identify() = 0; + inline const time::Duration& duration(); + protected: virtual ErrorOr doNext(bool& hasNext) = 0; void beforeNext(); @@ -57,24 +62,52 @@ class IndexNode { std::vector> children_; std::string name_; time::Duration duration_; + bool profileDetail_{false}; }; /* Defination of inline function */ inline IndexNode::ErrorOr IndexNode::next(bool& hasNext) { beforeNext(); + if (context_->isPlanKilled()) { + return nebula::cpp2::ErrorCode::E_PLAN_IS_KILLED; + } auto ret = doNext(hasNext); afterNext(); return ret; } -inline void IndexNode::beforeNext() { duration_.resume(); } -inline void IndexNode::afterNext() { duration_.pause(); } +inline void IndexNode::beforeNext() { + if (UNLIKELY(profileDetail_)) { + duration_.resume(); + } +} +inline void IndexNode::afterNext() { + if (UNLIKELY(profileDetail_)) { + duration_.pause(); + } +} inline nebula::cpp2::ErrorCode IndexNode::execute(PartitionID partId) { beforeExecute(); auto ret = doExecute(partId); afterExecute(); return std::move(ret); } -inline void IndexNode::beforeExecute() { duration_.resume(); } -inline void IndexNode::afterExecute() { duration_.pause(); } +inline void IndexNode::beforeExecute() { + if (UNLIKELY(profileDetail_)) { + duration_.resume(); + } +} +inline void IndexNode::afterExecute() { + if (UNLIKELY(profileDetail_)) { + duration_.pause(); + } +} +inline void IndexNode::enableProfileDetail() { + profileDetail_ = true; + for (auto& child : children_) { + child->enableProfileDetail(); + } +} +inline const time::Duration& IndexNode::duration() { return duration_; } + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp index b7379d23e8c..fd6536e919b 100644 --- a/src/storage/exec/IndexProjectionNode.cpp +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -52,6 +52,9 @@ Row IndexProjectionNode::project(Row&& row) { std::unique_ptr IndexProjectionNode::copy() { return std::make_unique(*this); } +std::string IndexProjectionNode::identify() { + return fmt::format("{}(projectColumn=[{}])", name_, folly::join(",", requiredColumns_)); +} } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index b252cadfc86..0d61246b9c3 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -15,6 +15,7 @@ class IndexProjectionNode : public IndexNode { IndexProjectionNode(RuntimeContext* context, const std::vector& requiredColumns); nebula::cpp2::ErrorCode init(InitContext& ctx) override; std::unique_ptr copy() override; + std::string identify() override; private: ErrorOr doNext(bool& hasNext) override; diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index 5e4fd35c86c..3c529d72b39 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -100,6 +100,8 @@ std::string Path::encodeValue(const Value& value, key.append(val); return val; } +const std::string& Path::toString() { return serializeString_; } + // End of Path // Define of RangePath @@ -170,10 +172,22 @@ void RangePath::buildKey() { auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); CHECK(type != Value::Type::STRING || fieldIter->get_type().type_length_ref().has_value()); encodeValue(hint.get_begin_value(), fieldIter->get_type(), i, common); + serializeString_ += + fmt::format("{}={}, ", hint.get_column_name(), hint.get_begin_value().toString()); } auto& hint = hints_.back(); size_t index = hints_.size() - 1; auto [a, b] = encodeRange(hint, fieldIter->get_type(), index, common.size()); + std::string left = + hint.begin_value_ref().is_set() + ? fmt::format( + "{}{}", hint.get_include_begin() ? '[' : '(', hint.get_begin_value().toString()) + : "[-INF"; + std::string right = + hint.end_value_ref().is_set() + ? fmt::format("{}{}", hint.get_end_value().toString(), hint.get_include_end() ? ']' : ')') + : "INF]"; + serializeString_ += fmt::format("{}={},{}", hint.get_column_name(), left, right); startKey_ = common + a; endKey_ = common + b; if (!hint.end_value_ref().is_set()) { @@ -346,6 +360,8 @@ void PrefixPath::buildKey() { auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); CHECK(type != Value::Type::STRING || fieldIter->get_type().type_length_ref().has_value()); encodeValue(hint.get_begin_value(), fieldIter->get_type(), i, common); + serializeString_ += + fmt::format("{}={}, ", hint.get_column_name(), hint.get_begin_value().toString()); } prefix_ = std::move(common); } @@ -361,6 +377,7 @@ IndexScanNode::IndexScanNode(const IndexScanNode& node) columnHints_(node.columnHints_), kvstore_(node.kvstore_), requiredColumns_(node.requiredColumns_), + requiredAndHintColumns_(node.requiredAndHintColumns_), ttlProps_(node.ttlProps_), needAccessBase_(node.needAccessBase_) { if (node.path_->isRange()) { @@ -371,28 +388,15 @@ IndexScanNode::IndexScanNode(const IndexScanNode& node) } ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { + DCHECK(requiredColumns_.empty()); ttlProps_ = CommonUtils::ttlProps(getSchema()); - DLOG(INFO) << columnHints_.size(); - CHECK(requiredColumns_.empty()); + requiredAndHintColumns_ = ctx.requiredColumns; + for (auto& hint : columnHints_) { - ctx.requiredColumns.insert(hint.get_column_name()); - } - DLOG(INFO) << "vvvvvv"; - for (auto& col : requiredColumns_) { - DLOG(INFO) << col; - } - DLOG(INFO) << "aaaaaaaaaaa"; - for (auto& x : ctx.requiredColumns) { - DLOG(INFO) << x.size() << '\t' << x; + requiredAndHintColumns_.insert(hint.get_column_name()); } - DLOG(INFO) << "bbbbbbbbbbb"; for (auto& col : ctx.requiredColumns) { requiredColumns_.push_back(col); - DLOG(INFO) << col; - } - DLOG(INFO) << "xxxxxxxxxxxxxxxxxxxx"; - for (auto& col : requiredColumns_) { - DLOG(INFO) << col; } ctx.returnColumns = requiredColumns_; for (size_t i = 0; i < ctx.returnColumns.size(); i++) { @@ -414,10 +418,7 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { tmp.erase(kDst); tmp.erase(kType); needAccessBase_ = !tmp.empty(); - DLOG(INFO) << needAccessBase_; - DLOG(INFO) << tmp.size(); path_ = Path::make(index_.get(), getSchema(), columnHints_, context_->vIdLen()); - DLOG(INFO) << columnHints_.size(); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { @@ -588,7 +589,9 @@ void IndexScanNode::decodePropFromIndex(folly::StringPiece key, nullableColPosit -= 1; } } - +std::string IndexScanNode::identify() { + return fmt::format("{}(IndexID={}, Path=({}))", name_, indexId_, path_->toString()); +} // End of IndexScan } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index 9ba084d2234..9a7e2ec95d0 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -37,6 +37,7 @@ class IndexScanNode : public IndexNode { ::nebula::kvstore::KVStore* kvstore) : IndexNode(context, name), indexId_(indexId), columnHints_(columnHints), kvstore_(kvstore) {} ::nebula::cpp2::ErrorCode init(InitContext& ctx) override; + std::string identify() override; protected: nebula::cpp2::ErrorCode doExecute(PartitionID partId) final; @@ -61,6 +62,7 @@ class IndexScanNode : public IndexNode { std::unique_ptr iter_; nebula::kvstore::KVStore* kvstore_; std::vector requiredColumns_; + Set requiredAndHintColumns_; std::pair> ttlProps_; bool needAccessBase_{false}; bool fatalOnBaseNotFound_{false}; @@ -90,6 +92,7 @@ class Path { virtual Qualified qualified(const Map& rowData) = 0; virtual void resetPart(PartitionID partId) = 0; + const std::string& toString(); protected: std::string encodeValue(const Value& value, @@ -104,6 +107,7 @@ class Path { int64_t index_nullable_offset_{8}; int64_t totalKeyLength_{8}; int64_t suffixLength_; + std::string serializeString_; }; class PrefixPath : public Path { public: @@ -161,6 +165,7 @@ class RangePath : public Path { }; /* define inline functions */ + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index 0e10908d33a..36e6272b03c 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -48,6 +48,9 @@ IndexNode::ErrorOr IndexSelectionNode::doNext(bool& hasNext) { std::unique_ptr IndexSelectionNode::copy() { return std::make_unique(*this); } +std::string IndexSelectionNode::identify() { + return fmt::format("{}(expr=[{}])", name_, expr_->toString()); +} Value IndexSelectionNode::ExprContext::getEdgeProp(const std::string& edgeType, const std::string& prop) const { UNUSED(edgeType); diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index eaa21f38406..019def98d0d 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -18,6 +18,7 @@ class IndexSelectionNode : public IndexNode { IndexSelectionNode(RuntimeContext *context, Expression *expr); nebula::cpp2::ErrorCode init(InitContext &ctx) override; std::unique_ptr copy() override; + std::string identify() override; private: ErrorOr doNext(bool &hasNext) override; diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index 7017fae697e..65185a8939f 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -87,7 +87,7 @@ Map IndexVertexScanNode::decodeFromBase(const std::string& k const std::string& value) { Map values; auto reader = RowReaderWrapper::getRowReader(tag_.get(), value); - for (auto& col : requiredColumns_) { + for (auto& col : requiredAndHintColumns_) { DLOG(INFO) << col; switch (QueryUtils::toReturnColType(col)) { case QueryUtils::ReturnColType::kVid: { diff --git a/src/storage/index/LookupProcessor2.cpp b/src/storage/index/LookupProcessor2.cpp index df082998ab0..b0bfb18f721 100644 --- a/src/storage/index/LookupProcessor2.cpp +++ b/src/storage/index/LookupProcessor2.cpp @@ -31,6 +31,9 @@ void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { } } void LookupProcessor::doProcess(const cpp2::LookupIndexRequest& req) { + if (req.common_ref().has_value() && req.get_common()->profile_detail_ref().value_or(false)) { + profileDetailFlag_ = true; + } prepare(req); auto plan = buildPlan(req); if (!FLAGS_query_concurrently) { @@ -101,6 +104,10 @@ std::unique_ptr LookupProcessor::buildPlan(const cpp2::LookupIndexReq } InitContext ctx; auto result = nodes[0]->init(ctx); + if (profileDetailFlag_) { + nodes[0]->enableProfileDetail(); + } + // TODO(hs.zhang): check init result if (result == ::nebula::cpp2::ErrorCode::SUCCEEDED) { } else { } @@ -170,6 +177,9 @@ void LookupProcessor::runInSingleThread(const std::vector& parts, handleErrorCode(codeList[i], context_->spaceId(), parts[i]); } } + if (UNLIKELY(profileDetailFlag_)) { + profilePlan(plan.get()); + } onProcessFinished(); onFinished(); } @@ -198,6 +208,9 @@ void LookupProcessor::runInMultipleThread(const std::vector& parts, break; } } while (true); + if (UNLIKELY(profileDetailFlag_)) { + profilePlan(plan.get()); + } return {part, code, dataset}; })); } @@ -236,6 +249,24 @@ std::vector> LookupProcessor::reproducePlan(IndexNode } return ret; } +void LookupProcessor::profilePlan(IndexNode* root) { + std::queue q; + q.push(root); + while (!q.empty()) { + auto node = q.front(); + q.pop(); + auto id = node->identify(); + auto iter = profileDetail_.find(id); + if (iter == profileDetail_.end()) { + profileDetail_[id] = node->duration().elapsedInUSec(); + } else { + iter->second += node->duration().elapsedInUSec(); + } + for (auto& child : node->children()) { + q.push(child.get()); + } + } +} } // namespace storage } // namespace nebula diff --git a/src/storage/index/LookupProcessor2.h b/src/storage/index/LookupProcessor2.h index ed22562ffee..3767f395ca7 100644 --- a/src/storage/index/LookupProcessor2.h +++ b/src/storage/index/LookupProcessor2.h @@ -29,7 +29,7 @@ class LookupProcessor : public BaseProcessor { void onProcessFinished() { BaseProcessor::resp_.set_data(std::move(resultDataSet_)); } - + void profilePlan(IndexNode* plan); void runInSingleThread(const std::vector& parts, std::unique_ptr plan); void runInMultipleThread(const std::vector& parts, std::unique_ptr plan); ::nebula::cpp2::ErrorCode prepare(const cpp2::LookupIndexRequest& req); diff --git a/src/storage/test/DeleteTagsTest.cpp b/src/storage/test/DeleteTagsTest.cpp index abd9e7889bd..1f2cccbbf99 100644 --- a/src/storage/test/DeleteTagsTest.cpp +++ b/src/storage/test/DeleteTagsTest.cpp @@ -14,7 +14,7 @@ #include "interface/gen-cpp2/storage_types.h" #include "mock/MockCluster.h" #include "mock/MockData.h" -#include "storage/index/LookupProcessor.h" +#include "storage/index/LookupProcessor2.h" #include "storage/mutate/DeleteTagsProcessor.h" #include "storage/query/GetPropProcessor.h" #include "storage/test/QueryTestUtils.h" diff --git a/src/storage/test/IndexScanTest.cpp b/src/storage/test/IndexScanTest.cpp index 73072fe8ec9..78ba4e050ef 100644 --- a/src/storage/test/IndexScanTest.cpp +++ b/src/storage/test/IndexScanTest.cpp @@ -77,7 +77,8 @@ static std::string genEdgeIndexKey(meta::SchemaManager* schemaMan, VertexID dst) { auto reader = RowReaderWrapper::getEdgePropReader(schemaMan, prop, spaceId, type); auto values = collectIndexValues(reader.get(), index->get_fields()); - auto indexKey = NebulaKeyUtils::edgeIndexKey(partId, index->get_index_id(), src, 0, dst, values); + auto indexKey = + NebulaKeyUtils::edgeIndexKeys(partId, index->get_index_id(), src, 0, dst, values)[0]; return indexKey; } diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 50f710a4bca..5747c900d08 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -145,8 +145,8 @@ class IndexScanTest : public ::testing::Test { auto& index = indices[j]; auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index->get_fields()).value(); DVLOG(1) << folly::hexDump(indexValue.data(), indexValue.size()); - auto indexKey = IndexKeyUtils::vertexIndexKey( - 8, 0, index->get_index_id(), std::to_string(i), std::move(indexValue)); + auto indexKey = IndexKeyUtils::vertexIndexKeys( + 8, 0, index->get_index_id(), std::to_string(i), std::move(indexValue))[0]; assert(ret[j + 1].insert({indexKey, ""}).second); } } @@ -172,13 +172,13 @@ class IndexScanTest : public ::testing::Test { for (size_t j = 0; j < indices.size(); j++) { auto& index = indices[j]; auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index->get_fields()).value(); - auto indexKey = IndexKeyUtils::edgeIndexKey(8, - 0, - index->get_index_id(), - std::to_string(i), - i, - std::to_string(i), - std::move(indexValue)); + auto indexKey = IndexKeyUtils::edgeIndexKeys(8, + 0, + index->get_index_id(), + std::to_string(i), + i, + std::to_string(i), + std::move(indexValue))[0]; DVLOG(1) << '\n' << folly::hexDump(indexKey.data(), indexKey.size()); assert(ret[j + 1].insert({indexKey, ""}).second); } @@ -224,8 +224,8 @@ TEST_F(IndexScanTest, Base) { }; IndexID indexId = 2; auto context = makeContext(1, 0); - auto scanNode = std::make_unique(context.get(), indexId, columnHints); - scanNode->kvstore_ = kvstore.get(); + auto scanNode = + std::make_unique(context.get(), indexId, columnHints, kvstore.get()); scanNode->getIndex = [index = indices[0]]() { return index; }; scanNode->getTag = [schema]() { return schema; }; InitContext initCtx; @@ -262,9 +262,8 @@ TEST_F(IndexScanTest, Base) { }; IndexID indexId = 3; auto context = makeContext(1, 0); - auto scanNode = std::make_unique(context.get(), indexId, columnHints); - DVLOG(1) << kvstore.get(); - scanNode->kvstore_ = kvstore.get(); + auto scanNode = + std::make_unique(context.get(), indexId, columnHints, kvstore.get()); scanNode->getIndex = [index = indices[1]]() { return index; }; scanNode->getTag = [schema]() { return schema; }; InitContext initCtx; @@ -321,8 +320,8 @@ TEST_F(IndexScanTest, Vertex) { for (auto& item : kv[1]) { kvstore->put(item.first, item.second); } - auto scanNode = std::make_unique(context.get(), indexId, columnHints); - scanNode->kvstore_ = kvstore.get(); + auto scanNode = + std::make_unique(context.get(), indexId, columnHints, kvstore.get()); scanNode->getIndex = [index = indices[0]]() { return index; }; scanNode->getTag = [schema]() { return schema; }; InitContext initCtx; @@ -358,8 +357,8 @@ TEST_F(IndexScanTest, Vertex) { for (auto& item : kv[0]) { kvstore->put(item.first, item.second); } - auto scanNode = std::make_unique(context.get(), indexId, columnHints); - scanNode->kvstore_ = kvstore.get(); + auto scanNode = + std::make_unique(context.get(), indexId, columnHints, kvstore.get()); scanNode->getIndex = [index = indices[0]]() { return index; }; scanNode->getTag = [schema]() { return schema; }; InitContext initCtx; @@ -419,8 +418,8 @@ TEST_F(IndexScanTest, Edge) { for (auto& item : kv[1]) { kvstore->put(item.first, item.second); } - auto scanNode = std::make_unique(context.get(), indexId, columnHints); - scanNode->kvstore_ = kvstore.get(); + auto scanNode = + std::make_unique(context.get(), indexId, columnHints, kvstore.get()); scanNode->getIndex = [index = indices[0]]() { return index; }; scanNode->getEdge = [schema]() { return schema; }; InitContext initCtx; @@ -456,9 +455,8 @@ TEST_F(IndexScanTest, Edge) { for (auto& item : kv[0]) { kvstore->put(item.first, item.second); } - auto scanNode = std::make_unique(context.get(), indexId, columnHints); - scanNode->kvstore_ = kvstore.get(); - scanNode->kvstore_ = kvstore.get(); + auto scanNode = + std::make_unique(context.get(), indexId, columnHints, kvstore.get()); scanNode->getIndex = [index = indices[0]]() { return index; }; scanNode->getEdge = [schema]() { return schema; }; InitContext initCtx; diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index ecd5139596a..622e0cdb7d6 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -383,6 +383,7 @@ class MockIndexNode : public IndexNode { std::function(bool&)> nextFunc; std::function<::nebula::cpp2::ErrorCode(PartitionID)> executeFunc; std::function<::nebula::cpp2::ErrorCode(InitContext& initCtx)> initFunc; + std::string identify() override { return "MockIndexNode"; } private: ErrorOr doNext(bool& hasNext) override { return nextFunc(hasNext); } diff --git a/src/storage/test/KillQueryTest.cpp b/src/storage/test/KillQueryTest.cpp index 11ccacc746c..19846fb2b27 100644 --- a/src/storage/test/KillQueryTest.cpp +++ b/src/storage/test/KillQueryTest.cpp @@ -9,7 +9,7 @@ #include "common/fs/TempDir.h" #include "interface/gen-cpp2/common_types.h" #include "mock/MockCluster.h" -#include "storage/index/LookupProcessor.h" +#include "storage/index/LookupProcessor2.h" #include "storage/query/GetNeighborsProcessor.h" #include "storage/test/QueryTestUtils.h" diff --git a/src/tools/db-dump/CMakeLists.txt b/src/tools/db-dump/CMakeLists.txt index 893bd53e494..35b6b0754e1 100644 --- a/src/tools/db-dump/CMakeLists.txt +++ b/src/tools/db-dump/CMakeLists.txt @@ -45,6 +45,7 @@ set(tools_test_deps $ $ $ + $ $ ) diff --git a/src/tools/meta-dump/CMakeLists.txt b/src/tools/meta-dump/CMakeLists.txt index 46c508f3b1f..abfde220aba 100644 --- a/src/tools/meta-dump/CMakeLists.txt +++ b/src/tools/meta-dump/CMakeLists.txt @@ -50,6 +50,7 @@ nebula_add_executable( $ $ $ + $ $ LIBRARIES ${ROCKSDB_LIBRARIES} From 40ec7d021e45bae9c364f95bb56420813b45ed94 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Thu, 21 Oct 2021 18:01:21 +0800 Subject: [PATCH 18/38] remove debug log --- src/storage/CMakeLists.txt | 2 +- src/storage/GraphStorageServiceHandler.cpp | 2 +- src/storage/exec/IndexEdgeScanNode.cpp | 2 - src/storage/exec/IndexNode.h | 1 + src/storage/exec/IndexScanNode2.cpp | 40 +------------------ src/storage/exec/IndexSelectionNode.cpp | 2 - src/storage/exec/IndexSelectionNode.h | 1 - src/storage/exec/IndexVertexScanNode.cpp | 1 - ...okupProcessor2.cpp => LookupProcessor.cpp} | 2 +- .../{LookupProcessor2.h => LookupProcessor.h} | 0 src/storage/test/DeleteTagsTest.cpp | 2 +- src/storage/test/IndexScanLimitTest.cpp | 2 +- src/storage/test/IndexWithTTLTest.cpp | 2 +- src/storage/test/IndexWriteTest.cpp | 2 +- src/storage/test/KillQueryTest.cpp | 2 +- src/storage/test/LookupIndexTest.cpp | 2 +- 16 files changed, 12 insertions(+), 53 deletions(-) rename src/storage/index/{LookupProcessor2.cpp => LookupProcessor.cpp} (99%) rename src/storage/index/{LookupProcessor2.h => LookupProcessor.h} (100%) diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 115fd7aa96b..980c6aa378e 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -40,7 +40,7 @@ nebula_add_library( query/GetPropProcessor.cpp query/ScanVertexProcessor.cpp query/ScanEdgeProcessor.cpp - index/LookupProcessor2.cpp + index/LookupProcessor.cpp exec/IndexNode.cpp exec/IndexDedupNode.cpp exec/IndexEdgeScanNode.cpp diff --git a/src/storage/GraphStorageServiceHandler.cpp b/src/storage/GraphStorageServiceHandler.cpp index 1c03dba8f13..ec77f713943 100644 --- a/src/storage/GraphStorageServiceHandler.cpp +++ b/src/storage/GraphStorageServiceHandler.cpp @@ -6,7 +6,7 @@ #include "storage/GraphStorageServiceHandler.h" -#include "storage/index/LookupProcessor2.h" +#include "storage/index/LookupProcessor.h" #include "storage/mutate/AddEdgesProcessor.h" #include "storage/mutate/AddVerticesProcessor.h" #include "storage/mutate/DeleteEdgesProcessor.h" diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index 1b4d0acd89c..15937c117f3 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -135,8 +135,6 @@ Map IndexEdgeScanNode::decodeFromBase(const std::string& key default: LOG(FATAL) << "Unexpect column name:" << col; } - DLOG(INFO) << col; - DLOG(INFO) << values[col]; } return values; } diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index acef0649394..03e058ca60d 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -23,6 +23,7 @@ struct InitContext { std::vector returnColumns; Map retColMap; }; +/***/ class IndexNode { public: template diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index 3c529d72b39..8e656dc60c2 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -55,14 +55,10 @@ std::unique_ptr Path::make(nebula::meta::cpp2::IndexItem* index, return ret; } Path::Qualified Path::qualified(const folly::StringPiece& key) { - DLOG(INFO) << "x"; - Qualified ret = Qualified::COMPATIBLE; for (auto& func : QFList_) { ret = std::min(ret, func(key.toString())); } - DLOG(INFO) << "x"; - return ret; } std::string Path::encodeValue(const Value& value, @@ -113,29 +109,13 @@ RangePath::RangePath(nebula::meta::cpp2::IndexItem* index, buildKey(); } void RangePath::resetPart(PartitionID partId) { - DLOG(INFO) << hints_.size(); std::string p = IndexKeyUtils::indexPrefix(partId); - DLOG(INFO) << hints_.size(); - DLOG(INFO) << folly::hexDump(&hints_, sizeof(hints_)); - // DVLOG(3) << '\n' << folly::hexDump(startKey_.data(), startKey_.size()); - DLOG(INFO) << folly::hexDump(&hints_, sizeof(hints_)); - DLOG(INFO) << folly::hexDump(&hints_, sizeof(hints_)); - - DLOG(INFO) << hints_.size(); startKey_ = startKey_.replace(0, p.size(), p); - DLOG(INFO) << hints_.size(); - // DVLOG(3) << '\n' << folly::hexDump(startKey_.data(), startKey_.size()); - DVLOG(3) << '\n' << folly::hexDump(endKey_.data(), endKey_.size()); - DLOG(INFO) << hints_.size(); endKey_ = endKey_.replace(0, p.size(), p); - DLOG(INFO) << hints_.size(); - // DVLOG(3) << '\n' << folly::hexDump(endKey_.data(), endKey_.size()); } Path::Qualified RangePath::qualified(const Map& rowData) { - DLOG(INFO) << hints_.size(); for (size_t i = 0; i < hints_.size() - 1; i++) { auto& hint = hints_[i]; - DLOG(INFO) << ::apache::thrift::SimpleJSONSerializer::serialize(hint); if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { return Qualified::INCOMPATIBLE; } @@ -143,7 +123,6 @@ Path::Qualified RangePath::qualified(const Map& rowData) { auto& hint = hints_.back(); // TODO(hs.zhang): improve performance.Check include or not during build key. if (hint.begin_value_ref().is_set()) { - DLOG(INFO) << hint.get_column_name(); bool ret = includeStart_ ? hint.get_begin_value() <= rowData.at(hint.get_column_name()) : hint.get_begin_value() < rowData.at(hint.get_column_name()); if (!ret) { @@ -152,7 +131,6 @@ Path::Qualified RangePath::qualified(const Map& rowData) { } if (hint.end_value_ref().is_set()) { DVLOG(2) << includeEnd_; - DLOG(INFO) << hint.get_column_name(); bool ret = includeEnd_ ? hint.get_end_value() >= rowData.at(hint.get_column_name()) : hint.get_end_value() > rowData.at(hint.get_column_name()); DVLOG(2) << ret; @@ -290,8 +268,8 @@ std::string RangePath::encodeEndValue(const Value& value, greater |= isNaN; if (UNLIKELY(isNaN)) { QFList_.emplace_back([startPos = offset, length = val.size()](const std::string& k) { - DLOG(INFO) << "check NaN:" << startPos << "\t" << length; - DLOG(INFO) << '\n' << folly::hexDump(k.data(), k.size()); + DVLOG(3) << "check NaN:" << startPos << "\t" << length; + DVLOG(3) << '\n' << folly::hexDump(k.data(), k.size()); int ret = memcmp("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", k.data() + startPos, length); return ret == 0 ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); @@ -339,7 +317,6 @@ PrefixPath::PrefixPath(nebula::meta::cpp2::IndexItem* index, } Path::Qualified PrefixPath::qualified(const Map& rowData) { for (auto& hint : hints_) { - DLOG(INFO) << hint.get_column_name(); if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { return Qualified::INCOMPATIBLE; } @@ -422,10 +399,8 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { - DLOG(INFO) << columnHints_.size(); partId_ = partId; auto ret = resetIter(partId); - DLOG(INFO) << columnHints_.size(); return ret; } IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { @@ -444,8 +419,6 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { } DVLOG(3) << "x"; bool compatible = q == Path::Qualified::COMPATIBLE; - DLOG(INFO) << compatible; - DLOG(INFO) << needAccessBase_; if (compatible && !needAccessBase_) { DVLOG(3) << 123; auto key = iter_->key().toString(); @@ -510,31 +483,22 @@ bool IndexScanNode::checkTTL() { nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { path_->resetPart(partId); - DLOG(INFO) << columnHints_.size(); - DLOG(INFO) << path_.get(); nebula::cpp2::ErrorCode ret; if (path_->isRange()) { auto rangePath = dynamic_cast(path_.get()); - DLOG(INFO) << rangePath; - DLOG(INFO) << columnHints_.size(); DVLOG(1) << '\n' << folly::hexDump(rangePath->getStartKey().data(), rangePath->getStartKey().size()); - DLOG(INFO) << columnHints_.size(); DVLOG(1) << '\n' << folly::hexDump(rangePath->getEndKey().data(), rangePath->getEndKey().size()); - DLOG(INFO) << columnHints_.size(); std::unique_ptr<::nebula::kvstore::KVIterator> iter; kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter); - DLOG(INFO) << columnHints_.size(); iter_ = std::move(iter); - DLOG(INFO) << columnHints_.size(); } else { auto prefixPath = dynamic_cast(path_.get()); DVLOG(1) << '\n' << folly::hexDump(prefixPath->getPrefixKey().data(), prefixPath->getPrefixKey().size()); ret = kvstore_->prefix(spaceId_, partId, prefixPath->getPrefixKey(), &iter_); - DLOG(INFO) << columnHints_.size(); } return ret; } diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index 36e6272b03c..eac6841c311 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -58,7 +58,6 @@ Value IndexSelectionNode::ExprContext::getEdgeProp(const std::string& edgeType, auto iter = colPos_.find(prop); DCHECK(iter != colPos_.end()); DCHECK(iter->second < row_->size()); - DLOG(INFO) << (*row_)[iter->second]; return (*row_)[iter->second]; } Value IndexSelectionNode::ExprContext::getTagProp(const std::string& tag, @@ -68,7 +67,6 @@ Value IndexSelectionNode::ExprContext::getTagProp(const std::string& tag, auto iter = colPos_.find(prop); DCHECK(iter != colPos_.end()); DCHECK(iter->second < row_->size()); - DLOG(INFO) << (*row_)[iter->second]; return (*row_)[iter->second]; } diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 019def98d0d..c1dc4fd50b9 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -23,7 +23,6 @@ class IndexSelectionNode : public IndexNode { private: ErrorOr doNext(bool &hasNext) override; inline bool filter(const Row &row) { - DLOG(INFO) << row; ctx_->setRow(row); auto &result = expr_->eval(*ctx_); return result.type() == Value::Type::BOOL ? result.getBool() : false; diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index 65185a8939f..7dcbd0d1e4a 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -88,7 +88,6 @@ Map IndexVertexScanNode::decodeFromBase(const std::string& k Map values; auto reader = RowReaderWrapper::getRowReader(tag_.get(), value); for (auto& col : requiredAndHintColumns_) { - DLOG(INFO) << col; switch (QueryUtils::toReturnColType(col)) { case QueryUtils::ReturnColType::kVid: { auto vId = NebulaKeyUtils::getVertexId(context_->vIdLen(), key); diff --git a/src/storage/index/LookupProcessor2.cpp b/src/storage/index/LookupProcessor.cpp similarity index 99% rename from src/storage/index/LookupProcessor2.cpp rename to src/storage/index/LookupProcessor.cpp index b0bfb18f721..843ac1c45b9 100644 --- a/src/storage/index/LookupProcessor2.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -3,7 +3,7 @@ * This source code is licensed under Apache 2.0 License, * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include "storage/index/LookupProcessor2.h" +#include "storage/index/LookupProcessor.h" #include #include diff --git a/src/storage/index/LookupProcessor2.h b/src/storage/index/LookupProcessor.h similarity index 100% rename from src/storage/index/LookupProcessor2.h rename to src/storage/index/LookupProcessor.h diff --git a/src/storage/test/DeleteTagsTest.cpp b/src/storage/test/DeleteTagsTest.cpp index 1f2cccbbf99..abd9e7889bd 100644 --- a/src/storage/test/DeleteTagsTest.cpp +++ b/src/storage/test/DeleteTagsTest.cpp @@ -14,7 +14,7 @@ #include "interface/gen-cpp2/storage_types.h" #include "mock/MockCluster.h" #include "mock/MockData.h" -#include "storage/index/LookupProcessor2.h" +#include "storage/index/LookupProcessor.h" #include "storage/mutate/DeleteTagsProcessor.h" #include "storage/query/GetPropProcessor.h" #include "storage/test/QueryTestUtils.h" diff --git a/src/storage/test/IndexScanLimitTest.cpp b/src/storage/test/IndexScanLimitTest.cpp index e28f3e78b92..a73e0186a80 100644 --- a/src/storage/test/IndexScanLimitTest.cpp +++ b/src/storage/test/IndexScanLimitTest.cpp @@ -14,7 +14,7 @@ #include "mock/AdHocIndexManager.h" #include "mock/AdHocSchemaManager.h" #include "mock/MockCluster.h" -#include "storage/index/LookupProcessor2.h" +#include "storage/index/LookupProcessor.h" #include "storage/query/GetNeighborsProcessor.h" #include "storage/test/TestUtils.h" diff --git a/src/storage/test/IndexWithTTLTest.cpp b/src/storage/test/IndexWithTTLTest.cpp index 733bd8514fd..00f9d62d587 100644 --- a/src/storage/test/IndexWithTTLTest.cpp +++ b/src/storage/test/IndexWithTTLTest.cpp @@ -19,7 +19,7 @@ #include "storage/admin/AdminTaskManager.h" #include "storage/admin/RebuildEdgeIndexTask.h" #include "storage/admin/RebuildTagIndexTask.h" -#include "storage/index/LookupProcessor2.h" +#include "storage/index/LookupProcessor.h" #include "storage/mutate/AddEdgesProcessor.h" #include "storage/mutate/AddVerticesProcessor.h" #include "storage/mutate/UpdateEdgeProcessor.h" diff --git a/src/storage/test/IndexWriteTest.cpp b/src/storage/test/IndexWriteTest.cpp index ba22ab5e6b6..7a8dc0bd85b 100644 --- a/src/storage/test/IndexWriteTest.cpp +++ b/src/storage/test/IndexWriteTest.cpp @@ -16,7 +16,7 @@ #include "mock/AdHocSchemaManager.h" #include "mock/MockCluster.h" #include "mock/MockData.h" -#include "storage/index/LookupProcessor2.h" +#include "storage/index/LookupProcessor.h" #include "storage/mutate/AddEdgesProcessor.h" #include "storage/mutate/AddVerticesProcessor.h" #include "storage/mutate/DeleteEdgesProcessor.h" diff --git a/src/storage/test/KillQueryTest.cpp b/src/storage/test/KillQueryTest.cpp index 19846fb2b27..11ccacc746c 100644 --- a/src/storage/test/KillQueryTest.cpp +++ b/src/storage/test/KillQueryTest.cpp @@ -9,7 +9,7 @@ #include "common/fs/TempDir.h" #include "interface/gen-cpp2/common_types.h" #include "mock/MockCluster.h" -#include "storage/index/LookupProcessor2.h" +#include "storage/index/LookupProcessor.h" #include "storage/query/GetNeighborsProcessor.h" #include "storage/test/QueryTestUtils.h" diff --git a/src/storage/test/LookupIndexTest.cpp b/src/storage/test/LookupIndexTest.cpp index bb56067c724..904168c23fb 100644 --- a/src/storage/test/LookupIndexTest.cpp +++ b/src/storage/test/LookupIndexTest.cpp @@ -22,7 +22,7 @@ #include "mock/AdHocSchemaManager.h" #include "mock/MockCluster.h" #include "mock/MockData.h" -#include "storage/index/LookupProcessor2.h" +#include "storage/index/LookupProcessor.h" #include "storage/mutate/AddVerticesProcessor.h" #include "storage/test/QueryTestUtils.h" From eab73bec23ffcf86353b377fbd2116564fd09a25 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Fri, 22 Oct 2021 19:11:53 +0800 Subject: [PATCH 19/38] fix bug --- src/graph/optimizer/OptimizerUtils.cpp | 13 +---------- src/storage/exec/IndexDedupNode.cpp | 13 +++++++++-- src/storage/exec/IndexDedupNode.h | 6 ++++- src/storage/exec/IndexScanNode2.cpp | 31 +++++++++++++++----------- src/storage/index/LookupProcessor.cpp | 14 +++++++----- 5 files changed, 43 insertions(+), 34 deletions(-) diff --git a/src/graph/optimizer/OptimizerUtils.cpp b/src/graph/optimizer/OptimizerUtils.cpp index f2046707f63..5b2fe8ad42f 100644 --- a/src/graph/optimizer/OptimizerUtils.cpp +++ b/src/graph/optimizer/OptimizerUtils.cpp @@ -465,18 +465,7 @@ Value OptimizerUtils::normalizeValue(const meta::cpp2::ColumnDef& col, const Val if (!col.type.type_length_ref().has_value()) { return Value::kNullBadType; } - if (!v.isStr()) { - return v; - } - auto len = static_cast(*col.get_type().get_type_length()); - if (v.getStr().size() > len) { - return Value(v.getStr().substr(0, len)); - } else { - std::string s; - s.reserve(len); - s.append(v.getStr()).append(len - v.getStr().size(), '\0'); - return Value(std::move(s)); - } + return v; } case Value::Type::__EMPTY__: case Value::Type::NULLVALUE: diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index a768d0e0eee..c8f59872097 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -17,13 +17,19 @@ ::nebula::cpp2::ErrorCode IndexDedupNode::init(InitContext& ctx) { } // The return Row format of each child node must be the same InitContext childCtx = ctx; + InitContext ctx2; for (auto& child : children_) { child->init(childCtx); + ctx2 = childCtx; childCtx = ctx; } - ctx = childCtx; + ctx = ctx2; + for (auto& iter : ctx.retColMap) { + DLOG(INFO) << iter.first << ":" << iter.second; + } for (auto& col : dedupColumns_) { dedupPos_.push_back(ctx.retColMap[col]); + DLOG(INFO) << dedupPos_.back(); } return ::nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -37,6 +43,7 @@ IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) { hasNext = false; while (currentChild_ < children_.size()) { auto& child = *children_[currentChild_]; + DLOG(INFO) << currentChild_; do { auto result = child.next(hasNext); if (!nebula::ok(result)) { @@ -45,7 +52,9 @@ IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) { if (!hasNext) { break; } - if (dedup(::nebula::value(result))) { + auto d = dedup(::nebula::value(result)); + DLOG(INFO) << d << "\t" << ::nebula::value(result); + if (d) { ret = ::nebula::value(std::move(result)); hasNext = true; break; diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index 6e53fbc910c..3c9b044e860 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -39,7 +39,11 @@ class IndexDedupNode : public IndexNode { }; struct Equal { bool operator()(const RowWrapper& a, const RowWrapper& b) const { - return a.values() == b.values(); + bool cmp = a.values() == b.values(); + DLOG(INFO) << a.values(); + DLOG(INFO) << b.values(); + DLOG(INFO) << cmp; + return cmp; } }; std::vector dedupColumns_; diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index 8e656dc60c2..cd7be4eaf9d 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -217,10 +217,10 @@ std::string RangePath::encodeBeginValue(const Value& value, val = encodeString(value, *colDef.get_type_length(), truncated); greater &= !truncated; if (UNLIKELY(truncated)) { - QFList_.emplace_back([&startKey = this->startKey_, - startPos = key.size(), - length = val.size()](const std::string& k) { - int ret = memcmp(startKey.data() + startPos, k.data() + startPos, length); + QFList_.emplace_back([val, startPos = offset](const std::string& k) { + int ret = memcmp(val.data(), k.data() + startPos, val.size()); + DVLOG(1) << folly::hexDump(val.data(), val.size()); + DVLOG(1) << folly::hexDump(k.data(), k.size()); CHECK_LE(ret, 0); return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; }); @@ -255,10 +255,11 @@ std::string RangePath::encodeEndValue(const Value& value, val = encodeString(value, *colDef.get_type_length(), truncated); greater |= truncated; if (UNLIKELY(truncated)) { - QFList_.emplace_back([&endKey = this->endKey_, startPos = key.size(), length = val.size()]( - const std::string& k) { - int ret = memcmp(endKey.data() + startPos, k.data() + startPos, length); - CHECK_LE(ret, 0); + QFList_.emplace_back([val, startPos = offset](const std::string& k) { + int ret = memcmp(val.data(), k.data() + startPos, val.size()); + DVLOG(3) << folly::hexDump(val.data(), val.size()); + DVLOG(3) << folly::hexDump(k.data(), k.size()); + CHECK_GE(ret, 0); return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; }); } @@ -386,6 +387,9 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { if (field.get_type().get_type() == PropertyType::FIXED_STRING) { continue; } + if (field.get_type().get_type() == PropertyType::GEOGRAPHY) { + continue; + } tmp.erase(field.get_name()); } tmp.erase(kVid); @@ -408,7 +412,7 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { DVLOG(3) << columnHints_.size(); hasNext = true; for (; iter_ && iter_->valid(); iter_->next()) { - DVLOG(1) << '\n' << folly::hexDump(iter_->key().data(), iter_->key().size()); + DLOG(INFO) << '\n' << folly::hexDump(iter_->key().data(), iter_->key().size()); if (!checkTTL()) { continue; } @@ -424,7 +428,7 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { auto key = iter_->key().toString(); iter_->next(); auto ret = decodeFromIndex(key); - DVLOG(3) << ret; + DLOG(INFO) << ret; return ret; } DVLOG(3) << "x"; @@ -463,7 +467,7 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { row.emplace_back(std::move(rowData.at(col))); } iter_->next(); - DVLOG(3) << row; + DLOG(INFO) << row; return row; } hasNext = false; @@ -483,7 +487,7 @@ bool IndexScanNode::checkTTL() { nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { path_->resetPart(partId); - nebula::cpp2::ErrorCode ret; + nebula::cpp2::ErrorCode ret = nebula::cpp2::ErrorCode::SUCCEEDED; if (path_->isRange()) { auto rangePath = dynamic_cast(path_.get()); DVLOG(1) << '\n' @@ -491,7 +495,8 @@ nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { DVLOG(1) << '\n' << folly::hexDump(rangePath->getEndKey().data(), rangePath->getEndKey().size()); std::unique_ptr<::nebula::kvstore::KVIterator> iter; - kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter); + ret = + kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter); iter_ = std::move(iter); } else { auto prefixPath = dynamic_cast(path_.get()); diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index 843ac1c45b9..89b1b614c08 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -30,12 +30,19 @@ void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { doProcess(req); } } +void printPlan(IndexNode* node, int tab = 0) { + LOG(INFO) << std::string(tab, '\t') << node->identify(); + for (auto& child : node->children()) { + printPlan(child.get(), tab + 1); + } +} void LookupProcessor::doProcess(const cpp2::LookupIndexRequest& req) { if (req.common_ref().has_value() && req.get_common()->profile_detail_ref().value_or(false)) { profileDetailFlag_ = true; } prepare(req); auto plan = buildPlan(req); + printPlan(plan.get()); if (!FLAGS_query_concurrently) { runInSingleThread(req.get_parts(), std::move(plan)); } else { @@ -134,12 +141,7 @@ std::unique_ptr LookupProcessor::buildOneContext(const cpp2::IndexQue } return node; } -void printPlan(IndexNode* node, int tab = 0) { - DLOG(INFO) << std::string(tab, '\t') << node->name() << "(" << node << ")"; - for (auto& child : node->children()) { - printPlan(child.get(), tab + 1); - } -} + void LookupProcessor::runInSingleThread(const std::vector& parts, std::unique_ptr plan) { printPlan(plan.get()); From 4ec5d8aa10ec63c71ee83c53f963aeed25fd229f Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Sat, 23 Oct 2021 21:12:28 +0800 Subject: [PATCH 20/38] fix bug --- src/storage/exec/IndexEdgeScanNode.cpp | 29 ++++++++---------------- src/storage/exec/IndexEdgeScanNode.h | 6 ++--- src/storage/exec/IndexScanNode2.cpp | 10 ++++---- src/storage/exec/IndexScanNode2.h | 2 +- src/storage/exec/IndexVertexScanNode.cpp | 12 +++++----- src/storage/exec/IndexVertexScanNode.h | 8 ++++--- src/storage/test/IndexTest.cpp | 27 +++++++++++++++------- src/storage/test/IndexTestUtil.h | 2 +- 8 files changed, 50 insertions(+), 46 deletions(-) diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index 15937c117f3..58103041428 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -8,21 +8,7 @@ namespace nebula { namespace storage { IndexEdgeScanNode::IndexEdgeScanNode(const IndexEdgeScanNode& node) - : IndexScanNode(node), edge_(node.edge_) { - getIndex = std::function([this]() { - auto env = this->context_->env(); - auto indexMgr = env->indexMan_; - auto index = indexMgr->getTagIndex(this->spaceId_, this->indexId_).value(); - return index; - }); - getEdge = std::function([this]() { - auto env = this->context_->env(); - auto schemaMgr = env->schemaMan_; - auto schema = - schemaMgr->getEdgeSchema(this->spaceId_, this->index_->get_schema_id().get_edge_type()); - return schema; - }); -} + : IndexScanNode(node), edge_(node.edge_) {} IndexEdgeScanNode::IndexEdgeScanNode(RuntimeContext* context, IndexID indexId, const std::vector& columnHint, @@ -37,8 +23,8 @@ IndexEdgeScanNode::IndexEdgeScanNode(RuntimeContext* context, getEdge = std::function([this]() { auto env = this->context_->env(); auto schemaMgr = env->schemaMan_; - auto schema = - schemaMgr->getEdgeSchema(this->spaceId_, this->index_->get_schema_id().get_edge_type()); + auto schema = schemaMgr->getAllVerEdgeSchema(this->spaceId_) + .value()[this->index_->get_schema_id().get_edge_type()]; return schema; }); } @@ -100,7 +86,7 @@ nebula::cpp2::ErrorCode IndexEdgeScanNode::getBaseData(folly::StringPiece key, Map IndexEdgeScanNode::decodeFromBase(const std::string& key, const std::string& value) { Map values; - auto reader = RowReaderWrapper::getRowReader(edge_.get(), value); + auto reader = RowReaderWrapper::getRowReader(edge_, value); for (auto& col : requiredAndHintColumns_) { switch (QueryUtils::toReturnColType(col)) { case QueryUtils::ReturnColType::kType: { @@ -126,7 +112,7 @@ Map IndexEdgeScanNode::decodeFromBase(const std::string& key values[col] = Value(NebulaKeyUtils::getRank(context_->vIdLen(), key)); } break; case QueryUtils::ReturnColType::kOther: { - auto retVal = QueryUtils::readValue(reader.get(), col, edge_->field(col)); + auto retVal = QueryUtils::readValue(reader.get(), col, edge_.back()->field(col)); if (!retVal.ok()) { LOG(FATAL) << "Bad value for field" << col; } @@ -138,7 +124,10 @@ Map IndexEdgeScanNode::decodeFromBase(const std::string& key } return values; } -const meta::SchemaProviderIf* IndexEdgeScanNode::getSchema() { return edge_.get(); } +const std::vector>& +IndexEdgeScanNode::getSchema() { + return edge_; +} std::unique_ptr IndexEdgeScanNode::copy() { return std::make_unique(*this); diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index 3f243bdc97c..a5c351e77df 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -27,13 +27,13 @@ class IndexEdgeScanNode : public IndexScanNode { nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, std::pair& kv) override; Map decodeFromBase(const std::string& key, const std::string& value) override; - const meta::SchemaProviderIf* getSchema() override; + const std::vector>& getSchema() override; - std::shared_ptr edge_; + std::vector> edge_; // Convenient for testing std::function>()> getIndex; - std::function()> getEdge; + std::function>()> getEdge; FRIEND_TEST(IndexScanTest, Edge); friend class IndexScanTestHelper; diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index cd7be4eaf9d..edd70d787e8 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -367,7 +367,7 @@ IndexScanNode::IndexScanNode(const IndexScanNode& node) ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { DCHECK(requiredColumns_.empty()); - ttlProps_ = CommonUtils::ttlProps(getSchema()); + ttlProps_ = CommonUtils::ttlProps(getSchema().back().get()); requiredAndHintColumns_ = ctx.requiredColumns; for (auto& hint : columnHints_) { @@ -399,7 +399,7 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { tmp.erase(kDst); tmp.erase(kType); needAccessBase_ = !tmp.empty(); - path_ = Path::make(index_.get(), getSchema(), columnHints_, context_->vIdLen()); + path_ = Path::make(index_.get(), getSchema().back().get(), columnHints_, context_->vIdLen()); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { @@ -478,8 +478,10 @@ bool IndexScanNode::checkTTL() { return true; } auto v = IndexKeyUtils::parseIndexTTL(iter_->val()); - if (CommonUtils::checkDataExpiredForTTL( - getSchema(), std::move(v), ttlProps_.second.second, ttlProps_.second.first)) { + if (CommonUtils::checkDataExpiredForTTL(getSchema().back().get(), + std::move(v), + ttlProps_.second.second, + ttlProps_.second.first)) { return false; } return true; diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode2.h index 9a7e2ec95d0..fbbda690cd4 100644 --- a/src/storage/exec/IndexScanNode2.h +++ b/src/storage/exec/IndexScanNode2.h @@ -50,7 +50,7 @@ class IndexScanNode : public IndexNode { std::pair& kv) = 0; virtual Map decodeFromBase(const std::string& key, const std::string& value) = 0; - virtual const meta::SchemaProviderIf* getSchema() = 0; + virtual const std::vector>& getSchema() = 0; bool checkTTL(); nebula::cpp2::ErrorCode resetIter(PartitionID partId); PartitionID partId_; diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index 7dcbd0d1e4a..c21d7c63eaa 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -22,8 +22,8 @@ IndexVertexScanNode::IndexVertexScanNode(const IndexVertexScanNode& node) getTag = std::function([this]() { auto env = this->context_->env(); auto schemaMgr = env->schemaMan_; - auto schema = - schemaMgr->getTagSchema(this->spaceId_, this->index_->get_schema_id().get_tag_id()); + auto schema = schemaMgr->getAllVerTagSchema(this->spaceId_) + .value()[this->index_->get_schema_id().get_tag_id()]; return schema; }); } @@ -42,8 +42,8 @@ IndexVertexScanNode::IndexVertexScanNode(RuntimeContext* context, getTag = std::function([this]() { auto env = this->context_->env(); auto schemaMgr = env->schemaMan_; - auto schema = - schemaMgr->getTagSchema(this->spaceId_, this->index_->get_schema_id().get_tag_id()); + auto schema = schemaMgr->getAllVerTagSchema(this->spaceId_) + .value()[this->index_->get_schema_id().get_tag_id()]; return schema; }); } @@ -86,7 +86,7 @@ Row IndexVertexScanNode::decodeFromIndex(folly::StringPiece key) { Map IndexVertexScanNode::decodeFromBase(const std::string& key, const std::string& value) { Map values; - auto reader = RowReaderWrapper::getRowReader(tag_.get(), value); + auto reader = RowReaderWrapper::getRowReader(tag_, folly::StringPiece(value)); for (auto& col : requiredAndHintColumns_) { switch (QueryUtils::toReturnColType(col)) { case QueryUtils::ReturnColType::kVid: { @@ -101,7 +101,7 @@ Map IndexVertexScanNode::decodeFromBase(const std::string& k values[col] = Value(context_->tagId_); } break; case QueryUtils::ReturnColType::kOther: { - auto retVal = QueryUtils::readValue(reader.get(), col, tag_->field(col)); + auto retVal = QueryUtils::readValue(reader.get(), col, tag_.back()->field(col)); if (!retVal.ok()) { LOG(FATAL) << "Bad value for field" << col; } diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index 951326a4661..6b3ceb6be8a 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -31,13 +31,15 @@ class IndexVertexScanNode final : public IndexScanNode { std::pair& kv) override; Row decodeFromIndex(folly::StringPiece key) override; Map decodeFromBase(const std::string& key, const std::string& value) override; - const meta::SchemaProviderIf* getSchema() override { return tag_.get(); } + const std::vector>& getSchema() override { + return tag_; + } - std::shared_ptr tag_; + std::vector> tag_; // Convenient for testing std::function>()> getIndex; - std::function()> getTag; + std::function>()> getTag; FRIEND_TEST(IndexScanTest, VertexIndexOnlyScan); FRIEND_TEST(IndexScanTest, VertexBase); diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 5747c900d08..bdf5b34fec2 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -76,11 +76,15 @@ struct IndexScanTestHelper { } void setTag(IndexVertexScanNode* node, std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema) { - node->getTag = [schema]() { return schema; }; + node->getTag = [schema]() { + return std::vector>{schema}; + }; } void setEdge(IndexEdgeScanNode* node, std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema) { - node->getEdge = [schema]() { return schema; }; + node->getEdge = [schema]() { + return std::vector>{schema}; + }; } void setFatal(IndexScanNode* node, bool value) { node->fatalOnBaseNotFound_ = value; } }; @@ -226,8 +230,9 @@ TEST_F(IndexScanTest, Base) { auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); + IndexScanTestHelper helper; scanNode->getIndex = [index = indices[0]]() { return index; }; - scanNode->getTag = [schema]() { return schema; }; + helper.setTag(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kVid, "a"}; scanNode->init(initCtx); @@ -264,8 +269,9 @@ TEST_F(IndexScanTest, Base) { auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); + IndexScanTestHelper helper; scanNode->getIndex = [index = indices[1]]() { return index; }; - scanNode->getTag = [schema]() { return schema; }; + helper.setTag(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kVid, "b"}; scanNode->init(initCtx); @@ -322,8 +328,10 @@ TEST_F(IndexScanTest, Vertex) { } auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); + IndexScanTestHelper helper; + scanNode->getIndex = [index = indices[0]]() { return index; }; - scanNode->getTag = [schema]() { return schema; }; + helper.setTag(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kVid, "a"}; scanNode->init(initCtx); @@ -359,8 +367,9 @@ TEST_F(IndexScanTest, Vertex) { } auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); + IndexScanTestHelper helper; scanNode->getIndex = [index = indices[0]]() { return index; }; - scanNode->getTag = [schema]() { return schema; }; + helper.setTag(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kVid, "b"}; scanNode->init(initCtx); @@ -420,8 +429,9 @@ TEST_F(IndexScanTest, Edge) { } auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); + IndexScanTestHelper helper; scanNode->getIndex = [index = indices[0]]() { return index; }; - scanNode->getEdge = [schema]() { return schema; }; + helper.setEdge(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kSrc, kRank, kDst, "c"}; scanNode->init(initCtx); @@ -457,8 +467,9 @@ TEST_F(IndexScanTest, Edge) { } auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); + IndexScanTestHelper helper; scanNode->getIndex = [index = indices[0]]() { return index; }; - scanNode->getEdge = [schema]() { return schema; }; + helper.setEdge(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kSrc, kRank, kDst, "a"}; scanNode->init(initCtx); diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index 622e0cdb7d6..ff1d80bf4db 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -464,7 +464,7 @@ class RowParser { class SchemaParser { public: explicit SchemaParser(const std::string& str) { - schema = std::make_shared<::nebula::meta::NebulaSchemaProvider>(2); + schema = std::make_shared<::nebula::meta::NebulaSchemaProvider>(0); ss = std::stringstream(folly::trimWhitespace(folly::StringPiece(str)).toString()); parse(); } From 8b9dc5d7e0687a739218594e3ce26e64a64fe500 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Mon, 25 Oct 2021 10:33:56 +0800 Subject: [PATCH 21/38] fix bug --- src/storage/exec/IndexDedupNode.cpp | 8 ++----- src/storage/exec/IndexDedupNode.h | 3 --- src/storage/exec/IndexScanNode2.cpp | 37 ++++++++++------------------- 3 files changed, 14 insertions(+), 34 deletions(-) diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index c8f59872097..07ae2640332 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -24,12 +24,8 @@ ::nebula::cpp2::ErrorCode IndexDedupNode::init(InitContext& ctx) { childCtx = ctx; } ctx = ctx2; - for (auto& iter : ctx.retColMap) { - DLOG(INFO) << iter.first << ":" << iter.second; - } for (auto& col : dedupColumns_) { dedupPos_.push_back(ctx.retColMap[col]); - DLOG(INFO) << dedupPos_.back(); } return ::nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -43,7 +39,7 @@ IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) { hasNext = false; while (currentChild_ < children_.size()) { auto& child = *children_[currentChild_]; - DLOG(INFO) << currentChild_; + DVLOG(3) << currentChild_; do { auto result = child.next(hasNext); if (!nebula::ok(result)) { @@ -53,7 +49,7 @@ IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) { break; } auto d = dedup(::nebula::value(result)); - DLOG(INFO) << d << "\t" << ::nebula::value(result); + DVLOG(3) << d << "\t" << ::nebula::value(result); if (d) { ret = ::nebula::value(std::move(result)); hasNext = true; diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index 3c9b044e860..4e2673bed82 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -40,9 +40,6 @@ class IndexDedupNode : public IndexNode { struct Equal { bool operator()(const RowWrapper& a, const RowWrapper& b) const { bool cmp = a.values() == b.values(); - DLOG(INFO) << a.values(); - DLOG(INFO) << b.values(); - DLOG(INFO) << cmp; return cmp; } }; diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode2.cpp index edd70d787e8..5a37b5f40c9 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode2.cpp @@ -24,10 +24,6 @@ Path::Path(nebula::meta::cpp2::IndexItem* index, index_nullable_offset_ += tmpStr.size(); totalKeyLength_ += tmpStr.size(); } - for (auto x : nullable_) { - DVLOG(1) << x; - } - DVLOG(1) << nullFlag; if (!nullFlag) { nullable_.clear(); } else { @@ -203,6 +199,15 @@ std::tuple RangePath::encodeRange( return nullableBit.test(15 - colIndex) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); } + // One GEO data will generator more than one index key. So need dedup. + if (UNLIKELY(colTypeDef.get_type() == nebula::meta::cpp2::PropertyType::GEOGRAPHY)) { + QFList_.emplace_back([suffixSet = Set(), + suffixLength = suffixLength_](const std::string& k) mutable { + auto suffix = k.substr(k.size() - suffixLength, suffixLength); + auto [iter, result] = suffixSet.insert(suffix); + return result ? Qualified::COMPATIBLE : Qualified::INCOMPATIBLE; + }); + } return {startKey, endKey}; } std::string RangePath::encodeBeginValue(const Value& value, @@ -408,31 +413,24 @@ nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { return ret; } IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { - DVLOG(3) << "x"; - DVLOG(3) << columnHints_.size(); hasNext = true; for (; iter_ && iter_->valid(); iter_->next()) { - DLOG(INFO) << '\n' << folly::hexDump(iter_->key().data(), iter_->key().size()); + DVLOG(3) << '\n' << folly::hexDump(iter_->key().data(), iter_->key().size()); if (!checkTTL()) { continue; } - DVLOG(3) << "x"; auto q = path_->qualified(iter_->key()); if (q == Path::Qualified::INCOMPATIBLE) { continue; } - DVLOG(3) << "x"; bool compatible = q == Path::Qualified::COMPATIBLE; if (compatible && !needAccessBase_) { - DVLOG(3) << 123; auto key = iter_->key().toString(); iter_->next(); auto ret = decodeFromIndex(key); - DLOG(INFO) << ret; + DVLOG(3) << ret; return ret; } - DVLOG(3) << "x"; - DVLOG(3) << 123; std::pair kv; auto ret = getBaseData(iter_->key(), kv); if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { @@ -443,31 +441,20 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { } continue; } - DVLOG(3) << "x"; - Map rowData = decodeFromBase(kv.first, kv.second); - DVLOG(3) << "x"; - for (auto& iter : rowData) { - DVLOG(3) << iter.first << ":" << iter.second; - } if (!compatible) { - DVLOG(3) << "x"; - DVLOG(3) << path_.get(); q = path_->qualified(rowData); CHECK(q != Path::Qualified::UNCERTAIN); if (q == Path::Qualified::INCOMPATIBLE) { continue; } } - DVLOG(3) << 123; Row row; for (auto& col : requiredColumns_) { - DVLOG(3) << col; - DVLOG(3) << rowData.at(col); row.emplace_back(std::move(rowData.at(col))); } iter_->next(); - DLOG(INFO) << row; + DVLOG(3) << row; return row; } hasNext = false; From c432c95373b945e42bc89d2c3999c3a092ae1369 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Mon, 25 Oct 2021 10:59:41 +0800 Subject: [PATCH 22/38] clear file --- src/storage/CMakeLists.txt | 2 +- src/storage/exec/IndexEdgeScanNode.h | 2 +- .../{IndexScanNode2.cpp => IndexScanNode.cpp} | 2 +- .../{IndexScanNode2.h => IndexScanNode.h} | 0 src/storage/exec/IndexVertexScanNode.h | 2 +- .../index/LookupBaseProcessor-inl.h.bk | 473 ------------------ src/storage/index/LookupBaseProcessor.h.bk | 93 ---- src/storage/index/LookupProcessor.cpp.bk | 146 ------ src/storage/index/LookupProcessor.h.bk | 48 -- 9 files changed, 4 insertions(+), 764 deletions(-) rename src/storage/exec/{IndexScanNode2.cpp => IndexScanNode.cpp} (99%) rename src/storage/exec/{IndexScanNode2.h => IndexScanNode.h} (100%) delete mode 100644 src/storage/index/LookupBaseProcessor-inl.h.bk delete mode 100644 src/storage/index/LookupBaseProcessor.h.bk delete mode 100644 src/storage/index/LookupProcessor.cpp.bk delete mode 100644 src/storage/index/LookupProcessor.h.bk diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 980c6aa378e..c182f8b4f99 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -46,7 +46,7 @@ nebula_add_library( exec/IndexEdgeScanNode.cpp exec/IndexLimitNode.cpp exec/IndexProjectionNode.cpp - exec/IndexScanNode2.cpp + exec/IndexScanNode.cpp exec/IndexSelectionNode.cpp exec/IndexVertexScanNode.cpp ) diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index a5c351e77df..ecdca4a73c4 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -6,7 +6,7 @@ #pragma once #include "common/base/Base.h" #include "common/utils/NebulaKeyUtils.h" -#include "storage/exec/IndexScanNode2.h" +#include "storage/exec/IndexScanNode.h" #include "storage/exec/QueryUtils.h" #include "storage/exec/StorageIterator.h" namespace nebula { diff --git a/src/storage/exec/IndexScanNode2.cpp b/src/storage/exec/IndexScanNode.cpp similarity index 99% rename from src/storage/exec/IndexScanNode2.cpp rename to src/storage/exec/IndexScanNode.cpp index 5a37b5f40c9..60f9d8ccbce 100644 --- a/src/storage/exec/IndexScanNode2.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -3,7 +3,7 @@ * This source code is licensed under Apache 2.0 License, * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include "storage/exec/IndexScanNode2.h" +#include "storage/exec/IndexScanNode.h" namespace nebula { namespace storage { diff --git a/src/storage/exec/IndexScanNode2.h b/src/storage/exec/IndexScanNode.h similarity index 100% rename from src/storage/exec/IndexScanNode2.h rename to src/storage/exec/IndexScanNode.h diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index 6b3ceb6be8a..689c72ec584 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -11,7 +11,7 @@ #include #include "common/base/Base.h" -#include "storage/exec/IndexScanNode2.h" +#include "storage/exec/IndexScanNode.h" #include "storage/exec/StorageIterator.h" namespace nebula { diff --git a/src/storage/index/LookupBaseProcessor-inl.h.bk b/src/storage/index/LookupBaseProcessor-inl.h.bk deleted file mode 100644 index 6a280f078bb..00000000000 --- a/src/storage/index/LookupBaseProcessor-inl.h.bk +++ /dev/null @@ -1,473 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#pragma once - -#include "LookupBaseProcessor.h" -#include "folly/container/Enumerate.h" -namespace nebula { -namespace storage { - -template -nebula::cpp2::ErrorCode LookupBaseProcessor::requestCheck( - const cpp2::LookupIndexRequest& req) { - spaceId_ = req.get_space_id(); - auto retCode = this->getSpaceVidLen(spaceId_); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - return retCode; - } - this->planContext_ = std::make_unique( - this->env_, spaceId_, this->spaceVidLen_, this->isIntId_, req.common_ref()); - const auto& indices = req.get_indices(); - const auto& schemaId = indices.get_schema_id(); - this->planContext_->isEdge_ = schemaId.getType() == nebula::cpp2::SchemaID::Type::edge_type; - this->context_ = std::make_unique(this->planContext_.get()); - if (context_->isEdge()) { - context_->edgeType_ = schemaId.get_edge_type(); - auto edgeName = this->env_->schemaMan_->toEdgeName(spaceId_, context_->edgeType_); - if (!edgeName.ok()) { - return nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND; - } - context_->edgeName_ = std::move(edgeName.value()); - auto allEdges = this->env_->schemaMan_->getAllVerEdgeSchema(spaceId_); - if (!allEdges.ok()) { - return nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND; - } - if (!allEdges.value().count(context_->edgeType_)) { - return nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND; - } - schemas_ = std::move(allEdges).value()[context_->edgeType_]; - context_->edgeSchema_ = schemas_.back().get(); - } else { - context_->tagId_ = schemaId.get_tag_id(); - auto tagName = this->env_->schemaMan_->toTagName(spaceId_, context_->tagId_); - if (!tagName.ok()) { - return nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND; - } - context_->tagName_ = std::move(tagName.value()); - auto allTags = this->env_->schemaMan_->getAllVerTagSchema(spaceId_); - if (!allTags.ok()) { - return nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND; - } - if (!allTags.value().count(context_->tagId_)) { - return nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND; - } - schemas_ = std::move(allTags).value()[context_->tagId_]; - context_->tagSchema_ = schemas_.back().get(); - } - - if (indices.get_contexts().empty() || !req.return_columns_ref().has_value() || - (*req.return_columns_ref()).empty()) { - return nebula::cpp2::ErrorCode::E_INVALID_OPERATION; - } - indexContexts_ = indices.get_contexts(); - - // setup yield columns. - if (req.return_columns_ref().has_value()) { - yieldCols_ = *req.return_columns_ref(); - } - - for (auto&& it : folly::enumerate(yieldCols_)) { - resultDataSet_.colNames.emplace_back(*it); - if (QueryUtils::toReturnColType(*it) != QueryUtils::ReturnColType::kOther) { - deDupColPos_.emplace_back(it.index); - } - } - - // limit - if (req.limit_ref().has_value()) { - if (*req.limit_ref() < 0) { - LOG(ERROR) << "Incorrect parameter : LIMIT = " << *req.limit_ref(); - return nebula::cpp2::ErrorCode::E_INVALID_PARM; - } - limit_ = *req.limit_ref(); - } - - return nebula::cpp2::ErrorCode::SUCCEEDED; -} - -template -bool LookupBaseProcessor::isOutsideIndex(Expression* filter, - const meta::cpp2::IndexItem* index) { - static const std::set propsInEdgeKey{kSrc, kType, kRank, kDst}; - auto fields = index->get_fields(); - switch (filter->kind()) { - case Expression::Kind::kLogicalOr: - case Expression::Kind::kLogicalAnd: { - auto* lExpr = static_cast(filter); - for (auto& expr : lExpr->operands()) { - auto ret = isOutsideIndex(expr, index); - if (ret) { - return ret; - } - } - break; - } - case Expression::Kind::kRelLE: - case Expression::Kind::kRelIn: - case Expression::Kind::kRelGE: - case Expression::Kind::kRelEQ: - case Expression::Kind::kRelLT: - case Expression::Kind::kRelGT: - case Expression::Kind::kRelNE: - case Expression::Kind::kRelNotIn: { - auto* rExpr = static_cast(filter); - auto ret = isOutsideIndex(rExpr->left(), index); - if (ret) { - return ret; - } - ret = isOutsideIndex(rExpr->right(), index); - if (ret) { - return ret; - } - break; - } - case Expression::Kind::kEdgeSrc: - case Expression::Kind::kEdgeType: - case Expression::Kind::kEdgeRank: - case Expression::Kind::kEdgeDst: { - auto* sExpr = static_cast(filter); - auto propName = sExpr->prop(); - return propsInEdgeKey.find(propName) == propsInEdgeKey.end(); - } - case Expression::Kind::kTagProperty: - case Expression::Kind::kEdgeProperty: { - auto* sExpr = static_cast(filter); - auto propName = sExpr->prop(); - auto it = std::find_if(fields.begin(), fields.end(), [&propName](const auto& f) { - return f.get_name() == propName; - }); - return it == fields.end(); - } - default: { - return false; - } - } - return false; -} - -/** - * lookup plan should be : - * +--------+---------+ - * | Plan | - * +--------+---------+ - * | - * +--------+---------+ - * | DeDupNode | - * +--------+---------+ - * | - * +----------+-----------+ - * + IndexOutputNode... + - * +----------+-----------+ - **/ - -template -StatusOr> LookupBaseProcessor::buildPlan( - IndexFilterItem* filterItem, nebula::DataSet* result) { - StoragePlan plan; - // TODO(sky) : Limit is not supported yet for de-dup node. - // Related to paging scan, the de-dup execution plan needs to be refactored - auto deDup = std::make_unique>(result, deDupColPos_); - int32_t filterId = 0; - std::unique_ptr> out; - auto pool = &planContext_->objPool_; - - for (const auto& ctx : indexContexts_) { - const auto& indexId = ctx.get_index_id(); - auto needFilter = ctx.filter_ref().is_set() && !(*ctx.filter_ref()).empty(); - - // Check whether a data node is required. - // If a non-indexed column appears in the WHERE clause or YIELD clause, - // That means need to query the corresponding data. - bool needData = false; - auto index = context_->isEdge() ? this->env_->indexMan_->getEdgeIndex(spaceId_, indexId) - : this->env_->indexMan_->getTagIndex(spaceId_, indexId); - if (!index.ok()) { - return Status::IndexNotFound(); - } - - // check nullable column - bool hasNullableCol = false; - - auto* indexItem = index.value().get(); - auto fields = indexItem->get_fields(); - - for (const auto& col : fields) { - if (!hasNullableCol && col.nullable_ref().value_or(false)) { - hasNullableCol = true; - break; - } - } - - for (const auto& yieldCol : yieldCols_) { - static const std::set propsInKey{kVid, kTag, kSrc, kType, kRank, kDst}; - if (propsInKey.count(yieldCol)) { - continue; - } - auto it = std::find_if(fields.begin(), fields.end(), [&yieldCol](const auto& columnDef) { - return yieldCol == columnDef.get_name(); - }); - if (it == fields.end() || - it->get_type().get_type() == - nebula::meta::cpp2::PropertyType::GEOGRAPHY) { // geography index just stores - // S2CellId, so must read the - // original geo data. - needData = true; - break; - } - } - auto colHints = ctx.get_column_hints(); - - // Check WHERE clause contains columns that ware not indexed - if (ctx.filter_ref().is_set() && !(*ctx.filter_ref()).empty()) { - auto filter = Expression::decode(pool, *ctx.filter_ref()); - auto isFieldsOutsideIndex = isOutsideIndex(filter, indexItem); - if (isFieldsOutsideIndex) { - needData = needFilter = true; - } - } - - if (!needData && !needFilter) { - out = buildPlanBasic(result, ctx, plan, hasNullableCol, fields); - } else if (needData && !needFilter) { - out = buildPlanWithData(result, ctx, plan); - } else if (!needData && needFilter) { - auto expr = Expression::decode(pool, ctx.get_filter()); - auto exprCtx = std::make_unique( - context_->vIdLen(), context_->isIntId(), hasNullableCol, fields); - filterItem->emplace(filterId, std::make_pair(std::move(exprCtx), expr)); - out = buildPlanWithFilter( - result, ctx, plan, (*filterItem)[filterId].first.get(), (*filterItem)[filterId].second); - filterId++; - } else { - auto expr = Expression::decode(pool, ctx.get_filter()); - // Need to get columns in data, expr ctx need to be aware of schema - const auto& schemaName = context_->isEdge() ? context_->edgeName_ : context_->tagName_; - if (schemas_.empty()) { - return Status::Error("Schema not found"); - } - auto exprCtx = std::make_unique(context_->vIdLen(), - context_->isIntId(), - schemaName, - schemas_.back().get(), - context_->isEdge()); - filterItem->emplace(filterId, std::make_pair(std::move(exprCtx), expr)); - out = buildPlanWithDataAndFilter( - result, ctx, plan, (*filterItem)[filterId].first.get(), (*filterItem)[filterId].second); - filterId++; - } - if (out == nullptr) { - return Status::Error("Index scan plan error"); - } - deDup->addDependency(out.get()); - plan.addNode(std::move(out)); - } - plan.addNode(std::move(deDup)); - return plan; -} - -/** - * - * +----------+-----------+ - * + IndexOutputNode + - * +----------+-----------+ - * | - * +----------+-----------+ - * + IndexScanNode + - * +----------+-----------+ - * - * If this is a simple index scan, Just having IndexScanNode is enough. for - *example : tag (c1, c2, c3) index on tag (c1, c2, c3) hint : lookup index where - *c1 == 1 and c2 == 1 and c3 == 1 yield c1,c2,c3 - **/ -template -std::unique_ptr> LookupBaseProcessor::buildPlanBasic( - nebula::DataSet* result, - const cpp2::IndexQueryContext& ctx, - StoragePlan& plan, - bool hasNullableCol, - const std::vector& fields) { - auto indexId = ctx.get_index_id(); - auto colHints = ctx.get_column_hints(); - auto indexScan = std::make_unique>( - context_.get(), indexId, std::move(colHints), limit_); - - auto output = std::make_unique>( - result, context_.get(), indexScan.get(), hasNullableCol, fields); - output->addDependency(indexScan.get()); - plan.addNode(std::move(indexScan)); - return output; -} - -/** - * - * +----------+-----------+ - * + IndexOutputNode + - * +----------+-----------+ - * | - * +----------------+-----------------+ - * + IndexEdgeNode or IndexVertexNode + - * +----------------+-----------------+ - * | - * +----------+-----------+ - * + IndexScanNode + - * +----------+-----------+ - * - * If a non-indexed column appears in the YIELD clause, and no expression - *filtering is required . for example : tag (c1, c2, c3) index on tag (c1, c2) - * hint : lookup index where c1 == 1 and c2 == 1 yield c3 - **/ -template -std::unique_ptr> LookupBaseProcessor::buildPlanWithData( - nebula::DataSet* result, const cpp2::IndexQueryContext& ctx, StoragePlan& plan) { - auto indexId = ctx.get_index_id(); - auto colHints = ctx.get_column_hints(); - - auto indexScan = - std::make_unique>(context_.get(), indexId, std::move(colHints)); - if (context_->isEdge()) { - auto edge = std::make_unique>( - context_.get(), indexScan.get(), schemas_, context_->edgeName_, limit_); - edge->addDependency(indexScan.get()); - auto output = std::make_unique>(result, context_.get(), edge.get()); - output->addDependency(edge.get()); - plan.addNode(std::move(indexScan)); - plan.addNode(std::move(edge)); - return output; - } else { - auto vertex = std::make_unique>( - context_.get(), indexScan.get(), schemas_, context_->tagName_, limit_); - vertex->addDependency(indexScan.get()); - auto output = std::make_unique>(result, context_.get(), vertex.get()); - output->addDependency(vertex.get()); - plan.addNode(std::move(indexScan)); - plan.addNode(std::move(vertex)); - return output; - } -} - -/** - * - * +----------+-----------+ - * + IndexOutputNode + - * +----------+-----------+ - * | - * +----------+-----------+ - * + IndexFilterNode + - * +----------+-----------+ - * | - * +----------+-----------+ - * + IndexScanNode + - * +----------+-----------+ - * - * If have not non-indexed column appears in the YIELD clause, and expression - *filtering is required . for example : tag (c1, c2, c3) index on tag (c1, c2) - * hint : lookup index where c1 > 1 and c2 > 1 - **/ -template -std::unique_ptr> LookupBaseProcessor::buildPlanWithFilter( - nebula::DataSet* result, - const cpp2::IndexQueryContext& ctx, - StoragePlan& plan, - StorageExpressionContext* exprCtx, - Expression* exp) { - auto indexId = ctx.get_index_id(); - auto colHints = ctx.get_column_hints(); - - auto indexScan = - std::make_unique>(context_.get(), indexId, std::move(colHints)); - - auto filter = std::make_unique>( - context_.get(), indexScan.get(), exprCtx, exp, context_->isEdge(), limit_); - filter->addDependency(indexScan.get()); - auto output = - std::make_unique>(result, context_.get(), filter.get(), true); - output->addDependency(filter.get()); - plan.addNode(std::move(indexScan)); - plan.addNode(std::move(filter)); - return output; -} - -/** - * - * +----------+-----------+ - * + IndexOutputNode + - * +----------+-----------+ - * | - * +----------+-----------+ - * + IndexFilterNode + - * +----------+-----------+ - * | - * +----------------+-----------------+ - * + IndexEdgeNode or IndexVertexNode + - * +----------------+-----------------+ - * | - * +----------+-----------+ - * + IndexScanNode + - * +----------+-----------+ - * - * If a non-indexed column appears in the WHERE clause or YIELD clause, - * and expression filtering is required . - * for example : - * tag (c1, c2, c3) - * index on tag (c1, c2) - * hint : lookup index where c1 == 1 and c2 == 1 and c3 > 1 yield c3 - * lookup index where c1 == 1 and c2 == 1 and c3 > 1 - * lookup index where c1 == 1 and c3 == 1 - **/ -template -std::unique_ptr> -LookupBaseProcessor::buildPlanWithDataAndFilter(nebula::DataSet* result, - const cpp2::IndexQueryContext& ctx, - StoragePlan& plan, - StorageExpressionContext* exprCtx, - Expression* exp) { - auto indexId = ctx.get_index_id(); - auto colHints = ctx.get_column_hints(); - - auto indexScan = - std::make_unique>(context_.get(), indexId, std::move(colHints)); - if (context_->isEdge()) { - auto edge = std::make_unique>( - context_.get(), indexScan.get(), schemas_, context_->edgeName_); - edge->addDependency(indexScan.get()); - auto filter = std::make_unique>( - context_.get(), edge.get(), exprCtx, exp, limit_); - filter->addDependency(edge.get()); - - auto output = std::make_unique>(result, context_.get(), filter.get()); - output->addDependency(filter.get()); - plan.addNode(std::move(indexScan)); - plan.addNode(std::move(edge)); - plan.addNode(std::move(filter)); - return output; - } else { - auto vertex = std::make_unique>( - context_.get(), indexScan.get(), schemas_, context_->tagName_); - vertex->addDependency(indexScan.get()); - auto filter = std::make_unique>( - context_.get(), vertex.get(), exprCtx, exp, limit_); - filter->addDependency(vertex.get()); - - auto output = std::make_unique>(result, context_.get(), filter.get()); - output->addDependency(filter.get()); - plan.addNode(std::move(indexScan)); - plan.addNode(std::move(vertex)); - plan.addNode(std::move(filter)); - return output; - } -} -template -void LookupBaseProcessor::profilePlan(StoragePlan& plan) { - auto& nodes = plan.getNodes(); - std::lock_guard lck(BaseProcessor::profileMut_); - for (auto& node : nodes) { - BaseProcessor::profileDetail(node->name_, node->duration_.elapsedInUSec()); - } -} - -} // namespace storage -} // namespace nebula diff --git a/src/storage/index/LookupBaseProcessor.h.bk b/src/storage/index/LookupBaseProcessor.h.bk deleted file mode 100644 index 1815e8a72fa..00000000000 --- a/src/storage/index/LookupBaseProcessor.h.bk +++ /dev/null @@ -1,93 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#ifndef STORAGE_QUERY_LOOKUPBASEPROCESSOR_H_ -#define STORAGE_QUERY_LOOKUPBASEPROCESSOR_H_ - -#include "common/base/Base.h" -#include "storage/BaseProcessor.h" -#include "storage/exec/AggregateNode.h" -#include "storage/exec/DeDupNode.h" -#include "storage/exec/IndexEdgeNode.h" -#include "storage/exec/IndexFilterNode.h" -#include "storage/exec/IndexOutputNode.h" -#include "storage/exec/IndexScanNode.h" -#include "storage/exec/IndexVertexNode.h" -#include "storage/exec/StoragePlan.h" - -namespace nebula { -namespace storage { -using IndexFilterItem = - std::unordered_map, Expression*>>; - -template -class LookupBaseProcessor : public BaseProcessor { - public: - virtual ~LookupBaseProcessor() = default; - - virtual void process(const REQ& req) = 0; - - protected: - LookupBaseProcessor(StorageEnv* env, - const ProcessorCounters* counters, - folly::Executor* executor = nullptr) - : BaseProcessor(env, counters), executor_(executor) {} - - virtual void onProcessFinished() = 0; - - nebula::cpp2::ErrorCode requestCheck(const cpp2::LookupIndexRequest& req); - - bool isOutsideIndex(Expression* filter, const meta::cpp2::IndexItem* index); - - StatusOr> buildPlan(IndexFilterItem* filterItem, nebula::DataSet* result); - - std::unique_ptr> buildPlanBasic( - nebula::DataSet* result, - const cpp2::IndexQueryContext& ctx, - StoragePlan& plan, - bool hasNullableCol, - const std::vector& fields); - - std::unique_ptr> buildPlanWithData(nebula::DataSet* result, - const cpp2::IndexQueryContext& ctx, - StoragePlan& plan); - - std::unique_ptr> buildPlanWithFilter(nebula::DataSet* result, - const cpp2::IndexQueryContext& ctx, - StoragePlan& plan, - StorageExpressionContext* exprCtx, - Expression* exp); - - std::unique_ptr> buildPlanWithDataAndFilter( - nebula::DataSet* result, - const cpp2::IndexQueryContext& ctx, - StoragePlan& plan, - StorageExpressionContext* exprCtx, - Expression* exp); - - void profilePlan(StoragePlan& plan); - - protected: - GraphSpaceID spaceId_; - std::unique_ptr planContext_; - std::unique_ptr context_; - folly::Executor* executor_{nullptr}; - nebula::DataSet resultDataSet_; - std::vector partResults_; - std::vector indexContexts_{}; - std::vector yieldCols_{}; - std::vector filterItems_; - // Save schemas when column is out of index, need to read from data - std::vector> schemas_; - std::vector deDupColPos_; - int64_t limit_ = -1; -}; - -} // namespace storage -} // namespace nebula - -#include "storage/index/LookupBaseProcessor-inl.h" -#endif // STORAGE_QUERY_LOOKUPBASEPROCESSOR_H_ diff --git a/src/storage/index/LookupProcessor.cpp.bk b/src/storage/index/LookupProcessor.cpp.bk deleted file mode 100644 index d25be24594a..00000000000 --- a/src/storage/index/LookupProcessor.cpp.bk +++ /dev/null @@ -1,146 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include "storage/index/LookupProcessor.h" - -#include "storage/exec/DeDupNode.h" - -namespace nebula { -namespace storage { - -ProcessorCounters kLookupCounters; - -void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { - if (executor_ != nullptr) { - executor_->add([req, this]() { this->doProcess(req); }); - } else { - doProcess(req); - } -} - -void LookupProcessor::doProcess(const cpp2::LookupIndexRequest& req) { - auto retCode = requestCheck(req); - if (limit_ == 0) { - onProcessFinished(); - onFinished(); - return; - } - if (req.common_ref().has_value() && req.get_common()->profile_detail_ref().value_or(false)) { - profileDetailFlag_ = true; - } - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - for (auto& p : req.get_parts()) { - pushResultCode(retCode, p); - } - onFinished(); - return; - } - - // todo(doodle): specify by each query - if (!FLAGS_query_concurrently) { - runInSingleThread(req); - } else { - runInMultipleThread(req); - } -} - -void LookupProcessor::runInSingleThread(const cpp2::LookupIndexRequest& req) { - filterItems_.emplace_back(IndexFilterItem()); - auto plan = buildPlan(&filterItems_.front(), &resultDataSet_); - if (!plan.ok()) { - for (auto& p : req.get_parts()) { - pushResultCode(nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND, p); - } - onFinished(); - return; - } - - std::unordered_set failedParts; - for (const auto& partId : req.get_parts()) { - auto ret = plan.value().go(partId); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - if (failedParts.find(partId) == failedParts.end()) { - failedParts.emplace(partId); - handleErrorCode(ret, spaceId_, partId); - } - } - } - if (UNLIKELY(profileDetailFlag_)) { - profilePlan(plan.value()); - } - onProcessFinished(); - onFinished(); -} - -void LookupProcessor::runInMultipleThread(const cpp2::LookupIndexRequest& req) { - // As for lookup, once requestCheck is done, the info in RunTimeContext won't - // be changed anymore. So we only use one RunTimeContext, could make it per - // partition later if necessary. - for (size_t i = 0; i < req.get_parts().size(); i++) { - nebula::DataSet result = resultDataSet_; - partResults_.emplace_back(std::move(result)); - filterItems_.emplace_back(IndexFilterItem()); - } - size_t i = 0; - std::vector>> futures; - for (const auto& partId : req.get_parts()) { - futures.emplace_back(runInExecutor(&filterItems_[i], &partResults_[i], partId)); - i++; - } - - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - CHECK(!t.hasException()); - const auto& tries = t.value(); - for (size_t j = 0; j < tries.size(); j++) { - CHECK(!tries[j].hasException()); - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - resultDataSet_.append(std::move(partResults_[j])); - } - } - // when run each part concurrently, we need to dedup again. - if (!deDupColPos_.empty()) { - DeDupNode::dedup(resultDataSet_.rows, deDupColPos_); - } - this->onProcessFinished(); - this->onFinished(); - }); -} - -folly::Future> LookupProcessor::runInExecutor( - IndexFilterItem* filterItem, nebula::DataSet* result, PartitionID partId) { - return folly::via(executor_, [this, filterItem, result, partId]() { - auto plan = buildPlan(filterItem, result); - if (!plan.ok()) { - return std::make_pair(nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND, partId); - } - auto ret = plan.value().go(partId); - if (UNLIKELY(this->profileDetailFlag_)) { - profilePlan(plan.value()); - } - return std::make_pair(ret, partId); - }); -} - -void LookupProcessor::onProcessFinished() { - if (context_->isEdge()) { - std::transform(resultDataSet_.colNames.begin(), - resultDataSet_.colNames.end(), - resultDataSet_.colNames.begin(), - [this](const auto& col) { return context_->edgeName_ + "." + col; }); - } else { - std::transform(resultDataSet_.colNames.begin(), - resultDataSet_.colNames.end(), - resultDataSet_.colNames.begin(), - [this](const auto& col) { return context_->tagName_ + "." + col; }); - } - resp_.set_data(std::move(resultDataSet_)); -} - -} // namespace storage -} // namespace nebula diff --git a/src/storage/index/LookupProcessor.h.bk b/src/storage/index/LookupProcessor.h.bk deleted file mode 100644 index 6d3cfc70312..00000000000 --- a/src/storage/index/LookupProcessor.h.bk +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#ifndef STORAGE_QUERY_LOOKUP_H_ -#define STORAGE_QUERY_LOOKUP_H_ - -#include "common/base/Base.h" -#include "storage/index/LookupBaseProcessor.h" - -namespace nebula { -namespace storage { - -extern ProcessorCounters kLookupCounters; - -class LookupProcessor - : public LookupBaseProcessor { - public: - static LookupProcessor* instance(StorageEnv* env, - const ProcessorCounters* counters = &kLookupCounters, - folly::Executor* executor = nullptr) { - return new LookupProcessor(env, counters, executor); - } - - void process(const cpp2::LookupIndexRequest& req) override; - - protected: - LookupProcessor(StorageEnv* env, const ProcessorCounters* counters, folly::Executor* executor) - : LookupBaseProcessor( - env, counters, executor) {} - - void onProcessFinished() override; - - private: - void runInSingleThread(const cpp2::LookupIndexRequest& req); - void runInMultipleThread(const cpp2::LookupIndexRequest& req); - - folly::Future> runInExecutor( - IndexFilterItem* filterItem, nebula::DataSet* result, PartitionID partId); - - void doProcess(const cpp2::LookupIndexRequest& req); -}; - -} // namespace storage -} // namespace nebula -#endif // STORAGE_QUERY_LOOKUP_H_ From c3d67344919c5b7c370d0b7a0887a5eb258429ab Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Mon, 25 Oct 2021 11:05:04 +0800 Subject: [PATCH 23/38] clear some useless code --- src/common/utils/IndexKeyUtils.h | 42 ++++---------------------------- 1 file changed, 5 insertions(+), 37 deletions(-) diff --git a/src/common/utils/IndexKeyUtils.h b/src/common/utils/IndexKeyUtils.h index 4e716a79615..7d3cb5899af 100644 --- a/src/common/utils/IndexKeyUtils.h +++ b/src/common/utils/IndexKeyUtils.h @@ -219,49 +219,17 @@ class IndexKeyUtils final { raw.append(c, sizeof(double)); return raw; } - if (v < 0) { - /** - * TODO : now, the -(std::numeric_limits::min()) - * have a problem of precision overflow. current return value is -nan. - */ - auto* c1 = reinterpret_cast(&v); - auto i = *reinterpret_cast(c1); - i = -(std::numeric_limits::max() + i); - auto* c2 = reinterpret_cast(&i); - v = *reinterpret_cast(c2); - } - auto val = folly::Endian::big(v); - auto* c = reinterpret_cast(&val); - c[0] ^= 0x80; - std::string raw; - raw.reserve(sizeof(double)); - raw.append(c, sizeof(double)); - return raw; } static double decodeDouble(const folly::StringPiece& raw) { - { - int64_t val = *reinterpret_cast(raw.data()); - val = folly::Endian::big(val); - if (val < 0) { - val &= 0x7fffffffffffffff; - } else { - val = ~val; - } - return *reinterpret_cast(&val); - } - char* v = const_cast(raw.data()); - v[0] ^= 0x80; - auto val = *reinterpret_cast(v); + int64_t val = *reinterpret_cast(raw.data()); val = folly::Endian::big(val); if (val < 0) { - auto* c1 = reinterpret_cast(&val); - auto i = *reinterpret_cast(c1); - i = -(std::numeric_limits::max() + i); - auto* c2 = reinterpret_cast(&i); - val = *reinterpret_cast(c2); + val &= 0x7fffffffffffffff; + } else { + val = ~val; } - return val; + return *reinterpret_cast(&val); } static std::string encodeTime(const nebula::Time& t) { From 46492857e0148b50ef4a844b753efaab53031185 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:23:46 +0800 Subject: [PATCH 24/38] add comments for IndexNode --- src/storage/exec/IndexNode.h | 65 +++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index 03e058ca60d..263a31c9544 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -13,40 +13,89 @@ #include "storage/CommonUtils.h" namespace nebula { namespace storage { +/** + * IndexNode + * + * Indexnode is the base class for each node that makes up the plan tree. + * + * Member: + * `children_` : all children of the node. + * `context_` : runtime context of plan. + * `name_` : node name which should be set in derive node. + * `duration_` : used to record execution time(exclude children node's time). + * `profileDetail_` : whether record execution time or not. + * + * Function: + * The functions is divided into three parts. + * + * First part is used to build node. This part include constructor/destructor, and + * `IndexNode(const IndexNode& node)` is used to cooperate with `copy` to realize + * the deep copy of node.`copy` function needs to be implemented by the derived + * class itself. + * In fact, the build process is divided into two stages. First, the user needs to + * make various derived classes and nd organize them into a plan tree(by + * `children_`).After that, the root node of plan tree needs to call the init + * function and recursively call the init function of all child nodes, `Initcontext` + * will pass parameters between nodes to determine the data format or other + * information to be returned between nodes during execution.Note that `init` needs + * to be executed before `copy`. + * + * Second part is used to access data. + * `execute` is used to initialize some variables at the beginning of each part(e.g + * dedup set, kv iterator, etc.) + * `next` is used to iterate data. Row format has been determined during `init`. + * Batch processing and loop unrolling can be used to optimize performance if + * necessary, but there are no serious performance problems at present. + * `end` and `finish` are used to release resources at the end of execute or plan + * (e.g, external sort free disk,release schema lease if support Online DDL,etc.). + * However, there are no relevant requirements, so it will not be implemented for + * the time being. + * `xxx` is the interface function.It will recursive call child node's `xxx`. `doXxx` + * is the actual execution logic, and the derived class needs to override this + * function + * + * The third part is used to assist in obtaining some detailed information + */ + using ErrorCode = ::nebula::cpp2::ErrorCode; template using Map = folly::F14FastMap; template using Set = folly::F14FastSet; struct InitContext { + // Column required by parent node Set requiredColumns; + // The format of the row returned to the parent node std::vector returnColumns; + // The index of name in `returncolumns` Map retColMap; }; -/***/ + class IndexNode { public: + /* typedef */ template using ErrorOr = ::nebula::ErrorOr; + + /* build */ IndexNode(const IndexNode& node); explicit IndexNode(RuntimeContext* context, const std::string& name); virtual ~IndexNode() = default; + virtual std::unique_ptr copy() = 0; + void addChild(std::unique_ptr child) { children_.emplace_back(std::move(child)); } + const std::vector>& children() { return children_; } virtual ::nebula::cpp2::ErrorCode init(InitContext& initCtx) { DCHECK_EQ(children_.size(), 1); return children_[0]->init(initCtx); } - + /* execution */ inline nebula::cpp2::ErrorCode execute(PartitionID partId); - inline ErrorOr next(bool& hasNext); + // inline nebula::cpp2::ErrorCode finish(); - void addChild(std::unique_ptr child) { children_.emplace_back(std::move(child)); } - virtual std::unique_ptr copy() = 0; - const std::vector>& children() { return children_; } + /* assist */ const std::string& name() { return name_; } - void enableProfileDetail(); - virtual std::string identify() = 0; inline const time::Duration& duration(); From 490a1bebfe2f3b8c3ce1d9ec23a7888857d48b1e Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Tue, 26 Oct 2021 12:52:16 +0800 Subject: [PATCH 25/38] add comments to IndexScanNode --- src/storage/exec/IndexEdgeScanNode.h | 6 ++ src/storage/exec/IndexNode.h | 3 +- src/storage/exec/IndexScanNode.cpp | 4 +- src/storage/exec/IndexScanNode.h | 109 +++++++++++++++++++++++-- src/storage/exec/IndexVertexScanNode.h | 6 ++ 5 files changed, 118 insertions(+), 10 deletions(-) diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index ecdca4a73c4..86fb093c423 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -12,6 +12,12 @@ namespace nebula { namespace storage { +/** + * IndexEdgeScanNode + * + * reference: IndexScanNode + */ + class IndexEdgeScanNode : public IndexScanNode { public: IndexEdgeScanNode(const IndexEdgeScanNode& node); diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index 263a31c9544..38410c40684 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -47,7 +47,8 @@ namespace storage { * Batch processing and loop unrolling can be used to optimize performance if * necessary, but there are no serious performance problems at present. * `end` and `finish` are used to release resources at the end of execute or plan - * (e.g, external sort free disk,release schema lease if support Online DDL,etc.). + * (e.g, external sort free disk,release schema lease if support Online DDL, commit + * write, etc.). * However, there are no relevant requirements, so it will not be implemented for * the time being. * `xxx` is the interface function.It will recursive call child node's `xxx`. `doXxx` diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index 60f9d8ccbce..ba1a48dd5cf 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -199,7 +199,9 @@ std::tuple RangePath::encodeRange( return nullableBit.test(15 - colIndex) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; }); } - // One GEO data will generator more than one index key. So need dedup. + // The data of a geo may generate multiple indexes, so it needs to be de duplicated. Because there + // is no duplicate data between different parts, the suffixset does not clear and does not affect + // the results. if (UNLIKELY(colTypeDef.get_type() == nebula::meta::cpp2::PropertyType::GEOGRAPHY)) { QFList_.emplace_back([suffixSet = Set(), suffixLength = suffixLength_](const std::string& k) mutable { diff --git a/src/storage/exec/IndexScanNode.h b/src/storage/exec/IndexScanNode.h index fbbda690cd4..b21e042805c 100644 --- a/src/storage/exec/IndexScanNode.h +++ b/src/storage/exec/IndexScanNode.h @@ -19,13 +19,112 @@ namespace nebula { namespace storage { +/** + * + * IndexScanNode + * + * reference: IndexNode, IndexVertexScanNode, IndexEdgeScanNode + * + * `IndexScanNode` is the base class of the node which need to access disk. It has two derive + * class `IndexVertexScanNode` and `IndexEdgeScanNode` + * + * ┌───────────┐ + * │ IndexNode │ + * └─────┬─────┘ + * │ + * ┌───────┴───────┐ + * │ IndexScanNode │ + * └───────┬───────┘ + * ┌───────────┴────────────┐ + * ┌──────────┴──────────┐ ┌───────────┴─────────┐ + * │ IndexVertexScanNode │ │ IndexEdgeScanNode │ + * └─────────────────────┘ └─────────────────────┘ + * + * `IndexScanNode` will access index data, and then access base data if necessary. + * + * Member: + * `indexId_` : index_ in this Node to access + * `partId_` : part to access.It will be modify while `doExecute` + * `index_` : index defination + * `indexNullable_` : if index contain nullable field or not + * `columnHints_` : + * `path_` : + * `iter_` : current kvstore iterator.It while be reseted `doExecute` and iterated + * during `doNext` + * `kvstore_` : server kvstore + * `requiredColumns_` : row format that `doNext` needs to return + * `requiredAndHintColumns_`: columns that `decodeFromBase` needs to decode + * `ttlProps` : ttl properties `needAccesBase_` : if need + * `fatalOnBaseNotFound_` : for debug + * + * Function: + * `decodePropFromIndex` : decode properties from Index key.It will be called by + * `decodeFromIndex` + * `decodeFromIndex` : decode all column in `requiredColumns_` by index + * key-value. + * `getBaseData` : get key-value of base data `decodeFromBase` : get + * all values that `requiredAndHintColumns_` required + * `checkTTL` : check data is + * expired or not + * ------------------------------------------------------------- + * + * Path + * + * `Path` is the most important part of `IndexScanNode`. By analyzing `ColumnHint`, it obtains + * the mode(Prefix or Range) and range(key of Prefix or [start,end) of Range) of keys that + * `IndexScanNode` need to query in kvstore. + * + * `Path` not only generate the key to access, but also `qualified` whether the key complies with + * the columnhint constraint or not.For example, if there is a truncated string index, we cannot + * simply compare bytes to determine whether the current key complies with the columnhints + * constraint, the result of `qulified(bytes)` should be `UNCERTAIN` and `IndexScanNode` will + * access base data then `Path` reconfirm `ColumnHint` constraint by `qulified(RowData)`. In + * addition to the above examples, there are other cases to deal with.`Path` and it's derive class + * will dynamic different strategy by `ColumnHint`,`IndexItem`,and `Schema`.All strategy will be + * added to `QFList_`(QualifiedFunctionList) during `buildKey`, and executed during `qualified`. + * + * `Path` whild be reseted when `IndexScanNode` execute on a new part. + * + * It should be noted that the range generated by `rangepath` is a certain left included and right + * excluded interval,like [startKey_, endKey_), although `ColumnHint` may have many different + * constraint ranges(e.g., (x, y],(INF,y),(x,INF)). Because the length of index key is fixed, the + * way to obtain **the smallest key greater than 'x'** is to append several '\xFF' after until the + * length of 'x' is greater than the length of the indexkey. + * + * + * Member: + * `QFList_` : all Qualified strategy need to executed during qualified + * `nullable_` : if `index_` contain nullable field, `nullable_[i]` is equal to + * `index_->fields[i].nullable`,else `nullable_` is empty + * `index_nullable_offset_` : Participate in the index key encode diagram + * `totalKeyLength_` : Participate in the index key encode diagram + * `suffixLength_` : Participate in the index key encode diagram + * `serializeString_` : a string express path + * + * Index Key Encode: + * ┌──────┬─────────────┬────────────────┬──────────┬─────────────────────────────────────────┐ + * │ type | PartitionID | Indexed Values | nullable | suffix({vid} or {srcId,rank,dstId}) | + * │ 1byte| 3 bytes | n bytes | 0/2 bytes| vid.length or vid.length*2+sizeof(rank) | + * └──────┴─────────────┴────────────────┴──────────┴─────────────────────────────────────────┘ + * │ └───────────────────┬─────────────────────┘ + * index_nullable_offset_ suffixLength_ + * └──────────────────────────────────┬───────────────────────────────────────────────────────┘ + * totalKeyLength_ + * + * Function: + * `make` : construct `PrefixPath` or `RangePath` according to `hints` + * `qualified(StringPiece)` : qulified key by bytes + * `qualified(Map)` : qulified row by value + * `resetPart` : reset current partitionID and reset `iter_` + * `encodeValue` : encode a Value to bytes + */ + class Path; -class RangePath; -class PrefixPath; class IndexScanNode : public IndexNode { FRIEND_TEST(IndexScanTest, Base); FRIEND_TEST(IndexScanTest, Vertex); FRIEND_TEST(IndexScanTest, Edge); + // There are too many unittests, so a helper is defined to access private data friend class IndexScanTestHelper; public: @@ -67,11 +166,6 @@ class IndexScanNode : public IndexNode { bool needAccessBase_{false}; bool fatalOnBaseNotFound_{false}; }; -/** - * Path - * - * Path表示一个Index查询范围（range或prefix）。 - */ class Path { public: enum class Qualified : int16_t { INCOMPATIBLE = 0, UNCERTAIN = 1, COMPATIBLE = 2 }; @@ -131,7 +225,6 @@ class RangePath : public Path { const meta::SchemaProviderIf* schema, const std::vector& hints, int64_t vidLen); - // Override Qualified qualified(const Map& rowData) override; void resetPart(PartitionID partId) override; diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index 689c72ec584..b4a294996ce 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -16,6 +16,12 @@ namespace nebula { namespace storage { + +/** + * IndexVertexScanNode + * + * reference: IndexScanNode + */ class IndexVertexScanNode final : public IndexScanNode { public: IndexVertexScanNode(const IndexVertexScanNode& node); From d6ebf10aa27d41b199da1dd7fb4f9c8594833aaa Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Tue, 26 Oct 2021 16:12:28 +0800 Subject: [PATCH 26/38] ad comment to Selection/Projection/DedupNode --- src/storage/exec/IndexDedupNode.h | 22 ++++++++++++++++++ src/storage/exec/IndexProjectionNode.h | 20 +++++++++++++++++ src/storage/exec/IndexSelectionNode.h | 31 ++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index 4e2673bed82..e7b45d8995e 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -9,6 +9,28 @@ #include "storage/exec/IndexNode.h" namespace nebula { namespace storage { +/** + * + * IndexDedupNode + * + * reference: IndexNode + * + * `IndexDedupNode` is the class which is used to eliminate duplicate rows of data returned by + * multiple child nodes. + * ┌───────────┐ + * │ IndexNode │ + * └─────┬─────┘ + * │ + * ┌────────┴───────┐ + * │ IndexDedupNode │ + * └────────────────┘ + * Member: + * `dedupColumns_`: columns' name which are used to dedup + * `dedupPos_` : dedup columns' position in child return row + * `dedupSet_` : the set which record the rows have been return to parent + * `currentChild_`: current iterate child + */ + class IndexDedupNode : public IndexNode { public: IndexDedupNode(const IndexDedupNode& node); diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index 0d61246b9c3..7f52ba2d907 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -9,6 +9,26 @@ #include "storage/exec/IndexNode.h" namespace nebula { namespace storage { +/** + * + * IndexProjectionNode + * + * reference: IndexNode + * + * `IndexProjectionNode` is the class which is used to reformat the row to ensure that the format of + * the returned row meets the requirements of RPC request. + * ┌───────────┐ + * │ IndexNode │ + * └─────┬─────┘ + * │ + * ┌──────────┴──────────┐ + * │ IndexProjectionNode │ + * └─────────────────────┘ + * + * Member: + * `requiredColumns_` : Row format required by parent node + * `colPos_` : each column position in child node return row + */ class IndexProjectionNode : public IndexNode { public: IndexProjectionNode(const IndexProjectionNode& node); diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index c1dc4fd50b9..f9d707e673c 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -12,6 +12,37 @@ #include "storage/exec/IndexNode.h" namespace nebula { namespace storage { +/** + * + * IndexSelectionNode + * + * reference: IndexNode + * + * `IndexSelectionNode` is the class which is used to filter data by given expression in RPC + * request. + * ┌───────────┐ + * │ IndexNode │ + * └─────┬─────┘ + * │ + * ┌─────────┴──────────┐ + * │ IndexSelectionNode │ + * └────────────────────┘ + * Member: + * `expr_` : expression used to filter + * `colPos_`: column's position in Row which is during eval `expr_` + * `ctx_` : used to eval expression + * Function: + * `filter` : compute `expr_` + * + * + * ------------------------------------------------------------------------------------------------ + * IndexSelectionNode::ExprContext + * + * `ExprContext` is a derive class of ExpressionContext which is needed in eval expression. + * NOTICE: There are many node in the entire storage plan tree where expressions need to be + * evaluated(e.g., Projection,Aggregate,etc.). So `ExprContext` may be not an internal + * class of IndexSelectionNode. + */ class IndexSelectionNode : public IndexNode { public: IndexSelectionNode(const IndexSelectionNode &node); From 27b15f913ba94cf87a6edc5709f4ff2728054777 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Wed, 27 Oct 2021 14:43:18 +0800 Subject: [PATCH 27/38] fix some init bug --- src/storage/exec/IndexDedupNode.cpp | 5 ++- src/storage/exec/IndexEdgeScanNode.cpp | 30 +++++++++---- src/storage/exec/IndexEdgeScanNode.h | 10 +++-- src/storage/exec/IndexScanNode.cpp | 2 +- src/storage/exec/IndexVertexScanNode.cpp | 46 ++++++++++---------- src/storage/exec/IndexVertexScanNode.h | 13 +++--- src/storage/index/LookupProcessor.cpp | 55 +++++++++++++++--------- src/storage/test/IndexTest.cpp | 33 ++++++++------ 8 files changed, 114 insertions(+), 80 deletions(-) diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index 07ae2640332..de0aedac4ce 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -19,7 +19,10 @@ ::nebula::cpp2::ErrorCode IndexDedupNode::init(InitContext& ctx) { InitContext childCtx = ctx; InitContext ctx2; for (auto& child : children_) { - child->init(childCtx); + auto ret = child->init(childCtx); + if (ret != ::nebula::cpp2::ErrorCode::SUCCEEDED) { + return ret; + } ctx2 = childCtx; childCtx = ctx; } diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index 58103041428..b600b661c77 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -14,23 +14,35 @@ IndexEdgeScanNode::IndexEdgeScanNode(RuntimeContext* context, const std::vector& columnHint, ::nebula::kvstore::KVStore* kvstore) : IndexScanNode(context, "IndexEdgeScanNode", indexId, columnHint, kvstore) { - getIndex = std::function([this]() { + getIndex = std::function([this](std::shared_ptr& index) { auto env = this->context_->env(); auto indexMgr = env->indexMan_; - auto index = indexMgr->getEdgeIndex(this->spaceId_, this->indexId_).value(); - return index; + auto indexVal = indexMgr->getEdgeIndex(this->spaceId_, this->indexId_); + if (!indexVal.ok()) { + return ::nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND; + } + index = indexVal.value(); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; }); - getEdge = std::function([this]() { + getEdge = std::function([this](EdgeSchemas& edge) { auto env = this->context_->env(); auto schemaMgr = env->schemaMan_; - auto schema = schemaMgr->getAllVerEdgeSchema(this->spaceId_) - .value()[this->index_->get_schema_id().get_edge_type()]; - return schema; + auto allSchema = schemaMgr->getAllVerEdgeSchema(this->spaceId_); + auto edgeType = this->index_->get_schema_id().get_edge_type(); + if (!allSchema.ok() || !allSchema.value().count(edgeType)) { + return ::nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND; + } + edge = allSchema.value().at(edgeType); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; }); } ::nebula::cpp2::ErrorCode IndexEdgeScanNode::init(InitContext& ctx) { - index_ = getIndex().value(); - edge_ = getEdge(); + if (auto ret = getIndex(this->index_); UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + return ret; + } + if (auto ret = getEdge(edge_); UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + return ret; + } return IndexScanNode::init(ctx); } Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index 86fb093c423..9c58aabb66b 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -33,13 +33,15 @@ class IndexEdgeScanNode : public IndexScanNode { nebula::cpp2::ErrorCode getBaseData(folly::StringPiece key, std::pair& kv) override; Map decodeFromBase(const std::string& key, const std::string& value) override; - const std::vector>& getSchema() override; - std::vector> edge_; + using EdgeSchemas = std::vector>; + using IndexItem = ::nebula::meta::cpp2::IndexItem; + const EdgeSchemas& getSchema() override; + EdgeSchemas edge_; // Convenient for testing - std::function>()> getIndex; - std::function>()> getEdge; + std::function<::nebula::cpp2::ErrorCode(std::shared_ptr&)> getIndex; + std::function<::nebula::cpp2::ErrorCode(EdgeSchemas&)> getEdge; FRIEND_TEST(IndexScanTest, Edge); friend class IndexScanTestHelper; diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index ba1a48dd5cf..bca1089d4e1 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -376,7 +376,7 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { DCHECK(requiredColumns_.empty()); ttlProps_ = CommonUtils::ttlProps(getSchema().back().get()); requiredAndHintColumns_ = ctx.requiredColumns; - + auto schema = getSchema().back(); for (auto& hint : columnHints_) { requiredAndHintColumns_.insert(hint.get_column_name()); } diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index c21d7c63eaa..c5c2d4e2065 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -12,44 +12,42 @@ namespace nebula { namespace storage { IndexVertexScanNode::IndexVertexScanNode(const IndexVertexScanNode& node) - : IndexScanNode(node), tag_(node.tag_) { - getIndex = std::function([this]() { - auto env = this->context_->env(); - auto indexMgr = env->indexMan_; - auto index = indexMgr->getTagIndex(this->spaceId_, this->indexId_).value(); - return index; - }); - getTag = std::function([this]() { - auto env = this->context_->env(); - auto schemaMgr = env->schemaMan_; - auto schema = schemaMgr->getAllVerTagSchema(this->spaceId_) - .value()[this->index_->get_schema_id().get_tag_id()]; - return schema; - }); -} + : IndexScanNode(node), tag_(node.tag_) {} IndexVertexScanNode::IndexVertexScanNode(RuntimeContext* context, IndexID indexId, const std::vector& clolumnHint, ::nebula::kvstore::KVStore* kvstore) : IndexScanNode(context, "IndexVertexScanNode", indexId, clolumnHint, kvstore) { - getIndex = std::function([this]() { + getIndex = std::function([this](std::shared_ptr& index) { auto env = this->context_->env(); auto indexMgr = env->indexMan_; - auto index = indexMgr->getTagIndex(this->spaceId_, this->indexId_).value(); - return index; + auto indexVal = indexMgr->getTagIndex(this->spaceId_, this->indexId_); + if (!indexVal.ok()) { + return ::nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND; + } + index = indexVal.value(); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; }); - getTag = std::function([this]() { + getTag = std::function([this](TagSchemas& tag) { auto env = this->context_->env(); auto schemaMgr = env->schemaMan_; - auto schema = schemaMgr->getAllVerTagSchema(this->spaceId_) - .value()[this->index_->get_schema_id().get_tag_id()]; - return schema; + auto allSchema = schemaMgr->getAllVerTagSchema(this->spaceId_); + auto tagId = this->index_->get_schema_id().get_tag_id(); + if (!allSchema.ok() || !allSchema.value().count(tagId)) { + return ::nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND; + } + tag = allSchema.value().at(tagId); + return ::nebula::cpp2::ErrorCode::SUCCEEDED; }); } ::nebula::cpp2::ErrorCode IndexVertexScanNode::init(InitContext& ctx) { - this->index_ = getIndex().value(); - this->tag_ = getTag(); + if (auto ret = getIndex(this->index_); UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + return ret; + } + if (auto ret = getTag(tag_); UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + return ret; + } return IndexScanNode::init(ctx); } nebula::cpp2::ErrorCode IndexVertexScanNode::getBaseData(folly::StringPiece key, diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index b4a294996ce..0c8b5baccf2 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -37,15 +37,14 @@ class IndexVertexScanNode final : public IndexScanNode { std::pair& kv) override; Row decodeFromIndex(folly::StringPiece key) override; Map decodeFromBase(const std::string& key, const std::string& value) override; - const std::vector>& getSchema() override { - return tag_; - } - - std::vector> tag_; + using TagSchemas = std::vector>; + const TagSchemas& getSchema() override { return tag_; } + TagSchemas tag_; + using IndexItem = ::nebula::meta::cpp2::IndexItem; // Convenient for testing - std::function>()> getIndex; - std::function>()> getTag; + std::function<::nebula::cpp2::ErrorCode(std::shared_ptr&)> getIndex; + std::function<::nebula::cpp2::ErrorCode(TagSchemas&)> getTag; FRIEND_TEST(IndexScanTest, VertexIndexOnlyScan); FRIEND_TEST(IndexScanTest, VertexBase); diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index 89b1b614c08..9e4c095fdd4 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -22,27 +22,45 @@ namespace nebula { namespace storage { ProcessorCounters kLookupCounters; +// print Plan for debug +inline void printPlan(IndexNode* node, int tab = 0); void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { - DLOG(INFO) << ::apache::thrift::SimpleJSONSerializer::serialize(req); + DVLOG(1) << ::apache::thrift::SimpleJSONSerializer::serialize(req); if (executor_ != nullptr) { executor_->add([req, this]() { this->doProcess(req); }); } else { doProcess(req); } } -void printPlan(IndexNode* node, int tab = 0) { - LOG(INFO) << std::string(tab, '\t') << node->identify(); - for (auto& child : node->children()) { - printPlan(child.get(), tab + 1); - } -} + void LookupProcessor::doProcess(const cpp2::LookupIndexRequest& req) { if (req.common_ref().has_value() && req.get_common()->profile_detail_ref().value_or(false)) { profileDetailFlag_ = true; } - prepare(req); + auto code = prepare(req); + if (UNLIKELY(code != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + for (auto& p : req.get_parts()) { + pushResultCode(code, p); + } + onFinished(); + DVLOG(1) << static_cast(code); + return; + } auto plan = buildPlan(req); - printPlan(plan.get()); + + if (UNLIKELY(profileDetailFlag_)) { + plan->enableProfileDetail(); + } + InitContext ctx; + code = plan->init(ctx); + if (UNLIKELY(code != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { + for (auto& p : req.get_parts()) { + pushResultCode(code, p); + } + onFinished(); + DVLOG(1) << static_cast(code); + return; + } if (!FLAGS_query_concurrently) { runInSingleThread(req.get_parts(), std::move(plan)); } else { @@ -109,15 +127,6 @@ std::unique_ptr LookupProcessor::buildPlan(const cpp2::LookupIndexReq node->addChild(std::move(nodes[0])); nodes[0] = std::move(node); } - InitContext ctx; - auto result = nodes[0]->init(ctx); - if (profileDetailFlag_) { - nodes[0]->enableProfileDetail(); - } - // TODO(hs.zhang): check init result - if (result == ::nebula::cpp2::ErrorCode::SUCCEEDED) { - } else { - } return std::move(nodes[0]); } @@ -144,7 +153,7 @@ std::unique_ptr LookupProcessor::buildOneContext(const cpp2::IndexQue void LookupProcessor::runInSingleThread(const std::vector& parts, std::unique_ptr plan) { - printPlan(plan.get()); + // printPlan(plan.get()); std::vector> datasetList; std::vector<::nebula::cpp2::ErrorCode> codeList; for (auto part : parts) { @@ -216,7 +225,6 @@ void LookupProcessor::runInMultipleThread(const std::vector& parts, return {part, code, dataset}; })); } - DLOG(INFO) << "xxxxxxxxxxxxxxxxxxxxxxx"; folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) { CHECK(!t.hasException()); const auto& tries = t.value(); @@ -269,6 +277,11 @@ void LookupProcessor::profilePlan(IndexNode* root) { } } } - +inline void printPlan(IndexNode* node, int tab) { + DVLOG(2) << std::string(tab, '\t') << node->identify(); + for (auto& child : node->children()) { + printPlan(child.get(), tab + 1); + } +} } // namespace storage } // namespace nebula diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index bdf5b34fec2..be8483396c8 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -69,21 +69,29 @@ using std::string_literals::operator""s; */ struct IndexScanTestHelper { void setIndex(IndexVertexScanNode* node, std::shared_ptr<::nebula::meta::cpp2::IndexItem> index) { - node->getIndex = [index]() { return index; }; + node->getIndex = [index](std::shared_ptr<::nebula::meta::cpp2::IndexItem>& ret) { + ret = index; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + }; } void setIndex(IndexEdgeScanNode* node, std::shared_ptr<::nebula::meta::cpp2::IndexItem> index) { - node->getIndex = [index]() { return index; }; + node->getIndex = [index](std::shared_ptr<::nebula::meta::cpp2::IndexItem>& ret) { + ret = index; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; + }; } void setTag(IndexVertexScanNode* node, std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema) { - node->getTag = [schema]() { - return std::vector>{schema}; + node->getTag = [schema](IndexVertexScanNode::TagSchemas& tag) { + tag = std::vector>{schema}; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; } void setEdge(IndexEdgeScanNode* node, std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema) { - node->getEdge = [schema]() { - return std::vector>{schema}; + node->getEdge = [schema](IndexEdgeScanNode::EdgeSchemas& edge) { + edge = std::vector>{schema}; + return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; } void setFatal(IndexScanNode* node, bool value) { node->fatalOnBaseNotFound_ = value; } @@ -231,7 +239,7 @@ TEST_F(IndexScanTest, Base) { auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); IndexScanTestHelper helper; - scanNode->getIndex = [index = indices[0]]() { return index; }; + helper.setIndex(scanNode.get(), indices[0]); helper.setTag(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kVid, "a"}; @@ -270,7 +278,7 @@ TEST_F(IndexScanTest, Base) { auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); IndexScanTestHelper helper; - scanNode->getIndex = [index = indices[1]]() { return index; }; + helper.setIndex(scanNode.get(), indices[1]); helper.setTag(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kVid, "b"}; @@ -329,8 +337,7 @@ TEST_F(IndexScanTest, Vertex) { auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); IndexScanTestHelper helper; - - scanNode->getIndex = [index = indices[0]]() { return index; }; + helper.setIndex(scanNode.get(), indices[0]); helper.setTag(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kVid, "a"}; @@ -368,7 +375,7 @@ TEST_F(IndexScanTest, Vertex) { auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); IndexScanTestHelper helper; - scanNode->getIndex = [index = indices[0]]() { return index; }; + helper.setIndex(scanNode.get(), indices[0]); helper.setTag(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kVid, "b"}; @@ -430,7 +437,7 @@ TEST_F(IndexScanTest, Edge) { auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); IndexScanTestHelper helper; - scanNode->getIndex = [index = indices[0]]() { return index; }; + helper.setIndex(scanNode.get(), indices[0]); helper.setEdge(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kSrc, kRank, kDst, "c"}; @@ -468,7 +475,7 @@ TEST_F(IndexScanTest, Edge) { auto scanNode = std::make_unique(context.get(), indexId, columnHints, kvstore.get()); IndexScanTestHelper helper; - scanNode->getIndex = [index = indices[0]]() { return index; }; + helper.setIndex(scanNode.get(), indices[0]); helper.setEdge(scanNode.get(), schema); InitContext initCtx; initCtx.requiredColumns = {kSrc, kRank, kDst, "a"}; From 5d8b86da3ea47fbd80f43b4761464e4d50de7931 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Fri, 29 Oct 2021 13:36:13 +0800 Subject: [PATCH 28/38] fix bug to support geo --- src/storage/exec/IndexScanNode.cpp | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index bca1089d4e1..4f79df7a438 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -63,7 +63,10 @@ std::string Path::encodeValue(const Value& value, std::string& key) { std::string val; bool isNull = false; - if (value.type() == Value::Type::STRING) { + if (colDef.get_type() == ::nebula::meta::cpp2::PropertyType::GEOGRAPHY) { + CHECK_EQ(value.type(), Value::Type::STRING); + val = value.getStr(); + } else if (value.type() == Value::Type::STRING) { val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); if (val.back() != '\0') { QFList_.clear(); @@ -219,7 +222,9 @@ std::string RangePath::encodeBeginValue(const Value& value, std::string val; bool greater = !includeStart_; CHECK_NE(value.type(), Value::Type::NULLVALUE); - if (value.type() == Value::Type::STRING) { + if (colDef.get_type() == ::nebula::meta::cpp2::PropertyType::GEOGRAPHY) { + val = value.getStr(); + } else if (value.type() == Value::Type::STRING) { bool truncated = false; val = encodeString(value, *colDef.get_type_length(), truncated); greater &= !truncated; @@ -257,7 +262,9 @@ std::string RangePath::encodeEndValue(const Value& value, CHECK_NE(value.type(), Value::Type::NULLVALUE); std::string val; bool greater = includeEnd_; - if (value.type() == Value::Type::STRING) { + if (colDef.get_type() == ::nebula::meta::cpp2::PropertyType::GEOGRAPHY) { + val = value.getStr(); + } else if (value.type() == Value::Type::STRING) { bool truncated = false; val = encodeString(value, *colDef.get_type_length(), truncated); greater |= truncated; @@ -348,6 +355,17 @@ void PrefixPath::buildKey() { serializeString_ += fmt::format("{}={}, ", hint.get_column_name(), hint.get_begin_value().toString()); } + for (; fieldIter != index_->get_fields().end(); fieldIter++) { + if (UNLIKELY(fieldIter->get_type().get_type() == nebula::meta::cpp2::PropertyType::GEOGRAPHY)) { + QFList_.emplace_back([suffixSet = Set(), + suffixLength = suffixLength_](const std::string& k) mutable { + auto suffix = k.substr(k.size() - suffixLength, suffixLength); + auto [iter, result] = suffixSet.insert(suffix); + DLOG(INFO) << "qualified geo dedup " << result; + return result ? Qualified::COMPATIBLE : Qualified::INCOMPATIBLE; + }); + } + } prefix_ = std::move(common); } // End of PrefixPath @@ -534,6 +552,9 @@ void IndexScanNode::decodePropFromIndex(folly::StringPiece key, case Value::Type::DATETIME: len = sizeof(int32_t) + sizeof(int16_t) + sizeof(int8_t) * 5; break; + case Value::Type::GEOGRAPHY: // colPosMap will never need GEOGRAPHY type + len = 8; + break; default: LOG(FATAL) << "Unexpect value type:" << int(field.type.get_type()); } From cd3c9cd7dbe83fe79242395e2d537443f6b4a9dc Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Fri, 29 Oct 2021 19:22:42 +0800 Subject: [PATCH 29/38] converge qualified strategy define together --- src/storage/exec/IndexScanNode.cpp | 110 +++++------------ src/storage/exec/IndexScanNode.h | 189 +++++++++++++++++++++++++++-- 2 files changed, 211 insertions(+), 88 deletions(-) diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index 4f79df7a438..e1eff160ef7 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -50,12 +50,8 @@ std::unique_ptr Path::make(nebula::meta::cpp2::IndexItem* index, } return ret; } -Path::Qualified Path::qualified(const folly::StringPiece& key) { - Qualified ret = Qualified::COMPATIBLE; - for (auto& func : QFList_) { - ret = std::min(ret, func(key.toString())); - } - return ret; +QualifiedStrategy::Result Path::qualified(const folly::StringPiece& key) { + return strategySet_(key); } std::string Path::encodeValue(const Value& value, const ColumnTypeDef& colDef, @@ -69,8 +65,7 @@ std::string Path::encodeValue(const Value& value, } else if (value.type() == Value::Type::STRING) { val = IndexKeyUtils::encodeValue(value, *colDef.get_type_length()); if (val.back() != '\0') { - QFList_.clear(); - QFList_.emplace_back([](const std::string&) { return Qualified::UNCERTAIN; }); + strategySet_.insert(QualifiedStrategy::constant()); } } else if (value.type() == Value::Type::NULLVALUE) { auto vtype = IndexKeyUtils::toValueType(colDef.get_type()); @@ -80,17 +75,13 @@ std::string Path::encodeValue(const Value& value, val = IndexKeyUtils::encodeValue(value); } if (!nullable_.empty() && nullable_[index] == true) { - QFList_.emplace_back([isNull, index, offset = index_nullable_offset_](const std::string& k) { - std::bitset<16> nullableBit; - auto v = *reinterpret_cast(k.data() + offset); - nullableBit = v; - DVLOG(3) << isNull; - DVLOG(3) << nullableBit.test(15 - index); - return nullableBit.test(15 - index) ^ isNull ? Qualified::INCOMPATIBLE - : Qualified::COMPATIBLE; - }); + if (isNull) { + strategySet_.insert(QualifiedStrategy::checkNull(index, index_nullable_offset_)); + } else { + strategySet_.insert(QualifiedStrategy::checkNull(index, index_nullable_offset_)); + } } else if (isNull) { - QFList_.emplace_back([](const std::string&) { return Qualified::INCOMPATIBLE; }); + strategySet_.insert(QualifiedStrategy::constant()); } key.append(val); return val; @@ -112,11 +103,11 @@ void RangePath::resetPart(PartitionID partId) { startKey_ = startKey_.replace(0, p.size(), p); endKey_ = endKey_.replace(0, p.size(), p); } -Path::Qualified RangePath::qualified(const Map& rowData) { +QualifiedStrategy::Result RangePath::qualified(const Map& rowData) { for (size_t i = 0; i < hints_.size() - 1; i++) { auto& hint = hints_[i]; if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { - return Qualified::INCOMPATIBLE; + return QualifiedStrategy::INCOMPATIBLE; } } auto& hint = hints_.back(); @@ -125,7 +116,7 @@ Path::Qualified RangePath::qualified(const Map& rowData) { bool ret = includeStart_ ? hint.get_begin_value() <= rowData.at(hint.get_column_name()) : hint.get_begin_value() < rowData.at(hint.get_column_name()); if (!ret) { - return Qualified::INCOMPATIBLE; + return QualifiedStrategy::INCOMPATIBLE; } } if (hint.end_value_ref().is_set()) { @@ -134,10 +125,10 @@ Path::Qualified RangePath::qualified(const Map& rowData) { : hint.get_end_value() > rowData.at(hint.get_column_name()); DVLOG(2) << ret; if (!ret) { - return Qualified::INCOMPATIBLE; + return QualifiedStrategy::INCOMPATIBLE; } } - return Qualified::COMPATIBLE; + return QualifiedStrategy::COMPATIBLE; } void RangePath::buildKey() { std::string common; @@ -192,26 +183,10 @@ std::tuple RangePath::encodeRange( encodeBeginValue(hint.get_begin_value(), colTypeDef, startKey, offset); } if (UNLIKELY(needCheckNullable)) { - QFList_.emplace_back([colIndex, offset = index_nullable_offset_](const std::string& k) { - DVLOG(1) << "check null"; - std::bitset<16> nullableBit; - auto v = *reinterpret_cast(k.data() + offset); - nullableBit = v; - DVLOG(1) << nullableBit; - DVLOG(1) << nullableBit.test(15 - colIndex); - return nullableBit.test(15 - colIndex) ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; - }); - } - // The data of a geo may generate multiple indexes, so it needs to be de duplicated. Because there - // is no duplicate data between different parts, the suffixset does not clear and does not affect - // the results. + strategySet_.insert(QualifiedStrategy::checkNull(colIndex, index_nullable_offset_)); + } if (UNLIKELY(colTypeDef.get_type() == nebula::meta::cpp2::PropertyType::GEOGRAPHY)) { - QFList_.emplace_back([suffixSet = Set(), - suffixLength = suffixLength_](const std::string& k) mutable { - auto suffix = k.substr(k.size() - suffixLength, suffixLength); - auto [iter, result] = suffixSet.insert(suffix); - return result ? Qualified::COMPATIBLE : Qualified::INCOMPATIBLE; - }); + strategySet_.insert(QualifiedStrategy::dedupGeoIndex(suffixLength_)); } return {startKey, endKey}; } @@ -229,23 +204,14 @@ std::string RangePath::encodeBeginValue(const Value& value, val = encodeString(value, *colDef.get_type_length(), truncated); greater &= !truncated; if (UNLIKELY(truncated)) { - QFList_.emplace_back([val, startPos = offset](const std::string& k) { - int ret = memcmp(val.data(), k.data() + startPos, val.size()); - DVLOG(1) << folly::hexDump(val.data(), val.size()); - DVLOG(1) << folly::hexDump(k.data(), k.size()); - CHECK_LE(ret, 0); - return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; - }); + strategySet_.insert(QualifiedStrategy::compareTruncated(val, offset)); } } else if (value.type() == Value::Type::FLOAT) { bool isNaN = false; val = encodeFloat(value, isNaN); greater |= isNaN; // TODO(hs.zhang): Optimize the logic of judging NaN - QFList_.emplace_back([startPos = offset, length = val.size()](const std::string& k) { - int ret = memcmp("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", k.data() + startPos, length); - return ret == 0 ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; - }); + strategySet_.insert(QualifiedStrategy::checkNaN(offset)); } else { val = IndexKeyUtils::encodeValue(value); } @@ -269,25 +235,14 @@ std::string RangePath::encodeEndValue(const Value& value, val = encodeString(value, *colDef.get_type_length(), truncated); greater |= truncated; if (UNLIKELY(truncated)) { - QFList_.emplace_back([val, startPos = offset](const std::string& k) { - int ret = memcmp(val.data(), k.data() + startPos, val.size()); - DVLOG(3) << folly::hexDump(val.data(), val.size()); - DVLOG(3) << folly::hexDump(k.data(), k.size()); - CHECK_GE(ret, 0); - return ret == 0 ? Qualified::UNCERTAIN : Qualified::COMPATIBLE; - }); + strategySet_.insert(QualifiedStrategy::compareTruncated(val, offset)); } } else if (value.type() == Value::Type::FLOAT) { bool isNaN = false; val = encodeFloat(value, isNaN); greater |= isNaN; if (UNLIKELY(isNaN)) { - QFList_.emplace_back([startPos = offset, length = val.size()](const std::string& k) { - DVLOG(3) << "check NaN:" << startPos << "\t" << length; - DVLOG(3) << '\n' << folly::hexDump(k.data(), k.size()); - int ret = memcmp("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", k.data() + startPos, length); - return ret == 0 ? Qualified::INCOMPATIBLE : Qualified::COMPATIBLE; - }); + strategySet_.insert(QualifiedStrategy::checkNaN(offset)); } } else { val = IndexKeyUtils::encodeValue(value); @@ -330,13 +285,13 @@ PrefixPath::PrefixPath(nebula::meta::cpp2::IndexItem* index, : Path(index, schema, hints, vidLen) { buildKey(); } -Path::Qualified PrefixPath::qualified(const Map& rowData) { +QualifiedStrategy::Result PrefixPath::qualified(const Map& rowData) { for (auto& hint : hints_) { if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { - return Qualified::INCOMPATIBLE; + return QualifiedStrategy::INCOMPATIBLE; } } - return Qualified::COMPATIBLE; + return QualifiedStrategy::COMPATIBLE; } void PrefixPath::resetPart(PartitionID partId) { std::string p = IndexKeyUtils::indexPrefix(partId); @@ -357,13 +312,8 @@ void PrefixPath::buildKey() { } for (; fieldIter != index_->get_fields().end(); fieldIter++) { if (UNLIKELY(fieldIter->get_type().get_type() == nebula::meta::cpp2::PropertyType::GEOGRAPHY)) { - QFList_.emplace_back([suffixSet = Set(), - suffixLength = suffixLength_](const std::string& k) mutable { - auto suffix = k.substr(k.size() - suffixLength, suffixLength); - auto [iter, result] = suffixSet.insert(suffix); - DLOG(INFO) << "qualified geo dedup " << result; - return result ? Qualified::COMPATIBLE : Qualified::INCOMPATIBLE; - }); + strategySet_.insert(QualifiedStrategy::dedupGeoIndex(suffixLength_)); + break; } } prefix_ = std::move(common); @@ -440,10 +390,10 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { continue; } auto q = path_->qualified(iter_->key()); - if (q == Path::Qualified::INCOMPATIBLE) { + if (q == QualifiedStrategy::INCOMPATIBLE) { continue; } - bool compatible = q == Path::Qualified::COMPATIBLE; + bool compatible = q == QualifiedStrategy::COMPATIBLE; if (compatible && !needAccessBase_) { auto key = iter_->key().toString(); iter_->next(); @@ -464,8 +414,8 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { Map rowData = decodeFromBase(kv.first, kv.second); if (!compatible) { q = path_->qualified(rowData); - CHECK(q != Path::Qualified::UNCERTAIN); - if (q == Path::Qualified::INCOMPATIBLE) { + CHECK(q != QualifiedStrategy::UNCERTAIN); + if (q == QualifiedStrategy::INCOMPATIBLE) { continue; } } diff --git a/src/storage/exec/IndexScanNode.h b/src/storage/exec/IndexScanNode.h index b21e042805c..d0f1a2730fc 100644 --- a/src/storage/exec/IndexScanNode.h +++ b/src/storage/exec/IndexScanNode.h @@ -117,9 +117,16 @@ namespace storage { * `qualified(Map)` : qulified row by value * `resetPart` : reset current partitionID and reset `iter_` * `encodeValue` : encode a Value to bytes + * + * + * ------------------------------------------------------------- + * + * + * */ class Path; +class QualifiedStrategySet; class IndexScanNode : public IndexNode { FRIEND_TEST(IndexScanTest, Base); FRIEND_TEST(IndexScanTest, Vertex); @@ -166,10 +173,164 @@ class IndexScanNode : public IndexNode { bool needAccessBase_{false}; bool fatalOnBaseNotFound_{false}; }; +class QualifiedStrategy { + public: + enum Result { INCOMPATIBLE = 0, UNCERTAIN = 1, COMPATIBLE = 2 }; + /** + * checkNull + * + * There are two overload `checkNull` functions: + * 1. First one which is with template arg `targetIsNull`, checks `columnIndex` at `nullable` + * whether equal to `targetIsNull` or not. + * 2. The other one which is without template, filters key whose `columnIndex` at `nullable` is + * true + * + * Args: + * `columnIndex` : Index of column. **NOTE** , however, that the order in nullable bytes is + * reversed + * `keyOffset` : Reference `Index Key Encode` -> `index_nullable_offset_` + * + * Return: + * For convenience, we define a variable x.When the value at `columnIndex` is null, x is true, + * Otherwise x is false. + * 1.With template.Return COMPATIBLE if `x`==`targetIsNull`,else INCOMPATIBLE + * 2.Without template.Return COMPATIBLE if `x`==false, else INCOMPATIBLE + */ + template + static QualifiedStrategy checkNull(size_t columnIndex, size_t keyOffset) { + QualifiedStrategy q; + q.func_ = [columnIndex, keyOffset](const folly::StringPiece& key) { + std::bitset<16> nullableBit; + auto v = *reinterpret_cast(key.data() + keyOffset); + nullableBit = v; + DVLOG(3) << targetIsNull; + DVLOG(3) << nullableBit.test(15 - columnIndex); + return nullableBit.test(15 - columnIndex) == targetIsNull ? Result::COMPATIBLE + : Result::INCOMPATIBLE; + }; + return q; + } + static QualifiedStrategy checkNull(size_t columnIndex, size_t keyOffset) { + QualifiedStrategy q; + q.func_ = [columnIndex, keyOffset](const folly::StringPiece& key) { + std::bitset<16> nullableBit; + auto v = *reinterpret_cast(key.data() + keyOffset); + nullableBit = v; + DVLOG(3) << nullableBit.test(15 - columnIndex); + return nullableBit.test(15 - columnIndex) ? Result::INCOMPATIBLE : Result::COMPATIBLE; + }; + return q; + } + /** + * checkNaN + * + * Only for double. Check the value at `keyOffset` in indexKey is NaN or not. The logic here needs + * to be coordinated with the encoding logic of double numbers. + * + * Args: + * `keyOffset` : value offset at indexKey + * + * Return: + * Return INCOMPATIBLE if v==Nan else COMPATIBLE; + */ + static QualifiedStrategy checkNaN(size_t keyOffset) { + const char* chr = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"; // '\xFF' * 8 + QualifiedStrategy q; + q.func_ = [chr, keyOffset](const folly::StringPiece& key) { + int ret = memcmp(chr, key.data() + keyOffset, 8); + return ret == 0 ? Result::INCOMPATIBLE : Result::COMPATIBLE; + }; + return q; + } + /** + * dedupGeoIndex + * + * Because a `GEOGRAPHY` type data will generate multiple index keys pointing to the same base + * data,the base data pointed to by the indexkey should be de duplicated. + * + * Args: + * `dedupSuffixLength` : If indexed schema is a tag, `dedupSuffixLength` should be vid.len; + * If the indexed schema is an edge, `dedupSuffixLength` shoule be + * srcId.len+sizeof(rank)+dstId.len + * Return: + * When suffix first appears, the function returns `COMPATIBLE`; otherwise, the function returns + * `INCOMPATIBLE` + */ + static QualifiedStrategy dedupGeoIndex(size_t dedupSuffixLength) { + QualifiedStrategy q; + q.func_ = [suffixSet = Set(), + suffixLength = dedupSuffixLength](const folly::StringPiece& key) mutable -> Result { + std::string suffix = key.subpiece(key.size() - suffixLength, suffixLength).toString(); + auto [iter, result] = suffixSet.insert(std::move(suffix)); + return result ? Result::COMPATIBLE : Result::INCOMPATIBLE; + }; + return q; + } + /** + * constant + * + * Always return `result` + */ + template + static QualifiedStrategy constant() { + QualifiedStrategy q; + q.func_ = [](const folly::StringPiece&) { return result; }; + return q; + } + /** + * compareTruncated + * + * For a `String` type index, `val` may be truncated, and it is not enough to determine whether + * the indexkey complies with the constraint of columnhint only through the interval limit of + * [start,end) which is generated by `RangeIndex`. Therefore, it is necessary to make additional + * judgment on the truncated string type index + * For example: + * (ab)c meas that string is "abc" but index val has been truncated to "ab". (ab)c > ab is + * `UNCERTAIN`, and (ab)c > aa is COMPATIBLE. + * + * Args: + * `LEorGE` : It's an assit arg. true means LE and false means GE. + * `val` : Truncated `String` index value,whose length has been define in `IndexItem`. + * `keyStartPos` : The position in indexKey where start compare with `val` + * + * Return: + * Return `COMPATIBLE` if `val` is `LEorGE` than indexKey.Otherwise, return `UNCERTAIN`. + */ + template + static QualifiedStrategy compareTruncated(const std::string& val, size_t keyStartPos) { + QualifiedStrategy q; + q.func_ = [val, keyStartPos](const folly::StringPiece& key) { + int ret = memcmp(val.data(), key.data() + keyStartPos, val.size()); + DVLOG(1) << folly::hexDump(val.data(), val.size()); + DVLOG(1) << folly::hexDump(key.data(), key.size()); + if constexpr (LEorGE == true) { + CHECK_LE(ret, 0); + } else { + CHECK_GE(ret, 0); + } + return ret == 0 ? Result::UNCERTAIN : Result::COMPATIBLE; + }; + return q; + } + // call + inline Result operator()(const folly::StringPiece& key); + + private: + std::function func_; +}; +class QualifiedStrategySet { + public: + inline void insert(QualifiedStrategy&& strategy); + inline QualifiedStrategy::Result operator()(const folly::StringPiece& key); + + private: + std::vector strategyList_; +}; + class Path { public: - enum class Qualified : int16_t { INCOMPATIBLE = 0, UNCERTAIN = 1, COMPATIBLE = 2 }; - using QualifiedFunction = std::function; + // enum class Qualified : int16_t { INCOMPATIBLE = 0, UNCERTAIN = 1, COMPATIBLE = 2 }; + // using QualifiedFunction = std::function; using ColumnTypeDef = ::nebula::meta::cpp2::ColumnTypeDef; Path(nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, @@ -181,10 +342,10 @@ class Path { const meta::SchemaProviderIf* schema, const std::vector& hints, int64_t vidLen); - Qualified qualified(const folly::StringPiece& key); + QualifiedStrategy::Result qualified(const folly::StringPiece& key); virtual bool isRange() { return false; } - virtual Qualified qualified(const Map& rowData) = 0; + virtual QualifiedStrategy::Result qualified(const Map& rowData) = 0; virtual void resetPart(PartitionID partId) = 0; const std::string& toString(); @@ -193,7 +354,7 @@ class Path { const ColumnTypeDef& colDef, size_t index, std::string& key); - std::vector QFList_; + QualifiedStrategySet strategySet_; ::nebula::meta::cpp2::IndexItem* index_; const meta::SchemaProviderIf* schema_; const std::vector hints_; @@ -210,7 +371,7 @@ class PrefixPath : public Path { const std::vector& hints, int64_t vidLen); // Override - Qualified qualified(const Map& rowData) override; + QualifiedStrategy::Result qualified(const Map& rowData) override; void resetPart(PartitionID partId) override; const std::string& getPrefixKey() { return prefix_; } @@ -225,7 +386,7 @@ class RangePath : public Path { const meta::SchemaProviderIf* schema, const std::vector& hints, int64_t vidLen); - Qualified qualified(const Map& rowData) override; + QualifiedStrategy::Result qualified(const Map& rowData) override; void resetPart(PartitionID partId) override; inline bool includeStart() { return includeStart_; } @@ -256,8 +417,20 @@ class RangePath : public Path { std::string& key, size_t offset); }; - /* define inline functions */ +QualifiedStrategy::Result QualifiedStrategySet::operator()(const folly::StringPiece& key) { + QualifiedStrategy::Result ret = QualifiedStrategy::COMPATIBLE; + for (auto& s : strategyList_) { + ret = std::min(ret, s(key)); + } + return ret; +} +void QualifiedStrategySet::insert(QualifiedStrategy&& strategy) { + strategyList_.emplace_back(std::move(strategy)); +} +inline QualifiedStrategy::Result QualifiedStrategy::operator()(const folly::StringPiece& key) { + return func_(key); +} } // namespace storage From bb26c9e6c0a9b259d9a783885603a144a8c086cf Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Mon, 1 Nov 2021 13:22:12 +0800 Subject: [PATCH 30/38] address comments --- src/storage/exec/IndexDedupNode.cpp | 7 +-- src/storage/exec/IndexDedupNode.h | 3 +- src/storage/exec/IndexEdgeScanNode.cpp | 32 +++++++------- src/storage/exec/IndexProjectionNode.cpp | 1 + src/storage/exec/IndexScanNode.cpp | 55 ++++++++++++++++-------- src/storage/exec/IndexScanNode.h | 1 + src/storage/exec/IndexSelectionNode.h | 1 + src/storage/exec/IndexVertexScanNode.cpp | 16 +++---- src/storage/index/LookupProcessor.cpp | 12 +++++- src/storage/test/IndexTest.cpp | 44 +++++++++++-------- src/storage/test/IndexTestUtil.h | 12 +++--- 11 files changed, 107 insertions(+), 77 deletions(-) diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index de0aedac4ce..bc056abdaef 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -51,9 +51,9 @@ IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) { if (!hasNext) { break; } - auto d = dedup(::nebula::value(result)); - DVLOG(3) << d << "\t" << ::nebula::value(result); - if (d) { + auto dedupResult = dedup(::nebula::value(result)); + DVLOG(3) << dedupResult << "\t" << ::nebula::value(result); + if (dedupResult) { ret = ::nebula::value(std::move(result)); hasNext = true; break; @@ -68,6 +68,7 @@ IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) { return ret; } IndexDedupNode::RowWrapper::RowWrapper(const Row& row, const std::vector& posList) { + values_.reserve(posList.size()); for (auto p : posList) { values_.emplace_back(row[p]); } diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index e7b45d8995e..8a0715d6460 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -61,8 +61,7 @@ class IndexDedupNode : public IndexNode { }; struct Equal { bool operator()(const RowWrapper& a, const RowWrapper& b) const { - bool cmp = a.values() == b.values(); - return cmp; + return a.values() == b.values(); } }; std::vector dedupColumns_; diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index b600b661c77..bd9e5000e8d 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -47,41 +47,39 @@ ::nebula::cpp2::ErrorCode IndexEdgeScanNode::init(InitContext& ctx) { } Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { std::vector values(requiredColumns_.size()); - Map colPosMap; - for (size_t i = 0; i < requiredColumns_.size(); i++) { - colPosMap[requiredColumns_[i]] = i; - } - if (colPosMap.count(kSrc)) { + if (colPosMap_.count(kSrc)) { auto vId = IndexKeyUtils::getIndexSrcId(context_->vIdLen(), key); DVLOG(1) << folly::hexDump(vId.data(), vId.size()); if (context_->isIntId()) { - values[colPosMap[kSrc]] = Value(*reinterpret_cast(vId.data())); + values[colPosMap_[kSrc]] = Value(*reinterpret_cast(vId.data())); } else { DVLOG(2) << vId.subpiece(0, vId.find_first_of('\0')); DVLOG(2) << vId.subpiece(0, vId.find_first_of('\0')).toString(); DVLOG(2) << Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); - values[colPosMap[kSrc]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + values[colPosMap_[kSrc]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } } - DVLOG(1) << values[colPosMap[kSrc]]; - if (colPosMap.count(kDst)) { + DVLOG(1) << values[colPosMap_[kSrc]]; + if (colPosMap_.count(kDst)) { auto vId = IndexKeyUtils::getIndexDstId(context_->vIdLen(), key); if (context_->isIntId()) { - values[colPosMap[kDst]] = Value(*reinterpret_cast(vId.data())); + values[colPosMap_[kDst]] = Value(*reinterpret_cast(vId.data())); } else { - values[colPosMap[kDst]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + values[colPosMap_[kDst]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } } - DVLOG(1) << values[colPosMap[kDst]]; - if (colPosMap.count(kRank)) { + DVLOG(1) << values[colPosMap_[kDst]]; + if (colPosMap_.count(kRank)) { auto rank = IndexKeyUtils::getIndexRank(context_->vIdLen(), key); - values[colPosMap[kRank]] = Value(rank); + values[colPosMap_[kRank]] = Value(rank); } - if (colPosMap.count(kType)) { - values[colPosMap[kType]] = Value(context_->edgeType_); + if (colPosMap_.count(kType)) { + values[colPosMap_[kType]] = Value(context_->edgeType_); } + // Truncate the src/rank/dst at the end to facilitate obtaining the two bytes representing the + // nullableBit directly at the end when needed key.subtract(context_->vIdLen() * 2 + sizeof(EdgeType)); - decodePropFromIndex(key, colPosMap, values); + decodePropFromIndex(key, colPosMap_, values); return Row(std::move(values)); } nebula::cpp2::ErrorCode IndexEdgeScanNode::getBaseData(folly::StringPiece key, diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp index fd6536e919b..4de084f0b7e 100644 --- a/src/storage/exec/IndexProjectionNode.cpp +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -44,6 +44,7 @@ IndexNode::ErrorOr IndexProjectionNode::doNext(bool& hasNext) { } Row IndexProjectionNode::project(Row&& row) { Row ret; + ret.reserve(requiredColumns_.size()); for (auto& col : requiredColumns_) { ret.emplace_back(std::move(row[colPos_[col]])); } diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index e1eff160ef7..75cff6af444 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -59,7 +59,7 @@ std::string Path::encodeValue(const Value& value, std::string& key) { std::string val; bool isNull = false; - if (colDef.get_type() == ::nebula::meta::cpp2::PropertyType::GEOGRAPHY) { + if (colDef.get_type() == ::nebula::cpp2::PropertyType::GEOGRAPHY) { CHECK_EQ(value.type(), Value::Type::STRING); val = value.getStr(); } else if (value.type() == Value::Type::STRING) { @@ -74,6 +74,9 @@ std::string Path::encodeValue(const Value& value, } else { val = IndexKeyUtils::encodeValue(value); } + // If the current colDef can be null, then it is necessary to additionally determine whether the + // corresponding value under a nullable is null when parsing the key (the encoding of the maximum + // value, for example, the encoding of INT_MAX and null are the same, both are 8*' \xFF') if (!nullable_.empty() && nullable_[index] == true) { if (isNull) { strategySet_.insert(QualifiedStrategy::checkNull(index, index_nullable_offset_)); @@ -131,33 +134,42 @@ QualifiedStrategy::Result RangePath::qualified(const Map& ro return QualifiedStrategy::COMPATIBLE; } void RangePath::buildKey() { - std::string common; - common.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); + std::string commonIndexPrefix; + commonIndexPrefix.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); auto fieldIter = index_->get_fields().begin(); for (size_t i = 0; i < hints_.size() - 1; i++, fieldIter++) { auto& hint = hints_[i]; CHECK(fieldIter->get_name() == hint.get_column_name()); auto type = IndexKeyUtils::toValueType(fieldIter->get_type().get_type()); CHECK(type != Value::Type::STRING || fieldIter->get_type().type_length_ref().has_value()); - encodeValue(hint.get_begin_value(), fieldIter->get_type(), i, common); + encodeValue(hint.get_begin_value(), fieldIter->get_type(), i, commonIndexPrefix); serializeString_ += fmt::format("{}={}, ", hint.get_column_name(), hint.get_begin_value().toString()); } auto& hint = hints_.back(); size_t index = hints_.size() - 1; - auto [a, b] = encodeRange(hint, fieldIter->get_type(), index, common.size()); + // The first n-1 columnHint has been spelled out the common prefix, and then according to the nth + // columnHint to determine the RangePath Scan range [a, b). Note that [a, b) must be the range of + // include begin but exclude end. + // [startKey, endKey) = common prefix + [a, b) + auto [a, b] = encodeRange(hint, fieldIter->get_type(), index, commonIndexPrefix.size()); + // left will be `[a`,`(a`, or `[INF` std::string left = hint.begin_value_ref().is_set() ? fmt::format( "{}{}", hint.get_include_begin() ? '[' : '(', hint.get_begin_value().toString()) : "[-INF"; + // left will be `b]`,`b)`, or `[INF` std::string right = hint.end_value_ref().is_set() ? fmt::format("{}{}", hint.get_end_value().toString(), hint.get_include_end() ? ']' : ')') : "INF]"; serializeString_ += fmt::format("{}={},{}", hint.get_column_name(), left, right); - startKey_ = common + a; - endKey_ = common + b; + startKey_ = commonIndexPrefix + a; + endKey_ = commonIndexPrefix + b; + // If `end_value` is not set, `b` will be empty. So `endKey_` should append '\xFF' until + // endKey_.size() > `totalKeyLength_` to indicate positive infinity prefixed with + // `commonIndexPrefix` if (!hint.end_value_ref().is_set()) { endKey_.append(totalKeyLength_ - endKey_.size() + 1, '\xFF'); } @@ -185,7 +197,7 @@ std::tuple RangePath::encodeRange( if (UNLIKELY(needCheckNullable)) { strategySet_.insert(QualifiedStrategy::checkNull(colIndex, index_nullable_offset_)); } - if (UNLIKELY(colTypeDef.get_type() == nebula::meta::cpp2::PropertyType::GEOGRAPHY)) { + if (UNLIKELY(colTypeDef.get_type() == nebula::cpp2::PropertyType::GEOGRAPHY)) { strategySet_.insert(QualifiedStrategy::dedupGeoIndex(suffixLength_)); } return {startKey, endKey}; @@ -197,7 +209,7 @@ std::string RangePath::encodeBeginValue(const Value& value, std::string val; bool greater = !includeStart_; CHECK_NE(value.type(), Value::Type::NULLVALUE); - if (colDef.get_type() == ::nebula::meta::cpp2::PropertyType::GEOGRAPHY) { + if (colDef.get_type() == ::nebula::cpp2::PropertyType::GEOGRAPHY) { val = value.getStr(); } else if (value.type() == Value::Type::STRING) { bool truncated = false; @@ -228,7 +240,7 @@ std::string RangePath::encodeEndValue(const Value& value, CHECK_NE(value.type(), Value::Type::NULLVALUE); std::string val; bool greater = includeEnd_; - if (colDef.get_type() == ::nebula::meta::cpp2::PropertyType::GEOGRAPHY) { + if (colDef.get_type() == ::nebula::cpp2::PropertyType::GEOGRAPHY) { val = value.getStr(); } else if (value.type() == Value::Type::STRING) { bool truncated = false; @@ -311,7 +323,7 @@ void PrefixPath::buildKey() { fmt::format("{}={}, ", hint.get_column_name(), hint.get_begin_value().toString()); } for (; fieldIter != index_->get_fields().end(); fieldIter++) { - if (UNLIKELY(fieldIter->get_type().get_type() == nebula::meta::cpp2::PropertyType::GEOGRAPHY)) { + if (UNLIKELY(fieldIter->get_type().get_type() == nebula::cpp2::PropertyType::GEOGRAPHY)) { strategySet_.insert(QualifiedStrategy::dedupGeoIndex(suffixLength_)); break; } @@ -332,7 +344,8 @@ IndexScanNode::IndexScanNode(const IndexScanNode& node) requiredColumns_(node.requiredColumns_), requiredAndHintColumns_(node.requiredAndHintColumns_), ttlProps_(node.ttlProps_), - needAccessBase_(node.needAccessBase_) { + needAccessBase_(node.needAccessBase_), + colPosMap_(node.colPosMap_) { if (node.path_->isRange()) { path_ = std::make_unique(*dynamic_cast(node.path_.get())); } else { @@ -355,14 +368,15 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { for (size_t i = 0; i < ctx.returnColumns.size(); i++) { ctx.retColMap[ctx.returnColumns[i]] = i; } + colPosMap_ = ctx.retColMap; // Analyze whether the scan needs to access base data. // TODO(hs.zhang): The performance is better to judge based on whether the string is truncated auto tmp = ctx.requiredColumns; for (auto& field : index_->get_fields()) { - if (field.get_type().get_type() == PropertyType::FIXED_STRING) { + if (field.get_type().get_type() == ::nebula::cpp2::PropertyType::FIXED_STRING) { continue; } - if (field.get_type().get_type() == PropertyType::GEOGRAPHY) { + if (field.get_type().get_type() == ::nebula::cpp2::PropertyType::GEOGRAPHY) { continue; } tmp.erase(field.get_name()); @@ -403,13 +417,16 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { } std::pair kv; auto ret = getBaseData(iter_->key(), kv); - if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { // do nothing + } else if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { if (LIKELY(!fatalOnBaseNotFound_)) { LOG(WARNING) << "base data not found"; } else { LOG(FATAL) << "base data not found"; } continue; + } else { + return ret; } Map rowData = decodeFromBase(kv.first, kv.second); if (!compatible) { @@ -453,10 +470,7 @@ nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { << folly::hexDump(rangePath->getStartKey().data(), rangePath->getStartKey().size()); DVLOG(1) << '\n' << folly::hexDump(rangePath->getEndKey().data(), rangePath->getEndKey().size()); - std::unique_ptr<::nebula::kvstore::KVIterator> iter; - ret = - kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter); - iter_ = std::move(iter); + kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter_); } else { auto prefixPath = dynamic_cast(path_.get()); DVLOG(1) << '\n' @@ -469,6 +483,9 @@ nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { void IndexScanNode::decodePropFromIndex(folly::StringPiece key, const Map& colPosMap, std::vector& values) { + if (colPosMap.empty()) { + return; + } size_t offset = sizeof(PartitionID) + sizeof(IndexID); std::bitset<16> nullableBit; int8_t nullableColPosit = 15; diff --git a/src/storage/exec/IndexScanNode.h b/src/storage/exec/IndexScanNode.h index d0f1a2730fc..7da6d851568 100644 --- a/src/storage/exec/IndexScanNode.h +++ b/src/storage/exec/IndexScanNode.h @@ -172,6 +172,7 @@ class IndexScanNode : public IndexNode { std::pair> ttlProps_; bool needAccessBase_{false}; bool fatalOnBaseNotFound_{false}; + Map colPosMap_; }; class QualifiedStrategy { public: diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index f9d707e673c..1dbf28a83dc 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -60,6 +60,7 @@ class IndexSelectionNode : public IndexNode { } Expression *expr_; Map colPos_; + // TODO(hs.zhang): `ExprContext` could be moved out later if we unify the valcano in go/lookup class ExprContext : public ExpressionContext { public: explicit ExprContext(const Map &colPos) : colPos_(colPos) {} diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index c5c2d4e2065..94e7962d9c3 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -62,23 +62,19 @@ nebula::cpp2::ErrorCode IndexVertexScanNode::getBaseData(folly::StringPiece key, } Row IndexVertexScanNode::decodeFromIndex(folly::StringPiece key) { std::vector values(requiredColumns_.size()); - Map colPosMap; - for (size_t i = 0; i < requiredColumns_.size(); i++) { - colPosMap[requiredColumns_[i]] = i; - } - if (colPosMap.count(kVid)) { + if (colPosMap_.count(kVid)) { auto vId = IndexKeyUtils::getIndexVertexID(context_->vIdLen(), key); if (context_->isIntId()) { - values[colPosMap[kVid]] = Value(*reinterpret_cast(vId.data())); + values[colPosMap_[kVid]] = Value(*reinterpret_cast(vId.data())); } else { - values[colPosMap[kVid]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); + values[colPosMap_[kVid]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } } - if (colPosMap.count(kTag)) { - values[colPosMap[kTag]] = Value(context_->tagId_); + if (colPosMap_.count(kTag)) { + values[colPosMap_[kTag]] = Value(context_->tagId_); } key.subtract(context_->vIdLen()); - decodePropFromIndex(key, colPosMap, values); + decodePropFromIndex(key, colPosMap_, values); return Row(std::move(values)); } Map IndexVertexScanNode::decodeFromBase(const std::string& key, diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index 9e4c095fdd4..fc3baadc0db 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -80,11 +80,19 @@ ::nebula::cpp2::ErrorCode LookupProcessor::prepare(const cpp2::LookupIndexReques std::string schemaName; if (planContext_->isEdge_) { auto edgeType = req.get_indices().get_schema_id().get_edge_type(); - schemaName = env_->schemaMan_->toEdgeName(req.get_space_id(), edgeType).value(); + auto schemaNameValue = env_->schemaMan_->toEdgeName(req.get_space_id(), edgeType); + if (!schemaNameValue.ok()) { + return ::nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND; + } + schemaName = schemaNameValue.value(); context_->edgeType_ = edgeType; } else { auto tagId = req.get_indices().get_schema_id().get_tag_id(); - schemaName = env_->schemaMan_->toTagName(req.get_space_id(), tagId).value(); + auto schemaNameValue = env_->schemaMan_->toTagName(req.get_space_id(), tagId); + if (!schemaNameValue.ok()) { + return ::nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND; + } + schemaName = schemaNameValue.value(); context_->tagId_ = tagId; } std::vector colNames; diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index be8483396c8..1b3e77f82f0 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -46,26 +46,34 @@ using std::string_literals::operator""s; * b. range with begin is include/exclude/-INF * c. range with end id include/exclude/+INF * 5. nullable + * 6. multiPart * Case: - * ┌──────────┬─────────────────────────────────────────┐ - * │ section1 │ name │ case │ description │ NOTICE │ - * Base | Base | - * | Vertex| IndexOnly - * | | BackToTable - * | Edge | indexOnly - * | BackToTable - * Value Type | Int | PrefixWithTruncate - * | Float | PrefixWithoutTruncate - * | Bool | INCLUDE_INF - * | String | INCLUDE_EXCLUDE - * | Time | EXCLUDE_INF - * | Date | EXCLUDE_INCLUDE - * | DateTime| INF_INCLUDE - * | Compound| INF_EXCLUDE - * | Nullable| - * - * ┌┬┐├┼┤└┴┘┌─┐│┼│└─┘ + * ┌────────────┬───────────┬───────────────┬─────────────────────────┬─────────┐ + * │ section1 │ name │ case │ description │ NOTICE │ + * ├────────────┼───────────┼───────────────┼─────────────────────────┼─────────┤ + * | Base | Base | | | | + * | ├───────────┼───────────────┼─────────────────────────┼─────────┤ + * | | Vertex | IndexOnly | | | + * | | Edge | BackToTable | | | + * | ├───────────┼───────────────┼─────────────────────────┼─────────┤ + * | | MultiPart | | | | + * ├────────────┼───────────┼───────────────┼─────────────────────────┼─────────┤ + * | Value Type | Int | Truncate | Test different interval | | + * | | Float | NoTruncate | with each type of Value | | + * | | Bool | INCLUDE_BEGIN | | | + * | | String | INCLUDE_END | | | + * | | Time | EXCLUDE_BEGIN | | | + * | | Date | EXCLUDE_END | | | + * | | DateTime | POSITIVE_INF | | | + * | | Compound | NEGATIVE_INF | | | + * | | Nullable | | | | + * | | Geography | | | | + * └────────────┴───────────┴───────────────┴─────────────────────────┴─────────┘ * + * ┌─┬┐ + * │ ││ + * ├─┼┤ + * └─┴┘ */ struct IndexScanTestHelper { void setIndex(IndexVertexScanNode* node, std::shared_ptr<::nebula::meta::cpp2::IndexItem> index) { diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index ff1d80bf4db..d4e987452a5 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -497,11 +497,11 @@ class SchemaParser { private: std::stringstream ss; std::shared_ptr<::nebula::meta::NebulaSchemaProvider> schema; - std::map typeMap{ - {"int", ::nebula::meta::cpp2::PropertyType::INT64}, - {"double", ::nebula::meta::cpp2::PropertyType::DOUBLE}, - {"string", ::nebula::meta::cpp2::PropertyType::STRING}, - {"bool", ::nebula::meta::cpp2::PropertyType::BOOL}}; + std::map typeMap{ + {"int", ::nebula::cpp2::PropertyType::INT64}, + {"double", ::nebula::cpp2::PropertyType::DOUBLE}, + {"string", ::nebula::cpp2::PropertyType::STRING}, + {"bool", ::nebula::cpp2::PropertyType::BOOL}}; }; /** @@ -583,7 +583,7 @@ class IndexParser { ::nebula::meta::cpp2::ColumnTypeDef type; if (length > 0) { type.set_type_length(length); - type.set_type(::nebula::meta::cpp2::PropertyType::FIXED_STRING); + type.set_type(::nebula::cpp2::PropertyType::FIXED_STRING); } else { type.set_type(field->type()); } From 4cc75067501f6f1775297d4bf6466d34ad2ae2d1 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Mon, 1 Nov 2021 16:15:48 +0800 Subject: [PATCH 31/38] address some comments; Modify IndexNode::next return Type --- src/storage/exec/IndexDedupNode.cpp | 71 +++++---- src/storage/exec/IndexDedupNode.h | 4 +- src/storage/exec/IndexLimitNode.cpp | 11 +- src/storage/exec/IndexLimitNode.h | 2 +- src/storage/exec/IndexNode.h | 40 +++-- src/storage/exec/IndexProjectionNode.cpp | 11 +- src/storage/exec/IndexProjectionNode.h | 2 +- src/storage/exec/IndexScanNode.cpp | 16 +- src/storage/exec/IndexScanNode.h | 2 +- src/storage/exec/IndexSelectionNode.cpp | 12 +- src/storage/exec/IndexSelectionNode.h | 2 +- src/storage/index/LookupProcessor.cpp | 22 ++- src/storage/test/IndexTest.cpp | 193 +++++++++++------------ src/storage/test/IndexTestUtil.h | 4 +- 14 files changed, 203 insertions(+), 189 deletions(-) diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index bc056abdaef..d2d3f8f8fea 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -15,18 +15,19 @@ ::nebula::cpp2::ErrorCode IndexDedupNode::init(InitContext& ctx) { for (auto& col : dedupColumns_) { ctx.requiredColumns.insert(col); } - // The return Row format of each child node must be the same - InitContext childCtx = ctx; - InitContext ctx2; - for (auto& child : children_) { - auto ret = child->init(childCtx); + // The contents of `ctx` should be the same when all child nodes are initialized, and `ctx` should + // be the same after initialization. + for (size_t i = 0; i < children_.size() - 1; i++) { + auto tmp = ctx; // + auto ret = children_[i]->init(tmp); if (ret != ::nebula::cpp2::ErrorCode::SUCCEEDED) { return ret; } - ctx2 = childCtx; - childCtx = ctx; } - ctx = ctx2; + auto ret = children_.back()->init(ctx); + if (ret != ::nebula::cpp2::ErrorCode::SUCCEEDED) { + return ret; + } for (auto& col : dedupColumns_) { dedupPos_.push_back(ctx.retColMap[col]); } @@ -37,35 +38,39 @@ ::nebula::cpp2::ErrorCode IndexDedupNode::doExecute(PartitionID partId) { dedupSet_.clear(); return IndexNode::doExecute(partId); } -IndexNode::ErrorOr IndexDedupNode::doNext(bool& hasNext) { - Row ret; - hasNext = false; +IndexNode::Result IndexDedupNode::iterateCurrentChild(size_t currentChild) { + auto& child = *children_[currentChild]; + Result result; + do { + result = child.next(); + // error or meet end + if (!result.hasData()) { + return result; + } + auto dedupResult = dedup(result.row()); + if (!dedupResult) { + continue; + } + return result; + } while (true); +} + +IndexNode::Result IndexDedupNode::doNext() { + Result result; while (currentChild_ < children_.size()) { - auto& child = *children_[currentChild_]; - DVLOG(3) << currentChild_; - do { - auto result = child.next(hasNext); - if (!nebula::ok(result)) { - return result; - } - if (!hasNext) { - break; - } - auto dedupResult = dedup(::nebula::value(result)); - DVLOG(3) << dedupResult << "\t" << ::nebula::value(result); - if (dedupResult) { - ret = ::nebula::value(std::move(result)); - hasNext = true; - break; - } - } while (true); - if (!hasNext) { + result = iterateCurrentChild(currentChild_); + // error + if (!result.success()) { + return result; + } + // finish iterate one child + if (!result.hasData()) { currentChild_++; - } else { - break; + continue; } + return result; } - return ret; + return Result(); } IndexDedupNode::RowWrapper::RowWrapper(const Row& row, const std::vector& posList) { values_.reserve(posList.size()); diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index 8a0715d6460..aa5a167904a 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -42,8 +42,8 @@ class IndexDedupNode : public IndexNode { private: inline bool dedup(const Row& row); ::nebula::cpp2::ErrorCode doExecute(PartitionID partId) override; - ErrorOr doNext(bool& hasNext) override; - + Result doNext() override; + Result iterateCurrentChild(size_t currentChild); // Define RowWrapper for dedup class RowWrapper { public: diff --git a/src/storage/exec/IndexLimitNode.cpp b/src/storage/exec/IndexLimitNode.cpp index afe5e661d19..aa82d97b843 100644 --- a/src/storage/exec/IndexLimitNode.cpp +++ b/src/storage/exec/IndexLimitNode.cpp @@ -17,22 +17,21 @@ nebula::cpp2::ErrorCode IndexLimitNode::doExecute(PartitionID partId) { currentOffset_ = 0; return children_[0]->execute(partId); } -IndexNode::ErrorOr IndexLimitNode::doNext(bool& hasNext) { +IndexNode::Result IndexLimitNode::doNext() { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; while (UNLIKELY(currentOffset_ < offset_)) { - auto result = child.next(hasNext); - if (!::nebula::ok(result) || !hasNext) { + auto result = child.next(); + if (!result.hasData()) { return result; } currentOffset_++; } if (currentOffset_ < offset_ + limit_) { currentOffset_++; - return child.next(hasNext); + return child.next(); } else { - hasNext = false; - return ErrorOr(Row()); + return Result(); } } std::unique_ptr IndexLimitNode::copy() { diff --git a/src/storage/exec/IndexLimitNode.h b/src/storage/exec/IndexLimitNode.h index 64b0087e43f..2d72c2b8e98 100644 --- a/src/storage/exec/IndexLimitNode.h +++ b/src/storage/exec/IndexLimitNode.h @@ -18,7 +18,7 @@ class IndexLimitNode : public IndexNode { private: nebula::cpp2::ErrorCode doExecute(PartitionID partId) override; - ErrorOr doNext(bool& hasNext) override; + Result doNext() override; const uint64_t offset_, limit_; uint64_t currentOffset_ = 0; }; diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index 38410c40684..c7682f76cf0 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -74,10 +74,32 @@ struct InitContext { class IndexNode { public: - /* typedef */ - template - using ErrorOr = ::nebula::ErrorOr; + /* Iterate result*/ + class Result { + public: + Result() : code_(ErrorCode::SUCCEEDED), empty_(true) {} + Result(const Result& result) : code_(result.code_), row_(result.row_), empty_(result.empty_) {} + Result(Result&& result) + : code_(result.code_), row_(std::move(result.row_)), empty_(result.empty_) {} + explicit Result(ErrorCode code) : code_(code), empty_(true) {} + explicit Result(Row&& row) : row_(row), empty_(false) {} + Result& operator=(Result&& result) { + this->code_ = result.code_; + this->row_ = std::move(result.row_); + this->empty_ = result.empty_; + return *this; + } + inline bool success() { return code_ == ErrorCode::SUCCEEDED; } + inline bool hasData() { return success() && empty_ == false; } + inline Row row() && { return std::move(row_); } + inline Row& row() & { return row_; } + ErrorCode code() { return code_; } + private: + ErrorCode code_{ErrorCode::SUCCEEDED}; + Row row_; + bool empty_{true}; + }; /* build */ IndexNode(const IndexNode& node); explicit IndexNode(RuntimeContext* context, const std::string& name); @@ -91,7 +113,7 @@ class IndexNode { } /* execution */ inline nebula::cpp2::ErrorCode execute(PartitionID partId); - inline ErrorOr next(bool& hasNext); + inline Result next(); // inline nebula::cpp2::ErrorCode finish(); /* assist */ @@ -101,7 +123,7 @@ class IndexNode { inline const time::Duration& duration(); protected: - virtual ErrorOr doNext(bool& hasNext) = 0; + virtual Result doNext() = 0; void beforeNext(); void afterNext(); virtual nebula::cpp2::ErrorCode doExecute(PartitionID partId); @@ -117,12 +139,12 @@ class IndexNode { }; /* Defination of inline function */ -inline IndexNode::ErrorOr IndexNode::next(bool& hasNext) { +inline IndexNode::Result IndexNode::next() { beforeNext(); if (context_->isPlanKilled()) { - return nebula::cpp2::ErrorCode::E_PLAN_IS_KILLED; + return Result(::nebula::cpp2::ErrorCode::E_PLAN_IS_KILLED); } - auto ret = doNext(hasNext); + Result ret = doNext(); afterNext(); return ret; } @@ -140,7 +162,7 @@ inline nebula::cpp2::ErrorCode IndexNode::execute(PartitionID partId) { beforeExecute(); auto ret = doExecute(partId); afterExecute(); - return std::move(ret); + return ret; } inline void IndexNode::beforeExecute() { if (UNLIKELY(profileDetail_)) { diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp index 4de084f0b7e..af18522157b 100644 --- a/src/storage/exec/IndexProjectionNode.cpp +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -32,15 +32,14 @@ nebula::cpp2::ErrorCode IndexProjectionNode::init(InitContext& ctx) { } return ::nebula::cpp2::ErrorCode::SUCCEEDED; } -IndexNode::ErrorOr IndexProjectionNode::doNext(bool& hasNext) { +IndexNode::Result IndexProjectionNode::doNext() { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; - auto result = child.next(hasNext); - if (::nebula::ok(result) && hasNext) { - return ErrorOr(project(::nebula::value(std::move(result)))); - } else { - return ErrorOr(Row()); + Result result = child.next(); + if (result.hasData()) { + result = Result(project(std::move(result).row())); } + return result; } Row IndexProjectionNode::project(Row&& row) { Row ret; diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index 7f52ba2d907..c5af9c41e97 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -38,7 +38,7 @@ class IndexProjectionNode : public IndexNode { std::string identify() override; private: - ErrorOr doNext(bool& hasNext) override; + Result doNext() override; Row project(Row&& row); std::vector requiredColumns_; Map colPos_; diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index 75cff6af444..77f7a65a1d5 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -396,8 +396,7 @@ nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { auto ret = resetIter(partId); return ret; } -IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { - hasNext = true; +IndexNode::Result IndexScanNode::doNext() { for (; iter_ && iter_->valid(); iter_->next()) { DVLOG(3) << '\n' << folly::hexDump(iter_->key().data(), iter_->key().size()); if (!checkTTL()) { @@ -411,9 +410,9 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { if (compatible && !needAccessBase_) { auto key = iter_->key().toString(); iter_->next(); - auto ret = decodeFromIndex(key); - DVLOG(3) << ret; - return ret; + Row row = decodeFromIndex(key); + DVLOG(3) << row; + return Result(std::move(row)); } std::pair kv; auto ret = getBaseData(iter_->key(), kv); @@ -426,7 +425,7 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { } continue; } else { - return ret; + return Result(ret); } Map rowData = decodeFromBase(kv.first, kv.second); if (!compatible) { @@ -442,10 +441,9 @@ IndexNode::ErrorOr IndexScanNode::doNext(bool& hasNext) { } iter_->next(); DVLOG(3) << row; - return row; + return Result(std::move(row)); } - hasNext = false; - return ErrorOr(Row()); + return Result(); } bool IndexScanNode::checkTTL() { if (iter_->val().empty() || ttlProps_.first == false) { diff --git a/src/storage/exec/IndexScanNode.h b/src/storage/exec/IndexScanNode.h index 7da6d851568..3dea609b99f 100644 --- a/src/storage/exec/IndexScanNode.h +++ b/src/storage/exec/IndexScanNode.h @@ -147,7 +147,7 @@ class IndexScanNode : public IndexNode { protected: nebula::cpp2::ErrorCode doExecute(PartitionID partId) final; - ErrorOr doNext(bool& hasNext) final; + Result doNext() final; void decodePropFromIndex(folly::StringPiece key, const Map& colPosMap, std::vector& values); diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index eac6841c311..232054b3abd 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -30,20 +30,20 @@ nebula::cpp2::ErrorCode IndexSelectionNode::init(InitContext& ctx) { ctx_ = std::make_unique(colPos_); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } -IndexNode::ErrorOr IndexSelectionNode::doNext(bool& hasNext) { +IndexNode::Result IndexSelectionNode::doNext() { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; do { - auto result = child.next(hasNext); - if (!hasNext || !nebula::ok(result)) { + auto result = child.next(); + if (!result.hasData()) { return result; } - DVLOG(1) << nebula::value(result); - if (filter(nebula::value(result))) { + DVLOG(1) << result.row(); + if (filter(result.row())) { return result; } } while (true); - return ErrorOr(Row()); + return Result(); } std::unique_ptr IndexSelectionNode::copy() { return std::make_unique(*this); diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 1dbf28a83dc..9250a119d03 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -52,7 +52,7 @@ class IndexSelectionNode : public IndexNode { std::string identify() override; private: - ErrorOr doNext(bool &hasNext) override; + Result doNext() override; inline bool filter(const Row &row) { ctx_->setRow(row); auto &result = expr_->eval(*ctx_); diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index fc3baadc0db..2a55d79fa23 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -167,17 +167,16 @@ void LookupProcessor::runInSingleThread(const std::vector& parts, for (auto part : parts) { DLOG(INFO) << "execute part:" << part; plan->execute(part); - bool hasNext = true; ::nebula::cpp2::ErrorCode code = ::nebula::cpp2::ErrorCode::SUCCEEDED; decltype(datasetList)::value_type dataset; do { - auto result = plan->next(hasNext); - if (!::nebula::ok(result)) { - code = ::nebula::error(result); + auto result = plan->next(); + if (!result.success()) { + code = result.code(); break; } - if (hasNext) { - dataset.emplace_back(::nebula::value(std::move(result))); + if (result.hasData()) { + dataset.emplace_back(std::move(result).row()); } else { break; } @@ -214,15 +213,14 @@ void LookupProcessor::runInMultipleThread(const std::vector& parts, ::nebula::cpp2::ErrorCode code = ::nebula::cpp2::ErrorCode::SUCCEEDED; std::deque dataset; plan->execute(part); - bool hasNext = true; do { - auto result = plan->next(hasNext); - if (!::nebula::ok(result)) { - code = ::nebula::error(result); + auto result = plan->next(); + if (!result.success()) { + code = result.code(); break; } - if (hasNext) { - dataset.emplace_back(::nebula::value(std::move(result))); + if (result.hasData()) { + dataset.emplace_back(std::move(result).row()); } else { break; } diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 1b3e77f82f0..a7ea775d38b 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -253,15 +253,14 @@ TEST_F(IndexScanTest, Base) { initCtx.requiredColumns = {kVid, "a"}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } auto expect = R"( string | int @@ -292,15 +291,15 @@ TEST_F(IndexScanTest, Base) { initCtx.requiredColumns = {kVid, "b"}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } auto expect = R"( string | int @@ -351,15 +350,15 @@ TEST_F(IndexScanTest, Vertex) { initCtx.requiredColumns = {kVid, "a"}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } auto expect = R"( string | int @@ -389,15 +388,15 @@ TEST_F(IndexScanTest, Vertex) { initCtx.requiredColumns = {kVid, "b"}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } auto expect = R"( string | int @@ -451,15 +450,15 @@ TEST_F(IndexScanTest, Edge) { initCtx.requiredColumns = {kSrc, kRank, kDst, "c"}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } auto expect = R"( string | int | string | int @@ -489,15 +488,15 @@ TEST_F(IndexScanTest, Edge) { initCtx.requiredColumns = {kSrc, kRank, kDst, "a"}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } auto expect = R"( string | int | string | int @@ -558,15 +557,15 @@ TEST_F(IndexScanTest, Int) { initCtx.requiredColumns = {kVid}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } EXPECT_EQ(result, expect) << "Fail at case " << case_; }; @@ -772,15 +771,15 @@ float | float | float | int initCtx.requiredColumns = {kSrc}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } EXPECT_EQ(result, expect) << "Fail at case " << case_; DVLOG(1) << "End case " << case_; @@ -995,15 +994,15 @@ TEST_F(IndexScanTest, Bool) { initCtx.requiredColumns = {kVid}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } EXPECT_EQ(result, expect) << "Fail at case " << case_; DVLOG(1) << "End case " << case_; @@ -1086,15 +1085,15 @@ TEST_F(IndexScanTest, String1) { initCtx.requiredColumns.insert(acquiredColumns.begin(), acquiredColumns.end()); scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } std::vector result2(result.size()); for (size_t j = 0; j < acquiredColumns.size(); j++) { @@ -1243,15 +1242,15 @@ TEST_F(IndexScanTest, String2) { initCtx.requiredColumns.insert(acquiredColumns.begin(), acquiredColumns.end()); scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } std::vector result2(result.size()); for (size_t j = 0; j < acquiredColumns.size(); j++) { @@ -1374,15 +1373,15 @@ TEST_F(IndexScanTest, String3) { initCtx.requiredColumns.insert(acquiredColumns.begin(), acquiredColumns.end()); scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } std::vector result2(result.size()); for (size_t j = 0; j < acquiredColumns.size(); j++) { @@ -1496,15 +1495,15 @@ TEST_F(IndexScanTest, String4) { initCtx.requiredColumns.insert(acquiredColumns.begin(), acquiredColumns.end()); scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } std::vector result2(result.size()); for (size_t j = 0; j < acquiredColumns.size(); j++) { @@ -1612,15 +1611,15 @@ TEST_F(IndexScanTest, Nullable) { initCtx.requiredColumns = {kVid}; scanNode->init(initCtx); scanNode->execute(0); - bool hasNext = false; + std::vector result; while (true) { - auto res = scanNode->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } EXPECT_EQ(result, expect) << "Fail at case " << case_; }; @@ -1757,15 +1756,14 @@ class IndexTest : public ::testing::Test { static std::vector collectResult(IndexNode* node) { std::vector result; InitContext initCtx; - bool hasNext = false; node->init(initCtx); while (true) { - auto res = node->next(hasNext); - ASSERT(::nebula::ok(res)); - if (!hasNext) { + auto res = node->next(); + ASSERT(res.success()); + if (!res.hasData()) { break; } - result.emplace_back(::nebula::value(std::move(res))); + result.emplace_back(std::move(res).row()); } return result; } @@ -1796,13 +1794,12 @@ TEST_F(IndexTest, Selection) { auto selection = std::make_unique(ctx.get(), expr); auto mockChild = std::make_unique(ctx.get()); mockChild->executeFunc = [&rows](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; - mockChild->nextFunc = [&rows, ¤tOffset](bool& hasNext) -> IndexNode::ErrorOr { + mockChild->nextFunc = [&rows, ¤tOffset]() -> IndexNode::Result { if (currentOffset < rows.size()) { - hasNext = true; - return rows[currentOffset++]; + auto row = rows[currentOffset++]; + return IndexNode::Result(std::move(row)); } else { - hasNext = false; - return {}; + return IndexNode::Result(); } }; mockChild->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { @@ -1827,13 +1824,12 @@ TEST_F(IndexTest, Projection) { std::make_unique(ctx.get(), std::vector{"c", "a", "b"}); auto mockChild = std::make_unique(ctx.get()); mockChild->executeFunc = [&rows](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; - mockChild->nextFunc = [&rows, ¤tOffset](bool& hasNext) -> IndexNode::ErrorOr { + mockChild->nextFunc = [&rows, ¤tOffset]() -> IndexNode::Result { if (currentOffset < rows.size()) { - hasNext = true; - return rows[currentOffset++]; + auto row = rows[currentOffset++]; + return IndexNode::Result(std::move(row)); } else { - hasNext = false; - return {}; + return IndexNode::Result(); } }; mockChild->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { @@ -1871,13 +1867,12 @@ TEST_F(IndexTest, Limit) { auto limit = std::make_unique(ctx.get(), 10); auto mockChild = std::make_unique(ctx.get()); mockChild->executeFunc = [&rows](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; - mockChild->nextFunc = [&rows, ¤tOffset](bool& hasNext) -> IndexNode::ErrorOr { + mockChild->nextFunc = [&rows, ¤tOffset]() -> IndexNode::Result { if (currentOffset < rows.size()) { - hasNext = true; - return rows[currentOffset++]; + auto row = rows[currentOffset++]; + return IndexNode::Result(std::move(row)); } else { - hasNext = false; - return {}; + return IndexNode::Result(); } }; mockChild->initFunc = [](InitContext&) -> ::nebula::cpp2::ErrorCode { @@ -1905,13 +1900,12 @@ TEST_F(IndexTest, Dedup) { auto dedup = std::make_unique(ctx.get(), std::vector{"a"}); auto child1 = std::make_unique(ctx.get()); child1->executeFunc = [&rows1](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; - child1->nextFunc = [&rows1, &offset1](bool& hasNext) -> IndexNode::ErrorOr { + child1->nextFunc = [&rows1, &offset1]() -> IndexNode::Result { if (offset1 < rows1.size()) { - hasNext = true; - return rows1[offset1++]; + auto row = rows1[offset1++]; + return IndexNode::Result(std::move(row)); } else { - hasNext = false; - return {}; + return IndexNode::Result(); } }; child1->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { @@ -1922,13 +1916,12 @@ TEST_F(IndexTest, Dedup) { }; auto child2 = std::make_unique(ctx.get()); child2->executeFunc = [&rows2](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; - child2->nextFunc = [&rows2, &offset2](bool& hasNext) -> IndexNode::ErrorOr { + child2->nextFunc = [&rows2, &offset2]() -> IndexNode::Result { if (offset2 < rows2.size()) { - hasNext = true; - return rows2[offset2++]; + auto row = rows2[offset2++]; + return IndexNode::Result(std::move(row)); } else { - hasNext = false; - return {}; + return IndexNode::Result(); } }; child2->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index d4e987452a5..efa798db52e 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -380,13 +380,13 @@ class MockIndexNode : public IndexNode { explicit MockIndexNode(RuntimeContext* context) : IndexNode(context, "MockIndexNode") {} ::nebula::cpp2::ErrorCode init(InitContext& initCtx) { return initFunc(initCtx); } std::unique_ptr copy() { LOG(FATAL) << "Unexpect"; } - std::function(bool&)> nextFunc; + std::function nextFunc; std::function<::nebula::cpp2::ErrorCode(PartitionID)> executeFunc; std::function<::nebula::cpp2::ErrorCode(InitContext& initCtx)> initFunc; std::string identify() override { return "MockIndexNode"; } private: - ErrorOr doNext(bool& hasNext) override { return nextFunc(hasNext); } + Result doNext() override { return nextFunc(); } ::nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { return executeFunc(partId); }; int offset_; int total_; From e8edf14de2113ba590668f8d27271e4a96cbd62c Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Tue, 2 Nov 2021 12:00:03 +0800 Subject: [PATCH 32/38] fix bug --- src/storage/exec/IndexEdgeScanNode.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index bd9e5000e8d..aef4a44d520 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -58,8 +58,8 @@ Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { DVLOG(2) << Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); values[colPosMap_[kSrc]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } + DVLOG(1) << values[colPosMap_[kSrc]]; } - DVLOG(1) << values[colPosMap_[kSrc]]; if (colPosMap_.count(kDst)) { auto vId = IndexKeyUtils::getIndexDstId(context_->vIdLen(), key); if (context_->isIntId()) { @@ -67,8 +67,8 @@ Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { } else { values[colPosMap_[kDst]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } + DVLOG(1) << values[colPosMap_[kDst]]; } - DVLOG(1) << values[colPosMap_[kDst]]; if (colPosMap_.count(kRank)) { auto rank = IndexKeyUtils::getIndexRank(context_->vIdLen(), key); values[colPosMap_[kRank]] = Value(rank); @@ -78,7 +78,7 @@ Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { } // Truncate the src/rank/dst at the end to facilitate obtaining the two bytes representing the // nullableBit directly at the end when needed - key.subtract(context_->vIdLen() * 2 + sizeof(EdgeType)); + key.subtract(context_->vIdLen() * 2 + sizeof(EdgeRanking)); decodePropFromIndex(key, colPosMap_, values); return Row(std::move(values)); } From ab0665574e7330ace81082a1241fff9b02d29fa1 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Tue, 2 Nov 2021 12:06:53 +0800 Subject: [PATCH 33/38] add some comments --- src/interface/storage.thrift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/interface/storage.thrift b/src/interface/storage.thrift index 6185797fe1d..404d7911f6b 100644 --- a/src/interface/storage.thrift +++ b/src/interface/storage.thrift @@ -507,6 +507,8 @@ struct IndexColumnHint { 2: ScanType scan_type, 3: common.Value begin_value, 4: common.Value end_value, + // When `columnhint` means ` >= begin_value`, `include_begin` is true + // and include_end is similar 5: bool include_begin = true, 6: bool include_end = false, } From 377655d48593e7ea1ef9ad3cab723837146a2bcd Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Wed, 3 Nov 2021 16:12:48 +0800 Subject: [PATCH 34/38] Add blank lines between function definitions --- src/storage/exec/IndexDedupNode.cpp | 5 +++++ src/storage/exec/IndexDedupNode.h | 1 + src/storage/exec/IndexEdgeScanNode.cpp | 5 +++++ src/storage/exec/IndexLimitNode.cpp | 3 +++ src/storage/exec/IndexNode.cpp | 1 + src/storage/exec/IndexNode.h | 7 +++++++ src/storage/exec/IndexProjectionNode.cpp | 4 ++++ src/storage/exec/IndexScanNode.cpp | 22 ++++++++++++++++++++++ src/storage/exec/IndexSelectionNode.cpp | 5 +++++ src/storage/exec/IndexVertexScanNode.cpp | 5 +++++ 10 files changed, 58 insertions(+) diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index d2d3f8f8fea..6421a2e5649 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -33,11 +33,13 @@ ::nebula::cpp2::ErrorCode IndexDedupNode::init(InitContext& ctx) { } return ::nebula::cpp2::ErrorCode::SUCCEEDED; } + ::nebula::cpp2::ErrorCode IndexDedupNode::doExecute(PartitionID partId) { currentChild_ = 0; dedupSet_.clear(); return IndexNode::doExecute(partId); } + IndexNode::Result IndexDedupNode::iterateCurrentChild(size_t currentChild) { auto& child = *children_[currentChild]; Result result; @@ -72,15 +74,18 @@ IndexNode::Result IndexDedupNode::doNext() { } return Result(); } + IndexDedupNode::RowWrapper::RowWrapper(const Row& row, const std::vector& posList) { values_.reserve(posList.size()); for (auto p : posList) { values_.emplace_back(row[p]); } } + std::unique_ptr IndexDedupNode::copy() { return std::make_unique(*this); } + std::string IndexDedupNode::identify() { return fmt::format("{}(dedup=[{}])", name_, folly::join(',', dedupColumns_)); } diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index aa5a167904a..c6eaca75c99 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -75,5 +75,6 @@ inline bool IndexDedupNode::dedup(const Row& row) { auto result = dedupSet_.emplace(row, dedupPos_); return result.second; } + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index aef4a44d520..e47f169e8c8 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -36,6 +36,7 @@ IndexEdgeScanNode::IndexEdgeScanNode(RuntimeContext* context, return ::nebula::cpp2::ErrorCode::SUCCEEDED; }); } + ::nebula::cpp2::ErrorCode IndexEdgeScanNode::init(InitContext& ctx) { if (auto ret = getIndex(this->index_); UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { return ret; @@ -45,6 +46,7 @@ ::nebula::cpp2::ErrorCode IndexEdgeScanNode::init(InitContext& ctx) { } return IndexScanNode::init(ctx); } + Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { std::vector values(requiredColumns_.size()); if (colPosMap_.count(kSrc)) { @@ -82,6 +84,7 @@ Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { decodePropFromIndex(key, colPosMap_, values); return Row(std::move(values)); } + nebula::cpp2::ErrorCode IndexEdgeScanNode::getBaseData(folly::StringPiece key, std::pair& kv) { auto vIdLen = context_->vIdLen(); @@ -93,6 +96,7 @@ nebula::cpp2::ErrorCode IndexEdgeScanNode::getBaseData(folly::StringPiece key, IndexKeyUtils::getIndexDstId(vIdLen, key).str()); return kvstore_->get(context_->spaceId(), partId_, kv.first, &kv.second); } + Map IndexEdgeScanNode::decodeFromBase(const std::string& key, const std::string& value) { Map values; @@ -134,6 +138,7 @@ Map IndexEdgeScanNode::decodeFromBase(const std::string& key } return values; } + const std::vector>& IndexEdgeScanNode::getSchema() { return edge_; diff --git a/src/storage/exec/IndexLimitNode.cpp b/src/storage/exec/IndexLimitNode.cpp index aa82d97b843..8bf1658f0ee 100644 --- a/src/storage/exec/IndexLimitNode.cpp +++ b/src/storage/exec/IndexLimitNode.cpp @@ -17,6 +17,7 @@ nebula::cpp2::ErrorCode IndexLimitNode::doExecute(PartitionID partId) { currentOffset_ = 0; return children_[0]->execute(partId); } + IndexNode::Result IndexLimitNode::doNext() { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; @@ -34,9 +35,11 @@ IndexNode::Result IndexLimitNode::doNext() { return Result(); } } + std::unique_ptr IndexLimitNode::copy() { return std::make_unique(*this); } + std::string IndexLimitNode::identify() { if (offset_ > 0) { return fmt::format("{}(offset={}, limit={})", name_, offset_, limit_); diff --git a/src/storage/exec/IndexNode.cpp b/src/storage/exec/IndexNode.cpp index 0b44a6424f2..3f5e288da6d 100644 --- a/src/storage/exec/IndexNode.cpp +++ b/src/storage/exec/IndexNode.cpp @@ -26,5 +26,6 @@ nebula::cpp2::ErrorCode IndexNode::doExecute(PartitionID partId) { } return ::nebula::cpp2::ErrorCode::SUCCEEDED; } + } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index c7682f76cf0..7d23deb7698 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -148,38 +148,45 @@ inline IndexNode::Result IndexNode::next() { afterNext(); return ret; } + inline void IndexNode::beforeNext() { if (UNLIKELY(profileDetail_)) { duration_.resume(); } } + inline void IndexNode::afterNext() { if (UNLIKELY(profileDetail_)) { duration_.pause(); } } + inline nebula::cpp2::ErrorCode IndexNode::execute(PartitionID partId) { beforeExecute(); auto ret = doExecute(partId); afterExecute(); return ret; } + inline void IndexNode::beforeExecute() { if (UNLIKELY(profileDetail_)) { duration_.resume(); } } + inline void IndexNode::afterExecute() { if (UNLIKELY(profileDetail_)) { duration_.pause(); } } + inline void IndexNode::enableProfileDetail() { profileDetail_ = true; for (auto& child : children_) { child->enableProfileDetail(); } } + inline const time::Duration& IndexNode::duration() { return duration_; } } // namespace storage diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp index af18522157b..65a13e7fcc1 100644 --- a/src/storage/exec/IndexProjectionNode.cpp +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -32,6 +32,7 @@ nebula::cpp2::ErrorCode IndexProjectionNode::init(InitContext& ctx) { } return ::nebula::cpp2::ErrorCode::SUCCEEDED; } + IndexNode::Result IndexProjectionNode::doNext() { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; @@ -41,6 +42,7 @@ IndexNode::Result IndexProjectionNode::doNext() { } return result; } + Row IndexProjectionNode::project(Row&& row) { Row ret; ret.reserve(requiredColumns_.size()); @@ -49,9 +51,11 @@ Row IndexProjectionNode::project(Row&& row) { } return ret; } + std::unique_ptr IndexProjectionNode::copy() { return std::make_unique(*this); } + std::string IndexProjectionNode::identify() { return fmt::format("{}(projectColumn=[{}])", name_, folly::join(",", requiredColumns_)); } diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index 77f7a65a1d5..a8e93d08078 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -37,6 +37,7 @@ Path::Path(nebula::meta::cpp2::IndexItem* index, suffixLength_ = vidLen * 2 + sizeof(EdgeRanking); } } + std::unique_ptr Path::make(nebula::meta::cpp2::IndexItem* index, const meta::SchemaProviderIf* schema, const std::vector& hints, @@ -50,9 +51,11 @@ std::unique_ptr Path::make(nebula::meta::cpp2::IndexItem* index, } return ret; } + QualifiedStrategy::Result Path::qualified(const folly::StringPiece& key) { return strategySet_(key); } + std::string Path::encodeValue(const Value& value, const ColumnTypeDef& colDef, size_t index, @@ -89,6 +92,7 @@ std::string Path::encodeValue(const Value& value, key.append(val); return val; } + const std::string& Path::toString() { return serializeString_; } // End of Path @@ -101,11 +105,13 @@ RangePath::RangePath(nebula::meta::cpp2::IndexItem* index, : Path(index, schema, hints, vidLen) { buildKey(); } + void RangePath::resetPart(PartitionID partId) { std::string p = IndexKeyUtils::indexPrefix(partId); startKey_ = startKey_.replace(0, p.size(), p); endKey_ = endKey_.replace(0, p.size(), p); } + QualifiedStrategy::Result RangePath::qualified(const Map& rowData) { for (size_t i = 0; i < hints_.size() - 1; i++) { auto& hint = hints_[i]; @@ -133,6 +139,7 @@ QualifiedStrategy::Result RangePath::qualified(const Map& ro } return QualifiedStrategy::COMPATIBLE; } + void RangePath::buildKey() { std::string commonIndexPrefix; commonIndexPrefix.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); @@ -176,6 +183,7 @@ void RangePath::buildKey() { DVLOG(2) << "start key:\n" << folly::hexDump(startKey_.data(), startKey_.size()); DVLOG(2) << "end key:\n" << folly::hexDump(endKey_.data(), endKey_.size()); } + std::tuple RangePath::encodeRange( const cpp2::IndexColumnHint& hint, const nebula::meta::cpp2::ColumnTypeDef& colTypeDef, @@ -202,6 +210,7 @@ std::tuple RangePath::encodeRange( } return {startKey, endKey}; } + std::string RangePath::encodeBeginValue(const Value& value, const ColumnTypeDef& colDef, std::string& key, @@ -233,6 +242,7 @@ std::string RangePath::encodeBeginValue(const Value& value, key += val; return val; } + std::string RangePath::encodeEndValue(const Value& value, const ColumnTypeDef& colDef, std::string& key, @@ -268,6 +278,7 @@ std::string RangePath::encodeEndValue(const Value& value, DVLOG(2) << '\n' << folly::hexDump(key.data(), key.size()); return val; } + inline std::string RangePath::encodeString(const Value& value, size_t len, bool& truncated) { std::string val = IndexKeyUtils::encodeValue(value); if (val.size() < len) { @@ -278,6 +289,7 @@ inline std::string RangePath::encodeString(const Value& value, size_t len, bool& } return val; } + std::string RangePath::encodeFloat(const Value& value, bool& isNaN) { std::string val = IndexKeyUtils::encodeValue(value); // check NaN @@ -297,6 +309,7 @@ PrefixPath::PrefixPath(nebula::meta::cpp2::IndexItem* index, : Path(index, schema, hints, vidLen) { buildKey(); } + QualifiedStrategy::Result PrefixPath::qualified(const Map& rowData) { for (auto& hint : hints_) { if (hint.get_begin_value() != rowData.at(hint.get_column_name())) { @@ -305,10 +318,12 @@ QualifiedStrategy::Result PrefixPath::qualified(const Map& r } return QualifiedStrategy::COMPATIBLE; } + void PrefixPath::resetPart(PartitionID partId) { std::string p = IndexKeyUtils::indexPrefix(partId); prefix_ = prefix_.replace(0, p.size(), p); } + void PrefixPath::buildKey() { std::string common; common.append(IndexKeyUtils::indexPrefix(0, index_->index_id_ref().value())); @@ -330,6 +345,7 @@ void PrefixPath::buildKey() { } prefix_ = std::move(common); } + // End of PrefixPath // Define of IndexScan @@ -391,11 +407,13 @@ ::nebula::cpp2::ErrorCode IndexScanNode::init(InitContext& ctx) { path_ = Path::make(index_.get(), getSchema().back().get(), columnHints_, context_->vIdLen()); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } + nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { partId_ = partId; auto ret = resetIter(partId); return ret; } + IndexNode::Result IndexScanNode::doNext() { for (; iter_ && iter_->valid(); iter_->next()) { DVLOG(3) << '\n' << folly::hexDump(iter_->key().data(), iter_->key().size()); @@ -445,6 +463,7 @@ IndexNode::Result IndexScanNode::doNext() { } return Result(); } + bool IndexScanNode::checkTTL() { if (iter_->val().empty() || ttlProps_.first == false) { return true; @@ -478,6 +497,7 @@ nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { } return ret; } + void IndexScanNode::decodePropFromIndex(folly::StringPiece key, const Map& colPosMap, std::vector& values) { @@ -535,9 +555,11 @@ void IndexScanNode::decodePropFromIndex(folly::StringPiece key, nullableColPosit -= 1; } } + std::string IndexScanNode::identify() { return fmt::format("{}(IndexID={}, Path=({}))", name_, indexId_, path_->toString()); } + // End of IndexScan } // namespace storage } // namespace nebula diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index 232054b3abd..8220f753387 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -30,6 +30,7 @@ nebula::cpp2::ErrorCode IndexSelectionNode::init(InitContext& ctx) { ctx_ = std::make_unique(colPos_); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } + IndexNode::Result IndexSelectionNode::doNext() { DCHECK_EQ(children_.size(), 1); auto& child = *children_[0]; @@ -45,12 +46,15 @@ IndexNode::Result IndexSelectionNode::doNext() { } while (true); return Result(); } + std::unique_ptr IndexSelectionNode::copy() { return std::make_unique(*this); } + std::string IndexSelectionNode::identify() { return fmt::format("{}(expr=[{}])", name_, expr_->toString()); } + Value IndexSelectionNode::ExprContext::getEdgeProp(const std::string& edgeType, const std::string& prop) const { UNUSED(edgeType); @@ -60,6 +64,7 @@ Value IndexSelectionNode::ExprContext::getEdgeProp(const std::string& edgeType, DCHECK(iter->second < row_->size()); return (*row_)[iter->second]; } + Value IndexSelectionNode::ExprContext::getTagProp(const std::string& tag, const std::string& prop) const { UNUSED(tag); diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index 94e7962d9c3..3733cd07223 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -41,6 +41,7 @@ IndexVertexScanNode::IndexVertexScanNode(RuntimeContext* context, return ::nebula::cpp2::ErrorCode::SUCCEEDED; }); } + ::nebula::cpp2::ErrorCode IndexVertexScanNode::init(InitContext& ctx) { if (auto ret = getIndex(this->index_); UNLIKELY(ret != ::nebula::cpp2::ErrorCode::SUCCEEDED)) { return ret; @@ -50,6 +51,7 @@ ::nebula::cpp2::ErrorCode IndexVertexScanNode::init(InitContext& ctx) { } return IndexScanNode::init(ctx); } + nebula::cpp2::ErrorCode IndexVertexScanNode::getBaseData(folly::StringPiece key, std::pair& kv) { kv.first = NebulaKeyUtils::vertexKey(context_->vIdLen(), @@ -60,6 +62,7 @@ nebula::cpp2::ErrorCode IndexVertexScanNode::getBaseData(folly::StringPiece key, DVLOG(1) << '\n' << folly::hexDump(kv.first.data(), kv.first.size()); return kvstore_->get(context_->spaceId(), partId_, kv.first, &kv.second); } + Row IndexVertexScanNode::decodeFromIndex(folly::StringPiece key) { std::vector values(requiredColumns_.size()); if (colPosMap_.count(kVid)) { @@ -77,6 +80,7 @@ Row IndexVertexScanNode::decodeFromIndex(folly::StringPiece key) { decodePropFromIndex(key, colPosMap_, values); return Row(std::move(values)); } + Map IndexVertexScanNode::decodeFromBase(const std::string& key, const std::string& value) { Map values; @@ -107,6 +111,7 @@ Map IndexVertexScanNode::decodeFromBase(const std::string& k } return values; } + std::unique_ptr IndexVertexScanNode::copy() { return std::make_unique(*this); } From 8d7ed4f86b914df12abeac2a350c43fbb1d6142f Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Thu, 4 Nov 2021 12:19:54 +0800 Subject: [PATCH 35/38] fix compile error --- src/common/utils/IndexKeyUtils.h | 4 +- src/storage/exec/IndexEdgeScanNode.cpp | 6 -- src/storage/exec/IndexScanNode.cpp | 19 +----- src/storage/exec/IndexScanNode.h | 7 +-- src/storage/exec/IndexSelectionNode.cpp | 1 - src/storage/exec/IndexSelectionNode.h | 2 +- src/storage/exec/IndexVertexScanNode.cpp | 2 - src/storage/index/LookupProcessor.cpp | 4 -- src/storage/test/IndexTest.cpp | 58 ++++++++----------- src/storage/test/IndexTestUtil.h | 73 +++++++++++------------- 10 files changed, 61 insertions(+), 115 deletions(-) diff --git a/src/common/utils/IndexKeyUtils.h b/src/common/utils/IndexKeyUtils.h index ab68260ac09..95590f9b6a4 100644 --- a/src/common/utils/IndexKeyUtils.h +++ b/src/common/utils/IndexKeyUtils.h @@ -229,7 +229,9 @@ class IndexKeyUtils final { } else { val = ~val; } - return *reinterpret_cast(&val); + double ret; + ::memcpy(&ret, &val, 8); + return ret; } static std::string encodeTime(const nebula::Time& t) { diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index e47f169e8c8..7f070615910 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -51,16 +51,11 @@ Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { std::vector values(requiredColumns_.size()); if (colPosMap_.count(kSrc)) { auto vId = IndexKeyUtils::getIndexSrcId(context_->vIdLen(), key); - DVLOG(1) << folly::hexDump(vId.data(), vId.size()); if (context_->isIntId()) { values[colPosMap_[kSrc]] = Value(*reinterpret_cast(vId.data())); } else { - DVLOG(2) << vId.subpiece(0, vId.find_first_of('\0')); - DVLOG(2) << vId.subpiece(0, vId.find_first_of('\0')).toString(); - DVLOG(2) << Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); values[colPosMap_[kSrc]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } - DVLOG(1) << values[colPosMap_[kSrc]]; } if (colPosMap_.count(kDst)) { auto vId = IndexKeyUtils::getIndexDstId(context_->vIdLen(), key); @@ -69,7 +64,6 @@ Row IndexEdgeScanNode::decodeFromIndex(folly::StringPiece key) { } else { values[colPosMap_[kDst]] = Value(vId.subpiece(0, vId.find_first_of('\0')).toString()); } - DVLOG(1) << values[colPosMap_[kDst]]; } if (colPosMap_.count(kRank)) { auto rank = IndexKeyUtils::getIndexRank(context_->vIdLen(), key); diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index a8e93d08078..4408c8624a1 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -129,10 +129,8 @@ QualifiedStrategy::Result RangePath::qualified(const Map& ro } } if (hint.end_value_ref().is_set()) { - DVLOG(2) << includeEnd_; bool ret = includeEnd_ ? hint.get_end_value() >= rowData.at(hint.get_column_name()) : hint.get_end_value() > rowData.at(hint.get_column_name()); - DVLOG(2) << ret; if (!ret) { return QualifiedStrategy::INCOMPATIBLE; } @@ -180,8 +178,6 @@ void RangePath::buildKey() { if (!hint.end_value_ref().is_set()) { endKey_.append(totalKeyLength_ - endKey_.size() + 1, '\xFF'); } - DVLOG(2) << "start key:\n" << folly::hexDump(startKey_.data(), startKey_.size()); - DVLOG(2) << "end key:\n" << folly::hexDump(endKey_.data(), endKey_.size()); } std::tuple RangePath::encodeRange( @@ -194,7 +190,7 @@ std::tuple RangePath::encodeRange( if (hint.end_value_ref().is_set()) { includeEnd_ = hint.get_include_end(); auto tmp = encodeEndValue(hint.get_end_value(), colTypeDef, endKey, offset); - if (memcmp(tmp.data(), std::string(tmp.size(), '\xFF').data(), tmp.size() != 0)) { + if (memcmp(tmp.data(), std::string(tmp.size(), '\xFF').data(), tmp.size()) != 0) { needCheckNullable &= false; } } @@ -269,13 +265,10 @@ std::string RangePath::encodeEndValue(const Value& value, } else { val = IndexKeyUtils::encodeValue(value); } - DVLOG(2) << includeEnd_; - DVLOG(2) << greater; if (greater) { val.append(suffixLength_ + 1, '\xFF'); } key += val; - DVLOG(2) << '\n' << folly::hexDump(key.data(), key.size()); return val; } @@ -416,7 +409,6 @@ nebula::cpp2::ErrorCode IndexScanNode::doExecute(PartitionID partId) { IndexNode::Result IndexScanNode::doNext() { for (; iter_ && iter_->valid(); iter_->next()) { - DVLOG(3) << '\n' << folly::hexDump(iter_->key().data(), iter_->key().size()); if (!checkTTL()) { continue; } @@ -429,7 +421,6 @@ IndexNode::Result IndexScanNode::doNext() { auto key = iter_->key().toString(); iter_->next(); Row row = decodeFromIndex(key); - DVLOG(3) << row; return Result(std::move(row)); } std::pair kv; @@ -458,7 +449,6 @@ IndexNode::Result IndexScanNode::doNext() { row.emplace_back(std::move(rowData.at(col))); } iter_->next(); - DVLOG(3) << row; return Result(std::move(row)); } return Result(); @@ -483,16 +473,9 @@ nebula::cpp2::ErrorCode IndexScanNode::resetIter(PartitionID partId) { nebula::cpp2::ErrorCode ret = nebula::cpp2::ErrorCode::SUCCEEDED; if (path_->isRange()) { auto rangePath = dynamic_cast(path_.get()); - DVLOG(1) << '\n' - << folly::hexDump(rangePath->getStartKey().data(), rangePath->getStartKey().size()); - DVLOG(1) << '\n' - << folly::hexDump(rangePath->getEndKey().data(), rangePath->getEndKey().size()); kvstore_->range(spaceId_, partId, rangePath->getStartKey(), rangePath->getEndKey(), &iter_); } else { auto prefixPath = dynamic_cast(path_.get()); - DVLOG(1) << '\n' - << folly::hexDump(prefixPath->getPrefixKey().data(), - prefixPath->getPrefixKey().size()); ret = kvstore_->prefix(spaceId_, partId, prefixPath->getPrefixKey(), &iter_); } return ret; diff --git a/src/storage/exec/IndexScanNode.h b/src/storage/exec/IndexScanNode.h index 3dea609b99f..8ce22bea970 100644 --- a/src/storage/exec/IndexScanNode.h +++ b/src/storage/exec/IndexScanNode.h @@ -204,8 +204,6 @@ class QualifiedStrategy { std::bitset<16> nullableBit; auto v = *reinterpret_cast(key.data() + keyOffset); nullableBit = v; - DVLOG(3) << targetIsNull; - DVLOG(3) << nullableBit.test(15 - columnIndex); return nullableBit.test(15 - columnIndex) == targetIsNull ? Result::COMPATIBLE : Result::INCOMPATIBLE; }; @@ -217,7 +215,6 @@ class QualifiedStrategy { std::bitset<16> nullableBit; auto v = *reinterpret_cast(key.data() + keyOffset); nullableBit = v; - DVLOG(3) << nullableBit.test(15 - columnIndex); return nullableBit.test(15 - columnIndex) ? Result::INCOMPATIBLE : Result::COMPATIBLE; }; return q; @@ -302,8 +299,6 @@ class QualifiedStrategy { QualifiedStrategy q; q.func_ = [val, keyStartPos](const folly::StringPiece& key) { int ret = memcmp(val.data(), key.data() + keyStartPos, val.size()); - DVLOG(1) << folly::hexDump(val.data(), val.size()); - DVLOG(1) << folly::hexDump(key.data(), key.size()); if constexpr (LEorGE == true) { CHECK_LE(ret, 0); } else { @@ -394,7 +389,7 @@ class RangePath : public Path { inline bool includeEnd() { return includeEnd_; } inline const std::string& getStartKey() { return startKey_; } inline const std::string& getEndKey() { return endKey_; } - virtual bool isRange() { return true; } + bool isRange() override { return true; } private: std::string startKey_, endKey_; diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index 8220f753387..f5e42cfe67d 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -39,7 +39,6 @@ IndexNode::Result IndexSelectionNode::doNext() { if (!result.hasData()) { return result; } - DVLOG(1) << result.row(); if (filter(result.row())) { return result; } diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 9250a119d03..49a19b61f81 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -121,7 +121,7 @@ class SelectionExprVisitor : public ExprVisitorBase { void visit(EdgeTypeExpression *expr) override { requiredColumns_.insert(expr->prop()); } void visit(EdgeRankExpression *expr) override { requiredColumns_.insert(expr->prop()); } void visit(EdgeDstIdExpression *expr) override { requiredColumns_.insert(expr->prop()); } - void visit(TagPropertyExpression *expr) { requiredColumns_.insert(expr->prop()); } + void visit(TagPropertyExpression *expr) override { requiredColumns_.insert(expr->prop()); } void visit(EdgePropertyExpression *expr) override { requiredColumns_.insert(expr->prop()); } const Set &getRequiredColumns() { return requiredColumns_; } ::nebula::cpp2::ErrorCode getCode() { return code_; } diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index 3733cd07223..529854f8c4e 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -58,8 +58,6 @@ nebula::cpp2::ErrorCode IndexVertexScanNode::getBaseData(folly::StringPiece key, partId_, key.subpiece(key.size() - context_->vIdLen()).toString(), context_->tagId_); - DVLOG(3) << partId_; - DVLOG(1) << '\n' << folly::hexDump(kv.first.data(), kv.first.size()); return kvstore_->get(context_->spaceId(), partId_, kv.first, &kv.second); } diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index 2a55d79fa23..fe28962abb8 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -25,7 +25,6 @@ ProcessorCounters kLookupCounters; // print Plan for debug inline void printPlan(IndexNode* node, int tab = 0); void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { - DVLOG(1) << ::apache::thrift::SimpleJSONSerializer::serialize(req); if (executor_ != nullptr) { executor_->add([req, this]() { this->doProcess(req); }); } else { @@ -43,7 +42,6 @@ void LookupProcessor::doProcess(const cpp2::LookupIndexRequest& req) { pushResultCode(code, p); } onFinished(); - DVLOG(1) << static_cast(code); return; } auto plan = buildPlan(req); @@ -58,7 +56,6 @@ void LookupProcessor::doProcess(const cpp2::LookupIndexRequest& req) { pushResultCode(code, p); } onFinished(); - DVLOG(1) << static_cast(code); return; } if (!FLAGS_query_concurrently) { @@ -284,7 +281,6 @@ void LookupProcessor::profilePlan(IndexNode* root) { } } inline void printPlan(IndexNode* node, int tab) { - DVLOG(2) << std::string(tab, '\t') << node->identify(); for (auto& child : node->children()) { printPlan(child.get(), tab + 1); } diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index a7ea775d38b..0d80d6dad93 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -75,7 +75,8 @@ using std::string_literals::operator""s; * ├─┼┤ * └─┴┘ */ -struct IndexScanTestHelper { +class IndexScanTestHelper { + public: void setIndex(IndexVertexScanNode* node, std::shared_ptr<::nebula::meta::cpp2::IndexItem> index) { node->getIndex = [index](std::shared_ptr<::nebula::meta::cpp2::IndexItem>& ret) { ret = index; @@ -159,15 +160,14 @@ class IndexScanTest : public ::testing::Test { } writer.finish(); auto value = writer.moveEncodedStr(); - assert(ret[0].insert({key, value}).second); + CHECK(ret[0].insert({key, value}).second); RowReaderWrapper reader(schema.get(), folly::StringPiece(value), schemaVer); for (size_t j = 0; j < indices.size(); j++) { auto& index = indices[j]; auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index->get_fields()).value(); - DVLOG(1) << folly::hexDump(indexValue.data(), indexValue.size()); auto indexKey = IndexKeyUtils::vertexIndexKeys( 8, 0, index->get_index_id(), std::to_string(i), std::move(indexValue))[0]; - assert(ret[j + 1].insert({indexKey, ""}).second); + CHECK(ret[j + 1].insert({indexKey, ""}).second); } } return ret; @@ -180,14 +180,13 @@ class IndexScanTest : public ::testing::Test { std::vector> ret(indices.size() + 1); for (size_t i = 0; i < rows.size(); i++) { auto key = NebulaKeyUtils::edgeKey(8, 0, std::to_string(i), edgeType, i, std::to_string(i)); - DVLOG(1) << '\n' << folly::hexDump(key.data(), key.size()); RowWriterV2 writer(schema.get()); for (size_t j = 0; j < rows[i].size(); j++) { writer.setValue(j, rows[i][j]); } writer.finish(); auto value = writer.moveEncodedStr(); - assert(ret[0].insert({key, value}).second); + CHECK(ret[0].insert({key, value}).second); RowReaderWrapper reader(schema.get(), folly::StringPiece(value), schemaVer); for (size_t j = 0; j < indices.size(); j++) { auto& index = indices[j]; @@ -199,8 +198,7 @@ class IndexScanTest : public ::testing::Test { i, std::to_string(i), std::move(indexValue))[0]; - DVLOG(1) << '\n' << folly::hexDump(indexKey.data(), indexKey.size()); - assert(ret[j + 1].insert({indexKey, ""}).second); + CHECK(ret[j + 1].insert({indexKey, ""}).second); } } return ret; @@ -469,7 +467,6 @@ TEST_F(IndexScanTest, Edge) { ASSERT_EQ(result.size(), expect.size()); for (size_t i = 0; i < result.size(); i++) { ASSERT_EQ(result[i].size(), expect[i].size()); - DVLOG(1) << result[i]; for (size_t j = 0; j < expect[i].size(); j++) { EXPECT_EQ(expect[i][j], result[i][initCtx.retColMap[colOrder[j]]]); } @@ -507,7 +504,6 @@ TEST_F(IndexScanTest, Edge) { ASSERT_EQ(result.size(), expect.size()); for (size_t i = 0; i < result.size(); i++) { ASSERT_EQ(result[i].size(), expect[i].size()); - DVLOG(1) << result[i]; for (size_t j = 0; j < expect[i].size(); j++) { EXPECT_EQ(expect[i][j], result[i][initCtx.retColMap[colOrder[j]]]); } @@ -546,7 +542,6 @@ TEST_F(IndexScanTest, Int) { const std::vector& columnHints, const std::vector& expect, const std::string& case_) { - DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), 0, columnHints, kvstore.get()); @@ -760,7 +755,6 @@ float | float | float | int const std::vector& columnHints, const std::vector& expect, const std::string& case_) { - DVLOG(1) << "Start case " << case_; auto context = makeContext(0, 1); auto scanNode = std::make_unique(context.get(), 0, columnHints, kvstore.get()); @@ -782,7 +776,6 @@ float | float | float | int result.emplace_back(std::move(res).row()); } EXPECT_EQ(result, expect) << "Fail at case " << case_; - DVLOG(1) << "End case " << case_; }; auto expect = [](auto... vidList) { std::vector ret; @@ -880,9 +873,13 @@ float | float | float | int }; check(indices[0], hint("a", 100), slice(aOrder, 11), "case4.1"); check(indices[1], hint("b", INF), {}, "case4.2"); - int64_t x = *reinterpret_cast(&INF); + int64_t x; + ::memcpy(&x, &INF, 8); + // int64_t x = *reinterpret_cast(&INF); x--; - double y = *reinterpret_cast(&x); + double y; + ::memcpy(&y, &x, 8); + // double y = *reinterpret_cast(&x); check(indices[1], hint("b", y), slice(bOrder, 11), "case4.3"); check(indices[2], hint("c", INF), {}, "case4.4"); check(indices[2], hint("c", y), slice(cOrder, 6), "case4.5"); @@ -946,9 +943,13 @@ float | float | float | int }; check(indices[0], hint("a", 100), slice(aOrder, 10), "case7.1"); check(indices[1], hint("b", -INF), {}, "case7.2"); - int64_t x = *reinterpret_cast(&INF); + int64_t x; + ::memcpy(&x, &INF, 8); + // int64_t x = *reinterpret_cast(&INF); x--; - double y = *reinterpret_cast(&x); + double y; + ::memcpy(&y, &x, 8); + // double y = *reinterpret_cast(&x); check(indices[1], hint("b", -y), slice(bOrder, 1), "case7.3"); check(indices[2], hint("c", -INF), {}, "case7.4"); check(indices[2], hint("c", -y), slice(cOrder, 2), "case7.5"); @@ -983,7 +984,6 @@ TEST_F(IndexScanTest, Bool) { const std::vector& columnHints, const std::vector& expect, const std::string& case_) { - DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), 0, columnHints, kvstore.get()); @@ -1005,7 +1005,6 @@ TEST_F(IndexScanTest, Bool) { result.emplace_back(std::move(res).row()); } EXPECT_EQ(result, expect) << "Fail at case " << case_; - DVLOG(1) << "End case " << case_; }; auto expect = [](auto... vidList) { std::vector ret; @@ -1073,7 +1072,6 @@ TEST_F(IndexScanTest, String1) { const std::vector& acquiredColumns, const std::vector& expect, const std::string& case_) { - DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), 0, columnHints, kvstore.get()); @@ -1230,7 +1228,6 @@ TEST_F(IndexScanTest, String2) { const std::vector& acquiredColumns, const std::vector& expect, const std::string& case_) { - DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), 0, columnHints, kvstore.get()); @@ -1361,7 +1358,6 @@ TEST_F(IndexScanTest, String3) { const std::vector& acquiredColumns, const std::vector& expect, const std::string& case_) { - DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), 0, columnHints, kvstore.get()); @@ -1483,7 +1479,6 @@ TEST_F(IndexScanTest, String4) { const std::vector& acquiredColumns, const std::vector& expect, const std::string& case_) { - DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), 0, columnHints, kvstore.get()); @@ -1599,7 +1594,6 @@ TEST_F(IndexScanTest, Nullable) { const std::vector& columnHints, const std::vector& expect, const std::string& case_) { - DVLOG(1) << "Start case " << case_; auto context = makeContext(1, 0); auto scanNode = std::make_unique(context.get(), 0, columnHints, kvstore.get()); @@ -1793,7 +1787,7 @@ TEST_F(IndexTest, Selection) { auto selection = std::make_unique(ctx.get(), expr); auto mockChild = std::make_unique(ctx.get()); - mockChild->executeFunc = [&rows](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + mockChild->executeFunc = [](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; mockChild->nextFunc = [&rows, ¤tOffset]() -> IndexNode::Result { if (currentOffset < rows.size()) { auto row = rows[currentOffset++]; @@ -1803,7 +1797,6 @@ TEST_F(IndexTest, Selection) { } }; mockChild->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { - ASSERT(initCtx.requiredColumns.find("a") != initCtx.requiredColumns.end()); initCtx.returnColumns = {"a", "b"}; initCtx.retColMap = {{"a", 0}, {"b", 1}}; return ::nebula::cpp2::ErrorCode::SUCCEEDED; @@ -1823,7 +1816,7 @@ TEST_F(IndexTest, Projection) { auto projection = std::make_unique(ctx.get(), std::vector{"c", "a", "b"}); auto mockChild = std::make_unique(ctx.get()); - mockChild->executeFunc = [&rows](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + mockChild->executeFunc = [](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; mockChild->nextFunc = [&rows, ¤tOffset]() -> IndexNode::Result { if (currentOffset < rows.size()) { auto row = rows[currentOffset++]; @@ -1833,9 +1826,6 @@ TEST_F(IndexTest, Projection) { } }; mockChild->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { - ASSERT(initCtx.requiredColumns.find("a") != initCtx.requiredColumns.end()); - ASSERT(initCtx.requiredColumns.find("b") != initCtx.requiredColumns.end()); - ASSERT(initCtx.requiredColumns.find("c") != initCtx.requiredColumns.end()); initCtx.returnColumns = {"a", "b", "c"}; initCtx.retColMap = {{"a", 0}, {"b", 1}, {"c", 2}}; return ::nebula::cpp2::ErrorCode::SUCCEEDED; @@ -1866,7 +1856,7 @@ TEST_F(IndexTest, Limit) { auto ctx = makeContext(); auto limit = std::make_unique(ctx.get(), 10); auto mockChild = std::make_unique(ctx.get()); - mockChild->executeFunc = [&rows](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + mockChild->executeFunc = [](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; mockChild->nextFunc = [&rows, ¤tOffset]() -> IndexNode::Result { if (currentOffset < rows.size()) { auto row = rows[currentOffset++]; @@ -1899,7 +1889,7 @@ TEST_F(IndexTest, Dedup) { auto ctx = makeContext(); auto dedup = std::make_unique(ctx.get(), std::vector{"a"}); auto child1 = std::make_unique(ctx.get()); - child1->executeFunc = [&rows1](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + child1->executeFunc = [](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; child1->nextFunc = [&rows1, &offset1]() -> IndexNode::Result { if (offset1 < rows1.size()) { auto row = rows1[offset1++]; @@ -1909,13 +1899,12 @@ TEST_F(IndexTest, Dedup) { } }; child1->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { - ASSERT(initCtx.requiredColumns.find("a") != initCtx.requiredColumns.end()); initCtx.returnColumns = {"a", "b"}; initCtx.retColMap = {{"a", 0}, {"b", 1}}; return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; auto child2 = std::make_unique(ctx.get()); - child2->executeFunc = [&rows2](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; + child2->executeFunc = [](PartitionID) { return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; child2->nextFunc = [&rows2, &offset2]() -> IndexNode::Result { if (offset2 < rows2.size()) { auto row = rows2[offset2++]; @@ -1925,7 +1914,6 @@ TEST_F(IndexTest, Dedup) { } }; child2->initFunc = [](InitContext& initCtx) -> ::nebula::cpp2::ErrorCode { - ASSERT(initCtx.requiredColumns.find("a") != initCtx.requiredColumns.end()); initCtx.returnColumns = {"a", "b"}; initCtx.retColMap = {{"a", 0}, {"b", 1}}; return ::nebula::cpp2::ErrorCode::SUCCEEDED; diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index efa798db52e..fa07a5297b1 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -28,14 +28,7 @@ class MockKVIterator : public KVIterator { public: MockKVIterator(const KVMap& kv, KVMap::iterator&& iter) : kv_(kv), iter_(std::move(iter)) {} bool valid() const { return iter_ != kv_.end() && validFunc_(iter_); } - void next() { - iter_++; - if (iter_ != kv_.end()) { - DVLOG(2) << '\n' << folly::hexDump(iter_->first.data(), iter_->first.size()); - } else { - DVLOG(2) << "InValid!!!"; - } - } + void next() { iter_++; } void prev() { iter_--; } folly::StringPiece key() const { return folly::StringPiece(iter_->first); } folly::StringPiece val() const { return folly::StringPiece(iter_->second); } @@ -57,14 +50,14 @@ class MockKVStore : public ::nebula::kvstore::KVStore { MockKVStore() {} // Return bit-OR of StoreCapability values; uint32_t capability() const override { - assert(false); + CHECK(false); return 0; }; void stop() override {} ErrorOr partLeader(GraphSpaceID spaceId, PartitionID partID) override { UNUSED(spaceId), UNUSED(partID); - assert(false); + CHECK(false); return nebula::cpp2::ErrorCode::SUCCEEDED; } // Read a single key @@ -130,23 +123,23 @@ class MockKVStore : public ::nebula::kvstore::KVStore { (*iter) = std::move(mockIter); return ::nebula::cpp2::ErrorCode::SUCCEEDED; } - virtual nebula::cpp2::ErrorCode prefix(GraphSpaceID spaceId, - PartitionID partId, - std::string&& prefix, - std::unique_ptr* iter, - bool canReadFromFollower = false) = delete; - virtual nebula::cpp2::ErrorCode rangeWithPrefix(GraphSpaceID spaceId, - PartitionID partId, - std::string&& start, - std::string&& prefix, - std::unique_ptr* iter, - bool canReadFromFollower = false) = delete; - virtual nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, - PartitionID partId, - std::string&& start, - std::string&& end, - std::unique_ptr* iter, - bool canReadFromFollower = false) = delete; + // virtual nebula::cpp2::ErrorCode prefix(GraphSpaceID spaceId, + // PartitionID partId, + // std::string&& prefix, + // std::unique_ptr* iter, + // bool canReadFromFollower = false) = delete override; + // virtual nebula::cpp2::ErrorCode rangeWithPrefix(GraphSpaceID spaceId, + // PartitionID partId, + // std::string&& start, + // std::string&& prefix, + // std::unique_ptr* iter, + // bool canReadFromFollower = false) = delete; + // virtual nebula::cpp2::ErrorCode range(GraphSpaceID spaceId, + // PartitionID partId, + // std::string&& start, + // std::string&& end, + // std::unique_ptr* iter, + // bool canReadFromFollower = false) = delete; nebula::cpp2::ErrorCode prefix(GraphSpaceID spaceId, PartitionID partId, const std::string& prefix, @@ -199,7 +192,7 @@ class MockKVStore : public ::nebula::kvstore::KVStore { return ::nebula::cpp2::ErrorCode::SUCCEEDED; } - nebula::cpp2::ErrorCode sync(GraphSpaceID spaceId, PartitionID partId) { + nebula::cpp2::ErrorCode sync(GraphSpaceID spaceId, PartitionID partId) override { UNUSED(spaceId); UNUSED(partId); LOG(FATAL) << "Unexpect"; @@ -316,14 +309,14 @@ class MockKVStore : public ::nebula::kvstore::KVStore { LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; }; - nebula::cpp2::ErrorCode dropCheckpoint(GraphSpaceID spaceId, const std::string& name) { + nebula::cpp2::ErrorCode dropCheckpoint(GraphSpaceID spaceId, const std::string& name) override { UNUSED(spaceId); UNUSED(name); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } - nebula::cpp2::ErrorCode setWriteBlocking(GraphSpaceID spaceId, bool sign) { + nebula::cpp2::ErrorCode setWriteBlocking(GraphSpaceID spaceId, bool sign) override { UNUSED(spaceId); UNUSED(sign); LOG(FATAL) << "Unexpect"; @@ -334,7 +327,7 @@ class MockKVStore : public ::nebula::kvstore::KVStore { GraphSpaceID spaceId, const std::string& name, const std::string& tablePrefix, - std::function filter) { + std::function filter) override { UNUSED(spaceId); UNUSED(name); UNUSED(tablePrefix); @@ -344,20 +337,20 @@ class MockKVStore : public ::nebula::kvstore::KVStore { } // for meta BR nebula::cpp2::ErrorCode restoreFromFiles(GraphSpaceID spaceId, - const std::vector& files) { + const std::vector& files) override { UNUSED(spaceId); UNUSED(files); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } - nebula::cpp2::ErrorCode multiPutWithoutReplicator(GraphSpaceID spaceId, - std::vector<::nebula::kvstore::KV> keyValues) { + nebula::cpp2::ErrorCode multiPutWithoutReplicator( + GraphSpaceID spaceId, std::vector<::nebula::kvstore::KV> keyValues) override { UNUSED(spaceId); UNUSED(keyValues); LOG(FATAL) << "Unexpect"; return ::nebula::cpp2::ErrorCode::SUCCEEDED; } - std::vector getDataRoot() const { + std::vector getDataRoot() const override { LOG(FATAL) << "Unexpect"; return {}; } @@ -378,8 +371,8 @@ class MockKVStore : public ::nebula::kvstore::KVStore { class MockIndexNode : public IndexNode { public: explicit MockIndexNode(RuntimeContext* context) : IndexNode(context, "MockIndexNode") {} - ::nebula::cpp2::ErrorCode init(InitContext& initCtx) { return initFunc(initCtx); } - std::unique_ptr copy() { LOG(FATAL) << "Unexpect"; } + ::nebula::cpp2::ErrorCode init(InitContext& initCtx) override { return initFunc(initCtx); } + std::unique_ptr copy() override { LOG(FATAL) << "Unexpect"; } std::function nextFunc; std::function<::nebula::cpp2::ErrorCode(PartitionID)> executeFunc; std::function<::nebula::cpp2::ErrorCode(InitContext& initCtx)> initFunc; @@ -388,8 +381,6 @@ class MockIndexNode : public IndexNode { private: Result doNext() override { return nextFunc(); } ::nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { return executeFunc(partId); }; - int offset_; - int total_; }; class RowParser { @@ -529,7 +520,7 @@ class IndexParser { std::smatch match; std::string line; std::getline(ss, line); - assert(std::regex_match(line, match, pattern)); + CHECK(std::regex_match(line, match, pattern)); std::string name = match.str(2); int32_t id = std::stoi(match.str(3)); schemaName_ = name; @@ -555,7 +546,7 @@ class IndexParser { ret->set_schema_name(schemaName_); static std::regex pattern(R"(\((.+),(\d+)\):(.+))"); std::smatch match; - assert(std::regex_match(line, match, pattern)); + CHECK(std::regex_match(line, match, pattern)); ret->set_index_name(folly::trimWhitespace(folly::StringPiece(match.str(1)).toString())); ret->set_index_id(std::stoi(match.str(2))); std::string columnStr = match.str(3); From 56bf4414edf878a02ef21d11df86aeb00ff7c30b Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Thu, 4 Nov 2021 12:26:39 +0800 Subject: [PATCH 36/38] modify new file license --- src/storage/ExprVisitorBase.cpp | 1 - src/storage/ExprVisitorBase.h | 1 - src/storage/exec/IndexDedupNode.cpp | 1 - src/storage/exec/IndexDedupNode.h | 1 - src/storage/exec/IndexEdgeScanNode.cpp | 1 - src/storage/exec/IndexEdgeScanNode.h | 1 - src/storage/exec/IndexLimitNode.cpp | 1 - src/storage/exec/IndexLimitNode.h | 1 - src/storage/exec/IndexNode.cpp | 1 - src/storage/exec/IndexNode.h | 1 - src/storage/exec/IndexProjectionNode.cpp | 1 - src/storage/exec/IndexProjectionNode.h | 1 - src/storage/exec/IndexScanNode.cpp | 1 - src/storage/exec/IndexSelectionNode.cpp | 1 - src/storage/exec/IndexSelectionNode.h | 1 - src/storage/exec/IndexVertexScanNode.cpp | 1 - src/storage/exec/IndexVertexScanNode.h | 1 - src/storage/test/IndexTest.cpp | 1 - src/storage/test/IndexTestUtil.h | 1 - 19 files changed, 19 deletions(-) diff --git a/src/storage/ExprVisitorBase.cpp b/src/storage/ExprVisitorBase.cpp index e921af37c24..082a5f292bd 100644 --- a/src/storage/ExprVisitorBase.cpp +++ b/src/storage/ExprVisitorBase.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/ExprVisitorBase.h" namespace nebula { diff --git a/src/storage/ExprVisitorBase.h b/src/storage/ExprVisitorBase.h index d085a902e7a..c8a60a39212 100644 --- a/src/storage/ExprVisitorBase.h +++ b/src/storage/ExprVisitorBase.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index 6421a2e5649..568286cc25b 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/exec/IndexDedupNode.h" namespace nebula { diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index c6eaca75c99..d3967348024 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once #include "common/datatypes/DataSet.h" diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index 7f070615910..8aca93625d7 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/exec/IndexEdgeScanNode.h" diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index 9c58aabb66b..ea7faa64439 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once #include "common/base/Base.h" diff --git a/src/storage/exec/IndexLimitNode.cpp b/src/storage/exec/IndexLimitNode.cpp index 8bf1658f0ee..1173845c4c6 100644 --- a/src/storage/exec/IndexLimitNode.cpp +++ b/src/storage/exec/IndexLimitNode.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/exec/IndexLimitNode.h" namespace nebula { diff --git a/src/storage/exec/IndexLimitNode.h b/src/storage/exec/IndexLimitNode.h index 2d72c2b8e98..4f6a8d6da81 100644 --- a/src/storage/exec/IndexLimitNode.h +++ b/src/storage/exec/IndexLimitNode.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once #include "folly/Likely.h" diff --git a/src/storage/exec/IndexNode.cpp b/src/storage/exec/IndexNode.cpp index 3f5e288da6d..e891426eeec 100644 --- a/src/storage/exec/IndexNode.cpp +++ b/src/storage/exec/IndexNode.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/exec/IndexNode.h" diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index 7d23deb7698..df287f97311 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once #include "common/base/ErrorOr.h" diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp index 65a13e7fcc1..5bd4c63ed15 100644 --- a/src/storage/exec/IndexProjectionNode.cpp +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/exec/IndexProjectionNode.h" namespace nebula { diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index c5af9c41e97..07643dd2201 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index 4408c8624a1..628f41192fb 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/exec/IndexScanNode.h" diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index f5e42cfe67d..32715a847a5 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/exec/IndexSelectionNode.h" namespace nebula { diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 49a19b61f81..257431e5745 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index 529854f8c4e..cad2722ca0a 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include "storage/exec/IndexVertexScanNode.h" diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index 0c8b5baccf2..ad7ba4623e1 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 0d80d6dad93..4139dd9b2e9 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -1,7 +1,6 @@ /* Copyright (c) 2018 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #include diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index fa07a5297b1..528d28d13ef 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -1,7 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ #pragma once From d4a6512cc543b9338a337c725b886447fc252efa Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Mon, 8 Nov 2021 10:13:28 +0800 Subject: [PATCH 37/38] Modify test to avoid the bug mentioned in issue3191 --- tests/tck/features/lookup/LookUp.feature | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/tck/features/lookup/LookUp.feature b/tests/tck/features/lookup/LookUp.feature index 6afca5f523f..ce6700e6697 100644 --- a/tests/tck/features/lookup/LookUp.feature +++ b/tests/tck/features/lookup/LookUp.feature @@ -840,7 +840,6 @@ Feature: LookUpTest_Vid_String "104":("yyy", 28), "105":("zzz", 21), "106":("kkk", 21), - "121":("Useless", 60), "121":("Useless", 20); INSERT VERTEX team(name) From 2ff061720ebf50d3be23f52a8b9788f1a5bd9903 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Mon, 8 Nov 2021 10:22:15 +0800 Subject: [PATCH 38/38] modify license error --- src/storage/ExprVisitorBase.cpp | 2 +- src/storage/ExprVisitorBase.h | 2 +- src/storage/exec/IndexDedupNode.cpp | 2 +- src/storage/exec/IndexDedupNode.h | 2 +- src/storage/exec/IndexEdgeScanNode.cpp | 2 +- src/storage/exec/IndexEdgeScanNode.h | 2 +- src/storage/exec/IndexLimitNode.cpp | 2 +- src/storage/exec/IndexLimitNode.h | 2 +- src/storage/exec/IndexNode.cpp | 2 +- src/storage/exec/IndexNode.h | 2 +- src/storage/exec/IndexProjectionNode.cpp | 2 +- src/storage/exec/IndexProjectionNode.h | 2 +- src/storage/exec/IndexScanNode.cpp | 2 +- src/storage/exec/IndexSelectionNode.cpp | 2 +- src/storage/exec/IndexSelectionNode.h | 2 +- src/storage/exec/IndexVertexScanNode.cpp | 2 +- src/storage/exec/IndexVertexScanNode.h | 2 +- src/storage/test/IndexTest.cpp | 2 +- src/storage/test/IndexTestUtil.h | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/storage/ExprVisitorBase.cpp b/src/storage/ExprVisitorBase.cpp index 082a5f292bd..9f56e8a977b 100644 --- a/src/storage/ExprVisitorBase.cpp +++ b/src/storage/ExprVisitorBase.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include "storage/ExprVisitorBase.h" namespace nebula { diff --git a/src/storage/ExprVisitorBase.h b/src/storage/ExprVisitorBase.h index c8a60a39212..015f02d5aca 100644 --- a/src/storage/ExprVisitorBase.h +++ b/src/storage/ExprVisitorBase.h @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #pragma once diff --git a/src/storage/exec/IndexDedupNode.cpp b/src/storage/exec/IndexDedupNode.cpp index 568286cc25b..c27faf11eec 100644 --- a/src/storage/exec/IndexDedupNode.cpp +++ b/src/storage/exec/IndexDedupNode.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include "storage/exec/IndexDedupNode.h" namespace nebula { diff --git a/src/storage/exec/IndexDedupNode.h b/src/storage/exec/IndexDedupNode.h index d3967348024..8025c08124b 100644 --- a/src/storage/exec/IndexDedupNode.h +++ b/src/storage/exec/IndexDedupNode.h @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #pragma once #include "common/datatypes/DataSet.h" diff --git a/src/storage/exec/IndexEdgeScanNode.cpp b/src/storage/exec/IndexEdgeScanNode.cpp index 8aca93625d7..e9242e6e556 100644 --- a/src/storage/exec/IndexEdgeScanNode.cpp +++ b/src/storage/exec/IndexEdgeScanNode.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include "storage/exec/IndexEdgeScanNode.h" diff --git a/src/storage/exec/IndexEdgeScanNode.h b/src/storage/exec/IndexEdgeScanNode.h index ea7faa64439..d21ef95faa5 100644 --- a/src/storage/exec/IndexEdgeScanNode.h +++ b/src/storage/exec/IndexEdgeScanNode.h @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #pragma once #include "common/base/Base.h" diff --git a/src/storage/exec/IndexLimitNode.cpp b/src/storage/exec/IndexLimitNode.cpp index 1173845c4c6..df5afed4130 100644 --- a/src/storage/exec/IndexLimitNode.cpp +++ b/src/storage/exec/IndexLimitNode.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include "storage/exec/IndexLimitNode.h" namespace nebula { diff --git a/src/storage/exec/IndexLimitNode.h b/src/storage/exec/IndexLimitNode.h index 4f6a8d6da81..8811ec77a87 100644 --- a/src/storage/exec/IndexLimitNode.h +++ b/src/storage/exec/IndexLimitNode.h @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #pragma once #include "folly/Likely.h" diff --git a/src/storage/exec/IndexNode.cpp b/src/storage/exec/IndexNode.cpp index e891426eeec..0ce1ab3f401 100644 --- a/src/storage/exec/IndexNode.cpp +++ b/src/storage/exec/IndexNode.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include "storage/exec/IndexNode.h" diff --git a/src/storage/exec/IndexNode.h b/src/storage/exec/IndexNode.h index df287f97311..0db70ec8030 100644 --- a/src/storage/exec/IndexNode.h +++ b/src/storage/exec/IndexNode.h @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #pragma once #include "common/base/ErrorOr.h" diff --git a/src/storage/exec/IndexProjectionNode.cpp b/src/storage/exec/IndexProjectionNode.cpp index 5bd4c63ed15..946edafb418 100644 --- a/src/storage/exec/IndexProjectionNode.cpp +++ b/src/storage/exec/IndexProjectionNode.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include "storage/exec/IndexProjectionNode.h" namespace nebula { diff --git a/src/storage/exec/IndexProjectionNode.h b/src/storage/exec/IndexProjectionNode.h index 07643dd2201..736971620e9 100644 --- a/src/storage/exec/IndexProjectionNode.h +++ b/src/storage/exec/IndexProjectionNode.h @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #pragma once diff --git a/src/storage/exec/IndexScanNode.cpp b/src/storage/exec/IndexScanNode.cpp index 628f41192fb..33f344844ac 100644 --- a/src/storage/exec/IndexScanNode.cpp +++ b/src/storage/exec/IndexScanNode.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include "storage/exec/IndexScanNode.h" diff --git a/src/storage/exec/IndexSelectionNode.cpp b/src/storage/exec/IndexSelectionNode.cpp index 32715a847a5..b69e6bbfcae 100644 --- a/src/storage/exec/IndexSelectionNode.cpp +++ b/src/storage/exec/IndexSelectionNode.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include "storage/exec/IndexSelectionNode.h" namespace nebula { diff --git a/src/storage/exec/IndexSelectionNode.h b/src/storage/exec/IndexSelectionNode.h index 257431e5745..a134ecf890d 100644 --- a/src/storage/exec/IndexSelectionNode.h +++ b/src/storage/exec/IndexSelectionNode.h @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #pragma once diff --git a/src/storage/exec/IndexVertexScanNode.cpp b/src/storage/exec/IndexVertexScanNode.cpp index cad2722ca0a..499a4d59d8a 100644 --- a/src/storage/exec/IndexVertexScanNode.cpp +++ b/src/storage/exec/IndexVertexScanNode.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include "storage/exec/IndexVertexScanNode.h" diff --git a/src/storage/exec/IndexVertexScanNode.h b/src/storage/exec/IndexVertexScanNode.h index ad7ba4623e1..fea56a19adb 100644 --- a/src/storage/exec/IndexVertexScanNode.h +++ b/src/storage/exec/IndexVertexScanNode.h @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #pragma once diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 4139dd9b2e9..6c4e34ad122 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -1,6 +1,6 @@ /* Copyright (c) 2018 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #include diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index 528d28d13ef..73e457e6167 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -1,6 +1,6 @@ /* Copyright (c) 2021 vesoft inc. All rights reserved. * - * This source code is licensed under Apache 2.0 License, + * This source code is licensed under Apache 2.0 License. */ #pragma once