diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index 4ecfe498f40..676b923adbb 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -2394,7 +2394,6 @@ folly::Future> MetaClient::heartbeat() { req.set_host(options_.localHost_); req.set_role(options_.role_); req.set_git_info_sha(options_.gitInfoSHA_); - req.set_version(getOriginVersion()); if (options_.role_ == cpp2::HostRole::STORAGE) { if (options_.clusterId_.load() == 0) { options_.clusterId_ = FileBasedClusterIdMan::getClusterIdFromFile(FLAGS_cluster_id_path); @@ -3505,6 +3504,7 @@ bool MetaClient::checkIsPlanKilled(SessionID sessionId, ExecutionPlanID planId) Status MetaClient::verifyVersion() { auto req = cpp2::VerifyClientVersionReq(); + req.set_host(options_.localHost_); folly::Promise> promise; auto future = promise.getFuture(); getResponse( diff --git a/src/common/utils/MetaKeyUtils.cpp b/src/common/utils/MetaKeyUtils.cpp index 48ed50172c1..88f4c5a5b19 100644 --- a/src/common/utils/MetaKeyUtils.cpp +++ b/src/common/utils/MetaKeyUtils.cpp @@ -20,6 +20,7 @@ namespace nebula { static const std::unordered_map> systemTableMaps = { {"users", {"__users__", true}}, {"hosts", {"__hosts__", false}}, + {"versions", {"__versions__", false}}, {"snapshots", {"__snapshots__", false}}, {"configs", {"__configs__", true}}, {"groups", {"__groups__", true}}, @@ -58,6 +59,7 @@ static const std::unordered_map< static const std::string kSpacesTable = tableMaps.at("spaces").first; // NOLINT static const std::string kPartsTable = tableMaps.at("parts").first; // NOLINT static const std::string kHostsTable = systemTableMaps.at("hosts").first; // NOLINT +static const std::string kVersionsTable = systemTableMaps.at("versions").first; // NOLINT static const std::string kTagsTable = tableMaps.at("tags").first; // NOLINT static const std::string kEdgesTable = tableMaps.at("edges").first; // NOLINT static const std::string kIndexesTable = tableMaps.at("indexes").first; // NOLINT @@ -269,6 +271,26 @@ HostAddr MetaKeyUtils::parseHostKeyV2(folly::StringPiece key) { return MetaKeyUtils::deserializeHostAddr(key); } +std::string MetaKeyUtils::versionKey(const HostAddr& h) { + std::string key; + key.append(kVersionsTable.data(), kVersionsTable.size()) + .append(MetaKeyUtils::serializeHostAddr(h)); + return key; +} + +std::string MetaKeyUtils::versionVal(const std::string& version) { + std::string val; + auto versionLen = version.size(); + val.reserve(sizeof(int64_t) + versionLen); + val.append(reinterpret_cast(&version), sizeof(int64_t)).append(version); + return val; +} + +std::string MetaKeyUtils::parseVersion(folly::StringPiece val) { + auto len = *reinterpret_cast(val.data()); + return val.subpiece(sizeof(size_t), len).str(); +} + std::string MetaKeyUtils::leaderKey(std::string addr, Port port) { LOG(ERROR) << "deprecated function\n" << boost::stacktrace::stacktrace(); return leaderKeyV2(addr, port); diff --git a/src/common/utils/MetaKeyUtils.h b/src/common/utils/MetaKeyUtils.h index ee698d3dffa..7ebce2d009a 100644 --- a/src/common/utils/MetaKeyUtils.h +++ b/src/common/utils/MetaKeyUtils.h @@ -122,6 +122,12 @@ class MetaKeyUtils final { static HostAddr parseHostKeyV2(folly::StringPiece key); + static std::string versionKey(const HostAddr& h); + + static std::string versionVal(const std::string& version); + + static std::string parseVersion(folly::StringPiece val); + static std::string leaderKey(std::string ip, Port port); static std::string leaderKeyV2(std::string addr, Port port); diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift index 670d0337668..f9bd4e1f92d 100644 --- a/src/interface/meta.thrift +++ b/src/interface/meta.thrift @@ -565,9 +565,7 @@ struct HBReq { 4: optional map> (cpp.template = "std::unordered_map") leader_partIds; 5: binary git_info_sha, - // version of binary - 6: optional binary version, - 7: optional map + 6: optional map (cpp.template = "std::unordered_map")> (cpp.template = "std::unordered_map") disk_parts; } @@ -1113,6 +1111,7 @@ struct VerifyClientVersionResp { struct VerifyClientVersionReq { 1: required binary version = common.version; + 2: common.HostAddr host; } service MetaService { diff --git a/src/meta/ActiveHostsMan.h b/src/meta/ActiveHostsMan.h index 2412c351d9b..7c1e4e9c26f 100644 --- a/src/meta/ActiveHostsMan.h +++ b/src/meta/ActiveHostsMan.h @@ -33,8 +33,6 @@ struct HostInfo { int64_t lastHBTimeInMilliSec_ = 0; cpp2::HostRole role_{cpp2::HostRole::UNKNOWN}; std::string gitInfoSha_; - // version of binary - folly::Optional version_; static HostInfo decode(const folly::StringPiece& data) { if (data.size() == sizeof(int64_t)) { @@ -71,12 +69,6 @@ struct HostInfo { if (!info.gitInfoSha_.empty()) { encode.append(info.gitInfoSha_.data(), len); } - - if (info.version_.has_value()) { - len = info.version_.value().size(); - encode.append(reinterpret_cast(&len), sizeof(std::size_t)); - encode.append(info.version_.value().data(), len); - } return encode; } @@ -104,20 +96,6 @@ struct HostInfo { } info.gitInfoSha_ = std::string(data.data() + offset, len); - offset += len; - - if (offset == data.size()) { - return info; - } - - len = *reinterpret_cast(data.data() + offset); - offset += sizeof(size_t); - - if (offset + len > data.size()) { - FLOG_FATAL("decode out of range, offset=%zu, actual=%zu", offset, data.size()); - } - - info.version_ = std::string(data.data() + offset, len); return info; } }; diff --git a/src/meta/processors/admin/HBProcessor.cpp b/src/meta/processors/admin/HBProcessor.cpp index beebe9b4810..04e806cb320 100644 --- a/src/meta/processors/admin/HBProcessor.cpp +++ b/src/meta/processors/admin/HBProcessor.cpp @@ -67,9 +67,6 @@ void HBProcessor::process(const cpp2::HBReq& req) { } HostInfo info(time::WallClock::fastNowInMilliSec(), req.get_role(), req.get_git_info_sha()); - if (req.version_ref().has_value()) { - info.version_ = *req.version_ref(); - } if (req.leader_partIds_ref().has_value()) { ret = ActiveHostsMan::updateHostInfo(kvstore_, host, info, &*req.leader_partIds_ref()); } else { diff --git a/src/meta/processors/admin/VerifyClientVersionProcessor.cpp b/src/meta/processors/admin/VerifyClientVersionProcessor.cpp index 314ab2e9ec8..d7328b5b145 100644 --- a/src/meta/processors/admin/VerifyClientVersionProcessor.cpp +++ b/src/meta/processors/admin/VerifyClientVersionProcessor.cpp @@ -25,6 +25,12 @@ void VerifyClientVersionProcessor::process(const cpp2::VerifyClientVersionReq& r req.get_version().c_str(), FLAGS_client_white_list.c_str())); } else { + auto host = req.get_host(); + auto versionKey = MetaKeyUtils::versionKey(host); + auto versionVal = MetaKeyUtils::versionVal(req.get_version().c_str()); + std::vector versionData; + versionData.emplace_back(std::move(versionKey), std::move(versionVal)); + doSyncPut(versionData); resp_.set_code(nebula::cpp2::ErrorCode::SUCCEEDED); } onFinished(); diff --git a/src/meta/processors/index/DropEdgeIndexProcessor.cpp b/src/meta/processors/index/DropEdgeIndexProcessor.cpp index fdbb57ce803..02f8aa9d661 100644 --- a/src/meta/processors/index/DropEdgeIndexProcessor.cpp +++ b/src/meta/processors/index/DropEdgeIndexProcessor.cpp @@ -39,6 +39,27 @@ void DropEdgeIndexProcessor::process(const cpp2::DropEdgeIndexReq& req) { keys.emplace_back(MetaKeyUtils::indexIndexKey(spaceID, indexName)); keys.emplace_back(MetaKeyUtils::indexKey(spaceID, edgeIndexID)); + auto indexItemRet = doGet(keys.back()); + if (!nebula::ok(indexItemRet)) { + auto retCode = nebula::error(indexItemRet); + if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + retCode = nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND; + } + LOG(ERROR) << "Drop Edge Index Failed: SpaceID " << spaceID << " Index Name: " << indexName + << " error: " << apache::thrift::util::enumNameSafe(retCode); + handleErrorCode(retCode); + onFinished(); + return; + } + + auto item = MetaKeyUtils::parseIndex(nebula::value(indexItemRet)); + if (item.get_schema_id().getType() != nebula::cpp2::SchemaID::Type::edge_type) { + LOG(ERROR) << "Drop Edge Index Failed: Index Name " << indexName << " is not EdgeIndex"; + resp_.set_code(nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND); + onFinished(); + return; + } + LOG(INFO) << "Drop Edge Index " << indexName; resp_.set_id(to(edgeIndexID, EntryType::INDEX)); doSyncMultiRemoveAndUpdate(std::move(keys)); diff --git a/src/meta/processors/index/DropTagIndexProcessor.cpp b/src/meta/processors/index/DropTagIndexProcessor.cpp index 4c392277645..b079a69aa71 100644 --- a/src/meta/processors/index/DropTagIndexProcessor.cpp +++ b/src/meta/processors/index/DropTagIndexProcessor.cpp @@ -38,6 +38,27 @@ void DropTagIndexProcessor::process(const cpp2::DropTagIndexReq& req) { keys.emplace_back(MetaKeyUtils::indexIndexKey(spaceID, indexName)); keys.emplace_back(MetaKeyUtils::indexKey(spaceID, tagIndexID)); + auto indexItemRet = doGet(keys.back()); + if (!nebula::ok(indexItemRet)) { + auto retCode = nebula::error(indexItemRet); + if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + retCode = nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND; + } + LOG(ERROR) << "Drop Tag Index Failed: SpaceID " << spaceID << " Index Name: " << indexName + << " error: " << apache::thrift::util::enumNameSafe(retCode); + handleErrorCode(retCode); + onFinished(); + return; + } + + auto item = MetaKeyUtils::parseIndex(nebula::value(indexItemRet)); + if (item.get_schema_id().getType() != nebula::cpp2::SchemaID::Type::tag_id) { + LOG(ERROR) << "Drop Tag Index Failed: Index Name " << indexName << " is not TagIndex"; + resp_.set_code(nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND); + onFinished(); + return; + } + LOG(INFO) << "Drop Tag Index " << indexName; resp_.set_id(to(tagIndexID, EntryType::INDEX)); doSyncMultiRemoveAndUpdate(std::move(keys)); diff --git a/src/meta/processors/parts/ListHostsProcessor.cpp b/src/meta/processors/parts/ListHostsProcessor.cpp index ef1e115f249..a2c425203bb 100644 --- a/src/meta/processors/parts/ListHostsProcessor.cpp +++ b/src/meta/processors/parts/ListHostsProcessor.cpp @@ -124,9 +124,14 @@ nebula::cpp2::ErrorCode ListHostsProcessor::allHostsWithStatus(cpp2::HostRole ro item.set_role(info.role_); item.set_git_info_sha(info.gitInfoSha_); - if (info.version_.has_value()) { - item.set_version(info.version_.value()); + + auto versionKey = MetaKeyUtils::versionKey(item.get_hostAddr()); + auto versionRet = doGet(versionKey); + if (nebula::ok(versionRet)) { + auto versionVal = MetaKeyUtils::parseVersion(value(versionRet)); + item.set_version(versionVal); } + if (now - info.lastHBTimeInMilliSec_ < FLAGS_removed_threshold_sec * 1000) { // If meta didn't receive heartbeat with 2 periods, regard hosts as // offline. Same as ActiveHostsMan::getActiveHosts diff --git a/src/meta/test/CMakeLists.txt b/src/meta/test/CMakeLists.txt index cf8bc5567f6..e8591bae05a 100644 --- a/src/meta/test/CMakeLists.txt +++ b/src/meta/test/CMakeLists.txt @@ -254,3 +254,18 @@ nebula_add_test( wangle gtest ) + +nebula_add_test( + NAME + verify_client_version_test + SOURCES + VerifyClientVersionTest.cpp + OBJECTS + ${meta_test_deps} + LIBRARIES + ${ROCKSDB_LIBRARIES} + ${THRIFT_LIBRARIES} + ${PROXYGEN_LIBRARIES} + wangle + gtest +) diff --git a/src/meta/test/VerifyClientVersionTest.cpp b/src/meta/test/VerifyClientVersionTest.cpp new file mode 100644 index 00000000000..9f72f559e00 --- /dev/null +++ b/src/meta/test/VerifyClientVersionTest.cpp @@ -0,0 +1,75 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/fs/TempDir.h" +#include "meta/processors/admin/HBProcessor.h" +#include "meta/processors/admin/VerifyClientVersionProcessor.h" +#include "meta/processors/parts/ListHostsProcessor.h" +#include "meta/test/TestUtils.h" + +namespace nebula { +namespace meta { + +TEST(VerifyClientVersionTest, VersionTest) { + fs::TempDir rootPath("/tmp/VersionTest.XXXXXX"); + std::unique_ptr kv(MockCluster::initMetaKV(rootPath.path())); + { + auto req = cpp2::VerifyClientVersionReq(); + req.set_version("1.0.1"); + auto* processor = VerifyClientVersionProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_CLIENT_SERVER_INCOMPATIBLE, resp.get_code()); + } + { + for (auto i = 0; i < 5; i++) { + auto req = cpp2::VerifyClientVersionReq(); + req.set_host(HostAddr(std::to_string(i), i)); + auto* processor = VerifyClientVersionProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + } + { + const ClusterID kClusterId = 10; + for (auto i = 0; i < 5; i++) { + auto req = cpp2::HBReq(); + req.set_role(cpp2::HostRole::GRAPH); + req.set_host(HostAddr(std::to_string(i), i)); + req.set_cluster_id(kClusterId); + auto* processor = HBProcessor::instance(kv.get(), nullptr, kClusterId); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + } + { + auto req = cpp2::ListHostsReq(); + req.set_type(cpp2::ListHostType::GRAPH); + auto* processor = ListHostsProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + ASSERT_EQ(resp.get_hosts().size(), 5); + } +} + +} // namespace meta +} // namespace nebula + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + return RUN_ALL_TESTS(); +} diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 04f1bf1699d..0ea8ede2672 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -213,7 +213,7 @@ static constexpr size_t kCommentLengthLimit = 256; %token BOOL %token INTEGER %token DOUBLE -%token STRING VARIABLE LABEL IPV4 +%token STRING VARIABLE LABEL IPV4 CHINESE_LABEL %type name_label unreserved_keyword predicate_name %type expression @@ -406,6 +406,7 @@ static constexpr size_t kCommentLengthLimit = 256; name_label : LABEL { $$ = $1; } + | CHINESE_LABEL { $$ = $1; } | unreserved_keyword { $$ = $1; } ; diff --git a/src/parser/scanner.lex b/src/parser/scanner.lex index 007a1f93153..4fe773332d2 100644 --- a/src/parser/scanner.lex +++ b/src/parser/scanner.lex @@ -43,7 +43,11 @@ HEX ([0-9a-fA-F]) OCT ([0-7]) IP_OCTET ([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]) - +U [\x80-\xbf] +U2 [\xc2-\xdf] +U3 [\xe0-\xef] +U4 [\xf0-\xf4] +CHINESE_LABEL ({U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U})+ %% @@ -467,6 +471,17 @@ IP_OCTET ([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]) // Must match /* */ throw GraphParser::syntax_error(*yylloc, "unterminated comment"); } +\`{CHINESE_LABEL}\` { + yylval->strval = new std::string(yytext + 1, yyleng - 2); + if (yylval->strval->size() > MAX_STRING) { + auto error = "Out of range of the LABEL length, " + "the max length of LABEL is " + + std::to_string(MAX_STRING) + ":"; + delete yylval->strval; + throw GraphParser::syntax_error(*yylloc, error); + } + return TokenType::CHINESE_LABEL; + } . { /** * Any other unmatched byte sequences will get us here, diff --git a/tests/tck/features/index/TagEdgeIndex.feature b/tests/tck/features/index/TagEdgeIndex.feature index 2d958badd44..40a12fbbd59 100644 --- a/tests/tck/features/index/TagEdgeIndex.feature +++ b/tests/tck/features/index/TagEdgeIndex.feature @@ -162,6 +162,12 @@ Feature: tag and edge index tests from pytest Then the result should be, in any order: | Tag Index Name | Create Tag Index | | 'multi_tag_index' | 'CREATE TAG INDEX `multi_tag_index` ON `tag_1` (\n `col2`,\n `col3`\n)' | + # Check if check tag/edge type before drop index + When executing query: + """ + DROP EDGE INDEX multi_tag_index + """ + Then an ExecutionError should be raised at runtime. When executing query: """ DROP TAG INDEX multi_tag_index @@ -405,6 +411,12 @@ Feature: tag and edge index tests from pytest Then the result should be, in any order: | Edge Index Name | Create Edge Index | | 'multi_edge_index' | 'CREATE EDGE INDEX `multi_edge_index` ON `edge_1` (\n `col2`,\n `col3`\n)' | + # Check if check tag/edge type before drop index + When executing query: + """ + DROP TAG INDEX multi_edge_index + """ + Then an ExecutionError should be raised at runtime. # Check if show create edge index works well When executing query: """ diff --git a/tests/tck/features/schema/Schema.feature b/tests/tck/features/schema/Schema.feature index b203ea87b62..f30f79a6d27 100644 --- a/tests/tck/features/schema/Schema.feature +++ b/tests/tck/features/schema/Schema.feature @@ -769,6 +769,69 @@ Feature: Insert string vid of vertex and edge ALTER EDGE edge_not_null_default1 CHANGE (name FIXED_STRING(10) DEFAULT 10) """ Then a ExecutionError should be raised at runtime: Invalid param! + # chinese tag without quote mark + When executing query: + """ + CREATE TAG 队伍( 名字 string); + """ + Then a SyntaxError should be raised at runtime: + # chinese tag and chinese prop + When executing query: + """ + CREATE TAG `队伍`(`名字` string); + """ + Then the execution should be successful + # show chinese tags + When executing query: + """ + SHOW TAGS + """ + Then the result should contain: + | Name | + | "队伍" | + # alter chinese tag + When executing query: + """ + ALTER TAG `队伍` ADD (`类别` string); + """ + Then the execution should be successful + # desc chinese tag + When executing query: + """ + DESCRIBE TAG `队伍` + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + | "名字" | "string" | "YES" | EMPTY | EMPTY | + | "类别" | "string" | "YES" | EMPTY | EMPTY | + # chinese edge and chinese prop + When executing query: + """ + CREATE EDGE `服役`(); + """ + Then the execution should be successful + # show chinese edge + When executing query: + """ + SHOW EDGES; + """ + Then the result should contain: + | Name | + | "服役" | + # alter chinese edge + When executing query: + """ + ALTER EDGE `服役` ADD (`时间` timestamp); + """ + Then the execution should be successful + # desc chinese edge + When executing query: + """ + DESCRIBE EDGE `服役` + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + | "时间" | "timestamp" | "YES" | EMPTY | EMPTY | When executing query: """ DROP SPACE issue2009;