From df2052611fdd812fc2011ec6a07595659e913004 Mon Sep 17 00:00:00 2001 From: "darion.yaphet" Date: Fri, 24 Dec 2021 10:16:02 +0800 Subject: [PATCH] Support meta upgrade v3 --- src/daemons/CMakeLists.txt | 1 + src/daemons/MetaDaemon.cpp | 32 +++--- src/meta/CMakeLists.txt | 2 + src/meta/MetaVersionMan.cpp | 92 ++++++++++++++- src/meta/MetaVersionMan.h | 9 +- src/meta/upgrade/CMakeLists.txt | 3 + src/meta/upgrade/MetaDataUpgrade.cpp | 128 ++++++++++++++++++++- src/meta/upgrade/MetaDataUpgrade.h | 10 +- src/meta/upgrade/v2/CMakeLists.txt | 24 ++++ src/meta/upgrade/v2/MetaServiceUtilsV2.cpp | 32 ++++++ src/meta/upgrade/v2/MetaServiceUtilsV2.h | 33 ++++++ src/meta/upgrade/v2/meta.thrift | 81 +++++++++++++ 12 files changed, 420 insertions(+), 27 deletions(-) create mode 100644 src/meta/upgrade/v2/CMakeLists.txt create mode 100644 src/meta/upgrade/v2/MetaServiceUtilsV2.cpp create mode 100644 src/meta/upgrade/v2/MetaServiceUtilsV2.h create mode 100644 src/meta/upgrade/v2/meta.thrift diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 292c3936d78..33dcf0d816e 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -91,6 +91,7 @@ nebula_add_executable( $ $ $ + $ $ ${common_deps} ${storage_meta_deps} diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp index 5aa7521107b..76b264ae2a3 100644 --- a/src/daemons/MetaDaemon.cpp +++ b/src/daemons/MetaDaemon.cpp @@ -140,26 +140,24 @@ std::unique_ptr initKV(std::vector p LOG(ERROR) << "Meta version is invalid"; return nullptr; } else if (version == nebula::meta::MetaVersion::V1) { - if (leader == localhost) { - LOG(INFO) << "I am leader, begin upgrade meta data"; - // need to upgrade the v1.0 meta data format to v2.0 meta data format - auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(kvstore.get()); - if (!ret.ok()) { - LOG(ERROR) << ret; - return nullptr; - } - } else { - LOG(INFO) << "I am follower, wait for leader to sync upgrade"; - while (version != nebula::meta::MetaVersion::V2) { - VLOG(1) << "Waiting for leader to upgrade"; - sleep(1); - version = nebula::meta::MetaVersionMan::getMetaVersionFromKV(kvstore.get()); - } + // need to upgrade the v1.0 meta data format to v2.0 + auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(kvstore.get()); + if (!ret.ok()) { + LOG(ERROR) << ret; + return nullptr; + } + } else if (version == nebula::meta::MetaVersion::V2) { + // need to upgrade the v2.0 meta data format to v3.0 + auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(kvstore.get()); + if (!ret.ok()) { + LOG(ERROR) << ret; + return nullptr; } } - if (leader == localhost) { - nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get()); + if (!nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get())) { + LOG(ERROR) << "Meta Version save failed"; + return nullptr; } LOG(INFO) << "Nebula store init succeeded, clusterId " << gClusterId; diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 77449574b61..6581d7b2ce9 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -106,12 +106,14 @@ nebula_add_library( add_dependencies( meta_version_man_obj meta_v1_thrift_obj + meta_v2_thrift_obj ) set(meta_test_deps $ $ $ + $ $ $ $ diff --git a/src/meta/MetaVersionMan.cpp b/src/meta/MetaVersionMan.cpp index fbe5a75db8b..23fa5986f90 100644 --- a/src/meta/MetaVersionMan.cpp +++ b/src/meta/MetaVersionMan.cpp @@ -9,6 +9,7 @@ #include "meta/processors/job/JobUtils.h" #include "meta/upgrade/MetaDataUpgrade.h" #include "meta/upgrade/v1/MetaServiceUtilsV1.h" +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" DEFINE_bool(null_type, true, "set schema to support null type"); DEFINE_bool(print_info, false, "enable to print the rewrite data"); @@ -51,10 +52,10 @@ MetaVersion MetaVersionMan::getVersionByHost(kvstore::KVStore* kv) { // static bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv) { CHECK_NOTNULL(kv); - auto v2 = MetaVersion::V2; + auto v3 = MetaVersion::V3; std::vector data; data.emplace_back(kMetaVersionKey, - std::string(reinterpret_cast(&v2), sizeof(MetaVersion))); + std::string(reinterpret_cast(&v3), sizeof(MetaVersion))); bool ret = true; folly::Baton baton; kv->asyncMultiPut( @@ -63,7 +64,7 @@ bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv) { LOG(ERROR) << "Put failed, error: " << static_cast(code); ret = false; } else { - LOG(INFO) << "Write meta version 2 succeeds"; + LOG(INFO) << "Write meta version 3 succeeds"; } baton.post(); }); @@ -80,7 +81,28 @@ Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { LOG(ERROR) << "Create snapshot failed: " << snapshot; return Status::Error("Create snapshot failed"); } - auto status = doUpgrade(kv); + auto status = doUpgradeV1ToV2(kv); + if (!status.ok()) { + // rollback by snapshot + return status; + } + // delete snapshot file + auto dmRet = kv->dropCheckpoint(kDefaultSpaceId, snapshot); + if (dmRet != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Delete snapshot: " << snapshot << " failed, You need to delete it manually"; + } + return Status::OK(); +} + +Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVStore* kv) { + CHECK_NOTNULL(kv); + auto snapshot = folly::format("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()).str(); + auto meteRet = kv->createCheckpoint(kDefaultSpaceId, snapshot); + if (meteRet.isLeftType()) { + LOG(ERROR) << "Create snapshot failed: " << snapshot; + return Status::Error("Create snapshot failed"); + } + auto status = doUpgradeV2ToV3(kv); if (!status.ok()) { // rollback by snapshot return status; @@ -94,7 +116,7 @@ Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { } // static -Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { +Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { MetaDataUpgrade upgrader(kv); { // kSpacesTable @@ -105,7 +127,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { Status status = Status::OK(); while (iter->valid()) { if (FLAGS_print_info) { - upgrader.printSpaces(iter->val()); + upgrader.printSpacesV1(iter->val()); } status = upgrader.rewriteSpaces(iter->key(), iter->val()); if (!status.ok()) { @@ -158,6 +180,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } + { // kLeadersTable auto prefix = nebula::meta::v1::kLeadersTable; @@ -178,6 +201,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } + { // kTagsTable auto prefix = nebula::meta::v1::kTagsTable; @@ -334,5 +358,61 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } +Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVStore* kv) { + MetaDataUpgrade upgrader(kv); + // Step 1: Upgrade HeartBeat into machine list + { + const auto& prefix = MetaKeyUtils::hostPrefix(); + std::unique_ptr iter; + auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get active hosts failed"; + return Status::Error("Get hosts failed"); + } + + std::vector data; + while (iter->valid()) { + // Save the machine information + auto host = MetaKeyUtils::parseHostKey(iter->key()); + auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); + data.emplace_back(std::move(machineKey), ""); + + // Save the zone information + auto zoneName = folly::stringPrintf("default_zone_%s_%d", host.host.c_str(), host.port); + auto zoneKey = MetaKeyUtils::zoneKey(std::move(zoneName)); + auto zoneVal = MetaKeyUtils::zoneVal({host}); + data.emplace_back(std::move(zoneKey), std::move(zoneVal)); + iter->next(); + } + } + + // Step 2: Update Create space properties about Group + { + const auto& prefix = MetaKeyUtils::spacePrefix(); + std::unique_ptr iter; + auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get spaces failed"; + return Status::Error("Get spaces failed"); + } + + while (iter->valid()) { + if (FLAGS_print_info) { + upgrader.printSpacesV2(iter->val()); + } + auto spaceProperties = meta::v2::MetaServiceUtilsV2::parseSpace(iter->val()); + if (spaceProperties.group_name_ref().has_value()) { + Status status = upgrader.rewriteSpacesV2ToV3(iter->key(), iter->val()); + if (!status.ok()) { + LOG(ERROR) << status; + return status; + } + } + iter->next(); + } + } + return Status::OK(); +} + } // namespace meta } // namespace nebula diff --git a/src/meta/MetaVersionMan.h b/src/meta/MetaVersionMan.h index 59737175430..1c10ec3701c 100644 --- a/src/meta/MetaVersionMan.h +++ b/src/meta/MetaVersionMan.h @@ -17,6 +17,7 @@ enum class MetaVersion { UNKNOWN = 0, V1 = 1, V2 = 2, + V3 = 3, }; /** @@ -32,13 +33,17 @@ class MetaVersionMan final { static Status updateMetaV1ToV2(kvstore::KVStore* kv); + static Status updateMetaV2ToV3(kvstore::KVStore* kv); + private: static MetaVersion getVersionByHost(kvstore::KVStore* kv); - static Status doUpgrade(kvstore::KVStore* kv); + static Status doUpgradeV1ToV2(kvstore::KVStore* kv); + + static Status doUpgradeV2ToV3(kvstore::KVStore* kv); }; } // namespace meta } // namespace nebula -#endif // META_ROOTUSERMAN_H_ +#endif // META_METAVERSIONMAN_H_ diff --git a/src/meta/upgrade/CMakeLists.txt b/src/meta/upgrade/CMakeLists.txt index 7b895c31e36..b0700952b86 100644 --- a/src/meta/upgrade/CMakeLists.txt +++ b/src/meta/upgrade/CMakeLists.txt @@ -6,11 +6,14 @@ nebula_add_library( meta_data_upgrade_obj OBJECT MetaDataUpgrade.cpp v1/MetaServiceUtilsV1.cpp + v2/MetaServiceUtilsV2.cpp ) add_dependencies( meta_data_upgrade_obj meta_v1_thrift_obj + meta_v2_thrift_obj ) nebula_add_subdirectory(v1) +nebula_add_subdirectory(v2) diff --git a/src/meta/upgrade/MetaDataUpgrade.cpp b/src/meta/upgrade/MetaDataUpgrade.cpp index bff64a063e5..e2c3c9b5a06 100644 --- a/src/meta/upgrade/MetaDataUpgrade.cpp +++ b/src/meta/upgrade/MetaDataUpgrade.cpp @@ -19,6 +19,7 @@ #include "meta/ActiveHostsMan.h" #include "meta/MetaServiceUtils.h" #include "meta/upgrade/v1/MetaServiceUtilsV1.h" +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" DECLARE_bool(null_type); DECLARE_uint32(string_index_limit); @@ -62,6 +63,65 @@ Status MetaDataUpgrade::rewriteSpaces(const folly::StringPiece &key, return Status::OK(); } +Status MetaDataUpgrade::rewriteSpacesV2ToV3(const folly::StringPiece &key, + const folly::StringPiece &val) { + auto oldProps = meta::v2::MetaServiceUtilsV2::parseSpace(val); + cpp2::SpaceDesc spaceDesc; + spaceDesc.set_space_name(oldProps.get_space_name()); + spaceDesc.set_partition_num(oldProps.get_partition_num()); + spaceDesc.set_replica_factor(oldProps.get_replica_factor()); + spaceDesc.set_charset_name(oldProps.get_charset_name()); + spaceDesc.set_collate_name(oldProps.get_collate_name()); + cpp2::ColumnTypeDef def; + auto &type = oldProps.get_vid_type(); + def.set_type_length(*type.get_type_length()); + def.set_type(convertToPropertyType(type.get_type())); + def.set_geo_shape(convertToGeoShape(*type.get_geo_shape())); + spaceDesc.set_vid_type(std::move(def)); + if (oldProps.isolation_level_ref().has_value()) { + if (*oldProps.isolation_level_ref() == nebula::meta::v2::cpp2::IsolationLevel::DEFAULT) { + spaceDesc.set_isolation_level(nebula::meta::cpp2::IsolationLevel::DEFAULT); + } else { + spaceDesc.set_isolation_level(nebula::meta::cpp2::IsolationLevel::TOSS); + } + } + + if (oldProps.comment_ref().has_value()) { + spaceDesc.set_comment(*oldProps.comment_ref()); + } + + if (oldProps.group_name_ref().has_value()) { + auto groupName = *oldProps.group_name_ref(); + auto groupKey = meta::v2::MetaServiceUtilsV2::groupKey(groupName); + std::string zoneValue; + auto code = kv_->get(kDefaultSpaceId, kDefaultPartId, std::move(groupKey), &zoneValue); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Get Group Failed"); + } + + auto zones = meta::v2::MetaServiceUtilsV2::parseZoneNames(std::move(zoneValue)); + spaceDesc.set_zone_names(std::move(zones)); + } else { + const auto &zonePrefix = MetaKeyUtils::zonePrefix(); + std::unique_ptr iter; + auto code = kv_->prefix(kDefaultSpaceId, kDefaultPartId, zonePrefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Get Zones Failed"); + } + + std::vector<::std::string> zones; + while (iter->valid()) { + auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); + zones.emplace_back(std::move(zoneName)); + iter->next(); + } + spaceDesc.set_zone_names(std::move(zones)); + } + + NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaKeyUtils::spaceVal(spaceDesc))); + return Status::OK(); +} + Status MetaDataUpgrade::rewriteParts(const folly::StringPiece &key, const folly::StringPiece &val) { auto oldHosts = meta::v1::MetaServiceUtilsV1::parsePartVal(val); std::vector newHosts; @@ -277,6 +337,60 @@ Status MetaDataUpgrade::convertToNewIndexColumns( return Status::OK(); } +nebula::cpp2::PropertyType MetaDataUpgrade::convertToPropertyType( + nebula::meta::v2::cpp2::PropertyType type) { + switch (type) { + case nebula::meta::v2::cpp2::PropertyType::BOOL: + return nebula::cpp2::PropertyType::BOOL; + case nebula::meta::v2::cpp2::PropertyType::INT64: + return nebula::cpp2::PropertyType::INT64; + case nebula::meta::v2::cpp2::PropertyType::VID: + return nebula::cpp2::PropertyType::VID; + case nebula::meta::v2::cpp2::PropertyType::FLOAT: + return nebula::cpp2::PropertyType::FLOAT; + case nebula::meta::v2::cpp2::PropertyType::DOUBLE: + return nebula::cpp2::PropertyType::DOUBLE; + case nebula::meta::v2::cpp2::PropertyType::STRING: + return nebula::cpp2::PropertyType::STRING; + case nebula::meta::v2::cpp2::PropertyType::FIXED_STRING: + return nebula::cpp2::PropertyType::FIXED_STRING; + case nebula::meta::v2::cpp2::PropertyType::INT8: + return nebula::cpp2::PropertyType::INT8; + case nebula::meta::v2::cpp2::PropertyType::INT16: + return nebula::cpp2::PropertyType::INT16; + case nebula::meta::v2::cpp2::PropertyType::INT32: + return nebula::cpp2::PropertyType::INT32; + case nebula::meta::v2::cpp2::PropertyType::TIMESTAMP: + return nebula::cpp2::PropertyType::TIMESTAMP; + case nebula::meta::v2::cpp2::PropertyType::DATE: + return nebula::cpp2::PropertyType::DATE; + case nebula::meta::v2::cpp2::PropertyType::DATETIME: + return nebula::cpp2::PropertyType::DATETIME; + case nebula::meta::v2::cpp2::PropertyType::TIME: + return nebula::cpp2::PropertyType::TIME; + case nebula::meta::v2::cpp2::PropertyType::GEOGRAPHY: + return nebula::cpp2::PropertyType::GEOGRAPHY; + default: + return nebula::cpp2::PropertyType::UNKNOWN; + } +} + +nebula::meta::cpp2::GeoShape MetaDataUpgrade::convertToGeoShape( + nebula::meta::v2::cpp2::GeoShape shape) { + switch (shape) { + case nebula::meta::v2::cpp2::GeoShape::ANY: + return nebula::meta::cpp2::GeoShape::ANY; + case nebula::meta::v2::cpp2::GeoShape::POINT: + return nebula::meta::cpp2::GeoShape::POINT; + case nebula::meta::v2::cpp2::GeoShape::LINESTRING: + return nebula::meta::cpp2::GeoShape::LINESTRING; + case nebula::meta::v2::cpp2::GeoShape::POLYGON: + return nebula::meta::cpp2::GeoShape::POLYGON; + default: + LOG(FATAL) << "Unimplemented"; + } +} + void MetaDataUpgrade::printHost(const folly::StringPiece &key, const folly::StringPiece &val) { auto host = meta::v1::MetaServiceUtilsV1::parseHostKey(key); auto info = HostInfo::decodeV1(val); @@ -287,7 +401,7 @@ void MetaDataUpgrade::printHost(const folly::StringPiece &key, const folly::Stri LOG(INFO) << "Host info: gitInfoSha_: " << info.gitInfoSha_; } -void MetaDataUpgrade::printSpaces(const folly::StringPiece &val) { +void MetaDataUpgrade::printSpacesV1(const folly::StringPiece &val) { auto oldProps = meta::v1::MetaServiceUtilsV1::parseSpace(val); LOG(INFO) << "Space name: " << oldProps.get_space_name(); LOG(INFO) << "Partition num: " << oldProps.get_partition_num(); @@ -296,6 +410,18 @@ void MetaDataUpgrade::printSpaces(const folly::StringPiece &val) { LOG(INFO) << "Collate name: " << oldProps.get_collate_name(); } +void MetaDataUpgrade::printSpacesV2(const folly::StringPiece &val) { + auto oldProps = meta::v2::MetaServiceUtilsV2::parseSpace(val); + LOG(INFO) << "Space name: " << oldProps.get_space_name(); + LOG(INFO) << "Partition num: " << oldProps.get_partition_num(); + LOG(INFO) << "Replica factor: " << oldProps.get_replica_factor(); + LOG(INFO) << "Charset name: " << oldProps.get_charset_name(); + LOG(INFO) << "Collate name: " << oldProps.get_collate_name(); + if (oldProps.group_name_ref().has_value()) { + LOG(INFO) << "Group name: " << *oldProps.group_name_ref(); + } +} + void MetaDataUpgrade::printParts(const folly::StringPiece &key, const folly::StringPiece &val) { auto spaceId = meta::v1::MetaServiceUtilsV1::parsePartKeySpaceId(key); auto partId = meta::v1::MetaServiceUtilsV1::parsePartKeyPartId(key); diff --git a/src/meta/upgrade/MetaDataUpgrade.h b/src/meta/upgrade/MetaDataUpgrade.h index dbea1afcbfc..172bd5dd77f 100644 --- a/src/meta/upgrade/MetaDataUpgrade.h +++ b/src/meta/upgrade/MetaDataUpgrade.h @@ -14,6 +14,7 @@ #include "kvstore/KVStore.h" #include "meta/processors/Common.h" #include "meta/upgrade/v1/gen-cpp2/meta_types.h" +#include "meta/upgrade/v2/gen-cpp2/meta_types.h" namespace nebula { namespace meta { @@ -33,10 +34,13 @@ class MetaDataUpgrade final { Status rewriteConfigs(const folly::StringPiece &key, const folly::StringPiece &val); Status rewriteJobDesc(const folly::StringPiece &key, const folly::StringPiece &val); + Status rewriteSpacesV2ToV3(const folly::StringPiece &key, const folly::StringPiece &val); + Status deleteKeyVal(const folly::StringPiece &key); void printHost(const folly::StringPiece &key, const folly::StringPiece &val); - void printSpaces(const folly::StringPiece &val); + void printSpacesV1(const folly::StringPiece &val); + void printSpacesV2(const folly::StringPiece &val); void printParts(const folly::StringPiece &key, const folly::StringPiece &val); void printLeaders(const folly::StringPiece &key); void printSchemas(const folly::StringPiece &val); @@ -94,6 +98,10 @@ class MetaDataUpgrade final { Status convertToNewIndexColumns(const std::vector &oldCols, std::vector &newCols); + nebula::cpp2::PropertyType convertToPropertyType(nebula::meta::v2::cpp2::PropertyType type); + + nebula::meta::cpp2::GeoShape convertToGeoShape(nebula::meta::v2::cpp2::GeoShape shape); + private: kvstore::KVStore *kv_ = nullptr; }; diff --git a/src/meta/upgrade/v2/CMakeLists.txt b/src/meta/upgrade/v2/CMakeLists.txt new file mode 100644 index 00000000000..df5d9d2549f --- /dev/null +++ b/src/meta/upgrade/v2/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +set(THRIFT1 ${Fbthrift_BIN}) + +set(meta_v2_sources + gen-cpp2/meta_constants.cpp + gen-cpp2/meta_data.cpp + gen-cpp2/meta_metadata.cpp + gen-cpp2/meta_types.cpp + ) + +add_custom_command( + OUTPUT ${meta_v2_sources} + COMMAND "${THRIFT1}" "--strict" "--allow-neg-enum-vals" "--gen" "mstch_cpp2:include_prefix=\"meta/upgrade/v2\",process_in_event_base,stack_arguments" "-o" "." "${CMAKE_CURRENT_SOURCE_DIR}/meta.thrift" + DEPENDS meta.thrift + ) + +nebula_add_library(meta_v2_thrift_obj OBJECT ${meta_v2_sources}) + +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-pedantic") +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-extra") +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-deprecated-declarations") diff --git a/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp b/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp new file mode 100644 index 00000000000..bb23cd29c74 --- /dev/null +++ b/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp @@ -0,0 +1,32 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" + +#include +#include + +namespace nebula::meta::v2 { + +std::string MetaServiceUtilsV2::groupKey(const std::string& group) { + std::string key; + key.reserve(kGroupsTable.size() + group.size()); + key.append(kGroupsTable.data(), kGroupsTable.size()).append(group); + return key; +} + +std::vector MetaServiceUtilsV2::parseZoneNames(folly::StringPiece rawData) { + std::vector zones; + folly::split(',', rawData.str(), zones); + return zones; +} + +cpp2::SpaceDesc MetaServiceUtilsV2::parseSpace(folly::StringPiece rawData) { + cpp2::SpaceDesc spaceDesc; + apache::thrift::CompactSerializer::deserialize(rawData, spaceDesc); + return spaceDesc; +} + +} // namespace nebula::meta::v2 diff --git a/src/meta/upgrade/v2/MetaServiceUtilsV2.h b/src/meta/upgrade/v2/MetaServiceUtilsV2.h new file mode 100644 index 00000000000..ff1fb38203a --- /dev/null +++ b/src/meta/upgrade/v2/MetaServiceUtilsV2.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ +#define TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ + +#include "common/base/Base.h" +#include "common/base/Status.h" +#include "common/thrift/ThriftTypes.h" +#include "interface/gen-cpp2/meta_types.h" +#include "kvstore/Common.h" +#include "meta/upgrade/v2/gen-cpp2/meta_types.h" + +namespace nebula::meta::v2 { + +const std::string kGroupsTable = "__groups__"; // NOLINT + +class MetaServiceUtilsV2 final { + public: + MetaServiceUtilsV2() = delete; + + static std::string groupKey(const std::string& group); + + static std::vector parseZoneNames(folly::StringPiece rawData); + + static cpp2::SpaceDesc parseSpace(folly::StringPiece rawData); +}; + +} // namespace nebula::meta::v2 + +#endif // TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ diff --git a/src/meta/upgrade/v2/meta.thrift b/src/meta/upgrade/v2/meta.thrift new file mode 100644 index 00000000000..d46922478b8 --- /dev/null +++ b/src/meta/upgrade/v2/meta.thrift @@ -0,0 +1,81 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +namespace cpp nebula.meta.v2 + +cpp_include "common/thrift/ThriftTypes.h" + +typedef i32 (cpp.type = "nebula::GraphSpaceID") GraphSpaceID +typedef i32 (cpp.type = "nebula::PartitionID") PartitionID +typedef i32 (cpp.type = "nebula::TagID") TagID +typedef i32 (cpp.type = "nebula::EdgeType") EdgeType +typedef i64 (cpp.type = "nebula::EdgeRanking") EdgeRanking +typedef i64 (cpp.type = "nebula::VertexID") VertexID +typedef i32 (cpp.type = "nebula::IndexID") IndexID + +typedef i32 IPv4 +typedef i32 (cpp.type = "nebula::Port") Port + +typedef i64 (cpp.type = "nebula::SchemaVer") SchemaVer + +struct SpaceDesc { + 1: binary space_name, + 2: i32 partition_num = 0, + 3: i32 replica_factor = 0, + 4: binary charset_name, + 5: binary collate_name, + 6: ColumnTypeDef vid_type = {"type": PropertyType.FIXED_STRING, "type_length": 8}, + 7: optional binary group_name, + 8: optional IsolationLevel isolation_level, + 9: optional binary comment, +} + +enum PropertyType { + UNKNOWN = 0, + + // Simple types + BOOL = 1, + INT64 = 2, // This is the same as INT in v1 + VID = 3, // Deprecated, only supported by v1 + FLOAT = 4, + DOUBLE = 5, + STRING = 6, + // String with fixed length. If the string content is shorteri + // than the given length, '\0' will be padded to the end + FIXED_STRING = 7, // New in v2 + INT8 = 8, // New in v2 + INT16 = 9, // New in v2 + INT32 = 10, // New in v2 + + // Date time + TIMESTAMP = 21, + DATE = 24, + DATETIME = 25, + TIME = 26, + + // Geo spatial + GEOGRAPHY = 31, +} (cpp.enum_strict) + +// Geo shape type +enum GeoShape { + ANY = 0, + POINT = 1, + LINESTRING = 2, + POLYGON = 3, +} (cpp.enum_strict) + +struct ColumnTypeDef { + 1: required PropertyType type, + // type_length is valid for fixed_string type + 2: optional i16 type_length = 0, + // geo_shape is valid for geography type + 3: optional GeoShape geo_shape, +} + +enum IsolationLevel { + DEFAULT = 0x00, // allow add half edge(either in or out edge succeeded) + TOSS = 0x01, // add in and out edge atomic +} (cpp.enum_strict) \ No newline at end of file