Merge branch 'add_tck_coverage' of https://github.com/HarrisChu/nebula …

…into add_tck_coverage
vesoft-inc · Sep 22, 2021 · 01d5f6e · 01d5f6e
2 parents b3a82d0 + fa18dc7
commit 01d5f6e
Show file tree

Hide file tree

Showing 47 changed files with 1,087 additions and 182 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -85,6 +85,7 @@ jobs:
                   -DENABLE_TESTING=on \
                   -B build
               echo "::set-output name=j::10"
+              echo "::set-output name=t::$(nproc)"
               ;;
             ubuntu2004)
               # build with Debug type
@@ -96,6 +97,7 @@ jobs:
                   -DENABLE_COVERAGE=on \
                   -B build
               echo "::set-output name=j::10"
+              echo "::set-output name=t::10"
               ;;
             esac
             ;;
@@ -109,12 +111,13 @@ jobs:
                 -DENABLE_TESTING=on \
                 -B build
             echo "::set-output name=j::6"
+            echo "::set-output name=t::10"
             ;;
           esac
       - name: Make
         run: |
           ccache -z
-          cmake --build build/ -j $(($(nproc)/2+1))
+          cmake --build build/ -j ${{ steps.cmake.outputs.t }}
           ccache -s
       - name: CTest
         env:

diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp
@@ -24,13 +24,17 @@
 #include "version/Version.h"
 #include "webservice/Common.h"
 
+DECLARE_int32(ws_meta_http_port);
+DECLARE_int32(ws_meta_h2_port);
+
 DEFINE_uint32(expired_time_factor, 5, "The factor of expired time based on heart beat interval");
 DEFINE_int32(heartbeat_interval_secs, 10, "Heartbeat interval in seconds");
 DEFINE_int32(meta_client_retry_times, 3, "meta client retry times, 0 means no retry");
 DEFINE_int32(meta_client_retry_interval_secs, 1, "meta client sleep interval between retry");
 DEFINE_int32(meta_client_timeout_ms, 60 * 1000, "meta client timeout");
 DEFINE_string(cluster_id_path, "cluster.id", "file path saved clusterId");
 DEFINE_int32(check_plan_killed_frequency, 8, "check plan killed every 1<<n times");
+
 namespace nebula {
 namespace meta {
 

diff --git a/src/clients/storage/GraphStorageClient.cpp b/src/clients/storage/GraphStorageClient.cpp
@@ -27,7 +27,7 @@ folly::SemiFuture<StorageRpcResponse<cpp2::GetNeighborsResponse>> GraphStorageCl
     bool random,
     const std::vector<cpp2::OrderBy>& orderBy,
     int64_t limit,
-    std::string filter,
+    const Expression* filter,
     folly::EventBase* evb) {
   auto cbStatus = getIdFromRow(space, false);
   if (!cbStatus.ok()) {
@@ -72,8 +72,8 @@ folly::SemiFuture<StorageRpcResponse<cpp2::GetNeighborsResponse>> GraphStorageCl
       spec.set_order_by(orderBy);
     }
     spec.set_limit(limit);
-    if (filter.size() > 0) {
-      spec.set_filter(filter);
+    if (filter != nullptr) {
+      spec.set_filter(filter->encode());
     }
     req.set_traverse_spec(std::move(spec));
   }
@@ -180,7 +180,7 @@ folly::SemiFuture<StorageRpcResponse<cpp2::GetPropResponse>> GraphStorageClient:
     bool dedup,
     const std::vector<cpp2::OrderBy>& orderBy,
     int64_t limit,
-    std::string filter,
+    const Expression* filter,
     folly::EventBase* evb) {
   auto cbStatus = getIdFromRow(space, edgeProps != nullptr);
   if (!cbStatus.ok()) {
@@ -216,8 +216,8 @@ folly::SemiFuture<StorageRpcResponse<cpp2::GetPropResponse>> GraphStorageClient:
       req.set_order_by(orderBy);
     }
     req.set_limit(limit);
-    if (filter.size() > 0) {
-      req.set_filter(filter);
+    if (filter != nullptr) {
+      req.set_filter(filter->encode());
     }
     req.set_common(common);
   }

diff --git a/src/clients/storage/GraphStorageClient.h b/src/clients/storage/GraphStorageClient.h
@@ -49,7 +49,7 @@ class GraphStorageClient : public StorageClientBase<cpp2::GraphStorageServiceAsy
       bool random = false,
       const std::vector<cpp2::OrderBy>& orderBy = std::vector<cpp2::OrderBy>(),
       int64_t limit = std::numeric_limits<int64_t>::max(),
-      std::string filter = std::string(),
+      const Expression* filter = nullptr,
       folly::EventBase* evb = nullptr);
 
   folly::SemiFuture<StorageRpcResponse<cpp2::GetPropResponse>> getProps(
@@ -63,7 +63,7 @@ class GraphStorageClient : public StorageClientBase<cpp2::GraphStorageServiceAsy
       bool dedup = false,
       const std::vector<cpp2::OrderBy>& orderBy = std::vector<cpp2::OrderBy>(),
       int64_t limit = std::numeric_limits<int64_t>::max(),
-      std::string filter = std::string(),
+      const Expression* filter = nullptr,
       folly::EventBase* evb = nullptr);
 
   folly::SemiFuture<StorageRpcResponse<cpp2::ExecResponse>> addVertices(

diff --git a/src/common/base/test/LoggingBenchmark.cpp b/src/common/base/test/LoggingBenchmark.cpp
@@ -6,6 +6,9 @@
 
 #include <folly/Benchmark.h>
 #include <folly/init/Init.h>
+#include <folly/logging/FileHandlerFactory.h>
+#include <folly/logging/Init.h>
+#include <folly/logging/LoggerDB.h>
 #include <sys/types.h>
 
 #include <iostream>
@@ -18,6 +21,26 @@
               << "123";                 \
   }
 
+#define XLOG_SOMETHING(iters)           \
+  for (int64_t i = 0; i < iters; i++) { \
+    XLOG(INFO) << "Hello"               \
+               << " "                   \
+               << "Wolrd"               \
+               << "123";                \
+  }
+
+class XlogInit {
+ public:
+  explicit XlogInit(folly::StringPiece config) { folly::initLogging(config); }
+};
+
+static void xlogRegistFileHandler() {
+  folly::LoggerDB::get().registerHandlerFactory(std::make_unique<folly::FileHandlerFactory>());
+  // Since glog outputs the logs to /tmp by default, so we explicitly set a file handler for
+  // xlog and output logs to /tmp.
+  folly::initLogging(";default=file:path=/tmp/logging_bm.log");
+}
+
 /***************************
  *
  * <glog/logging.h> native
@@ -34,6 +57,9 @@ void loggingUsingGlog(int64_t iters) { LOG_SOMETHING(iters); }
 #include "common/base/Logging.h"
 void loggingOptimized(int64_t iters) { LOG_SOMETHING(iters); }
 
+#include <folly/logging/xlog.h>
+void loggingUsingXlog(int64_t iters) { XLOG_SOMETHING(iters); }
+
 /***************************
  *
  * Run benchmarks
@@ -49,6 +75,18 @@ BENCHMARK_RELATIVE(optimized_output_logs, iters) {
   loggingOptimized(iters);
 }
 
+BENCHMARK_RELATIVE(xlog_output_logs, iters) {
+  BENCHMARK_SUSPEND { static XlogInit init(".=INFO:default"); }
+  loggingUsingXlog(iters);
+}
+
+BENCHMARK_RELATIVE(xlog_output_logs_async, iters) {
+  BENCHMARK_SUSPEND {
+    static XlogInit init(".=INFO:default;default:async=true,max_buffer_size=4096");
+  }
+  loggingUsingXlog(iters);
+}
+
 BENCHMARK_DRAW_LINE();
 
 BENCHMARK(glog_skip_logs, iters) {
@@ -61,28 +99,35 @@ BENCHMARK_RELATIVE(optimized_skip_logs, iters) {
   loggingOptimized(iters);
 }
 
+BENCHMARK_RELATIVE(xlog_skip_logs, iters) {
+  BENCHMARK_SUSPEND { static XlogInit init(".=WARN:default"); }
+  loggingUsingXlog(iters);
+}
 /***************************
  *
  * main()
  *
  **************************/
 int main(int argc, char** argv) {
   folly::init(&argc, &argv, true);
+  xlogRegistFileHandler();
 
   folly::runBenchmarks();
   return 0;
 }
 
 /*
-Benchmark number is taken from WSL running on i7-8650
-
+Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz
 ============================================================================
-LoggingBenchmark.cpp                            relative  time/iter  iters/s
+src/common/base/test/LoggingBenchmark.cpp       relative  time/iter  iters/s
 ============================================================================
-glog_output_logs                                             3.13us  319.07K
-optimized_output_logs                            100.10%     3.13us  319.39K
+glog_output_logs                                             1.86us  536.82K
+optimized_output_logs                            100.26%     1.86us  538.24K
+xlog_output_logs                                  52.73%     3.53us  283.09K
+xlog_output_logs_async                            53.40%     3.49us  286.68K
 ----------------------------------------------------------------------------
-glog_skip_logs                                               1.76us  567.45K
-optimized_skip_logs                                 inf%     0.00fs  Infinity
+glog_skip_logs                                               1.27us  789.36K
+optimized_skip_logs                             94753.03%     1.34ns  747.94M
+xlog_skip_logs                                  5215.83%    24.29ns   41.17M
 ============================================================================
 */
diff --git a/src/common/datatypes/DataSet.h b/src/common/datatypes/DataSet.h
@@ -7,6 +7,8 @@
 #ifndef COMMON_DATATYPES_DATASET_H_
 #define COMMON_DATATYPES_DATASET_H_
 
+#include <folly/dynamic.h>
+
 #include <iostream>
 #include <iterator>
 #include <sstream>
@@ -153,6 +155,44 @@ struct DataSet {
     return os.str();
   }
 
+  // format:
+  // [
+  //   {
+  //     "row": [ row-data ],
+  //     "meta": [ metadata ]
+  //   },
+  // ]
+  folly::dynamic toJson() const {
+    // parse rows to json
+    auto dataBody = folly::dynamic::array();
+    for (auto& row : rows) {
+      dataBody.push_back(rowToJson(row));
+    }
+
+    return dataBody;
+  }
+
+  // parse Nebula::Row to json
+  // format:
+  // {
+  //   "row": [ row-data ],
+  //   "meta": [ metadata ]
+  // }
+  folly::dynamic rowToJson(const Row& row) const {
+    folly::dynamic rowJsonObj = folly::dynamic::object();
+    auto rowDataList = folly::dynamic::array();
+    auto metaDataList = folly::dynamic::array();
+
+    for (const auto& ele : row.values) {
+      rowDataList.push_back(ele.toJson());
+      metaDataList.push_back(ele.getMetaData());
+    }
+
+    rowJsonObj.insert("row", rowDataList);
+    rowJsonObj.insert("meta", metaDataList);
+    return rowJsonObj;
+  }
+
   bool operator==(const DataSet& rhs) const { return colNames == rhs.colNames && rows == rhs.rows; }
 };
 

diff --git a/src/common/datatypes/Date.h b/src/common/datatypes/Date.h
@@ -7,6 +7,8 @@
 #ifndef COMMON_DATATYPES_DATE_H_
 #define COMMON_DATATYPES_DATE_H_
 
+#include <folly/dynamic.h>
+
 #include <string>
 
 namespace nebula {
@@ -62,6 +64,7 @@ struct Date {
   Date operator-(int64_t days) const;
 
   std::string toString() const;
+  folly::dynamic toJson() const { return toString(); }
 
   // Return the number of days since -32768/1/1
   int64_t toInt() const;
@@ -113,6 +116,8 @@ struct Time {
   }
 
   std::string toString() const;
+  // 'Z' representing UTC timezone
+  folly::dynamic toJson() const { return toString() + "Z"; }
 };
 
 inline std::ostream& operator<<(std::ostream& os, const Time& d) {
@@ -203,6 +208,8 @@ struct DateTime {
   }
 
   std::string toString() const;
+  // 'Z' representing UTC timezone
+  folly::dynamic toJson() const { return toString() + "Z"; }
 };
 
 inline std::ostream& operator<<(std::ostream& os, const DateTime& d) {

diff --git a/src/common/datatypes/Edge.cpp b/src/common/datatypes/Edge.cpp
@@ -32,6 +32,45 @@ std::string Edge::toString() const {
   return os.str();
 }
 
+// format:
+// {
+//    "prop1": val1,
+//    "prop2": val2,
+// }
+folly::dynamic Edge::toJson() const {
+  folly::dynamic propObj = folly::dynamic::object();
+
+  for (const auto& iter : props) {
+    propObj.insert(iter.first, iter.second.toJson());
+  }
+
+  return propObj;
+}
+
+// Used in Json form query result
+// format:
+// {
+//   "id": {
+//      "name": _name,
+//      "src": srcVID,
+//      "dst": dstVID,
+//      "type": _type,
+//      "ranking": _rankding
+//    }
+//   "type": "edge"
+// }
+folly::dynamic Edge::getMetaData() const {
+  folly::dynamic edgeMetadataObj = folly::dynamic::object();
+
+  folly::dynamic edgeIdObj = folly::dynamic::object("name", name)("src", src.toJson())(
+      "dst", dst.toJson())("type", type)("ranking", ranking);
+
+  edgeMetadataObj.insert("id", edgeIdObj);
+  edgeMetadataObj.insert("type", "edge");
+
+  return edgeMetadataObj;
+}
+
 bool Edge::contains(const Value& key) const {
   if (!key.isStr()) {
     return false;

diff --git a/src/common/datatypes/Edge.h b/src/common/datatypes/Edge.h
@@ -50,6 +50,9 @@ struct Edge {
   void __clear() { clear(); }
 
   std::string toString() const;
+  folly::dynamic toJson() const;
+  // Used in Json form query result
+  folly::dynamic getMetaData() const;
 
   bool operator==(const Edge& rhs) const;
 

diff --git a/src/common/datatypes/List.cpp b/src/common/datatypes/List.cpp
@@ -22,4 +22,24 @@ std::string List::toString() const {
   return os.str();
 }
 
+folly::dynamic List::toJson() const {
+  auto listJsonObj = folly::dynamic::array();
+
+  for (const auto& val : values) {
+    listJsonObj.push_back(val.toJson());
+  }
+
+  return listJsonObj;
+}
+
+folly::dynamic List::getMetaData() const {
+  auto listMetadataObj = folly::dynamic::array();
+
+  for (const auto& val : values) {
+    listMetadataObj.push_back(val.getMetaData());
+  }
+
+  return listMetadataObj;
+}
+
 }  // namespace nebula
diff --git a/src/common/datatypes/List.h b/src/common/datatypes/List.h
@@ -65,6 +65,9 @@ struct List {
   size_t size() const { return values.size(); }
 
   std::string toString() const;
+  folly::dynamic toJson() const;
+  // Extract the metadata of each element
+  folly::dynamic getMetaData() const;
 };
 
 inline std::ostream& operator<<(std::ostream& os, const List& l) { return os << l.toString(); }