Skip to content

Commit

Permalink
support s2 index params
Browse files Browse the repository at this point in the history
  • Loading branch information
@jievince
jievince committed Dec 8, 2021
1 parent d38fc07 commit 064ed80
Show file tree
Hide file tree
Showing 41 changed files with 783 additions and 158 deletions.
12 changes: 4 additions & 8 deletions src/clients/meta/MetaClient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1712,16 +1712,14 @@ folly::Future<StatusOr<IndexID>> MetaClient::createTagIndex(GraphSpaceID spaceID
std::string tagName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment) {
cpp2::IndexParams indexParams) {
cpp2::CreateTagIndexReq req;
req.set_space_id(spaceID);
req.set_index_name(std::move(indexName));
req.set_tag_name(std::move(tagName));
req.set_fields(std::move(fields));
req.set_if_not_exists(ifNotExists);
if (comment != nullptr) {
req.set_comment(*comment);
}
req.set_index_params(std::move(indexParams));

folly::Promise<StatusOr<IndexID>> promise;
auto future = promise.getFuture();
Expand Down Expand Up @@ -1824,16 +1822,14 @@ folly::Future<StatusOr<IndexID>> MetaClient::createEdgeIndex(
std::string edgeName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment) {
cpp2::IndexParams indexParams) {
cpp2::CreateEdgeIndexReq req;
req.set_space_id(spaceID);
req.set_index_name(std::move(indexName));
req.set_edge_name(std::move(edgeName));
req.set_fields(std::move(fields));
req.set_if_not_exists(ifNotExists);
if (comment != nullptr) {
req.set_comment(*comment);
}
req.set_index_params(std::move(indexParams));

folly::Promise<StatusOr<IndexID>> promise;
auto future = promise.getFuture();
Expand Down
26 changes: 14 additions & 12 deletions src/clients/meta/MetaClient.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,12 +311,13 @@ class MetaClient {
bool ifExists = false);

// Operations for index
folly::Future<StatusOr<IndexID>> createTagIndex(GraphSpaceID spaceID,
std::string indexName,
std::string tagName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
const std::string* comment = nullptr);
folly::Future<StatusOr<IndexID>> createTagIndex(
GraphSpaceID spaceID,
std::string indexName,
std::string tagName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
meta::cpp2::IndexParams indexParams = cpp2::IndexParams{});

// Remove the define of tag index
folly::Future<StatusOr<bool>> dropTagIndex(GraphSpaceID spaceId,
Expand All @@ -331,12 +332,13 @@ class MetaClient {

folly::Future<StatusOr<std::vector<cpp2::IndexStatus>>> listTagIndexStatus(GraphSpaceID spaceId);

folly::Future<StatusOr<IndexID>> createEdgeIndex(GraphSpaceID spaceID,
std::string indexName,
std::string edgeName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
const std::string* comment = nullptr);
folly::Future<StatusOr<IndexID>> createEdgeIndex(
GraphSpaceID spaceID,
std::string indexName,
std::string edgeName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
cpp2::IndexParams indexParams = cpp2::IndexParams{});

// Remove the definition of edge index
folly::Future<StatusOr<bool>> dropEdgeIndex(GraphSpaceID spaceId,
Expand Down
2 changes: 1 addition & 1 deletion src/common/datatypes/Geography.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ struct Geography {
bool needNormalize = false,
bool verifyValidity = false);

Geography() {}
Geography() = default;
Geography(const Point& v) : geo_(v) {} // NOLINT
Geography(Point&& v) : geo_(std::move(v)) {} // NOLINT
Geography(const LineString& v) : geo_(v) {} // NOLINT
Expand Down
12 changes: 11 additions & 1 deletion src/common/geo/GeoIndex.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,17 @@
namespace nebula {
namespace geo {

nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() {
bool ScanRange::operator==(const ScanRange& rhs) const {
if (isRangeScan != rhs.isRangeScan) {
return false;
}
if (isRangeScan) {
return rangeMin == rhs.rangeMin && rangeMax == rhs.rangeMax;
}
return rangeMin == rhs.rangeMin;
}

nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() const {
nebula::storage::cpp2::IndexColumnHint hint;
// column_name should be set by the caller
if (isRangeScan) {
Expand Down
5 changes: 4 additions & 1 deletion src/common/geo/GeoIndex.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ struct ScanRange {

explicit ScanRange(uint64_t v) : rangeMin(v), isRangeScan(false) {}

nebula::storage::cpp2::IndexColumnHint toIndexColumnHint();
bool operator==(const ScanRange& rhs) const;

nebula::storage::cpp2::IndexColumnHint toIndexColumnHint() const;
};

class GeoIndex {
Expand Down Expand Up @@ -82,6 +84,7 @@ class GeoIndex {

private:
RegionCoverParams rcParams_;
// pointsOnly_ indicates whether the indexed column only has points.
// For the column Geography(Point), we don't need to build ancestor cells
bool pointsOnly_{false};
};
Expand Down
161 changes: 154 additions & 7 deletions src/common/geo/test/GeoIndexTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

#include <gtest/gtest.h>

#include <cstdint>
#include <unordered_set>

#include "common/base/Base.h"
Expand All @@ -13,22 +14,23 @@
namespace nebula {
namespace geo {

std::vector<uint64_t> toUint64Vector(std::vector<int64_t> expect) {
auto reinterpretInt64AsUint64 = [](int64_t i) -> uint64_t {
const char* c = reinterpret_cast<const char*>(&i);
uint64_t u = *reinterpret_cast<const uint64_t*>(c);
return u;
};
uint64_t asUint64(int64_t i) {
const char* c = reinterpret_cast<const char*>(&i);
uint64_t u = *reinterpret_cast<const uint64_t*>(c);
return u;
}

std::vector<uint64_t> toUint64Vector(std::vector<int64_t> expect) {
std::vector<uint64_t> transformedExpect;
transformedExpect.reserve(expect.size());
for (int64_t i : expect) {
transformedExpect.push_back(reinterpretInt64AsUint64(i));
transformedExpect.push_back(asUint64(i));
}
return transformedExpect;
}

// The tested wkt data is generated by https://clydedacruz.github.io/openstreetmap-wkt-playground/
// And the expected result comes from BigQuery
TEST(indexCells, point) {
geo::RegionCoverParams rc;
geo::GeoIndex geoIndex(rc);
Expand Down Expand Up @@ -105,6 +107,7 @@ TEST(indexCells, lineString) {
EXPECT_EQ(toUint64Vector(expect), cells);
}
}

TEST(indexCells, polygon) {
geo::RegionCoverParams rc;
geo::GeoIndex geoIndex(rc);
Expand Down Expand Up @@ -172,6 +175,150 @@ TEST(indexCells, polygon) {
}
}

TEST(indexCells, tunningRegionCoverParamsForPoint) {
// TODO(jie)
}
TEST(indexCells, tunningRegionCoverParamsForLineString) {
// TODO(jie)
}
TEST(indexCells, tunningRegionCoverParamsForPolygon) {
// TODO(jie)
}

TEST(intersects, point) {
geo::RegionCoverParams rc(0, 30, 8);
{
bool pointsOnly = false;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto point = Geography::fromWKT("POINT(1.0 1.0)").value();
auto got = geoIndex.intersects(point);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({1153277837650709461});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
{
bool pointsOnly = false; // The indexed geo column only has points;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto point = Geography::fromWKT("POINT(1.0 1.0)").value();
auto got = geoIndex.intersects(point);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({1153277837650709461});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
}

TEST(intersects, lineString) {
geo::RegionCoverParams rc(0, 30, 8);
{
bool pointsOnly = false;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto line = Geography::fromWKT("LINESTRING(68.9 48.9,76.1 35.5,125.7 28.2)").value();
auto got = geoIndex.intersects(line);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({3765009288481734656,
3818771009033469952,
3909124476557590528,
3928264774973915136,
4017210867614482432,
4053239664633446400,
4089268461652410368,
4773815605012725760});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
{
bool pointsOnly = true; // The indexed geo column only has points;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto line = Geography::fromWKT("POINT(1.0 1.0)").value();
auto got = geoIndex.intersects(line);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({3765009288481734656,
3818771009033469952,
3909124476557590528,
3928264774973915136,
4017210867614482432,
4053239664633446400,
4089268461652410368,
4773815605012725760});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
}

TEST(intersects, polygon) {
geo::RegionCoverParams rc(0, 30, 8);
{
bool pointsOnly = false;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto point =
Geography::fromWKT("POLYGON((102.5 33.5, 110.6 36.9,113.6 30.4,102.7 27.3,102.5 33.5))")
.value();
auto got = geoIndex.intersects(point);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({3759379788947521536,
3879094614979772416,
3915809507254468608,
3917005775905488896,
3922635275439702016,
3931642474694443008,
3949656873203924992,
3958664072458665984});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
{
bool pointsOnly = false; // The indexed geo column only has points;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto point =
Geography::fromWKT("POLYGON((102.5 33.5, 110.6 36.9,113.6 30.4,102.7 27.3,102.5 33.5))")
.value();
auto got = geoIndex.intersects(point);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({3759379788947521536,
3879094614979772416,
3915809507254468608,
3917005775905488896,
3922635275439702016,
3931642474694443008,
3949656873203924992,
3958664072458665984});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
}

} // namespace geo
} // namespace nebula

Expand Down
21 changes: 16 additions & 5 deletions src/common/utils/IndexKeyUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@

#include <thrift/lib/cpp2/protocol/Serializer.h>

#include "common/geo/GeoIndex.h"

namespace nebula {

// static
std::vector<std::string> IndexKeyUtils::encodeValues(
std::vector<Value>&& values, const std::vector<nebula::meta::cpp2::ColumnDef>& cols) {
std::vector<std::string> IndexKeyUtils::encodeValues(std::vector<Value>&& values,
const meta::cpp2::IndexItem* indexItem) {
auto& cols = indexItem->get_fields();
bool hasNullCol = false;
// An index has a maximum of 16 columns. 2 byte (16 bit) is enough.
u_short nullableBitSet = 0;
Expand Down Expand Up @@ -52,7 +55,14 @@ std::vector<std::string> IndexKeyUtils::encodeValues(
const auto& value = values.back();
if (!value.isNull()) {
DCHECK(value.type() == Value::Type::GEOGRAPHY);
indexes = encodeGeography(value.getGeography());
geo::RegionCoverParams rc;
if (indexItem->get_s2_max_level()) {
rc.maxCellLevel_ = *indexItem->get_s2_max_level();
}
if (indexItem->get_s2_max_cells()) {
rc.maxCellNum_ = *indexItem->get_s2_max_level();
}
indexes = encodeGeography(value.getGeography(), rc);
} else {
nullableBitSet |= 0x8000;
auto type = IndexKeyUtils::toValueType(cols.back().type.get_type());
Expand Down Expand Up @@ -157,10 +167,11 @@ Value IndexKeyUtils::parseIndexTTL(const folly::StringPiece& raw) {

// static
StatusOr<std::vector<std::string>> IndexKeyUtils::collectIndexValues(
RowReader* reader, const std::vector<nebula::meta::cpp2::ColumnDef>& cols) {
RowReader* reader, const meta::cpp2::IndexItem* indexItem) {
if (reader == nullptr) {
return Status::Error("Invalid row reader");
}
auto& cols = indexItem->get_fields();
std::vector<Value> values;
for (const auto& col : cols) {
auto v = reader->getValueByName(col.get_name());
Expand All @@ -172,7 +183,7 @@ StatusOr<std::vector<std::string>> IndexKeyUtils::collectIndexValues(
}
values.emplace_back(std::move(v));
}
return encodeValues(std::move(values), cols);
return encodeValues(std::move(values), indexItem);
}

// static
Expand Down
Loading

0 comments on commit 064ed80

Please sign in to comment.