Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support for nullable prop #239

Merged
merged 1 commit into from
Dec 6, 2022
Merged

feat: support for nullable prop #239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,13 @@ schema:
index: 1
- name: gender
type: string
- name: phone
type: string
nullable: true
- name: wechat
type: string
nullable: true
nullValue: "__NULL__"
```

##### `schema.vertex.vid`
Expand All @@ -225,6 +232,8 @@ Each tag contains the following two properties:
* `name`: **Required**. The property name, must be the same with the tag property in Nebula Graph.
* `type`: **Optional**. The property type, currently `bool`, `int`, `float`, `double`, `string`, `time`, `timestamp`, `date`, `datetime`, `geography`, `geography(point)`, `geography(linestring)` and `geography(polygon)` are supported.
* `index`: **Optional**. The column number in the CSV file.
* `nullable`: **Optional**. Whether this prop property can be `NULL`, optional values is `true` or `false`, default `false`.
* `nullValue`: **Optional**. If `nullable` is set to `true`, the property will be set to `NULL` when the value is equal to `nullValue`, default `""`.

> **NOTE**: The properties in the preceding `prop` parameter must be sorted in the **same** way as in the CSV data file.

Expand Down
2 changes: 2 additions & 0 deletions README_zh-CN.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,8 @@ schema:
- `name`：**必填**。属性名称，同 Nebula Graph 中创建的 TAG 的属性名称一致。
- `type`：**必填**。属性类型，目前支持 `bool`、`int`、`float`、`double`、`timestamp`、`string`、`geography`、`geography(point)`、`geography(linestring)`和`geography(polygon)` 几种类型。
- `index`：**可选**。在 CSV 文件中的列标。
- `nullable`：**可选**。此属性是否可以为 `NULL`，可选 `true` 或者 `false`，默认值为 `false` 。
- `nullValue`：**可选**。如果 `nullable` 设置为 `true`，则当值等于 `nullValue` 的时候属性将被设置为 `NULL` ，默认值为 `""`。

> **注意**：上述 `props` 中的属性描述**顺序**必须同数据文件中的对应数据排列顺序一致。

Expand Down
2 changes: 1 addition & 1 deletion examples/v1/example.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ clientSettings:
CREATE TAG course_no_props();
CREATE TAG building_no_props();
CREATE EDGE follow_no_props();
afterPeriod: 8s
afterPeriod: 10s
preStop:
commands: |
UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = false };
Expand Down
2 changes: 2 additions & 0 deletions examples/v2/basic_type_test.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@ b2,false,0,0,0.0,0
b3,true,1,2.0,3.3,abc
b4,false,3,2.0,3.3,0a bd
b5,true,-3,2,3,abcd efg
bnull1,,,,,
bnull2,,,,,__NULL__
1 change: 1 addition & 0 deletions examples/v2/date_test.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
d1,2020-01-01,18:28:23.284,2020-01-01T18:28:23.284,2020-01-01T18:28:23
d2,2020-01-02,18:38:23.284,2020-01-11T19:28:23.284,1578770903
dnull,,,,
42 changes: 35 additions & 7 deletions examples/v2/example.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,14 @@ clientSettings:
CREATE TAG course_no_props();
CREATE TAG building_no_props();
CREATE EDGE follow_no_props();
CREATE TAG basic_type_test(b bool, i int, f float, d double, s string);
CREATE EDGE edge_basic_type_test(b bool, i int, f float, d double, s string);
CREATE TAG date_test(c1 date, c2 time, c3 datetime, c4 timestamp);
CREATE EDGE edge_date_test(c1 date, c2 time, c3 datetime, c4 timestamp);
CREATE TAG geography_test(any_shape geography, only_point geography(point), only_linestring geography(linestring), only_polygon geography(polygon));
CREATE EDGE edge_geography_test(any_shape geography, only_point geography(point), only_linestring geography(linestring), only_polygon geography(polygon))
afterPeriod: 8s
CREATE TAG basic_type_test(b bool NULL, i int NULL, f float NULL, d double NULL, s string NULL);
CREATE EDGE edge_basic_type_test(b bool NULL, i int NULL, f float NULL, d double NULL, s string NULL);
CREATE TAG date_test(c1 date NULL, c2 time NULL, c3 datetime NULL, c4 timestamp NULL);
CREATE EDGE edge_date_test(c1 date NULL, c2 time NULL, c3 datetime NULL, c4 timestamp NULL);
CREATE TAG geography_test(any_shape geography NULL, only_point geography(point) NULL, only_linestring geography(linestring) NULL, only_polygon geography(polygon) NULL);
CREATE EDGE edge_geography_test(any_shape geography NULL, only_point geography(point) NULL, only_linestring geography(linestring) NULL, only_polygon geography(polygon) NULL)

afterPeriod: 10s
preStop:
commands: |
UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = false };
Expand Down Expand Up @@ -478,18 +479,23 @@ files:
- name: b
type: bool
index: 1
nullable: true
- name: i
type: int
index: 2
nullable: true
- name: f
type: float
index: 3
nullable: true
- name: d
type: double
index: 4
nullable: true
- name: s
type: string
index: 5
nullable: true

- path: ./basic_type_test.csv
failDataPath: ./err/edge_basic_type_test.csv
Expand All @@ -513,18 +519,24 @@ files:
- name: b
type: bool
index: 1
nullable: true
- name: i
type: int
index: 2
nullable: true
- name: f
type: float
index: 3
nullable: true
- name: d
type: double
index: 4
nullable: true
- name: s
type: string
index: 5
nullable: true
nullValue: "__NULL__"

- path: ./date_test.csv
failDataPath: ./err/date_test.csv
Expand All @@ -546,15 +558,19 @@ files:
- name: c1
type: date
index: 1
nullable: true
- name: c2
type: time
index: 2
nullable: true
- name: c3
type: datetime
index: 3
nullable: true
- name: c4
type: timestamp
index: 4
nullable: true

- path: ./date_test.csv
failDataPath: ./err/edge_date_test.csv
Expand All @@ -578,15 +594,19 @@ files:
- name: c1
type: date
index: 1
nullable: true
- name: c2
type: time
index: 2
nullable: true
- name: c3
type: datetime
index: 3
nullable: true
- name: c4
type: timestamp
index: 4
nullable: true

- path: ./geography_test.csv
failDataPath: ./err/geography_test.csv
Expand All @@ -608,15 +628,19 @@ files:
- name: any_shape
type: geography
index: 1
nullable: true
- name: only_point
type: geography(point)
index: 2
nullable: true
- name: only_linestring
type: geography(linestring)
index: 3
nullable: true
- name: only_polygon
type: geography(polygon)
index: 4
nullable: true

- path: ./geography_test.csv
failDataPath: ./err/edge_geography_test.csv
Expand All @@ -640,12 +664,16 @@ files:
- name: any_shape
type: geography
index: 1
nullable: true
- name: only_point
type: geography(point)
index: 2
nullable: true
- name: only_linestring
type: geography(linestring)
index: 3
nullable: true
- name: only_polygon
type: geography(polygon)
index: 4
nullable: true
1 change: 1 addition & 0 deletions examples/v2/geography_test.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
g1,POINT (-82.3764154 42.6452196),"Point(3 8)","LINEstring (-108.7 35.0,-100.0 46.5,-90.7 34.9,-108.7 35.0)","POlygon ( (-100.1 41.4,-102.9 37.6,-96.8 37.5,-100.1 41.4))"
g2,"LineString(0 1, 1 2, 2 3)","point(4.6 5.7 )","LINESTRING(43.8 52.6, -78.99 84.323)","POLYGON ((-108.7 35.0,-100.0 46.5,-90.7 34.9,-108.7 35.0))"
g3,"Polygon((-85.1 34.8,-80.7 28.4,-76.9 34.9,-85.1 34.8))", Point(0.0 0.0),"linestring(0 1, 179.99 89.99)","polygon((0 1, 2 4, 3 5, 4 9, 0 1))"
gnull,,,,
27 changes: 18 additions & 9 deletions pkg/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ import (
"gopkg.in/yaml.v2"
)

const (
dbNULL = "NULL"
)

var (
reTimestampInteger = regexp.MustCompile(`^(0[xX][0-9a-fA-F]+|0[0-7]+|\d+)$`)
)
Expand Down Expand Up @@ -47,9 +51,11 @@ type NebulaClientSettings struct {
}

type Prop struct {
Name *string `json:"name" yaml:"name"`
Type *string `json:"type" yaml:"type"`
Index *int `json:"index" yaml:"index"`
Name *string `json:"name" yaml:"name"`
Type *string `json:"type" yaml:"type"`
Index *int `json:"index" yaml:"index"`
Nullable bool `json:"nullable" yaml:"nullable"`
NullValue string `json:"nullValue" yaml:"nullValue"`
}

type VID struct {
Expand Down Expand Up @@ -466,17 +472,17 @@ func (s *Schema) validateAndReset(prefix string) error {
var err error = nil
switch strings.ToLower(*s.Type) {
case "edge":
if s.Edge != nil {
err = s.Edge.validateAndReset(fmt.Sprintf("%s.edge", prefix))
} else {
if s.Edge == nil {
logger.Log.Infof("%s.edge is nil", prefix)
s.Edge = &Edge{}
}
err = s.Edge.validateAndReset(fmt.Sprintf("%s.edge", prefix))
case "vertex":
if s.Vertex != nil {
err = s.Vertex.validateAndReset(fmt.Sprintf("%s.vertex", prefix))
} else {
if s.Vertex == nil {
logger.Log.Infof("%s.vertex is nil", prefix)
s.Vertex = &Vertex{}
}
err = s.Vertex.validateAndReset(fmt.Sprintf("%s.vertex", prefix))
default:
err = fmt.Errorf("Error schema type(%s) in %s.type only edge and vertex are supported", *s.Type, prefix)
}
Expand Down Expand Up @@ -830,6 +836,9 @@ func (p *Prop) FormatValue(record base.Record) (string, error) {
return "", fmt.Errorf("Prop index %d out range %d of record(%v)", *p.Index, len(record), record)
}
r := record[*p.Index]
if p.Nullable && r == p.NullValue {
return dbNULL, nil
}
if p.IsStringType() {
return fmt.Sprintf("%q", r), nil
}
Expand Down
Loading