From ca510f9d4dd218f7824ee856ffb8d2a911617296 Mon Sep 17 00:00:00 2001 From: "jie.wang" <38901892+jievince@users.noreply.github.com> Date: Thu, 21 Oct 2021 10:56:43 +0800 Subject: [PATCH] Support to import data of type geography (#169) * support geo value * support geo value --- .gitignore | 3 +++ README.md | 4 ++-- README_zh-CN.md | 4 ++-- examples/v2/example.yaml | 33 ++++++++++++++++++++++++++++++++- examples/v2/geography_test.csv | 3 +++ pkg/base/tools.go | 2 +- pkg/base/tools_test.go | 4 ++++ pkg/config/config.go | 9 +++++++++ pkg/logger/adapter.go | 8 ++++---- 9 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 examples/v2/geography_test.csv diff --git a/.gitignore b/.gitignore index 1a86b285..50f6a154 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ err/ vendor/ nebula-importer + +# IDE +.vscode/ diff --git a/README.md b/README.md index 5dc6fbeb..e2771046 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Each tag contains the following two properties: * `name`: The tag name. * `prop`: A property of the tag. Each property contains the following two fields: * `name`: **Required**. The property name, must be the same with the tag property in Nebula Graph. - * `type`: **Optional**. The property type, currently `bool`, `int`, `float`, `double`, `string`, `time`, `timestamp`, `date` and `datetime` are supported. + * `type`: **Optional**. The property type, currently `bool`, `int`, `float`, `double`, `string`, `time`, `timestamp`, `date`, `datetime`, `geography`, `geography(point)`, `geography(linestring)` and `geography(polygon)` are supported. * `index`: **Optional**. The column number in the CSV file. > **NOTE**: The properties in the preceding `prop` parameter must be sorted in the **same** way as in the CSV data file. @@ -315,7 +315,7 @@ The format for each column is `.:`: * `` is the name for the vertex or edge. * `` is the property name. -* `` is the property type. It can be `bool`, `int`, `float`, `double`, `string`, `time`, `timestamp`, `date` and `datetime`. The default type is `string`. +* `` is the property type. It can be `bool`, `int`, `float`, `double`, `string`, `time`, `timestamp`, `date`, `datetime`, `geography`, `geography(point)`, `geography(linestring)` and `geography(polygon)`. The default type is `string`. In the above `` field, the following keywords contain special semantics: diff --git a/README_zh-CN.md b/README_zh-CN.md index af3e83ac..a145607b 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -192,7 +192,7 @@ schema: - `name`：TAG 的名称。 - `props`：TAG 的属性字段数组，每个属性字段又由如下两个字段构成： - `name`：**必填**。属性名称，同 Nebula Graph 中创建的 TAG 的属性名称一致。 - - `type`：**必填**。属性类型，目前支持 `bool`、`int`、`float`、`double`、`timestamp` 和 `string` 几种类型。 + - `type`：**必填**。属性类型，目前支持 `bool`、`int`、`float`、`double`、`timestamp`、`string`、`geography`、`geography(point)`、`geography(linestring)`和`geography(polygon)` 几种类型。 - `index`：**可选**。在 CSV 文件中的列标。 > **注意**：上述 `props` 中的属性描述**顺序**必须同数据文件中的对应数据排列顺序一致。 @@ -268,7 +268,7 @@ example 中 choose 类型的边的样例数据： - `` 表示 TAG 或者 EDGE 的名称。 - `` 表示属性名称。 -- `` 表示属性类型。可以是 `bool`、`int`、`float`、`double`、`string` 和 `timestamp`，不设置默认为 `string`。 +- `` 表示属性类型。可以是 `bool`、`int`、`float`、`double`、`string`、`timestamp`、`geography`、`geography(point)`、`geography(linestring)`和`geography(polygon)`，不设置默认为 `string`。 在上述的 `` 字段中有如下几个关键词含有特殊语义： diff --git a/examples/v2/example.yaml b/examples/v2/example.yaml index 580b96a9..632638a9 100644 --- a/examples/v2/example.yaml +++ b/examples/v2/example.yaml @@ -25,7 +25,8 @@ clientSettings: CREATE TAG course_no_props(); CREATE TAG building_no_props(); CREATE EDGE follow_no_props(); - CREATE TAG date_test(c1 date, c2 time, c3 datetime, c4 timestamp) + CREATE TAG date_test(c1 date, c2 time, c3 datetime, c4 timestamp); + CREATE TAG geography_test(any_shape geography, only_point geography(point), only_linestring geography(linestring), only_polygon geography(polygon)) afterPeriod: 8s preStop: commands: | @@ -440,3 +441,33 @@ files: - name: c4 type: timestamp index: 4 + + - path: ./geography_test.csv + failDataPath: ./err/geography_test.csv + batchSize: 2 + inOrder: true + type: csv + csv: + withHeader: false + withLabel: false + delimiter: "," + schema: + type: vertex + vertex: + vid: + index: 0 + tags: + - name: geography_test + props: + - name: any_shape + type: geography + index: 1 + - name: only_point + type: geography(point) + index: 2 + - name: only_linestring + type: geography(linestring) + index: 3 + - name: only_polygon + type: geography(polygon) + index: 4 diff --git a/examples/v2/geography_test.csv b/examples/v2/geography_test.csv new file mode 100644 index 00000000..20cd8392 --- /dev/null +++ b/examples/v2/geography_test.csv @@ -0,0 +1,3 @@ +g1,POINT (-82.3764154 42.6452196),"Point(3 8)","LINEstring (-108.7 35.0,-100.0 46.5,-90.7 34.9,-108.7 35.0)","POlygon ( (-100.1 41.4,-102.9 37.6,-96.8 37.5,-100.1 41.4))" +g2,"LineString(0 1, 1 2, 2 3)","point(4.6 5.7 )","LINESTRING(43.8 52.6, -78.99 84.323)","POLYGON ((-108.7 35.0,-100.0 46.5,-90.7 34.9,-108.7 35.0))" +g3,"Polygon((-85.1 34.8,-80.7 28.4,-76.9 34.9,-85.1 34.8))", Point(0.0 0.0),"linestring(0 1, 179.99 89.99)","polygon((0 1, 2 4, 3 5, 4 9, 0 1))" diff --git a/pkg/base/tools.go b/pkg/base/tools.go index 5a6c6ecb..76af6a8a 100644 --- a/pkg/base/tools.go +++ b/pkg/base/tools.go @@ -29,7 +29,7 @@ func FileExists(filename string) bool { func IsValidType(t string) bool { switch strings.ToLower(t) { - case "string", "int", "float", "double", "bool", "date", "time", "datetime", "timestamp": + case "string", "int", "float", "double", "bool", "date", "time", "datetime", "timestamp", "geography", "geography(point)", "geography(linestring)", "geography(polygon)": return true default: return false diff --git a/pkg/base/tools_test.go b/pkg/base/tools_test.go index 2d0a9238..8d4a2cf6 100644 --- a/pkg/base/tools_test.go +++ b/pkg/base/tools_test.go @@ -28,4 +28,8 @@ func TestIsValidType(t *testing.T) { assert.True(t, IsValidType("bool")) assert.True(t, IsValidType("timestamp")) assert.True(t, IsValidType("double")) + assert.True(t, IsValidType("geography")) + assert.True(t, IsValidType("geography(point)")) + assert.True(t, IsValidType("geography(linestring)")) + assert.True(t, IsValidType("geography(polygon)")) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 0be557f2..591714cd 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -791,6 +791,11 @@ func (p *Prop) IsDateOrTimeType() bool { return t == "date" || t == "time" || t == "datetime" || t == "timestamp" } +func (p *Prop) IsGeographyType() bool { + t := strings.ToLower(*p.Type) + return t == "geography" || t == "geography(point)" || t == "geography(linestring)" || t == "geography(polygon)" +} + func (p *Prop) FormatValue(record base.Record) (string, error) { if p.Index != nil && *p.Index >= len(record) { return "", fmt.Errorf("Prop index %d out range %d of record(%v)", *p.Index, len(record), record) @@ -802,6 +807,10 @@ func (p *Prop) FormatValue(record base.Record) (string, error) { if p.IsDateOrTimeType() { return fmt.Sprintf("%s(%q)", strings.ToLower(*p.Type), r), nil } + // Only support wkt for geography currently + if p.IsGeographyType() { + return fmt.Sprintf("ST_GeogFromText(%q)", r), nil + } return r, nil } diff --git a/pkg/logger/adapter.go b/pkg/logger/adapter.go index 7761c782..0fcf9cc6 100644 --- a/pkg/logger/adapter.go +++ b/pkg/logger/adapter.go @@ -7,17 +7,17 @@ import ( type NebulaLogger struct{} func (l NebulaLogger) Info(msg string) { - infoWithSkip(2, fmt.Sprintf("[nebula-clients] %s", msg)) + infoWithSkip(2, fmt.Sprintf("[nebula-go] %s", msg)) } func (l NebulaLogger) Warn(msg string) { - warnWithSkip(2, fmt.Sprintf("[nebula-clients] %s", msg)) + warnWithSkip(2, fmt.Sprintf("[nebula-go] %s", msg)) } func (l NebulaLogger) Error(msg string) { - errorWithSkip(2, fmt.Sprintf("[nebula-clients] %s", msg)) + errorWithSkip(2, fmt.Sprintf("[nebula-go] %s", msg)) } func (l NebulaLogger) Fatal(msg string) { - fatalWithSkip(2, fmt.Sprintf("[nebula-clients] %s", msg)) + fatalWithSkip(2, fmt.Sprintf("[nebula-go] %s", msg)) }