diff --git a/x-pack/plugin/sql/sql-shared-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java b/x-pack/plugin/sql/sql-shared-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java index 7d6e5f8c9a983..6dcb420ec76b6 100644 --- a/x-pack/plugin/sql/sql-shared-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java +++ b/x-pack/plugin/sql/sql-shared-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java @@ -42,14 +42,16 @@ public enum DataType { // see https://github.com/elastic/elasticsearch/issues/30386#issuecomment-386807288 DATE( JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES, 24, 24), // TODO: This should map to some Geography class instead of String - GEO_SHAPE( JDBCType.OTHER, String.class, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false); + GEO_SHAPE( JDBCType.OTHER, String.class, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false), + GEO_POINT( JDBCType.OTHER, String.class, Double.BYTES*2, Integer.MAX_VALUE, 0, false, false, false); // @formatter:on private static final Map jdbcToEs; static { jdbcToEs = Arrays.stream(DataType.values()) - .filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT && dataType != GEO_SHAPE) + .filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT && dataType != GEO_SHAPE && + dataType != GEO_POINT) .collect(Collectors.toMap(dataType -> dataType.jdbcType, dataType -> dataType)); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java index d4c354552f879..60f1459b04bb2 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java @@ -58,11 +58,11 @@ public void testSysTypes() throws Exception { Command cmd = sql("SYS TYPES").v1(); List names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE", - "KEYWORD", "TEXT", "BOOLEAN", "DATE", "UNSUPPORTED", "GEO_SHAPE", "OBJECT", "NESTED"); + "KEYWORD", "TEXT", "BOOLEAN", "DATE", "UNSUPPORTED", "GEO_SHAPE", "GEO_POINT", "OBJECT", "NESTED"); cmd.execute(null, ActionListener.wrap(r -> { assertEquals(19, r.columnCount()); - assertEquals(18, r.size()); + assertEquals(names.size(), r.size()); assertFalse(r.schema().types().contains(DataType.NULL)); // test numeric as signed assertFalse(r.column(9, Boolean.class)); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java index f478ddf14f852..3992b86a2819b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java @@ -15,6 +15,7 @@ import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.sql.type.DataType.DATE; +import static org.elasticsearch.xpack.sql.type.DataType.GEO_POINT; import static org.elasticsearch.xpack.sql.type.DataType.GEO_SHAPE; import static org.elasticsearch.xpack.sql.type.DataType.INTEGER; import static org.elasticsearch.xpack.sql.type.DataType.KEYWORD; @@ -39,13 +40,12 @@ public void testEmptyMap() { public void testBasicMapping() { Map mapping = loadMapping("mapping-basic.json"); - assertThat(mapping.size(), is(7)); + assertThat(mapping.size(), is(6)); assertThat(mapping.get("emp_no").getDataType(), is(INTEGER)); assertThat(mapping.get("first_name"), instanceOf(TextEsField.class)); assertThat(mapping.get("last_name").getDataType(), is(TEXT)); assertThat(mapping.get("gender").getDataType(), is(KEYWORD)); assertThat(mapping.get("salary").getDataType(), is(INTEGER)); - assertThat(mapping.get("site").getDataType(), is(GEO_SHAPE)); } public void testDefaultStringMapping() { @@ -181,8 +181,11 @@ public void testNestedDoc() { public void testGeoField() { Map mapping = loadMapping("mapping-geo.json"); - EsField dt = mapping.get("location"); - assertThat(dt.getDataType().esType, is("unsupported")); + assertThat(mapping.size(), is(2)); + EsField gp = mapping.get("location"); + assertThat(gp.getDataType().esType, is("geo_point")); + EsField gs = mapping.get("site"); + assertThat(gs.getDataType().esType, is("geo_shape")); } public void testUnsupportedTypes() { @@ -202,4 +205,4 @@ public static Map loadMapping(String name, boolean ordered) { assertNotNull("Could not find mapping resource:" + name, stream); return Types.fromEs(XContentHelper.convertToMap(JsonXContent.jsonXContent, stream, ordered)); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/test/resources/mapping-basic.json b/x-pack/plugin/sql/src/test/resources/mapping-basic.json index 3c6d937bd7042..c1747d1561c6d 100644 --- a/x-pack/plugin/sql/src/test/resources/mapping-basic.json +++ b/x-pack/plugin/sql/src/test/resources/mapping-basic.json @@ -17,9 +17,6 @@ }, "salary" : { "type" : "integer" - }, - "site": { - "type": "geo_shape" } } } diff --git a/x-pack/plugin/sql/src/test/resources/mapping-geo.json b/x-pack/plugin/sql/src/test/resources/mapping-geo.json index 3c958ff37edfc..e6e499ef82e83 100644 --- a/x-pack/plugin/sql/src/test/resources/mapping-geo.json +++ b/x-pack/plugin/sql/src/test/resources/mapping-geo.json @@ -2,6 +2,9 @@ "properties" : { "location" : { "type" : "geo_point" + }, + "site": { + "type" : "geo_shape" } } } diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoCsvSpecTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoCsvSpecTestCase.java index 73ff9f4f4205f..e02e55612ace4 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoCsvSpecTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoCsvSpecTestCase.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.qa.sql.geo; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.client.Request; import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase; import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase; import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration; @@ -34,6 +35,7 @@ public static List readScriptSpec() throws Exception { Parser parser = specParser(); List tests = new ArrayList<>(); tests.addAll(readScriptSpec("/ogc/ogc.csv-spec", parser)); + tests.addAll(readScriptSpec("/geo/geosql.csv-spec", parser)); return tests; } @@ -45,8 +47,11 @@ public GeoCsvSpecTestCase(String fileName, String groupName, String testName, In @Before public void setupTestGeoDataIfNeeded() throws Exception { - if (client().performRequest("HEAD", "/ogc").getStatusLine().getStatusCode() == 404) { - GeoDataLoader.loadDatasetIntoEs(client()); + if (client().performRequest(new Request("HEAD", "/ogc")).getStatusLine().getStatusCode() == 404) { + GeoDataLoader.loadOGCDatasetIntoEs(client(), "ogc"); + } + if (client().performRequest(new Request("HEAD", "/geo")).getStatusLine().getStatusCode() == 404) { + GeoDataLoader.loadGeoDatasetIntoEs(client(), "geo"); } } diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoDataLoader.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoDataLoader.java index ec48384a73bad..29c00257c5bda 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoDataLoader.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoDataLoader.java @@ -11,6 +11,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; @@ -37,26 +38,28 @@ public class GeoDataLoader { public static void main(String[] args) throws Exception { try (RestClient client = RestClient.builder(new HttpHost("localhost", 9200)).build()) { - loadDatasetIntoEs(client); + loadOGCDatasetIntoEs(client, "ogc"); + loadGeoDatasetIntoEs(client, "geo"); Loggers.getLogger(GeoDataLoader.class).info("Geo data loaded"); } } - protected static void loadDatasetIntoEs(RestClient client) throws Exception { - loadDatasetIntoEs(client, "ogc"); - makeFilteredAlias(client, "lakes", "ogc", "\"term\" : { \"ogc_type\" : \"lakes\" }"); - makeFilteredAlias(client, "road_segments", "ogc", "\"term\" : { \"ogc_type\" : \"road_segments\" }"); - makeFilteredAlias(client, "divided_routes", "ogc", "\"term\" : { \"ogc_type\" : \"divided_routes\" }"); - makeFilteredAlias(client, "forests", "ogc", "\"term\" : { \"ogc_type\" : \"forests\" }"); - makeFilteredAlias(client, "bridges", "ogc", "\"term\" : { \"ogc_type\" : \"bridges\" }"); - makeFilteredAlias(client, "streams", "ogc", "\"term\" : { \"ogc_type\" : \"streams\" }"); - makeFilteredAlias(client, "buildings", "ogc", "\"term\" : { \"ogc_type\" : \"buildings\" }"); - makeFilteredAlias(client, "ponds", "ogc", "\"term\" : { \"ogc_type\" : \"ponds\" }"); - makeFilteredAlias(client, "named_places", "ogc", "\"term\" : { \"ogc_type\" : \"named_places\" }"); - makeFilteredAlias(client, "map_neatlines", "ogc", "\"term\" : { \"ogc_type\" : \"map_neatlines\" }"); + protected static void loadOGCDatasetIntoEs(RestClient client, String index) throws Exception { + createIndex(client, index, createOGCIndexRequest()); + loadData(client, index, readResource("/ogc/ogc.json")); + makeFilteredAlias(client, "lakes", index, "\"term\" : { \"ogc_type\" : \"lakes\" }"); + makeFilteredAlias(client, "road_segments", index, "\"term\" : { \"ogc_type\" : \"road_segments\" }"); + makeFilteredAlias(client, "divided_routes", index, "\"term\" : { \"ogc_type\" : \"divided_routes\" }"); + makeFilteredAlias(client, "forests", index, "\"term\" : { \"ogc_type\" : \"forests\" }"); + makeFilteredAlias(client, "bridges", index, "\"term\" : { \"ogc_type\" : \"bridges\" }"); + makeFilteredAlias(client, "streams", index, "\"term\" : { \"ogc_type\" : \"streams\" }"); + makeFilteredAlias(client, "buildings", index, "\"term\" : { \"ogc_type\" : \"buildings\" }"); + makeFilteredAlias(client, "ponds", index, "\"term\" : { \"ogc_type\" : \"ponds\" }"); + makeFilteredAlias(client, "named_places", index, "\"term\" : { \"ogc_type\" : \"named_places\" }"); + makeFilteredAlias(client, "map_neatlines", index, "\"term\" : { \"ogc_type\" : \"map_neatlines\" }"); } - protected static void loadDatasetIntoEs(RestClient client, String index) throws Exception { + private static String createOGCIndexRequest() throws Exception { XContentBuilder createIndex = JsonXContent.contentBuilder().startObject(); createIndex.startObject("settings"); { @@ -101,12 +104,21 @@ protected static void loadDatasetIntoEs(RestClient client, String index) throws createIndex.endObject(); } createIndex.endObject().endObject(); + return Strings.toString(createIndex); + } - client.performRequest("PUT", "/" + index, emptyMap(), new StringEntity(Strings.toString(createIndex), - ContentType.APPLICATION_JSON)); + private static void createIndex(RestClient client, String index, String settingsMappings) throws IOException { + Request createIndexRequest = new Request("PUT", "/" + index); + createIndexRequest.setEntity(new StringEntity(settingsMappings, ContentType.APPLICATION_JSON)); + client.performRequest(createIndexRequest); + } - String bulk = readResource("/ogc/ogc.json"); + static void loadGeoDatasetIntoEs(RestClient client, String index) throws Exception { + createIndex(client, index, readResource("/geo/geosql.json")); + loadData(client, index, readResource("/geo/geosql-bulk.json")); + } + private static void loadData(RestClient client, String index, String bulk) throws IOException { Response response = client.performRequest("POST", "/" + index + "/doc/_bulk", singletonMap("refresh", "true"), new StringEntity(bulk, ContentType.APPLICATION_JSON)); diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoSqlSpecTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoSqlSpecTestCase.java index 54ee615d73adf..f0ddbd060cb95 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoSqlSpecTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoSqlSpecTestCase.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.qa.sql.geo; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.client.Request; import org.elasticsearch.xpack.qa.sql.jdbc.LocalH2; import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase; import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration; @@ -47,8 +48,8 @@ public static List readScriptSpec() throws Exception { public void setupTestGeoDataIfNeeded() throws Exception { assumeTrue("Cannot support locales that don't use Hindu-Arabic numerals due to H2", "42".equals(NumberFormat.getInstance(Locale.getDefault()).format(42))); - if (client().performRequest("HEAD", "/ogc").getStatusLine().getStatusCode() == 404) { - GeoDataLoader.loadDatasetIntoEs(client()); + if (client().performRequest(new Request("HEAD", "/ogc")).getStatusLine().getStatusCode() == 404) { + GeoDataLoader.loadOGCDatasetIntoEs(client(), "ogc"); } } diff --git a/x-pack/qa/sql/src/main/resources/geo/geosql-bulk.json b/x-pack/qa/sql/src/main/resources/geo/geosql-bulk.json new file mode 100644 index 0000000000000..495b37174fc94 --- /dev/null +++ b/x-pack/qa/sql/src/main/resources/geo/geosql-bulk.json @@ -0,0 +1,33 @@ +{"index":{"_id": "1"}} +{"region": "Americas", "city": "Mountain View", "location": "37.386483,-122.083843"} +{"index":{"_id": "2"}} +{"region": "Americas", "city": "Chicago", "location": "41.888783,-87.637874"} +{"index":{"_id": "3"}} +{"region": "Americas", "city": "New York", "location": "40.745171,-73.990027"} +{"index":{"_id": "4"}} +{"region": "Americas", "city": "San Francisco", "location": "37.789541,-122.394228"} +{"index":{"_id": "5"}} +{"region": "Americas", "city": "Phoenix", "location": "33.376242,-111.973505"} +{"index":{"_id": "6"}} +{"region": "Europe", "city": "Amsterdam", "location": "52.347557,4.850312"} +{"index":{"_id": "7"}} +{"region": "Europe", "city": "Berlin", "location": "52.486701,13.390889"} +{"index":{"_id": "8"}} +{"region": "Europe", "city": "Munich", "location": "48.146321,11.537505"} +{"index":{"_id": "9"}} +{"region": "Europe", "city": "London", "location": "51.510871,-0.121672"} +{"index":{"_id": "10"}} +{"region": "Europe", "city": "Paris", "location": "48.845538,2.351773"} +{"index":{"_id": "11"}} +{"region": "Asia", "city": "Singapore", "location": "1.295868,103.855535"} +{"index":{"_id": "12"}} +{"region": "Asia", "city": "Hong Kong", "location": "22.281397,114.183925"} +{"index":{"_id": "13"}} +{"region": "Asia", "city": "Seoul", "location": "37.509132,127.060851"} +{"index":{"_id": "14"}} +{"region": "Asia", "city": "Tokyo", "location": "35.669616,139.76402225"} +{"index":{"_id": "15"}} +{"region": "Asia", "city": "Sydney", "location": "-33.863385,151.208629"} + + + diff --git a/x-pack/qa/sql/src/main/resources/geo/geosql.csv-spec b/x-pack/qa/sql/src/main/resources/geo/geosql.csv-spec new file mode 100644 index 0000000000000..bcf93d224738d --- /dev/null +++ b/x-pack/qa/sql/src/main/resources/geo/geosql.csv-spec @@ -0,0 +1,45 @@ +// +// Commands on geo test data +// + +showTables +SHOW TABLES 'geo'; + + name:s | type:s +geo |BASE TABLE +; + +// DESCRIBE + +describe +DESCRIBE "geo"; + + column:s | type:s +city | VARCHAR +location | OTHER +region | VARCHAR +; + +// SELECT ALL +// TODO: For now we just get geopoint formatted as is and we also need to convert it to STRING to work with CSV + +selectAllPoints +SELECT city, CAST(location AS STRING) location, region FROM "geo" ORDER BY "city"; + + city:s | location:s | region:s +Amsterdam |52.347557,4.850312 |Europe +Berlin |52.486701,13.390889 |Europe +Chicago |41.888783,-87.637874 |Americas +Hong Kong |22.281397,114.183925 |Asia +London |51.510871,-0.121672 |Europe +Mountain View |37.386483,-122.083843 |Americas +Munich |48.146321,11.537505 |Europe +New York |40.745171,-73.990027 |Americas +Paris |48.845538,2.351773 |Europe +Phoenix |33.376242,-111.973505 |Americas +San Francisco |37.789541,-122.394228 |Americas +Seoul |37.509132,127.060851 |Asia +Singapore |1.295868,103.855535 |Asia +Sydney |-33.863385,151.208629 |Asia +Tokyo |35.669616,139.76402225|Asia +; \ No newline at end of file diff --git a/x-pack/qa/sql/src/main/resources/geo/geosql.json b/x-pack/qa/sql/src/main/resources/geo/geosql.json new file mode 100644 index 0000000000000..2231f583667e9 --- /dev/null +++ b/x-pack/qa/sql/src/main/resources/geo/geosql.json @@ -0,0 +1,20 @@ +{ + "settings": { + "number_of_shards": 1 + }, + "mappings": { + "doc": { + "properties": { + "region": { + "type": "keyword" + }, + "city": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + } + } +} \ No newline at end of file