From 80209c00fa1f31cb50ff3d52db41d4de5a2fa8fe Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Mon, 15 Feb 2021 20:53:16 +0200 Subject: [PATCH 1/6] feat: add connector for CrateDB --- setup.py | 1 + superset/db_engine_specs/crate.py | 64 +++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 superset/db_engine_specs/crate.py diff --git a/setup.py b/setup.py index e39284f147e2f..40cb32e7b5eee 100644 --- a/setup.py +++ b/setup.py @@ -121,6 +121,7 @@ def get_git_sha(): "clickhouse": ["clickhouse-sqlalchemy>= 0.1.4, <0.2"], "cockroachdb": ["cockroachdb>=0.3.5, <0.4"], "cors": ["flask-cors>=2.0.0"], + "crate": ["crate[sqlalchemy]>=0.26.0, <0.27"], "db2": ["ibm-db-sa>=0.3.5, <0.4"], "dremio": ["sqlalchemy-dremio>=1.1.5, <1.2"], "drill": ["sqlalchemy-drill==0.1.dev"], diff --git a/superset/db_engine_specs/crate.py b/superset/db_engine_specs/crate.py new file mode 100644 index 0000000000000..3a9b2c0ea9f9f --- /dev/null +++ b/superset/db_engine_specs/crate.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from datetime import datetime +from typing import Optional, TYPE_CHECKING + +from superset.db_engine_specs.base import BaseEngineSpec +from superset.utils import core as utils + +if TYPE_CHECKING: + from superset.connectors.sqla.models import TableColumn + + +class CrateEngineSpec(BaseEngineSpec): + + engine = "crate" + engine_name = "CrateDB" + + _time_grain_expressions = { + None: "{col}", + "PT1S": "DATE_TRUNC('second', {col})", + "PT1M": "DATE_TRUNC('minute', {col})", + "PT1H": "DATE_TRUNC('hour', {col})", + "P1D": "DATE_TRUNC('day', {col})", + "P1W": "DATE_TRUNC('week', {col})", + "P1M": "DATE_TRUNC('month', {col})", + "P0.25Y": "DATE_TRUNC('quarter', {col})", + "P1Y": "DATE_TRUNC('year', {col})", + } + + @classmethod + def epoch_to_dttm(cls) -> str: + return "{col} * 1000" + + @classmethod + def epoch_ms_to_dttm(cls) -> str: + return "{col}" + + @classmethod + def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + tt = target_type.upper() + if tt == utils.TemporalType.TIMESTAMP: + return f"{dttm.timestamp() * 1000}" + return None + + @classmethod + def alter_new_orm_column(cls, orm_col: "TableColumn") -> None: + if orm_col.type == "TIMESTAMP": + orm_col.type = "INT" + orm_col.is_dttm = True + orm_col.python_date_format = "epoch_ms" From 0c4070b7e64568553b7e336751b60a79ba704c48 Mon Sep 17 00:00:00 2001 From: Srini Kadamati Date: Tue, 16 Feb 2021 11:57:58 -0500 Subject: [PATCH 2/6] added crate --- .../docs/Connecting to Databases/crate.mdx | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 docs/src/pages/docs/Connecting to Databases/crate.mdx diff --git a/docs/src/pages/docs/Connecting to Databases/crate.mdx b/docs/src/pages/docs/Connecting to Databases/crate.mdx new file mode 100644 index 0000000000000..e157b20a468bb --- /dev/null +++ b/docs/src/pages/docs/Connecting to Databases/crate.mdx @@ -0,0 +1,18 @@ +--- +name: CrateDB +menu: Connecting to Databases +route: /docs/databases/cratedb +index: 30 +version: 1 +--- + +## CrateDB + +The recommended connector library for CrateDB is +[crate](https://pypi.org/project/crate/0.17.0/). + +The expected connection string is formatted as follows: + +``` +crate://crate@127.0.0.1:4200 +``` From d38dfb043e018c1e9b3af49ffec07495de8315d1 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Thu, 18 Feb 2021 10:43:00 +0200 Subject: [PATCH 3/6] fix PyPI reference --- docs/src/pages/docs/Connecting to Databases/crate.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/pages/docs/Connecting to Databases/crate.mdx b/docs/src/pages/docs/Connecting to Databases/crate.mdx index e157b20a468bb..b903ab8ba8c9e 100644 --- a/docs/src/pages/docs/Connecting to Databases/crate.mdx +++ b/docs/src/pages/docs/Connecting to Databases/crate.mdx @@ -9,7 +9,7 @@ version: 1 ## CrateDB The recommended connector library for CrateDB is -[crate](https://pypi.org/project/crate/0.17.0/). +[crate](https://pypi.org/project/crate/). The expected connection string is formatted as follows: From 27f003bd5b690f9033a2b796c191ec716e48b87e Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Thu, 18 Feb 2021 11:33:24 +0200 Subject: [PATCH 4/6] add tests --- superset/db_engine_specs/crate.py | 2 -- tests/db_engine_specs/crate_tests.py | 51 ++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 tests/db_engine_specs/crate_tests.py diff --git a/superset/db_engine_specs/crate.py b/superset/db_engine_specs/crate.py index 3a9b2c0ea9f9f..a55d72fd0632c 100644 --- a/superset/db_engine_specs/crate.py +++ b/superset/db_engine_specs/crate.py @@ -59,6 +59,4 @@ def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: @classmethod def alter_new_orm_column(cls, orm_col: "TableColumn") -> None: if orm_col.type == "TIMESTAMP": - orm_col.type = "INT" - orm_col.is_dttm = True orm_col.python_date_format = "epoch_ms" diff --git a/tests/db_engine_specs/crate_tests.py b/tests/db_engine_specs/crate_tests.py new file mode 100644 index 0000000000000..74e544b7ed985 --- /dev/null +++ b/tests/db_engine_specs/crate_tests.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from superset.connectors.sqla.models import SqlaTable, TableColumn +from superset.db_engine_specs.crate import CrateEngineSpec +from superset.models.core import Database +from tests.db_engine_specs.base_tests import TestDbEngineSpec + + +class TestCrateDbEngineSpec(TestDbEngineSpec): + def test_convert_dttm(self): + """ + DB Eng Specs (crate): Test conversion to date time + """ + dttm = self.get_dttm() + assert CrateEngineSpec.convert_dttm("TIMESTAMP", dttm) == "1546391045678.9" + + def test_epoch_to_dttm(self): + """ + DB Eng Specs (crate): Test epoch to dttm + """ + assert CrateEngineSpec.epoch_to_dttm() == "{col} * 1000" + + def test_epoch_ms_to_dttm(self): + """ + DB Eng Specs (crate): Test epoch ms to dttm + """ + assert CrateEngineSpec.epoch_ms_to_dttm() == "{col}" + + def test_alter_new_orm_column(self): + """ + DB Eng Specs (crate): Test alter orm column + """ + database = Database(database_name="crate", sqlalchemy_uri="crate://db") + tbl = SqlaTable(table_name="druid_tbl", database=database) + col = TableColumn(column_name="ts", type="TIMESTAMP", table=tbl) + CrateEngineSpec.alter_new_orm_column(col) + assert col.python_date_format == "epoch_ms" From d868d6a1bb820cb29d8dc1fee8f7313cfc04bdef Mon Sep 17 00:00:00 2001 From: Srini Kadamati Date: Thu, 11 Mar 2021 10:39:46 -0500 Subject: [PATCH 5/6] added more clarity on installing crate --- docs/src/pages/docs/Connecting to Databases/crate.mdx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/src/pages/docs/Connecting to Databases/crate.mdx b/docs/src/pages/docs/Connecting to Databases/crate.mdx index b903ab8ba8c9e..64ee991a0ce81 100644 --- a/docs/src/pages/docs/Connecting to Databases/crate.mdx +++ b/docs/src/pages/docs/Connecting to Databases/crate.mdx @@ -10,6 +10,13 @@ version: 1 The recommended connector library for CrateDB is [crate](https://pypi.org/project/crate/). +You need to install the extras as well for this library. +We recommend adding something like the following +text to your requirements file: + +``` +crate[sqlalchemy]==0.26.0 +``` The expected connection string is formatted as follows: From 22138b28260f43707c0f18ef1be319af417e9ddd Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Thu, 11 Mar 2021 19:23:40 +0200 Subject: [PATCH 6/6] fix test --- tests/db_engine_specs/crate_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/db_engine_specs/crate_tests.py b/tests/db_engine_specs/crate_tests.py index 74e544b7ed985..50cf511ea3871 100644 --- a/tests/db_engine_specs/crate_tests.py +++ b/tests/db_engine_specs/crate_tests.py @@ -26,7 +26,7 @@ def test_convert_dttm(self): DB Eng Specs (crate): Test conversion to date time """ dttm = self.get_dttm() - assert CrateEngineSpec.convert_dttm("TIMESTAMP", dttm) == "1546391045678.9" + assert CrateEngineSpec.convert_dttm("TIMESTAMP", dttm) == "1546398245678.9" def test_epoch_to_dttm(self): """