From f3b781551876ceaad66e0e27f973d6389151de7f Mon Sep 17 00:00:00 2001 From: apurva-sigmoid <89530372+apurva-sigmoid@users.noreply.github.com> Date: Fri, 1 Oct 2021 22:13:15 +0530 Subject: [PATCH] feat: add Firebolt DB engine spec (#16903) * New branch from superset for integration with firebolt sqlalchemy adapter * Added db_engine_spec file for Firebolt * Removed firebolt code from superset repo * Deleted virtual env commit * Adding time grain changes to firebolt.py * Updated README.md Added steps to install and run Superset with Firebolt SQLAlchemy Adapter * Update README.md Reduced installation steps. Using PyPi installation for adapter now * Revert "Update README.md" This reverts commit 5ed17c7a4545998ed07693c93738ebe7bff2580e. * Revert "Updated README.md" This reverts commit 45c5072649a7926fc4d15d2eb0b9cc5a89e5d7b2. * added epoch methods, added test cases for firebolt db engine spec and edited setup.py * Added license to files * Added documentation for Firebolt-SQLAlchemy * Removed trailing whitespace Co-authored-by: raghavsharma Co-authored-by: raghavSharmaSigmoid <88667094+raghavSharmaSigmoid@users.noreply.github.com> --- .../docs/Connecting to Databases/firebolt.mdx | 24 ++++++++ .../docs/Connecting to Databases/index.mdx | 1 + setup.py | 1 + superset/db_engine_specs/firebolt.py | 56 +++++++++++++++++++ .../db_engine_specs/firebolt_tests.py | 39 +++++++++++++ 5 files changed, 121 insertions(+) create mode 100644 docs/src/pages/docs/Connecting to Databases/firebolt.mdx create mode 100644 superset/db_engine_specs/firebolt.py create mode 100644 tests/integration_tests/db_engine_specs/firebolt_tests.py diff --git a/docs/src/pages/docs/Connecting to Databases/firebolt.mdx b/docs/src/pages/docs/Connecting to Databases/firebolt.mdx new file mode 100644 index 0000000000000..c1d11ff10e3be --- /dev/null +++ b/docs/src/pages/docs/Connecting to Databases/firebolt.mdx @@ -0,0 +1,24 @@ +--- +name: Firebolt +menu: Connecting to Databases +route: /docs/databases/firebolt +index: 31 +version: 1 +--- + +## Firebolt + +The recommended connector library for Firebolt is [firebolt-sqlalchemy](https://pypi.org/project/firebolt-sqlalchemy/). +Superset has been tested on `firebolt-sqlalchemy>=0.0.1`. + +The recommended connection string is: + +``` +firebolt://{username}:{password}@{host}/{database} +``` + +Here's a connection string example of Superset connecting to a Firebolt database: + +``` +firebolt://email@domain:password@host/sample_database +``` diff --git a/docs/src/pages/docs/Connecting to Databases/index.mdx b/docs/src/pages/docs/Connecting to Databases/index.mdx index 0704dc7956940..5d7ec9f38e237 100644 --- a/docs/src/pages/docs/Connecting to Databases/index.mdx +++ b/docs/src/pages/docs/Connecting to Databases/index.mdx @@ -42,6 +42,7 @@ A list of some of the recommended packages. |[Elasticsearch](/docs/databases/elasticsearch)|```pip install elasticsearch-dbapi```|```elasticsearch+http://{user}:{password}@{host}:9200/```| |[Exasol](/docs/databases/exasol)|```pip install sqlalchemy-exasol```|```exa+pyodbc://{username}:{password}@{hostname}:{port}/my_schema?CONNECTIONLCALL=en_US.UTF-8&driver=EXAODBC```| |[Google Sheets](/docs/databases/google-sheets)|```pip install shillelagh[gsheetsapi]```|```gsheets://```| +|[Firebolt](/docs/databases/firebolt)|```pip install firebolt-sqlalchemy```|```firebolt://{username}:{password}@{host}/{database}```| |[Hologres](/docs/databases/hologres)|```pip install psycopg2```|```postgresql+psycopg2://:@/```| |[IBM Db2](/docs/databases/ibm-db2)|```pip install ibm_db_sa```|```db2+ibm_db://```| |[IBM Netezza Performance Server](/docs/databases/netezza)|```pip install nzalchemy```|```netezza+nzpy://:@/```| diff --git a/setup.py b/setup.py index 8f72eb7940638..7baf6480985ca 100644 --- a/setup.py +++ b/setup.py @@ -134,6 +134,7 @@ def get_git_sha() -> str: "exasol": ["sqlalchemy-exasol>=2.1.0, <2.2"], "excel": ["xlrd>=1.2.0, <1.3"], "firebird": ["sqlalchemy-firebird>=0.7.0, <0.8"], + "firebolt": ["firebolt-sqlalchemy>=0.0.1"], "gsheets": ["shillelagh[gsheetsapi]>=1.0.3, <2"], "hana": ["hdbcli==2.4.162", "sqlalchemy_hana==0.4.0"], "hive": ["pyhive[hive]>=0.6.1", "tableschema", "thrift>=0.11.0, <1.0.0"], diff --git a/superset/db_engine_specs/firebolt.py b/superset/db_engine_specs/firebolt.py new file mode 100644 index 0000000000000..ea5091f69ffa0 --- /dev/null +++ b/superset/db_engine_specs/firebolt.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from datetime import datetime +from typing import Optional + +from superset.db_engine_specs.base import BaseEngineSpec +from superset.utils import core as utils + + +class FireboltEngineSpec(BaseEngineSpec): + """Engine spec for Firebolt""" + + engine = "firebolt" + engine_name = "Firebolt" + default_driver = "firebolt" + + _time_grain_expressions = { + None: "{col}", + "PT1S": "date_trunc('second', CAST({col} AS TIMESTAMP))", + "PT1M": "date_trunc('minute', CAST({col} AS TIMESTAMP))", + "PT1H": "date_trunc('hour', CAST({col} AS TIMESTAMP))", + "P1D": "date_trunc('day', CAST({col} AS TIMESTAMP))", + "P1W": "date_trunc('week', CAST({col} AS TIMESTAMP))", + "P1M": "date_trunc('month', CAST({col} AS TIMESTAMP))", + "P0.25Y": "date_trunc('quarter', CAST({col} AS TIMESTAMP))", + "P1Y": "date_trunc('year', CAST({col} AS TIMESTAMP))", + } + + @classmethod + def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + tt = target_type.upper() + if tt == utils.TemporalType.DATE: + return f"CAST('{dttm.date().isoformat()}' AS DATE)" + if tt == utils.TemporalType.DATETIME: + return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS DATETIME)""" + if tt == utils.TemporalType.TIMESTAMP: + return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS TIMESTAMP)""" + return None + + @classmethod + def epoch_to_dttm(cls) -> str: + return "from_unixtime({col})" diff --git a/tests/integration_tests/db_engine_specs/firebolt_tests.py b/tests/integration_tests/db_engine_specs/firebolt_tests.py new file mode 100644 index 0000000000000..793b32970bddb --- /dev/null +++ b/tests/integration_tests/db_engine_specs/firebolt_tests.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from superset.db_engine_specs.firebolt import FireboltEngineSpec +from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec + + +class TestFireboltDbEngineSpec(TestDbEngineSpec): + def test_convert_dttm(self): + dttm = self.get_dttm() + test_cases = { + "DATE": "CAST('2019-01-02' AS DATE)", + "DATETIME": "CAST('2019-01-02T03:04:05' AS DATETIME)", + "TIMESTAMP": "CAST('2019-01-02T03:04:05' AS TIMESTAMP)", + "UNKNOWNTYPE": None, + } + + for target_type, expected in test_cases.items(): + actual = FireboltEngineSpec.convert_dttm(target_type, dttm) + self.assertEqual(actual, expected) + + def test_epoch_to_dttm(self): + assert ( + FireboltEngineSpec.epoch_to_dttm().format(col="timestamp_column") + == "from_unixtime(timestamp_column)" + )