From aa535b215c18f1eb2c7505b21f4766658029d289 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 11 Sep 2017 15:42:34 -0400 Subject: [PATCH 1/2] Feature: Implementing incremental search for a column values --- superset/connectors/base/models.py | 2 +- superset/connectors/druid/models.py | 23 +++++++++++++++++++-- superset/connectors/sqla/models.py | 16 ++++++++++++--- superset/utils.py | 2 ++ superset/views/core.py | 16 ++++++++------- tests/core_tests.py | 24 ++++++++++++++++++++-- tests/druid_tests.py | 32 +++++++++++++++++++++++++++++ 7 files changed, 100 insertions(+), 15 deletions(-) diff --git a/superset/connectors/base/models.py b/superset/connectors/base/models.py index 940cc446ee87c..ea9f0b465366b 100644 --- a/superset/connectors/base/models.py +++ b/superset/connectors/base/models.py @@ -194,7 +194,7 @@ def query(self, query_obj): """ raise NotImplementedError() - def values_for_column(self, column_name, limit=10000): + def values_for_column(self, column_name, limit=10000, search_string=None): """Given a column, returns an iterable of distinct values This is used to populate the dropdown showing a list of diff --git a/superset/connectors/druid/models.py b/superset/connectors/druid/models.py index 57038097b7d11..55db4a393cd1c 100644 --- a/superset/connectors/druid/models.py +++ b/superset/connectors/druid/models.py @@ -5,6 +5,7 @@ import json import logging from multiprocessing.pool import ThreadPool +import re from dateutil.parser import parse as dparse from flask import escape, Markup @@ -913,7 +914,8 @@ def metrics_and_post_aggs(metrics, metrics_dict): def values_for_column(self, column_name, - limit=10000): + limit=10000, + search_string=None): """Retrieve some values for the given column""" logging.info( 'Getting values for columns [{}] limited to [{}]' @@ -934,10 +936,27 @@ def values_for_column(self, threshold=limit, ) + if search_string: + # Druid can't make the regex case-insensitive :( + pattern = ''.join([ + '[{0}{1}]'.format(c.upper(), c.lower()) + if c.isalpha() else re.escape(c) + for c in search_string]) + + filter_params = { + 'type': 'regex', + 'dimension': column_name, + 'pattern': '.*{}.*'.format(pattern), + } + qry['filter'] = Filter(**filter_params) + client = self.cluster.get_pydruid_client() client.topn(**qry) df = client.export_pandas() - return [row[column_name] for row in df.to_records(index=False)] + if (df.values.any()): + return [row[column_name] for row in df.to_records(index=False)] + else: + return [] def get_query_str(self, query_obj, phase=1, client=None): return self.run_query(client=client, phase=phase, **query_obj) diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index 6ccddbe79c4bd..647cd54e48513 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -9,7 +9,7 @@ import six import sqlalchemy as sa from sqlalchemy import ( - and_, asc, Boolean, Column, DateTime, desc, ForeignKey, Integer, or_, + and_, asc, Boolean, cast, Column, DateTime, desc, ForeignKey, Integer, or_, select, String, Text, ) from sqlalchemy.orm import backref, relationship @@ -19,7 +19,9 @@ import sqlparse from superset import db, import_util, sm, utils -from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric +from superset.connectors.base.models import ( + BaseColumn, BaseDatasource, BaseMetric, +) from superset.jinja_context import get_template_processor from superset.models.annotations import Annotation from superset.models.core import Database @@ -366,7 +368,7 @@ def data(self): d['time_grain_sqla'] = grains return d - def values_for_column(self, column_name, limit=10000): + def values_for_column(self, column_name, limit=10000, search_string=None): """Runs query against sqla to retrieve some sample values for the given column. """ @@ -380,6 +382,14 @@ def values_for_column(self, column_name, limit=10000): .select_from(self.get_from_clause(tp, db_engine_spec)) .distinct(column_name) ) + + if search_string: + # cast to String in case we want to search for numeric values + qry = qry.where( + cast(target_col.sqla_col, String(length=100)).ilike( + '%%{}%%'.format(search_string))).order_by( + target_col.sqla_col) + if limit: qry = qry.limit(limit) diff --git a/superset/utils.py b/superset/utils.py index 8224843213d2a..4570ca22e815b 100644 --- a/superset/utils.py +++ b/superset/utils.py @@ -676,6 +676,8 @@ def setup_cache(app, cache_config): """Setup the flask-cache on a flask app""" if cache_config and cache_config.get('CACHE_TYPE') != 'null': return Cache(app, config=cache_config) + # By default setup a no-op cache + return Cache(app, config={'CACHE_TYPE': 'null'}) def zlib_compress(data): diff --git a/superset/views/core.py b/superset/views/core.py index ec4cce1fb1568..5210d8116d3c5 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -57,7 +57,6 @@ can_access = utils.can_access DAR = models.DatasourceAccessRequest - ALL_DATASOURCE_ACCESS_ERR = __( 'This endpoint requires the `all_datasource_access` permission') DATASOURCE_MISSING_ERR = __('The datasource seems to have been deleted') @@ -1236,15 +1235,19 @@ def explore(self, datasource_type, datasource_id): @api @has_access_api + @cache.memoize(timeout=300) @expose('/filter////') - def filter(self, datasource_type, datasource_id, column): + @expose("/filter/////") + @expose("/filter/////") + def filter(self, datasource_type, datasource_id, column, limit=0, search_string=None): """ Endpoint to retrieve values for specified column. :param datasource_type: Type of datasource e.g. table :param datasource_id: Datasource id :param column: Column name to retrieve values for - :return: + :param limit: Return at most these entries (default: 10000) + :return: search_string: Only return columns containing the search_string """ # TODO: Cache endpoint by user, datasource and column datasource = ConnectorRegistry.get_datasource( @@ -1255,10 +1258,9 @@ def filter(self, datasource_type, datasource_id, column): return json_error_response(DATASOURCE_ACCESS_ERR) payload = json.dumps( - datasource.values_for_column( - column, - config.get('FILTER_SELECT_ROW_LIMIT', 10000), - ), + datasource.values_for_column(column_name=column, + limit=limit if limit else config.get('FILTER_SELECT_ROW_LIMIT', 10000), + search_string=search_string), default=utils.json_int_dttm_ser) return json_success(payload) diff --git a/tests/core_tests.py b/tests/core_tests.py index a7edc4ec16c2e..aca68666ca92f 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -201,8 +201,28 @@ def test_filter_endpoint(self): 'datasource_id=1&datasource_type=table') # Changing name - resp = self.get_resp(url.format(tbl_id, slice_id)) - assert len(resp) > 0 + resp = json.loads(self.get_resp(url.format(tbl_id, slice_id))) + assert len(resp) > 1 + assert 'Carbon Dioxide' in resp + + # Limit to 3 + url = ( + '/superset/filter/table/{}/target/3?viz_type=sankey&groupby=source' + '&metric=sum__value&flt_col_0=source&flt_op_0=in&flt_eq_0=&' + 'slice_id={}&datasource_name=energy_usage&' + 'datasource_id=1&datasource_type=table') + resp = json.loads(self.get_resp(url.format(tbl_id, slice_id))) + assert len(resp) == 3 + + # With search_string = 'carbon' + url = ( + '/superset/filter/table/{}/target/100/carbon?' + 'viz_type=sankey&groupby=source&' + 'metric=sum__value&flt_col_0=source&flt_op_0=in&flt_eq_0=&' + 'slice_id={}&datasource_name=energy_usage&' + 'datasource_id=1&datasource_type=table') + resp = json.loads(self.get_resp(url.format(tbl_id, slice_id))) + assert len(resp) == 1 assert 'Carbon Dioxide' in resp def test_slices(self): diff --git a/tests/druid_tests.py b/tests/druid_tests.py index c280da790a293..3f80f732507f6 100644 --- a/tests/druid_tests.py +++ b/tests/druid_tests.py @@ -328,6 +328,38 @@ def test_sync_druid_perm(self, PyDruid): permission=permission, view_menu=view_menu).first() assert pv is not None + @patch('superset.connectors.druid.models.PyDruid') + def test_values_for_column(self, py_druid): + ds = 'test_datasource' + column = 'test_column' + search_string = '$t1' # difficult test string + + datasource = self.get_or_create( + DruidDatasource, {'datasource_name': ds}, + db.session) + druid = py_druid() + datasource.cluster.get_pydruid_client = Mock(return_value=druid) + + # search_string + datasource.values_for_column(column_name=column, limit=5, + search_string=search_string) + + assert druid.topn.call_args[1]['datasource'] == ds + assert druid.topn.call_args[1]['granularity'] == 'all' + assert druid.topn.call_args[1]['metric'] == 'count' + assert druid.topn.call_args[1]['dimension'] == column + assert druid.topn.call_args[1]['threshold'] == 5 + + # test filter + assert(druid.topn.call_args[1]['filter'] + .filter['filter']['dimension'] == column) + assert(druid.topn.call_args[1]['filter'] + .filter['filter']['pattern'] == '.*\\$[Tt]1.*') + + # no search_string + datasource.values_for_column(column_name=column) + assert not druid.topn.call_args[1].get('filter') + if __name__ == '__main__': unittest.main() From 7fd15d84e3717789fdae53ed923d5cd6eba8d4c9 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 19 Jul 2017 12:08:21 -0400 Subject: [PATCH 2/2] Feature: Adding new exploration UI --- superset/assets/backendSync.json | 12 + .../components/ExploreActionButtons.jsx | 15 + .../javascripts/explore/exploreUtils.js | 34 +- .../explore/reducers/exploreReducer.js | 1 + .../assets/javascripts/swivel/ColumnTypes.jsx | 5 + .../javascripts/swivel/ContainerTypes.jsx | 4 + .../assets/javascripts/swivel/FilterTypes.jsx | 5 + .../assets/javascripts/swivel/ItemTypes.jsx | 3 + .../javascripts/swivel/SessionManager.js | 98 ++++++ .../swivel/actions/globalActions.js | 51 +++ .../swivel/actions/keyBindingsActions.js | 9 + .../swivel/actions/querySettingsActions.js | 105 ++++++ .../swivel/actions/refDataActions.js | 91 +++++ .../swivel/actions/vizDataActions.js | 63 ++++ .../swivel/actions/vizSettingsActions.js | 23 ++ .../swivel/components/ChartContainer.jsx | 187 +++++++++++ .../javascripts/swivel/components/Column.jsx | 116 +++++++ .../swivel/components/ColumnContainer.jsx | 126 +++++++ .../swivel/components/ColumnDropTarget.jsx | 46 +++ .../swivel/components/Container.jsx | 52 +++ .../swivel/components/DateFilter.jsx | 196 +++++++++++ .../swivel/components/FilterTile.jsx | 312 ++++++++++++++++++ .../swivel/components/LabeledSelect.jsx | 65 ++++ .../swivel/components/MetricContainer.jsx | 141 ++++++++ .../swivel/components/RangeFilter.jsx | 58 ++++ .../swivel/components/RunToolbar.jsx | 161 +++++++++ .../swivel/components/SettingsPanel.jsx | 36 ++ .../swivel/components/SplitTile.jsx | 239 ++++++++++++++ .../swivel/components/ValueFilter.jsx | 152 +++++++++ .../assets/javascripts/swivel/constants.js | 3 + .../swivel/containers/DatasourceSelect.js | 22 ++ .../swivel/containers/FilterContainer.jsx | 64 ++++ .../swivel/containers/SplitContainer.jsx | 60 ++++ .../swivel/containers/VizTypeSelect.js | 22 ++ .../swivel/formDataUtils/convertToFormData.js | 224 +++++++++++++ .../formDataUtils/importQuerySettings.js | 196 +++++++++++ .../swivel/formDataUtils/importVizSettings.js | 7 + .../swivel/formDataUtils/sliceObject.js | 52 +++ superset/assets/javascripts/swivel/index.jsx | 139 ++++++++ .../swivel/listeners/QuerySettingsListener.js | 59 ++++ .../swivel/listeners/VizSettingsListener.jsx | 41 +++ superset/assets/javascripts/swivel/main.css | 81 +++++ .../swivel/reducers/controlReducer.js | 63 ++++ .../swivel/reducers/keyBindingsReducer.js | 19 ++ .../swivel/reducers/querySettingsReducer.js | 200 +++++++++++ .../swivel/reducers/refDataReducer.js | 28 ++ .../swivel/reducers/settingsReducer.js | 40 +++ .../swivel/reducers/vizDataReducer.js | 33 ++ .../swivel/reducers/vizSettingsReducer.js | 34 ++ .../assets/javascripts/swivel/shortcuts.js | 61 ++++ .../javascripts/swivel/stores/ControlStore.js | 19 ++ .../swivel/stores/FormDataStore.js | 147 +++++++++ .../swivel/stores/QuerySettingsStore.js | 27 ++ .../javascripts/swivel/stores/RefDataStore.js | 21 ++ .../javascripts/swivel/stores/VizDataStore.js | 12 + .../swivel/stores/VizSettingsStore.js | 14 + superset/assets/package.json | 10 +- .../components/ExploreActionButtons_spec.jsx | 1 + .../javascripts/swivel/formDataUtils/data.js | 263 +++++++++++++++ .../formDataUtils/formDataToQuery_spec.js | 76 +++++ .../formDataUtils/queryToFormData_spec.js | 113 +++++++ superset/assets/visualizations/nvd3_vis.js | 113 +++++-- superset/assets/webpack.config.js | 1 + superset/config.py | 3 +- superset/templates/superset/swivel.html | 20 ++ superset/views/__init__.py | 1 + superset/views/core.py | 31 +- superset/views/swivel.py | 116 +++++++ 68 files changed, 4790 insertions(+), 52 deletions(-) create mode 100644 superset/assets/javascripts/swivel/ColumnTypes.jsx create mode 100644 superset/assets/javascripts/swivel/ContainerTypes.jsx create mode 100644 superset/assets/javascripts/swivel/FilterTypes.jsx create mode 100644 superset/assets/javascripts/swivel/ItemTypes.jsx create mode 100644 superset/assets/javascripts/swivel/SessionManager.js create mode 100644 superset/assets/javascripts/swivel/actions/globalActions.js create mode 100644 superset/assets/javascripts/swivel/actions/keyBindingsActions.js create mode 100644 superset/assets/javascripts/swivel/actions/querySettingsActions.js create mode 100644 superset/assets/javascripts/swivel/actions/refDataActions.js create mode 100644 superset/assets/javascripts/swivel/actions/vizDataActions.js create mode 100644 superset/assets/javascripts/swivel/actions/vizSettingsActions.js create mode 100644 superset/assets/javascripts/swivel/components/ChartContainer.jsx create mode 100644 superset/assets/javascripts/swivel/components/Column.jsx create mode 100644 superset/assets/javascripts/swivel/components/ColumnContainer.jsx create mode 100644 superset/assets/javascripts/swivel/components/ColumnDropTarget.jsx create mode 100644 superset/assets/javascripts/swivel/components/Container.jsx create mode 100644 superset/assets/javascripts/swivel/components/DateFilter.jsx create mode 100644 superset/assets/javascripts/swivel/components/FilterTile.jsx create mode 100644 superset/assets/javascripts/swivel/components/LabeledSelect.jsx create mode 100644 superset/assets/javascripts/swivel/components/MetricContainer.jsx create mode 100644 superset/assets/javascripts/swivel/components/RangeFilter.jsx create mode 100644 superset/assets/javascripts/swivel/components/RunToolbar.jsx create mode 100644 superset/assets/javascripts/swivel/components/SettingsPanel.jsx create mode 100644 superset/assets/javascripts/swivel/components/SplitTile.jsx create mode 100644 superset/assets/javascripts/swivel/components/ValueFilter.jsx create mode 100644 superset/assets/javascripts/swivel/constants.js create mode 100644 superset/assets/javascripts/swivel/containers/DatasourceSelect.js create mode 100644 superset/assets/javascripts/swivel/containers/FilterContainer.jsx create mode 100644 superset/assets/javascripts/swivel/containers/SplitContainer.jsx create mode 100644 superset/assets/javascripts/swivel/containers/VizTypeSelect.js create mode 100644 superset/assets/javascripts/swivel/formDataUtils/convertToFormData.js create mode 100644 superset/assets/javascripts/swivel/formDataUtils/importQuerySettings.js create mode 100644 superset/assets/javascripts/swivel/formDataUtils/importVizSettings.js create mode 100644 superset/assets/javascripts/swivel/formDataUtils/sliceObject.js create mode 100644 superset/assets/javascripts/swivel/index.jsx create mode 100644 superset/assets/javascripts/swivel/listeners/QuerySettingsListener.js create mode 100644 superset/assets/javascripts/swivel/listeners/VizSettingsListener.jsx create mode 100644 superset/assets/javascripts/swivel/main.css create mode 100644 superset/assets/javascripts/swivel/reducers/controlReducer.js create mode 100644 superset/assets/javascripts/swivel/reducers/keyBindingsReducer.js create mode 100644 superset/assets/javascripts/swivel/reducers/querySettingsReducer.js create mode 100644 superset/assets/javascripts/swivel/reducers/refDataReducer.js create mode 100644 superset/assets/javascripts/swivel/reducers/settingsReducer.js create mode 100644 superset/assets/javascripts/swivel/reducers/vizDataReducer.js create mode 100644 superset/assets/javascripts/swivel/reducers/vizSettingsReducer.js create mode 100644 superset/assets/javascripts/swivel/shortcuts.js create mode 100644 superset/assets/javascripts/swivel/stores/ControlStore.js create mode 100644 superset/assets/javascripts/swivel/stores/FormDataStore.js create mode 100644 superset/assets/javascripts/swivel/stores/QuerySettingsStore.js create mode 100644 superset/assets/javascripts/swivel/stores/RefDataStore.js create mode 100644 superset/assets/javascripts/swivel/stores/VizDataStore.js create mode 100644 superset/assets/javascripts/swivel/stores/VizSettingsStore.js create mode 100644 superset/assets/spec/javascripts/swivel/formDataUtils/data.js create mode 100644 superset/assets/spec/javascripts/swivel/formDataUtils/formDataToQuery_spec.js create mode 100644 superset/assets/spec/javascripts/swivel/formDataUtils/queryToFormData_spec.js create mode 100644 superset/templates/superset/swivel.html create mode 100644 superset/views/swivel.py diff --git a/superset/assets/backendSync.json b/superset/assets/backendSync.json index ba91b47cde46c..631b53ae90ce0 100644 --- a/superset/assets/backendSync.json +++ b/superset/assets/backendSync.json @@ -2059,6 +2059,18 @@ "default": "linear", "description": "Line interpolation as defined by d3.js" }, + "overlays": { + "type": "SelectControl", + "multi": true, + "label": "Overlays", + "default": [] + }, + "offset_overlays": { + "type": "CheckboxControl", + "label": "Auto Offset Overlays", + "default": false, + "description": "Auto offset overlay to match the time frame of the current config." + }, "pie_label_type": { "type": "SelectControl", "label": "Label Type", diff --git a/superset/assets/javascripts/explore/components/ExploreActionButtons.jsx b/superset/assets/javascripts/explore/components/ExploreActionButtons.jsx index 57f2dfd744c45..2604c81e3b291 100644 --- a/superset/assets/javascripts/explore/components/ExploreActionButtons.jsx +++ b/superset/assets/javascripts/explore/components/ExploreActionButtons.jsx @@ -5,6 +5,8 @@ import URLShortLinkButton from './URLShortLinkButton'; import EmbedCodeButton from './EmbedCodeButton'; import DisplayQueryButton from './DisplayQueryButton'; import { t } from '../../locales'; +import { getSwivelUrl } from '../exploreUtils'; +import { isSupportedBySwivel } from '../../swivel/formDataUtils/convertToFormData'; const propTypes = { canDownload: PropTypes.oneOfType([PropTypes.string, PropTypes.bool]).isRequired, @@ -22,6 +24,19 @@ export default function ExploreActionButtons({ if (slice) { return (
+ { + queryResponse && + isSupportedBySwivel(queryResponse.form_data) && + + Open in Swivel + + } diff --git a/superset/assets/javascripts/explore/exploreUtils.js b/superset/assets/javascripts/explore/exploreUtils.js index 8a01745d9a39f..476e5142f5d8d 100644 --- a/superset/assets/javascripts/explore/exploreUtils.js +++ b/superset/assets/javascripts/explore/exploreUtils.js @@ -1,5 +1,16 @@ /* eslint camelcase: 0 */ import URI from 'urijs'; +import { compressToBase64 } from 'lz-string'; + +export function trimFormData(formData) { + const cleaned = { ...formData }; + Object.entries(formData).forEach(([k, v]) => { + if (v === null || v === undefined) { + delete cleaned[k]; + } + }); + return cleaned; +} export function getChartKey(explore) { const slice = explore.slice; @@ -14,8 +25,7 @@ export function getAnnotationJsonUrl(slice_id, form_data, isNative) { const endpoint = isNative ? 'annotation_json' : 'slice_json'; return uri.pathname(`/superset/${endpoint}/${slice_id}`) .search({ - form_data: JSON.stringify(form_data, - (key, value) => value === null ? undefined : value), + form_data: JSON.stringify(trimFormData(form_data)), }).toString(); } @@ -43,7 +53,7 @@ export function getExploreUrl(form_data, endpointType = 'base', force = false, // Building the querystring (search) part of the URI const search = uri.search(true); - search.form_data = JSON.stringify(form_data); + search.form_data = JSON.stringify(trimFormData(form_data)); if (force) { search.force = 'true'; } @@ -67,3 +77,21 @@ export function getExploreUrl(form_data, endpointType = 'base', force = false, uri = uri.search(search).directory(directory); return uri.toString(); } + +export function getSwivelUrl(formData, lzCompress) { + if (!formData || !formData.datasource) { + return null; + } + + const uri = URI(window.location.search); + + // Building the query + if (lzCompress) { + return uri.pathname('/swivel') + .search({ lz_form_data: compressToBase64(JSON.stringify(trimFormData(formData))) }) + .toString(); + } + return uri.pathname('/swivel') + .search({ form_data: JSON.stringify(trimFormData(formData)) }) + .toString(); +} diff --git a/superset/assets/javascripts/explore/reducers/exploreReducer.js b/superset/assets/javascripts/explore/reducers/exploreReducer.js index 7b55748800ff8..afc5e6ed9d683 100644 --- a/superset/assets/javascripts/explore/reducers/exploreReducer.js +++ b/superset/assets/javascripts/explore/reducers/exploreReducer.js @@ -79,3 +79,4 @@ export default function exploreReducer(state = {}, action) { } return state; } + diff --git a/superset/assets/javascripts/swivel/ColumnTypes.jsx b/superset/assets/javascripts/swivel/ColumnTypes.jsx new file mode 100644 index 0000000000000..2324d7df752a0 --- /dev/null +++ b/superset/assets/javascripts/swivel/ColumnTypes.jsx @@ -0,0 +1,5 @@ +export default { + TIMESTAMP: 'TIMESTAMP', + NUMERIC: 'NUMERIC', + STRING: 'NVARCHAR', +}; diff --git a/superset/assets/javascripts/swivel/ContainerTypes.jsx b/superset/assets/javascripts/swivel/ContainerTypes.jsx new file mode 100644 index 0000000000000..b7138f7fff452 --- /dev/null +++ b/superset/assets/javascripts/swivel/ContainerTypes.jsx @@ -0,0 +1,4 @@ +export default { + SPLIT: 'SPLIT', + FILTER: 'FILTER', +}; diff --git a/superset/assets/javascripts/swivel/FilterTypes.jsx b/superset/assets/javascripts/swivel/FilterTypes.jsx new file mode 100644 index 0000000000000..0cf0fb3303ae3 --- /dev/null +++ b/superset/assets/javascripts/swivel/FilterTypes.jsx @@ -0,0 +1,5 @@ +export default { + INTERVAL: 'INTERVAL', + SELECT: 'SELECT', + UNBOUND: 'UNBOUND', +}; diff --git a/superset/assets/javascripts/swivel/ItemTypes.jsx b/superset/assets/javascripts/swivel/ItemTypes.jsx new file mode 100644 index 0000000000000..f7e8b4950df6c --- /dev/null +++ b/superset/assets/javascripts/swivel/ItemTypes.jsx @@ -0,0 +1,3 @@ +export default { + DIMENSION: 'dimension', +}; diff --git a/superset/assets/javascripts/swivel/SessionManager.js b/superset/assets/javascripts/swivel/SessionManager.js new file mode 100644 index 0000000000000..889aa7ecfd221 --- /dev/null +++ b/superset/assets/javascripts/swivel/SessionManager.js @@ -0,0 +1,98 @@ +import moment from 'moment'; +import uuidv4 from 'uuid/v4'; + +import { LOCAL_STORAGE_SESSIONS_KEY, + LOCAL_STORAGE_KEY_PREFIX, MAX_NUM_SESSIONS } from './constants'; + +export function getSessions() { + let sessions = JSON.parse(localStorage.getItem(LOCAL_STORAGE_SESSIONS_KEY)); + if (!Array.isArray(sessions)) { + sessions = []; + } else { + sessions = sessions.filter(x => !!x && !!x.ts).sort((a, b) => b.ts - a.ts); + } + return sessions; +} + +function saveSessions(sessions) { + localStorage.setItem(LOCAL_STORAGE_SESSIONS_KEY, JSON.stringify(sessions)); + return sessions; +} + +function cleanup(sessions) { + // Cleanup old sessions + Object.keys(localStorage) + .filter(x => x.startsWith(LOCAL_STORAGE_KEY_PREFIX) && + !sessions.find(s => `${LOCAL_STORAGE_KEY_PREFIX}${s.id}` === x)) + .forEach(x => localStorage.removeItem(x)); + saveSessions(sessions); +} + +export function deleteSessions() { + cleanup([]); +} + +export function deleteSession(id) { + const sessions = getSessions(); + cleanup(sessions.filter(x => x.id !== id)); +} + +export function createNewSession(name, id) { + let sessions = getSessions(); + if (sessions.length > MAX_NUM_SESSIONS) { + localStorage.removeItem(sessions[sessions.length - 1]); + sessions.pop(); + } + const newSession = { ts: moment.now(), id: id || uuidv4(), name }; + sessions = [newSession, ...sessions]; + cleanup(sessions); + return newSession; +} + +export function updateSession(id, name) { + const sessions = getSessions(); + let session = sessions.find(x => x.id === id); + if (session) { + session.name = name || session.name; + session.ts = moment.now(); + saveSessions(sessions); + } else { + session = createNewSession(name, id); + } + window.document.title = `Swivel - ${session.name} (${session.id.substring(0, 7)})`; +} + +export function getSessionKey(bootstrapData) { + let swivelSession = null; + const sessions = getSessions(); + + const createNew = bootstrapData.new || + bootstrapData.reset || + bootstrapData.lz_form_data || + bootstrapData.form_data; + + const now = moment.now(); + // Read the current session from Local Storage + if (bootstrapData.session && + sessions.find(x => x.id === bootstrapData.session)) { + // Session was passed in with bootstrapData + const s = sessions.find(x => x.id === bootstrapData.session); + s.ts = now; + swivelSession = s.id; + } else if (sessions.length && !createNew) { + // Get the most recent session. + const s = sessions[0]; + swivelSession = s.id; + s.ts = now; + } + + // Create a new Session + if (!swivelSession || createNew) { + swivelSession = createNewSession('').id; + } else { + saveSessions(sessions); + } + + window.history.pushState('', '', `${location.pathname}?session=${swivelSession}`); + return swivelSession; +} diff --git a/superset/assets/javascripts/swivel/actions/globalActions.js b/superset/assets/javascripts/swivel/actions/globalActions.js new file mode 100644 index 0000000000000..6cd313ea5cdcd --- /dev/null +++ b/superset/assets/javascripts/swivel/actions/globalActions.js @@ -0,0 +1,51 @@ +// Aborts the current query +export const ABORT = 'ABORT'; +export function abort() { + return { type: ABORT }; +} + +export const RESET = 'RESET'; +export const CLEAR_HISTORY = 'CLEAR_HISTORY'; +export function reset(clearHistory) { + if (clearHistory) { + return dispatch => + // We need the sandwich to make sure there is enough space in the + // local storage to RESET + Promise.resolve(dispatch({ type: ABORT })) + .then(() => dispatch({ type: CLEAR_HISTORY })) + .then(() => dispatch({ type: RESET })) + .then(() => dispatch({ type: CLEAR_HISTORY })); + } + return dispatch => + Promise.resolve(dispatch({ type: ABORT })) + .then(() => dispatch({ type: RESET })); +} + +// This controls whether a query should be run +export const SET_RUN = 'SET_RUN'; +export function setRun(run) { + return { type: SET_RUN, run }; +} + +// This controls if a query should automatically run if the query settings change +export const SET_AUTO_RUN = 'SET_AUTO_RUN'; +export function setAutoRun(autoRun) { + return { type: SET_AUTO_RUN, autoRun }; +} + +// This indicates if a query is currently running +export const SET_IS_RUNNING = 'SET_IS_RUNNING'; +export function setIsRunning(isRunning, queryRequest) { + return { type: SET_IS_RUNNING, isRunning, queryRequest }; +} + +export const SET_ERROR = 'SET_ERROR'; +export function setError(error) { + return { type: SET_ERROR, error }; +} + +export const UPDATE_FORM_DATA = 'UPDATE_FORM_DATA'; +export const IMPORT_FORM_DATA = 'IMPORT_FORM_DATA'; +export function importFormData(formData, refData) { + return { type: IMPORT_FORM_DATA, formData, refData }; +} diff --git a/superset/assets/javascripts/swivel/actions/keyBindingsActions.js b/superset/assets/javascripts/swivel/actions/keyBindingsActions.js new file mode 100644 index 0000000000000..cb808213ea312 --- /dev/null +++ b/superset/assets/javascripts/swivel/actions/keyBindingsActions.js @@ -0,0 +1,9 @@ +export const SEARCH_COLUMNS = 'SEARCH_COLUMNS'; +export function searchColumns() { + return { type: SEARCH_COLUMNS }; +} + +export const SEARCH_METRICS = 'SEARCH_METRICS'; +export function searchMetrics() { + return { type: SEARCH_METRICS }; +} diff --git a/superset/assets/javascripts/swivel/actions/querySettingsActions.js b/superset/assets/javascripts/swivel/actions/querySettingsActions.js new file mode 100644 index 0000000000000..6b4fdc0e78d8e --- /dev/null +++ b/superset/assets/javascripts/swivel/actions/querySettingsActions.js @@ -0,0 +1,105 @@ +import { convertQuerySettingsToFormData } from '../formDataUtils/convertToFormData'; + +import { fetchDatasources, fetchDatasourceMetadata } from './refDataActions'; +import { UPDATE_FORM_DATA, setError, setAutoRun, importFormData } from './globalActions'; +import { runQuery } from './vizDataActions'; + +export const SET_DEFAULTS = 'SET_DEFAULTS'; +export function setDefaults(refData) { + return { type: SET_DEFAULTS, refData }; +} + +export const SET_DATASOURCE = 'SET_DATASOURCE'; +export function setDatasource(uid, init = true) { + return (dispatch, getState) => { + if (getState().settings.future.length === 0 && + getState().settings.present.query.datasource === uid && + getState().refData.columns.length) { + return Promise.resolve(); + } + return dispatch(fetchDatasourceMetadata(uid)) + .then(() => dispatch({ + type: SET_DATASOURCE, + uid, + name: (getState() + .refData + .datasources.find(x => x.uid === uid) || {}).name, + })) + .then(() => init ? dispatch( + setDefaults(getState().refData)) : Promise.resolve()); + }; +} + +export const BOOTSTRAP = 'BOOTSTRAP'; +export function bootstrap(formData) { + return (dispatch, getState) => + dispatch(fetchDatasources()).then(() => { + const datasource = getState().settings.present.query.datasource; + if (formData.datasource) { + return Promise.resolve(dispatch(setAutoRun(false))) + .then(() => dispatch(setDatasource(formData.datasource, false))) + .then(() => dispatch(importFormData(formData, getState().refData))) + .then(() => dispatch(setAutoRun(true))); + } else if (datasource) { + return dispatch(setDatasource(datasource, false)); + } + return Promise.resolve(); + }); +} + +export function updateFormDataAndRunQuery(settings) { + return (dispatch) => { + const formData = convertQuerySettingsToFormData(settings); + return Promise.resolve( + dispatch({ type: UPDATE_FORM_DATA, formData, wipeData: true })) + .then(() => dispatch(setError(formData.error))) + .then(() => dispatch(runQuery())); + }; +} + +export const TOGGLE_METRIC = 'TOGGLE_METRIC'; +export function toggleMetric(metric) { + return { type: TOGGLE_METRIC, metric }; +} + +export const ADD_FILTER = 'ADD_FILTER'; +export function addFilter(filter) { + return { type: ADD_FILTER, filter }; +} + +export const CONFIGURE_FILTER = 'CONFIGURE_FILTER'; +export function configureFilter(filter) { + return { type: CONFIGURE_FILTER, filter }; +} + +export const REMOVE_FILTER = 'REMOVE_FILTER'; +export function removeFilter(filter) { + return { type: REMOVE_FILTER, filter }; +} + +export const ADD_SPLIT = 'ADD_SPLIT'; +export function addSplit(split) { + return { type: ADD_SPLIT, split }; +} + +export const CONFIGURE_SPLIT = 'CONFIGURE_SPLIT'; +export function configureSplit(split) { + return { type: CONFIGURE_SPLIT, split }; +} + +export const REMOVE_SPLIT = 'REMOVE_SPLIT'; +export function removeSplit(split) { + return { type: REMOVE_SPLIT, split }; +} + +export const CHANGE_INTERVAL = 'CHANGE_INTERVAL'; +export function changeInterval(intervalStart, intervalEnd) { + return { type: CHANGE_INTERVAL, intervalStart, intervalEnd }; +} + +// TODO need to move those to the vizSettings +export const SET_VIZTYPE = 'SET_VIZTYPE'; +export function setVizType(vizType) { + return { type: SET_VIZTYPE, vizType }; +} + diff --git a/superset/assets/javascripts/swivel/actions/refDataActions.js b/superset/assets/javascripts/swivel/actions/refDataActions.js new file mode 100644 index 0000000000000..c3fb0b8e6ac9d --- /dev/null +++ b/superset/assets/javascripts/swivel/actions/refDataActions.js @@ -0,0 +1,91 @@ +import ColumnTypes from '../ColumnTypes'; + +export const SET_COLUMNS = 'SET_COLUMNS'; +export function setColumns(columns) { + return { type: SET_COLUMNS, columns }; +} + +export const SET_DATASOURCES = 'SET_DATASOURCES'; +export function setDatasources(datasources) { + return { type: SET_DATASOURCES, datasources }; +} + +export const SET_METRICS = 'SET_METRICS'; +export function setMetrics(metrics) { + return { type: SET_METRICS, metrics }; +} + +export const SET_TIME_GRAINS = 'SET_TIME_GRAINS'; +export function setTimeGrains(timeGrains) { + return { type: SET_TIME_GRAINS, timeGrains }; +} + +export function fetchDatasources() { + return dispatch => fetch('/superset/datasources/', { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'same-origin', + }).then(response => response.json()).then((data) => { + const datasources = data.map(x => ({ + uid: x.uid, + name: x.name, + type: x.type, + id: x.id })); + return datasources; + }).then(datasources => dispatch(setDatasources(datasources))); +} + +export function fetchDatasourceMetadata(uid) { + return function (dispatch) { + if (!uid) { + return Promise.resolve(); + } + const url = `/swivel/fetch_datasource_metadata?uid=${uid}`; + return fetch(url, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'same-origin', + }).then(response => response.json()).then((data) => { + const columns = data.columns + .filter(x => x.groupable || + x.filterable || + (x.type !== ColumnTypes.NUMERIC)).map(x => ({ + name: x.name, + id: x.id, + columnType: x.type, + groupable: x.groupable, + })); + const metrics = data.metrics.map(x => ({ + name: x.name, + id: x.id, + format: x.format, + })); + const timeGrains = data.time_grains; + + // Todo: this is hacky should be done better + if (uid.endsWith('druid')) { + let timeColumn = columns.find(x => + x.id.toLowerCase() === '__time'); + if (!timeColumn) { + timeColumn = { + name: 'Time', + id: '__time', + }; + columns.push(timeColumn); + } + timeColumn.columnType = ColumnTypes.TIMESTAMP; + timeColumn.groupable = false; + } + + return Promise.all([ + dispatch(setColumns(columns)), + dispatch(setMetrics(metrics)), + dispatch(setTimeGrains(timeGrains)), + ]); + }); + }; +} diff --git a/superset/assets/javascripts/swivel/actions/vizDataActions.js b/superset/assets/javascripts/swivel/actions/vizDataActions.js new file mode 100644 index 0000000000000..d3b1ebab42dff --- /dev/null +++ b/superset/assets/javascripts/swivel/actions/vizDataActions.js @@ -0,0 +1,63 @@ +import { setError, setRun, setIsRunning } from './globalActions'; + +export const SET_DATA = 'SET_DATA'; +export function setData(data) { + return { type: SET_DATA, data }; +} + +export const SET_OUTDATED = 'SET_OUTDATED'; +export function setOutdated(outdated) { + return { type: SET_OUTDATED, outdated }; +} + +export const RESET_DATA = 'RESET_DATA'; +export function resetData() { + return { type: RESET_DATA }; +} + +export function runQuery() { + return (dispatch, getState) => { + const { vizData, controls, settings } = getState(); + const payload = vizData.formData; + if (!controls.error) { + const datasource = settings.present.query.datasource; + const [dsId, dsType] = datasource.split('__'); + const url = `${window.location.origin}/superset/explore_json/${dsType}/${dsId}`; + const queryRequest = $.ajax({ + method: 'POST', + url, + dataType: 'json', + contentType: 'application/json; charset=UTF-8', + timeout: 300000, // 5 Min + data: payload.toJson() }); + return Promise.resolve(dispatch(setIsRunning(true, queryRequest))) + .then(() => queryRequest) + .then(data => Promise.all([ + dispatch(setData(data.data)), + dispatch(setRun(false)), + dispatch(setOutdated(false)), + dispatch(setIsRunning(false, queryRequest))])) + .catch(function (res) { + if (res.status === 0) { + return Promise.all([ + dispatch(setRun(false)), + dispatch(setOutdated(true)), + dispatch(setIsRunning(false, queryRequest))]); + } + let error; + if (res.responseJSON) { + error = res.responseJSON.error; + } else { + error = 'Server error'; + } + return Promise.all([ + dispatch(setIsRunning(false, queryRequest)), + dispatch(setData()), + dispatch(setError(error)), + dispatch(setOutdated(false)), + ]); + }); + } + return Promise.resolve(dispatch(setOutdated(false))); + }; +} diff --git a/superset/assets/javascripts/swivel/actions/vizSettingsActions.js b/superset/assets/javascripts/swivel/actions/vizSettingsActions.js new file mode 100644 index 0000000000000..4ff7673b8d1a6 --- /dev/null +++ b/superset/assets/javascripts/swivel/actions/vizSettingsActions.js @@ -0,0 +1,23 @@ +import { convertVizSettingsToFormData } from '../formDataUtils/convertToFormData'; +import { UPDATE_FORM_DATA } from './globalActions'; + +export const TOGGLE_SHOW_LEGEND = 'TOGGLE_SHOW_LEGEND'; +export function toggleShowLegend() { + return { type: TOGGLE_SHOW_LEGEND }; +} + +export const TOGGLE_RICH_TOOLTIP = 'TOGGLE_RICH_TOOLTIP'; +export function toggleRichTooltip() { + return { type: TOGGLE_RICH_TOOLTIP }; +} + +export function updateFormData(vizSettings) { + const formData = convertVizSettingsToFormData(vizSettings); + return { type: UPDATE_FORM_DATA, formData }; +} + +export const TOGGLE_SEPARATE_CHARTS = 'TOGGLE_SEPARATE_CHARTS'; +export function toggleSeparateCharts() { + return { type: TOGGLE_SEPARATE_CHARTS }; +} + diff --git a/superset/assets/javascripts/swivel/components/ChartContainer.jsx b/superset/assets/javascripts/swivel/components/ChartContainer.jsx new file mode 100644 index 0000000000000..115adacacdea6 --- /dev/null +++ b/superset/assets/javascripts/swivel/components/ChartContainer.jsx @@ -0,0 +1,187 @@ +import $ from 'jquery'; +import { connect } from 'react-redux'; +import React, { PureComponent } from 'react'; +import PropTypes from 'prop-types'; + +import visMap from '../../../visualizations/main'; + +import { addFilter, removeFilter, changeInterval } from '../actions/querySettingsActions'; +import { getMockedSliceObject } from '../formDataUtils/sliceObject'; + + +const propTypes = { + containerId: PropTypes.string.isRequired, + error: PropTypes.string, + outdated: PropTypes.bool, + isRunning: PropTypes.bool, + formData: PropTypes.object, + data: PropTypes.oneOfType( + [PropTypes.object, PropTypes.arrayOf(PropTypes.object)]), + columns: PropTypes.arrayOf(PropTypes.object), + selectedMetrics: PropTypes.object, + metrics: PropTypes.arrayOf(PropTypes.object), + + intervalCallback: PropTypes.func, + handleAddFilter: PropTypes.func, + handleRemoveFilter: PropTypes.func, +}; + +/** + * This component takes care of rendering one of multiple charts + */ +class ChartContainer extends PureComponent { + constructor(props) { + super(props); + this.update = this.update.bind(this); + this.getCharts = this.getCharts.bind(this); + } + + componentDidMount() { this.update(); } + componentDidUpdate(prevProps) { + if (prevProps.formData !== this.props.formData || + prevProps.data !== this.props.data) { + this.update(); + } + } + + /** + * This function splits the data into multiple charts + * @param data to be rendered + * @param metrics to adapt legend and axis labels + * @param formData which defines the visualization configuration + */ + getCharts(data, metrics, separateCharts) { + if (!data) { + return []; + } else if (separateCharts && Array.isArray(data) && metrics && metrics.length > 1) { + const separatedData = []; + for (const metric of metrics) { + let series = data.filter((c) => { + if (Array.isArray(c.key)) { + return c.key[0] === metric.id; + } + return c.key === metric.id; + }); + series = series.map(c => ({ + ...c, + })); + if (series.length) { + separatedData.push(series); + } + } + return separatedData; + } + return [data]; + } + + deleteGraphs(containerId) { + $(`[id^=${containerId}-]`).toArray() + .forEach(x => $(x).children().remove()); + } + + update() { + const { error, data, formData, containerId, selectedMetrics, columns, + intervalCallback, handleAddFilter, handleRemoveFilter } = this.props; + const metrics = this.props.metrics.filter(x => selectedMetrics[x.id]).sort(x => x.name); + if (!error && data && formData.viz_type) { + // eslint-disable-next-line camelcase + const charts = this.getCharts(data, metrics, formData.separate_charts); + if (charts.length > 1) { + charts.filter(x => x).forEach((series, i) => { + const fd = Object.assign({}, formData); + const dataObj = { + data: series, + intervalCallback, + }; + const container = `${containerId}-${i}`; + fd.y_axis_label = metrics[i].name; + visMap[formData.viz_type]( + getMockedSliceObject(container, + fd, + metrics.concat(columns), + metrics.length, + handleAddFilter, + handleRemoveFilter), + dataObj, + ); + }); + } else if (charts.length && charts[0]) { + this.deleteGraphs(containerId); + const dataObj = { + data: charts[0], + intervalCallback, + }; + visMap[formData.viz_type](getMockedSliceObject(`${containerId}-0`, + formData, + metrics.concat(columns), + 1, + handleAddFilter, + handleRemoveFilter), dataObj); + } + } + } + + render() { + const { selectedMetrics, containerId, error, isRunning, formData, outdated } = this.props; + + const chartContainers = [`${containerId}-0`]; + const num = Object.keys(selectedMetrics).length; + if (formData.separate_charts && num > 1) { + for (let i = 1; i < num; i++) { + chartContainers.push(`${containerId}-${i}`); + } + } + + if (error) { + this.deleteGraphs(containerId); + } + + return ( +
+
+ {chartContainers.map((chart, index) => + (
+ {error} +
), + )} +
+
+
+ ); + } +} + +ChartContainer.propTypes = propTypes; + +const mapStateToProps = state => ({ + isRunning: state.controls.isRunning, + outdated: state.vizData.outdated, + error: state.controls.error, + data: state.vizData.data, + formData: state.vizData.formData, + metrics: state.refData.metrics, + columns: state.refData.columns, + selectedMetrics: state.settings.present.query.metrics, +}); + +const mapDispatchToProps = dispatch => ({ + intervalCallback: (intervalStart, intervalEnd) => + dispatch(changeInterval(intervalStart, intervalEnd)), + handleAddFilter: filter => dispatch(addFilter(filter)), + handleRemoveFilter: filter => dispatch(removeFilter(filter)), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(ChartContainer); diff --git a/superset/assets/javascripts/swivel/components/Column.jsx b/superset/assets/javascripts/swivel/components/Column.jsx new file mode 100644 index 0000000000000..3d8711499da27 --- /dev/null +++ b/superset/assets/javascripts/swivel/components/Column.jsx @@ -0,0 +1,116 @@ +import React, { Component } from 'react'; +import PropTypes from 'prop-types'; +import { Button, ButtonGroup, OverlayTrigger, Popover } from 'react-bootstrap'; +import { DragSource } from 'react-dnd'; + +import ItemTypes from '../ItemTypes'; +import ColumnTypes from '../ColumnTypes'; + +const style = { + cursor: 'move', + display: 'flex', + flexWrap: 'nowrap', + justifyContent: 'space-between', +}; + +const boxSource = { + beginDrag(props) { + return { + name: props.name, + id: props.id, + columnType: props.columnType, + groupable: props.groupable, + }; + }, +}; + +function collect(connect, monitor) { + return { + connectDragSource: connect.dragSource(), + isDragging: monitor.isDragging(), + }; +} + +const propTypes = { + connectDragSource: PropTypes.func.isRequired, + isDragging: PropTypes.bool.isRequired, + name: PropTypes.string.isRequired, + id: PropTypes.string.isRequired, + columnType: PropTypes.string.isRequired, + groupable: PropTypes.bool.isRequired, + + handleAddFilter: PropTypes.func.isRequired, + handleAddSplit: PropTypes.func.isRequired, +}; + +class Column extends Component { + constructor(props) { + super(props); + this.popoverRight.bind(this); + } + + popoverRight() { + const { handleAddFilter, handleAddSplit } = this.props; + return ( + + + + + + + ); + } + + render() { + const { name, isDragging, connectDragSource, columnType } = this.props; + const opacity = isDragging ? 0.4 : 1; + let icon = ''; + if (columnType === ColumnTypes.TIMESTAMP) { + icon = 'fa-clock-o'; + } else if (columnType === ColumnTypes.NUMERIC) { + icon = 'fa-hashtag'; + } else if (columnType === ColumnTypes.STRING) { + icon = 'fa-language'; + } + + return connectDragSource( +
+ +
+
+ {name} +
+