diff --git a/superset/assets/images/viz_thumbnails/deck_zipcodes.png b/superset/assets/images/viz_thumbnails/deck_zipcodes.png new file mode 100644 index 0000000000000..b15d521556dfa Binary files /dev/null and b/superset/assets/images/viz_thumbnails/deck_zipcodes.png differ diff --git a/superset/assets/images/viz_thumbnails_large/deck_zipcodes.png b/superset/assets/images/viz_thumbnails_large/deck_zipcodes.png new file mode 100644 index 0000000000000..21be5bf8aa81c Binary files /dev/null and b/superset/assets/images/viz_thumbnails_large/deck_zipcodes.png differ diff --git a/superset/assets/src/explore/visTypes.jsx b/superset/assets/src/explore/visTypes.jsx index f9cc723eebb3e..a67636c5700b3 100644 --- a/superset/assets/src/explore/visTypes.jsx +++ b/superset/assets/src/explore/visTypes.jsx @@ -701,6 +701,53 @@ export const visTypes = { ], }, + deck_zipcodes: { + label: t('Deck.gl - Zip codes'), + requiresTime: true, + controlPanelSections: [ + { + label: t('Query'), + expanded: true, + controlSetRows: [ + ['geojson', 'autozoom'], + ['color_picker', 'size'], + ['adhoc_filters'], + ], + }, + { + label: t('Map'), + controlSetRows: [ + ['mapbox_style', 'viewport'], + // TODO ['autozoom', null], + ], + }, + { + label: t('Advanced'), + controlSetRows: [ + ['js_columns'], + ['js_data_mutator'], + ['js_tooltip'], + ['js_onclick_href'], + ], + }, + ], + controlOverrides: { + adhoc_filters: { + validators: [v.nonEmpty], + }, + geojson: { + label: t('ZIP code'), + description: t('Column with ZIP codes'), + }, + size: { + label: t('Weight'), + description: t("Metric used as a weight for the grid's coloring"), + validators: [v.nonEmpty], + }, + time_grain_sqla: timeGrainSqlaAnimationOverrides, + }, + }, + deck_polygon: { label: t('Deck.gl - Polygon'), requiresTime: true, diff --git a/superset/assets/src/visualizations/PlaySlider.css b/superset/assets/src/visualizations/PlaySlider.css index 7de07de54ab1c..04687a87a06c2 100644 --- a/superset/assets/src/visualizations/PlaySlider.css +++ b/superset/assets/src/visualizations/PlaySlider.css @@ -1,8 +1,7 @@ .play-slider { position: absolute; - bottom: -16px; height: 20px; - width: 100%; + width: 90%; } .slider-selection { @@ -21,3 +20,7 @@ color: #b3b3b3; margin-right: 5px; } + +div.tooltip.tooltip-main.top.in { + margin-left: 0 !important; +} diff --git a/superset/assets/src/visualizations/PlaySlider.jsx b/superset/assets/src/visualizations/PlaySlider.jsx index b72dc635ca832..408f08df5c3aa 100644 --- a/superset/assets/src/visualizations/PlaySlider.jsx +++ b/superset/assets/src/visualizations/PlaySlider.jsx @@ -21,6 +21,7 @@ const propTypes = { orientation: PropTypes.oneOf(['horizontal', 'vertical']), reversed: PropTypes.bool, disabled: PropTypes.bool, + range: PropTypes.bool, }; const defaultProps = { @@ -30,6 +31,7 @@ const defaultProps = { orientation: 'horizontal', reversed: false, disabled: false, + range: true, }; export default class PlaySlider extends React.PureComponent { @@ -87,7 +89,11 @@ export default class PlaySlider extends React.PureComponent { if (this.props.disabled) { return; } - let values = this.props.values.map(value => value + this.increment); + let values = this.props.values; + if (!Array.isArray(values)) { + values = [values, values + this.props.step]; + } + values = values.map(value => value + this.increment); if (values[1] > this.props.end) { const cr = values[0] - this.props.start; values = values.map(value => value - cr); @@ -116,7 +122,8 @@ export default class PlaySlider extends React.PureComponent { this.setState({ values: newValues })} + range={!this.props.aggregation} + onChange={this.onChange} /> } {this.props.children} diff --git a/superset/assets/src/visualizations/deckgl/layers/index.js b/superset/assets/src/visualizations/deckgl/layers/index.js index d8d25d5b0e91b..00e170a537e49 100644 --- a/superset/assets/src/visualizations/deckgl/layers/index.js +++ b/superset/assets/src/visualizations/deckgl/layers/index.js @@ -7,6 +7,7 @@ import { getLayer as deck_scatter } from './scatter'; import { getLayer as deck_geojson } from './geojson'; import { getLayer as deck_arc } from './arc'; import { getLayer as deck_polygon } from './polygon'; +import { getLayer as deck_zipcodes } from './zipcodes'; const layerGenerators = { deck_grid, @@ -17,5 +18,6 @@ const layerGenerators = { deck_geojson, deck_arc, deck_polygon, + deck_zipcodes, }; export default layerGenerators; diff --git a/superset/assets/src/visualizations/deckgl/layers/zipcodes.jsx b/superset/assets/src/visualizations/deckgl/layers/zipcodes.jsx new file mode 100644 index 0000000000000..a07ee61d6559c --- /dev/null +++ b/superset/assets/src/visualizations/deckgl/layers/zipcodes.jsx @@ -0,0 +1,201 @@ +/* eslint no-underscore-dangle: ["error", { "allow": ["", "__timestamp"] }] */ + +import React from 'react'; +import ReactDOM from 'react-dom'; +import PropTypes from 'prop-types'; + +import { GeoJsonLayer } from 'deck.gl'; + +import AnimatableDeckGLContainer from '../AnimatableDeckGLContainer'; + +import * as common from './common'; +import { getPlaySliderParams } from '../../../modules/time'; +import sandboxedEval from '../../../modules/sandbox'; + +function getPoints(features) { + const points = []; + features.forEach((multipolygon) => { + if (multipolygon.geometry !== null) { + multipolygon.geometry.coordinates.forEach((polygon) => { + polygon.forEach((coordinates) => { + coordinates.forEach((point) => { + points.push(point); + }); + }); + }); + } + }); + return points; +} + +function getLayer(formData, payload, slice, filters) { + const fd = formData; + let data = payload.data.features; + + if (filters != null) { + filters.forEach((f) => { + data = data.filter(f); + }); + } + + // find values range + let minValue = Infinity; + let maxValue = -Infinity; + data.forEach((d) => { + if (d.geometry !== null) { + minValue = Math.min(minValue, d.metric); + maxValue = Math.max(maxValue, d.metric); + } + }); + + const c = fd.color_picker || { r: 0, g: 0, b: 0, a: 1 }; + data = data.map(d => ({ + ...d, + properties: { + color: [c.r, c.g, c.b, 255 * c.a * (d.metric - minValue) / (maxValue - minValue)], + metric: d.metric, + zipcode: d.zipcode, + }, + })); + data = data.filter(d => d.geometry !== null); + + if (fd.js_data_mutator) { + // Applying user defined data mutator if defined + const jsFnMutator = sandboxedEval(fd.js_data_mutator); + data = jsFnMutator(data); + } + + const layerProps = common.commonLayerProps(fd, slice); + if (layerProps.onHover === undefined) { + layerProps.pickable = true; + layerProps.onHover = (o) => { + if (o.picked) { + slice.setTooltip({ + content: 'ZIP code: ' + o.object.zipcode + '
Metric: ' + o.object.metric, + x: o.x, + y: o.y + 75, // weird offset + }); + } else { + slice.setTooltip(null); + } + }; + } + + return new GeoJsonLayer({ + id: `zipcodes-layer-${fd.slice_id}`, + data, + pickable: true, + stroked: true, + filled: true, + extruded: false, + lineWidthScale: 20, + lineWidthMinPixels: 1, + getFillColor: d => d.properties.color, + getLineColor: [0, 0, 0, 100], + getRadius: 100, + getLineWidth: 1, + getElevation: 30, + ...layerProps, + }); +} + +const propTypes = { + slice: PropTypes.object.isRequired, + payload: PropTypes.object.isRequired, + setControlValue: PropTypes.func.isRequired, + viewport: PropTypes.object.isRequired, +}; + +class DeckGLZipCodes extends React.PureComponent { + /* eslint-disable-next-line react/sort-comp */ + static getDerivedStateFromProps(nextProps) { + const fd = nextProps.slice.formData; + const features = nextProps.payload.data.features || []; + + const timeGrain = fd.time_grain_sqla || fd.granularity || 'PT1M'; + const timestamps = features.map(f => f.__timestamp); + const { start, end, step, values, disabled } = getPlaySliderParams(timestamps, timeGrain); + + return { start, end, step, values, disabled }; + } + constructor(props) { + super(props); + this.state = DeckGLZipCodes.getDerivedStateFromProps(props); + + this.getLayers = this.getLayers.bind(this); + } + componentWillReceiveProps(nextProps) { + this.setState(DeckGLZipCodes.getDerivedStateFromProps(nextProps, this.state)); + } + getLayers(values) { + if (this.props.payload.data.features === undefined) { + return []; + } + + const filters = []; + + // time filter + if (values[0] === values[1] || values[1] === this.end) { + filters.push(d => d.__timestamp >= values[0] && d.__timestamp <= values[1]); + } else { + filters.push(d => d.__timestamp >= values[0] && d.__timestamp < values[1]); + } + + const layer = getLayer( + this.props.slice.formData, + this.props.payload, + this.props.slice, + filters); + + return [layer]; + } + render() { + return ( +
+ +
+ ); + } +} + +DeckGLZipCodes.propTypes = propTypes; + +function deckZipCodes(slice, payload, setControlValue) { + const fd = slice.formData; + let viewport = { + ...fd.viewport, + width: slice.width(), + height: slice.height(), + }; + + if (fd.autozoom && payload.data.features) { + viewport = common.fitViewport(viewport, getPoints(payload.data.features)); + } + + ReactDOM.render( + , + document.getElementById(slice.containerId), + ); +} + +module.exports = { + default: deckZipCodes, + getLayer, +}; diff --git a/superset/assets/src/visualizations/index.js b/superset/assets/src/visualizations/index.js index 098079ea500dd..5192b09d1aa86 100644 --- a/superset/assets/src/visualizations/index.js +++ b/superset/assets/src/visualizations/index.js @@ -49,6 +49,7 @@ export const VIZ_TYPES = { deck_multi: 'deck_multi', deck_arc: 'deck_arc', deck_polygon: 'deck_polygon', + deck_zipcodes: 'deck_zipcodes', rose: 'rose', }; @@ -136,6 +137,8 @@ const vizMap = { loadVis(import(/* webpackChunkName: "deckgl/layers/polygon" */ './deckgl/layers/polygon.jsx')), [VIZ_TYPES.deck_multi]: () => loadVis(import(/* webpackChunkName: "deckgl/multi" */ './deckgl/multi.jsx')), + [VIZ_TYPES.deck_zipcodes]: () => + loadVis(import(/* webpackChunkName: "deckgl/layers/zipcodes" */ './deckgl/layers/zipcodes.jsx')), [VIZ_TYPES.rose]: () => loadVis(import(/* webpackChunkName: "rose" */ './rose.js')), }; diff --git a/superset/views/lyft.py b/superset/views/lyft.py index c64e688781ea9..ad8636e134e35 100644 --- a/superset/views/lyft.py +++ b/superset/views/lyft.py @@ -1,15 +1,20 @@ +# -*- coding: utf-8 -*- +# pylint: disable=broad-except,no-self-use from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from flask import Response, request, g +import os +import logging +from flask import Response, request +from flask_babel import gettext as __ from flask_appbuilder import expose +from flask_appbuilder.security.decorators import has_access_api -from superset import app, appbuilder, security_manager +from superset import app, appbuilder, utils import superset.models.core as models from superset.views.core import Superset -from .base import json_error_response config = app.config stats_logger = config.get('STATS_LOGGER') @@ -17,32 +22,35 @@ DAR = models.DatasourceAccessRequest +ALL_DATASOURCE_ACCESS_ERR = __( + 'This endpoint requires the `all_datasource_access` permission') +DATASOURCE_MISSING_ERR = __('The datasource seems to have been deleted') +ACCESS_REQUEST_MISSING_ERR = __( + 'The access requests seem to have been deleted') +USER_MISSING_ERR = __('The user seems to have been deleted') +DATASOURCE_ACCESS_ERR = __("You don't have access to this datasource") +SECRET_KEY = os.getenv("CREDENTIALS_SUPERSET_SECRET_KEY") or None + + def json_success(json_msg, status=200): return Response(json_msg, status=status, mimetype='application/json') class Lyft(Superset): - def authorize(self): - """Provides access if token, impersonates if specified""" - if not security_manager.has_tom_key(): - raise Exception("Wrong key") + def datasource_access(self, datasource, user=None): + if SECRET_KEY is None: + logging.error('No secret loaded') + return False - email = request.headers.get('IMPERSONATE') - if email: - user = security_manager.find_user(email=email) - if not user: - raise Exception("Email to impersonate not found") - g.user = user + tom_request_key = request.headers.get('TOM_ACCESS_KEY') + return tom_request_key == SECRET_KEY + @has_access_api @expose('/sql_json/', methods=['POST', 'GET']) @log_this def sql_json(self): - try: - self.authorize() - except Exception as e: - return json_error_response('{}'.format(e)) - return self.sql_json_call(request) + return super(Lyft, self).sql_json() appbuilder.add_view_no_menu(Lyft) diff --git a/superset/viz.py b/superset/viz.py index b0a71d5a9d3cc..30e8e80da1cda 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -18,6 +18,7 @@ from itertools import product import logging import math +import os import re import traceback import uuid @@ -36,6 +37,7 @@ import simplejson as json from six import string_types, text_type from six.moves import cPickle as pkl, reduce +import sqlalchemy from superset import app, cache, get_css_manifest_files, utils from superset.exceptions import NullValueException, SpatialException @@ -2229,7 +2231,6 @@ class DeckScreengrid(BaseDeckGLViz): viz_type = 'deck_screengrid' verbose_name = _('Deck.gl - Screen Grid') spatial_control_keys = ['spatial'] - is_timeseries = True def query_obj(self): fd = self.form_data @@ -2346,6 +2347,79 @@ def get_properties(self, d): return json.loads(geojson) +class DeckZipCodes(BaseDeckGLViz): + + """Custom viz for Lyft, shows zip codes as geojson.""" + + viz_type = 'deck_zipcodes' + verbose_name = _('Deck.gl - ZIP codes') + is_timeseries = True + + user = os.environ.get('CREDENTIALS_LYFTPG_USER', '') + password = os.environ.get('CREDENTIALS_LYFTPG_PASSWORD', '') + url = ( + 'postgresql+psycopg2://' + '{user}:{password}' + '@analytics-platform-vpc.c067nfzisc99.us-east-1.rds.amazonaws.com:5432' + '/platform'.format(user=user, password=password) + ) + + def query_obj(self): + fd = self.form_data + self.is_timeseries = fd.get('time_grain_sqla') or fd.get('granularity') + + d = super(DeckZipCodes, self).query_obj() + self.zipcode_col = self.form_data.get('geojson') + d['groupby'] = [self.zipcode_col] + return d + + def get_geojson(self, zipcodes): + out = {} + missing = set() + for zipcode in zipcodes: + cache_key = 'zipcode_geojson_{}'.format(zipcode) + geojson = cache and cache.get(cache_key) + if geojson: + out[zipcode] = geojson + else: + missing.add(str(zipcode)) + + if not missing: + return out + + # fetch missing geojson from lyftpg + in_clause = ', '.join(['%s'] * len(missing)) + query = ( + 'SELECT zipcode, geojson FROM zip_codes WHERE zipcode IN ({0})' + .format(in_clause)) + conn = sqlalchemy.create_engine(self.url, client_encoding='utf8') + results = conn.execute(query, tuple(missing)).fetchall() + + for zipcode, geojson in results: + out[zipcode] = geojson + if cache and len(results) < 10000: # avoid storing too much + cache_key = 'zipcode_geojson_{}'.format(zipcode) + try: + cache.set(cache_key, geojson, timeout=86400) + except Exception: + pass + + return out + + def get_data(self, df): + self.metric_label = self.get_metric_label(self.metric) + self.zipcodes = self.get_geojson(set(df[self.zipcode_col])) + return super(DeckZipCodes, self).get_data(df) + + def get_properties(self, d): + return { + 'metric': d.get(self.metric_label) or 1, + 'zipcode': d.get(self.zipcode_col), + 'geometry': self.zipcodes.get(str(d[self.zipcode_col])), + DTTM_ALIAS: d.get(DTTM_ALIAS), + } + + class DeckArc(BaseDeckGLViz): """deck.gl's Arc Layer"""