Reduce dashboard position_json data size (apache#5543)

(cherry picked from commit 2e2c980)

superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js

@@ -10,94 +10,91 @@ import {
 describe('findFirstParentContainer', () => {
   const mockGridLayout = {
     DASHBOARD_VERSION_KEY: 'v2',
-    DASHBOARD_ROOT_ID: {
-      type: 'DASHBOARD_ROOT_TYPE',
-      id: 'DASHBOARD_ROOT_ID',
-      children: ['DASHBOARD_GRID_ID'],
-    },
-    DASHBOARD_GRID_ID: {
-      type: 'DASHBOARD_GRID_TYPE',
-      id: 'DASHBOARD_GRID_ID',
-      children: ['DASHBOARD_ROW_TYPE-Bk45URrlQ'],
-    },
-    'DASHBOARD_ROW_TYPE-Bk45URrlQ': {
-      type: 'DASHBOARD_ROW_TYPE',
-      id: 'DASHBOARD_ROW_TYPE-Bk45URrlQ',
-      children: ['DASHBOARD_CHART_TYPE-ryxVc8RHlX'],
-    },
-    'DASHBOARD_CHART_TYPE-ryxVc8RHlX': {
-      type: 'DASHBOARD_CHART_TYPE',
-      id: 'DASHBOARD_CHART_TYPE-ryxVc8RHlX',
+    ROOT_ID: {
+      type: 'ROOT',
+      id: 'ROOT_ID',
+      children: ['GRID_ID'],
+    },
+    GRID_ID: {
+      type: 'GRID',
+      id: 'GRID_ID',
+      children: ['ROW-Bk45URrlQ'],
+    },
+    'ROW-Bk45URrlQ': {
+      type: 'ROW',
+      id: 'ROW-Bk45URrlQ',
+      children: ['CHART-ryxVc8RHlX'],
+    },
+    'CHART-ryxVc8RHlX': {
+      type: 'CHART',
+      id: 'CHART-ryxVc8RHlX',
       children: [],
     },
-    DASHBOARD_HEADER_ID: {
-      id: 'DASHBOARD_HEADER_ID',
-      type: 'DASHBOARD_HEADER_TYPE',
+    HEADER_ID: {
+      id: 'HEADER_ID',
+      type: 'HEADER',
     },
   };
   const mockTabsLayout = {
-    'DASHBOARD_CHART_TYPE-S1gilYABe7': {
+    'CHART-S1gilYABe7': {
       children: [],
-      id: 'DASHBOARD_CHART_TYPE-S1gilYABe7',
-      type: 'DASHBOARD_CHART_TYPE',
+      id: 'CHART-S1gilYABe7',
+      type: 'CHART',
     },
-    'DASHBOARD_CHART_TYPE-SJli5K0HlQ': {
+    'CHART-SJli5K0HlQ': {
       children: [],
-      id: 'DASHBOARD_CHART_TYPE-SJli5K0HlQ',
-      type: 'DASHBOARD_CHART_TYPE',
+      id: 'CHART-SJli5K0HlQ',
+      type: 'CHART',
     },
-    DASHBOARD_GRID_ID: {
+    GRID_ID: {
       children: [],
-      id: 'DASHBOARD_GRID_ID',
-      type: 'DASHBOARD_GRID_TYPE',
-    },
-    DASHBOARD_HEADER_ID: {
-      id: 'DASHBOARD_HEADER_ID',
-      type: 'DASHBOARD_HEADER_TYPE',
-    },
-    DASHBOARD_ROOT_ID: {
-      children: ['DASHBOARD_TABS_TYPE-SkgJ5t0Bem'],
-      id: 'DASHBOARD_ROOT_ID',
-      type: 'DASHBOARD_ROOT_TYPE',
-    },
-    'DASHBOARD_ROW_TYPE-S1B8-JLgX': {
-      children: ['DASHBOARD_CHART_TYPE-SJli5K0HlQ'],
-      id: 'DASHBOARD_ROW_TYPE-S1B8-JLgX',
-      type: 'DASHBOARD_ROW_TYPE',
-    },
-    'DASHBOARD_ROW_TYPE-S1bUb1Ilm': {
-      children: ['DASHBOARD_CHART_TYPE-S1gilYABe7'],
-      id: 'DASHBOARD_ROW_TYPE-S1bUb1Ilm',
-      type: 'DASHBOARD_ROW_TYPE',
-    },
-    'DASHBOARD_TABS_TYPE-ByeLSWyLe7': {
-      children: ['DASHBOARD_TAB_TYPE-BJbLSZ1UeQ'],
-      id: 'DASHBOARD_TABS_TYPE-ByeLSWyLe7',
-      type: 'DASHBOARD_TABS_TYPE',
-    },
-    'DASHBOARD_TABS_TYPE-SkgJ5t0Bem': {
-      children: [
-        'DASHBOARD_TAB_TYPE-HkWJcFCHxQ',
-        'DASHBOARD_TAB_TYPE-ByDBbkLlQ',
-      ],
-      id: 'DASHBOARD_TABS_TYPE-SkgJ5t0Bem',
+      id: 'GRID_ID',
+      type: 'GRID',
+    },
+    HEADER_ID: {
+      id: 'HEADER_ID',
+      type: 'HEADER',
+    },
+    ROOT_ID: {
+      children: ['TABS-SkgJ5t0Bem'],
+      id: 'ROOT_ID',
+      type: 'ROOT',
+    },
+    'ROW-S1B8-JLgX': {
+      children: ['CHART-SJli5K0HlQ'],
+      id: 'ROW-S1B8-JLgX',
+      type: 'ROW',
+    },
+    'ROW-S1bUb1Ilm': {
+      children: ['CHART-S1gilYABe7'],
+      id: 'ROW-S1bUb1Ilm',
+      type: 'ROW',
+    },
+    'TABS-ByeLSWyLe7': {
+      children: ['TAB-BJbLSZ1UeQ'],
+      id: 'TABS-ByeLSWyLe7',
+      type: 'TABS',
+    },
+    'TABS-SkgJ5t0Bem': {
+      children: ['TAB-HkWJcFCHxQ', 'TAB-ByDBbkLlQ'],
+      id: 'TABS-SkgJ5t0Bem',
       meta: {},
-      type: 'DASHBOARD_TABS_TYPE',
-    },
-    'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ': {
-      children: ['DASHBOARD_ROW_TYPE-S1bUb1Ilm'],
-      id: 'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ',
-      type: 'DASHBOARD_TAB_TYPE',
-    },
-    'DASHBOARD_TAB_TYPE-ByDBbkLlQ': {
-      children: ['DASHBOARD_ROW_TYPE-S1B8-JLgX'],
-      id: 'DASHBOARD_TAB_TYPE-ByDBbkLlQ',
-      type: 'DASHBOARD_TAB_TYPE',
-    },
-    'DASHBOARD_TAB_TYPE-HkWJcFCHxQ': {
-      children: ['DASHBOARD_TABS_TYPE-ByeLSWyLe7'],
-      id: 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ',
-      type: 'DASHBOARD_TAB_TYPE',
+      type: 'TABS',
+    },
+    'TAB-BJbLSZ1UeQ': {
+      children: ['ROW-S1bUb1Ilm'],
+      id: 'TAB-BJbLSZ1UeQ',
+      type: 'TAB',
+    },
+    'TAB-ByDBbkLlQ': {
+      children: ['ROW-S1B8-JLgX'],
+      id: 'TAB-ByDBbkLlQ',
+      type: 'TAB',
+    },
+    'TAB-HkWJcFCHxQ': {
+      children: ['TABS-ByeLSWyLe7'],
+      id: 'TAB-HkWJcFCHxQ',
+      type: 'TAB',
     },
     DASHBOARD_VERSION_KEY: 'v2',
   };

superset/assets/src/dashboard/components/Header.jsx

@@ -10,11 +10,16 @@ import UndoRedoKeylisteners from './UndoRedoKeylisteners';
 
 import { chartPropShape } from '../util/propShapes';
 import { t } from '../../locales';
-import { UNDO_LIMIT, SAVE_TYPE_OVERWRITE } from '../util/constants';
+import {
+  UNDO_LIMIT,
+  SAVE_TYPE_OVERWRITE,
+  DASHBOARD_POSITION_DATA_LIMIT,
+} from '../util/constants';
 
 const propTypes = {
   addSuccessToast: PropTypes.func.isRequired,
   addDangerToast: PropTypes.func.isRequired,
+  addWarningToast: PropTypes.func.isRequired,
   dashboardInfo: PropTypes.object.isRequired,
   dashboardTitle: PropTypes.string.isRequired,
   charts: PropTypes.objectOf(chartPropShape).isRequired,
@@ -143,7 +148,24 @@ class Header extends React.PureComponent {
       default_filters: JSON.stringify(filters),
     };
 
-    this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE);
+    // make sure positions data less than DB storage limitation:
+    const positionJSONLength = JSON.stringify(positions).length;
+    const limit =
+      dashboardInfo.common.conf.SUPERSET_DASHBOARD_POSITION_DATA_LIMIT ||
+      DASHBOARD_POSITION_DATA_LIMIT;
+    if (positionJSONLength >= limit) {
+      this.props.addDangerToast(
+        t(
+          'Your dashboard is too large. Please reduce the size before save it.',
+        ),
+      );
+    } else {
+      if (positionJSONLength >= limit * 0.9) {
+        this.props.addWarningToast('Your dashboard is near the size limit.');
+      }
+
+      this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE);
+    }
   }
 
   render() {

superset/assets/src/dashboard/containers/DashboardHeader.jsx

@@ -23,7 +23,11 @@ import {
   updateDashboardTitle,
 } from '../actions/dashboardLayout';
 
-import { addSuccessToast, addDangerToast } from '../actions/messageToasts';
+import {
+  addSuccessToast,
+  addDangerToast,
+  addWarningToast,
+} from '../actions/messageToasts';
 
 import { DASHBOARD_HEADER_ID } from '../util/constants';
 
@@ -59,6 +63,7 @@ function mapDispatchToProps(dispatch) {
     {
       addSuccessToast,
       addDangerToast,
+      addWarningToast,
       onUndo: undoLayoutAction,
       onRedo: redoLayoutAction,
       setEditMode,

superset/assets/src/dashboard/util/componentTypes.js

@@ -1,15 +1,15 @@
-export const CHART_TYPE = 'DASHBOARD_CHART_TYPE';
-export const COLUMN_TYPE = 'DASHBOARD_COLUMN_TYPE';
-export const DASHBOARD_HEADER_TYPE = 'DASHBOARD_HEADER_TYPE';
-export const DASHBOARD_GRID_TYPE = 'DASHBOARD_GRID_TYPE';
-export const DASHBOARD_ROOT_TYPE = 'DASHBOARD_ROOT_TYPE';
-export const DIVIDER_TYPE = 'DASHBOARD_DIVIDER_TYPE';
-export const HEADER_TYPE = 'DASHBOARD_HEADER_TYPE';
-export const MARKDOWN_TYPE = 'DASHBOARD_MARKDOWN_TYPE';
-export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE_TYPE';
-export const ROW_TYPE = 'DASHBOARD_ROW_TYPE';
-export const TABS_TYPE = 'DASHBOARD_TABS_TYPE';
-export const TAB_TYPE = 'DASHBOARD_TAB_TYPE';
+export const CHART_TYPE = 'CHART';
+export const COLUMN_TYPE = 'COLUMN';
+export const DASHBOARD_HEADER_TYPE = 'HEADER';
+export const DASHBOARD_GRID_TYPE = 'GRID';
+export const DASHBOARD_ROOT_TYPE = 'ROOT';
+export const DIVIDER_TYPE = 'DIVIDER';
+export const HEADER_TYPE = 'HEADER';
+export const MARKDOWN_TYPE = 'MARKDOWN';
+export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE';
+export const ROW_TYPE = 'ROW';
+export const TABS_TYPE = 'TABS';
+export const TAB_TYPE = 'TAB';
 
 export default {
   CHART_TYPE,

superset/assets/src/dashboard/util/constants.js

@@ -1,7 +1,7 @@
 // Ids
-export const DASHBOARD_GRID_ID = 'DASHBOARD_GRID_ID';
-export const DASHBOARD_HEADER_ID = 'DASHBOARD_HEADER_ID';
-export const DASHBOARD_ROOT_ID = 'DASHBOARD_ROOT_ID';
+export const DASHBOARD_GRID_ID = 'GRID_ID';
+export const DASHBOARD_HEADER_ID = 'HEADER_ID';
+export const DASHBOARD_ROOT_ID = 'ROOT_ID';
 export const DASHBOARD_VERSION_KEY = 'DASHBOARD_VERSION_KEY';
 
 export const NEW_COMPONENTS_SOURCE_ID = 'NEW_COMPONENTS_SOURCE_ID';
@@ -45,3 +45,7 @@ export const UNDO_LIMIT = 50;
 // save dash options
 export const SAVE_TYPE_OVERWRITE = 'overwrite';
 export const SAVE_TYPE_NEWDASHBOARD = 'newDashboard';
+
+// default dashboard layout data size limit
+// could be overwritten by server-side config
+export const DASHBOARD_POSITION_DATA_LIMIT = 65535;

superset/config.py

@@ -49,6 +49,7 @@
 SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0'
 SUPERSET_WEBSERVER_PORT = 8088
 SUPERSET_WEBSERVER_TIMEOUT = 60  # deprecated
+SUPERSET_DASHBOARD_POSITION_DATA_LIMIT = 65535
 EMAIL_NOTIFICATIONS = False
 CUSTOM_SECURITY_MANAGER = None
 SQLALCHEMY_TRACK_MODIFICATIONS = False

superset/migrations/versions/7fcdcde0761c_.py

@@ -0,0 +1,69 @@
+"""Reduce position_json size by remove extra space and component id prefix
+
+Revision ID: 7fcdcde0761c
+Revises: c18bd4186f15
+Create Date: 2018-08-01 11:47:02.233971
+
+"""
+
+# revision identifiers, used by Alembic.
+import json
+import re
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import (
+    Table, Column,
+    Integer, String, Text, ForeignKey,
+)
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+from superset import db
+
+revision = '7fcdcde0761c'
+down_revision = 'c18bd4186f15'
+
+Base = declarative_base()
+
+
+class Dashboard(Base):
+    """Declarative class to do query in upgrade"""
+    __tablename__ = 'dashboards'
+    id = sa.Column(sa.Integer, primary_key=True)
+    dashboard_title = sa.Column(String(500))
+    position_json = sa.Column(sa.Text)
+
+
+def is_v2_dash(positions):
+    return (
+        isinstance(positions, dict) and
+        positions.get('DASHBOARD_VERSION_KEY') == 'v2'
+    )
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+
+    dashboards = session.query(Dashboard).all()
+    for i, dashboard in enumerate(dashboards):
+        original_text = dashboard.position_json or ''
+        position_json = json.loads(original_text or '{}')
+        if is_v2_dash(position_json):
+            # re-dump the json data and remove leading and trailing white spaces
+            text = json.dumps(
+                position_json, indent=None, separators=(',', ':'), sort_keys=True)
+            # remove DASHBOARD_ and _TYPE prefix/suffix in all the component ids
+            text = re.sub(r'DASHBOARD_(?!VERSION)', '', text)
+            text = text.replace('_TYPE', '')
+
+            dashboard.position_json = text
+            print('dash id:{} position_json size from {} to {}'
+                .format(dashboard.id, len(original_text), len(text)))
+            session.merge(dashboard)
+            session.commit()
+
+
+def downgrade():
+    pass

superset/views/base.py

@@ -26,6 +26,7 @@
 
 FRONTEND_CONF_KEYS = (
     'SUPERSET_WEBSERVER_TIMEOUT',
+    'SUPERSET_DASHBOARD_POSITION_DATA_LIMIT',
     'ENABLE_JAVASCRIPT_CONTROLS',
 )
 

superset/views/core.py

@@ -1631,6 +1631,9 @@ def _set_dash_metadata(dashboard, data):
                 session.merge(slc)
                 session.flush()
 
+        # remove leading and trailing white spaces in the dumped json
+        dashboard.position_json = json.dumps(
+            positions, indent=None, separators=(',', ':'), sort_keys=True)
         dashboard.position_json = json.dumps(positions, sort_keys=True)
         md = dashboard.params_dict
         dashboard.css = data.get('css')