From 78408e50c52e0068b8f2523391252864dc0aa5ab Mon Sep 17 00:00:00 2001 From: yuua Date: Thu, 29 Dec 2016 10:54:57 +0900 Subject: [PATCH 001/149] impala query_runner get_tables error fix --- redash/query_runner/impala_ds.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/redash/query_runner/impala_ds.py b/redash/query_runner/impala_ds.py index e3337006e5..dec4c3c079 100644 --- a/redash/query_runner/impala_ds.py +++ b/redash/query_runner/impala_ds.py @@ -82,11 +82,11 @@ def __init__(self, configuration): def _get_tables(self, schema_dict): schemas_query = "show schemas;" tables_query = "show tables in %s;" - columns_query = "show column stats %s;" + columns_query = "show column stats %s.%s;" - for schema_name in map(lambda a: a['name'], self._run_query_internal(schemas_query)): - for table_name in map(lambda a: a['name'], self._run_query_internal(tables_query % schema_name)): - columns = map(lambda a: a['Column'], self._run_query_internal(columns_query % table_name)) + for schema_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['name']), self._run_query_internal(schemas_query))): + for table_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['name']), self._run_query_internal(tables_query % schema_name))): + columns = filter(lambda a: len(a) > 0, map(lambda a: str(a['Column']), self._run_query_internal(columns_query % (schema_name, table_name)))) if schema_name != 'default': table_name = '{}.{}'.format(schema_name, table_name) From 96bc3a5d0e16e48b4aed260f18bb684e905874ad Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Tue, 3 Jan 2017 18:14:36 -0800 Subject: [PATCH 002/149] Adding global params to dashboard by prefixing with '$' --- client/app/pages/dashboards/dashboard.html | 4 +++ client/app/pages/dashboards/dashboard.js | 35 ++++++++++++++++++++-- client/app/pages/dashboards/widget.js | 4 ++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/client/app/pages/dashboards/dashboard.html b/client/app/pages/dashboards/dashboard.html index 34af7e21b7..2037bbe6b8 100644 --- a/client/app/pages/dashboards/dashboard.html +++ b/client/app/pages/dashboards/dashboard.html @@ -54,6 +54,10 @@

This dashboard is archived and won't appear in the dashboards list or search results. +
+ +
+
diff --git a/client/app/pages/dashboards/dashboard.js b/client/app/pages/dashboards/dashboard.js index 5bd1873b73..06f05fbeab 100644 --- a/client/app/pages/dashboards/dashboard.js +++ b/client/app/pages/dashboards/dashboard.js @@ -2,12 +2,13 @@ import * as _ from 'underscore'; import template from './dashboard.html'; import shareDashboardTemplate from './share-dashboard.html'; -function DashboardCtrl($rootScope, $routeParams, $location, $timeout, $q, $uibModal, +function DashboardCtrl($rootScope, $scope, $routeParams, $location, $timeout, $q, $uibModal, Title, AlertDialog, Dashboard, currentUser, clientConfig, Events) { this.isFullscreen = false; this.refreshRate = null; this.showPermissionsControl = clientConfig.showPermissionsControl; this.currentUser = currentUser; + this.globalParameters = []; this.refreshRates = [ { name: '10 seconds', rate: 10 }, { name: '30 seconds', rate: 30 }, @@ -26,6 +27,34 @@ function DashboardCtrl($rootScope, $routeParams, $location, $timeout, $q, $uibMo } }; + const extractGlobalParameters = () => { + let globalParams = {}; + this.dashboard.widgets.forEach(row => + row.forEach((widget) => { + widget.getQuery().getParametersDefs().forEach((param) => { + if (param.name[0] === '$') { + const defaults = {}; + defaults[param.name] = _.clone(param); + defaults[param.name].locals = []; + globalParams = _.defaults(globalParams, defaults); + globalParams[param.name].locals.push(param); + } + }); + }) + ); + this.globalParameters = _.values(globalParams); + }; + + $scope.$watch(() => this.globalParameters, (parameters) => { + _.each(parameters, (global) => { + _.each(global.locals, (local) => { + local.value = global.value; + }); + }); + }, true); + + $scope.$on('deleteDashboardWidget', extractGlobalParameters); + const renderDashboard = (dashboard, force) => { Title.set(dashboard.name); const promises = []; @@ -42,6 +71,8 @@ function DashboardCtrl($rootScope, $routeParams, $location, $timeout, $q, $uibMo }) ); + extractGlobalParameters(); + $q.all(promises).then((queryResults) => { const filters = {}; queryResults.forEach((queryResult) => { @@ -139,7 +170,7 @@ function DashboardCtrl($rootScope, $routeParams, $location, $timeout, $q, $uibMo resolve: { dashboard: () => this.dashboard, }, - }); + }).result.then(() => extractGlobalParameters()); }; this.toggleFullscreen = () => { diff --git a/client/app/pages/dashboards/widget.js b/client/app/pages/dashboards/widget.js index a6c714ad30..81bb809af3 100644 --- a/client/app/pages/dashboards/widget.js +++ b/client/app/pages/dashboards/widget.js @@ -24,7 +24,7 @@ const EditTextBoxComponent = { }, }; -function DashboardWidgetCtrl($location, $uibModal, $window, Events, currentUser) { +function DashboardWidgetCtrl($location, $uibModal, $window, $scope, Events, currentUser) { this.canViewQuery = currentUser.hasPermission('view_query'); this.editTextBox = () => { @@ -51,6 +51,8 @@ function DashboardWidgetCtrl($location, $uibModal, $window, Events, currentUser) this.dashboard.layout = response.layout; this.dashboard.version = response.version; + + $scope.$emit('deleteDashboardWidget'); }); }; From b1fd2101df18e766f6f2f9674c63f0de15a82ba0 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Wed, 4 Jan 2017 08:29:25 -0800 Subject: [PATCH 003/149] Refactored to make more efficent and cleaner --- client/app/components/parameters.js | 4 ++++ client/app/pages/dashboards/dashboard.html | 2 +- client/app/pages/dashboards/dashboard.js | 20 +++++++++----------- client/app/pages/dashboards/widget.html | 2 +- client/app/services/query.js | 15 +++++++++++++++ 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/client/app/components/parameters.js b/client/app/components/parameters.js index 59d314ba04..8e5439e788 100644 --- a/client/app/components/parameters.js +++ b/client/app/components/parameters.js @@ -23,12 +23,16 @@ function ParametersDirective($location, $uibModal) { parameters: '=', syncValues: '=?', editable: '=?', + change: '&onChange', }, template, link(scope) { // is this the correct location for this logic? if (scope.syncValues !== false) { scope.$watch('parameters', () => { + if (scope.change) { + scope.change({}); + } scope.parameters.forEach((param) => { if (param.value !== null || param.value !== '') { $location.search(`p_${param.name}`, param.value); diff --git a/client/app/pages/dashboards/dashboard.html b/client/app/pages/dashboards/dashboard.html index 2037bbe6b8..bcce3fd868 100644 --- a/client/app/pages/dashboards/dashboard.html +++ b/client/app/pages/dashboards/dashboard.html @@ -55,7 +55,7 @@

- +
diff --git a/client/app/pages/dashboards/dashboard.js b/client/app/pages/dashboards/dashboard.js index 06f05fbeab..e04da7108c 100644 --- a/client/app/pages/dashboards/dashboard.js +++ b/client/app/pages/dashboards/dashboard.js @@ -31,27 +31,25 @@ function DashboardCtrl($rootScope, $scope, $routeParams, $location, $timeout, $q let globalParams = {}; this.dashboard.widgets.forEach(row => row.forEach((widget) => { - widget.getQuery().getParametersDefs().forEach((param) => { - if (param.name[0] === '$') { - const defaults = {}; - defaults[param.name] = _.clone(param); - defaults[param.name].locals = []; - globalParams = _.defaults(globalParams, defaults); - globalParams[param.name].locals.push(param); - } + widget.getQuery().getGlobalParametersDefs().forEach((param) => { + const defaults = {}; + defaults[param.name] = _.clone(param); + defaults[param.name].locals = []; + globalParams = _.defaults(globalParams, defaults); + globalParams[param.name].locals.push(param); }); }) ); this.globalParameters = _.values(globalParams); }; - $scope.$watch(() => this.globalParameters, (parameters) => { - _.each(parameters, (global) => { + this.onGlobalParametersChange = () => { + _.each(this.globalParameters, (global) => { _.each(global.locals, (local) => { local.value = global.value; }); }); - }, true); + }; $scope.$on('deleteDashboardWidget', extractGlobalParameters); diff --git a/client/app/pages/dashboards/widget.html b/client/app/pages/dashboards/widget.html index 3109c4dc6a..1bd1622a50 100644 --- a/client/app/pages/dashboards/widget.html +++ b/client/app/pages/dashboards/widget.html @@ -25,7 +25,7 @@
- +
diff --git a/client/app/services/query.js b/client/app/services/query.js index 13b84716ce..fdc1d050b2 100644 --- a/client/app/services/query.js +++ b/client/app/services/query.js @@ -293,6 +293,21 @@ function QueryResource($resource, $http, $q, $location, currentUser, QueryResult return this.getParameters().get(); }; + Query.prototype.getLocalParametersDefs = function getLocalParametersDefs() { + if (!this.$localParameters) { + this.$localParameters = this.getParametersDefs().filter(p => p.name[0] !== '$'); + } + + return this.$localParameters; + }; + + Query.prototype.getGlobalParametersDefs = function getGlobalParametersDefs() { + if (!this.$globalParameters) { + this.$globalParameters = this.getParametersDefs().filter(p => p.name[0] === '$'); + } + return this.$globalParameters; + }; + return Query; } From 2aae4971693000700fbd66d42aca86abfbc62154 Mon Sep 17 00:00:00 2001 From: sseifert Date: Sat, 7 Jan 2017 00:41:42 +0100 Subject: [PATCH 004/149] include custom fields in JQL query results convert list values to concatenated strings support definition of fieldMapping property in query which allows to rename output columns or pick specific members from dict values --- redash/query_runner/jql.py | 51 +++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/redash/query_runner/jql.py b/redash/query_runner/jql.py index 50f110adfd..533f7ac562 100644 --- a/redash/query_runner/jql.py +++ b/redash/query_runner/jql.py @@ -26,15 +26,33 @@ def to_json(self): return json.dumps({'rows': self.rows, 'columns': self.columns.values()}) -def parse_issue(issue): +def parse_issue(issue, fieldMapping): result = OrderedDict() result['key'] = issue['key'] for k, v in issue['fields'].iteritems(): - if k.startswith('customfield_'): - continue - if isinstance(v, dict): + # if field mapping is defined optionally change output key and parsing rules for value + if k in fieldMapping: + mapping = fieldMapping[k] + output_key = k + if 'name' in mapping: + output_key = mapping['name'] + put_value(result, output_key, v, mapping) + + else: + put_value(result, k, v, {}) + + return result + + +def put_value(result, k, v, mapping): + if isinstance(v, dict): + if 'member' in mapping: + result[k] = v[mapping['member']] + + else: + # these special mapping rules are kept for backwards compatibility if 'key' in v: result['{}_key'.format(k)] = v['key'] if 'name' in v: @@ -45,19 +63,27 @@ def parse_issue(issue): if 'watchCount' in v: result[k] = v['watchCount'] - # elif isinstance(v, list): - # pass - else: - result[k] = v + + elif isinstance(v, list): + listValues = [] + for listItem in v: + if isinstance(listItem, dict): + if 'member' in mapping: + listValues.append(listItem[mapping['member']]) + else: + listValues.append(listItem) - return result + result[k] = ','.join(listValues) + + else: + result[k] = v -def parse_issues(data): +def parse_issues(data, fieldMapping): results = ResultSet() for issue in data['issues']: - results.add_row(parse_issue(issue)) + results.add_row(parse_issue(issue, fieldMapping)) return results @@ -109,6 +135,7 @@ def run_query(self, query, user): try: query = json.loads(query) query_type = query.pop('queryType', 'select') + fieldMapping = query.pop('fieldMapping', {}) if query_type == 'count': query['maxResults'] = 1 @@ -127,7 +154,7 @@ def run_query(self, query, user): if query_type == 'count': results = parse_count(data) else: - results = parse_issues(data) + results = parse_issues(data, fieldMapping) return results.to_json(), None except KeyboardInterrupt: From 689a1aac4d27f4e5baedb8ba4f4888e07a96c71b Mon Sep 17 00:00:00 2001 From: sseifert Date: Sat, 7 Jan 2017 12:15:30 +0100 Subject: [PATCH 005/149] should set nothing instead of empty string as value when no valid list items are found --- redash/query_runner/jql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redash/query_runner/jql.py b/redash/query_runner/jql.py index 533f7ac562..dea1426c69 100644 --- a/redash/query_runner/jql.py +++ b/redash/query_runner/jql.py @@ -72,8 +72,8 @@ def put_value(result, k, v, mapping): listValues.append(listItem[mapping['member']]) else: listValues.append(listItem) - - result[k] = ','.join(listValues) + if len(listValues) > 0: + result[k] = ','.join(listValues) else: result[k] = v From 9126874c879183bfe685961f65f47219ecd1fd8a Mon Sep 17 00:00:00 2001 From: sseifert Date: Mon, 9 Jan 2017 12:27:50 +0100 Subject: [PATCH 006/149] apply snake_case naming conventions --- redash/query_runner/jql.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/redash/query_runner/jql.py b/redash/query_runner/jql.py index dea1426c69..be17a306ae 100644 --- a/redash/query_runner/jql.py +++ b/redash/query_runner/jql.py @@ -26,15 +26,15 @@ def to_json(self): return json.dumps({'rows': self.rows, 'columns': self.columns.values()}) -def parse_issue(issue, fieldMapping): +def parse_issue(issue, field_mapping): result = OrderedDict() result['key'] = issue['key'] for k, v in issue['fields'].iteritems(): # if field mapping is defined optionally change output key and parsing rules for value - if k in fieldMapping: - mapping = fieldMapping[k] + if k in field_mapping: + mapping = field_mapping[k] output_key = k if 'name' in mapping: output_key = mapping['name'] @@ -79,11 +79,11 @@ def put_value(result, k, v, mapping): result[k] = v -def parse_issues(data, fieldMapping): +def parse_issues(data, field_mapping): results = ResultSet() for issue in data['issues']: - results.add_row(parse_issue(issue, fieldMapping)) + results.add_row(parse_issue(issue, field_mapping)) return results @@ -135,7 +135,7 @@ def run_query(self, query, user): try: query = json.loads(query) query_type = query.pop('queryType', 'select') - fieldMapping = query.pop('fieldMapping', {}) + field_mapping = query.pop('fieldMapping', {}) if query_type == 'count': query['maxResults'] = 1 @@ -154,7 +154,7 @@ def run_query(self, query, user): if query_type == 'count': results = parse_count(data) else: - results = parse_issues(data, fieldMapping) + results = parse_issues(data, field_mapping) return results.to_json(), None except KeyboardInterrupt: From 187b557eeeea767e3cede203d3173ad17e1fa967 Mon Sep 17 00:00:00 2001 From: sseifert Date: Mon, 9 Jan 2017 14:37:18 +0100 Subject: [PATCH 007/149] adapt to new field mapping syntax and add unit tests --- redash/query_runner/jql.py | 127 ++++++++++++++++++++++----------- tests/query_runner/test_jql.py | 104 +++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 42 deletions(-) create mode 100644 tests/query_runner/test_jql.py diff --git a/redash/query_runner/jql.py b/redash/query_runner/jql.py index be17a306ae..2b37339c48 100644 --- a/redash/query_runner/jql.py +++ b/redash/query_runner/jql.py @@ -1,5 +1,6 @@ import json import requests +import re from collections import OrderedDict @@ -30,53 +31,55 @@ def parse_issue(issue, field_mapping): result = OrderedDict() result['key'] = issue['key'] - for k, v in issue['fields'].iteritems(): + for k, v in issue['fields'].iteritems():# + output_name = field_mapping.get_output_field_name(k) + member_names = field_mapping.get_dict_members(k) - # if field mapping is defined optionally change output key and parsing rules for value - if k in field_mapping: - mapping = field_mapping[k] - output_key = k - if 'name' in mapping: - output_key = mapping['name'] - put_value(result, output_key, v, mapping) - - else: - put_value(result, k, v, {}) - - return result + if isinstance(v, dict): + if len(member_names) > 0: + # if field mapping with dict member mappings defined get value of each member + for member_name in member_names: + if member_name in v: + result[field_mapping.get_dict_output_field_name(k,member_name)] = v[member_name] + else: + # these special mapping rules are kept for backwards compatibility + if 'key' in v: + result['{}_key'.format(output_name)] = v['key'] + if 'name' in v: + result['{}_name'.format(output_name)] = v['name'] + + if k in v: + result[output_name] = v[k] + + if 'watchCount' in v: + result[output_name] = v['watchCount'] + + elif isinstance(v, list): + if len(member_names) > 0: + # if field mapping with dict member mappings defined get value of each member + for member_name in member_names: + listValues = [] + for listItem in v: + if isinstance(listItem, dict): + if member_name in listItem: + listValues.append(listItem[member_name]) + if len(listValues) > 0: + result[field_mapping.get_dict_output_field_name(k,member_name)] = ','.join(listValues) -def put_value(result, k, v, mapping): - if isinstance(v, dict): - if 'member' in mapping: - result[k] = v[mapping['member']] + else: + # otherwise support list values only for non-dict items + listValues = [] + for listItem in v: + if not isinstance(listItem, dict): + listValues.append(listItem) + if len(listValues) > 0: + result[output_name] = ','.join(listValues) else: - # these special mapping rules are kept for backwards compatibility - if 'key' in v: - result['{}_key'.format(k)] = v['key'] - if 'name' in v: - result['{}_name'.format(k)] = v['name'] - - if k in v: - result[k] = v[k] - - if 'watchCount' in v: - result[k] = v['watchCount'] - - elif isinstance(v, list): - listValues = [] - for listItem in v: - if isinstance(listItem, dict): - if 'member' in mapping: - listValues.append(listItem[mapping['member']]) - else: - listValues.append(listItem) - if len(listValues) > 0: - result[k] = ','.join(listValues) + result[output_name] = v - else: - result[k] = v + return result def parse_issues(data, field_mapping): @@ -94,6 +97,46 @@ def parse_count(data): return results +class FieldMapping: + + def __init__(cls, query_field_mapping): + cls.mapping = [] + for k, v in query_field_mapping.iteritems(): + field_name = k + member_name = None + + # check for member name contained in field name + member_parser = re.search('(\w+)\.(\w+)', k) + if (member_parser): + field_name = member_parser.group(1) + member_name = member_parser.group(2) + + cls.mapping.append({ + 'field_name': field_name, + 'member_name': member_name, + 'output_field_name': v + }) + + def get_output_field_name(cls,field_name): + for item in cls.mapping: + if item['field_name'] == field_name and not item['member_name']: + return item['output_field_name'] + return field_name + + def get_dict_members(cls,field_name): + member_names = [] + for item in cls.mapping: + if item['field_name'] == field_name and item['member_name']: + member_names.append(item['member_name']) + return member_names + + def get_dict_output_field_name(cls,field_name, member_name): + for item in cls.mapping: + if item['field_name'] == field_name and item['member_name'] == member_name: + return item['output_field_name'] + return None + + class JiraJQL(BaseQueryRunner): noop_query = '{"queryType": "count"}' @@ -135,7 +178,7 @@ def run_query(self, query, user): try: query = json.loads(query) query_type = query.pop('queryType', 'select') - field_mapping = query.pop('fieldMapping', {}) + field_mapping = FieldMapping(query.pop('fieldMapping', {})) if query_type == 'count': query['maxResults'] = 1 diff --git a/tests/query_runner/test_jql.py b/tests/query_runner/test_jql.py new file mode 100644 index 0000000000..18428d1ab2 --- /dev/null +++ b/tests/query_runner/test_jql.py @@ -0,0 +1,104 @@ +from unittest import TestCase +from redash.query_runner.jql import FieldMapping, parse_issue + + +class TestFieldMapping(TestCase): + + def test_empty(self): + field_mapping = FieldMapping({}) + + self.assertEqual(field_mapping.get_output_field_name('field1'), 'field1') + self.assertEqual(field_mapping.get_dict_output_field_name('field1','member1'), None) + self.assertEqual(field_mapping.get_dict_members('field1'), []) + + def test_with_mappings(self): + field_mapping = FieldMapping({ + 'field1': 'output_name_1', + 'field2.member1': 'output_name_2', + 'field2.member2': 'output_name_3' + }) + + self.assertEqual(field_mapping.get_output_field_name('field1'), 'output_name_1') + self.assertEqual(field_mapping.get_dict_output_field_name('field1','member1'), None) + self.assertEqual(field_mapping.get_dict_members('field1'), []) + + self.assertEqual(field_mapping.get_output_field_name('field2'), 'field2') + self.assertEqual(field_mapping.get_dict_output_field_name('field2','member1'), 'output_name_2') + self.assertEqual(field_mapping.get_dict_output_field_name('field2','member2'), 'output_name_3') + self.assertEqual(field_mapping.get_dict_output_field_name('field2','member3'), None) + self.assertEqual(field_mapping.get_dict_members('field2'), ['member1','member2']) + + +class TestParseIssue(TestCase): + issue = { + 'key': 'KEY-1', + 'fields': { + 'string_field': 'value1', + 'int_field': 123, + 'string_list_field': ['value1','value2'], + 'dict_field': {'member1':'value1','member2': 'value2'}, + 'dict_list_field': [ + {'member1':'value1a','member2': 'value2a'}, + {'member1':'value1b','member2': 'value2b'} + ], + 'dict_legacy': {'key':'legacyKey','name':'legacyName','dict_legacy':'legacyValue'}, + 'watchers': {'watchCount':10} + } + } + + def test_no_mapping(self): + result = parse_issue(self.issue, FieldMapping({})) + + self.assertEqual(result['key'], 'KEY-1') + self.assertEqual(result['string_field'], 'value1') + self.assertEqual(result['int_field'], 123) + self.assertEqual(result['string_list_field'], 'value1,value2') + self.assertEqual('dict_field' in result, False) + self.assertEqual('dict_list_field' in result, False) + self.assertEqual(result['dict_legacy'], 'legacyValue') + self.assertEqual(result['dict_legacy_key'], 'legacyKey') + self.assertEqual(result['dict_legacy_name'], 'legacyName') + self.assertEqual(result['watchers'], 10) + + def test_mapping(self): + result = parse_issue(self.issue, FieldMapping({ + 'string_field': 'string_output_field', + 'string_list_field': 'string_output_list_field', + 'dict_field.member1': 'dict_field_1', + 'dict_field.member2': 'dict_field_2', + 'dict_list_field.member1': 'dict_list_field_1', + 'dict_legacy.key': 'dict_legacy', + 'watchers.watchCount': 'watchCount', + })) + + self.assertEqual(result['key'], 'KEY-1') + self.assertEqual(result['string_output_field'], 'value1') + self.assertEqual(result['int_field'], 123) + self.assertEqual(result['string_output_list_field'], 'value1,value2') + self.assertEqual(result['dict_field_1'], 'value1') + self.assertEqual(result['dict_field_2'], 'value2') + self.assertEqual(result['dict_list_field_1'], 'value1a,value1b') + self.assertEqual(result['dict_legacy'], 'legacyKey') + self.assertEqual('dict_legacy_key' in result, False) + self.assertEqual('dict_legacy_name' in result, False) + self.assertEqual('watchers' in result, False) + self.assertEqual(result['watchCount'], 10) + + + def test_mapping_nonexisting_field(self): + result = parse_issue(self.issue, FieldMapping({ + 'non_existing_field': 'output_name1', + 'dict_field.non_existing_member': 'output_name2', + 'dict_list_field.non_existing_member': 'output_name3' + })) + + self.assertEqual(result['key'], 'KEY-1') + self.assertEqual(result['string_field'], 'value1') + self.assertEqual(result['int_field'], 123) + self.assertEqual(result['string_list_field'], 'value1,value2') + self.assertEqual('dict_field' in result, False) + self.assertEqual('dict_list_field' in result, False) + self.assertEqual(result['dict_legacy'], 'legacyValue') + self.assertEqual(result['dict_legacy_key'], 'legacyKey') + self.assertEqual(result['dict_legacy_name'], 'legacyName') + self.assertEqual(result['watchers'], 10) From 894a85ae6368e6bc7992511c0b7d3c2b8f9a6ab5 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Tue, 10 Jan 2017 08:25:32 -0800 Subject: [PATCH 008/149] Remove events on widget delete --- client/app/components/parameters.js | 6 +++--- client/app/pages/dashboards/dashboard.html | 2 +- client/app/pages/dashboards/dashboard.js | 12 +++++------- client/app/pages/dashboards/widget.js | 5 ++++- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/client/app/components/parameters.js b/client/app/components/parameters.js index 8e5439e788..021eda44a1 100644 --- a/client/app/components/parameters.js +++ b/client/app/components/parameters.js @@ -23,15 +23,15 @@ function ParametersDirective($location, $uibModal) { parameters: '=', syncValues: '=?', editable: '=?', - change: '&onChange', + changed: '&onChange', }, template, link(scope) { // is this the correct location for this logic? if (scope.syncValues !== false) { scope.$watch('parameters', () => { - if (scope.change) { - scope.change({}); + if (scope.changed) { + scope.changed({}); } scope.parameters.forEach((param) => { if (param.value !== null || param.value !== '') { diff --git a/client/app/pages/dashboards/dashboard.html b/client/app/pages/dashboards/dashboard.html index bcce3fd868..8731e282a7 100644 --- a/client/app/pages/dashboards/dashboard.html +++ b/client/app/pages/dashboards/dashboard.html @@ -63,6 +63,6 @@

- +
diff --git a/client/app/pages/dashboards/dashboard.js b/client/app/pages/dashboards/dashboard.js index e04da7108c..c46c3632a4 100644 --- a/client/app/pages/dashboards/dashboard.js +++ b/client/app/pages/dashboards/dashboard.js @@ -27,7 +27,7 @@ function DashboardCtrl($rootScope, $scope, $routeParams, $location, $timeout, $q } }; - const extractGlobalParameters = () => { + this.extractGlobalParameters = () => { let globalParams = {}; this.dashboard.widgets.forEach(row => row.forEach((widget) => { @@ -44,15 +44,13 @@ function DashboardCtrl($rootScope, $scope, $routeParams, $location, $timeout, $q }; this.onGlobalParametersChange = () => { - _.each(this.globalParameters, (global) => { - _.each(global.locals, (local) => { + this.globalParameters.forEach((global) => { + global.locals.forEach((local) => { local.value = global.value; }); }); }; - $scope.$on('deleteDashboardWidget', extractGlobalParameters); - const renderDashboard = (dashboard, force) => { Title.set(dashboard.name); const promises = []; @@ -69,7 +67,7 @@ function DashboardCtrl($rootScope, $scope, $routeParams, $location, $timeout, $q }) ); - extractGlobalParameters(); + this.extractGlobalParameters(); $q.all(promises).then((queryResults) => { const filters = {}; @@ -168,7 +166,7 @@ function DashboardCtrl($rootScope, $scope, $routeParams, $location, $timeout, $q resolve: { dashboard: () => this.dashboard, }, - }).result.then(() => extractGlobalParameters()); + }).result.then(() => this.extractGlobalParameters()); }; this.toggleFullscreen = () => { diff --git a/client/app/pages/dashboards/widget.js b/client/app/pages/dashboards/widget.js index 81bb809af3..d7a6f9916a 100644 --- a/client/app/pages/dashboards/widget.js +++ b/client/app/pages/dashboards/widget.js @@ -52,7 +52,9 @@ function DashboardWidgetCtrl($location, $uibModal, $window, $scope, Events, curr this.dashboard.layout = response.layout; this.dashboard.version = response.version; - $scope.$emit('deleteDashboardWidget'); + if (this.deleted) { + this.deleted({}); + } }); }; @@ -90,6 +92,7 @@ export default function (ngModule) { widget: '<', public: '<', dashboard: '<', + deleted: '&onDelete', }, }); } From a2be7bf060fed793b5e38fcd0698829c9bf82f95 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Tue, 10 Jan 2017 08:46:03 -0800 Subject: [PATCH 009/149] Adding an explict toggle for global parameters --- client/app/components/parameter-settings.html | 4 ++++ client/app/services/query.js | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/client/app/components/parameter-settings.html b/client/app/components/parameter-settings.html index ebd52b8ada..5315bed8f5 100644 --- a/client/app/components/parameter-settings.html +++ b/client/app/components/parameter-settings.html @@ -18,5 +18,9 @@

{{$ctrl.parameter.name}}

+
+ + +
diff --git a/client/app/services/query.js b/client/app/services/query.js index fdc1d050b2..dabcf61679 100644 --- a/client/app/services/query.js +++ b/client/app/services/query.js @@ -78,6 +78,7 @@ class Parameters { name: param, type: 'text', value: null, + global: false, }); } }); @@ -295,7 +296,7 @@ function QueryResource($resource, $http, $q, $location, currentUser, QueryResult Query.prototype.getLocalParametersDefs = function getLocalParametersDefs() { if (!this.$localParameters) { - this.$localParameters = this.getParametersDefs().filter(p => p.name[0] !== '$'); + this.$localParameters = this.getParametersDefs().filter(p => !p.global); } return this.$localParameters; @@ -303,7 +304,7 @@ function QueryResource($resource, $http, $q, $location, currentUser, QueryResult Query.prototype.getGlobalParametersDefs = function getGlobalParametersDefs() { if (!this.$globalParameters) { - this.$globalParameters = this.getParametersDefs().filter(p => p.name[0] === '$'); + this.$globalParameters = this.getParametersDefs().filter(p => p.global); } return this.$globalParameters; }; From cb2e6fddf44d32fc22d54e5f2a2adffcbb44dce1 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Tue, 10 Jan 2017 08:52:03 -0800 Subject: [PATCH 010/149] Remove unused $scope --- client/app/pages/dashboards/dashboard.js | 2 +- client/app/pages/dashboards/widget.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/app/pages/dashboards/dashboard.js b/client/app/pages/dashboards/dashboard.js index c46c3632a4..f472b0c904 100644 --- a/client/app/pages/dashboards/dashboard.js +++ b/client/app/pages/dashboards/dashboard.js @@ -2,7 +2,7 @@ import * as _ from 'underscore'; import template from './dashboard.html'; import shareDashboardTemplate from './share-dashboard.html'; -function DashboardCtrl($rootScope, $scope, $routeParams, $location, $timeout, $q, $uibModal, +function DashboardCtrl($rootScope, $routeParams, $location, $timeout, $q, $uibModal, Title, AlertDialog, Dashboard, currentUser, clientConfig, Events) { this.isFullscreen = false; this.refreshRate = null; diff --git a/client/app/pages/dashboards/widget.js b/client/app/pages/dashboards/widget.js index d7a6f9916a..d77db21a73 100644 --- a/client/app/pages/dashboards/widget.js +++ b/client/app/pages/dashboards/widget.js @@ -24,7 +24,7 @@ const EditTextBoxComponent = { }, }; -function DashboardWidgetCtrl($location, $uibModal, $window, $scope, Events, currentUser) { +function DashboardWidgetCtrl($location, $uibModal, $window, Events, currentUser) { this.canViewQuery = currentUser.hasPermission('view_query'); this.editTextBox = () => { From 361abac7ad28a567c6aefa1853bdb6ac1090525e Mon Sep 17 00:00:00 2001 From: Vladislav Denisov Date: Thu, 12 Jan 2017 14:09:37 +0300 Subject: [PATCH 011/149] clickhouse: convert UInt64 to integer type --- redash/query_runner/clickhouse.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/redash/query_runner/clickhouse.py b/redash/query_runner/clickhouse.py index 9bc8b3f424..c22f9dd57c 100644 --- a/redash/query_runner/clickhouse.py +++ b/redash/query_runner/clickhouse.py @@ -68,6 +68,7 @@ def _send_query(self, data, stream=False): }) if r.status_code != 200: raise Exception(r.text) + # logging.warning(r.json()) return r.json() @staticmethod @@ -89,7 +90,13 @@ def _clickhouse_query(self, query): result = self._send_query(query) columns = [{'name': r['name'], 'friendly_name': r['name'], 'type': self._define_column_type(r['type'])} for r in result['meta']] - return {'columns': columns, 'rows': result['data']} + # db converts value to string if its type equals UInt64 + columns_uint64 = [r['name'] for r in result['meta'] if r['type'] == 'UInt64'] + rows = result['data'] + for row in rows: + for column in columns_uint64: + row[column] = int(row[column]) + return {'columns': columns, 'rows': rows} def run_query(self, query, user): logger.debug("Clickhouse is about to execute query: %s", query) From c67519e45ddd2b741a48a2a215e93d6c7c804401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karri=20Niemel=C3=A4?= Date: Sun, 15 Jan 2017 19:44:46 +0200 Subject: [PATCH 012/149] Update amazon linux bootstrap.sh Changed this so it works with recent redash version. --- setup/amazon_linux/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/amazon_linux/bootstrap.sh b/setup/amazon_linux/bootstrap.sh index 2505b25ca5..500475b44d 100755 --- a/setup/amazon_linux/bootstrap.sh +++ b/setup/amazon_linux/bootstrap.sh @@ -180,7 +180,7 @@ if [ $pg_user_exists -ne 0 ]; then sudo -u redash psql -c "grant select on events, queries, dashboards, widgets, visualizations, query_results to redash_reader;" redash cd /opt/redash/current - sudo -u redash bin/run ./manage.py ds new -n "Redash Metadata" -t "pg" -o "{\"user\": \"redash_reader\", \"password\": \"$REDASH_READER_PASSWORD\", \"host\": \"localhost\", \"dbname\": \"redash\"}" + sudo -u redash bin/run ./manage.py ds new "Redash Metadata" --type "pg" --options "{\"user\": \"redash_reader\", \"password\": \"$REDASH_READER_PASSWORD\", \"host\": \"localhost\", \"dbname\": \"redash\"}" fi From 9d36d39f479690128edc6b8e9d006e7e8802a75d Mon Sep 17 00:00:00 2001 From: Adam Griffiths Date: Mon, 16 Jan 2017 11:45:06 +1100 Subject: [PATCH 013/149] Fix for #1521 The merge from #1521 fixed the invalid variable error but it fixed in in a way that introduced another error: "dictionary changed size during iteration" This correctly applies the fix. --- redash/query_runner/elasticsearch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/redash/query_runner/elasticsearch.py b/redash/query_runner/elasticsearch.py index 2e70e4672c..d0dff323e6 100644 --- a/redash/query_runner/elasticsearch.py +++ b/redash/query_runner/elasticsearch.py @@ -119,12 +119,13 @@ def _get_mappings(self, url): return mappings, error def _get_query_mappings(self, url): - mappings, error = self._get_mappings(url) + mappings_data, error = self._get_mappings(url) if error: - return mappings, error + return mappings_data, error - for index_name in mappings: - index_mappings = mappings[index_name] + mappings = {} + for index_name in mappings_data: + index_mappings = mappings_data[index_name] for m in index_mappings.get("mappings", {}): for property_name in index_mappings["mappings"][m]["properties"]: property_data = index_mappings["mappings"][m]["properties"][property_name] From 1a06a5c789902b478910648a9b9a96cc0223b973 Mon Sep 17 00:00:00 2001 From: lab79 Date: Mon, 16 Jan 2017 11:36:30 +0300 Subject: [PATCH 014/149] add read_timeout https://github.com/getredash/redash/issues/1527 --- redash/query_runner/vertica.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/redash/query_runner/vertica.py b/redash/query_runner/vertica.py index 068579f490..c6106e127f 100644 --- a/redash/query_runner/vertica.py +++ b/redash/query_runner/vertica.py @@ -109,7 +109,8 @@ def run_query(self, query, user): 'port': self.configuration.get('port', 5433), 'user': self.configuration.get('user', ''), 'password': self.configuration.get('password', ''), - 'database': self.configuration.get('database', '') + 'database': self.configuration.get('database', ''), + 'read_timeout': self.configuration.get('read_timeout', 60000) } connection = vertica_python.connect(**conn_info) cursor = connection.cursor() From 34f7b2f50a7a33ae91b440bc92fcd3985e801503 Mon Sep 17 00:00:00 2001 From: lab79 Date: Mon, 16 Jan 2017 12:42:47 +0300 Subject: [PATCH 015/149] add read_timeout to configuration_schema --- redash/query_runner/vertica.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/redash/query_runner/vertica.py b/redash/query_runner/vertica.py index c6106e127f..55bbcc22fb 100644 --- a/redash/query_runner/vertica.py +++ b/redash/query_runner/vertica.py @@ -53,6 +53,9 @@ def configuration_schema(cls): "port": { "type": "number" }, + "read_timeout": { + "type": "number" + }, }, 'required': ['database'], 'secret': ['password'] @@ -110,7 +113,7 @@ def run_query(self, query, user): 'user': self.configuration.get('user', ''), 'password': self.configuration.get('password', ''), 'database': self.configuration.get('database', ''), - 'read_timeout': self.configuration.get('read_timeout', 60000) + 'read_timeout': self.configuration.get('read_timeout', 600) } connection = vertica_python.connect(**conn_info) cursor = connection.cursor() From 681fd213daba169b35eaa3c6ea19d823b8210ba9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Jan 2017 15:09:15 +0200 Subject: [PATCH 016/149] Fix package name in npm-shrinkwrap --- npm-shrinkwrap.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 18008cb765..bf6fdcb32b 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,5 +1,5 @@ { - "name": "redash-frontend", + "name": "redash-client", "version": "1.0.0", "dependencies": { "3d-view": { From 7e986db979d0719d0739d9a298c88e5d4e314e56 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Jan 2017 15:09:45 +0200 Subject: [PATCH 017/149] Make webpack-dev-server config more modular --- webpack.config.js | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/webpack.config.js b/webpack.config.js index 33c3807f8e..7211557af9 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -6,6 +6,7 @@ var ExtractTextPlugin = require("extract-text-webpack-plugin"); var WebpackBuildNotifierPlugin = require('webpack-build-notifier'); var path = require('path'); +var redashBackend = process.env.REDASH_BACKEND || 'http://localhost:5000'; var config = { entry: { @@ -82,25 +83,41 @@ var config = { historyApiFallback: true, proxy: { '/login': { - target: 'http://localhost:5000/', + target: redashBackend + '/', + secure: false + }, + '/images': { + target: redashBackend + '/', + secure: false + }, + '/js': { + target: redashBackend + '/', + secure: false + }, + '/styles': { + target: redashBackend + '/', secure: false }, '/status.json': { - target: 'http://localhost:5000/', + target: redashBackend + '/', secure: false }, '/api/admin': { - target: 'http://localhost:5000/', + target: redashBackend + '/', secure: false }, '/api': { - target: 'http://localhost:5000', + target: redashBackend, secure: false } } } }; +if (process.env.DEV_SERVER_HOST) { + config.devServer.host = process.env.DEV_SERVER_HOST; +} + if (process.env.NODE_ENV === 'production') { config.output.path = __dirname + '/client/dist'; config.plugins.push(new webpack.optimize.UglifyJsPlugin()); From e586f99cb16a95da6b4e51dcf36230fc54648f59 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Jan 2017 15:10:01 +0200 Subject: [PATCH 018/149] Fix content-base option of webpack-dev-server --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 08ee227b42..420000012f 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "The frontend part of Redash.", "main": "index.js", "scripts": { - "start": "webpack-dev-server --content-base app", + "start": "webpack-dev-server --content-base client/app", "build": "rm -rf ./client/dist/ && NODE_ENV=production node node_modules/.bin/webpack", "watch": "webpack --watch --progress --colors -d" }, From 785b6fc412cc81218b4d15afb83ef367647d1a45 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Jan 2017 22:22:23 +0200 Subject: [PATCH 019/149] Rename codeclimate file --- .codeclimate => .codeclimate.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .codeclimate => .codeclimate.yml (100%) diff --git a/.codeclimate b/.codeclimate.yml similarity index 100% rename from .codeclimate rename to .codeclimate.yml From bca7d08082dff7ab5334d8332b06a1d4c7a30a4b Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Jan 2017 22:24:02 +0200 Subject: [PATCH 020/149] Disable eslint packages check --- .codeclimate.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.codeclimate.yml b/.codeclimate.yml index a2571ecac2..c17ce2e20a 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -6,6 +6,9 @@ engines: channel: "eslint-3" config: config: client/.eslintrc.js + checks: + import/no-unresolved: + enabled: false ratings: paths: - "redash/**/*.py" From 509c17632a15921b54741d8e4c8a1cd2e674f88c Mon Sep 17 00:00:00 2001 From: lab79 Date: Tue, 17 Jan 2017 14:46:09 +0300 Subject: [PATCH 021/149] Update vertica.py --- redash/query_runner/vertica.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/redash/query_runner/vertica.py b/redash/query_runner/vertica.py index 55bbcc22fb..0bfe2b6611 100644 --- a/redash/query_runner/vertica.py +++ b/redash/query_runner/vertica.py @@ -54,7 +54,8 @@ def configuration_schema(cls): "type": "number" }, "read_timeout": { - "type": "number" + "type": "number", + "title": "Read Timeout" }, }, 'required': ['database'], From d93a3259021d330a98aa8a6fa8b5e3f65eafa6c2 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Wed, 18 Jan 2017 11:46:43 -0600 Subject: [PATCH 022/149] API docstrings suitable for generating sphinx docs --- redash/__init__.py | 7 ++-- redash/handlers/dashboards.py | 68 +++++++++++++++++++++++++++++++++++ redash/handlers/widgets.py | 31 +++++++++++++++- 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/redash/__init__.py b/redash/__init__.py index 60af6b265b..9910b6fa07 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -77,7 +77,7 @@ def to_url(self, value): return value -def create_app(): +def create_app(load_admin=True): from redash import handlers from redash.admin import init_admin from redash.models import db @@ -113,10 +113,11 @@ def create_app(): provision_app(app) db.init_app(app) migrate.init_app(app, db) - init_admin(app) + if load_admin: + init_admin(app) mail.init_app(app) setup_authentication(app) - handlers.init_app(app) limiter.init_app(app) + handlers.init_app(app) return app diff --git a/redash/handlers/dashboards.py b/redash/handlers/dashboards.py index e29b81a7e8..8ff790a052 100644 --- a/redash/handlers/dashboards.py +++ b/redash/handlers/dashboards.py @@ -15,6 +15,9 @@ class RecentDashboardsResource(BaseResource): @require_permission('list_dashboards') def get(self): + """ + Lists dashboards modified in the last 7 days. + """ recent = [d.to_dict() for d in models.Dashboard.recent(self.current_org, self.current_user.group_ids, self.current_user.id, for_user=True)] global_recent = [] @@ -27,11 +30,21 @@ def get(self): class DashboardListResource(BaseResource): @require_permission('list_dashboards') def get(self): + """ + Lists all accessible dashboards. + """ results = models.Dashboard.all(self.current_org, self.current_user.group_ids, self.current_user.id) return [q.to_dict() for q in results] @require_permission('create_dashboard') def post(self): + """ + Creates a new dashboard. + + :`. + """ dashboard_properties = request.get_json(force=True) dashboard = models.Dashboard(name=dashboard_properties['name'], org=self.current_org, @@ -46,6 +59,26 @@ def post(self): class DashboardResource(BaseResource): @require_permission('list_dashboards') def get(self, dashboard_slug=None): + """ + Retrieves a dashboard. + + :qparam string slug: Slug of dashboard to retrieve. + + .. _dashboard-response-label: + + :>json number id: Dashboard ID + :>json string name: + :>json string slug: + :>json number user_id: ID of the dashboard creator + :>json string created_at: ISO format timestamp for dashboard creation + :>json string updated_at: ISO format timestamp for last dashboard modification + :>json number version: Revision number of dashboard + :>json boolean dashboard_filters_enabled: Whether filters are enabled or not + :>json boolean is_archived: Whether this dashboard has been removed from the index or not + :>json boolean is_draft: Whether this dashboard is a draft or not. + :>json array layout: Array of arrays containing widget IDs, corresponding to the rows and columns the widgets are displayed in + :>json array widgets: Array of arrays containing :ref:`widget ` data + """ dashboard = get_object_or_404(models.Dashboard.get_by_slug_and_org, dashboard_slug, self.current_org) response = dashboard.to_dict(with_widgets=True, user=self.current_user) @@ -60,6 +93,16 @@ def get(self, dashboard_slug=None): @require_permission('edit_dashboard') def post(self, dashboard_slug): + """ + Modifies a dashboard. + + :qparam string slug: Slug of dashboard to retrieve. + + Responds with the updated :ref:`dashboard `. + + :status 200: success + :status 409: Version conflict -- dashboard modified since last read + """ dashboard_properties = request.get_json(force=True) # TODO: either convert all requests to use slugs or ids dashboard = models.Dashboard.get_by_id_and_org(dashboard_slug, self.current_org) @@ -89,6 +132,13 @@ def post(self, dashboard_slug): @require_permission('edit_dashboard') def delete(self, dashboard_slug): + """ + Archives a dashboard. + + :qparam string slug: Slug of dashboard to retrieve. + + Responds with the archived :ref:`dashboard `. + """ dashboard = models.Dashboard.get_by_slug_and_org(dashboard_slug, self.current_org) dashboard.is_archived = True dashboard.record_changes(changed_by=self.current_user) @@ -100,6 +150,12 @@ def delete(self, dashboard_slug): class PublicDashboardResource(BaseResource): def get(self, token): + """ + Retrieve a public dashboard. + + :param token: An API key for a public dashboard. + :>json array widgets: An array of arrays of :ref:`public widgets `, corresponding to the rows and columns the widgets are displayed in + """ if not isinstance(self.current_user, models.ApiUser): api_key = get_object_or_404(models.ApiKey.get_by_api_key, token) dashboard = api_key.object @@ -111,6 +167,13 @@ def get(self, token): class DashboardShareResource(BaseResource): def post(self, dashboard_id): + """ + Allow anonymous access to a dashboard. + + :param dashboard_id: The numeric ID of the dashboard to share. + :>json string public_url: The URL for anonymous access to the dashboard. + :>json api_key: The API key to use when accessing it. + """ dashboard = models.Dashboard.get_by_id_and_org(dashboard_id, self.current_org) require_admin_or_owner(dashboard.user_id) api_key = models.ApiKey.create_for_object(dashboard, self.current_user) @@ -128,6 +191,11 @@ def post(self, dashboard_id): return {'public_url': public_url, 'api_key': api_key.api_key} def delete(self, dashboard_id): + """ + Disable anonymous access to a dashboard. + + :param dashboard_id: The numeric ID of the dashboard to unshare. + """ dashboard = models.Dashboard.get_by_id_and_org(dashboard_id, self.current_org) require_admin_or_owner(dashboard.user_id) api_key = models.ApiKey.get_by_object(dashboard) diff --git a/redash/handlers/widgets.py b/redash/handlers/widgets.py index 545c0039da..22acd9cfe0 100644 --- a/redash/handlers/widgets.py +++ b/redash/handlers/widgets.py @@ -11,6 +11,20 @@ class WidgetListResource(BaseResource): @require_permission('edit_dashboard') def post(self): + """ + Add a widget to a dashboard. + + :json object widget: The created widget + :>json array layout: The new layout of the dashboard this widget was added to + :>json boolean new_row: Whether this widget was added on a new row or not + :>json number version: The revision number of the dashboard + """ widget_properties = request.get_json(force=True) dashboard = models.Dashboard.get_by_id_and_org(widget_properties.pop('dashboard_id'), self.current_org) require_object_modify_permission(dashboard, self.current_user) @@ -56,7 +70,14 @@ def post(self): class WidgetResource(BaseResource): @require_permission('edit_dashboard') def post(self, widget_id): - # This method currently handles Text Box widgets only. + """ + Updates a widget in a dashboard. + This method currently handles Text Box widgets only. + + :param number widget_id: The ID of the widget to modify + + :json array layout: New layout of dashboard this widget was removed from + :>json number version: Revision number of dashboard + """ widget = models.Widget.get_by_id_and_org(widget_id, self.current_org) require_object_modify_permission(widget.dashboard, self.current_user) widget.delete() From af70f34f057a28d27e203125951d3c1cbb314743 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Jan 2017 16:04:03 +0200 Subject: [PATCH 023/149] Remove Vagrant related files --- Procfile.dev | 2 -- Vagrantfile | 15 --------------- bin/vagrant_ctl.sh | 21 --------------------- setup/vagrant/provision.sh | 20 -------------------- 4 files changed, 58 deletions(-) delete mode 100644 Procfile.dev delete mode 100644 Vagrantfile delete mode 100755 bin/vagrant_ctl.sh delete mode 100755 setup/vagrant/provision.sh diff --git a/Procfile.dev b/Procfile.dev deleted file mode 100644 index 289b30418e..0000000000 --- a/Procfile.dev +++ /dev/null @@ -1,2 +0,0 @@ -web: ./manage.py runserver -p $PORT --host 0.0.0.0 -worker: ./bin/run celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 875cca8fa6..0000000000 --- a/Vagrantfile +++ /dev/null @@ -1,15 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "redash/dev" - config.vm.synced_folder "./", "/opt/redash/current" - config.vm.network "forwarded_port", guest: 5000, host: 9001 - config.vm.provision "shell" do |s| - s.inline = "/opt/redash/current/setup/vagrant/provision.sh" - s.privileged = false - end -end diff --git a/bin/vagrant_ctl.sh b/bin/vagrant_ctl.sh deleted file mode 100755 index da08abb236..0000000000 --- a/bin/vagrant_ctl.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -e - -help() { - echo "Usage: " - echo "`basename "$0"` {start, test}" -} - -case "$1" in - start) - vagrant up - vagrant ssh -c "cd /opt/redash/current; bin/run honcho start -f Procfile.dev;" - ;; - test) - vagrant up - vagrant ssh -c "cd /opt/redash/current; make test" - ;; - *) - help - ;; -esac \ No newline at end of file diff --git a/setup/vagrant/provision.sh b/setup/vagrant/provision.sh deleted file mode 100755 index aad6210c51..0000000000 --- a/setup/vagrant/provision.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - - -cd /opt/redash/current -cp /opt/redash/.env /opt/redash/current -bower install - -#install requirements -sudo pip install -r /opt/redash/current/requirements_dev.txt -sudo pip install -r /opt/redash/current/requirements.txt -sudo pip install pymongo==3.2.1 - -#update database -bin/run ./manage.py database drop_tables -bin/run ./manage.py database create_tables -bin/run ./manage.py users create --admin --password admin "Admin" "admin" - -#Purge Redis cache -redis-cli -n 1 FLUSHALL - From ddf6fc50a580fd71b758450c0081890712ecb1dc Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 16 Jan 2017 16:51:24 +0200 Subject: [PATCH 024/149] New Dockerfile and docker-compose for dev --- Dockerfile | 58 +++++---------------- bin/docker-entrypoint | 61 +++++++++++++++++++++++ docker-compose.dev.yml | 50 +++++++++++++++++++ setup/docker/supervisord/supervisord.conf | 56 --------------------- 4 files changed, 123 insertions(+), 102 deletions(-) create mode 100755 bin/docker-entrypoint create mode 100644 docker-compose.dev.yml delete mode 100644 setup/docker/supervisord/supervisord.conf diff --git a/Dockerfile b/Dockerfile index 4824c50398..617a3689b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,51 +1,17 @@ -FROM ubuntu:trusty +FROM redash/base:latest -# Ubuntu packages -RUN apt-get update && \ - apt-get install -y python-pip python-dev curl build-essential pwgen libffi-dev sudo git-core wget \ - # Postgres client - libpq-dev \ - # Additional packages required for data sources: - libssl-dev libmysqlclient-dev freetds-dev libsasl2-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +# Webpack dev server +EXPOSE 8080 -# Users creation -RUN useradd --system --comment " " --create-home redash +# We first copy only the requirements file, to avoid rebuilding on every file +# change. +COPY requirements.txt requirements_dev.txt requirements_all_ds.txt ./ +RUN pip install -r requirements.txt -r requirements_dev.txt -r requirements_all_ds.txt -# Pip requirements for all data source types -RUN pip install -U setuptools==23.1.0 && \ - pip install supervisor==3.1.2 +COPY package.json ./ +RUN npm install -COPY . /opt/redash/current -RUN chown -R redash /opt/redash/current +COPY . ./ +RUN npm run build -# Setting working directory -WORKDIR /opt/redash/current - -# Install project specific dependencies -RUN pip install -r requirements_all_ds.txt && \ - pip install -r requirements.txt - -RUN curl https://deb.nodesource.com/setup_4.x | bash - && \ - apt-get install -y nodejs && \ - sudo -u redash -H make deps && \ - rm -rf node_modules client/node_modules /home/redash/.npm /home/redash/.cache && \ - apt-get purge -y nodejs && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Setup supervisord -RUN mkdir -p /opt/redash/supervisord && \ - mkdir -p /opt/redash/logs && \ - cp /opt/redash/current/setup/docker/supervisord/supervisord.conf /opt/redash/supervisord/supervisord.conf - -# Fix permissions -RUN chown -R redash /opt/redash - -# Expose ports -EXPOSE 5000 -EXPOSE 9001 - -# Startup script -CMD ["supervisord", "-c", "/opt/redash/supervisord/supervisord.conf"] +ENTRYPOINT ["/app/bin/docker-entrypoint"] diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint new file mode 100755 index 0000000000..e8cf90f7ca --- /dev/null +++ b/bin/docker-entrypoint @@ -0,0 +1,61 @@ +#!/bin/bash +set -e + +worker() { + WORKERS_COUNT=${WORKERS_COUNT:-2} + QUEUES=${QUEUES:-queries,scheduled_queries,celery} + + echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..." + exec sudo -E -u redash /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair +} + +scheduler() { + WORKERS_COUNT=${WORKERS_COUNT:-1} + QUEUES=${QUEUES:-celery} + + echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..." + + exec sudo -E -u redash /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair +} + +api() { + exec sudo -E -u redash /usr/local/bin/gunicorn -b 0.0.0.0:5000 -k gevent --name redash -w5 redash_saas:app +} + +dev_server() { + exec sudo -E -u redash /app/manage.py runserver --debugger --reload -h 0.0.0.0 +} + +create_db() { + exec sudo -E -u redash /app/manage.py database create_tables +} + +help() { + echo "Usage: " + echo "`basename "$0"` {worker, scheduler, api, shell}" +} + +case "$1" in + worker) + shift + worker + ;; + api) + shift + api + ;; + scheduler) + shift + scheduler + ;; + dev_server) + shift + dev_server + ;; + shell) + exec sudo -E -u redash ./manage.py shell + ;; + *) + help + ;; +esac diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000000..f80cb13224 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,50 @@ +version: '2' +services: + api: + build: . + command: dev_server + volumes_from: + - webpack + depends_on: + - webpack + - postgres + - redis + ports: + - "5002:5000" + environment: + PYTHONUNBUFFERED: 0 + REDASH_LOG_LEVEL: "INFO" + REDASH_REDIS_URL: "redis://redis:6379/0" + REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres" + worker: + build: . + command: scheduler + volumes_from: + - webpack + depends_on: + - postgres + - redis + - api + environment: + REDASH_LOG_LEVEL: "INFO" + REDASH_REDIS_URL: "redis://redis:6379/0" + REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres" + QUEUES: "queries,scheduled_queries,celery" + WORKERS_COUNT: 2 + webpack: + build: . + command: run start + entrypoint: /usr/bin/npm + environment: + DEV_SERVER_HOST: '0.0.0.0' + REDASH_BACKEND: 'http://api:5000' + ports: + - "8080:8080" + volumes: + - ".:/app" + - "/app/client/dist" + - "/app/node_modules" + redis: + image: redis:2.8 + postgres: + image: postgres:9.3 diff --git a/setup/docker/supervisord/supervisord.conf b/setup/docker/supervisord/supervisord.conf deleted file mode 100644 index 9933dfa0fd..0000000000 --- a/setup/docker/supervisord/supervisord.conf +++ /dev/null @@ -1,56 +0,0 @@ -[supervisord] -nodaemon=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -directory=/opt/redash/current - -[inet_http_server] -port = 0.0.0.0:9001 - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[program:redash_server] -command=gunicorn -b 0.0.0.0:5000 --name redash -w 4 --max-requests 1000 redash.wsgi:app -directory=/opt/redash/current -process_name=redash_server -numprocs=1 -priority=999 -autostart=true -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -user=redash - -# There are two queue types here: one for ad-hoc queries, and one for the refresh of scheduled queries -# (note that "scheduled_queries" appears only in the queue list of "redash_celery_scheduled"). -# The default concurrency level for each is 2 (-c2), you can increase based on your machine's resources. -[program:redash_celery] -command=celery worker --app=redash.worker --beat -c2 -Qqueries,celery --maxtasksperchild=10 -Ofair -directory=/opt/redash/current -process_name=redash_celery -numprocs=1 -priority=999 -autostart=true -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -user=redash - -[program:redash_celery_scheduled] -command=celery worker --app=redash.worker -c1 -Qscheduled_queries --maxtasksperchild=10 -Ofair -directory=/opt/redash/current -process_name=redash_celery_scheduled -numprocs=1 -priority=999 -autostart=true -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -user=redash From 06186d9e8d41a45187d60512909ba62c7e29c191 Mon Sep 17 00:00:00 2001 From: Ofer Segev Date: Tue, 17 Jan 2017 20:27:47 +0200 Subject: [PATCH 025/149] Close #242: added ui indication for offline state --- client/app/index.js | 2 +- client/app/services/index.js | 1 + client/app/services/offline-listener.js | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 client/app/services/offline-listener.js diff --git a/client/app/index.js b/client/app/index.js index b6da3b4332..71230fed3e 100644 --- a/client/app/index.js +++ b/client/app/index.js @@ -107,7 +107,7 @@ ngModule.config(($routeProvider, $locationProvider, $compileProvider, }); // Update ui-select's template to use Font-Awesome instead of glyphicon. -ngModule.run(($templateCache) => { +ngModule.run(($templateCache, OfflineListener) => { // eslint-disable-line no-unused-vars const templateName = 'bootstrap/match.tpl.html'; let template = $templateCache.get(templateName); template = template.replace('glyphicon glyphicon-remove', 'fa fa-remove'); diff --git a/client/app/services/index.js b/client/app/services/index.js index 864f8c4e9f..1a00c038df 100644 --- a/client/app/services/index.js +++ b/client/app/services/index.js @@ -12,5 +12,6 @@ export { default as DataSource } from './data-source'; export { default as QuerySnippet } from './query-snippet'; export { default as Notifications } from './notifications'; export { default as KeyboardShortcuts } from './keyboard-shortcuts'; +export { default as OfflineListener } from './offline-listener'; export { default as AlertDialog } from './alert-dialog'; export { default as Auth } from './auth'; diff --git a/client/app/services/offline-listener.js b/client/app/services/offline-listener.js new file mode 100644 index 0000000000..e2d47bd40d --- /dev/null +++ b/client/app/services/offline-listener.js @@ -0,0 +1,23 @@ +function OfflineListener(toastr) { + function addOnlineListener(toast) { + function onlineStateHandler() { + toastr.remove(toast.toastId); + window.removeEventListener('online', onlineStateHandler); + } + window.addEventListener('online', onlineStateHandler); + } + + window.addEventListener('offline', () => { + const toast = toastr.warning('
Please check your Internet connection.
', '', { + allowHtml: true, + autoDismiss: false, + timeOut: false, + tapToDismiss: true, + }); + addOnlineListener(toast); + }); +} + +export default function (ngModule) { + ngModule.service('OfflineListener', OfflineListener); +} From 81fca9329e5b1f9233700205c55d50dcc6a427fa Mon Sep 17 00:00:00 2001 From: Ofer Segev Date: Tue, 17 Jan 2017 21:04:06 +0200 Subject: [PATCH 026/149] App header: query search css fix --- client/app/components/app-header/app-header.css | 7 +++++++ client/app/components/app-header/app-header.html | 6 ++++-- client/app/components/app-header/index.js | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 client/app/components/app-header/app-header.css diff --git a/client/app/components/app-header/app-header.css b/client/app/components/app-header/app-header.css new file mode 100644 index 0000000000..11c7c3fd84 --- /dev/null +++ b/client/app/components/app-header/app-header.css @@ -0,0 +1,7 @@ +.menu-search { + margin-top: 3px; +} + +.menu-search input[type="text"] { + height: 30px; +} \ No newline at end of file diff --git a/client/app/components/app-header/app-header.html b/client/app/components/app-header/app-header.html index 62f9b168e0..aef9bec1af 100644 --- a/client/app/components/app-header/app-header.html +++ b/client/app/components/app-header/app-header.html @@ -37,10 +37,12 @@