From e5fcc1d83d78113811710f8c4a664c1f48103a00 Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Tue, 5 Dec 2017 07:43:42 +0200 Subject: [PATCH] Flake8 fixes - see #3302 --- contrib/cache/dataframe.py | 1 - contrib/connectors/pandas/models.py | 40 +++--- contrib/connectors/pandas/views.py | 212 ++++++++++++++-------------- contrib/tests/cache_tests.py | 3 +- contrib/tests/connector_tests.py | 28 ++-- 5 files changed, 137 insertions(+), 147 deletions(-) diff --git a/contrib/cache/dataframe.py b/contrib/cache/dataframe.py index b4ec9fbd4521c..9e091ce778ad5 100644 --- a/contrib/cache/dataframe.py +++ b/contrib/cache/dataframe.py @@ -15,7 +15,6 @@ import pandas as pd from six import u - from werkzeug.contrib.cache import FileSystemCache from werkzeug.posixemulation import rename diff --git a/contrib/connectors/pandas/models.py b/contrib/connectors/pandas/models.py index 9af950c77b888..28488c1a57c02 100644 --- a/contrib/connectors/pandas/models.py +++ b/contrib/connectors/pandas/models.py @@ -6,33 +6,31 @@ from collections import OrderedDict from datetime import datetime import hashlib -import logging from io import BytesIO -from past.builtins import basestring -import requests +import logging try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse +from flask import escape, Markup +from flask_appbuilder import Model +from flask_babel import lazy_gettext as _ import pandas as pd from pandas.api.types import ( - is_string_dtype, is_numeric_dtype, is_datetime64_any_dtype) - + is_datetime64_any_dtype, is_numeric_dtype, is_string_dtype) +from past.builtins import basestring +import requests +import sqlalchemy as sa from sqlalchemy import ( - Column, Integer, String, ForeignKey, Text, and_, or_ + and_, Column, ForeignKey, Integer, or_, String, Text, ) -import sqlalchemy as sa from sqlalchemy.orm import backref, relationship from sqlalchemy_utils import ChoiceType, JSONType -from flask import escape, Markup -from flask_appbuilder import Model -from flask_babel import lazy_gettext as _ - -from superset import dataframe_cache, db, utils, sm +from superset import dataframe_cache, db, sm, utils from superset.connectors.base.models import ( - BaseDatasource, BaseColumn, BaseMetric) + BaseColumn, BaseDatasource, BaseMetric) from superset.models.helpers import QueryResult, set_perm from superset.utils import QueryStatus @@ -240,12 +238,12 @@ def link(self): def get_perm(self): return ( - "pandas.{obj.name}" - "(id:{obj.id})").format(obj=self) + 'pandas.{obj.name}' + '(id:{obj.id})').format(obj=self) @property def dttm_cols(self): - l = [c.column_name for c in self.columns if c.is_dttm] + l = [c.column_name for c in self.columns if c.is_dttm] # noqa: E741 if self.main_dttm_col and self.main_dttm_col not in l: l.append(self.main_dttm_col) return l @@ -268,8 +266,8 @@ def html(self): return df.to_html( index=False, classes=( - "dataframe table table-striped table-bordered " - "table-condensed")) + 'dataframe table table-striped table-bordered ' + 'table-condensed')) @property def data(self): @@ -419,7 +417,7 @@ def get_filter_query(self, filter): # col_obj is None, probably because the col is a metric, # in which case it is numeric anyway pass - query += "({col} {op} {eq})".format(col=col, op=op, eq=eq) + query += '({col} {op} {eq})'.format(col=col, op=op, eq=eq) return query def get_agg_function(self, expr): @@ -488,8 +486,8 @@ def process_dataframe( # Standard tests (copied from SqlaTable) if not granularity and is_timeseries: raise Exception(_( - "Datetime column not provided as part table configuration " - "and is required by this type of chart")) + 'Datetime column not provided as part table configuration ' + 'and is required by this type of chart')) # Filter the DataFrame by the time column, and resample if necessary timestamp_cols = [] diff --git a/contrib/connectors/pandas/views.py b/contrib/connectors/pandas/views.py index 0d514e4530935..197fc05ee7063 100644 --- a/contrib/connectors/pandas/views.py +++ b/contrib/connectors/pandas/views.py @@ -2,27 +2,24 @@ import json import logging -from past.builtins import basestring - -from flask import Markup, flash, redirect +from flask import flash, Markup, redirect from flask_appbuilder import CompactCRUDMixin, expose -from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, BS3TextAreaFieldWidget +from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, BS3TextFieldWidget from flask_appbuilder.models.sqla.interface import SQLAInterface -import sqlalchemy as sa - -from flask_babel import lazy_gettext as _ from flask_babel import gettext as __ +from flask_babel import lazy_gettext as _ +from past.builtins import basestring +import sqlalchemy as sa from wtforms import SelectField, StringField, validators -from superset import appbuilder, db, utils, security, sm -from superset.utils import has_access +from superset import appbuilder, db, security, sm, utils from superset.connectors.base.views import DatasourceModelView +from superset.utils import has_access from superset.views.base import ( - SupersetModelView, ListWidgetWithCheckboxes, DeleteMixin, DatasourceFilter, - get_datasource_exist_error_mgs, + DatasourceFilter, DeleteMixin, get_datasource_exist_error_mgs, + ListWidgetWithCheckboxes, SupersetModelView, ) - -from .models import FORMATS, PandasDatasource, PandasColumn, PandasMetric +from .models import FORMATS, PandasColumn, PandasDatasource, PandasMetric class ChoiceTypeSelectField(SelectField): @@ -84,30 +81,30 @@ class PandasColumnInlineView(CompactCRUDMixin, SupersetModelView): # noqa page_size = 500 description_columns = { 'is_dttm': _( - "Whether to make this column available as a " - "[Time Granularity] option, column has to be DATETIME or " - "DATETIME-like"), + 'Whether to make this column available as a ' + '[Time Granularity] option, column has to be DATETIME or ' + 'DATETIME-like'), 'filterable': _( - "Whether this column is exposed in the `Filters` section " - "of the explore view."), + 'Whether this column is exposed in the `Filters` section ' + 'of the explore view.'), 'type': _( - "The data type that was inferred by Pandas. " - "It may be necessary to input a type manually for " - "expression-defined columns in some cases. In most case " - "users should not need to alter this."), + 'The data type that was inferred by Pandas. ' + 'It may be necessary to input a type manually for ' + 'expression-defined columns in some cases. In most case ' + 'users should not need to alter this.'), } label_columns = { - 'column_name': _("Column"), - 'verbose_name': _("Verbose Name"), - 'description': _("Description"), - 'groupby': _("Groupable"), - 'filterable': _("Filterable"), - 'datasource': _("Datasource"), - 'count_distinct': _("Count Distinct"), - 'sum': _("Sum"), - 'avg': _("Average"), - 'min': _("Min"), - 'max': _("Max"), + 'column_name': _('Column'), + 'verbose_name': _('Verbose Name'), + 'description': _('Description'), + 'groupby': _('Groupable'), + 'filterable': _('Filterable'), + 'datasource': _('Datasource'), + 'count_distinct': _('Count Distinct'), + 'sum': _('Sum'), + 'avg': _('Average'), + 'min': _('Min'), + 'max': _('Max'), 'type': _('Type'), } @@ -130,36 +127,36 @@ class PandasMetricInlineView(CompactCRUDMixin, SupersetModelView, DeleteMixin): 'warning_text'] description_columns = { 'source': utils.markdown( - "a comma-separated list of column(s) used to calculate " - " the metric. Example: `claim_amount`", True), + 'a comma-separated list of column(s) used to calculate ' + ' the metric. Example: `claim_amount`', True), 'expression': utils.markdown( - "a valid Pandas expression as supported by the underlying " - "backend. Example: `count()`", True), - 'is_restricted': _("Whether the access to this metric is restricted " - "to certain roles. Only roles with the permission " + 'a valid Pandas expression as supported by the underlying ' + 'backend. Example: `count()`', True), + 'is_restricted': _('Whether the access to this metric is restricted ' + 'to certain roles. Only roles with the permission ' "'metric access on XXX (the name of this metric)' " - "are allowed to access this metric"), + 'are allowed to access this metric'), 'd3format': utils.markdown( - "d3 formatting string as defined [here]" - "(https://github.com/d3/d3-format/blob/master/README.md#format). " - "For instance, this default formatting applies in the Table " - "visualization and allow for different metric to use different " - "formats", True + 'd3 formatting string as defined [here]' + '(https://github.com/d3/d3-format/blob/master/README.md#format). ' + 'For instance, this default formatting applies in the Table ' + 'visualization and allow for different metric to use different ' + 'formats', True, ), } add_columns = edit_columns page_size = 500 label_columns = { - 'metric_name': _("Metric"), - 'description': _("Description"), - 'verbose_name': _("Verbose Name"), - 'metric_type': _("Type"), - 'source': _("Pandas Source Columns"), - 'expression': _("Pandas Expression"), - 'datasource': _("Datasource"), - 'd3format': _("D3 Format"), + 'metric_name': _('Metric'), + 'description': _('Description'), + 'verbose_name': _('Verbose Name'), + 'metric_type': _('Type'), + 'source': _('Pandas Source Columns'), + 'expression': _('Pandas Expression'), + 'datasource': _('Datasource'), + 'd3format': _('D3 Format'), 'is_restricted': _('Is Restricted'), - 'warning_text': _("Warning Message"), + 'warning_text': _('Warning Message'), } def post_add(self, metric): @@ -194,26 +191,26 @@ class PandasDatasourceModelView(DatasourceModelView, DeleteMixin): # noqa [validators.optional(), validators.length(max=100)], widget=BS3TextFieldWidget(), description=( - "Credentials required to access the raw data, if required. " - "Can be either a username and password in the form " + 'Credentials required to access the raw data, if required. ' + 'Can be either a username and password in the form ' "'[\"username\", \"password\"]' which will be authenticated " - "using HTTP Basic Auth, or a string which will be used as " - "an Authorization header")), + 'using HTTP Basic Auth, or a string which will be used as ' + 'an Authorization header')), 'source_parameters': JSONField( - _("Additional Query Parameters"), + _('Additional Query Parameters'), [validators.optional(), validators.length(max=500)], widget=BS3TextAreaFieldWidget(), description=( - "A JSON-formatted dictionary of additional parameters " - "used to request the remote file")), + 'A JSON-formatted dictionary of additional parameters ' + 'used to request the remote file')), 'format': ChoiceTypeSelectField(_('Format'), choices=FORMATS), 'additional_parameters': JSONField( - _("Additional Read Parameters"), + _('Additional Read Parameters'), [validators.optional(), validators.length(max=500)], widget=BS3TextAreaFieldWidget(), description=( - "A JSON-formatted dictionary of additional parameters " - "passed to the Pandas read function")), + 'A JSON-formatted dictionary of additional parameters ' + 'passed to the Pandas read function')), } edit_columns = [ 'name', 'source_url', 'source_auth', 'source_parameters', @@ -227,57 +224,56 @@ class PandasDatasourceModelView(DatasourceModelView, DeleteMixin): # noqa related_views = [PandasColumnInlineView, PandasMetricInlineView] base_order = ('changed_on', 'desc') search_columns = ( - 'owner', 'name', 'source_url' + 'owner', 'name', 'source_url', ) description_columns = { 'slices': _( - "The list of slices associated with this datasource. By " - "altering this datasource, you may change how these associated " - "slices behave. " - "Also note that slices need to point to a datasource, so " - "this form will fail at saving if removing slices from a " - "datasource. If you want to change the datasource for a slice, " + 'The list of slices associated with this datasource. By ' + 'altering this datasource, you may change how these associated ' + 'slices behave. ' + 'Also note that slices need to point to a datasource, so ' + 'this form will fail at saving if removing slices from a ' + 'datasource. If you want to change the datasource for a slice, ' "overwrite the slice from the 'explore view'"), - 'offset': _("Timezone offset (in hours) for this datasource"), + 'offset': _('Timezone offset (in hours) for this datasource'), 'name': _( - "The name of this datasource"), + 'The name of this datasource'), 'source_url': _( - "The URL used to access the raw data"), + 'The URL used to access the raw data'), 'format': _( - "The format of the raw data, e.g. csv"), + 'The format of the raw data, e.g. csv'), 'description': Markup( "Supports " - "markdown"), + 'markdown'), 'fetch_values_predicate': _( - "Predicate applied when fetching distinct value to " - "populate the filter control component. Supports " - "jinja template syntax. Applies only when " - "`Enable Filter Select` is on." - ), + 'Predicate applied when fetching distinct value to ' + 'populate the filter control component. Supports ' + 'jinja template syntax. Applies only when ' + '`Enable Filter Select` is on.'), 'default_endpoint': _( - "Redirects to this endpoint when clicking on the datasource " - "from the datasource list"), + 'Redirects to this endpoint when clicking on the datasource ' + 'from the datasource list'), 'filter_select_enabled': _( "Whether to populate the filter's dropdown in the explore " "view's filter section with a list of distinct values fetched " - "from the backend on the fly"), + 'from the backend on the fly'), } base_filters = [['id', DatasourceFilter, lambda: []]] label_columns = { - 'slices': _("Associated Slices"), - 'link': _("Datasource"), - 'changed_by_': _("Changed By"), - 'changed_on_': _("Last Changed"), - 'filter_select_enabled': _("Enable Filter Select"), + 'slices': _('Associated Slices'), + 'link': _('Datasource'), + 'changed_by_': _('Changed By'), + 'changed_on_': _('Last Changed'), + 'filter_select_enabled': _('Enable Filter Select'), 'default_endpoint': _('Default Endpoint'), - 'offset': _("Offset"), - 'cache_timeout': _("Cache Timeout"), - 'name': _("Name"), - 'source_url': _("Source URL"), - 'format': _("Format"), + 'offset': _('Offset'), + 'cache_timeout': _('Cache Timeout'), + 'name': _('Name'), + 'source_url': _('Source URL'), + 'format': _('Format'), 'fetch_values_predicate': _('Fetch Values Predicate'), - 'owner': _("Owner"), - 'main_dttm_col': _("Main Datetime Column"), + 'owner': _('Owner'), + 'main_dttm_col': _('Main Datetime Column'), 'description': _('Description'), } @@ -298,9 +294,9 @@ def pre_add(self, datasource): except Exception as e: logging.exception(e) raise Exception(_( - "File [{}] could not be read, " - "please double check the " - "Source URL and Read Method").format(datasource.name)) + 'File [{}] could not be read, ' + 'please double check the ' + 'Source URL and Read Method').format(datasource.name)) def post_add(self, datasource, flash_message=True): datasource.get_metadata() @@ -308,10 +304,10 @@ def post_add(self, datasource, flash_message=True): if flash_message: flash(_( - "The datasource was created. " - "As part of this two phase configuration " - "process, you should now click the edit button by " - "the new datasource to configure it."), "info") + 'The datasource was created. ' + 'As part of this two phase configuration ' + 'process, you should now click the edit button by ' + 'the new datasource to configure it.'), 'info') def post_update(self, datasource): self.post_add(datasource, flash_message=False) @@ -331,10 +327,10 @@ def edit(self, pk): appbuilder.add_view( PandasDatasourceModelView, - "File Datasources", - label=__("File Datasources"), - category="Sources", - category_label=__("Sources"), + 'File Datasources', + label=__('File Datasources'), + category='Sources', + category_label=__('Sources'), icon='fa-file',) -appbuilder.add_separator("Sources") +appbuilder.add_separator('Sources') diff --git a/contrib/tests/cache_tests.py b/contrib/tests/cache_tests.py index 0ee8ad9482a03..5039339e50c2d 100644 --- a/contrib/tests/cache_tests.py +++ b/contrib/tests/cache_tests.py @@ -12,10 +12,9 @@ import pandas as pd from pandas.testing import assert_frame_equal - from tests.base_tests import SupersetTestCase -from contrib.cache.dataframe import DataFrameCache +from ..cache.dataframe import DataFrameCache class DataFrameCacheTestCase(SupersetTestCase): diff --git a/contrib/tests/connector_tests.py b/contrib/tests/connector_tests.py index b64e377e31b09..f9e8925c65f36 100644 --- a/contrib/tests/connector_tests.py +++ b/contrib/tests/connector_tests.py @@ -14,15 +14,14 @@ import pandas as pd from pandas.testing import assert_frame_equal from sqlalchemy import Date +from tests.base_tests import SupersetTestCase +from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn +from superset.models.core import Database from superset.models.helpers import QueryResult from superset.utils import QueryStatus - -from tests.base_tests import SupersetTestCase -from superset.models.core import Database -from superset.connectors.sqla.models import SqlaTable, TableColumn, SqlMetric -from contrib.connectors.pandas.models import ( - PandasDatasource, PandasColumn, PandasMetric) +from ..connectors.pandas.models import ( + PandasColumn, PandasDatasource, PandasMetric) class BaseConnectorTestCase(SupersetTestCase): @@ -43,7 +42,7 @@ def __init__(self, methodName='runTest'): @classmethod def setUpClass(cls): if cls is BaseConnectorTestCase: - raise unittest.SkipTest("Skip tests from BaseConnectorTestCase") + raise unittest.SkipTest('Skip tests from BaseConnectorTestCase') super(BaseConnectorTestCase, cls).setUpClass() data = """ @@ -100,8 +99,7 @@ def md_to_html(self, md_raw): _md.append(line.strip()) md = '\n'.join(_md) return io.StringIO( - markdown.markdown(md, extensions=['markdown.extensions.tables']) - ) + markdown.markdown(md, extensions=['markdown.extensions.tables'])) def md_to_df(self, md_raw): """ @@ -512,7 +510,7 @@ def test_orderby_with_columns(self): 'orderby': [ ['project', False], ['region', True], - ] + ], } result = self.datasource.query(parameters) self.assertIsInstance(result, QueryResult) @@ -943,10 +941,10 @@ class SqlaConnectorTestCase(BaseConnectorTestCase): SqlMetric(metric_name='ratio', metric_type='avg', expression='AVG(value/value2)'), SqlMetric(metric_name='value_percentage', metric_type='custom', - expression="SUM(value)/SUM(value + value2)"), + expression='SUM(value)/SUM(value + value2)'), SqlMetric(metric_name='category_percentage', metric_type='custom', expression="SUM(CASE WHEN category='CategoryA' THEN 1 ELSE 0 END)/" - "CAST(COUNT(*) AS REAL)"), + 'CAST(COUNT(*) AS REAL)'), ] def setUp(self): @@ -977,9 +975,9 @@ class PandasConnectorTestCase(BaseConnectorTestCase): PandasColumn(column_name='received', type='datetime64[D]'), PandasColumn(column_name='value', type='int64'), PandasColumn(column_name='ratio', type='float64', - expression="value / value2"), + expression='value / value2'), PandasColumn(column_name='inverse_ratio', type='float64', - expression="value2 / value"), + expression='value2 / value'), ] metrics = [ @@ -992,7 +990,7 @@ class PandasConnectorTestCase(BaseConnectorTestCase): PandasMetric(metric_name='value_percentage', metric_type='custom', source=None, expression='calc_value_percentage'), PandasMetric(metric_name='category_percentage', metric_type='custom', - source='category', expression="calc_category_percentage"), + source='category', expression='calc_category_percentage'), ] def setUp(self):