From 6bc98bc1ee857355f2141c98b3e41da84343f8df Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Wed, 5 Sep 2018 16:19:31 -0700 Subject: [PATCH 1/3] Fix cache for multiple time comparisons --- superset/assets/yarn.lock | 40 +++++++++------------------------------ superset/config.py | 2 +- superset/viz.py | 9 +++++---- 3 files changed, 15 insertions(+), 36 deletions(-) diff --git a/superset/assets/yarn.lock b/superset/assets/yarn.lock index e1a0d885015ee..519c593b0818a 100644 --- a/superset/assets/yarn.lock +++ b/superset/assets/yarn.lock @@ -1035,14 +1035,14 @@ ansi-html@0.0.7: version "0.0.7" resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e" -ansi-regex@*, ansi-regex@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" - ansi-regex@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" +ansi-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" + ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" @@ -3798,7 +3798,7 @@ debug@^3.1.0: dependencies: ms "2.0.0" -debuglog@*, debuglog@^1.0.1: +debuglog@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" @@ -6031,7 +6031,7 @@ imports-loader@^0.7.1: loader-utils "^1.0.2" source-map "^0.5.6" -imurmurhash@*, imurmurhash@^0.1.4: +imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" @@ -7170,10 +7170,6 @@ lodash._basecreate@^3.0.0: version "3.0.3" resolved "https://registry.yarnpkg.com/lodash._basecreate/-/lodash._basecreate-3.0.3.tgz#1bc661614daa7fc311b7d03bf16806a0213cf821" -lodash._baseindexof@*: - version "3.1.0" - resolved "https://registry.yarnpkg.com/lodash._baseindexof/-/lodash._baseindexof-3.1.0.tgz#fe52b53a1c6761e42618d654e4a25789ed61822c" - lodash._baseuniq@~4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz#0ebb44e456814af7905c6212fa2c9b2d51b841e8" @@ -7181,25 +7177,11 @@ lodash._baseuniq@~4.6.0: lodash._createset "~4.0.0" lodash._root "~3.0.0" -lodash._bindcallback@*: - version "3.0.1" - resolved "https://registry.yarnpkg.com/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz#e531c27644cf8b57a99e17ed95b35c748789392e" - -lodash._cacheindexof@*: - version "3.0.2" - resolved "https://registry.yarnpkg.com/lodash._cacheindexof/-/lodash._cacheindexof-3.0.2.tgz#3dc69ac82498d2ee5e3ce56091bafd2adc7bde92" - -lodash._createcache@*: - version "3.1.2" - resolved "https://registry.yarnpkg.com/lodash._createcache/-/lodash._createcache-3.1.2.tgz#56d6a064017625e79ebca6b8018e17440bdcf093" - dependencies: - lodash._getnative "^3.0.0" - lodash._createset@~4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26" -lodash._getnative@*, lodash._getnative@^3.0.0: +lodash._getnative@^3.0.0: version "3.9.1" resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5" @@ -7311,10 +7293,6 @@ lodash.reject@^4.4.0: version "4.6.0" resolved "https://registry.yarnpkg.com/lodash.reject/-/lodash.reject-4.6.0.tgz#80d6492dc1470864bbf583533b651f42a9f52415" -lodash.restparam@*: - version "3.6.1" - resolved "https://registry.yarnpkg.com/lodash.restparam/-/lodash.restparam-3.6.1.tgz#936a4e309ef330a7645ed4145986c85ae5b20805" - lodash.some@^4.4.0: version "4.6.0" resolved "https://registry.yarnpkg.com/lodash.some/-/lodash.some-4.6.0.tgz#1bb9f314ef6b8baded13b549169b2a945eb68e4d" @@ -10218,7 +10196,7 @@ readable-stream@~2.1.5: string_decoder "~0.10.x" util-deprecate "~1.0.1" -readdir-scoped-modules@*, readdir-scoped-modules@^1.0.0: +readdir-scoped-modules@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.0.2.tgz#9fafa37d286be5d92cbaebdee030dc9b5f406747" dependencies: @@ -12098,7 +12076,7 @@ v8flags@^2.1.1: dependencies: user-home "^1.1.1" -validate-npm-package-license@*, validate-npm-package-license@^3.0.1: +validate-npm-package-license@^3.0.1: version "3.0.4" resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a" dependencies: diff --git a/superset/config.py b/superset/config.py index a5e4f2988c4e1..b1653072552e2 100644 --- a/superset/config.py +++ b/superset/config.py @@ -185,7 +185,7 @@ # IMG_SIZE = (300, 200, True) CACHE_DEFAULT_TIMEOUT = 60 * 60 * 24 -CACHE_CONFIG = {'CACHE_TYPE': 'null'} +CACHE_CONFIG = {'CACHE_TYPE': 'simple'} TABLE_NAMES_CACHE_CONFIG = {'CACHE_TYPE': 'null'} # CORS Options diff --git a/superset/viz.py b/superset/viz.py index 6a18dfab17dca..39d750ffe4f20 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -327,7 +327,7 @@ def get_json(self): self.get_payload(), default=utils.json_int_dttm_ser, ignore_nan=True) - def cache_key(self, query_obj): + def cache_key(self, query_obj, **extra): """ The cache key is made out of the key/values in `query_obj` @@ -336,6 +336,7 @@ def cache_key(self, query_obj): may be time-relative (as in "5 days ago" or "now"). """ cache_dict = copy.copy(query_obj) + cache_dict.update(extra) for k in ['from_dttm', 'to_dttm']: del cache_dict[k] @@ -360,11 +361,11 @@ def get_payload(self, query_obj=None): del payload['df'] return payload - def get_df_payload(self, query_obj=None): + def get_df_payload(self, query_obj=None, **kwargs): """Handles caching around the df payload retrieval""" if not query_obj: query_obj = self.query_obj() - cache_key = self.cache_key(query_obj) if query_obj else None + cache_key = self.cache_key(query_obj, **kwargs) if query_obj else None logging.info('Cache key: {}'.format(cache_key)) is_loaded = False stacktrace = None @@ -1204,7 +1205,7 @@ def run_extra_queries(self): query_object['from_dttm'] -= delta query_object['to_dttm'] -= delta - df2 = self.get_df_payload(query_object).get('df') + df2 = self.get_df_payload(query_object, time_compare=option).get('df') if df2 is not None and DTTM_ALIAS in df2: label = '{} offset'. format(option) df2[DTTM_ALIAS] += delta From b745e3cf1411dd62ebe58d76dc413b38e80f0154 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Wed, 5 Sep 2018 16:33:30 -0700 Subject: [PATCH 2/3] Remove simple cache --- superset/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/config.py b/superset/config.py index b1653072552e2..a5e4f2988c4e1 100644 --- a/superset/config.py +++ b/superset/config.py @@ -185,7 +185,7 @@ # IMG_SIZE = (300, 200, True) CACHE_DEFAULT_TIMEOUT = 60 * 60 * 24 -CACHE_CONFIG = {'CACHE_TYPE': 'simple'} +CACHE_CONFIG = {'CACHE_TYPE': 'null'} TABLE_NAMES_CACHE_CONFIG = {'CACHE_TYPE': 'null'} # CORS Options From 79c0203e363bb5d7f2fd52a7b6315cac4d9f6bb0 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Fri, 7 Sep 2018 09:55:33 -0700 Subject: [PATCH 3/3] Improve docstring --- superset/viz.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/superset/viz.py b/superset/viz.py index 39d750ffe4f20..da4f32c805d84 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -329,11 +329,16 @@ def get_json(self): def cache_key(self, query_obj, **extra): """ - The cache key is made out of the key/values in `query_obj` + The cache key is made out of the key/values in `query_obj`, plus any + other key/values in `extra`. - We remove datetime bounds that are hard values, - and replace them with the use-provided inputs to bounds, which - may be time-relative (as in "5 days ago" or "now"). + We remove datetime bounds that are hard values, and replace them with + the use-provided inputs to bounds, which may be time-relative (as in + "5 days ago" or "now"). + + The `extra` arguments are currently used by time shift queries, since + different time shifts wil differ only in the `from_dttm` and `to_dttm` + values which are stripped. """ cache_dict = copy.copy(query_obj) cache_dict.update(extra)