Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Return cached data source schema when available #4934

Merged
merged 1 commit into from
Jun 2, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 23 additions & 18 deletions client/app/pages/queries/hooks/useDataSourceSchema.js
Original file line number Diff line number Diff line change
@@ -1,36 +1,39 @@
import { reduce } from "lodash";
import { reduce, has } from "lodash";
import { useCallback, useEffect, useRef, useState, useMemo } from "react";
import { axios } from "@/services/axios";
import DataSource, { SCHEMA_NOT_SUPPORTED } from "@/services/data-source";
import notification from "@/services/notification";

function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
return new Promise(resolve => setTimeout(resolve, ms));
}

function getSchema(dataSource, refresh = undefined) {
if (!dataSource) {
return Promise.resolve([]);
}

const fetchSchemaFromJob = (data) => {
return sleep(1000).then(() => {
Copy link
Member Author

@gabrieldutra gabrieldutra Jun 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also removed the first request sleeping time, from a few tests a good portion of the results would be done already in the first request without the sleeping time

return axios.get(`api/jobs/${data.job.id}`).then((data) => {
if (data.job.status < 3) {
return fetchSchemaFromJob(data);
} else if (data.job.status === 3) {
return data.job.result;
} else if (data.job.status === 4 && data.job.error.code === SCHEMA_NOT_SUPPORTED) {
return [];
} else {
return Promise.reject(new Error(data.job.error));
}
});
const fetchSchemaFromJob = data => {
return axios.get(`api/jobs/${data.job.id}`).then(data => {
if (data.job.status < 3) {
return sleep(1000).then(() => fetchSchemaFromJob(data));
} else if (data.job.status === 3) {
return data.job.result;
} else if (data.job.status === 4 && data.job.error.code === SCHEMA_NOT_SUPPORTED) {
return [];
} else {
return Promise.reject(new Error(data.job.error));
}
});
};

return DataSource.fetchSchema(dataSource, refresh)
.then(fetchSchemaFromJob)
.then(data => {
if (has(data, "job")) {
return fetchSchemaFromJob(data);
}
return has(data, "schema") ? data.schema : Promise.reject();
})
.catch(() => {
notification.error("Schema refresh failed.", "Please try again later.");
return Promise.resolve([]);
Expand All @@ -48,9 +51,11 @@ export default function useDataSourceSchema(dataSource) {

const reloadSchema = useCallback(
(refresh = undefined) => {
const refreshToken = Math.random().toString(36).substr(2);
const refreshToken = Math.random()
.toString(36)
.substr(2);
refreshSchemaTokenRef.current = refreshToken;
getSchema(dataSource, refresh).then((data) => {
getSchema(dataSource, refresh).then(data => {
if (refreshSchemaTokenRef.current === refreshToken) {
setSchema(prepareSchema(data));
}
Expand Down
6 changes: 6 additions & 0 deletions redash/handlers/data_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,12 @@ def get(self, data_source_id):
require_access(data_source, self.current_user, view_only)
refresh = request.args.get("refresh") is not None

if not refresh:
cached_schema = data_source.get_cached_schema()

if cached_schema is not None:
return {"schema": cached_schema}

job = get_schema.delay(data_source.id, refresh)

return serialize_job(job)
Expand Down
12 changes: 7 additions & 5 deletions redash/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,12 +182,16 @@ def delete(self):

return res

def get_cached_schema(self):
cache = redis_connection.get(self._schema_key)
return json_loads(cache) if cache else None

def get_schema(self, refresh=False):
cache = None
out_schema = None
if not refresh:
cache = redis_connection.get(self._schema_key)
out_schema = self.get_cached_schema()

if cache is None:
if out_schema is None:
query_runner = self.query_runner
schema = query_runner.get_schema(get_stats=refresh)

Expand All @@ -200,8 +204,6 @@ def get_schema(self, refresh=False):
out_schema = schema
finally:
redis_connection.set(self._schema_key, json_dumps(out_schema))
else:
out_schema = json_loads(cache)

return out_schema

Expand Down