Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: API to allow importing old exports (JSON/YAML) #13444

Merged
merged 4 commits into from
Mar 5, 2021
Merged

fix: API to allow importing old exports (JSON/YAML) #13444

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2c9ffb0
fix: fix API to allow importing old exports (JSON/YAML)
betodealmeida Mar 3, 2021
778b592
Fix test
betodealmeida Mar 3, 2021
0b1b0c9
Fix lint
betodealmeida Mar 3, 2021
9a7c3dc
Add description to API schema
betodealmeida Mar 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions superset/dashboards/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from datetime import datetime
from io import BytesIO
from typing import Any, Dict
from zipfile import ZipFile
from zipfile import is_zipfile, ZipFile

from flask import g, make_response, redirect, request, Response, send_file, url_for
from flask_appbuilder.api import expose, protect, rison, safe
Expand Down Expand Up @@ -787,8 +787,12 @@ def import_(self) -> Response:
upload = request.files.get("formData")
if not upload:
return self.response_400()
with ZipFile(upload) as bundle:
contents = get_contents_from_bundle(bundle)
if is_zipfile(upload):
with ZipFile(upload) as bundle:
contents = get_contents_from_bundle(bundle)
else:
upload.seek(0)
contents = {upload.filename: upload.read()}

passwords = (
json.loads(request.form["passwords"])
Expand Down
5 changes: 4 additions & 1 deletion superset/dashboards/commands/importers/v0.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,10 @@ class ImportDashboardsCommand(BaseCommand):
in Superset.
"""

def __init__(self, contents: Dict[str, str], database_id: Optional[int] = None):
# pylint: disable=unused-argument
def __init__(
self, contents: Dict[str, str], database_id: Optional[int] = None, **kwargs: Any
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is contents a flat Dict? also why the kwargs here, they receive passwords and overwrite but nothing is done with them, I'm probably missing something.

It would be nice also to add a description for the fields on the API spec:

                schema:
                  type: object
                  properties:
                    formData:
                      type: string
                      format: binary
                    passwords:
                      type: string
                    overwrite:
                      type: bool

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

contents is a flat dict, with filename => file_contents.

*kwargs is needed because different version (v0 vs v1) take different arguments, so they just discard what they don't need.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add the description to the API, I thought I had added it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dpgaspar, the spec is up-to-date in api.py: https://github.com/apache/superset/pull/13444/files#diff-f6a877e9c3a368e3c41256960e6bddff588b7d26f5ad750d5378f2fd65722508R761-R767

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mean a description for the passwords, overwrite, formData fields

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, a description! Will do! :)

):
self.contents = contents
self.database_id = database_id

Expand Down
10 changes: 7 additions & 3 deletions superset/datasets/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from distutils.util import strtobool
from io import BytesIO
from typing import Any
from zipfile import ZipFile
from zipfile import is_zipfile, ZipFile

import yaml
from flask import g, request, Response, send_file
Expand Down Expand Up @@ -687,8 +687,12 @@ def import_(self) -> Response:
upload = request.files.get("formData")
if not upload:
return self.response_400()
with ZipFile(upload) as bundle:
contents = get_contents_from_bundle(bundle)
if is_zipfile(upload):
with ZipFile(upload) as bundle:
contents = get_contents_from_bundle(bundle)
else:
upload.seek(0)
contents = {upload.filename: upload.read()}

passwords = (
json.loads(request.form["passwords"])
Expand Down
31 changes: 31 additions & 0 deletions tests/dashboards/api_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
chart_config,
database_config,
dashboard_config,
dashboard_export,
dashboard_metadata_config,
dataset_config,
dataset_metadata_config,
Expand Down Expand Up @@ -1316,6 +1317,36 @@ def test_import_dashboard(self):
db.session.delete(database)
db.session.commit()

def test_import_dashboard_v0_export(self):
num_dashboards = db.session.query(Dashboard).count()

self.login(username="admin")
uri = "api/v1/dashboard/import/"

buf = BytesIO()
buf.write(json.dumps(dashboard_export).encode())
buf.seek(0)
form_data = {
"formData": (buf, "20201119_181105.json"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))

assert rv.status_code == 200
assert response == {"message": "OK"}
assert db.session.query(Dashboard).count() == num_dashboards + 1

dashboard = (
db.session.query(Dashboard).filter_by(dashboard_title="Births 2").one()
)
chart = dashboard.slices[0]
dataset = chart.table

db.session.delete(dashboard)
db.session.delete(chart)
db.session.delete(dataset)
db.session.commit()

def test_import_dashboard_overwrite(self):
"""
Dashboard API: Test import existing dashboard
Expand Down
26 changes: 26 additions & 0 deletions tests/datasets/api_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
database_metadata_config,
dataset_config,
dataset_metadata_config,
dataset_ui_export,
)


Expand Down Expand Up @@ -1275,6 +1276,31 @@ def test_import_dataset(self):
db.session.delete(database)
db.session.commit()

def test_import_dataset_v0_export(self):
num_datasets = db.session.query(SqlaTable).count()

self.login(username="admin")
uri = "api/v1/dataset/import/"

buf = BytesIO()
buf.write(json.dumps(dataset_ui_export).encode())
buf.seek(0)
form_data = {
"formData": (buf, "dataset_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))

assert rv.status_code == 200
assert response == {"message": "OK"}
assert db.session.query(SqlaTable).count() == num_datasets + 1

dataset = (
db.session.query(SqlaTable).filter_by(table_name="birth_names_2").one()
)
db.session.delete(dataset)
db.session.commit()

def test_import_dataset_overwrite(self):
"""
Dataset API: Test import existing dataset
Expand Down