diff --git a/superset/charts/api.py b/superset/charts/api.py index 436dffe615c1c..05eb0ab0c2af9 100644 --- a/superset/charts/api.py +++ b/superset/charts/api.py @@ -793,7 +793,7 @@ def export(self, **kwargs: Any) -> Response: try: for file_name, file_content in ExportChartsCommand(requested_ids).run(): with bundle.open(f"{root}/{file_name}", "w") as fp: - fp.write(file_content.encode()) + fp.write(file_content().encode()) except ChartNotFoundError: return self.response_404() buf.seek(0) diff --git a/superset/cli/importexport.py b/superset/cli/importexport.py index ebf94b444ae1c..54322aaec2b9c 100755 --- a/superset/cli/importexport.py +++ b/superset/cli/importexport.py @@ -83,7 +83,7 @@ def export_dashboards(dashboard_file: Optional[str] = None) -> None: with ZipFile(dashboard_file, "w") as bundle: for file_name, file_content in ExportDashboardsCommand(dashboard_ids).run(): with bundle.open(f"{root}/{file_name}", "w") as fp: - fp.write(file_content.encode()) + fp.write(file_content().encode()) except Exception: # pylint: disable=broad-except logger.exception( "There was an error when exporting the dashboards, please check " @@ -116,7 +116,7 @@ def export_datasources(datasource_file: Optional[str] = None) -> None: with ZipFile(datasource_file, "w") as bundle: for file_name, file_content in ExportDatasetsCommand(dataset_ids).run(): with bundle.open(f"{root}/{file_name}", "w") as fp: - fp.write(file_content.encode()) + fp.write(file_content().encode()) except Exception: # pylint: disable=broad-except logger.exception( "There was an error when exporting the datasets, please check " diff --git a/superset/commands/chart/export.py b/superset/commands/chart/export.py index fcb721c7032db..0a188aee4bb5b 100644 --- a/superset/commands/chart/export.py +++ b/superset/commands/chart/export.py @@ -19,6 +19,7 @@ import json import logging from collections.abc import Iterator +from typing import Callable import yaml @@ -42,10 +43,12 @@ class ExportChartsCommand(ExportModelsCommand): not_found = ChartNotFoundError @staticmethod - def _export(model: Slice, export_related: bool = True) -> Iterator[tuple[str, str]]: + def _file_name(model: Slice) -> str: file_name = get_filename(model.slice_name, model.id) - file_path = f"charts/{file_name}.yaml" + return f"charts/{file_name}.yaml" + @staticmethod + def _file_content(model: Slice) -> str: payload = model.export_to_dict( recursive=False, include_parent_ref=False, @@ -69,7 +72,15 @@ def _export(model: Slice, export_related: bool = True) -> Iterator[tuple[str, st payload["dataset_uuid"] = str(model.table.uuid) file_content = yaml.safe_dump(payload, sort_keys=False) - yield file_path, file_content + return file_content + + @staticmethod + def _export( + model: Slice, export_related: bool = True + ) -> Iterator[tuple[str, Callable[[], str]]]: + yield ExportChartsCommand._file_name( + model + ), lambda: ExportChartsCommand._file_content(model) if model.table and export_related: yield from ExportDatasetsCommand([model.table.id]).run() diff --git a/superset/commands/dashboard/export.py b/superset/commands/dashboard/export.py index fd06c60fa06c0..3447387466afe 100644 --- a/superset/commands/dashboard/export.py +++ b/superset/commands/dashboard/export.py @@ -20,7 +20,7 @@ import logging import random import string -from typing import Any, Optional +from typing import Any, Optional, Callable from collections.abc import Iterator import yaml @@ -106,14 +106,13 @@ class ExportDashboardsCommand(ExportModelsCommand): dao = DashboardDAO not_found = DashboardNotFoundError - # pylint: disable=too-many-locals @staticmethod - def _export( - model: Dashboard, export_related: bool = True - ) -> Iterator[tuple[str, str]]: + def _file_name(model: Dashboard) -> str: file_name = get_filename(model.dashboard_title, model.id) - file_path = f"dashboards/{file_name}.yaml" + return f"dashboards/{file_name}.yaml" + @staticmethod + def _file_content(model: Dashboard) -> str: payload = model.export_to_dict( recursive=False, include_parent_ref=False, @@ -131,20 +130,6 @@ def _export( logger.info("Unable to decode `%s` field: %s", key, value) payload[new_name] = {} - # Extract all native filter datasets and replace native - # filter dataset references with uuid - for native_filter in payload.get("metadata", {}).get( - "native_filter_configuration", [] - ): - for target in native_filter.get("targets", []): - dataset_id = target.pop("datasetId", None) - if dataset_id is not None: - dataset = DatasetDAO.find_by_id(dataset_id) - if dataset: - target["datasetUuid"] = str(dataset.uuid) - if export_related: - yield from ExportDatasetsCommand([dataset_id]).run() - # the mapping between dashboard -> charts is inferred from the position # attribute, so if it's not present we need to add a default config if not payload.get("position"): @@ -163,8 +148,47 @@ def _export( payload["version"] = EXPORT_VERSION file_content = yaml.safe_dump(payload, sort_keys=False) - yield file_path, file_content + return file_content + + @staticmethod + def _export( + model: Dashboard, export_related: bool = True + ) -> Iterator[tuple[str, Callable[[], str]]]: + yield ExportDashboardsCommand._file_name( + model + ), lambda: ExportDashboardsCommand._file_content(model) if export_related: chart_ids = [chart.id for chart in model.slices] yield from ExportChartsCommand(chart_ids).run() + + payload = model.export_to_dict( + recursive=False, + include_parent_ref=False, + include_defaults=True, + export_uuids=True, + ) + # TODO (betodealmeida): move this logic to export_to_dict once this + # becomes the default export endpoint + for key, new_name in JSON_KEYS.items(): + value: Optional[str] = payload.pop(key, None) + if value: + try: + payload[new_name] = json.loads(value) + except (TypeError, json.decoder.JSONDecodeError): + logger.info("Unable to decode `%s` field: %s", key, value) + payload[new_name] = {} + + # Extract all native filter datasets and replace native + # filter dataset references with uuid + for native_filter in payload.get("metadata", {}).get( + "native_filter_configuration", [] + ): + for target in native_filter.get("targets", []): + dataset_id = target.pop("datasetId", None) + if dataset_id is not None: + dataset = DatasetDAO.find_by_id(dataset_id) + if dataset: + target["datasetUuid"] = str(dataset.uuid) + if export_related: + yield from ExportDatasetsCommand([dataset_id]).run() diff --git a/superset/commands/database/export.py b/superset/commands/database/export.py index 82c22ea801948..555a9c327a680 100644 --- a/superset/commands/database/export.py +++ b/superset/commands/database/export.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. # isort:skip_file - +import functools import json import logging -from typing import Any +from typing import Any, Callable from collections.abc import Iterator import yaml @@ -56,12 +56,12 @@ class ExportDatabasesCommand(ExportModelsCommand): not_found = DatabaseNotFoundError @staticmethod - def _export( - model: Database, export_related: bool = True - ) -> Iterator[tuple[str, str]]: + def _file_name(model: Database) -> str: db_file_name = get_filename(model.database_name, model.id, skip_id=True) - file_path = f"databases/{db_file_name}.yaml" + return f"databases/{db_file_name}.yaml" + @staticmethod + def _file_content(model: Database) -> str: payload = model.export_to_dict( recursive=False, include_parent_ref=False, @@ -100,9 +100,18 @@ def _export( payload["version"] = EXPORT_VERSION file_content = yaml.safe_dump(payload, sort_keys=False) - yield file_path, file_content + return file_content + + @staticmethod + def _export( + model: Database, export_related: bool = True + ) -> Iterator[tuple[str, Callable[[], str]]]: + yield ExportDatabasesCommand._file_name( + model + ), lambda: ExportDatabasesCommand._file_content(model) if export_related: + db_file_name = get_filename(model.database_name, model.id, skip_id=True) for dataset in model.tables: ds_file_name = get_filename( dataset.table_name, dataset.id, skip_id=True @@ -118,5 +127,6 @@ def _export( payload["version"] = EXPORT_VERSION payload["database_uuid"] = str(model.uuid) - file_content = yaml.safe_dump(payload, sort_keys=False) - yield file_path, file_content + yield file_path, functools.partial( # type: ignore + yaml.safe_dump, payload, sort_keys=False + ) diff --git a/superset/commands/dataset/export.py b/superset/commands/dataset/export.py index afecdd2fea261..4dd64119079b1 100644 --- a/superset/commands/dataset/export.py +++ b/superset/commands/dataset/export.py @@ -19,6 +19,7 @@ import json import logging from collections.abc import Iterator +from typing import Callable import yaml @@ -41,15 +42,15 @@ class ExportDatasetsCommand(ExportModelsCommand): not_found = DatasetNotFoundError @staticmethod - def _export( - model: SqlaTable, export_related: bool = True - ) -> Iterator[tuple[str, str]]: + def _file_name(model: SqlaTable) -> str: db_file_name = get_filename( model.database.database_name, model.database.id, skip_id=True ) ds_file_name = get_filename(model.table_name, model.id, skip_id=True) - file_path = f"datasets/{db_file_name}/{ds_file_name}.yaml" + return f"datasets/{db_file_name}/{ds_file_name}.yaml" + @staticmethod + def _file_content(model: SqlaTable) -> str: payload = model.export_to_dict( recursive=True, include_parent_ref=False, @@ -78,10 +79,21 @@ def _export( payload["database_uuid"] = str(model.database.uuid) file_content = yaml.safe_dump(payload, sort_keys=False) - yield file_path, file_content + return file_content + + @staticmethod + def _export( + model: SqlaTable, export_related: bool = True + ) -> Iterator[tuple[str, Callable[[], str]]]: + yield ExportDatasetsCommand._file_name( + model + ), lambda: ExportDatasetsCommand._file_content(model) # include database as well if export_related: + db_file_name = get_filename( + model.database.database_name, model.database.id, skip_id=True + ) file_path = f"databases/{db_file_name}.yaml" payload = model.database.export_to_dict( @@ -109,5 +121,4 @@ def _export( payload["version"] = EXPORT_VERSION - file_content = yaml.safe_dump(payload, sort_keys=False) - yield file_path, file_content + yield file_path, lambda: yaml.safe_dump(payload, sort_keys=False) diff --git a/superset/commands/export/assets.py b/superset/commands/export/assets.py index 61d805acafc19..ff76dab03dae5 100644 --- a/superset/commands/export/assets.py +++ b/superset/commands/export/assets.py @@ -1,63 +1,64 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from collections.abc import Iterator -from datetime import datetime, timezone - -import yaml - -from superset.commands.base import BaseCommand -from superset.commands.chart.export import ExportChartsCommand -from superset.commands.dashboard.export import ExportDashboardsCommand -from superset.commands.database.export import ExportDatabasesCommand -from superset.commands.dataset.export import ExportDatasetsCommand -from superset.commands.query.export import ExportSavedQueriesCommand -from superset.utils.dict_import_export import EXPORT_VERSION - -METADATA_FILE_NAME = "metadata.yaml" - - -class ExportAssetsCommand(BaseCommand): - """ - Command that exports all databases, datasets, charts, dashboards and saved queries. - """ - - def run(self) -> Iterator[tuple[str, str]]: - metadata = { - "version": EXPORT_VERSION, - "type": "assets", - "timestamp": datetime.now(tz=timezone.utc).isoformat(), - } - yield METADATA_FILE_NAME, yaml.safe_dump(metadata, sort_keys=False) - seen = {METADATA_FILE_NAME} - - commands = [ - ExportDatabasesCommand, - ExportDatasetsCommand, - ExportChartsCommand, - ExportDashboardsCommand, - ExportSavedQueriesCommand, - ] - for command in commands: - ids = [model.id for model in command.dao.find_all()] - for file_name, file_content in command(ids, export_related=False).run(): - if file_name not in seen: - yield file_name, file_content - seen.add(file_name) - - def validate(self) -> None: - pass +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from collections.abc import Iterator +from datetime import datetime, timezone +from typing import Callable + +import yaml + +from superset.commands.base import BaseCommand +from superset.commands.chart.export import ExportChartsCommand +from superset.commands.dashboard.export import ExportDashboardsCommand +from superset.commands.database.export import ExportDatabasesCommand +from superset.commands.dataset.export import ExportDatasetsCommand +from superset.commands.query.export import ExportSavedQueriesCommand +from superset.utils.dict_import_export import EXPORT_VERSION + +METADATA_FILE_NAME = "metadata.yaml" + + +class ExportAssetsCommand(BaseCommand): + """ + Command that exports all databases, datasets, charts, dashboards and saved queries. + """ + + def run(self) -> Iterator[tuple[str, Callable[[], str]]]: + metadata = { + "version": EXPORT_VERSION, + "type": "assets", + "timestamp": datetime.now(tz=timezone.utc).isoformat(), + } + yield METADATA_FILE_NAME, lambda: yaml.safe_dump(metadata, sort_keys=False) + seen = {METADATA_FILE_NAME} + + commands = [ + ExportDatabasesCommand, + ExportDatasetsCommand, + ExportChartsCommand, + ExportDashboardsCommand, + ExportSavedQueriesCommand, + ] + for command in commands: + ids = [model.id for model in command.dao.find_all()] + for file_name, file_content in command(ids, export_related=False).run(): + if file_name not in seen: + yield file_name, file_content + seen.add(file_name) + + def validate(self) -> None: + pass diff --git a/superset/commands/export/models.py b/superset/commands/export/models.py index 61532d4a031c4..f46368d5321db 100644 --- a/superset/commands/export/models.py +++ b/superset/commands/export/models.py @@ -17,6 +17,7 @@ from collections.abc import Iterator from datetime import datetime, timezone +from typing import Callable import yaml from flask_appbuilder import Model @@ -41,10 +42,20 @@ def __init__(self, model_ids: list[int], export_related: bool = True): self._models: list[Model] = [] @staticmethod - def _export(model: Model, export_related: bool = True) -> Iterator[tuple[str, str]]: + def _file_name(model: Model) -> str: + raise NotImplementedError("Subclasses MUST implement _file_name") + + @staticmethod + def _file_content(model: Model) -> str: + raise NotImplementedError("Subclasses MUST implement _export") + + @staticmethod + def _export( + model: Model, export_related: bool = True + ) -> Iterator[tuple[str, Callable[[], str]]]: raise NotImplementedError("Subclasses MUST implement _export") - def run(self) -> Iterator[tuple[str, str]]: + def run(self) -> Iterator[tuple[str, Callable[[], str]]]: self.validate() metadata = { @@ -52,7 +63,7 @@ def run(self) -> Iterator[tuple[str, str]]: "type": self.dao.model_cls.__name__, # type: ignore "timestamp": datetime.now(tz=timezone.utc).isoformat(), } - yield METADATA_FILE_NAME, yaml.safe_dump(metadata, sort_keys=False) + yield METADATA_FILE_NAME, lambda: yaml.safe_dump(metadata, sort_keys=False) seen = {METADATA_FILE_NAME} for model in self._models: diff --git a/superset/commands/query/export.py b/superset/commands/query/export.py index 43a110c3b9117..5997f0c18b975 100644 --- a/superset/commands/query/export.py +++ b/superset/commands/query/export.py @@ -19,6 +19,7 @@ import json import logging from collections.abc import Iterator +from typing import Callable import yaml from werkzeug.utils import secure_filename @@ -37,10 +38,8 @@ class ExportSavedQueriesCommand(ExportModelsCommand): not_found = SavedQueryNotFoundError @staticmethod - def _export( - model: SavedQuery, export_related: bool = True - ) -> Iterator[tuple[str, str]]: - # build filename based on database, optional schema, and label. + def _file_name(model: SavedQuery) -> str: + # build filename based on database, optional schema, and label # we call secure_filename() multiple times and join the directories afterwards, # as secure_filename() replaces "/" with "_". database_slug = secure_filename(model.database.database_name) @@ -50,7 +49,10 @@ def _export( else: schema_slug = secure_filename(model.schema) file_name = f"queries/{database_slug}/{schema_slug}/{query_slug}.yaml" + return file_name + @staticmethod + def _file_content(model: SavedQuery) -> str: payload = model.export_to_dict( recursive=False, include_parent_ref=False, @@ -61,10 +63,19 @@ def _export( payload["database_uuid"] = str(model.database.uuid) file_content = yaml.safe_dump(payload, sort_keys=False) - yield file_name, file_content + return file_content + + @staticmethod + def _export( + model: SavedQuery, export_related: bool = True + ) -> Iterator[tuple[str, Callable[[], str]]]: + yield ExportSavedQueriesCommand._file_name( + model + ), lambda: ExportSavedQueriesCommand._file_content(model) - # include database as well - if export_related: + if export_related: # TODO: Maybe we can use database export command here? + # include database as well + database_slug = secure_filename(model.database.database_name) file_name = f"databases/{database_slug}.yaml" payload = model.database.export_to_dict( @@ -84,4 +95,4 @@ def _export( payload["version"] = EXPORT_VERSION file_content = yaml.safe_dump(payload, sort_keys=False) - yield file_name, file_content + yield file_name, lambda: file_content diff --git a/superset/dashboards/api.py b/superset/dashboards/api.py index e719a27fc743b..ee8f1f73aeddb 100644 --- a/superset/dashboards/api.py +++ b/superset/dashboards/api.py @@ -756,7 +756,7 @@ def export(self, **kwargs: Any) -> Response: requested_ids ).run(): with bundle.open(f"{root}/{file_name}", "w") as fp: - fp.write(file_content.encode()) + fp.write(file_content().encode()) except DashboardNotFoundError: return self.response_404() buf.seek(0) diff --git a/superset/databases/api.py b/superset/databases/api.py index 2f95bd0442754..ceea8230c1353 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -1096,7 +1096,7 @@ def export(self, **kwargs: Any) -> Response: requested_ids ).run(): with bundle.open(f"{root}/{file_name}", "w") as fp: - fp.write(file_content.encode()) + fp.write(file_content().encode()) except DatabaseNotFoundError: return self.response_404() buf.seek(0) diff --git a/superset/datasets/api.py b/superset/datasets/api.py index 8cc1b2df6301d..f6dedc97ebec3 100644 --- a/superset/datasets/api.py +++ b/superset/datasets/api.py @@ -529,7 +529,7 @@ def export(self, **kwargs: Any) -> Response: requested_ids ).run(): with bundle.open(f"{root}/{file_name}", "w") as fp: - fp.write(file_content.encode()) + fp.write(file_content().encode()) except DatasetNotFoundError: return self.response_404() buf.seek(0) diff --git a/superset/importexport/api.py b/superset/importexport/api.py index f385d04fafe6b..fe170d2593a0c 100644 --- a/superset/importexport/api.py +++ b/superset/importexport/api.py @@ -80,7 +80,7 @@ def export(self) -> Response: with ZipFile(buf, "w") as bundle: for file_name, file_content in ExportAssetsCommand().run(): with bundle.open(f"{root}/{file_name}", "w") as fp: - fp.write(file_content.encode()) + fp.write(file_content().encode()) buf.seek(0) response = send_file( diff --git a/superset/queries/saved_queries/api.py b/superset/queries/saved_queries/api.py index ce283dd6d6797..be9a3f00b35a7 100644 --- a/superset/queries/saved_queries/api.py +++ b/superset/queries/saved_queries/api.py @@ -276,7 +276,7 @@ def export(self, **kwargs: Any) -> Response: requested_ids ).run(): with bundle.open(f"{root}/{file_name}", "w") as fp: - fp.write(file_content.encode()) + fp.write(file_content().encode()) except SavedQueryNotFoundError: return self.response_404() buf.seek(0) diff --git a/tests/integration_tests/charts/commands_tests.py b/tests/integration_tests/charts/commands_tests.py index 6ee3e45b5f045..28f9d42d6dd60 100644 --- a/tests/integration_tests/charts/commands_tests.py +++ b/tests/integration_tests/charts/commands_tests.py @@ -75,7 +75,7 @@ def test_export_chart_command(self, mock_g): assert expected == list(contents.keys()) metadata = yaml.safe_load( - contents[f"charts/Energy_Sankey_{example_chart.id}.yaml"] + contents[f"charts/Energy_Sankey_{example_chart.id}.yaml"]() ) assert metadata == { @@ -133,7 +133,7 @@ def test_export_chart_command_key_order(self, mock_g): contents = dict(command.run()) metadata = yaml.safe_load( - contents[f"charts/Energy_Sankey_{example_chart.id}.yaml"] + contents[f"charts/Energy_Sankey_{example_chart.id}.yaml"]() ) assert list(metadata.keys()) == [ "slice_name", diff --git a/tests/integration_tests/dashboards/commands_tests.py b/tests/integration_tests/dashboards/commands_tests.py index c1b65ee74e615..94473a2d4d485 100644 --- a/tests/integration_tests/dashboards/commands_tests.py +++ b/tests/integration_tests/dashboards/commands_tests.py @@ -78,7 +78,7 @@ def test_export_dashboard_command(self, mock_g1, mock_g2): assert expected_paths == set(contents.keys()) metadata = yaml.safe_load( - contents[f"dashboards/World_Banks_Data_{example_dashboard.id}.yaml"] + contents[f"dashboards/World_Banks_Data_{example_dashboard.id}.yaml"]() ) # remove chart UUIDs from metadata so we can compare @@ -269,7 +269,7 @@ def test_export_dashboard_command_key_order(self, mock_g1, mock_g2): contents = dict(command.run()) metadata = yaml.safe_load( - contents[f"dashboards/World_Banks_Data_{example_dashboard.id}.yaml"] + contents[f"dashboards/World_Banks_Data_{example_dashboard.id}.yaml"]() ) assert list(metadata.keys()) == [ "dashboard_title", diff --git a/tests/integration_tests/databases/commands_tests.py b/tests/integration_tests/databases/commands_tests.py index b46e1b7ea3a1a..ecdf1b7a88953 100644 --- a/tests/integration_tests/databases/commands_tests.py +++ b/tests/integration_tests/databases/commands_tests.py @@ -158,7 +158,7 @@ def test_export_database_command(self, mock_g): big_int_type = "BIGINT(20)" else: big_int_type = "BIGINT" - metadata = yaml.safe_load(contents["databases/examples.yaml"]) + metadata = yaml.safe_load(contents["databases/examples.yaml"]()) assert metadata == ( { "allow_csv_upload": True, @@ -176,7 +176,7 @@ def test_export_database_command(self, mock_g): } ) - metadata = yaml.safe_load(contents["datasets/examples/birth_names.yaml"]) + metadata = yaml.safe_load(contents["datasets/examples/birth_names.yaml"]()) metadata.pop("uuid") metadata["columns"].sort(key=lambda x: x["column_name"]) @@ -359,7 +359,7 @@ def test_export_database_command_key_order(self, mock_g): command = ExportDatabasesCommand([example_db.id]) contents = dict(command.run()) - metadata = yaml.safe_load(contents["databases/examples.yaml"]) + metadata = yaml.safe_load(contents["databases/examples.yaml"]()) assert list(metadata.keys()) == [ "database_name", "sqlalchemy_uri", diff --git a/tests/integration_tests/datasets/commands_tests.py b/tests/integration_tests/datasets/commands_tests.py index 7b6066a22af67..aa2156bdfddac 100644 --- a/tests/integration_tests/datasets/commands_tests.py +++ b/tests/integration_tests/datasets/commands_tests.py @@ -82,7 +82,7 @@ def test_export_dataset_command(self, mock_g): "databases/examples.yaml", ] - metadata = yaml.safe_load(contents["datasets/examples/energy_usage.yaml"]) + metadata = yaml.safe_load(contents["datasets/examples/energy_usage.yaml"]()) # sort columns for deterministic comparison metadata["columns"] = sorted(metadata["columns"], key=itemgetter("column_name")) @@ -216,7 +216,7 @@ def test_export_dataset_command_key_order(self, mock_g): command = ExportDatasetsCommand([example_dataset.id]) contents = dict(command.run()) - metadata = yaml.safe_load(contents["datasets/examples/energy_usage.yaml"]) + metadata = yaml.safe_load(contents["datasets/examples/energy_usage.yaml"]()) assert list(metadata.keys()) == [ "table_name", "main_dttm_col", diff --git a/tests/integration_tests/importexport/commands_tests.py b/tests/integration_tests/importexport/commands_tests.py index 9e8f79026057f..80892bdabcec4 100644 --- a/tests/integration_tests/importexport/commands_tests.py +++ b/tests/integration_tests/importexport/commands_tests.py @@ -38,7 +38,7 @@ def test_export_models_command(self, mock_g): command = ExportDatabasesCommand([example_db.id]) contents = dict(command.run()) - metadata = yaml.safe_load(contents["metadata.yaml"]) + metadata = yaml.safe_load(contents["metadata.yaml"]()) assert metadata == ( { "version": "1.0.0", diff --git a/tests/integration_tests/queries/saved_queries/commands_tests.py b/tests/integration_tests/queries/saved_queries/commands_tests.py index cccc40998583b..ce03c7da00dce 100644 --- a/tests/integration_tests/queries/saved_queries/commands_tests.py +++ b/tests/integration_tests/queries/saved_queries/commands_tests.py @@ -70,7 +70,9 @@ def test_export_query_command(self, mock_g): ] assert expected == list(contents.keys()) - metadata = yaml.safe_load(contents["queries/examples/schema1/The_answer.yaml"]) + metadata = yaml.safe_load( + contents["queries/examples/schema1/The_answer.yaml"]() + ) assert metadata == { "schema": "schema1", "label": "The answer", @@ -127,7 +129,9 @@ def test_export_query_command_key_order(self, mock_g): command = ExportSavedQueriesCommand([self.example_query.id]) contents = dict(command.run()) - metadata = yaml.safe_load(contents["queries/examples/schema1/The_answer.yaml"]) + metadata = yaml.safe_load( + contents["queries/examples/schema1/The_answer.yaml"]() + ) assert list(metadata.keys()) == [ "schema", "label", diff --git a/tests/unit_tests/commands/export_test.py b/tests/unit_tests/commands/export_test.py index 24fa491664042..68f6a821b63bd 100644 --- a/tests/unit_tests/commands/export_test.py +++ b/tests/unit_tests/commands/export_test.py @@ -32,9 +32,9 @@ def test_export_assets_command(mocker: MockFixture) -> None: ExportDatabasesCommand.return_value.run.return_value = [ ( "metadata.yaml", - "version: 1.0.0\ntype: Database\ntimestamp: '2022-01-01T00:00:00+00:00'\n", + lambda: "version: 1.0.0\ntype: Database\ntimestamp: '2022-01-01T00:00:00+00:00'\n", ), - ("databases/example.yaml", ""), + ("databases/example.yaml", lambda: ""), ] ExportDatasetsCommand = mocker.patch( "superset.commands.export.assets.ExportDatasetsCommand" @@ -42,9 +42,9 @@ def test_export_assets_command(mocker: MockFixture) -> None: ExportDatasetsCommand.return_value.run.return_value = [ ( "metadata.yaml", - "version: 1.0.0\ntype: Dataset\ntimestamp: '2022-01-01T00:00:00+00:00'\n", + lambda: "version: 1.0.0\ntype: Dataset\ntimestamp: '2022-01-01T00:00:00+00:00'\n", ), - ("datasets/example/dataset.yaml", ""), + ("datasets/example/dataset.yaml", lambda: ""), ] ExportChartsCommand = mocker.patch( "superset.commands.export.assets.ExportChartsCommand" @@ -52,9 +52,9 @@ def test_export_assets_command(mocker: MockFixture) -> None: ExportChartsCommand.return_value.run.return_value = [ ( "metadata.yaml", - "version: 1.0.0\ntype: Slice\ntimestamp: '2022-01-01T00:00:00+00:00'\n", + lambda: "version: 1.0.0\ntype: Slice\ntimestamp: '2022-01-01T00:00:00+00:00'\n", ), - ("charts/pie.yaml", ""), + ("charts/pie.yaml", lambda: ""), ] ExportDashboardsCommand = mocker.patch( "superset.commands.export.assets.ExportDashboardsCommand" @@ -62,9 +62,9 @@ def test_export_assets_command(mocker: MockFixture) -> None: ExportDashboardsCommand.return_value.run.return_value = [ ( "metadata.yaml", - "version: 1.0.0\ntype: Dashboard\ntimestamp: '2022-01-01T00:00:00+00:00'\n", + lambda: "version: 1.0.0\ntype: Dashboard\ntimestamp: '2022-01-01T00:00:00+00:00'\n", ), - ("dashboards/sales.yaml", ""), + ("dashboards/sales.yaml", lambda: ""), ] ExportSavedQueriesCommand = mocker.patch( "superset.commands.export.assets.ExportSavedQueriesCommand" @@ -72,14 +72,14 @@ def test_export_assets_command(mocker: MockFixture) -> None: ExportSavedQueriesCommand.return_value.run.return_value = [ ( "metadata.yaml", - "version: 1.0.0\ntype: SavedQuery\ntimestamp: '2022-01-01T00:00:00+00:00'\n", + lambda: "version: 1.0.0\ntype: SavedQuery\ntimestamp: '2022-01-01T00:00:00+00:00'\n", ), - ("queries/example/metric.yaml", ""), + ("queries/example/metric.yaml", lambda: ""), ] with freeze_time("2022-01-01T00:00:00Z"): command = ExportAssetsCommand() - output = list(command.run()) + output = [(file[0], file[1]()) for file in list(command.run())] assert output == [ ( diff --git a/tests/unit_tests/datasets/commands/export_test.py b/tests/unit_tests/datasets/commands/export_test.py index 73f383859b794..550d885f8faff 100644 --- a/tests/unit_tests/datasets/commands/export_test.py +++ b/tests/unit_tests/datasets/commands/export_test.py @@ -88,9 +88,22 @@ def test_export(session: Session) -> None: extra=json.dumps({"warning_markdown": "*WARNING*"}), ) - export = list( - ExportDatasetsCommand._export(sqla_table) # pylint: disable=protected-access + export = [ + (file[0], file[1]()) + for file in list( + ExportDatasetsCommand._export( + sqla_table + ) # pylint: disable=protected-access + ) + ] + + payload = sqla_table.export_to_dict( + recursive=True, + include_parent_ref=False, + include_defaults=True, + export_uuids=True, ) + assert export == [ ( "datasets/my_database/my_table.yaml", @@ -114,7 +127,7 @@ def test_export(session: Session) -> None: warning_markdown: '*WARNING*' normalize_columns: false always_filter_main_dttm: false -uuid: null +uuid: {payload['uuid']} metrics: - metric_name: cnt verbose_name: null @@ -129,12 +142,12 @@ def test_export(session: Session) -> None: columns: - column_name: profit verbose_name: null - is_dttm: null - is_active: null + is_dttm: false + is_active: true type: INTEGER advanced_data_type: null - groupby: null - filterable: null + groupby: true + filterable: true expression: revenue-expenses description: null python_date_format: null @@ -143,47 +156,47 @@ def test_export(session: Session) -> None: - column_name: ds verbose_name: null is_dttm: 1 - is_active: null + is_active: true type: TIMESTAMP advanced_data_type: null - groupby: null - filterable: null + groupby: true + filterable: true expression: null description: null python_date_format: null extra: null - column_name: user_id verbose_name: null - is_dttm: null - is_active: null + is_dttm: false + is_active: true type: INTEGER advanced_data_type: null - groupby: null - filterable: null + groupby: true + filterable: true expression: null description: null python_date_format: null extra: null - column_name: expenses verbose_name: null - is_dttm: null - is_active: null + is_dttm: false + is_active: true type: INTEGER advanced_data_type: null - groupby: null - filterable: null + groupby: true + filterable: true expression: null description: null python_date_format: null extra: null - column_name: revenue verbose_name: null - is_dttm: null - is_active: null + is_dttm: false + is_active: true type: INTEGER advanced_data_type: null - groupby: null - filterable: null + groupby: true + filterable: true expression: null description: null python_date_format: null diff --git a/tests/unit_tests/importexport/api_test.py b/tests/unit_tests/importexport/api_test.py index 86fdd723089ba..ddbe8e285132b 100644 --- a/tests/unit_tests/importexport/api_test.py +++ b/tests/unit_tests/importexport/api_test.py @@ -45,9 +45,16 @@ def test_export_assets( ), ("databases/example.yaml", ""), ] + mocked_export_result = [ + ( + "metadata.yaml", + lambda: "version: 1.0.0\ntype: assets\ntimestamp: '2022-01-01T00:00:00+00:00'\n", + ), + ("databases/example.yaml", lambda: ""), + ] ExportAssetsCommand = mocker.patch("superset.importexport.api.ExportAssetsCommand") - ExportAssetsCommand().run.return_value = mocked_contents[:] + ExportAssetsCommand().run.return_value = mocked_export_result[:] response = client.get("/api/v1/assets/export/") assert response.status_code == 200