diff --git a/projects/vdk-plugins/vdk-notebook/src/vdk/plugin/notebook/notebook.py b/projects/vdk-plugins/vdk-notebook/src/vdk/plugin/notebook/notebook.py index ad527b21ab..a9640b315c 100644 --- a/projects/vdk-plugins/vdk-notebook/src/vdk/plugin/notebook/notebook.py +++ b/projects/vdk-plugins/vdk-notebook/src/vdk/plugin/notebook/notebook.py @@ -82,7 +82,7 @@ def register_notebook_steps(file_path: Path, context: JobContext): ) notebook_steps.append(step) context.step_builder.add_step(step) - + context.step_builder._StepBuilder__steps.sort(key=lambda step: step.name) log.debug(f"{len(notebook_steps)} " f"cells with vdk tag were detected!") except json.JSONDecodeError as e: errors.log_and_rethrow( diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/10_delete_table.sql b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/10_delete_table.sql new file mode 100644 index 0000000000..a569a6543f --- /dev/null +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/10_delete_table.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS rest_target_table; diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/20_create_table.ipynb b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/20_create_table.ipynb new file mode 100644 index 0000000000..9775a3caf1 --- /dev/null +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/20_create_table.ipynb @@ -0,0 +1,42 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "826a105f-1874-4251-8abd-75fb898ba71c", + "metadata": { + "pycharm": { + "name": "#%%\n" + }, + "tags": [ + "vdk" + ] + }, + "outputs": [], + "source": [ + "job_input.execute_query(\"CREATE TABLE rest_target_table (userId, id, title, completed);\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/30_rest_ingest.py b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/30_rest_ingest.py new file mode 100644 index 0000000000..6f26277734 --- /dev/null +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/30_rest_ingest.py @@ -0,0 +1,13 @@ +# Copyright 2021-2023 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 +import requests + + +def run(job_input): + response = requests.get("https://jsonplaceholder.typicode.com/todos/1") + response.raise_for_status() + payload = response.json() + + job_input.send_object_for_ingestion( + payload=payload, destination_table="rest_target_table" + ) diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/config.ini b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/config.ini new file mode 100644 index 0000000000..adc5f80a56 --- /dev/null +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/config.ini @@ -0,0 +1,16 @@ +; Supported format: https://docs.python.org/3/library/configparser.html#supported-ini-file-structure + +; This is the only file required to deploy a Data Job. +; Read more to understand what each option means: + +; Information about the owner of the Data Job +[owner] + +; Team is a way to group Data Jobs that belonged to the same team. +team = jupyter-test-jobs + +[vdk] +; Key value pairs of any configuration options that can be passed to vdk. +; For possible options in your vdk installation execute command vdk config-help +db_default_type=SQLITE +ingest_method_default=SQLITE diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/requirements.txt b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/requirements.txt new file mode 100644 index 0000000000..15d8cd6f62 --- /dev/null +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/requirements.txt @@ -0,0 +1,5 @@ +# Python jobs can specify extra library dependencies in requirements.txt file. +# See https://pip.readthedocs.io/en/stable/user_guide/#requirements-files +# The file is optional and can be deleted if no extra library dependencies are necessary. + +requests diff --git a/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py b/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py index 9f266044c7..c9f8dcdb23 100644 --- a/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py +++ b/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py @@ -54,3 +54,17 @@ def test_failing_job_with_sql_error(self) -> None: ["run", jobs_path_from_caller_directory("rest-api-job-sql-error")] ) cli_assert_equal(1, result) + + def test_mixed_job_with_py_and_sql(self) -> None: + result: Result = self.__runner.invoke( + ["run", jobs_path_from_caller_directory("mixed-rest-api")] + ) + cli_assert_equal(0, result) + actual_rs: Result = self.__runner.invoke( + ["sqlite-query", "--query", "SELECT * FROM rest_target_table"] + ) + assert actual_rs.stdout == ( + " userId id title completed\n" + "-------- ---- ------------------ -----------\n" + " 1 1 delectus aut autem 0\n" + )