From 0918a53b8a40363eba7ed6ab96724b3d44678003 Mon Sep 17 00:00:00 2001 From: Duygu Hasan <87015481+duyguHsnHsn@users.noreply.github.com> Date: Wed, 28 Jun 2023 16:36:54 +0300 Subject: [PATCH] vdk-notebook: remove rest_api from tests (#2336) What: modified the jobs used in vdk-notebook tests so they do not use rest api anymore Why: The build on main is failing: https://gitlab.com/vmware-analytics/versatile-data-kit/-/jobs/4555592828. It is failing to make a request to the url https://jsonplaceholder.typicode.com/todos/1 with a 403 error Signed-off-by: Duygu Hasan [hduygu@vmware.com](mailto:hduygu@vmware.com) --- .../config.ini | 0 .../requirements.txt | 0 .../steps.ipynb | 40 ++++------------ .../config.ini | 0 .../requirements.txt | 0 .../steps.ipynb | 26 ++++------ .../config.ini | 0 .../requirements.txt | 0 .../steps.ipynb | 48 ++++++------------- .../config.ini | 0 .../requirements.txt | 0 .../{rest-api-job => ingest-job}/steps.ipynb | 26 ++++------ .../10_delete_table.sql | 0 .../20_create_table.ipynb | 0 .../30_ingest.py} | 5 +- .../{rest-api-job => mixed-job}/config.ini | 0 .../requirements.txt | 0 .../vdk-notebook/tests/test_plugin.py | 10 ++-- 18 files changed, 50 insertions(+), 105 deletions(-) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job-code-error => ingest-job-code-error}/config.ini (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job-code-error => ingest-job-code-error}/requirements.txt (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job-code-error => ingest-job-code-error}/steps.ipynb (77%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{mixed-rest-api => ingest-job-fail-syntax-error}/config.ini (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job-fail-syntax-error => ingest-job-fail-syntax-error}/requirements.txt (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job-fail-syntax-error => ingest-job-fail-syntax-error}/steps.ipynb (84%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job-fail-syntax-error => ingest-job-sql-error}/config.ini (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{mixed-rest-api => ingest-job-sql-error}/requirements.txt (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job-sql-error => ingest-job-sql-error}/steps.ipynb (75%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job-sql-error => ingest-job}/config.ini (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job-sql-error => ingest-job}/requirements.txt (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job => ingest-job}/steps.ipynb (85%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{mixed-rest-api => mixed-job}/10_delete_table.sql (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{mixed-rest-api => mixed-job}/20_create_table.ipynb (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{mixed-rest-api/30_rest_ingest.py => mixed-job/30_ingest.py} (57%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job => mixed-job}/config.ini (100%) rename projects/vdk-plugins/vdk-notebook/tests/jobs/{rest-api-job => mixed-job}/requirements.txt (100%) diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-code-error/config.ini b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-code-error/config.ini similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-code-error/config.ini rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-code-error/config.ini diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-code-error/requirements.txt b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-code-error/requirements.txt similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-code-error/requirements.txt rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-code-error/requirements.txt diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-code-error/steps.ipynb b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-code-error/steps.ipynb similarity index 77% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-code-error/steps.ipynb rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-code-error/steps.ipynb index 525486d9a9..dd6389c7f9 100644 --- a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-code-error/steps.ipynb +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-code-error/steps.ipynb @@ -1,19 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "c635d290-99d5-4354-b95c-a3210e4cf6e1", - "metadata": { - "tags": [ - "vdk" - ] - }, - "outputs": [], - "source": [ - "import requests" - ] - }, { "cell_type": "code", "execution_count": null, @@ -49,11 +35,19 @@ "metadata": { "tags": [ "vdk" - ] + ], + "pycharm": { + "name": "#%%\n" + } }, "outputs": [], "source": [ - "response = requests.get(\"https://jsonplaceholder.typicode.com/todos/1\")" + "payload = {\n", + " \"userId\": 1,\n", + " \"id\": 1,\n", + " \"title\": \"delectus aut autem\",\n", + " \"completed\": False\n", + " }" ] }, { @@ -71,20 +65,6 @@ "hello1 = hello + 1" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "f04c82ad-b8df-484e-9903-e064d61ca5c5", - "metadata": { - "tags": [ - "vdk" - ] - }, - "outputs": [], - "source": [ - "payload = response.json()" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/config.ini b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-fail-syntax-error/config.ini similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/config.ini rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-fail-syntax-error/config.ini diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-fail-syntax-error/requirements.txt b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-fail-syntax-error/requirements.txt similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-fail-syntax-error/requirements.txt rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-fail-syntax-error/requirements.txt diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-fail-syntax-error/steps.ipynb b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-fail-syntax-error/steps.ipynb similarity index 84% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-fail-syntax-error/steps.ipynb rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-fail-syntax-error/steps.ipynb index e00ed7ee08..1ef3412996 100644 --- a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-fail-syntax-error/steps.ipynb +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-fail-syntax-error/steps.ipynb @@ -49,28 +49,22 @@ "metadata": { "tags": [ "vdk" - ] + ], + "pycharm": { + "name": "#%%\n" + } }, "outputs": [], "source": [ - "response = requests.get(\"https://jsonplaceholder.typicode.com/todos/1\")\n", + "payload = {\n", + " \"userId\": 1,\n", + " \"id\": 1,\n", + " \"title\": \"delectus aut autem\",\n", + " \"completed\": False\n", + " }\n", "ss" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "211b1800-6699-49c0-83bc-fe6d9878adcf", - "metadata": { - "tags": [ - "vdk" - ] - }, - "outputs": [], - "source": [ - "payload = response.json()" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-fail-syntax-error/config.ini b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-sql-error/config.ini similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-fail-syntax-error/config.ini rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-sql-error/config.ini diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/requirements.txt b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-sql-error/requirements.txt similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/requirements.txt rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-sql-error/requirements.txt diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-sql-error/steps.ipynb b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-sql-error/steps.ipynb similarity index 75% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-sql-error/steps.ipynb rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-sql-error/steps.ipynb index f7e549dd52..c2a7695fb5 100644 --- a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-sql-error/steps.ipynb +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job-sql-error/steps.ipynb @@ -1,22 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "c635d290-99d5-4354-b95c-a3210e4cf6e1", - "metadata": { - "pycharm": { - "name": "#%%\n" - }, - "tags": [ - "vdk" - ] - }, - "outputs": [], - "source": [ - "import requests" - ] - }, { "cell_type": "code", "execution_count": null, @@ -58,25 +41,19 @@ "metadata": { "tags": [ "vdk" - ] - }, - "outputs": [], - "source": [ - "response = requests.get(\"https://jsonplaceholder.typicode.com/todos/1\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "654db028-2d0e-4f95-a377-e1fac1255f08", - "metadata": { - "tags": [ - "vdk" - ] + ], + "pycharm": { + "name": "#%%\n" + } }, "outputs": [], "source": [ - "payload = response.json()" + "payload = {\n", + " \"userId\": 1,\n", + " \"id\": 1,\n", + " \"title\": \"delectus aut autem\",\n", + " \"completed\": False\n", + " }" ] }, { @@ -86,7 +63,10 @@ "metadata": { "tags": [ "vdk" - ] + ], + "pycharm": { + "name": "#%%\n" + } }, "outputs": [], "source": [ diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-sql-error/config.ini b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job/config.ini similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-sql-error/config.ini rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job/config.ini diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-sql-error/requirements.txt b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job/requirements.txt similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job-sql-error/requirements.txt rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job/requirements.txt diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job/steps.ipynb b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job/steps.ipynb similarity index 85% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job/steps.ipynb rename to projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job/steps.ipynb index 74b62b29b7..146d3e8e49 100644 --- a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job/steps.ipynb +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/ingest-job/steps.ipynb @@ -51,20 +51,6 @@ "job_input.execute_query(\"CREATE TABLE rest_target_table (userId, id, title, completed);\")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "40c5181f-7a29-4842-b3f4-d1258522c83c", - "metadata": { - "tags": [ - "vdk" - ] - }, - "outputs": [], - "source": [ - "response = requests.get(\"https://jsonplaceholder.typicode.com/todos/1\")" - ] - }, { "cell_type": "code", "execution_count": null, @@ -72,11 +58,19 @@ "metadata": { "tags": [ "vdk" - ] + ], + "pycharm": { + "name": "#%%\n" + } }, "outputs": [], "source": [ - "payload = response.json()" + "payload = {\n", + " \"userId\": 1,\n", + " \"id\": 1,\n", + " \"title\": \"delectus aut autem\",\n", + " \"completed\": False\n", + " }" ] }, { diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/10_delete_table.sql b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/10_delete_table.sql similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/10_delete_table.sql rename to projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/10_delete_table.sql diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/20_create_table.ipynb b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/20_create_table.ipynb similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/20_create_table.ipynb rename to projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/20_create_table.ipynb diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/30_rest_ingest.py b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/30_ingest.py similarity index 57% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/30_rest_ingest.py rename to projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/30_ingest.py index 6f26277734..da01902ee6 100644 --- a/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-rest-api/30_rest_ingest.py +++ b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/30_ingest.py @@ -1,12 +1,9 @@ # Copyright 2021-2023 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 -import requests def run(job_input): - response = requests.get("https://jsonplaceholder.typicode.com/todos/1") - response.raise_for_status() - payload = response.json() + payload = {"userId": 1, "id": 1, "title": "delectus aut autem", "completed": False} job_input.send_object_for_ingestion( payload=payload, destination_table="rest_target_table" diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job/config.ini b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/config.ini similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job/config.ini rename to projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/config.ini diff --git a/projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job/requirements.txt b/projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/requirements.txt similarity index 100% rename from projects/vdk-plugins/vdk-notebook/tests/jobs/rest-api-job/requirements.txt rename to projects/vdk-plugins/vdk-notebook/tests/jobs/mixed-job/requirements.txt diff --git a/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py b/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py index c9f8dcdb23..425f2b2ebc 100644 --- a/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py +++ b/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py @@ -25,7 +25,7 @@ def setUp(self) -> None: def test_successful_job(self) -> None: result: Result = self.__runner.invoke( - ["run", jobs_path_from_caller_directory("rest-api-job")] + ["run", jobs_path_from_caller_directory("ingest-job")] ) cli_assert_equal(0, result) actual_rs: Result = self.__runner.invoke( @@ -39,25 +39,25 @@ def test_successful_job(self) -> None: def test_failing_job_with_syntax_error(self) -> None: result: Result = self.__runner.invoke( - ["run", jobs_path_from_caller_directory("rest-api-job-fail-syntax-error")] + ["run", jobs_path_from_caller_directory("ingest-job-fail-syntax-error")] ) cli_assert_equal(1, result) def test_failing_job_with_code_error(self) -> None: result: Result = self.__runner.invoke( - ["run", jobs_path_from_caller_directory("rest-api-job-fail-code-error")] + ["run", jobs_path_from_caller_directory("ingest-job-fail-code-error")] ) cli_assert_equal(2, result) def test_failing_job_with_sql_error(self) -> None: result: Result = self.__runner.invoke( - ["run", jobs_path_from_caller_directory("rest-api-job-sql-error")] + ["run", jobs_path_from_caller_directory("ingest-job-sql-error")] ) cli_assert_equal(1, result) def test_mixed_job_with_py_and_sql(self) -> None: result: Result = self.__runner.invoke( - ["run", jobs_path_from_caller_directory("mixed-rest-api")] + ["run", jobs_path_from_caller_directory("mixed-job")] ) cli_assert_equal(0, result) actual_rs: Result = self.__runner.invoke(