diff --git a/projects/vdk-plugins/vdk-ipython/.plugin-ci.yml b/projects/vdk-plugins/vdk-ipython/.plugin-ci.yml new file mode 100644 index 0000000000..211ab1e968 --- /dev/null +++ b/projects/vdk-plugins/vdk-ipython/.plugin-ci.yml @@ -0,0 +1,35 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + +image: "python:3.7" + +.build-vdk-ipython-ext: + variables: + PLUGIN_NAME: vdk-ipython-ext + extends: .build-plugin + +build-py37-vdk-ipython-ext: + extends: .build-vdk-ipython-ext + image: "python:3.7" + + +build-py38-vdk-ipython-ext: + extends: .build-vdk-ipython-ext + image: "python:3.8" + +build-py39-vdk-ipython-ext: + extends: .build-vdk-ipython-ext + image: "python:3.9" + +build-py310-vdk-ipython-ext: + extends: .build-vdk-ipython-ext + image: "python:3.10" + +build-py311-vdk-ipython-ext: + extends: .build-vdk-ipython-ext + image: "python:3.11" + +release-vdk-ipython-ext: + variables: + PLUGIN_NAME: vdk-ipython-ext + extends: .release-plugin diff --git a/projects/vdk-plugins/vdk-ipython/README.md b/projects/vdk-plugins/vdk-ipython/README.md new file mode 100644 index 0000000000..df15639188 --- /dev/null +++ b/projects/vdk-plugins/vdk-ipython/README.md @@ -0,0 +1,53 @@ +# vdk-ipython + +Ipython extension for VDK + +This extension introduces a magic command for Jupyter. +The command enables the user to load job_input for his current data job and use it freely while working with Jupyter. + +See more about magic commands: https://ipython.readthedocs.io/en/stable/interactive/magics.html + + +## Usage +To use the extension it must be firstly installed with pip as a python package. +Then to load the extension in Jupyter the user should use: +``` +%reload_ext vdk_ipython +``` +And to load the job_input: +``` +%reload_job_input +``` +The %reload_job_input magic can be used with arguments such as passing the job's path with --path +or giving the job a new with --name, etc. + +### Example +The output of this example is "myjob" +``` +%reload_ext vdk_ipython + +%reload_job_input --name=myjob + +job_input.get_name() +``` + +### Build and testing + +``` +pip install -r requirements.txt +pip install -e . +pytest +``` + +In VDK repo [../build-plugin.sh](https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins/build-plugin.sh) script can be used also. + + +#### Note about the CICD: + +.plugin-ci.yaml is needed only for plugins part of [Versatile Data Kit Plugin repo](https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins). + +The CI/CD is separated in two stages, a build stage and a release stage. +The build stage is made up of a few jobs, all which inherit from the same +job configuration and only differ in the Python version they use (3.7, 3.8, 3.9, 3.10 and 3.11). +They run according to rules, which are ordered in a way such that changes to a +plugin's directory trigger the plugin CI, but changes to a different plugin does not. diff --git a/projects/vdk-plugins/vdk-ipython/requirements.txt b/projects/vdk-plugins/vdk-ipython/requirements.txt new file mode 100644 index 0000000000..2ffb45f8f0 --- /dev/null +++ b/projects/vdk-plugins/vdk-ipython/requirements.txt @@ -0,0 +1,8 @@ +# this file is used to provide testing requirements +# for requirements (dependencies) needed during and after installation of the plugin see (and update) setup.py install_requires section + +IPython + +pytest +vdk-core +vdk-test-utils diff --git a/projects/vdk-plugins/vdk-ipython/setup.py b/projects/vdk-plugins/vdk-ipython/setup.py new file mode 100644 index 0000000000..2151a2cf21 --- /dev/null +++ b/projects/vdk-plugins/vdk-ipython/setup.py @@ -0,0 +1,31 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 +import pathlib + +import setuptools + + +__version__ = "0.1.0" + +setuptools.setup( + name="vdk-ipython", + version=__version__, + url="https://github.com/vmware/versatile-data-kit", + description="Ipython extension for VDK", + long_description=pathlib.Path("README.md").read_text(), + long_description_content_type="text/markdown", + install_requires=["vdk-core"], + package_dir={"": "src"}, + packages=setuptools.find_namespace_packages(where="src"), + entry_points={"vdk.plugin.run": ["ipython = vdk_ipython"]}, + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Framework :: IPython", + ], +) diff --git a/projects/vdk-plugins/vdk-ipython/src/vdk_ipython.py b/projects/vdk-plugins/vdk-ipython/src/vdk_ipython.py new file mode 100644 index 0000000000..4c693bece7 --- /dev/null +++ b/projects/vdk-plugins/vdk-ipython/src/vdk_ipython.py @@ -0,0 +1,46 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + +import os +import pathlib + +from IPython import get_ipython +from IPython.core.magic_arguments import argument +from IPython.core.magic_arguments import magic_arguments +from IPython.core.magic_arguments import parse_argstring +from vdk.internal.builtin_plugins.run.standalone_data_job import ( + StandaloneDataJobFactory, +) + + +def load_ipython_extension(ipython): + """ + IPython will look for this function specifically. + See https://ipython.readthedocs.io/en/stable/config/extensions/index.html + """ + ipython.register_magic_function(magic_load_job, magic_name="reload_job_input") + + +@magic_arguments() +@argument("-p", "--path", type=str, default=None) +@argument("-n", "--name", type=str, default=None) +@argument("-a", "--arguments", type=str, default=None) +@argument("-t", "--template", type=str, default=None) +def magic_load_job(line: str): + """ + You can use %initialize_vdk_job line magic within your Notebook to reload the job_input variable + for your current job + See more for line magic: https://ipython.readthedocs.io/en/stable/interactive/magics.html + """ + args = parse_argstring(magic_load_job, line) + load_job(args.path, args.name, args.arguments, args.template) + + +def load_job( + path: str = None, name: str = None, arguments: str = None, template: str = None +): + path = pathlib.Path(path) if path else pathlib.Path(os.getcwd()) + with StandaloneDataJobFactory.create( + data_job_directory=path, name=name, job_args=arguments, template_name=template + ) as job_input: + get_ipython().push(variables={"job_input": job_input}) diff --git a/projects/vdk-plugins/vdk-ipython/tests/test_plugin.py b/projects/vdk-plugins/vdk-ipython/tests/test_plugin.py new file mode 100644 index 0000000000..78dfded210 --- /dev/null +++ b/projects/vdk-plugins/vdk-ipython/tests/test_plugin.py @@ -0,0 +1,39 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +from IPython.core.error import UsageError +from IPython.testing.globalipapp import start_ipython +from vdk.api.job_input import IJobInput + + +@pytest.fixture(scope="session") +def session_ip(): + yield start_ipython() + + +@pytest.fixture(scope="function") +def ip(session_ip): + session_ip.run_line_magic(magic_name="load_ext", line="vdk_ipython") + yield session_ip + session_ip.run_line_magic(magic_name="reset", line="-f") + + +def test_load_job_input_with_no_arguments(ip): + ip.run_line_magic(magic_name="reload_job_input", line="") + assert ip.user_global_ns["job_input"] is not None + assert isinstance(ip.user_global_ns["job_input"], IJobInput) + + +def test_load_job_input_with_valid_argument(ip): + ip.run_line_magic(magic_name="reload_job_input", line="--name=test") + assert ip.user_global_ns["job_input"] is not None + assert isinstance(ip.user_global_ns["job_input"], IJobInput) + assert ip.user_global_ns["job_input"].get_name() == "test" + + +def test_load_job_input_with_invalid_argument(ip): + with pytest.raises( + UsageError, match=r"unrecognized arguments: --invalid_arg=dummy" + ): + ip.run_line_magic(magic_name="reload_job_input", line="--invalid_arg=dummy")