vdk-kerberos-auth: introduce a new plugin

projects/vdk-plugins/vdk-kerberos-auth/.plugin-ci.yml

@@ -0,0 +1,32 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+image: "python:3.7"
+
+.build-vdk-kerberos-auth:
+  variables:
+    PLUGIN_NAME: vdk-kerberos-auth
+  extends: .build-plugin
+
+build-py37-vdk-kerberos-auth:
+  extends: .build-vdk-kerberos-auth
+  image: "python:3.7"
+
+
+build-py38-vdk-kerberos-auth:
+  extends: .build-vdk-kerberos-auth
+  image: "python:3.8"
+
+
+build-py39-vdk-kerberos-auth:
+  extends: .build-vdk-kerberos-auth
+  image: "python:3.9"
+
+build-py310-vdk-kerberos-auth:
+  extends: .build-vdk-kerberos-auth
+  image: "python:3.10"
+
+release-vdk-kerberos-auth:
+  variables:
+    PLUGIN_NAME: vdk-kerberos-auth
+  extends: .release-plugin

projects/vdk-plugins/vdk-kerberos-auth/README.md

@@ -0,0 +1,8 @@
+The plugin provides GSSAPI Kerberos authentication on data job startup. The plugin also adds Kerberos/GSSAPI support for HTTP requests.
+
+# Usage
+
+Run
+```bash
+pip install vdk-kerberos-auth
+```

projects/vdk-plugins/vdk-kerberos-auth/requirements.txt

@@ -0,0 +1,6 @@
+vdk-core
+minikerberos==0.1.0
+requests-kerberos
+
+vdk-test-utils
+pytest

projects/vdk-plugins/vdk-kerberos-auth/setup.py

@@ -0,0 +1,31 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import pathlib
+
+import setuptools
+
+
+__version__ = "0.1.0"
+
+setuptools.setup(
+    name="vdk-kerberos-auth",
+    version=__version__,
+    url="https://github.com/vmware/versatile-data-kit",
+    description="Versatile Data Kit SDK plugin adds Kerberos/GSSAPI support.",
+    long_description=pathlib.Path("README.md").read_text(),
+    long_description_content_type="text/markdown",
+    install_requires=["vdk-core", "minikerberos", "requests-kerberos"],
+    package_dir={"": "src"},
+    packages=setuptools.find_namespace_packages(where="src"),
+    entry_points={
+        "vdk.plugin.run": ["vdk-kerberos-auth = vdk.plugin.kerberos.kerberos_plugin"]
+    },
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+    ],
+)

projects/vdk-plugins/vdk-kerberos-auth/src/vdk/plugin/kerberos/authenticator_factory.py

@@ -0,0 +1,48 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import logging
+from typing import Optional
+
+from vdk.internal.core import errors
+from vdk.plugin.kerberos.base_authenticator import BaseAuthenticator
+from vdk.plugin.kerberos.kerberos_configuration import KerberosPluginConfiguration
+from vdk.plugin.kerberos.kinit_authenticator import KinitGSSAPIAuthenticator
+from vdk.plugin.kerberos.minikerberos_authenticator import (
+    MinikerberosGSSAPIAuthenticator,
+)
+
+log = logging.getLogger(__name__)
+
+
+class KerberosAuthenticatorFactory:
+    def __init__(self, configuration: KerberosPluginConfiguration):
+        self.__configuration = configuration
+
+    def create_authenticator(
+        self, authentication_type: str
+    ) -> Optional[BaseAuthenticator]:
+        if authentication_type == "minikerberos":
+            return MinikerberosGSSAPIAuthenticator(
+                self.__configuration.keytab_pathname(),
+                self.__configuration.keytab_principal(),
+                self.__configuration.keytab_realm(),
+                self.__configuration.kerberos_host(),
+            )
+        elif authentication_type == "kinit":
+            return KinitGSSAPIAuthenticator(
+                self.__configuration.keytab_pathname(),
+                self.__configuration.keytab_principal(),
+            )  # Can kinit the whole process
+        elif authentication_type is None:
+            log.debug("No Kerberos authentication specified")
+            return None
+        else:
+            errors.log_and_throw(
+                to_be_fixed_by=errors.ResolvableBy.CONFIG_ERROR,
+                log=log,
+                what_happened=f"Provided environment variable {'VDK_KRB_AUTH'} has invalid value.",
+                why_it_happened=f"VDK was run with environment variable {'VDK_KRB_AUTH'}={authentication_type}, "
+                f"however '{authentication_type}' is invalid value for this variable.",
+                consequences=errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE,
+                countermeasures=f"Provide either 'minikerberos' or 'kinit' for environment variable {'VDK_KRB_AUTH'}.",
+            )

projects/vdk-plugins/vdk-kerberos-auth/src/vdk/plugin/kerberos/base_authenticator.py

@@ -0,0 +1,76 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import os
+from abc import ABC
+from abc import abstractmethod
+
+from vdk.internal.core import errors
+
+log = logging.getLogger(__name__)
+
+
+class BaseAuthenticator(ABC):
+    def __init__(
+        self,
+        keytab_pathname: str,
+        kerberos_principal: str,
+        kerberos_realm: str = None,
+        kerberos_kdc_hostname: str = None,
+    ):
+        if not os.path.isfile(keytab_pathname):
+            f = os.path.abspath(keytab_pathname)
+            errors.log_and_throw(
+                to_be_fixed_by=errors.ResolvableBy.CONFIG_ERROR,
+                log=log,
+                what_happened=f"Cannot locate keytab file {keytab_pathname}.",
+                why_it_happened=f"Keytab file at {f} does not exist",
+                consequences="Kerberos authentication is impossible. "
+                "Subsequent operation that require authentication will fail.",
+                countermeasures=f"Ensure a keytab file is located at {f}.",
+            )
+        self._keytab_pathname = os.path.abspath(keytab_pathname)
+        self._kerberos_principal = kerberos_principal
+        self._kerberos_realm = kerberos_realm
+        self._kerberos_kdc_hostname = kerberos_kdc_hostname
+        self._is_authenticated = False
+
+        self.__configure_krb5_config()
+
+    def __str__(self):
+        return str(self.__repr__())
+
+    @staticmethod
+    def __configure_krb5_config():
+        kerberos_module_dir = os.path.dirname(os.path.abspath(__file__))
+        krb5_conf_path = os.path.join(kerberos_module_dir, "krb5.conf")
+        if os.path.exists(krb5_conf_path):
+            os.environ["KRB5_CONFIG"] = krb5_conf_path
+
+    def authenticate(self):
+        if not self.is_authenticated():
+            if self._kinit():
+                self.set_authenticated()
+        else:
+            log.debug(
+                f"Already authenticated, skipping authentication for principal {self._kerberos_principal}."
+            )
+
+    def get_principal(self):
+        return self._kerberos_principal
+
+    def is_authenticated(self):
+        # TODO add support for renewal
+        return self._is_authenticated
+
+    def set_authenticated(self):
+        # TODO add support for renewal
+        self._is_authenticated = True
+
+    @abstractmethod
+    def _kinit(self) -> None:
+        """
+        Obtain and cache a Kerberos ticket-granting ticket (TGT)
+        that will be used for subsequent Kerberos authentication.
+        This method either succeeds or throws an exception.
+        """

projects/vdk-plugins/vdk-kerberos-auth/src/vdk/plugin/kerberos/kerberos_configuration.py

@@ -0,0 +1,85 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import os
+
+from vdk.internal.core.config import Configuration
+from vdk.internal.core.config import ConfigurationBuilder
+
+KRB_AUTH = "KRB_AUTH"
+KEYTAB_FOLDER = "KEYTAB_FOLDER"
+KEYTAB_FILENAME = "KEYTAB_FILENAME"
+KEYTAB_PRINCIPAL = "KEYTAB_PRINCIPAL"
+KEYTAB_REALM = "KEYTAB_REALM"
+KERBEROS_KDC_HOST = "KERBEROS_KDC_HOST"
+
+
+class KerberosPluginConfiguration:
+    def __init__(self, job_name: str, job_directory: str, config: Configuration):
+        self.__job_name = job_name
+        self.__job_directory = job_directory
+        self.__config = config
+
+    def authentication_type(self):
+        return self.__config.get_value(KRB_AUTH)
+
+    def keytab_folder(self):
+        keytab_folder = self.__config.get_value(KEYTAB_FOLDER)
+        if keytab_folder is None:
+            keytab_folder = self.__job_directory
+        return keytab_folder
+
+    def keytab_filename(self):
+        keytab_filename = self.__config.get_value(KEYTAB_FILENAME)
+        if keytab_filename is None:
+            keytab_filename = f"{self.__job_name}.keytab"
+        return keytab_filename
+
+    def keytab_pathname(self):
+        return os.path.join(self.keytab_folder(), self.keytab_filename())
+
+    def keytab_principal(self):
+        keytab_principal = self.__config.get_value(KEYTAB_PRINCIPAL)
+        if keytab_principal is None:
+            keytab_principal = f"pa__view_{self.__job_name}"
+        return keytab_principal
+
+    def keytab_realm(self):
+        return self.__config.get_value(KEYTAB_REALM)
+
+    def kerberos_host(self):
+        return self.__config.get_value(KERBEROS_KDC_HOST)
+
+
+def add_definitions(config_builder: ConfigurationBuilder) -> None:
+    config_builder.add(
+        key=KRB_AUTH,
+        default_value=None,
+        description="Specifies the Kerberos authentication type to use. "
+        "Possible values are 'minikerberos' and 'kinit'. "
+        "If left empty, the authentication is disabled.",
+    )
+    config_builder.add(
+        key=KEYTAB_FILENAME,
+        default_value=None,
+        description="Specifies the name of the keytab file. "
+        "If left empty, the name of the keytab file is assumed to be the same "
+        "as the name of the data job with '.keytab' suffix.",
+    )
+    config_builder.add(
+        key=KEYTAB_PRINCIPAL,
+        default_value=None,
+        description="Specifies the Kerberos principal. "
+        "If left empty, the principal will be the job name prepended with 'pa__view_'.",
+    )
+    config_builder.add(
+        key=KEYTAB_REALM,
+        default_value="default_realm",
+        description="Specifies the Kerberos realm. This value is used only with "
+        "the 'minikerberos' authentication type. The default value is 'default_realm'.",
+    )
+    config_builder.add(
+        key=KERBEROS_KDC_HOST,
+        default_value="localhost",
+        description="Specifies the name of the Kerberos KDC (Key Distribution Center) host. "
+        "This value is used only with the 'minikerberos' authentication type.",
+    )

projects/vdk-plugins/vdk-kerberos-auth/src/vdk/plugin/kerberos/kerberos_plugin.py

@@ -0,0 +1,51 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import logging
+from typing import List
+
+from vdk.api.plugin.hook_markers import hookimpl
+from vdk.api.plugin.plugin_registry import IPluginRegistry
+from vdk.internal.builtin_plugins.run.job_context import JobContext
+from vdk.internal.core.config import ConfigurationBuilder
+from vdk.plugin.kerberos.authenticator_factory import KerberosAuthenticatorFactory
+from vdk.plugin.kerberos.kerberos_configuration import add_definitions
+from vdk.plugin.kerberos.kerberos_configuration import KerberosPluginConfiguration
+
+log = logging.getLogger(__name__)
+
+
+class KerberosPlugin:
+    def __init__(self):
+        self.__authenticator = None
+
+    @staticmethod
+    @hookimpl
+    def vdk_configure(config_builder: ConfigurationBuilder) -> None:
+        add_definitions(config_builder)
+
+    @hookimpl(tryfirst=True)
+    def initialize_job(self, context: JobContext) -> None:
+        kerberos_configuration = KerberosPluginConfiguration(
+            context.name, str(context.job_directory), context.core_context.configuration
+        )
+        authenticator_factory = KerberosAuthenticatorFactory(kerberos_configuration)
+
+        try:
+            self.__authenticator = authenticator_factory.create_authenticator(
+                kerberos_configuration.authentication_type()
+            )
+        except Exception:
+            log.error(
+                f"Kerberos authenticator cannot be created. You can provide configuration that specifies "
+                f"keytab info using the following environment variables: {'VDK_KEYTAB_FOLDER'}, or "
+                f"{'VDK_KEYTAB_FILENAME'} and {'VDK_KEYTAB_PRINCIPAL'}"
+            )
+            self.__authenticator = None
+
+        if self.__authenticator:
+            self.__authenticator.authenticate()
+
+
+@hookimpl
+def vdk_start(plugin_registry: IPluginRegistry, command_line_args: List):
+    plugin_registry.load_plugin_with_hooks_impl(KerberosPlugin())