python: install without distutils

Instead of using distutils/setuptools for installing the Python code,
just install it into our own Python directory.

Distutils is being removed from Python, and setuptools doesn't work well
when trying to install into your own location. For our usage its just
simpler to install with make.

In addition to removing the configure check for distutils, also remove
the check for pyyaml. This lets the user install pyyaml after Suricata
is installed, and Suricata-Update does handle this case gracefully.

Issue: OISF#5313

(cherry picked from commit 9a1d6af)

configure.ac

@@ -117,42 +117,6 @@
        pymv="$($HAVE_PYTHON -c 'import sys; print(sys.version_info[[0]]);')"
     fi
 
-    # Check for python-distutils (setup).
-    have_python_distutils="no"
-    if test "x$enable_python" = "xyes"; then
-        AC_MSG_CHECKING([for python-distutils])
-	if $HAVE_PYTHON -c "import distutils; from distutils.core import setup" 2>/dev/null; then
-	   AC_MSG_RESULT([yes])
-	   have_python_distutils="yes"
-	else
-	   AC_MSG_RESULT([no])
-	fi
-    fi
-    AM_CONDITIONAL([HAVE_PYTHON_DISTUTILS],
-	[test "x$have_python_distutils" = "xyes"])
-    if test "$have_python_distutils" = "no"; then
-       echo ""
-       echo "    Warning: Python distutils not found. Python tools will"
-       echo "        not be installed."
-       echo ""
-       echo "    Install the distutils module for Python ${pymv} to enable"
-       echo "    the Python tools."
-       echo ""
-    fi
-
-    # Check for python-yaml.
-    have_python_yaml="no"
-    if test "x$enable_python" = "xyes"; then
-        AC_MSG_CHECKING([for python-yaml])
-	if $HAVE_PYTHON -c "import yaml" 2>/dev/null; then
-	   have_python_yaml="yes"
-	   AC_MSG_RESULT([yes])
-	else
-	   AC_MSG_RESULT([no])
-	fi
-    fi
-    AM_CONDITIONAL([HAVE_PYTHON_YAML], [test "x$have_python_yaml" = "xyes"])
-
     AC_PATH_PROG(HAVE_WGET, wget, "no")
     if test "$HAVE_WGET" = "no"; then
         AC_PATH_PROG(HAVE_CURL, curl, "no")
@@ -1610,13 +1574,11 @@
     fi
 
     if test "$have_suricata_update" = "yes"; then
-        if test "$have_python_yaml" != "yes"; then
+        if test "$enable_python" != "yes"; then
 	    echo ""
-	    echo "    Warning: suricata-update will not be installed as the"
-	    echo "        Python yaml module is not installed.."
+	    echo "    Warning: suricata-update will not be installed as"
+	    echo "        Python is not installed."
 	    echo ""
-            echo "    Install the yaml module for Python ${pymv} to enable"
-            echo "    suricata-update."
 	    echo
 	else
             SURICATA_UPDATE_DIR="suricata-update"
@@ -1630,8 +1592,6 @@
     # Test to see if suricatactl (and suricatasc) can be installed.
     if test "x$enable_python" != "xyes"; then
         install_suricatactl="requires python"
-    elif test "x$have_python_distutils" != "xyes"; then
-        install_suricatactl="no, requires distutils"
     else
         install_suricatactl="yes"
     fi
@@ -1640,11 +1600,8 @@
     if test "x$have_suricata_update" != "xyes"; then
         install_suricata_update="not bundled"
     elif test "x$enable_python" != "xyes"; then
-        install_suricata_update="no, requires python"
-    elif test "x$have_python_distutils" != "xyes"; then
-        install_suricata_update="no, requires distutils"
-    elif test "x$have_python_yaml" != "xyes"; then
-        install_suricata_update="no, requires pyyaml"
+        install_suricata_update="no, "
+        install_suricata_update_reason="requires python"
     else
         install_suricata_update="yes"
     fi
@@ -2861,8 +2818,6 @@ SURICATA_BUILD_CONF="Suricata Configuration:
 
   Python support:                          ${enable_python}
   Python path:                             ${python_path}
-  Python distutils                         ${have_python_distutils}
-  Python yaml                              ${have_python_yaml}
   Install suricatactl:                     ${install_suricatactl}
   Install suricatasc:                      ${install_suricatactl}
   Install suricata-update:                 ${install_suricata_update}

python/Makefile.am

@@ -1,34 +1,47 @@
-EXTRA_DIST =	setup.py \
-		bin \
-		suricata \
-		suricatasc
+LIBS =	\
+		suricata/__init__.py \
+		suricata/config/__init__.py \
+		suricata/ctl/__init__.py \
+		suricata/ctl/filestore.py \
+		suricata/ctl/loghandler.py \
+		suricata/ctl/main.py \
+		suricata/ctl/test_filestore.py \
+		suricata/sc/__init__.py \
+		suricata/sc/specs.py \
+		suricata/sc/suricatasc.py \
+		suricatasc/__init__.py
+
+BINS =	\
+		suricatasc \
+		suricatactl
+
+EXTRA_DIST = $(LIBS) bin suricata/config/defaults.py
 
 if HAVE_PYTHON
-if HAVE_PYTHON_DISTUTILS
-all-local:
-	cd $(srcdir) && \
-		$(HAVE_PYTHON) setup.py build --build-base "$(abs_builddir)"
 
 install-exec-local:
-	cd $(srcdir) && \
-		$(HAVE_PYTHON) setup.py build --build-base "$(abs_builddir)" \
-		install --prefix $(DESTDIR)$(prefix)
+	install -d -m 0755 "$(DESTDIR)$(prefix)/lib/suricata/python/suricata/config"
+	install -d -m 0755 "$(DESTDIR)$(prefix)/lib/suricata/python/suricata/ctl"
+	install -d -m 0755 "$(DESTDIR)$(prefix)/lib/suricata/python/suricata/sc"
+	install -d -m 0755 "$(DESTDIR)$(prefix)/lib/suricata/python/suricatasc"
+	install -d -m 0755 "$(DESTDIR)$(prefix)/bin"
+	for src in $(LIBS); do \
+		install $(srcdir)/$$src "$(DESTDIR)$(prefix)/lib/suricata/python/$$src"; \
+	done
+	install suricata/config/defaults.py \
+		"$(DESTDIR)$(prefix)/lib/suricata/python/suricata/config/defaults.py"
+	for bin in $(BINS); do \
+		cat "$(srcdir)/bin/$$bin" | \
+		    sed -e "1 s,.*,#"'!'" ${HAVE_PYTHON}," > "${DESTDIR}$(bindir)/$$bin"; \
+		chmod 0755 "$(DESTDIR)$(bindir)/$$bin"; \
+	done
 
 uninstall-local:
 	rm -f $(DESTDIR)$(bindir)/suricatactl
 	rm -f $(DESTDIR)$(bindir)/suricatasc
-	rm -rf $(DESTDIR)$(prefix)/lib*/python*/site-packages/suricata
-	rm -rf $(DESTDIR)$(prefix)/lib*/python*/site-packages/suricatasc
-	rm -rf $(DESTDIR)$(prefix)/lib*/python*/site-packages/suricata-[0-9]*.egg-info
+	rm -rf $(DESTDIR)$(prefix)/lib/suricata/python
 
 clean-local:
-	cd $(srcdir) && \
-		$(HAVE_PYTHON) setup.py clean \
-		--build-base "$(abs_builddir)"
-	rm -rf scripts-* lib* build
 	find . -name \*.pyc -print0 | xargs -0 rm -f
 
-distclean-local:
-	rm -f version
-endif
 endif

python/bin/suricatactl

@@ -1,6 +1,6 @@
 #! /usr/bin/env python
 #
-# Copyright (C) 2017 Open Information Security Foundation
+# Copyright (C) 2017-2022 Open Information Security Foundation
 #
 # You can copy, redistribute or modify this Program under the terms of
 # the GNU General Public License version 2 as published by the Free
@@ -26,13 +26,12 @@ if os.path.exists(os.path.join(exec_dir, "..", "suricata", "ctl", "main.py")):
     # Looks like we're running from the development directory.
     sys.path.insert(0, ".")
 else:
-    # This is to find the suricata module in the case of being installed
-    # to a non-standard prefix.
+    # Check if the Python modules are installed in the Suricata installation
+    # prefix.
     version_info = sys.version_info
     pyver = "%d.%d" % (version_info.major, version_info.minor)
-    path = os.path.join(
-        exec_dir, "..", "lib", "python%s" % (pyver), "site-packages",
-        "suricata")
+    path = os.path.realpath(os.path.join(
+        exec_dir, "..", "lib", "suricata", "python", "suricata"))
     if os.path.exists(path):
         sys.path.insert(0, os.path.dirname(path))
 

python/bin/suricatasc

@@ -1,14 +1,15 @@
-#!/usr/bin/python
-# Copyright(C) 2013-2020 Open Information Security Foundation
-
+#! /usr/bin/env python
+#
+# Copyright(C) 2013-2022 Open Information Security Foundation
+#
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, version 2 of the License.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU General Public License for mo re details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
@@ -26,13 +27,12 @@ if os.path.exists(os.path.join(exec_dir, "..", "suricata", "ctl", "main.py")):
     # Looks like we're running from the development directory.
     sys.path.insert(0, ".")
 else:
-    # This is to find the suricata module in the case of being installed
-    # to a non-standard prefix.
+    # Check if the Python modules are installed in the Suricata installation
+    # prefix.
     version_info = sys.version_info
     pyver = "%d.%d" % (version_info.major, version_info.minor)
-    path = os.path.join(
-        exec_dir, "..", "lib", "python%s" % (pyver), "site-packages",
-        "suricata")
+    path = os.path.realpath(os.path.join(
+        exec_dir, "..", "lib", "suricata", "python", "suricata"))
     if os.path.exists(path):
         sys.path.insert(0, os.path.dirname(path))
 

suricata-update/Makefile.am

@@ -1,30 +1,62 @@
-if HAVE_PYTHON
-if HAVE_PYTHON_DISTUTILS
-if HAVE_PYTHON_YAML
+LIBS =	\
+	suricata/__init__.py \
+	suricata/update/commands/__init__.py \
+	suricata/update/commands/addsource.py \
+	suricata/update/commands/checkversions.py \
+	suricata/update/commands/disablesource.py \
+	suricata/update/commands/enablesource.py \
+	suricata/update/commands/listsources.py \
+	suricata/update/commands/removesource.py \
+	suricata/update/commands/updatesources.py \
+	suricata/update/compat/__init__.py \
+	suricata/update/compat/ordereddict.py \
+	suricata/update/compat/argparse/__init__.py \
+	suricata/update/compat/argparse/argparse.py \
+	suricata/update/configs/__init__.py \
+	suricata/update/config.py \
+	suricata/update/data/__init__.py \
+	suricata/update/data/index.py \
+	suricata/update/data/update.py \
+	suricata/update/__init__.py \
+	suricata/update/engine.py \
+	suricata/update/exceptions.py \
+	suricata/update/extract.py \
+	suricata/update/loghandler.py \
+	suricata/update/main.py \
+	suricata/update/maps.py \
+	suricata/update/matchers.py \
+	suricata/update/net.py \
+	suricata/update/notes.py \
+	suricata/update/osinfo.py \
+	suricata/update/parsers.py \
+	suricata/update/rule.py \
+	suricata/update/sources.py \
+	suricata/update/util.py \
+	suricata/update/version.py
+
+BINS =	suricata-update
 
-all-local:
-	cd $(srcdir) && \
-		$(HAVE_PYTHON) setup.py build --build-base $(abs_builddir)
+if HAVE_PYTHON
 
 install-exec-local:
-	cd $(srcdir) && \
-		$(HAVE_PYTHON) setup.py build --build-base "$(abs_builddir)" \
-		install --prefix $(DESTDIR)$(prefix)
+	install -d -m 0755 "$(DESTDIR)$(prefix)/lib/suricata/python/suricata/update/commands"
+	install -d -m 0755 "$(DESTDIR)$(prefix)/lib/suricata/python/suricata/update/compat/argparse"
+	install -d -m 0755 "$(DESTDIR)$(prefix)/lib/suricata/python/suricata/update/configs"
+	install -d -m 0755 "$(DESTDIR)$(prefix)/lib/suricata/python/suricata/update/data"
+	for lib in $(LIBS); do \
+		install $(srcdir)/$$lib "$(DESTDIR)$(prefix)/lib/suricata/python/$$lib"; \
+	done
+	for bin in $(BINS); do \
+		cat "$(srcdir)/bin/$$bin" | \
+		    sed -e "1 s,.*,#"'!'" ${HAVE_PYTHON}," > "${DESTDIR}$(bindir)/$$bin"; \
+		chmod 0755 "$(DESTDIR)$(bindir)/$$bin"; \
+	done
 
 uninstall-local:
 	rm -f $(DESTDIR)$(bindir)/suricata-update
-	rm -rf $(DESTDIR)$(prefix)/lib*/python*/site-packages/suricata-update
-	rm -rf $(DESTDIR)$(prefix)/lib*/python*/site-packages/suricata_update-[0-9]*.egg-info
+	rm -rf $(DESTDIR)$(prefix)/lib/suricata/python
 
 clean-local:
-	cd $(srcdir) && \
-		$(HAVE_PYTHON) setup.py clean \
-		--build-base "$(abs_builddir)"
-	rm -rf scripts-* lib* build
 	find . -name \*.pyc -print0 | xargs -0 rm -f
 
-distclean-local:
-
-endif
-endif
 endif