Python: use separate output for tkinter

doc/languages-frameworks/python.md

@@ -416,29 +416,14 @@ aliases `python` and `python3` correspond to respectively `python27` and
 `python35`. The Nix expressions for the interpreters can be found in
 `pkgs/development/interpreters/python`.
 
-
-#### Missing modules standard library
-
-The interpreters `python26` and `python27` do not include modules that
-require external dependencies. This is done in order to reduce the closure size.
-The following modules need to be added as `buildInput` explicitly:
-
-* `python.modules.bsddb`
-* `python.modules.curses`
-* `python.modules.curses_panel`
-* `python.modules.crypt`
-* `python.modules.gdbm`
-* `python.modules.sqlite3`
-* `python.modules.tkinter`
-* `python.modules.readline`
-
-For convenience `python27Full` and `python26Full` are provided with all
-modules included.
-
 All packages depending on any Python interpreter get appended
 `out/{python.sitePackages}` to `$PYTHONPATH` if such directory
 exists.
 
+#### Missing `tkinter` module standard library
+
+To reduce closure size the `Tkinter`/`tkinter` is put in a separate output. The `tkinter` is available as `python35Packages.tkinter`.
+
 #### Attributes on interpreters packages
 
 Each interpreter has the following attributes:
@@ -448,7 +433,7 @@ Each interpreter has the following attributes:
 - `buildEnv`. Function to build python interpreter environments with extra packages bundled together. See section *python.buildEnv function* for usage and documentation.
 - `withPackages`. Simpler interface to `buildEnv`. See section *python.withPackages function* for usage and documentation.
 - `sitePackages`. Alias for `lib/${libPrefix}/site-packages`.
-- `executable`. Name of the interpreter executable, ie `python3.4`.
+- `executable`. Name of the interpreter executable, e.g. `python3.4`.
 
 ### Building packages and applications
 

nixos/modules/services/web-servers/apache-httpd/trac.nix

@@ -102,7 +102,6 @@ in
             pkgs.setuptools
             pkgs.pythonPackages.genshi
             pkgs.pythonPackages.psycopg2
-            pkgs.python.modules.sqlite3
             subversion
           ];
     };

pkgs/applications/misc/loxodo/default.nix

@@ -12,7 +12,7 @@ py.buildPythonApplication rec {
     sha256 = "1cg0dfcv57ps54f1a0ksib7hgkrbdi9q699w302xyyfyvjcb5dd2";
   };
 
-  propagatedBuildInputs = with py; [ wxPython python.modules.readline ];
+  propagatedBuildInputs = with py; [ wxPython ];
 
   postInstall = ''
     mv $out/bin/loxodo.py $out/bin/loxodo

pkgs/applications/misc/ranger/default.nix

@@ -15,7 +15,7 @@ pythonPackages.buildPythonApplication rec {
     sha256 = "0yaviybviwdvfg2a0pf2kk28g10k245499xmbpqlai7fv91f7xll";
   };
 
-  propagatedBuildInputs = [ pythonPackages.python.modules.curses file ];
+  propagatedBuildInputs = [ file ];
 
   preConfigure = ''
     substituteInPlace ranger/ext/img_display.py \

pkgs/applications/misc/roxterm/default.nix

@@ -1,7 +1,7 @@
 { stdenv, fetchurl, docbook_xsl, dbus_libs, dbus_glib, expat, gettext
 , gsettings_desktop_schemas, gdk_pixbuf, gtk2, gtk3, hicolor_icon_theme
 , imagemagick, itstool, librsvg, libtool, libxslt, lockfile, makeWrapper
-, pkgconfig, pythonFull, pythonPackages, vte }:
+, pkgconfig, python, pythonPackages, vte }:
 
 # TODO: Still getting following warning.
 # WARNING **: Error retrieving accessibility bus address: org.freedesktop.DBus.Error.ServiceUnknown: The name org.a11y.Bus was not provided by any .service files
@@ -19,7 +19,7 @@ in stdenv.mkDerivation rec {
 
   buildInputs =
     [ docbook_xsl expat imagemagick itstool librsvg libtool libxslt
-      makeWrapper pkgconfig pythonFull pythonPackages.lockfile ];
+      makeWrapper pkgconfig python pythonPackages.lockfile ];
 
   propagatedBuildInputs =
     [ dbus_libs dbus_glib gdk_pixbuf gettext gsettings_desktop_schemas gtk2 gtk3 hicolor_icon_theme vte ];
@@ -29,7 +29,7 @@ in stdenv.mkDerivation rec {
                          "-I${dbus_libs.lib}/lib/dbus-1.0/include" ];
 
   # Fix up python path so the lockfile library is on it.
-  PYTHONPATH = stdenv.lib.makeSearchPathOutput "lib" pythonFull.sitePackages [
+  PYTHONPATH = stdenv.lib.makeSearchPathOutput "lib" python.sitePackages [
     pythonPackages.curses pythonPackages.lockfile
   ];
 

pkgs/applications/misc/rtv/default.nix

@@ -17,7 +17,6 @@ pythonPackages.buildPythonApplication rec {
     six
     praw
     kitchen
-    python.modules.curses
     praw
   ] ++ lib.optional (!pythonPackages.isPy3k) futures;
 

pkgs/applications/networking/mailreaders/mailpile/default.nix

@@ -16,7 +16,7 @@ pythonPackages.buildPythonApplication rec {
 
   propagatedBuildInputs = with pythonPackages; [
     makeWrapper pillow jinja2 spambayes pythonPackages.lxml
-    python.modules.readline pgpdump gnupg1orig
+    pgpdump gnupg1orig
   ];
 
   postInstall = ''

pkgs/applications/version-management/bazaar/default.nix

@@ -10,9 +10,6 @@ pythonPackages.buildPythonApplication rec {
     sha256 = "1cysix5k3wa6y7jjck3ckq3abls4gvz570s0v0hxv805nwki4i8d";
   };
 
-  # Readline support is needed by bzrtools.
-  propagatedBuildInputs = [ pythonPackages.python.modules.readline ];
-
   doCheck = false;
 
   # Bazaar can't find the certificates alone

pkgs/applications/video/kodi/default.nix

@@ -1,5 +1,5 @@
 { stdenv, lib, fetchurl, makeWrapper
-, pkgconfig, cmake, gnumake, yasm, pythonFull
+, pkgconfig, cmake, gnumake, yasm, python
 , boost, avahi, libdvdcss, lame, autoreconfHook
 , gettext, pcre-cpp, yajl, fribidi, which
 , openssl, gperf, tinyxml2, taglib, libssh, swig, jre
@@ -54,7 +54,7 @@ in stdenv.mkDerivation rec {
 
     buildInputs = [
       makeWrapper libxml2 gnutls
-      pkgconfig cmake gnumake yasm pythonFull
+      pkgconfig cmake gnumake yasm python
       boost libmicrohttpd autoreconfHook
       gettext pcre-cpp yajl fribidi libva
       openssl gperf tinyxml2 taglib libssh swig jre
@@ -107,7 +107,7 @@ in stdenv.mkDerivation rec {
     postInstall = ''
       for p in $(ls $out/bin/) ; do
         wrapProgram $out/bin/$p \
-          --prefix PATH ":" "${pythonFull}/bin" \
+          --prefix PATH ":" "${python}/bin" \
           --prefix PATH ":" "${glxinfo}/bin" \
           --prefix PATH ":" "${xdpyinfo}/bin" \
           --prefix LD_LIBRARY_PATH ":" "${lib.makeLibraryPath

pkgs/development/compilers/edk2/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, fetchgit, libuuid, pythonFull, iasl }:
+{ stdenv, fetchgit, libuuid, python, iasl }:
 
 let
 
@@ -18,7 +18,7 @@ edk2 = stdenv.mkDerivation {
     sha256 = "0s9ywb8w7xzlnmm4kwzykxkrdaw53b7pky121cc9wjkllzqwyxrb";
   };
 
-  buildInputs = [ libuuid pythonFull ];
+  buildInputs = [ libuuid python python.tkinter ];
 
   makeFlags = "-C BaseTools";
 
@@ -40,7 +40,7 @@ edk2 = stdenv.mkDerivation {
 
   passthru = {
     setup = projectDscPath: attrs: {
-      buildInputs = [ pythonFull ] ++
+      buildInputs = [ python python.tkinter ] ++
         stdenv.lib.optionals (attrs ? buildInputs) attrs.buildInputs;
 
       configurePhase = ''

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -1,8 +1,5 @@
 { stdenv, fetchurl, fetchpatch, self, callPackage, python27Packages
 , bzip2, openssl, gettext
-
-, includeModules ? false
-
 , db, gdbm, ncurses, sqlite, readline
 
 , tcl ? null, tk ? null, xlibsWrapper ? null, libX11 ? null, x11Support ? !stdenv.isCygwin
@@ -27,6 +24,7 @@ let
   pythonVersion = majorVersion;
   version = "${majorVersion}.${minorVersion}${minorVersionSuffix}";
   libPrefix = "python${majorVersion}";
+  sitePackages = "lib/${libPrefix}/site-packages";
 
   src = fetchurl {
     url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz";
@@ -113,10 +111,8 @@ let
     optional (stdenv ? cc && stdenv.cc.libc != null) stdenv.cc.libc ++
     [ bzip2 openssl ]
     ++ optionals stdenv.isCygwin [ expat libffi ]
-    ++ optionals includeModules (
-        [ db gdbm ncurses sqlite readline
-        ] ++ optionals x11Support [ tcl tk xlibsWrapper libX11 ]
-    )
+    ++ [ db gdbm ncurses sqlite readline ]
+    ++ optionals x11Support [ tcl tk xlibsWrapper libX11 ]
     ++ optional zlibSupport zlib
     ++ optional stdenv.isDarwin CF;
 
@@ -129,7 +125,8 @@ let
 
   # Build the basic Python interpreter without modules that have
   # external dependencies.
-  python = stdenv.mkDerivation {
+
+in stdenv.mkDerivation {
     name = "python-${version}";
     pythonVersion = majorVersion;
 
@@ -165,20 +162,25 @@ let
         # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
         echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
 
-        ${optionalString includeModules "$out/bin/python ./setup.py build_ext"}
-
         rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev
       '';
 
+    postFixup = optionalString x11Support ''
+      # tkinter goes in a separate output
+      mkdir -p $tkinter/${sitePackages}
+      mv $out/lib/${libPrefix}/lib-dynload/_tkinter* $tkinter/${sitePackages}/
+    '';
+
+    outputs = ["out"] ++ optional x11Support "tkinter";
+
     passthru = rec {
-      inherit libPrefix;
+      inherit libPrefix sitePackages;
       inherit zlibSupport;
       isPy2 = true;
       isPy27 = true;
       buildEnv = callPackage ../../wrapper.nix { python = self; };
       withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python27Packages; };
       executable = libPrefix;
-      sitePackages = "lib/${libPrefix}/site-packages";
       interpreter = "${self}/bin/${executable}";
     };
 
@@ -200,99 +202,4 @@ let
       platforms = stdenv.lib.platforms.all;
       maintainers = with stdenv.lib.maintainers; [ chaoflow domenkozar ];
     };
-  };
-
-
-  # This function builds a Python module included in the main Python
-  # distribution in a separate derivation.
-  buildInternalPythonModule =
-    { moduleName
-    , internalName ? "_" + moduleName
-    , deps
-    }:
-    if includeModules then null else stdenv.mkDerivation rec {
-      name = "python-${moduleName}-${python.version}";
-
-      inherit src patches preConfigure postConfigure configureFlags;
-
-      buildInputs = [ python ] ++ deps;
-
-      # We need to set this for python.buildEnv
-      pythonPath = [];
-
-      inherit (mkPaths buildInputs) C_INCLUDE_PATH LIBRARY_PATH;
-
-      # non-python gdbm has a libintl dependency on i686-cygwin, not on x86_64-cygwin
-      buildPhase = (if (stdenv.system == "i686-cygwin" && moduleName == "gdbm") then ''
-          sed -i setup.py -e "s:libraries = \['gdbm'\]:libraries = ['gdbm', 'intl']:"
-      '' else '''') + ''
-          substituteInPlace setup.py --replace 'self.extensions = extensions' \
-            'self.extensions = [ext for ext in self.extensions if ext.name in ["${internalName}"]]'
-
-          python ./setup.py build_ext
-          [ -z "$(find build -name '*_failed.so' -print)" ]
-        '';
-
-      installPhase =
-        ''
-          dest=$out/lib/${python.libPrefix}/site-packages
-          mkdir -p $dest
-          cp -p $(find . -name "*.${if stdenv.isCygwin then "dll" else "so"}") $dest/
-        '';
-    };
-
-
-  # The Python modules included in the main Python distribution, built
-  # as separate derivations.
-  modules = {
-
-    bsddb = buildInternalPythonModule {
-      moduleName = "bsddb";
-      deps = [ db ];
-    };
-
-    curses = buildInternalPythonModule {
-      moduleName = "curses";
-      deps = [ ncurses ];
-    };
-
-    curses_panel = buildInternalPythonModule {
-      moduleName = "curses_panel";
-      deps = [ ncurses modules.curses ];
-    };
-
-    crypt = buildInternalPythonModule {
-      moduleName = "crypt";
-      internalName = "crypt";
-      deps = optional (stdenv ? glibc) stdenv.glibc;
-    };
-
-    gdbm = buildInternalPythonModule {
-      moduleName = "gdbm";
-      internalName = "gdbm";
-      deps = [ gdbm ] ++ stdenv.lib.optional stdenv.isCygwin gettext;
-    };
-
-    sqlite3 = buildInternalPythonModule {
-      moduleName = "sqlite3";
-      deps = [ sqlite ];
-    };
-
-  } // optionalAttrs x11Support {
-
-    tkinter = if stdenv.isCygwin then null else (buildInternalPythonModule {
-      moduleName = "tkinter";
-      deps = [ tcl tk xlibsWrapper libX11 ];
-    });
-
-  } // {
-
-    readline = buildInternalPythonModule {
-      moduleName = "readline";
-      internalName = "readline";
-      deps = [ readline ];
-    };
-
-  };
-
-in python // { inherit modules; }
+  }

pkgs/development/interpreters/python/cpython/3.3/default.nix

@@ -26,6 +26,7 @@ let
   pythonVersion = majorVersion;
   version = "${majorVersion}.${minorVersion}${minorVersionSuffix}";
   libPrefix = "python${majorVersion}";
+  sitePackages = "lib/${libPrefix}/site-packages";
 
   buildInputs = filter (p: p != null) [
     zlib bzip2 lzma gdbm sqlite db readline ncurses openssl tcl tk libX11 xproto
@@ -79,8 +80,25 @@ stdenv.mkDerivation {
     echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
   '';
 
+  postFixup = ''
+    # Get rid of retained dependencies on -dev packages, and remove
+    # some $TMPDIR references to improve binary reproducibility.
+    for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
+      sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
+    done
+
+    # FIXME: should regenerate this.
+    rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
+
+    # tkinter goes in a separate output
+    mkdir -p $tkinter/${sitePackages}
+    mv $out/lib/${libPrefix}/lib-dynload/_tkinter* $tkinter/${sitePackages}/
+  '';
+
+  outputs = ["out" "tkinter"];
+
   passthru = rec {
-    inherit libPrefix;
+    inherit libPrefix sitePackages;
     zlibSupport = zlib != null;
     sqliteSupport = sqlite != null;
     dbSupport = db != null;
@@ -93,7 +111,6 @@ stdenv.mkDerivation {
     isPy3 = true;
     isPy33 = true;
     is_py3k = true;  # deprecated
-    sitePackages = "lib/${libPrefix}/site-packages";
     interpreter = "${self}/bin/${executable}";
   };