From 70598d36b35f7136cb67cb38c30aceb2f785e1ae Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Mon, 26 Sep 2016 16:20:36 +0200 Subject: [PATCH] Python: fixed-point combinator --- .../python/cpython/2.6/default.nix | 9 +- .../python/cpython/2.7/default.nix | 10 +- .../python/cpython/3.3/default.nix | 8 +- .../python/cpython/3.4/default.nix | 10 +- .../python/cpython/3.5/default.nix | 9 +- .../python/cpython/3.6/default.nix | 9 +- .../interpreters/python/interpreter.nix | 71 ++++++++++++++ .../interpreters/python/pypy/2.7/default.nix | 4 +- .../interpreters/python/support.nix | 40 ++++++++ pkgs/os-specific/linux/dstat/default.nix | 4 +- pkgs/top-level/all-packages.nix | 93 +++++++------------ pkgs/top-level/python-packages.nix | 49 +--------- 12 files changed, 177 insertions(+), 139 deletions(-) create mode 100644 pkgs/development/interpreters/python/interpreter.nix create mode 100644 pkgs/development/interpreters/python/support.nix diff --git a/pkgs/development/interpreters/python/cpython/2.6/default.nix b/pkgs/development/interpreters/python/cpython/2.6/default.nix index 5311be697afea..3e1f2f6832b8c 100644 --- a/pkgs/development/interpreters/python/cpython/2.6/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.6/default.nix @@ -1,6 +1,8 @@ { stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, includeModules ? false , sqlite, tcl, tk, xlibsWrapper, openssl, readline, db, ncurses, gdbm, self, callPackage -, python26Packages }: +, pkgOverrides ? (self: super: {}) +, pkgs +}: assert zlibSupport -> zlib != null; @@ -97,13 +99,12 @@ let inherit zlibSupport; isPy2 = true; isPy26 = true; - buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python26Packages; }; libPrefix = "python${majorVersion}"; executable = libPrefix; sitePackages = "lib/${libPrefix}/site-packages"; interpreter = "${self}/bin/${executable}"; - }; + } // (import ../../interpreter.nix {inherit stdenv pkgs; overrides=pkgOverrides; python=self;}); + enableParallelBuilding = true; diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index 5f15db26ebc35..d7bbbcb44840a 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -1,4 +1,4 @@ -{ stdenv, fetchurl, self, callPackage, python27Packages +{ stdenv, fetchurl, self, callPackage , bzip2, openssl, gettext , includeModules ? false @@ -9,6 +9,10 @@ , zlib ? null, zlibSupport ? true , expat, libffi + +, pkgOverrides ? (self: super: {}) +, pkgs + , CF, configd }: @@ -159,13 +163,11 @@ let inherit zlibSupport; isPy2 = true; isPy27 = true; - buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python27Packages; }; libPrefix = "python${majorVersion}"; executable = libPrefix; sitePackages = "lib/${libPrefix}/site-packages"; interpreter = "${self}/bin/${executable}"; - }; + } // (import ../../interpreter.nix {inherit stdenv pkgs; overrides=pkgOverrides; python=self;}); enableParallelBuilding = true; diff --git a/pkgs/development/interpreters/python/cpython/3.3/default.nix b/pkgs/development/interpreters/python/cpython/3.3/default.nix index cb48186a69e7b..d22ceda6a275e 100644 --- a/pkgs/development/interpreters/python/cpython/3.3/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.3/default.nix @@ -12,7 +12,8 @@ , zlib , callPackage , self -, python33Packages +, pkgOverrides ? (self: super: {}) +, pkgs }: assert readline != null -> ncurses != null; @@ -82,14 +83,13 @@ stdenv.mkDerivation { tkSupport = (tk != null) && (tcl != null) && (libX11 != null) && (xproto != null); libPrefix = "python${majorVersion}"; executable = "python3.3m"; - buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python33Packages; }; isPy3 = true; isPy33 = true; is_py3k = true; # deprecated sitePackages = "lib/${libPrefix}/site-packages"; interpreter = "${self}/bin/${executable}"; - }; + } // (import ../../interpreter.nix {inherit stdenv pkgs; overrides=pkgOverrides; python=self;}); + enableParallelBuilding = true; diff --git a/pkgs/development/interpreters/python/cpython/3.4/default.nix b/pkgs/development/interpreters/python/cpython/3.4/default.nix index 2e7d3a03141ea..12a221ff17b37 100644 --- a/pkgs/development/interpreters/python/cpython/3.4/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.4/default.nix @@ -12,8 +12,8 @@ , zlib , callPackage , self -, python34Packages - +, pkgOverrides ? (self: super: {}) +, pkgs , CF, configd }: @@ -106,14 +106,12 @@ stdenv.mkDerivation { tkSupport = (tk != null) && (tcl != null) && (libX11 != null) && (xproto != null); libPrefix = "python${majorVersion}"; executable = "python3.4m"; - buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python34Packages; }; isPy3 = true; isPy34 = true; is_py3k = true; # deprecated sitePackages = "lib/${libPrefix}/site-packages"; - interpreter = "${self}/bin/${executable}"; - }; + } // (import ../../interpreter.nix {inherit stdenv pkgs; overrides=pkgOverrides; python=self;}); + enableParallelBuilding = true; diff --git a/pkgs/development/interpreters/python/cpython/3.5/default.nix b/pkgs/development/interpreters/python/cpython/3.5/default.nix index 69d3df32a326c..6111aaab3722e 100644 --- a/pkgs/development/interpreters/python/cpython/3.5/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.5/default.nix @@ -11,9 +11,10 @@ , zlib , callPackage , self -, python35Packages - +, pkgOverrides ? (self: super: {}) +, pkgs , CF, configd + }: assert readline != null -> ncurses != null; @@ -114,14 +115,12 @@ stdenv.mkDerivation { tkSupport = (tk != null) && (tcl != null) && (libX11 != null) && (xproto != null); libPrefix = "python${majorVersion}"; executable = "python${majorVersion}m"; - buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python35Packages; }; isPy3 = true; isPy35 = true; is_py3k = true; # deprecated sitePackages = "lib/${libPrefix}/site-packages"; interpreter = "${self}/bin/${executable}"; - }; + } // (import ../../interpreter.nix {inherit stdenv pkgs; overrides=pkgOverrides; python=self;}); enableParallelBuilding = true; diff --git a/pkgs/development/interpreters/python/cpython/3.6/default.nix b/pkgs/development/interpreters/python/cpython/3.6/default.nix index 3a6635cd8b621..bb002059e1ede 100644 --- a/pkgs/development/interpreters/python/cpython/3.6/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.6/default.nix @@ -13,8 +13,8 @@ , zlib , callPackage , self -, python36Packages - +, pkgOverrides ? (self: super: {}) +, pkgs , CF, configd }: @@ -109,14 +109,13 @@ stdenv.mkDerivation { tkSupport = (tk != null) && (tcl != null) && (libX11 != null) && (xproto != null); libPrefix = "python${majorVersion}"; executable = "python${majorVersion}m"; - buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python36Packages; }; isPy3 = true; isPy35 = true; is_py3k = true; # deprecated sitePackages = "lib/${libPrefix}/site-packages"; interpreter = "${self}/bin/${executable}"; - }; + } // (import ../../interpreter.nix {inherit stdenv pkgs; overrides=pkgOverrides; python=self;}); + enableParallelBuilding = true; diff --git a/pkgs/development/interpreters/python/interpreter.nix b/pkgs/development/interpreters/python/interpreter.nix new file mode 100644 index 0000000000000..7f8b7beed4261 --- /dev/null +++ b/pkgs/development/interpreters/python/interpreter.nix @@ -0,0 +1,71 @@ +{ stdenv +, pkgs +, python +#, pythonConfig ? (self: super: {}) +, overrides ? (self: super: {}) +}: + +let + inherit (stdenv) lib; + + # Package set + packages = let + + inherit (lib) fix' extends fold; + + pythonPackages = self: import ./support.nix { + inherit pkgs python setuptools; + } self; + + commonConfiguration = import ../../../top-level/python-packages.nix { inherit pkgs stdenv; }; + + #in fix' (fold extends [ overrides commonConfiguration pythonPackages ]); + in fix' (extends overrides (extends commonConfiguration pythonPackages)); + + # Function to build an environment + buildEnv = import ./wrapper.nix { + inherit stdenv python; + inherit (pkgs) buildEnv makeWrapper; + }; + + # Function to build an environment, but accepting only a list of packages + withPackages = import ./with-packages.nix { + inherit buildEnv; + pythonPackages = packages; + }; + + # Derivation with scripts for building wrappers + wrapPython = import ./wrap-python.nix { + inherit lib python; + inherit (pkgs) makeSetupHook makeWrapper; + }; + + # Setuptools derivation + setuptools = import ../../python-modules/setuptools { + inherit stdenv lib python wrapPython; + inherit (pkgs) fetchurl; + }; + + # Function to build a package that doesn't use setuptools + mkPythonDerivation = import ./mk-python-derivation.nix { + inherit lib python wrapPython setuptools; + inherit (pkgs) unzip ensureNewerSourcesHook; + }; + + # Function to build a package that uses setuptools or installs a wheel + buildPythonPackage = lib.makeOverridable ( import ./build-python-package.nix { + inherit lib python mkPythonDerivation; + bootstrapped-pip = import ../../python-modules/bootstrapped-pip { + inherit stdenv python; + inherit (pkgs) fetchurl makeWrapper unzip; + }; + }); + +in { +# For each Python interpreter version we have this set. + pkgs = packages; + inherit withPackages; + inherit buildEnv; + inherit mkPythonDerivation; + inherit buildPythonPackage; +} diff --git a/pkgs/development/interpreters/python/pypy/2.7/default.nix b/pkgs/development/interpreters/python/pypy/2.7/default.nix index c0bd10a0d4372..aa17ba334fd46 100644 --- a/pkgs/development/interpreters/python/pypy/2.7/default.nix +++ b/pkgs/development/interpreters/python/pypy/2.7/default.nix @@ -1,6 +1,6 @@ { stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, pkgconfig, libffi , sqlite, openssl, ncurses, pythonFull, expat, tcl, tk, xlibsWrapper, libX11 -, makeWrapper, callPackage, self, pypyPackages, gdbm, db }: +, makeWrapper, callPackage, self, gdbm, db }: assert zlibSupport -> zlib != null; @@ -119,10 +119,8 @@ let inherit zlibSupport libPrefix; executable = "pypy"; isPypy = true; - buildEnv = callPackage ../../wrapper.nix { python = self; }; interpreter = "${self}/bin/${executable}"; sitePackages = "site-packages"; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = pypyPackages; }; }; enableParallelBuilding = true; # almost no parallelization without STM diff --git a/pkgs/development/interpreters/python/support.nix b/pkgs/development/interpreters/python/support.nix new file mode 100644 index 0000000000000..6cb5bac0d700a --- /dev/null +++ b/pkgs/development/interpreters/python/support.nix @@ -0,0 +1,40 @@ +{ pkgs +, python +, setuptools }: + +with pkgs.lib; + +self: + +let + modules = python.modules or { + readline = null; + sqlite3 = null; + curses = null; + curses_panel = null; + crypt = null; + }; + +in rec { + + inherit python; + inherit setuptools; # Packages shouldn't depend explicitly on setuptools. + inherit (python) mkPythonDerivation buildPythonPackage wrapPython ; + inherit modules; + + pythonAtLeast = versionAtLeast python.pythonVersion; + pythonOlder = versionOlder python.pythonVersion; + isPy26 = python.majorVersion == "2.6"; + isPy27 = python.majorVersion == "2.7"; + isPy33 = python.majorVersion == "3.3"; + isPy34 = python.majorVersion == "3.4"; + isPy35 = python.majorVersion == "3.5"; + isPy36 = python.majorVersion == "3.6"; + isPyPy = python.executable == "pypy"; + isPy3k = strings.substring 0 1 python.majorVersion == "3"; + + callPackage = pkgs.newScope self; + + buildPythonApplication = args: buildPythonPackage ({namePrefix="";} // args ); + +} // modules diff --git a/pkgs/os-specific/linux/dstat/default.nix b/pkgs/os-specific/linux/dstat/default.nix index 8f7772de1fdce..4ce251e059f56 100644 --- a/pkgs/os-specific/linux/dstat/default.nix +++ b/pkgs/os-specific/linux/dstat/default.nix @@ -1,4 +1,4 @@ -{ stdenv, fetchurl, python, pythonPackages }: +{ stdenv, fetchurl, pythonPackages }: stdenv.mkDerivation rec { name = "dstat-${version}"; @@ -14,7 +14,7 @@ stdenv.mkDerivation rec { pythonPath = with pythonPackages; [ python-wifi ]; patchPhase = '' - sed -i -e 's|/usr/bin/env python|${python}/bin/python|' \ + sed -i -e 's|/usr/bin/env python|${pythonPackages.python.interpreter}|' \ -e "s|/usr/share/dstat|$out/share/dstat|" dstat ''; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 434b0674c9a0a..a9691a6642b5e 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6090,47 +6090,36 @@ in }; purePackages = recurseIntoAttrs (callPackage ./pure-packages.nix {}); - python = python2; - python2 = python27; - python3 = python35; - - # pythonPackages further below, but assigned here because they need to be in sync - pythonPackages = python2Packages; - python2Packages = python27Packages; - python3Packages = python35Packages; + ## PYTHON + # Interpreters python26 = callPackage ../development/interpreters/python/cpython/2.6 { - db = db47; self = python26; }; python27 = callPackage ../development/interpreters/python/cpython/2.7 { - self = python27; inherit (darwin) CF configd; + self = python27; }; python33 = callPackage ../development/interpreters/python/cpython/3.3 { self = python33; }; - python34 = hiPrio (callPackage ../development/interpreters/python/cpython/3.4 { + python34 = callPackage ../development/interpreters/python/cpython/3.4 { inherit (darwin) CF configd; self = python34; - }); - python35 = hiPrio (callPackage ../development/interpreters/python/cpython/3.5 { + }; + python35 = callPackage ../development/interpreters/python/cpython/3.5 { inherit (darwin) CF configd; self = python35; - }); + }; python36 = callPackage ../development/interpreters/python/cpython/3.6 { inherit (darwin) CF configd; self = python36; }; - - pypy = pypy27; - pypy27 = callPackage ../development/interpreters/python/pypy/2.7 { self = pypy27; }; - pythonFull = python2Full; - python2Full = python27Full; + # Interpreters - full versions python26Full = python26.override { includeModules = true; self = python26Full; @@ -6140,6 +6129,29 @@ in self = python27Full; }; + # Interpreters - aliases + python = python2; + python2 = python27; + python3 = python35; + pythonFull = python27Full; + pypy = pypy27; + + + ## Aliases package sets. Deprecated + python26Packages = python26.pkgs; + python27Packages = python27.pkgs; + python33Packages = python33.pkgs; + python34Packages = python34.pkgs; + python35Packages = python35.pkgs; + python36Packages = python36.pkgs; + pypy27Packages = pypy27.pkgs; + + pythonPackages = python2Packages; + python2Packages = python27Packages; + python3Packages = python35Packages; + pypyPackages = pypy27Packages; + pythonFullPackages = pythonFull.pkgs; + python2nix = callPackage ../tools/package-management/python2nix { }; pythonDocs = recurseIntoAttrs (callPackage ../development/interpreters/python/cpython/docs {}); @@ -10277,47 +10289,6 @@ in sqitchModule = perlPackages.AppSqitch; }; - ### DEVELOPMENT / PYTHON MODULES - - # `nix-env -i python-nose` installs for 2.7, the default python. - # Therefore we do not recurse into attributes here, in contrast to - # python27Packages. `nix-env -iA python26Packages.nose` works - # regardless. - python26Packages = callPackage ./python-packages.nix { - python = python26; - self = python26Packages; - }; - - python27Packages = lib.hiPrioSet (recurseIntoAttrs (callPackage ./python-packages.nix { - python = python27; - self = python27Packages; - })); - - python33Packages = callPackage ./python-packages.nix { - python = python33; - self = python33Packages; - }; - - python34Packages = callPackage ./python-packages.nix { - python = python34; - self = python34Packages; - }; - - python35Packages = recurseIntoAttrs (callPackage ./python-packages.nix { - python = python35; - self = python35Packages; - }); - - python36Packages = (callPackage ./python-packages.nix { - python = python36; - self = python36Packages; - }); - - pypyPackages = callPackage ./python-packages.nix { - python = pypy; - self = pypyPackages; - }; - ### DEVELOPMENT / R MODULES R = callPackage ../applications/science/math/R { @@ -11113,7 +11084,7 @@ in dstat = callPackage ../os-specific/linux/dstat { # pythonFull includes the "curses" standard library module, for pretty # dstat color output - python = pythonFull; + pythonPackages = pythonFull.pkgs; }; libossp_uuid = callPackage ../development/libraries/libossp-uuid { }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 1f5ed0bfa88dc..5cc6f6a127b40 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -1,54 +1,13 @@ -{ pkgs, stdenv, python, self }: +# This file contains all Python package overrides. -with pkgs.lib; - -let - pythonAtLeast = versionAtLeast python.pythonVersion; - pythonOlder = versionOlder python.pythonVersion; - isPy26 = python.majorVersion == "2.6"; - isPy27 = python.majorVersion == "2.7"; - isPy33 = python.majorVersion == "3.3"; - isPy34 = python.majorVersion == "3.4"; - isPy35 = python.majorVersion == "3.5"; - isPy36 = python.majorVersion == "3.6"; - isPyPy = python.executable == "pypy"; - isPy3k = strings.substring 0 1 python.majorVersion == "3"; - - callPackage = pkgs.newScope self; - - bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { }; - - mkPythonDerivation = makeOverridable( callPackage ../development/interpreters/python/mk-python-derivation.nix { - }); - buildPythonPackage = makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix { - inherit mkPythonDerivation; - inherit bootstrapped-pip; - }); - - buildPythonApplication = args: buildPythonPackage ({namePrefix="";} // args ); +{ stdenv, pkgs }: - modules = python.modules or { - readline = null; - sqlite3 = null; - curses = null; - curses_panel = null; - crypt = null; - }; - -in modules // { - - inherit python bootstrapped-pip isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k mkPythonDerivation buildPythonPackage buildPythonApplication; - - # helpers - - wrapPython = callPackage ../development/interpreters/python/wrap-python.nix {inherit python; inherit (pkgs) makeSetupHook makeWrapper; }; +with pkgs.lib; - # specials +self: super: with super; { recursivePthLoader = callPackage ../development/python-modules/recursive-pth-loader { }; - setuptools = callPackage ../development/python-modules/setuptools { }; - agate = buildPythonPackage rec { name = "agate-1.2.2"; disabled = isPy3k;