pythonPackages.fetchpypi: fetch tarballs from pypi

This function grabs the url, version and hash for a specified package from a file that was generated by update-hashes.

pkgs/development/python-modules/support/fetchpypi/default.nix

@@ -0,0 +1,6 @@
+{ fetchurl, filename }:
+
+let
+  data = builtins.fromJSON (builtins.readFile filename);
+in pname: fetchurl {url=data.${pname}.url; sha256=data.${pname}.sha256; meta.version=data.${pname}.version;}
+

pkgs/development/python-modules/support/fetchpypi/hashes.json

@@ -0,0 +1,17 @@
+{
+  "notebook": {
+    "sha256": "39a9603d3fe88b60de2903680c965cf643acf2c16fb2c6bac1d905e1042b5851",
+    "url": "https://files.pythonhosted.org/packages/81/a1/20af1a3ea6090343b029d31f882c7e4c061133e0c25808835b1b59a187f8/notebook-4.2.3.tar.gz",
+    "version": "4.2.3"
+  },
+  "numpy": {
+    "sha256": "04db2fbd64e2e7c68e740b14402b25af51418fc43a59d9e54172b38b906b0f69",
+    "url": "https://files.pythonhosted.org/packages/16/f5/b432f028134dd30cfbf6f21b8264a9938e5e0f75204e72453af08d67eb0b/numpy-1.11.2.tar.gz",
+    "version": "1.11.2"
+  },
+  "scipy": {
+    "sha256": "8ab6e9c808bf2fb3e8576cd8cf07226d9cdc18b012c06d9708429a821ac6634e",
+    "url": "https://files.pythonhosted.org/packages/22/41/b1538a75309ae4913cdbbdc8d1cc54cae6d37981d2759532c1aa37a41121/scipy-0.18.1.tar.gz",
+    "version": "0.18.1"
+  }
+}

pkgs/development/python-modules/support/fetchpypi/packages.txt

@@ -0,0 +1,3 @@
+notebook
+numpy
+scipy

pkgs/development/python-modules/support/fetchpypi/update-hashes

@@ -0,0 +1,107 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i python3 -p python3Packages.aiohttp
+#  #! nix-shell -i python -p 'python3.withPackages(ps: [ps.aiohttp])'
+import asyncio
+import aiohttp
+
+import json
+
+INDEX = "https://pypi.io/pypi"
+"""url of PyPI"""
+
+FILENAME_PACKAGES = "packages.txt"
+FILENAME_HASHES = "hashes.json"
+
+NSEMAPHORE = 200
+"""Maximum amount of concurrent requests"""
+
+NTIMEOUT = 2
+"""Timeout in seconds"""
+
+EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', 'whl']
+"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned."""
+
+def load_package_names(filename):
+    """Load names of packages we like to retrieve hashes for."""
+    with open(filename, 'r') as f:
+        names = f.read().splitlines()
+    return names
+
+def write_hashes(filename, data):
+    """Write hashes to json file."""
+
+    with open(filename, 'w') as f:
+        json.dump(data, f, indent=2, sort_keys=True)
+
+
+async def _fetch(session, url, sem):
+    async with sem:
+        with aiohttp.Timeout(NTIMEOUT):
+            async with session.get(url) as response:
+                return await response.json()
+
+
+async def _fetch_all(session, urls, loop, sem):
+    tasks = [loop.create_task(_fetch(session, url, sem)) for url in urls]
+    results = await asyncio.gather(*tasks, return_exceptions=False)
+    return results
+
+def _get_json_from_api(names, index=INDEX):
+#def _get_and_write_data(folder, packages, index=INDEX):
+    """Yield JSON information obtained from PyPI index given an iterable of package names.
+
+    :param packages: Iterable of package names.
+    :param index: url with packages index. By default `INDEX` is used.
+    """
+    loop = asyncio.get_event_loop()
+    urls = ("{}/{}/json".format(index, package) for package in names)
+    connector = aiohttp.TCPConnector(share_cookies=True, loop=loop)
+    with aiohttp.ClientSession(loop=loop, connector=connector) as session:
+        sem = asyncio.Semaphore(NSEMAPHORE)
+        data = loop.run_until_complete(_fetch_all(session, urls, loop, sem))
+        #logger.info("Finished retrieved JSON from PyPI")
+    loop.close()
+    return data
+
+def _extract_src_and_hash(json, version, extensions=EXTENSIONS):
+    """Obtain url and hash for a given version and list of allowable extensions.
+    :param json: json retrieved from PyPI
+    """
+    if not json['releases']:
+        msg = "Package {}: No releases available.".format(json['info']['name'])
+        raise ValueError(msg)
+    else:
+        # We use ['releases'] and not ['urls'] because we want to have the possibility for different version.
+        for extension in extensions:
+            for possible_file in json['releases'][version]:
+                if possible_file['filename'].endswith(extension):
+                    src = {'url': str(possible_file['url']),
+                            'sha256': str(possible_file['digests']['sha256']),
+                            }
+                    return src
+        else:
+            msg = "Package {}: No release for version {} with valid file extension available.".format(json['info']['name'], version)
+            raise ValueError(msg)
+
+
+def _get_relevant_data(json):
+    name = json['info']['name']
+    data = {}
+    data['version'] = json['info']['version']
+    # Get source archive of this version
+    data.update(_extract_src_and_hash(json, data['version']))
+    return name, data
+
+def get_hashes(names):
+    """Retrieve hashes of packages."""
+    raw_data = _get_json_from_api(names)
+    #print(raw_data)
+
+    return {name : x for name, x in map(_get_relevant_data, raw_data)}
+
+
+if __name__ == '__main__':
+
+    names = load_package_names(FILENAME_PACKAGES)
+    data = get_hashes(names)
+    write_hashes(FILENAME_HASHES, data)

pkgs/top-level/python-packages.nix

@@ -26,10 +26,12 @@ let
   });
 
   buildPythonApplication = args: buildPythonPackage ({namePrefix="";} // args );
+  fetchpypi = callPackage ../development/python-modules/support/fetchpypi {filename=../development/python-modules/support/fetchpypi/hashes.json; };
 
 in {
 
   inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k mkPythonDerivation buildPythonPackage buildPythonApplication;
+  inherit fetchpypi;
 
   # helpers
 
@@ -15594,13 +15596,10 @@ in {
   };
 
   notebook = buildPythonPackage rec {
-    version = "4.2.3";
-    name = "notebook-${version}";
-
-    src = pkgs.fetchurl {
-      url = "mirror://pypi/n/notebook/${name}.tar.gz";
-      sha256 = "39a9603d3fe88b60de2903680c965cf643acf2c16fb2c6bac1d905e1042b5851";
-    };
+    pname = "notebook";
+    name = "${pname}-${version}";
+    src = fetchpypi pname;
+    version = src.meta.version;
 
     LC_ALL = "en_US.UTF-8";
 
@@ -15820,7 +15819,7 @@ in {
     blas = pkgs.openblasCompat;
   };
 
-  numpy = self.numpy_1_11;
+  numpy = self.numpy_latest;
 
   numpy_1_10 = self.buildNumpyPackage rec {
     version = "1.10.4";
@@ -15838,6 +15837,12 @@ in {
     };
   };
 
+  numpy_latest = self.buildNumpyPackage rec {
+    pname = "numpy";
+    src = fetchpypi pname;
+    version = src.meta.version;
+  };
+
   numpydoc = buildPythonPackage rec {
     name = "numpydoc-${version}";
     version = "0.5";
@@ -22671,7 +22676,7 @@ in {
     gfortran = pkgs.gfortran;
   };
 
-  scipy = self.scipy_0_18;
+  scipy = self.scipy_latest;
 
   scipy_0_17 = self.buildScipyPackage rec {
     version = "0.17.1";
@@ -22691,6 +22696,13 @@ in {
     numpy = self.numpy;
   };
 
+  scipy_latest = self.buildNumpyPackage rec {
+    pname = "numpy";
+    src = fetchpypi pname;
+    version = src.meta.version;
+    numpy = self.numpy;
+  };
+
   scikitimage = buildPythonPackage rec {
     name = "scikit-image-${version}";
     version = "0.12.3";