diff --git a/pkgs/development/python-modules/support/fetchpypi/default.nix b/pkgs/development/python-modules/support/fetchpypi/default.nix new file mode 100644 index 0000000000000..12018fcb42388 --- /dev/null +++ b/pkgs/development/python-modules/support/fetchpypi/default.nix @@ -0,0 +1,6 @@ +{ fetchurl, filename }: + +let + data = builtins.fromJSON (builtins.readFile filename); +in pname: fetchurl {url=data.${pname}.url; sha256=data.${pname}.sha256; meta.version=data.${pname}.version;} + diff --git a/pkgs/development/python-modules/support/fetchpypi/hashes.json b/pkgs/development/python-modules/support/fetchpypi/hashes.json new file mode 100644 index 0000000000000..689e07b8af6da --- /dev/null +++ b/pkgs/development/python-modules/support/fetchpypi/hashes.json @@ -0,0 +1,17 @@ +{ + "notebook": { + "sha256": "39a9603d3fe88b60de2903680c965cf643acf2c16fb2c6bac1d905e1042b5851", + "url": "https://files.pythonhosted.org/packages/81/a1/20af1a3ea6090343b029d31f882c7e4c061133e0c25808835b1b59a187f8/notebook-4.2.3.tar.gz", + "version": "4.2.3" + }, + "numpy": { + "sha256": "04db2fbd64e2e7c68e740b14402b25af51418fc43a59d9e54172b38b906b0f69", + "url": "https://files.pythonhosted.org/packages/16/f5/b432f028134dd30cfbf6f21b8264a9938e5e0f75204e72453af08d67eb0b/numpy-1.11.2.tar.gz", + "version": "1.11.2" + }, + "scipy": { + "sha256": "8ab6e9c808bf2fb3e8576cd8cf07226d9cdc18b012c06d9708429a821ac6634e", + "url": "https://files.pythonhosted.org/packages/22/41/b1538a75309ae4913cdbbdc8d1cc54cae6d37981d2759532c1aa37a41121/scipy-0.18.1.tar.gz", + "version": "0.18.1" + } +} \ No newline at end of file diff --git a/pkgs/development/python-modules/support/fetchpypi/packages.txt b/pkgs/development/python-modules/support/fetchpypi/packages.txt new file mode 100644 index 0000000000000..9e42136f424da --- /dev/null +++ b/pkgs/development/python-modules/support/fetchpypi/packages.txt @@ -0,0 +1,3 @@ +notebook +numpy +scipy diff --git a/pkgs/development/python-modules/support/fetchpypi/update-hashes b/pkgs/development/python-modules/support/fetchpypi/update-hashes new file mode 100755 index 0000000000000..e6d15c5f6a1b2 --- /dev/null +++ b/pkgs/development/python-modules/support/fetchpypi/update-hashes @@ -0,0 +1,107 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i python3 -p python3Packages.aiohttp +# #! nix-shell -i python -p 'python3.withPackages(ps: [ps.aiohttp])' +import asyncio +import aiohttp + +import json + +INDEX = "https://pypi.io/pypi" +"""url of PyPI""" + +FILENAME_PACKAGES = "packages.txt" +FILENAME_HASHES = "hashes.json" + +NSEMAPHORE = 200 +"""Maximum amount of concurrent requests""" + +NTIMEOUT = 2 +"""Timeout in seconds""" + +EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', 'whl'] +"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned.""" + +def load_package_names(filename): + """Load names of packages we like to retrieve hashes for.""" + with open(filename, 'r') as f: + names = f.read().splitlines() + return names + +def write_hashes(filename, data): + """Write hashes to json file.""" + + with open(filename, 'w') as f: + json.dump(data, f, indent=2, sort_keys=True) + + +async def _fetch(session, url, sem): + async with sem: + with aiohttp.Timeout(NTIMEOUT): + async with session.get(url) as response: + return await response.json() + + +async def _fetch_all(session, urls, loop, sem): + tasks = [loop.create_task(_fetch(session, url, sem)) for url in urls] + results = await asyncio.gather(*tasks, return_exceptions=False) + return results + +def _get_json_from_api(names, index=INDEX): +#def _get_and_write_data(folder, packages, index=INDEX): + """Yield JSON information obtained from PyPI index given an iterable of package names. + + :param packages: Iterable of package names. + :param index: url with packages index. By default `INDEX` is used. + """ + loop = asyncio.get_event_loop() + urls = ("{}/{}/json".format(index, package) for package in names) + connector = aiohttp.TCPConnector(share_cookies=True, loop=loop) + with aiohttp.ClientSession(loop=loop, connector=connector) as session: + sem = asyncio.Semaphore(NSEMAPHORE) + data = loop.run_until_complete(_fetch_all(session, urls, loop, sem)) + #logger.info("Finished retrieved JSON from PyPI") + loop.close() + return data + +def _extract_src_and_hash(json, version, extensions=EXTENSIONS): + """Obtain url and hash for a given version and list of allowable extensions. + :param json: json retrieved from PyPI + """ + if not json['releases']: + msg = "Package {}: No releases available.".format(json['info']['name']) + raise ValueError(msg) + else: + # We use ['releases'] and not ['urls'] because we want to have the possibility for different version. + for extension in extensions: + for possible_file in json['releases'][version]: + if possible_file['filename'].endswith(extension): + src = {'url': str(possible_file['url']), + 'sha256': str(possible_file['digests']['sha256']), + } + return src + else: + msg = "Package {}: No release for version {} with valid file extension available.".format(json['info']['name'], version) + raise ValueError(msg) + + +def _get_relevant_data(json): + name = json['info']['name'] + data = {} + data['version'] = json['info']['version'] + # Get source archive of this version + data.update(_extract_src_and_hash(json, data['version'])) + return name, data + +def get_hashes(names): + """Retrieve hashes of packages.""" + raw_data = _get_json_from_api(names) + #print(raw_data) + + return {name : x for name, x in map(_get_relevant_data, raw_data)} + + +if __name__ == '__main__': + + names = load_package_names(FILENAME_PACKAGES) + data = get_hashes(names) + write_hashes(FILENAME_HASHES, data) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 13b5f0a91053c..e57e13b05bac3 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -26,10 +26,12 @@ let }); buildPythonApplication = args: buildPythonPackage ({namePrefix="";} // args ); + fetchpypi = callPackage ../development/python-modules/support/fetchpypi {filename=../development/python-modules/support/fetchpypi/hashes.json; }; in { inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k mkPythonDerivation buildPythonPackage buildPythonApplication; + inherit fetchpypi; # helpers @@ -15594,13 +15596,10 @@ in { }; notebook = buildPythonPackage rec { - version = "4.2.3"; - name = "notebook-${version}"; - - src = pkgs.fetchurl { - url = "mirror://pypi/n/notebook/${name}.tar.gz"; - sha256 = "39a9603d3fe88b60de2903680c965cf643acf2c16fb2c6bac1d905e1042b5851"; - }; + pname = "notebook"; + name = "${pname}-${version}"; + src = fetchpypi pname; + version = src.meta.version; LC_ALL = "en_US.UTF-8"; @@ -15820,7 +15819,7 @@ in { blas = pkgs.openblasCompat; }; - numpy = self.numpy_1_11; + numpy = self.numpy_latest; numpy_1_10 = self.buildNumpyPackage rec { version = "1.10.4"; @@ -15838,6 +15837,12 @@ in { }; }; + numpy_latest = self.buildNumpyPackage rec { + pname = "numpy"; + src = fetchpypi pname; + version = src.meta.version; + }; + numpydoc = buildPythonPackage rec { name = "numpydoc-${version}"; version = "0.5"; @@ -22671,7 +22676,7 @@ in { gfortran = pkgs.gfortran; }; - scipy = self.scipy_0_18; + scipy = self.scipy_latest; scipy_0_17 = self.buildScipyPackage rec { version = "0.17.1"; @@ -22691,6 +22696,13 @@ in { numpy = self.numpy; }; + scipy_latest = self.buildNumpyPackage rec { + pname = "numpy"; + src = fetchpypi pname; + version = src.meta.version; + numpy = self.numpy; + }; + scikitimage = buildPythonPackage rec { name = "scikit-image-${version}"; version = "0.12.3";