From 22a1fc21994df688c6d4d6cf69a8a7cbeabf1e34 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Mon, 11 Feb 2019 14:35:34 +1300 Subject: [PATCH] :white_check_mark: :rewind: Revert rasterio to 1.0.13 to fix unit tests Revert #90 to fix broken unit tests on the get_window_bounds and selective_tile functions in data_prep.ipynb. I've tracked it down to an incorrect affine output from dataset.transform. Seems to be a 'bug' introduced in rasterio version 1.0.14 (see #84), something with how the GDAL env is now set? Also smallfix to remove read_table deprecation warning in pandas 0.24 (see #89), replacing with read_csv instead. --- Pipfile | 2 +- Pipfile.lock | 100 ++++++++++++++++++++++++------------------------ data_prep.ipynb | 4 +- data_prep.py | 2 +- 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/Pipfile b/Pipfile index 6219575..0506499 100644 --- a/Pipfile +++ b/Pipfile @@ -24,7 +24,7 @@ packaging = "==19.0" pandas = "==0.24.1" pyproj = "==1.9.6" quilt = "==2.9.14" -rasterio = "==1.0.16" +rasterio = "==1.0.13" requests = "==2.21.0" scikit-image = "==0.14.2" shapely = "==1.7a1" diff --git a/Pipfile.lock b/Pipfile.lock index 1fb0017..22a8e1a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3e9d8b7d395f9e6ee8faf23feed19b649455ec93232fe7a175c39ae83ac99660" + "sha256": "8fbccb5fe635a3f5f6a0870af05a41e334edc7204af4c5a6897b92e39a8a87d0" }, "pipfile-spec": 6, "requires": { @@ -513,10 +513,10 @@ }, "nbconvert": { "hashes": [ - "sha256:08d21cf4203fabafd0d09bbd63f06131b411db8ebeede34b0fd4be4548351779", - "sha256:a8a2749f972592aa9250db975304af6b7337f32337e523a2c995cc9e12c07807" + "sha256:302554a2e219bc0fc84f3edd3e79953f3767b46ab67626fdec16e38ba3f7efe4", + "sha256:5de8fb2284422272a1d45abc77c07b888127550a6d602ce619592a2b08a474ff" ], - "version": "==5.4.0" + "version": "==5.4.1" }, "nbformat": { "hashes": [ @@ -859,9 +859,9 @@ }, "pyrsistent": { "hashes": [ - "sha256:5a3827d57ad3e46820e5ee4ed5b9e0ee7bc4686df6634a7368bc1863a5c48a77" + "sha256:07f7ae71291af8b0dbad8c2ab630d8223e4a8c4e10fc37badda158c02e753acf" ], - "version": "==0.14.9" + "version": "==0.14.10" }, "python-dateutil": { "hashes": [ @@ -958,20 +958,20 @@ }, "rasterio": { "hashes": [ - "sha256:17f2582d928e0cd9057ed4588a333ad6c453549d2cfec068d1972b6c4260c922", - "sha256:1dc84b3c472bbc51f6ab1ca571633d19ba48b000df99afebc2067fe1c8199332", - "sha256:2280687d5386cdc4c7a642fced71f7174dbb420ab833490c412c840111ca7649", - "sha256:8010425eb7804cfa3785ae2f59ddcd9f8de51d03516719d084e84e80c89ee955", - "sha256:81d1feeeb87e9ce7cd75ad5ba3adef68a561ef4b37167cfcd80ea85cac16d393", - "sha256:8628389d7bd56a2c1ba6f4628e5382b106ee4043966dce9f7be9b151dc943cc3", - "sha256:a6db2bb1b000c5f1acb94d6e451f057d64961b1ab1310f430be828958da58d94", - "sha256:cc90656e1fe05b74a4296d7e15932bfd3a23834a24b9f2009ba949c6ca2ed274", - "sha256:e92b34e36459a4e2a39e83cbc02b389efcb2f44a4d5118c5d610ffc77a10f80f", - "sha256:ecc324e22dc298a3fe26646a60a3cd33bf9c17bfa2afe15b15e442f9e1889c04", - "sha256:f08d19fe38b2480d1766b44d5e74463e8ca5508a862c14415c234a484f60537a" + "sha256:142ebafd76d4164c7a70e7ca741237aa5dead40e4ef16767b1eb3f7664d7015e", + "sha256:179bbcf843bab6f13158bb45c81d18a8dd62233520901c7356ea9cd034e70300", + "sha256:1dbda9578538cb2acc432e7c10caf55d533d993fc29b77f3b29b2ccdcdebdcb3", + "sha256:3d09c9bc9fc18110894691512e2685ac5a645039f9e46567fccd4f23158034b9", + "sha256:43b1eaea2c2ed501ad236ebba98b588a9e0a0e104c25125b64910b3bbf032162", + "sha256:7f1157c78c1a0649ea1d6ec7655e9e05e357072c787a6e9cd25b0a463facc343", + "sha256:812c54ba444a9a7967bfd73256a058d6dd8cdfa810145fe44320eb5b5dfc9c48", + "sha256:c7921a34a299b43842233840bce0803553a346a0e387e1cb09c0afc4a342d247", + "sha256:e4186cfaaf4ca4960cc0ab3da87dc5b31fb9ffdb7b5f71ff20a64c47574bc421", + "sha256:e490ef4d2c34bc0d958dece284c42f2e2a7fcfb240094c902f701e34f9e5ba57", + "sha256:f69aac9d62d14065cfcb6f1927325a57ed3988ce86e8be14ccd94aedf2b80506" ], "index": "pypi", - "version": "==1.0.16" + "version": "==1.0.13" }, "requests": { "hashes": [ @@ -1009,36 +1009,36 @@ }, "scipy": { "hashes": [ - "sha256:02cb79ea38114dc480e9b08d6b87095728e8fb39b9a49b449ee443d678001611", - "sha256:03c827cdbc584e935264040b958e5fa0570a16095bee23a013482ba3f0e963a2", - "sha256:04f2b23258139c109d0524f111597dd095a505d9cb2c71e381d688d653877fa3", - "sha256:3132a9fab3f3545c8b0ba15688d11857efdd4a58d388d3785dc474f56fea7138", - "sha256:4b1f0883cb9d8ee963cf0a31c87341e9e758abb2cf1e5bcc0d7b066ef6b17573", - "sha256:4cce25c6e7ff7399c67dfe1b5423c36c391cf9b4b2be390c1675ab410f1ef503", - "sha256:51a2424c8ed80e60bdb9a896806e7adaf24a58253b326fbad10f80a6d06f2214", - "sha256:5706b785ca289fdfd91aa05066619e51d140613b613e35932601f2315f5d8470", - "sha256:58f0435f052cb60f1472c77f52a8f6642f8877b70559e5e0b9a1744f33f5cbe5", - "sha256:63e1d5ca9e5e1984f1a275276991b036e25d39d37dd7edbb3f4f6165c2da7dbb", - "sha256:64b2c35824da3ef6bb1e722216e4ef28802af6413c7586136500e343d34ba179", - "sha256:6f791987899532305126309578727c0197bddbafab9596bafe3e7bfab6e1ce13", - "sha256:72bd766f753fd32f072d30d7bc2ad492d56dbcbf3e13764e27635d5c41079339", - "sha256:7413080b381766a22d52814edb65631f0e323a7cea945c70021a672f38acc73f", - "sha256:78a67ee4845440e81cfbfabde20537ca12051d0eeac951fe4c6d8751feac3103", - "sha256:7994c044bf659b0a24ad7673ec7db85c2fadb87e4980a379a9fd5b086fe3649a", - "sha256:7dc4002f0a0a688774ef04878afe769ecd1ac21fe9b4b1d7125e2cf16170afd3", - "sha256:854bd87cc23824d5db4983956bc30f3790e1c7448f1a9e6a8fb7bff7601aef87", - "sha256:8608316d0cc01f8b25111c8adfe6efbc959cbba037a62c784551568d7ffbf280", - "sha256:8f5fcc87b48fc3dd6d901669c89af4feeb856dffb6f671539a238b7e8af1799c", - "sha256:937147086e8b4338bf139ca8fa98da650e3a46bf2ca617f8e9dd68c3971ec420", - "sha256:bc294841f6c822714af362095b181a853f47ed5ce757354bd2e4776d579ff3a4", - "sha256:bc6a88b0009a1b60eab5c22ac3a006f6968d6328de10c6a64ebb0d64a05548d3", - "sha256:c5eae911cf26b3c7eda889ec98d3c226f312c587acfaaf02602473f98b4c16d6", - "sha256:cca33a01a5987c650b87a1a910aa311ffa44e67cca1ff502ef9efdae5d9a8624", - "sha256:d1ae77b79fd5e27a10ba7c4e7c3a62927b9d29a4dccf28f6905c25d983aaf183", - "sha256:fb36064047e6bf87b6320a9b6eb7f525ef6863c7a4aef1a84a4bbfb043612617", - "sha256:fc1a19d95649439dbd50baca676bceb29bbfcd600aed2c5bd71d9bad82a87cfe" - ], - "version": "==1.2.0" + "sha256:014cb900c003b5ac81a53f2403294e8ecf37aedc315b59a6b9370dce0aa7627a", + "sha256:281a34da34a5e0de42d26aed692ab710141cad9d5d218b20643a9cb538ace976", + "sha256:588f9cc4bfab04c45fbd19c1354b5ade377a8124d6151d511c83730a9b6b2338", + "sha256:5a10661accd36b6e2e8855addcf3d675d6222006a15795420a39c040362def66", + "sha256:628f60be272512ca1123524969649a8cb5ae8b31cca349f7c6f8903daf9034d7", + "sha256:6dcc43a88e25b815c2dea1c6fac7339779fc988f5df8396e1de01610604a7c38", + "sha256:70e37cec0ac0fe95c85b74ca4e0620169590fd5d3f44765f3c3a532cedb0e5fd", + "sha256:7274735fb6fb5d67d3789ddec2cd53ed6362539b41aa6cc0d33a06c003aaa390", + "sha256:78e12972e144da47326958ac40c2bd1c1cca908edc8b01c26a36f9ffd3dce466", + "sha256:790cbd3c8d09f3a6d9c47c4558841e25bac34eb7a0864a9def8f26be0b8706af", + "sha256:79792c8fe8e9d06ebc50fe23266522c8c89f20aa94ac8e80472917ecdce1e5ba", + "sha256:865afedf35aaef6df6344bee0de391ee5e99d6e802950a237f9fb9b13e441f91", + "sha256:870fd401ec7b64a895cff8e206ee16569158db00254b2f7157b4c9a5db72c722", + "sha256:963815c226b29b0176d5e3d37fc9de46e2778ce4636a5a7af11a48122ef2577c", + "sha256:9726791484f08e394af0b59eb80489ad94d0a53bbb58ab1837dcad4d58489863", + "sha256:9de84a71bb7979aa8c089c4fb0ea0e2ed3917df3fb2a287a41aaea54bbad7f5d", + "sha256:b2c324ddc5d6dbd3f13680ad16a29425841876a84a1de23a984236d1afff4fa6", + "sha256:b86ae13c597fca087cb8c193870507c8916cefb21e52e1897da320b5a35075e5", + "sha256:ba0488d4dbba2af5bf9596b849873102d612e49a118c512d9d302ceafa36e01a", + "sha256:d78702af4102a3a4e23bb7372cec283e78f32f5573d92091aa6aaba870370fe1", + "sha256:def0e5d681dd3eb562b059d355ae8bebe27f5cc455ab7c2b6655586b63d3a8ea", + "sha256:e085d1babcb419bbe58e2e805ac61924dac4ca45a07c9fa081144739e500aa3c", + "sha256:e2cfcbab37c082a5087aba5ff00209999053260441caadd4f0e8f4c2d6b72088", + "sha256:e742f1f5dcaf222e8471c37ee3d1fd561568a16bb52e031c25674ff1cf9702d5", + "sha256:f06819b028b8ef9010281e74c59cb35483933583043091ed6b261bb1540f11cc", + "sha256:f15f2d60a11c306de7700ee9f65df7e9e463848dbea9c8051e293b704038da60", + "sha256:f31338ee269d201abe76083a990905473987371ff6f3fdb76a3f9073a361cf37", + "sha256:f6b88c8d302c3dac8dff7766955e38d670c82e0d79edfc7eae47d6bb2c186594" + ], + "version": "==1.2.1" }, "send2trash": { "hashes": [ @@ -1426,9 +1426,9 @@ }, "pyrsistent": { "hashes": [ - "sha256:5a3827d57ad3e46820e5ee4ed5b9e0ee7bc4686df6634a7368bc1863a5c48a77" + "sha256:07f7ae71291af8b0dbad8c2ab630d8223e4a8c4e10fc37badda158c02e753acf" ], - "version": "==0.14.9" + "version": "==0.14.10" }, "pytest": { "hashes": [ diff --git a/data_prep.ipynb b/data_prep.ipynb index 73531f4..27f5f3e 100644 --- a/data_prep.ipynb +++ b/data_prep.ipynb @@ -29,7 +29,7 @@ "Geopandas : 0.4.0+26.g9e584cc\n", "GMT : 0.0.1a0+16.g7004aa0\n", "Numpy : 1.14.5\n", - "Rasterio : 1.0.16\n", + "Rasterio : 1.0.13\n", "Scikit-image : 0.14.2\n", "Xarray : 0.11.3\n" ] @@ -622,7 +622,7 @@ " assert len(files) > 0 # check that there are actually files being matched!\n", "\n", " df = pd.concat(\n", - " pd.read_table(f, sep=sep, header=skip, names=names, usecols=usecols)\n", + " pd.read_csv(f, sep=sep, header=skip, names=names, usecols=usecols)\n", " for f in files\n", " )\n", " df.reset_index(drop=True, inplace=True) # reset index after concatenation\n", diff --git a/data_prep.py b/data_prep.py index b34f9ac..ba14c8c 100644 --- a/data_prep.py +++ b/data_prep.py @@ -265,7 +265,7 @@ def ascii_to_xyz(pipeline_file: str) -> pd.DataFrame: assert len(files) > 0 # check that there are actually files being matched! df = pd.concat( - pd.read_table(f, sep=sep, header=skip, names=names, usecols=usecols) + pd.read_csv(f, sep=sep, header=skip, names=names, usecols=usecols) for f in files ) df.reset_index(drop=True, inplace=True) # reset index after concatenation