diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c58850972..ce00d2b58 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,6 +1,10 @@ name: Lint -on: push +on: + push: + pull_request: + branches: + - develop jobs: formatting: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 19bff140e..7f25c83c1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,10 @@ name: Pytest -on: push +on: + push: + pull_request: + branches: + - develop jobs: test: @@ -9,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.8, 3.9, '3.10'] name: Python ${{ matrix.python-version }} steps: diff --git a/Pipfile b/Pipfile index e29208c62..395acd908 100644 --- a/Pipfile +++ b/Pipfile @@ -19,3 +19,4 @@ tqdm = "==4.29.1" requests = "~=2.25" fs = "*" "fs.sshfs" = "*" +fs-azureblob = "*" diff --git a/Pipfile.lock b/Pipfile.lock index ed23d585d..88493743d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "cc7beb7fb5a62576ebd376a38d1581f3ab941c6c4267a4dd17551a0a299c13c9" + "sha256": "81ea83f6f3c165fff79189a14b7235a2e9596f9a0e7784a6a125b1dfc98552f8" }, "pipfile-spec": 6, "requires": {}, @@ -21,12 +21,29 @@ ], "version": "==1.4.4" }, + "azure-core": { + "hashes": [ + "sha256:3d70e9ec64de92dfae330c15bc69085caceb2d83813ef6c01cc45326f2a4be83", + "sha256:88d2db5cf9a135a7287dc45fdde6b96f9ca62c9567512a3bb3e20e322ce7deb2" + ], + "version": "==1.21.1" + }, + "azure-storage-blob": { + "hashes": [ + "sha256:e74c2c49fd04b80225f5b9734f1dbd417d89f280abfedccced3ac21509e1659d", + "sha256:eb37b50ddfb6e558b29f6c8c03b0666514e55d6170bf4624e7261a3af93c6401" + ], + "version": "==12.8.1" + }, "bcrypt": { "hashes": [ + "sha256:56e5da069a76470679f312a7d3d23deb3ac4519991a0361abc11da837087b61d", "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29", "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7", "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34", + "sha256:a0584a92329210fcd75eb8a3250c5a941633f8bfaf2a18f81009b097732839b7", "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55", + "sha256:b589229207630484aefe5899122fb938a5b017b0f4349f769b8c13e78d99a8fd", "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6", "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1", "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d" @@ -36,122 +53,145 @@ }, "certifi": { "hashes": [ - "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", - "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" + "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", + "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569" ], - "version": "==2021.5.30" + "version": "==2021.10.8" }, "cffi": { "hashes": [ - "sha256:016ba30a6c3e93e48fd57b1a8624516aeba61f85ed1d5c31d9dd34f77fdff4f6", - "sha256:0e4d1c32812d5885956ba1489052ecd86a8cdde2f334568343e8bc646c7e2cb4", - "sha256:104516acf2608c22b5d44a58e0819da275c21e1751df69bd077a70fc8f016e18", - "sha256:1edddb00facd25f2fb0cc2f271b3a795cbbe804b36319ace01d15fb962309293", - "sha256:23feec3ff73fd47d9bf3fed96e5debb32727c67c9dcbfea67e172f8fac4500e6", - "sha256:28036db6711fcc83d154e3022270508bcfe20a4839e79b8f9f0776555512f2a4", - "sha256:2941c0ae3161b185b116eb939386c9c1dc8d4294ea0a3886c7248c50c23defe8", - "sha256:2bcca3d8cbaf4e3250f5c4916bf32e2da8ce3fd8163303ce41d9b703df38d31e", - "sha256:3d804d22084ba6cee9922cab3b0f665021be93efe40d528c6c32a73caae587eb", - "sha256:3e11347d5d19bd13359587270f2b4580fc3a6761fedb7d4a81ffac2e68c66b6c", - "sha256:415529aaf4d9b4c00b3a75fb30f82cafbb4487f96b3c7fbf424cebc978c96a32", - "sha256:42e4cb8e0705f7757816bfa6a771db1c5933f69cbab0d96a1a5a2d955237086e", - "sha256:577c7c7a5f7065ec9bf33b9ca2de96d8f579e39fff3bc94a9acd018c0db7dac1", - "sha256:66dee5853862e63d521354f7034714ad1c925e025641a4a19548e32580790a47", - "sha256:6a78361a30688afc1c79f09cee629dc5f6d627f7fbd43ab9a37b239e7609940f", - "sha256:6de0c4a6e53877696115221aefbea22eb9d31a4b2fc0756c7e4c970668376a60", - "sha256:6e1db0b63067f1a96f1b92fc8149aa20327a11d18ab3e3ad23be2d8aeb3a6bbb", - "sha256:71e842ed506fd588a427cbc127dc91d0727468f73d581fed02955256cd4bbdbb", - "sha256:720985a41a7be5ca5d17fa9196b4965c226b85f6c57e7b2f6c6debed657faa23", - "sha256:7623bbfc421d26698cc9dc4b8907ca6a149c7547d5c94ac6f55b1685e9f187b1", - "sha256:78f691b8c882d65162062bb10a69423f9d3005b3c8ec9977c60e14fe0036525f", - "sha256:8a2b1001ece83aabd818d334659a97cf3c0158501442f20fee7305c8ac3059cb", - "sha256:8c16099cb2d5a61b9ac4aa77de25a6376bb96c8f94ea96e2c69e625cd222e7ae", - "sha256:94261a9a318c6620b1e079f2ce15ff13c90305e880eaba1f9450e47ac4ea8b4a", - "sha256:9660de460a5cae80b896e560dcdbf4a4f1d055a88512207d582a361f47eb0dca", - "sha256:966df3527b71319f28fc6a723b727d5d6abc3298dff56262b20cc4236c26cb8f", - "sha256:98b299690d7a0c544bf0f211eabe426ad57751c657139aa4d3333d357c43f638", - "sha256:9c2e5e75c7c3d65131a366e0ae460542a1ace97ae93cc0f283f9efff1a97ebfa", - "sha256:afc696d8e7f3db5cbd32ae13ba53848aa6eec0a85d19ada7363443a27505c651", - "sha256:b6cf0921851b21ee5337e48a487e21a52d629453b42c85a6225776d9300c39ef", - "sha256:b8f4891d4f4772bab7d39f748ffcce39be83dac3bdf93d4e40bbd9e2ce1eb5c4", - "sha256:b96741e5ab3290aae9712e708362d4843e77d8a2418a3af3c08dda14d9691bab", - "sha256:c248be3c030cf7e0d035479b3d8d7ce4e4107959551196413897876c90c77d85", - "sha256:c4a469421402b50e48cc2ffa05229cbdd7dc0164ce2d31eae6b2ac8195ef9634", - "sha256:c5c8578e7fe6aca1dd2a1335ca977efce8bf2d38bc54568f120b912d7ab8d9c7", - "sha256:c7ed3d86d61284d5c0910c8c0988a645cf6723fa2deabe1b149112b2ebe0547e", - "sha256:ccec0368227d02c2ae5dc0881d8075bfa8fa87232f027d375a795d8a0f70c73a", - "sha256:d475cef45640f69ee779720c2fecfcef7cf2a869a0e7e7297ec7854db6cb6a04", - "sha256:d6491e6e623abb27d20f42e37011d28380792ffbba3657bc9fef464c8b5ca1cb", - "sha256:d6f8782c19a0b12b4c5d9cefe1b2ca1636591531c7f20c4ae4d7c5b4077d3af5", - "sha256:d89fcd91a1e0da07b35fbe09f0e10fff880e9ca5f76478f6bbfab10c9c9ef6f2", - "sha256:e0874c097458ce6933cdb1918f2e601e48772d10b8aff478f02d949c757aed97", - "sha256:e185aa34d2ab08e0fe1f6bd60aa6950bfcb84059e7debb8bcfb4f1818eafd6e3", - "sha256:e29accdb21f2186b80b60b07af29d752666fe7bbf057e08c54741740877909b2", - "sha256:e34d93929b4cf93e736be64c0c926af6bf90f0887de0253de233509665d36c9c", - "sha256:e42f7138056b2428cd6197351940ef36d214b8923dbb496af73ac717552cdfec", - "sha256:e4bb67bdef349768f3e662b63d3fc6a939e9e425326acebe4c0c1b48b6f38267", - "sha256:f3765dc391a99564c9b0e6030398a68d5f862e48b96e37fa66ba8101790642e8", - "sha256:f909436b96a032aed0a8529464910ae7665b5826b35801a03d9012dde3fcc9cb", - "sha256:fb2efcf7e942cf58c221adf0b39e8d55d252d2962ed7e8c511f3eec57a903f56" - ], - "version": "==1.15.0rc1" + "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3", + "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2", + "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636", + "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20", + "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728", + "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27", + "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66", + "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443", + "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0", + "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7", + "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39", + "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605", + "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a", + "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37", + "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029", + "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139", + "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc", + "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df", + "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14", + "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880", + "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2", + "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a", + "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e", + "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474", + "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024", + "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8", + "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0", + "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e", + "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a", + "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e", + "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032", + "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6", + "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e", + "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b", + "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e", + "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954", + "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962", + "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c", + "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4", + "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55", + "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962", + "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023", + "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c", + "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6", + "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8", + "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382", + "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7", + "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc", + "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997", + "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796" + ], + "version": "==1.15.0" }, "charset-normalizer": { "hashes": [ - "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6", - "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f" + "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd", + "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455" ], "markers": "python_version >= '3'", - "version": "==2.0.6" + "version": "==2.0.10" }, "cryptography": { "hashes": [ - "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e", - "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b", - "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7", - "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085", - "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc", - "sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d", - "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a", - "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498", - "sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89", - "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9", - "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c", - "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7", - "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb", - "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14", - "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af", - "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e", - "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5", - "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06", - "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7" + "sha256:0a817b961b46894c5ca8a66b599c745b9a3d9f822725221f0e0fe49dc043a3a3", + "sha256:2d87cdcb378d3cfed944dac30596da1968f88fb96d7fc34fdae30a99054b2e31", + "sha256:30ee1eb3ebe1644d1c3f183d115a8c04e4e603ed6ce8e394ed39eea4a98469ac", + "sha256:391432971a66cfaf94b21c24ab465a4cc3e8bf4a939c1ca5c3e3a6e0abebdbcf", + "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316", + "sha256:4caa4b893d8fad33cf1964d3e51842cd78ba87401ab1d2e44556826df849a8ca", + "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638", + "sha256:596f3cd67e1b950bc372c33f1a28a0692080625592ea6392987dba7f09f17a94", + "sha256:5d59a9d55027a8b88fd9fd2826c4392bd487d74bf628bb9d39beecc62a644c12", + "sha256:6c0c021f35b421ebf5976abf2daacc47e235f8b6082d3396a2fe3ccd537ab173", + "sha256:73bc2d3f2444bcfeac67dd130ff2ea598ea5f20b40e36d19821b4df8c9c5037b", + "sha256:74d6c7e80609c0f4c2434b97b80c7f8fdfaa072ca4baab7e239a15d6d70ed73a", + "sha256:7be0eec337359c155df191d6ae00a5e8bbb63933883f4f5dffc439dac5348c3f", + "sha256:94ae132f0e40fe48f310bba63f477f14a43116f05ddb69d6fa31e93f05848ae2", + "sha256:bb5829d027ff82aa872d76158919045a7c1e91fbf241aec32cb07956e9ebd3c9", + "sha256:ca238ceb7ba0bdf6ce88c1b74a87bffcee5afbfa1e41e173b1ceb095b39add46", + "sha256:ca28641954f767f9822c24e927ad894d45d5a1e501767599647259cbf030b903", + "sha256:e0344c14c9cb89e76eb6a060e67980c9e35b3f36691e15e1b7a9e58a0a6c6dc3", + "sha256:ebc15b1c22e55c4d5566e3ca4db8689470a0ca2babef8e3a9ee057a8b82ce4b1", + "sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee" ], "markers": "python_version >= '3.6'", - "version": "==3.4.8" + "version": "==36.0.1" }, "fs": { "hashes": [ - "sha256:1d10cc8f9c55fbcf7b23775289a13f6796dca7acd5a135c379f49e87a56a7230", - "sha256:caab4dc1561d63c92f36ee78976f6a4a01381830d8420ce34a78d4f1bb1dc95f" + "sha256:9555dc2bc58c58cac03478ac7e9f622d29fe2d20a4384c24c90ab50de2c7b36c", + "sha256:b298013377f51125b3d7f0c86920de4e3e2d4a83731bd5caf1f1e5bddabe7798" + ], + "index": "pypi", + "version": "==2.4.14" + }, + "fs-azureblob": { + "hashes": [ + "sha256:485f9669ac9a212a64391012c2f1689930f408f13e3d00a83a8c4fa8ce97b5c5", + "sha256:812049faff88d465130b6e51ea2cb0d4aa7fa5c4baa274447cdc0bf58a2c76bb" ], "index": "pypi", - "version": "==2.4.13" + "version": "==0.1.0" }, "fs.sshfs": { "hashes": [ - "sha256:cf0c3675c325b4e988a093a4ad76b0eff37889ab57d6e1ea5788266875ac8278", - "sha256:d1e4c8ff2f68ad1cdc6a7f08753d3a2737b590c3eadd62af4b7539960ddf0951" + "sha256:05cb3dac1a932d5c4f44d1c2a43963ae5ff13450af1d6fb9ee6ce0139754ed1a", + "sha256:f9f30a027b6c93bada0527d0a8d8440c2222cf5f3ba2a481138a99c8e572636b" ], "index": "pypi", - "version": "==1.0.0" + "version": "==1.0.1" }, "idna": { "hashes": [ - "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", - "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], "markers": "python_version >= '3'", - "version": "==3.2" + "version": "==3.3" + }, + "isodate": { + "hashes": [ + "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", + "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" + ], + "version": "==0.6.1" + }, + "msrest": { + "hashes": [ + "sha256:72661bc7bedc2dc2040e8f170b6e9ef226ee6d3892e01affd4d26b06474d68d8", + "sha256:c840511c845330e96886011a236440fafc2c9aff7b2df9c0a92041ee2dee3782" + ], + "version": "==0.6.21" }, "networkx": { "hashes": [ @@ -163,66 +203,70 @@ }, "numpy": { "hashes": [ - "sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf", - "sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2", - "sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6", - "sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad", - "sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc", - "sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254", - "sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0", - "sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218", - "sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13", - "sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef", - "sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737", - "sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723", - "sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3", - "sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1", - "sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d", - "sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52", - "sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3", - "sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5", - "sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f", - "sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496", - "sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f", - "sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724", - "sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931", - "sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f", - "sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7", - "sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38", - "sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557", - "sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57", - "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310", - "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419" + "sha256:0cfe07133fd00b27edee5e6385e333e9eeb010607e8a46e1cd673f05f8596595", + "sha256:11a1f3816ea82eed4178102c56281782690ab5993251fdfd75039aad4d20385f", + "sha256:2762331de395739c91f1abb88041f94a080cb1143aeec791b3b223976228af3f", + "sha256:283d9de87c0133ef98f93dfc09fad3fb382f2a15580de75c02b5bb36a5a159a5", + "sha256:3d22662b4b10112c545c91a0741f2436f8ca979ab3d69d03d19322aa970f9695", + "sha256:41388e32e40b41dd56eb37fcaa7488b2b47b0adf77c66154d6b89622c110dfe9", + "sha256:42c16cec1c8cf2728f1d539bd55aaa9d6bb48a7de2f41eb944697293ef65a559", + "sha256:47ee7a839f5885bc0c63a74aabb91f6f40d7d7b639253768c4199b37aede7982", + "sha256:5a311ee4d983c487a0ab546708edbdd759393a3dc9cd30305170149fedd23c88", + "sha256:5dc65644f75a4c2970f21394ad8bea1a844104f0fe01f278631be1c7eae27226", + "sha256:6ed0d073a9c54ac40c41a9c2d53fcc3d4d4ed607670b9e7b0de1ba13b4cbfe6f", + "sha256:76ba7c40e80f9dc815c5e896330700fd6e20814e69da9c1267d65a4d051080f1", + "sha256:818b9be7900e8dc23e013a92779135623476f44a0de58b40c32a15368c01d471", + "sha256:a024181d7aef0004d76fb3bce2a4c9f2e67a609a9e2a6ff2571d30e9976aa383", + "sha256:a955e4128ac36797aaffd49ab44ec74a71c11d6938df83b1285492d277db5397", + "sha256:a97a954a8c2f046d3817c2bce16e3c7e9a9c2afffaf0400f5c16df5172a67c9c", + "sha256:a97e82c39d9856fe7d4f9b86d8a1e66eff99cf3a8b7ba48202f659703d27c46f", + "sha256:b55b953a1bdb465f4dc181758570d321db4ac23005f90ffd2b434cc6609a63dd", + "sha256:bb02929b0d6bfab4c48a79bd805bd7419114606947ec8284476167415171f55b", + "sha256:bece0a4a49e60e472a6d1f70ac6cdea00f9ab80ff01132f96bd970cdd8a9e5a9", + "sha256:e41e8951749c4b5c9a2dc5fdbc1a4eec6ab2a140fdae9b460b0f557eed870f4d", + "sha256:f71d57cc8645f14816ae249407d309be250ad8de93ef61d9709b45a0ddf4050c" ], "index": "pypi", - "version": "==1.21.2" + "version": "==1.22.0" + }, + "oauthlib": { + "hashes": [ + "sha256:42bf6354c2ed8c6acb54d971fce6f88193d97297e18602a3a886603f9d7730cc", + "sha256:8f0215fcc533dd8dd1bee6f4c412d4f0cd7297307d43ac61666389e3bc3198a3" + ], + "markers": "python_version >= '3.6'", + "version": "==3.1.1" }, "pandas": { "hashes": [ - "sha256:272c8cb14aa9793eada6b1ebe81994616e647b5892a370c7135efb2924b701df", - "sha256:3334a5a9eeaca953b9db1b2b165dcdc5180b5011f3bec3a57a3580c9c22eae68", - "sha256:37d63e78e87eb3791da7be4100a65da0383670c2b59e493d9e73098d7a879226", - "sha256:3f5020613c1d8e304840c34aeb171377dc755521bf5e69804991030c2a48aec3", - "sha256:45649503e167d45360aa7c52f18d1591a6d5c70d2f3a26bc90a3297a30ce9a66", - "sha256:49fd2889d8116d7acef0709e4c82b8560a8b22b0f77471391d12c27596e90267", - "sha256:4def2ef2fb7fcd62f2aa51bacb817ee9029e5c8efe42fe527ba21f6a3ddf1a9f", - "sha256:53e2fb11f86f6253bb1df26e3aeab3bf2e000aaa32a953ec394571bec5dc6fd6", - "sha256:629138b7cf81a2e55aa29ce7b04c1cece20485271d1f6c469c6a0c03857db6a4", - "sha256:68408a39a54ebadb9014ee5a4fae27b2fe524317bc80adf56c9ac59e8f8ea431", - "sha256:7326b37de08d42dd3fff5b7ef7691d0fd0bf2428f4ba5a2bdc3b3247e9a52e4c", - "sha256:7557b39c8e86eb0543a17a002ac1ea0f38911c3c17095bc9350d0a65b32d801c", - "sha256:86b16b1b920c4cb27fdd65a2c20258bcd9c794be491290660722bb0ea765054d", - "sha256:a800df4e101b721e94d04c355e611863cc31887f24c0b019572e26518cbbcab6", - "sha256:a9f1b54d7efc9df05320b14a48fb18686f781aa66cc7b47bb62fabfc67a0985c", - "sha256:c399200631db9bd9335d013ec7fce4edb98651035c249d532945c78ad453f23a", - "sha256:e574c2637c9d27f322e911650b36e858c885702c5996eda8a5a60e35e6648cf2", - "sha256:e9bc59855598cb57f68fdabd4897d3ed2bc3a3b3bef7b868a0153c4cd03f3207", - "sha256:ebbed7312547a924df0cbe133ff1250eeb94cdff3c09a794dc991c5621c8c735", - "sha256:ed2f29b4da6f6ae7c68f4b3708d9d9e59fa89b2f9e87c2b64ce055cbd39f729e", - "sha256:f7d84f321674c2f0f31887ee6d5755c54ca1ea5e144d6d54b3bbf566dd9ea0cc" + "sha256:1e4285f5de1012de20ca46b188ccf33521bff61ba5c5ebd78b4fb28e5416a9f1", + "sha256:2651d75b9a167cc8cc572cf787ab512d16e316ae00ba81874b560586fa1325e0", + "sha256:2c21778a688d3712d35710501f8001cdbf96eb70a7c587a3d5613573299fdca6", + "sha256:32e1a26d5ade11b547721a72f9bfc4bd113396947606e00d5b4a5b79b3dcb006", + "sha256:3345343206546545bc26a05b4602b6a24385b5ec7c75cb6059599e3d56831da2", + "sha256:344295811e67f8200de2390093aeb3c8309f5648951b684d8db7eee7d1c81fb7", + "sha256:37f06b59e5bc05711a518aa10beaec10942188dccb48918bb5ae602ccbc9f1a0", + "sha256:552020bf83b7f9033b57cbae65589c01e7ef1544416122da0c79140c93288f56", + "sha256:5cce0c6bbeb266b0e39e35176ee615ce3585233092f685b6a82362523e59e5b4", + "sha256:5f261553a1e9c65b7a310302b9dbac31cf0049a51695c14ebe04e4bfd4a96f02", + "sha256:60a8c055d58873ad81cae290d974d13dd479b82cbb975c3e1fa2cf1920715296", + "sha256:62d5b5ce965bae78f12c1c0df0d387899dd4211ec0bdc52822373f13a3a022b9", + "sha256:7d28a3c65463fd0d0ba8bbb7696b23073efee0510783340a44b08f5e96ffce0c", + "sha256:8025750767e138320b15ca16d70d5cdc1886e8f9cc56652d89735c016cd8aea6", + "sha256:8b6dbec5f3e6d5dc80dcfee250e0a2a652b3f28663492f7dab9a24416a48ac39", + "sha256:a395692046fd8ce1edb4c6295c35184ae0c2bbe787ecbe384251da609e27edcb", + "sha256:a62949c626dd0ef7de11de34b44c6475db76995c2064e2d99c6498c3dba7fe58", + "sha256:aaf183a615ad790801fa3cf2fa450e5b6d23a54684fe386f7e3208f8b9bfbef6", + "sha256:adfeb11be2d54f275142c8ba9bf67acee771b7186a5745249c7d5a06c670136b", + "sha256:b6b87b2fb39e6383ca28e2829cddef1d9fc9e27e55ad91ca9c435572cdba51bf", + "sha256:bd971a3f08b745a75a86c00b97f3007c2ea175951286cdda6abe543e687e5f2f", + "sha256:c69406a2808ba6cf580c2255bcf260b3f214d2664a3a4197d0e640f573b46fd3", + "sha256:d3bc49af96cd6285030a64779de5b3688633a07eb75c124b0747134a63f4c05f", + "sha256:fd541ab09e1f80a2a1760032d665f6e032d8e44055d602d65eeea6e6e85498cb", + "sha256:fe95bae4e2d579812865db2212bb733144e34d0c6785c0685329e5b60fcb85dd" ], "index": "pypi", - "version": "==1.3.3" + "version": "==1.3.5" }, "paramiko": { "hashes": [ @@ -242,11 +286,11 @@ }, "pycparser": { "hashes": [ - "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", - "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.20" + "version": "==2.21" }, "pynacl": { "hashes": [ @@ -282,43 +326,55 @@ }, "pytz": { "hashes": [ - "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da", - "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798" + "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c", + "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326" ], - "version": "==2021.1" + "version": "==2021.3" }, "requests": { "hashes": [ - "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", - "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" + "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", + "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" ], "index": "pypi", - "version": "==2.26.0" + "version": "==2.27.1" + }, + "requests-oauthlib": { + "hashes": [ + "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d", + "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a", + "sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc" + ], + "version": "==1.3.0" }, "scipy": { "hashes": [ - "sha256:2a0eeaab01258e0870c4022a6cd329aef3b7c6c2b606bd7cf7bb2ba9820ae561", - "sha256:3304bd5bc32e00954ac4b3f4cc382ca8824719bf348aacbec6347337d6b125fe", - "sha256:3f52470e0548cdb74fb8ddf06773ffdcca7c97550f903b1c51312ec19243a7f7", - "sha256:4729b41a4cdaf4cd011aeac816b532f990bdf97710cef59149d3e293115cf467", - "sha256:4ee952f39a4a4c7ba775a32b664b1f4b74818548b65f765987adc14bb78f5802", - "sha256:611f9cb459d0707dd8e4de0c96f86e93f61aac7475fcb225e9ec71fecdc5cebf", - "sha256:6b47d5fa7ea651054362561a28b1ccc8da9368a39514c1bbf6c0977a1c376764", - "sha256:71cfc96297617eab911e22216e8a8597703202e95636d9406df9af5c2ac99a2b", - "sha256:787749110a23502031fb1643c55a2236c99c6b989cca703ea2114d65e21728ef", - "sha256:90c07ba5f34f33299a428b0d4fa24c30d2ceba44d63f8385b2b05be460819fcb", - "sha256:a496b42dbcd04ea9924f5e92be63af3d8e0f43a274b769bfaca0a297327d54ee", - "sha256:bc61e3e5ff92d2f32bb263621d54a9cff5e3f7c420af3d1fa122ce2529de2bd9", - "sha256:c9951e3746b68974125e5e3445008a4163dd6d20ae0bbdae22b38cb8951dc11b", - "sha256:d1388fbac9dd591ea630da75c455f4cc637a7ca5ecb31a6b6cef430914749cde", - "sha256:d13f31457f2216e5705304d9f28e2826edf75487410a57aa99263fa4ffd792c2", - "sha256:d648aa85dd5074b1ed83008ae987c3fbb53d68af619fce1dee231f4d8bd40e2f", - "sha256:da9c6b336e540def0b7fd65603da8abeb306c5fc9a5f4238665cbbb5ff95cf58", - "sha256:e101bceeb9e65a90dadbc5ca31283403a2d4667b9c178db29109750568e8d112", - "sha256:efdd3825d54c58df2cc394366ca4b9166cf940a0ebddeb87b6c10053deb625ea" + "sha256:28ae39a37a00f7279c1f200f0bbecd01837d489025951732556b271b38ce84f0", + "sha256:2e5a776dc7105b53204b3c9b948ee32c0325664279cccf87f7285d3cf6f4c06b", + "sha256:301ebcd624f6949e46f8f573564168fe59bdd6b56ce81df6a82f6e00db8e1d98", + "sha256:3b1d2af8a14bc393b6c763802e1e228f0f8918c29e01cef4542a712188303e78", + "sha256:3bbc009781112537a5e7969abebff60a4cbc641f4394c29cf08047f2826d3e77", + "sha256:4035b0f70d1bdbfe143005bb1033938529f684c0b93b4a90c2f59bbd88bcd0d3", + "sha256:55863c865d651aa2e49d72778db976c0d38db7ddf9ec26ee869e56c1a8c799d1", + "sha256:5b39372472fb79ea4a35684bb2f3a573e6ffe805179a07965a50d2df10e17ce3", + "sha256:62164c89c57e7bc8ca931e29edf25f534828e2cc4b7eb254f34eaa7c395dae83", + "sha256:63bea96f299c549febe7039e8c80942217e3e39851fb2258fae430b722610573", + "sha256:739ee3f6688c96516248f88725bbe1f241f3e0ab708f5eda98e2a9fe5cf38fba", + "sha256:7ae0f1c0bce0188bcb93666f7ffdc4dfcedd071e60a92fd6107bc6061cab79c6", + "sha256:83b7a82a56ccaf8b5accd8f4988898412806aecb3f52fc505eeccaee9d8cb889", + "sha256:874588522c8c97ec285effbd0bcd7c9de03e918674a9decc5779cc85de408e0d", + "sha256:8db99b6c017ab971b04a0781103a31ce745d4f0ac2b7db999523d4a94549ae15", + "sha256:9439aa6e8076df746e6d026328ba01a2160e886f185926e8915849f1e124422f", + "sha256:94a33efa21cffc5c3f8d416c0c1c79914019b9e3a82b5461176e4dc1c42218ea", + "sha256:aa31ae8d8cf0abba07bc795f75b1aacf46b6136be22eefb4435040386f0f2bec", + "sha256:bb3931cc7281fc0e5cc94f4cdc316a8628206bac24ed0ccd999be09266bc9970", + "sha256:d6ccda8592a30120ea4c3ab669a10ab6e0c45284df9066b6583165d5092e75a7", + "sha256:d73b13eb0452c178f946b4db60b27e400225df02e926609652ed67798054e77d", + "sha256:da36fd4d686ac936b7cd6b8348dfd4aebfd42650fcf15af62e5d9b582a9e7cb6", + "sha256:f3b64d3456c4f52e665019e08347c86618b702c190a9130df3c6e0a9f43eca78" ], "index": "pypi", - "version": "==1.7.1" + "version": "==1.8.0rc2" }, "six": { "hashes": [ @@ -348,63 +404,83 @@ "develop": { "attrs": { "hashes": [ - "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1", - "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb" + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==21.2.0" + "version": "==21.4.0" }, "black": { "hashes": [ - "sha256:380f1b5da05e5a1429225676655dddb96f5ae8c75bdf91e53d798871b902a115", - "sha256:7de4cfc7eb6b710de325712d40125689101d21d25283eed7e9998722cf10eb91" + "sha256:77b80f693a569e2e527958459634f18df9b0ba2625ba4e0c2d5da5be42e6f2b3", + "sha256:a615e69ae185e08fdd73e4715e260e2479c861b5740057fde6e8b4e3b7dd589f" ], "index": "pypi", - "version": "==21.9b0" + "version": "==21.12b0" }, "click": { "hashes": [ - "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a", - "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6" + "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", + "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" ], "markers": "python_version >= '3.6'", - "version": "==8.0.1" + "version": "==8.0.3" }, "coverage": { - "hashes": [ - "sha256:16db4173575901db8f3e6cc05e50fe19c7849b0256f6dc2e0979485184053417", - "sha256:18183948d5480e2ae30ad67edddf748149c778592b7e4ee649c058d5de2dcbb1", - "sha256:22888d3ce1b6fa1125f0be1602d8c634e00e7ec3a87bdb594ad87bde0b00b2b6", - "sha256:23c1611471cbfa2ac0e283862a76a333c13e5e7c4d499feb9919a5f52884610e", - "sha256:2dcc6d62b69a82759e5dddd788e09dd329124e493e62d92cfd01c0b918d7e511", - "sha256:40e30139113b141c238620b700aa5bd5c1b3a7b29ae47398936ff1c9166109d9", - "sha256:4528368196a90f11b70fb5668c13d92e88ba795eb4d37aab5855fd0479db417b", - "sha256:4cbdc51fc8c00ec6e53b30221d5757034aecf9839761bf97eaec0db7f0ff4955", - "sha256:6a585ba4087cc1fb5bfe34d1ecaaee183b854427992be2b42f1722ba8289fa82", - "sha256:79c136327e90ee46a2b3094263df94da5212890d6145678741eb805d79714971", - "sha256:7beec4df7542cf681356ef243fee3bf948775fc0d125bdcad3508e834229e07d", - "sha256:8394626a07e0a1b3695a16a4548d32e7259e00817d4bab1ef8172a1bd82a724e", - "sha256:84a1000f622d1df8824cd1ac629aa8392679c5c4de3f0de9e6889373f99ff3a0", - "sha256:91cd79f0f2996a4de737de89fdcbcd379a5bfd7b15129378ad1e5fc234e58d33", - "sha256:951e8d7bc98bceb61fc4fb426966fae854160301c0f8cd0945c62f2504f68615", - "sha256:95d2293d6a60da8952c675050231c02c9f4f1c1b9cf916315173e921d137d683", - "sha256:9981294b131023e63061ba88f4498fe27b9b15d908079d1866ee66a63d6e793f", - "sha256:a8826f6ecf079cb648534790ba59218a64e12a59bf2cd9ff00199abb39864a79", - "sha256:c1630e847ae0a2a366f18ddc3e017b69f80d729e95830579c61b5f9e9b94b91e", - "sha256:c6f46d5bbec8fe1ff25215356e819528a90d84b2801703514746b665742f1cd2", - "sha256:c8099c7033fb1ca73ac2246c3e52f45dd6a9c3826c59b3b5ad94e5be4e08d99b", - "sha256:ceb872b89c6461d4365be5f8fbf14f867be6b5217760980de7e014e54648f8ef", - "sha256:d6fbe69d52628b3e8a144265fd134f5da07cf287a00cf529730ae10380d315b2", - "sha256:da7de6e4162c69cc03cc56b7d051ae11147ac30872ff57df4ba4cac6d70ce5d9", - "sha256:ddb2287f66500ac57b24cce60341074b148977b74cd20eca755f95262928086f", - "sha256:e6a4260f0abf90c023b4f838905f645695b31666b76837152e2befad3d1ef5d6", - "sha256:e97b387f2744762b9984639b59abd7abb46ea6ae2ea24cb7c07893612328559b", - "sha256:ea784c96ca3b94912176d7adc9c4bb7d1988f36a0223a9ac128f4c834775202c", - "sha256:f0b250a03891255feb3ae69ac29d05cf9a62f5869bb8bac0e7f4968e7274efac", - "sha256:fdaa96733c9cf85491ad406fd78aa16025a1ea468951545b3da7ee133c150c7a" + "extras": [ + "toml" + ], + "hashes": [ + "sha256:01774a2c2c729619760320270e42cd9e797427ecfddd32c2a7b639cdc481f3c0", + "sha256:03b20e52b7d31be571c9c06b74746746d4eb82fc260e594dc662ed48145e9efd", + "sha256:0a7726f74ff63f41e95ed3a89fef002916c828bb5fcae83b505b49d81a066884", + "sha256:1219d760ccfafc03c0822ae2e06e3b1248a8e6d1a70928966bafc6838d3c9e48", + "sha256:13362889b2d46e8d9f97c421539c97c963e34031ab0cb89e8ca83a10cc71ac76", + "sha256:174cf9b4bef0db2e8244f82059a5a72bd47e1d40e71c68ab055425172b16b7d0", + "sha256:17e6c11038d4ed6e8af1407d9e89a2904d573be29d51515f14262d7f10ef0a64", + "sha256:215f8afcc02a24c2d9a10d3790b21054b58d71f4b3c6f055d4bb1b15cecce685", + "sha256:22e60a3ca5acba37d1d4a2ee66e051f5b0e1b9ac950b5b0cf4aa5366eda41d47", + "sha256:2641f803ee9f95b1f387f3e8f3bf28d83d9b69a39e9911e5bfee832bea75240d", + "sha256:276651978c94a8c5672ea60a2656e95a3cce2a3f31e9fb2d5ebd4c215d095840", + "sha256:3f7c17209eef285c86f819ff04a6d4cbee9b33ef05cbcaae4c0b4e8e06b3ec8f", + "sha256:3feac4084291642165c3a0d9eaebedf19ffa505016c4d3db15bfe235718d4971", + "sha256:49dbff64961bc9bdd2289a2bda6a3a5a331964ba5497f694e2cbd540d656dc1c", + "sha256:4e547122ca2d244f7c090fe3f4b5a5861255ff66b7ab6d98f44a0222aaf8671a", + "sha256:5829192582c0ec8ca4a2532407bc14c2f338d9878a10442f5d03804a95fac9de", + "sha256:5d6b09c972ce9200264c35a1d53d43ca55ef61836d9ec60f0d44273a31aa9f17", + "sha256:600617008aa82032ddeace2535626d1bc212dfff32b43989539deda63b3f36e4", + "sha256:619346d57c7126ae49ac95b11b0dc8e36c1dd49d148477461bb66c8cf13bb521", + "sha256:63c424e6f5b4ab1cf1e23a43b12f542b0ec2e54f99ec9f11b75382152981df57", + "sha256:6dbc1536e105adda7a6312c778f15aaabe583b0e9a0b0a324990334fd458c94b", + "sha256:6e1394d24d5938e561fbeaa0cd3d356207579c28bd1792f25a068743f2d5b282", + "sha256:86f2e78b1eff847609b1ca8050c9e1fa3bd44ce755b2ec30e70f2d3ba3844644", + "sha256:8bdfe9ff3a4ea37d17f172ac0dff1e1c383aec17a636b9b35906babc9f0f5475", + "sha256:8e2c35a4c1f269704e90888e56f794e2d9c0262fb0c1b1c8c4ee44d9b9e77b5d", + "sha256:92b8c845527eae547a2a6617d336adc56394050c3ed8a6918683646328fbb6da", + "sha256:9365ed5cce5d0cf2c10afc6add145c5037d3148585b8ae0e77cc1efdd6aa2953", + "sha256:9a29311bd6429be317c1f3fe4bc06c4c5ee45e2fa61b2a19d4d1d6111cb94af2", + "sha256:9a2b5b52be0a8626fcbffd7e689781bf8c2ac01613e77feda93d96184949a98e", + "sha256:a4bdeb0a52d1d04123b41d90a4390b096f3ef38eee35e11f0b22c2d031222c6c", + "sha256:a9c8c4283e17690ff1a7427123ffb428ad6a52ed720d550e299e8291e33184dc", + "sha256:b637c57fdb8be84e91fac60d9325a66a5981f8086c954ea2772efe28425eaf64", + "sha256:bf154ba7ee2fd613eb541c2bc03d3d9ac667080a737449d1a3fb342740eb1a74", + "sha256:c254b03032d5a06de049ce8bca8338a5185f07fb76600afff3c161e053d88617", + "sha256:c332d8f8d448ded473b97fefe4a0983265af21917d8b0cdcb8bb06b2afe632c3", + "sha256:c7912d1526299cb04c88288e148c6c87c0df600eca76efd99d84396cfe00ef1d", + "sha256:cfd9386c1d6f13b37e05a91a8583e802f8059bebfccde61a418c5808dea6bbfa", + "sha256:d5d2033d5db1d58ae2d62f095e1aefb6988af65b4b12cb8987af409587cc0739", + "sha256:dca38a21e4423f3edb821292e97cec7ad38086f84313462098568baedf4331f8", + "sha256:e2cad8093172b7d1595b4ad66f24270808658e11acf43a8f95b41276162eb5b8", + "sha256:e3db840a4dee542e37e09f30859f1612da90e1c5239a6a2498c473183a50e781", + "sha256:edcada2e24ed68f019175c2b2af2a8b481d3d084798b8c20d15d34f5c733fa58", + "sha256:f467bbb837691ab5a8ca359199d3429a11a01e6dfb3d9dcc676dc035ca93c0a9", + "sha256:f506af4f27def639ba45789fa6fde45f9a217da0be05f8910458e4557eed020c", + "sha256:f614fc9956d76d8a88a88bb41ddc12709caa755666f580af3a688899721efecd", + "sha256:f9afb5b746781fc2abce26193d1c817b7eb0e11459510fba65d2bd77fe161d9e", + "sha256:fb8b8ee99b3fffe4fd86f4c81b35a6bf7e4462cba019997af2fe679365db0c49" ], "index": "pypi", - "version": "==6.0b1" + "version": "==6.2" }, "iniconfig": { "hashes": [ @@ -422,11 +498,11 @@ }, "packaging": { "hashes": [ - "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7", - "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14" + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" ], "markers": "python_version >= '3.6'", - "version": "==21.0" + "version": "==21.3" }, "pathspec": { "hashes": [ @@ -437,11 +513,11 @@ }, "platformdirs": { "hashes": [ - "sha256:15b056538719b1c94bdaccb29e5f81879c7f7f0f4a153f46086d155dffcd4f0f", - "sha256:8003ac87717ae2c7ee1ea5a84a1a61e87f3fbd16eb5aadba194ea30a9019f648" + "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca", + "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda" ], - "markers": "python_version >= '3.6'", - "version": "==2.3.0" + "markers": "python_version >= '3.7'", + "version": "==2.4.1" }, "pluggy": { "hashes": [ @@ -453,105 +529,51 @@ }, "py": { "hashes": [ - "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3", - "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a" + "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", + "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.10.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.11.0" }, "pyparsing": { "hashes": [ - "sha256:14e99e14e11a14cadf4d99effd4c5c30a9f46b116551ee69b4e5f8f6004f62d5", - "sha256:61b247121581f50c7988eece6ebb2d87ccc3be46c5552daf910d56e20cf6da75" + "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4", + "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81" ], - "markers": "python_version >= '3.5'", - "version": "==3.0.0rc1" + "markers": "python_version >= '3.6'", + "version": "==3.0.6" }, "pytest": { "hashes": [ - "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89", - "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134" + "sha256:8fc363e0b7407a9397e660ef81e1634e4504faaeb6ad1d2416da4c38d29a0f45", + "sha256:e1af71303d633af3376130b388e028342815cff74d2f3be4aeb22f3fd94325e6" ], "index": "pypi", - "version": "==6.2.5" + "version": "==7.0.0rc1" }, "pytest-cov": { "hashes": [ - "sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a", - "sha256:261ceeb8c227b726249b376b8526b600f38667ee314f910353fa318caa01f4d7" + "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6", + "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470" ], "index": "pypi", - "version": "==2.12.1" - }, - "regex": { - "hashes": [ - "sha256:04f6b9749e335bb0d2f68c707f23bb1773c3fb6ecd10edf0f04df12a8920d468", - "sha256:08d74bfaa4c7731b8dac0a992c63673a2782758f7cfad34cf9c1b9184f911354", - "sha256:0fc1f8f06977c2d4f5e3d3f0d4a08089be783973fc6b6e278bde01f0544ff308", - "sha256:121f4b3185feaade3f85f70294aef3f777199e9b5c0c0245c774ae884b110a2d", - "sha256:1413b5022ed6ac0d504ba425ef02549a57d0f4276de58e3ab7e82437892704fc", - "sha256:1743345e30917e8c574f273f51679c294effba6ad372db1967852f12c76759d8", - "sha256:28fc475f560d8f67cc8767b94db4c9440210f6958495aeae70fac8faec631797", - "sha256:31a99a4796bf5aefc8351e98507b09e1b09115574f7c9dbb9cf2111f7220d2e2", - "sha256:328a1fad67445550b982caa2a2a850da5989fd6595e858f02d04636e7f8b0b13", - "sha256:473858730ef6d6ff7f7d5f19452184cd0caa062a20047f6d6f3e135a4648865d", - "sha256:4cde065ab33bcaab774d84096fae266d9301d1a2f5519d7bd58fc55274afbf7a", - "sha256:5f6a808044faae658f546dd5f525e921de9fa409de7a5570865467f03a626fc0", - "sha256:610b690b406653c84b7cb6091facb3033500ee81089867ee7d59e675f9ca2b73", - "sha256:66256b6391c057305e5ae9209941ef63c33a476b73772ca967d4a2df70520ec1", - "sha256:6eebf512aa90751d5ef6a7c2ac9d60113f32e86e5687326a50d7686e309f66ed", - "sha256:79aef6b5cd41feff359acaf98e040844613ff5298d0d19c455b3d9ae0bc8c35a", - "sha256:808ee5834e06f57978da3e003ad9d6292de69d2bf6263662a1a8ae30788e080b", - "sha256:8e44769068d33e0ea6ccdf4b84d80c5afffe5207aa4d1881a629cf0ef3ec398f", - "sha256:999ad08220467b6ad4bd3dd34e65329dd5d0df9b31e47106105e407954965256", - "sha256:9b006628fe43aa69259ec04ca258d88ed19b64791693df59c422b607b6ece8bb", - "sha256:9d05ad5367c90814099000442b2125535e9d77581855b9bee8780f1b41f2b1a2", - "sha256:a577a21de2ef8059b58f79ff76a4da81c45a75fe0bfb09bc8b7bb4293fa18983", - "sha256:a617593aeacc7a691cc4af4a4410031654f2909053bd8c8e7db837f179a630eb", - "sha256:abb48494d88e8a82601af905143e0de838c776c1241d92021e9256d5515b3645", - "sha256:ac88856a8cbccfc14f1b2d0b829af354cc1743cb375e7f04251ae73b2af6adf8", - "sha256:b4c220a1fe0d2c622493b0a1fd48f8f991998fb447d3cd368033a4b86cf1127a", - "sha256:b844fb09bd9936ed158ff9df0ab601e2045b316b17aa8b931857365ea8586906", - "sha256:bdc178caebd0f338d57ae445ef8e9b737ddf8fbc3ea187603f65aec5b041248f", - "sha256:c206587c83e795d417ed3adc8453a791f6d36b67c81416676cad053b4104152c", - "sha256:c61dcc1cf9fd165127a2853e2c31eb4fb961a4f26b394ac9fe5669c7a6592892", - "sha256:c7cb4c512d2d3b0870e00fbbac2f291d4b4bf2634d59a31176a87afe2777c6f0", - "sha256:d4a332404baa6665b54e5d283b4262f41f2103c255897084ec8f5487ce7b9e8e", - "sha256:d5111d4c843d80202e62b4fdbb4920db1dcee4f9366d6b03294f45ed7b18b42e", - "sha256:e1e8406b895aba6caa63d9fd1b6b1700d7e4825f78ccb1e5260551d168db38ed", - "sha256:e8690ed94481f219a7a967c118abaf71ccc440f69acd583cab721b90eeedb77c", - "sha256:ed283ab3a01d8b53de3a05bfdf4473ae24e43caee7dcb5584e86f3f3e5ab4374", - "sha256:ed4b50355b066796dacdd1cf538f2ce57275d001838f9b132fab80b75e8c84dd", - "sha256:ee329d0387b5b41a5dddbb6243a21cb7896587a651bebb957e2d2bb8b63c0791", - "sha256:f3bf1bc02bc421047bfec3343729c4bbbea42605bcfd6d6bfe2c07ade8b12d2a", - "sha256:f585cbbeecb35f35609edccb95efd95a3e35824cd7752b586503f7e6087303f1", - "sha256:f60667673ff9c249709160529ab39667d1ae9fd38634e006bec95611f632e759" - ], - "version": "==2021.8.28" - }, - "toml": { - "hashes": [ - "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", - "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.10.2" + "version": "==3.0.0" }, "tomli": { "hashes": [ - "sha256:8dd0e9524d6f386271a36b41dbf6c57d8e32fd96fd22b6584679dc569d20899f", - "sha256:a5b75cb6f3968abb47af1b40c1819dc519ea82bcc065776a866e8d74c5ca9442" + "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f", + "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c" ], "markers": "python_version >= '3.6'", - "version": "==1.2.1" + "version": "==1.2.3" }, "typing-extensions": { "hashes": [ - "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e", - "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7", - "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34" + "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e", + "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b" ], - "version": "==3.10.0.2" + "markers": "python_version >= '3.6'", + "version": "==4.0.1" } } } diff --git a/docker-compose.yml b/docker-compose.yml index 126279466..94c016463 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.7' - services: powersimdata: image: powersimdata:latest diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py index 541519446..a2e891587 100644 --- a/powersimdata/data_access/data_access.py +++ b/powersimdata/data_access/data_access.py @@ -5,7 +5,10 @@ import fs as fs2 from fs.tempfs import TempFS -from powersimdata.data_access.profile_helper import ProfileHelper +from powersimdata.data_access.profile_helper import ( + get_profile_version_cloud, + get_profile_version_local, +) from powersimdata.data_access.ssh_fs import WrapSSHFS from powersimdata.utility import server_setup @@ -127,8 +130,8 @@ def get_profile_version(self, grid_model, kind): :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*. :return: (*list*) -- available profile version. """ - blob_version = ProfileHelper.get_profile_version_cloud(grid_model, kind) - local_version = ProfileHelper.get_profile_version_local(grid_model, kind) + blob_version = get_profile_version_cloud(grid_model, kind) + local_version = get_profile_version_local(grid_model, kind) return list(set(blob_version + local_version)) diff --git a/powersimdata/data_access/profile_helper.py b/powersimdata/data_access/profile_helper.py index 20be22147..7a73e60af 100644 --- a/powersimdata/data_access/profile_helper.py +++ b/powersimdata/data_access/profile_helper.py @@ -7,6 +7,39 @@ from powersimdata.utility import server_setup +def _get_profile_version(_fs, kind): + """Returns available raw profiles from the give filesystem + :param fs.base.FS _fs: filesystem instance + :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*. + :return: (*list*) -- available profile version. + """ + matching = [f for f in _fs.listdir(".") if kind in f] + return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching] + + +def get_profile_version_cloud(grid_model, kind): + """Returns available raw profile from blob storage. + + :param str grid_model: grid model. + :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*. + :return: (*list*) -- available profile version. + """ + bfs = fs.open_fs("azblob://besciences@profiles").opendir(f"raw/{grid_model}") + return _get_profile_version(bfs, kind) + + +def get_profile_version_local(grid_model, kind): + """Returns available raw profile from local file. + + :param str grid_model: grid model. + :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*. + :return: (*list*) -- available profile version. + """ + profile_dir = fs.path.join(server_setup.LOCAL_DIR, "raw", grid_model) + lfs = fs.open_fs(profile_dir) + return _get_profile_version(lfs, kind) + + class ProfileHelper: BASE_URL = "https://besciences.blob.core.windows.net/profiles" @@ -52,42 +85,3 @@ def download_file(file_name, from_dir): pbar.update(len(chunk)) return dest - - @staticmethod - def parse_version(grid_model, kind, version): - """Parse available versions from the given spec. - - :param str grid_model: grid model. - :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*. - :param dict version: version information per grid model. - :return: (*list*) -- available profile version. - """ - if grid_model in version and kind in version[grid_model]: - return version[grid_model][kind] - print("No %s profiles available." % kind) - return [] - - @staticmethod - def get_profile_version_cloud(grid_model, kind): - """Returns available raw profile from blob storage. - - :param str grid_model: grid model. - :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*. - :return: (*list*) -- available profile version. - """ - - resp = requests.get(f"{ProfileHelper.BASE_URL}/version.json") - return ProfileHelper.parse_version(grid_model, kind, resp.json()) - - @staticmethod - def get_profile_version_local(grid_model, kind): - """Returns available raw profile from local file. - - :param str grid_model: grid model. - :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*. - :return: (*list*) -- available profile version. - """ - profile_dir = fs.path.join(server_setup.LOCAL_DIR, "raw", grid_model) - lfs = fs.open_fs(profile_dir) - matching = [f for f in lfs.listdir(".") if kind in f] - return [f.lstrip(f"{kind}_").rstrip(".csv") for f in matching] diff --git a/powersimdata/data_access/tests/test_profile_helper.py b/powersimdata/data_access/tests/test_profile_helper.py index ce67b0afd..8ed6825e5 100644 --- a/powersimdata/data_access/tests/test_profile_helper.py +++ b/powersimdata/data_access/tests/test_profile_helper.py @@ -1,20 +1,19 @@ -from powersimdata.data_access.profile_helper import ProfileHelper - - -def test_parse_version_default(): - assert [] == ProfileHelper.parse_version("usa_tamu", "solar", {}) - - -def test_parse_version_missing_key(): - version = {"solar": ["v123"]} - assert [] == ProfileHelper.parse_version("usa_tamu", "solar", version) - - -def test_parse_version(): - expected = ["v123", "v456"] - version = {"usa_tamu": {"solar": expected}} - assert expected == ProfileHelper.parse_version("usa_tamu", "solar", version) - assert [] == ProfileHelper.parse_version("usa_tamu", "hydro", version) +from fs.tempfs import TempFS + +from powersimdata.data_access.profile_helper import ProfileHelper, _get_profile_version + + +def test_get_profile_version(): + with TempFS() as tmp_fs: + tfs = tmp_fs.makedirs("raw/usa_tamu", recreate=True) + tfs.touch("solar_vOct2022.csv") + tfs.touch("foo_v1.0.1.csv") + v_solar = _get_profile_version(tfs, "solar") + v_foo = _get_profile_version(tfs, "foo") + v_missing = _get_profile_version(tfs, "missing") + assert "vOct2022" == v_solar[0] + assert "v1.0.1" == v_foo[0] + assert [] == v_missing def test_get_file_components(): diff --git a/powersimdata/design/generation/cost_curves.py b/powersimdata/design/generation/cost_curves.py index 5152e89fc..a2592d546 100644 --- a/powersimdata/design/generation/cost_curves.py +++ b/powersimdata/design/generation/cost_curves.py @@ -55,7 +55,7 @@ def linearize_gencost(input_grid, num_segments=1): price_label = "f" + str(i + 1) capacity_data = plant.Pmin + power_step * i price_data = ( - quad_term * capacity_data ** 2 + lin_term * capacity_data + const_term + quad_term * capacity_data**2 + lin_term * capacity_data + const_term ) gencost_after.loc[dispatchable_gens, capacity_label] = capacity_data[ dispatchable_gens @@ -76,7 +76,7 @@ def linearize_gencost(input_grid, num_segments=1): nondispatchable_gens, "n" ] power = plant.Pmax - price_data = quad_term * power ** 2 + lin_term * power + const_term + price_data = quad_term * power**2 + lin_term * power + const_term gencost_after.loc[nondispatchable_gens, ["c2", "c1"]] = 0 gencost_after.loc[nondispatchable_gens, "c0"] = price_data[nondispatchable_gens] diff --git a/powersimdata/input/check.py b/powersimdata/input/check.py index 842cc3c3f..939408af6 100644 --- a/powersimdata/input/check.py +++ b/powersimdata/input/check.py @@ -29,6 +29,7 @@ def check_grid(grid): _check_line_voltages, _check_plant_against_gencost, _check_connected_components, + _check_for_loop_branches, ]: try: check(grid, error_messages) @@ -225,6 +226,18 @@ def _check_connected_components(grid, error_messages): ) +def _check_for_loop_branches(grid, error_messages): + """Check whether any branches in a grid have the same start and end bus. + + :param powersimdata.input.grid.Grid grid: grid or grid-like object to check. + :param list error_messages: list, to be appended to with a str if: + there are any branches with the same start and end bus. + """ + if not all(grid.branch.from_bus_id != grid.branch.to_bus_id): + loop_lines = grid.branch.query("from_bus_id == to_bus_id").index # noqa: F841 + error_messages.append(f"This grid contains loop lines: {list(loop_lines)}") + + def _check_grid_models_match(grid1, grid2): """Check whether an object is an internally-consistent Grid object. diff --git a/powersimdata/input/export_data.py b/powersimdata/input/export_data.py index ca7a6c637..5222f4bb5 100644 --- a/powersimdata/input/export_data.py +++ b/powersimdata/input/export_data.py @@ -124,7 +124,7 @@ def export_case_mat(grid, filepath, storage_filepath=None): savemat(filepath, mpc, appendmat=False) -def export_transformed_profile(kind, scenario_info, grid, ct, filepath): +def export_transformed_profile(kind, scenario_info, grid, ct, filepath, slice=True): """Apply transformation to the given kind of profile and save the result locally. :param str kind: which profile to export. This parameter is passed to @@ -135,8 +135,9 @@ def export_transformed_profile(kind, scenario_info, grid, ct, filepath): transformed. :param dict ct: change table. :param str filepath: path to save the result, including the filename + :param bool slice: whether to slice the profiles by the Scenario's time range. """ - tp = TransformProfile(scenario_info, grid, ct) + tp = TransformProfile(scenario_info, grid, ct, slice) profile = tp.get_profile(kind) print(f"Writing scaled {kind} profile to {filepath} on local machine") profile.to_csv(filepath) diff --git a/powersimdata/input/input_data.py b/powersimdata/input/input_data.py index 7ed3deb9c..8a004a5cc 100644 --- a/powersimdata/input/input_data.py +++ b/powersimdata/input/input_data.py @@ -140,20 +140,28 @@ def _read_data(filepath): return data -def get_bus_demand(scenario_info, grid): - """Returns demand profiles by bus. - - :param dict scenario_info: scenario information. - :param powersimdata.input.grid.Grid grid: grid to construct bus demand for. - :return: (*pandas.DataFrame*) -- data frame of demand. +def distribute_demand_from_zones_to_buses(zone_demand, bus): + """Decomposes zone demand to bus demand based on bus 'Pd' column. + + :param pandas.DataFrame zone_demand: demand by zone. Index is timestamp, columns are + zone IDs, values are zone demand (MW). + :param pandas.DataFrame bus: table of bus data, containing at least 'zone_id' and + 'Pd' columns. + :return: (*pandas.DataFrame*) -- data frame of demand. Index is timestamp, columns + are bus IDs, values are bus demand (MW). + :raises ValueError: if the columns of ``zone_demand`` don't match the set of zone + IDs within the 'zone_id' column of ``bus``. """ - bus = grid.bus.copy() - demand = InputData().get_data(scenario_info, "demand")[bus.zone_id.unique()] - bus["zone_Pd"] = bus.groupby("zone_id")["Pd"].transform("sum") - bus["zone_share"] = bus["Pd"] / bus["zone_Pd"] - zone_bus_shares = pd.DataFrame( - {z: bus.groupby("zone_id").get_group(z).zone_share for z in demand.columns} - ).fillna(0) - bus_demand = demand.dot(zone_bus_shares.T) + if set(bus["zone_id"].unique()) != set(zone_demand.columns): + raise ValueError("zones don't match between zone_demand and bus dataframes") + grouped_buses = bus.groupby("zone_id") + bus_zone_pd = grouped_buses["Pd"].transform("sum") + bus_zone_share = pd.concat( + [pd.Series(bus["Pd"] / bus_zone_pd, name="zone_share"), bus["zone_id"]], axis=1 + ) + zone_bus_shares = bus_zone_share.pivot_table( + index="bus_id", columns="zone_id", values="zone_share", fill_value=0 + ) + bus_demand = zone_demand.dot(zone_bus_shares.T) return bus_demand diff --git a/powersimdata/input/transform_profile.py b/powersimdata/input/transform_profile.py index fbc09a752..0949b053d 100644 --- a/powersimdata/input/transform_profile.py +++ b/powersimdata/input/transform_profile.py @@ -6,16 +6,20 @@ class TransformProfile: """Transform profile according to operations listed in change table.""" - def __init__(self, scenario_info, grid, ct): + _default_dates = {"start_date": "2016-01-01 00:00", "end_date": "2016-12-31 23:00"} + + def __init__(self, scenario_info, grid, ct, slice=True): """Constructor. :param dict scenario_info: scenario information. :param powersimdata.input.grid.Grid grid: a Grid object previously transformed. :param dict ct: change table. + :param bool slice: whether to slice the profiles by the Scenario's time range. """ + self.slice = slice self._input_data = InputData() - self.scenario_info = scenario_info + self.scenario_info = {**self._default_dates, **scenario_info} self.ct = copy.deepcopy(ct) self.grid = copy.deepcopy(grid) @@ -113,6 +117,17 @@ def _get_demand_profile(self): demand.loc[:, key] *= value return demand + def _slice_df(self, df): + """Return dataframe, sliced by the times specified in scenario_info if and only + if ``self.slice`` = True. + + :param pandas.DataFrame df: data frame to be sliced. + :return: (*pandas.DataFrame*) -- sliced data frame. + """ + if not self.slice: + return df + return df.loc[self.scenario_info["start_date"] : self.scenario_info["end_date"]] + def get_profile(self, name): """Return profile. @@ -125,6 +140,6 @@ def get_profile(self, name): if name not in possible: raise ValueError("Choose from %s" % " | ".join(possible)) elif name == "demand": - return self._get_demand_profile() + return self._slice_df(self._get_demand_profile()) else: - return self._get_renewable_profile(name) + return self._slice_df(self._get_renewable_profile(name)) diff --git a/powersimdata/output/output_data.py b/powersimdata/output/output_data.py index 9f51d0ef4..af7a5cabd 100644 --- a/powersimdata/output/output_data.py +++ b/powersimdata/output/output_data.py @@ -6,7 +6,8 @@ from scipy.sparse import coo_matrix from powersimdata.data_access.context import Context -from powersimdata.input.input_data import get_bus_demand +from powersimdata.input.input_data import distribute_demand_from_zones_to_buses +from powersimdata.input.transform_profile import TransformProfile from powersimdata.utility import server_setup @@ -76,11 +77,12 @@ def _check_field(field_name): raise ValueError("Only %s data can be loaded" % " | ".join(possible)) -def construct_load_shed(scenario_info, grid, infeasibilities=None): +def construct_load_shed(scenario_info, grid, ct, infeasibilities=None): """Constructs load_shed dataframe from relevant scenario/grid data. :param dict scenario_info: info attribute of Scenario object. :param powersimdata.input.grid.Grid grid: grid to construct load_shed for. + :param dict ct: ChangeTable dictionary. :param dict/None infeasibilities: dictionary of {interval (int): load shed percentage (int)}, or None. :return: (*pandas.DataFrame*) -- data frame of load_shed. @@ -95,7 +97,8 @@ def construct_load_shed(scenario_info, grid, infeasibilities=None): load_shed = pd.DataFrame.sparse.from_spmatrix(load_shed_data) else: print("Infeasibilities, constructing DataFrame") - bus_demand = get_bus_demand(scenario_info, grid) + zone_demand = TransformProfile(scenario_info, grid, ct) + bus_demand = distribute_demand_from_zones_to_buses(zone_demand, grid.bus) load_shed = np.zeros((len(hours), len(buses))) # Convert '24H' to 24 interval = int(scenario_info["interval"][:-1]) diff --git a/powersimdata/scenario/create.py b/powersimdata/scenario/create.py index 0bac3a301..f89f1b6d8 100644 --- a/powersimdata/scenario/create.py +++ b/powersimdata/scenario/create.py @@ -7,7 +7,10 @@ from powersimdata.input.change_table import ChangeTable from powersimdata.input.grid import Grid -from powersimdata.input.input_data import InputData, get_bus_demand +from powersimdata.input.input_data import ( + InputData, + distribute_demand_from_zones_to_buses, +) from powersimdata.input.transform_grid import TransformGrid from powersimdata.input.transform_profile import TransformProfile from powersimdata.network.model import ModelImmutables @@ -86,8 +89,9 @@ def get_bus_demand(self): :return: (*pandas.DataFrame*) -- data frame of demand (hour, bus). """ self._update_scenario_info() + demand = self.get_demand() grid = self.get_grid() - return get_bus_demand(self._scenario_info, grid) + return distribute_demand_from_zones_to_buses(demand, grid.bus) def create_scenario(self): """Creates scenario.""" diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py index 46aeb41cf..ffd9b1539 100644 --- a/powersimdata/scenario/execute.py +++ b/powersimdata/scenario/execute.py @@ -217,17 +217,18 @@ def prepare_mpc_file(self): storage_file_name, self.REL_TMP_DIR, change_name_to="case_storage.mat" ) - def prepare_profile(self, kind, profile_as=None): + def prepare_profile(self, kind, profile_as=None, slice=False): """Prepares profile for simulation. :param kind: one of *demand*, *'hydro'*, *'solar'* or *'wind'*. :param int/str profile_as: if given, copy profile from this scenario. + :param bool slice: whether to slice the profiles by the Scenario's time range. """ if profile_as is None: file_name = "%s_%s.csv" % (self.scenario_id, kind) filepath = os.path.join(server_setup.LOCAL_DIR, file_name) export_transformed_profile( - kind, self._scenario_info, self.grid, self.ct, filepath + kind, self._scenario_info, self.grid, self.ct, filepath, slice ) self._data_access.move_to( diff --git a/powersimdata/scenario/ready.py b/powersimdata/scenario/ready.py index 12ce38101..4d8178fd5 100644 --- a/powersimdata/scenario/ready.py +++ b/powersimdata/scenario/ready.py @@ -1,7 +1,7 @@ import copy from powersimdata.input.grid import Grid -from powersimdata.input.input_data import get_bus_demand +from powersimdata.input.input_data import distribute_demand_from_zones_to_buses from powersimdata.input.transform_profile import TransformProfile from powersimdata.scenario.state import State @@ -120,5 +120,6 @@ def get_bus_demand(self): :return: (*pandas.DataFrame*) -- data frame of demand (hour, bus). """ + zone_demand = self.get_demand() grid = self.get_grid() - return get_bus_demand(self._scenario_info, grid) + return distribute_demand_from_zones_to_buses(zone_demand, grid.bus) diff --git a/powersimdata/scenario/tests/test_create.py b/powersimdata/scenario/tests/test_create.py index 37cd274e3..d60542c2f 100644 --- a/powersimdata/scenario/tests/test_create.py +++ b/powersimdata/scenario/tests/test_create.py @@ -1,12 +1,42 @@ import pytest +from pandas.testing import assert_series_equal from powersimdata.scenario.scenario import Scenario @pytest.mark.integration -@pytest.mark.ssh -def test_get_bus_demand(): +def test_get_demand_and_get_bus_demand(): scenario = Scenario("") - scenario.state.set_grid(interconnect="Texas") - scenario.state.builder.set_base_profile("demand", "vJan2021") - scenario.state.get_bus_demand() + scenario.set_grid(interconnect="Texas") + # Before we set the profile version, we should get errors trying to access + with pytest.raises(Exception): + scenario.get_bus_demand() + with pytest.raises(Exception): + scenario.get_demand() + # After we set the profile version, we should get the right len (default full year) + scenario.set_base_profile("demand", "vJan2021") + assert len(scenario.get_bus_demand()) == 8784 + scenario.set_time("2016-01-01 00:00", "2016-01-03 23:00", "24H") + demand = scenario.get_demand() + bus_demand = scenario.get_bus_demand() + assert len(demand) == 72 + assert len(bus_demand) == 72 + assert_series_equal(demand.sum(axis=1), bus_demand.sum(axis=1)) + unscaled_total_demand = demand.sum().sum() + scenario.change_table.scale_demand(zone_id={301: 1.5}) + new_demand = scenario.get_demand() + new_bus_demand = scenario.get_bus_demand() + assert_series_equal(new_demand.sum(axis=1), new_bus_demand.sum(axis=1)) + assert new_demand.sum().sum() > unscaled_total_demand + + +@pytest.mark.integration +def test_get_solar(): + scenario = Scenario("") + scenario.set_grid(interconnect="Texas") + with pytest.raises(Exception): + scenario.get_solar() + scenario.set_base_profile("solar", "vJan2021") + assert len(scenario.get_solar()) == 8784 + scenario.set_time("2016-01-01 00:00", "2016-01-03 23:00", "24H") + assert len(scenario.get_solar()) == 72 diff --git a/powersimdata/tests/mock_analyze.py b/powersimdata/tests/mock_analyze.py index 5174edf4e..8447e9486 100644 --- a/powersimdata/tests/mock_analyze.py +++ b/powersimdata/tests/mock_analyze.py @@ -43,6 +43,7 @@ class MockAnalyze: "get_congl", "get_congu", "get_ct", + "get_bus_demand", "get_demand", "get_grid", "get_lmp", @@ -63,6 +64,7 @@ def __init__( congu=None, ct=None, demand=None, + bus_demand=None, lmp=None, pf=None, pg=None, @@ -79,6 +81,7 @@ def __init__( self.congu = _ensure_ts_index(congu) self.ct = ct if ct is not None else {} self.demand = _ensure_ts_index(demand) + self.bus_demand = _ensure_ts_index(bus_demand) self.lmp = _ensure_ts_index(lmp) self.pf = _ensure_ts_index(pf) self.dcline_pf = _ensure_ts_index(dcline_pf) @@ -114,6 +117,12 @@ def get_demand(self, original=None): """ return self.demand + def get_bus_demand(self): + """Get bus demand. + :return: (pandas.DataFrame) -- dummy bus demand + """ + return self.bus_demand + def get_grid(self): """Get grid. :return: (MockGrid) -- mock grid diff --git a/requirements.txt b/requirements.txt index 2d0f4e15f..f6dff7235 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ coverage pytest-cov fs fs.sshfs +fs-azureblob diff --git a/setup.cfg b/setup.cfg index 3afca991d..321e4fced 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = powersimdata -version = 0.4.5 +version = 0.5.0 author = Breakthrough Energy author_email = sciences@breakthroughenergy.org description = Power Simulation Data @@ -13,14 +13,14 @@ classifiers = License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 [options] zip_safe = False packages = find: -python_requires = >=3.7 +python_requires = >=3.8 install_requires = networkx numpy @@ -31,6 +31,7 @@ install_requires = requests fs fs.sshfs + fs-azureblob [options.package_data] powersimdata =