Release 1.7.12, Merge pull request #355 from sentinel-hub/develop

Release 1.7.12

.github/workflows/ci_action.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - uses: pre-commit/[email protected]
         with:
@@ -46,7 +46,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
           cache: pip # caching the entire environment is faster when cache exists but slower for cache creation
 
       - name: Install packages

.pre-commit-config.yaml

@@ -20,18 +20,18 @@ repos:
         types_or: [json]
 
   - repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.8.0
     hooks:
       - id: black
         language_version: python3
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.4.9"
+    rev: "v0.6.8"
     hooks:
       - id: ruff
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.8.5
+    rev: 1.8.7
     hooks:
       - id: nbqa-black
       - id: nbqa-ruff

CHANGELOG.md

@@ -1,3 +1,9 @@
+## [Version 1.7.12] - 2024-09-27
+
+- Remove `aws_profile` from storage manager schema.
+- Remove `numpy<2` restriction.
+
+
 ## [Version 1.7.11] - 2024-06-19
 
 - `BatchDownloadPipeline` retries if the first connection to batch-id specific endpoints fails with a 404.

eogrow/__init__.py

@@ -1,3 +1,3 @@
 """The main module of the eo-grow package."""
 
-__version__ = "1.7.11"
+__version__ = "1.7.12"

eogrow/core/storage.py

@@ -2,7 +2,8 @@
 
 from __future__ import annotations
 
-from typing import Any, ClassVar, Dict, Literal, Optional
+import os
+from typing import Any, ClassVar, Dict, Literal
 
 import fs
 from pydantic import BaseSettings, Field
@@ -24,13 +25,6 @@ class Schema(ManagerSchema, BaseSettings):
                 "If on AWS, the path must be prefixed with s3://."
             ),
         )
-        aws_profile: Optional[str] = Field(
-            env="AWS_PROFILE",
-            description=(
-                "The AWS profile with credentials needed to access the S3 buckets. In case the profile isn't specified"
-                " with a parameter it can be read from an environmental variable."
-            ),
-        )
         filesystem_kwargs: Dict[str, Any] = Field(
             default_factory=dict, description="Optional kwargs to be passed on to FS specs."
         )
@@ -66,8 +60,9 @@ def _prepare_sh_config(self) -> SHConfig:
         will show a warning and return a config without AWS credentials."""
         sh_config = SHConfig()
 
-        if self.is_on_s3() and self.config.aws_profile:
-            sh_config = get_aws_credentials(aws_profile=self.config.aws_profile, config=sh_config)
+        aws_profile = os.getenv("AWS_PROFILE")
+        if self.is_on_s3() and aws_profile is not None:
+            sh_config = get_aws_credentials(aws_profile=aws_profile, config=sh_config)
 
         return sh_config
 

pyproject.toml

@@ -45,10 +45,12 @@ dependencies = [
     "colorlog",
     "boto3<=1.29.6",
     "eo-learn[VISUALIZATION]>=1.5.0",
-    "fiona",
+    "fiona>=1.8.18; python_version>='3.9'",
+    "fiona>=1.8.18,<1.10; python_version<'3.9'",
     "fs>=2.2.0",
-    "geopandas>=0.8.1",
-    "numpy<2",
+    "geopandas>=0.14.4,<1; python_version>='3.9'",
+    "geopandas>=0.11.0,<1; python_version<'3.9'",
+    "numpy",
     "opencv-python-headless",
     "pandas",
     "pydantic>=1.8.0, <2.0",
@@ -77,6 +79,7 @@ dev = [
     "build",
     "deepdiff",
     "fs_s3fs",
+    "numpy>=2.0.0; python_version>='3.9'",
     "moto[s3]>=5.0.0",
     "mypy>=0.990",
     "pre-commit",
@@ -165,7 +168,7 @@ lint.ignore = [
     "PT011", # complains for `pytest.raises(ValueError)` but we use it a lot
 ]
 lint.per-file-ignores = { "__init__.py" = ["F401"] }
-exclude = [".git", "__pycache__", "build", "dist"]
+exclude = [".git", "__pycache__", "build", "dist", "*.ipynb"]
 
 
 [tool.ruff.lint.isort]

tests/core/area/test_batch.py

@@ -80,7 +80,7 @@ def test_no_batch_id_error(storage, area_config):
         manager.get_grid()
 
 
-@pytest.mark.integration()
+@pytest.mark.integration
 @pytest.mark.usefixtures("configured_requests_mock")
 def test_grid(storage, area_config):
     manager = BatchAreaManager.from_raw_config(area_config, storage)

tests/core/test_storage.py

@@ -1,8 +1,6 @@
 import os
-from typing import Optional
 
 import pytest
-from botocore.exceptions import ProfileNotFound
 from fs.osfs import OSFS
 from fs_s3fs import S3FS
 
@@ -67,41 +65,6 @@ def test_get_custom_folder(local_storage_manager: StorageManager, project_folder
     assert local_storage_manager.get_folder("eopatches", full_path=True) == abs_path
 
 
-@pytest.mark.parametrize("config_profile", [None, "", "nonexistent-config-profile"])
-@pytest.mark.parametrize("env_profile", [None, "", "nonexistent-env-profile"])
-def test_aws_profile(aws_storage_config: RawConfig, config_profile: Optional[str], env_profile: Optional[str]):
-    """Checks different combinations of profile being set with a config parameter and environmental variable. Checks
-    also that config parameter takes priority over environmental variable.
-
-    In the first step of this test, we add given profile name parameters into the config dictionary and into the
-    dictionary of environmental variables. Note that if profile name is `None`, we instead remove the parameter from a
-    dictionary altogether.
-    """
-
-    for parameter_key, parameter_value, config_dict in [
-        ("aws_profile", config_profile, aws_storage_config),
-        ("AWS_PROFILE", env_profile, os.environ),
-    ]:
-        if parameter_value is not None:
-            config_dict[parameter_key] = parameter_value
-        elif parameter_key in config_dict:
-            del config_dict[parameter_key]
-
-    try:
-        expected_profile = config_profile if config_profile is not None else env_profile
-        if expected_profile:
-            with pytest.raises(ProfileNotFound) as exception_info:
-                StorageManager.from_raw_config(aws_storage_config)
-
-            assert str(exception_info.value) == f"The config profile ({expected_profile}) could not be found"
-        else:
-            storage = StorageManager.from_raw_config(aws_storage_config)
-            assert storage.config.aws_profile == expected_profile
-    finally:
-        if "AWS_PROFILE" in os.environ:
-            del os.environ["AWS_PROFILE"]
-
-
 @pytest.mark.parametrize(
     "config",
     [

tests/pipelines/test_byoc.py

@@ -70,7 +70,7 @@ def _get_tile_cover_geometry_mock(_: str) -> Geometry:
     return pipeline, relevant_requests
 
 
-@pytest.mark.chain()
+@pytest.mark.chain
 @pytest.mark.parametrize(("preparation_config", "config"), [("prepare_lulc_data", "ingest_lulc")])
 @pytest.mark.order(after=["test_rasterize.py::test_rasterize_feature_with_resolution"])
 def test_timeless_byoc(config_and_stats_paths, preparation_config, config, configured_requests_mock):

tests/pipelines/test_download.py

@@ -5,7 +5,7 @@
 pytestmark = pytest.mark.integration
 
 
-@pytest.mark.chain()
+@pytest.mark.chain
 @pytest.mark.order(before=["test_download_pipeline"])
 @pytest.mark.usefixtures("storage")
 def test_preparation():

tests/pipelines/test_import_vector.py

@@ -5,7 +5,7 @@
 pytestmark = pytest.mark.integration
 
 
-@pytest.mark.chain()
+@pytest.mark.chain
 @pytest.mark.parametrize("experiment_name", ["import_vector", "import_vector_temporal"])
 def test_import_vector_pipeline(config_and_stats_paths, experiment_name):
     config_path, stats_path = config_and_stats_paths("import_vector", experiment_name)

tests/pipelines/test_merge_samples.py

@@ -5,7 +5,7 @@
 pytestmark = pytest.mark.integration
 
 
-@pytest.mark.chain()
+@pytest.mark.chain
 @pytest.mark.order(after="test_features.py::test_features_pipeline")
 def test_merge_samples_pipeline(config_and_stats_paths):
     config_path, stats_path = config_and_stats_paths("merge_samples", "merge_features_samples")

tests/pipelines/test_rasterize.py

@@ -12,7 +12,7 @@ def test_rasterize_file(config_and_stats_paths, experiment_name):
     compare_content(output_path, stats_path)
 
 
-@pytest.mark.chain()
+@pytest.mark.chain
 @pytest.mark.order(before="test_rasterize_feature_with_resolution")
 @pytest.mark.parametrize(("preparation_config", "config"), [("load_crops_vector_data", "rasterize_feature_with_shape")])
 def test_rasterize_feature_with_shape(config_and_stats_paths, preparation_config, config):
@@ -24,7 +24,7 @@ def test_rasterize_feature_with_shape(config_and_stats_paths, preparation_config
     compare_content(output_path, stats_path)
 
 
-@pytest.mark.chain()
+@pytest.mark.chain
 @pytest.mark.parametrize(
     ("preparation_config", "config"), [("load_lulc_vector_data", "rasterize_feature_with_resolution")]
 )

tests/pipelines/test_testing.py

@@ -5,7 +5,7 @@
 pytestmark = pytest.mark.integration
 
 
-@pytest.mark.chain()
+@pytest.mark.chain
 @pytest.mark.parametrize("experiment_name", ["testing", "timestamps_only"])
 def test_data_generating_pipeline(config_and_stats_paths, experiment_name):
     config_path, stats_path = config_and_stats_paths("testing", experiment_name)

tests/tasks/test_common.py

@@ -31,12 +31,12 @@ def test_linear_function_task():
         [data_feature, mask_timeless_feature],
         [data_result_feature, mask_timeless_result_feature],
         slope=2,
-        intercept=-5,
+        intercept=+5,
     )
     task_double_minus_five(eopatch)
-    expected_result = np.arange(np.prod(data_shape)).reshape(data_shape).astype(float) * 2 - 5
+    expected_result = np.arange(np.prod(data_shape)).reshape(data_shape).astype(float) * 2 + 5
     assert np.array_equal(eopatch[data_result_feature], expected_result)
-    assert np.array_equal(eopatch[mask_timeless_result_feature], np.ones(mask_shape) * 2 - 5)
+    assert np.array_equal(eopatch[mask_timeless_result_feature], np.ones(mask_shape) * 2 + 5)
     assert eopatch[data_result_feature].dtype == np.float32
     # The value of the mask timeless changes here
 

tests/tasks/test_testing.py

@@ -16,7 +16,7 @@
 )
 
 
-@pytest.fixture()
+@pytest.fixture
 def dummy_eopatch() -> EOPatch:
     return EOPatch(bbox=BBox((0, 0, 1, 1), CRS.POP_WEB))
 

tests/test_cli.py

@@ -26,7 +26,7 @@ def test_pipeline_chain_validation(config_folder):
     assert subprocess.call(f"eogrow-validate {config_folder}/chain_pipeline.json", shell=True) == 0
 
 
-@pytest.mark.integration()
+@pytest.mark.integration
 @pytest.mark.order(after="tests/pipelines/test_zipmap.py::test_zipmap_pipeline")
 def test_pipeline_chain_execution(config_folder):
     """Tests a simple execution from command line"""

tests/test_config_files/other/aws_storage_test.json

@@ -1,7 +1,6 @@
 {
   "manager": "eogrow.core.storage.StorageManager",
   "project_folder": "s3://eogrow-test-storage/test",
-  "aws_profile": null,
   "structure": {
     "batch": "tiffs",
     "eopatches": "eopatches"

tests/test_stats/download_and_batch/download_l1c_yearly.json

@@ -17,7 +17,7 @@
         },
         "dtype": "float32",
         "histogram": {
-          "counts": [7046664, 5964028, 1206792, 268781, 43749, 4429, 63, 14],
+          "counts": [7046664, 5965840, 1204980, 268781, 43749, 4429, 63, 14],
           "edges": [
             0.0006, 0.15015, 0.2997, 0.44925, 0.5988, 0.74835, 0.8979, 1.04745,
             1.197

tests/test_stats/features/features_dtype.json

@@ -21,7 +21,7 @@
             140579, 624836, 1721408, 9943223, 2177532, 205367, 9027, 44
           ],
           "edges": [
-            -0.693802, -0.457451, -0.221101, 0.015249, 0.251599, 0.487949,
+            -0.693802, -0.457451, -0.221101, 0.0152491, 0.251599, 0.487949,
             0.7243, 0.96065, 1.197
           ]
         },

tests/test_stats/import_tiff/import_tiff_temporal.json

@@ -18,7 +18,7 @@
         "dtype": "float32",
         "histogram": {
           "counts": [
-            299415, 299725, 300061, 299722, 299399, 298483, 299305, 299690
+            299415, 299725, 300061, 299723, 299398, 298483, 299303, 299692
           ],
           "edges": [
             0.00125003, 1250.0, 2500.0, 3750.0, 5000.0, 6250.0, 7500.0, 8750.0,

tests/test_stats/sampling/sampling_block_fraction.json

@@ -164,7 +164,7 @@
         "dtype": "float32",
         "histogram": {
           "counts": [
-            2195347, 3494725, 1606303, 405891, 123943, 17915, 3865, 761
+            2195347, 3494725, 1606303, 406139, 123695, 17915, 3865, 761
           ],
           "edges": [
             0.0006, 0.110175, 0.21975, 0.329325, 0.4389, 0.548475, 0.65805,

tests/test_stats/sampling/sampling_chain.json

@@ -197,8 +197,8 @@
         "histogram": {
           "counts": [95600, 166829, 73548, 21212, 5658, 874, 236, 43],
           "edges": [
-            0.0006, 0.109563, 0.218525, 0.327487, 0.43645, 0.545413, 0.654375,
-            0.763338, 0.8723
+            0.0006, 0.109563, 0.218525, 0.327488, 0.43645, 0.545413, 0.654375,
+            0.763337, 0.8723
           ]
         },
         "random_values": [