Issue #2 memoizer_from_config: add ChainedMemoizer support

src/openeo_aggregator/caching.py

@@ -480,20 +480,30 @@ def memoizer_from_config(
         namespace: str,
 ) -> Memoizer:
     """Factory to create `ZkMemoizer` instance from config values."""
-    memoizer_type = config.memoizer.get("type", "null")
-    memoizer_conf = config.memoizer.get("config", {})
-    if memoizer_type == "null":
-        return NullMemoizer()
-    elif memoizer_type == "dict":
-        return DictMemoizer(default_ttl=memoizer_conf.get("default_ttl"))
-    elif memoizer_type == "jsondict":
-        return JsonDictMemoizer(default_ttl=memoizer_conf.get("default_ttl"))
-    elif memoizer_type == "zookeeper":
-        return ZkMemoizer(
-            client=KazooClient(hosts=memoizer_conf.get("zk_hosts", "localhost:2181")),
-            path_prefix=f"{config.zookeeper_prefix}/cache/{namespace}",
-            default_ttl=memoizer_conf.get("default_ttl"),
-            zk_timeout=memoizer_conf.get("zk_timeout"),
-        )
-    else:
-        raise ValueError(memoizer_type)
+
+    def get_memoizer(memoizer_type: str, memoizer_conf: dict) -> Memoizer:
+        if memoizer_type == "null":
+            return NullMemoizer()
+        elif memoizer_type == "dict":
+            return DictMemoizer(default_ttl=memoizer_conf.get("default_ttl"))
+        elif memoizer_type == "jsondict":
+            return JsonDictMemoizer(default_ttl=memoizer_conf.get("default_ttl"))
+        elif memoizer_type == "zookeeper":
+            return ZkMemoizer(
+                client=KazooClient(hosts=memoizer_conf.get("zk_hosts", "localhost:2181")),
+                path_prefix=f"{config.zookeeper_prefix}/cache/{namespace}",
+                default_ttl=memoizer_conf.get("default_ttl"),
+                zk_timeout=memoizer_conf.get("zk_timeout"),
+            )
+        elif memoizer_type == "chained":
+            return ChainedMemoizer([
+                get_memoizer(memoizer_type=part["type"], memoizer_conf=part["config"])
+                for part in memoizer_conf["parts"]
+            ])
+        else:
+            raise ValueError(memoizer_type)
+
+    return get_memoizer(
+        memoizer_type=config.memoizer.get("type", "null"),
+        memoizer_conf=config.memoizer.get("config", {}),
+    )

tests/test_caching.py

@@ -11,6 +11,7 @@
 from openeo_aggregator.backend import _InternalCollectionMetadata
 from openeo_aggregator.caching import TtlCache, CacheMissException, ZkMemoizer, memoizer_from_config, NullMemoizer, \
     DictMemoizer, ChainedMemoizer, JsonDictMemoizer, JsonSerDe, json_serde
+from openeo_aggregator.config import AggregatorConfig
 from openeo_aggregator.testing import clock_mock
 from openeo_aggregator.utils import Clock
 
@@ -292,6 +293,13 @@ def __init__(self):
 
 class TestChainedMemoizer(_TestMemoizer):
 
+    def test_empty(self):
+        cache = ChainedMemoizer(memoizers=[])
+        callback = self._build_callback()
+        assert cache.get_or_call(key="count", callback=callback) == 100
+        assert cache.get_or_call(key="count", callback=callback) == 101
+        assert cache.get_or_call(key="count", callback=callback) == 102
+
     def test_simple(self):
         dm1 = DictMemoizer(default_ttl=10)
         dm2 = DictMemoizer(default_ttl=100)
@@ -676,3 +684,57 @@ def fun2_cached():
         zk_client.start.assert_called_once()
         zk_client.stop.assert_called_once()
         assert fun2_cached() == "fun1:100+fun2:101"
+
+
+class TestMemoizerFromConfig:
+
+    def test_null_memoizer(self):
+        config = AggregatorConfig()
+        config.memoizer = {"type": "null"}
+        memoizer = memoizer_from_config(config, namespace="test")
+        assert isinstance(memoizer, NullMemoizer)
+
+    def test_dict_memoizer(self):
+        config = AggregatorConfig()
+        config.memoizer = {"type": "dict", "config": {"default_ttl": 99}}
+        memoizer = memoizer_from_config(config, namespace="test")
+        assert isinstance(memoizer, DictMemoizer)
+        assert memoizer._default_ttl == 99
+
+    def test_jsondict_memoizer(self):
+        config = AggregatorConfig()
+        config.memoizer = {"type": "jsondict", "config": {"default_ttl": 99}}
+        memoizer = memoizer_from_config(config, namespace="test")
+        assert isinstance(memoizer, JsonDictMemoizer)
+        assert memoizer._default_ttl == 99
+
+    def test_zookeeper_memoizer(self):
+        config = AggregatorConfig()
+        config.memoizer = {
+            "type": "zookeeper",
+            "config": {"zk_hosts": "zk.test:2181", "default_ttl": 99, "zk_timeout": 88}
+        }
+        config.zookeeper_prefix = "/oea/test"
+        memoizer = memoizer_from_config(config, namespace="test-ns")
+        assert isinstance(memoizer, ZkMemoizer)
+        assert memoizer._default_ttl == 99
+        assert memoizer._prefix == "/oea/test/cache/test-ns"
+        assert memoizer._zk_timeout == 88
+
+    def test_chained_memoizer(self):
+        config = AggregatorConfig()
+        config.memoizer = {
+            "type": "chained",
+            "config": {"parts": [
+                {"type": "jsondict", "config": {"default_ttl": 99}},
+                {"type": "dict", "config": {"default_ttl": 333}},
+            ]}
+        }
+        config.zookeeper_prefix = "/oea/test"
+        memoizer = memoizer_from_config(config, namespace="test-ns")
+        assert isinstance(memoizer, ChainedMemoizer)
+        assert len(memoizer._memoizers) == 2
+        assert isinstance(memoizer._memoizers[0], JsonDictMemoizer)
+        assert memoizer._memoizers[0]._default_ttl == 99
+        assert isinstance(memoizer._memoizers[1], DictMemoizer)
+        assert memoizer._memoizers[1]._default_ttl == 333