Merge pull request #234 from nolar/112-user-memos

Use `memo` for arbitrary per-resource payload during operator lifetime

docs/handlers.rst

@@ -56,6 +56,8 @@ Arguments
 The following keyword arguments are available to the handlers
 (though some handlers may have some of them empty):
 
+* ``memo`` for arbitrary in-memory runtime-only keys/fields and values stored
+  during the operator lifetime, per-object; they are lost on operator restarts.
 * ``body`` for the whole body of the handled objects.
 * ``spec`` as an alias for ``body['spec']``.
 * ``meta`` as an alias for ``body['metadata']``.

kopf/reactor/causation.py

@@ -26,6 +26,7 @@
 from typing import Any, Optional, Union, TypeVar
 
 from kopf.structs import bodies
+from kopf.structs import containers
 from kopf.structs import diffs
 from kopf.structs import finalizers
 from kopf.structs import lastseen
@@ -98,6 +99,7 @@ class ResourceCause(BaseCause):
     resource: resources.Resource
     patch: patches.Patch
     body: bodies.Body
+    memo: containers.ObjectDict
 
 
 @dataclasses.dataclass

kopf/reactor/handling.py

@@ -189,6 +189,7 @@ async def resource_handler(
             resource=resource,
             logger=logger,
             patch=patch,
+            memo=memory.user_data,
         )
         await handle_resource_watching_cause(
             lifecycle=lifecycles.all_at_once,
@@ -210,6 +211,7 @@ async def resource_handler(
             old=old,
             new=new,
             diff=diff,
+            memo=memory.user_data,
             initial=memory.noticed_by_listing and not memory.fully_handled_once,
             requires_finalizer=registry.requires_finalizer(resource=resource, body=body),
         )

kopf/reactor/invocation.py

@@ -75,6 +75,7 @@ async def invoke(
     if isinstance(cause, causation.ResourceCause):
         kwargs.update(
             patch=cause.patch,
+            memo=cause.memo,
             body=cause.body,
             spec=dicts.DictView(cause.body, 'spec'),
             meta=dicts.DictView(cause.body, 'metadata'),

kopf/structs/containers.py

@@ -8,14 +8,38 @@
 object, even if that object does not show up in the event streams for long time.
 """
 import dataclasses
-from typing import MutableMapping
+from typing import MutableMapping, Dict, Any
 
 from kopf.structs import bodies
 
 
+class ObjectDict(Dict[Any, Any]):
+    """ A container to hold arbitrary keys-fields assigned by the users. """
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        self[key] = value
+
+    def __delattr__(self, key: str) -> None:
+        try:
+            del self[key]
+        except KeyError as e:
+            raise AttributeError(str(e))
+
+    def __getattr__(self, key: str) -> Any:
+        try:
+            return self[key]
+        except KeyError as e:
+            raise AttributeError(str(e))
+
+
 @dataclasses.dataclass(frozen=False)
 class ResourceMemory:
-    """ A memo about a single resource/object. Usually stored in `Memories`. """
+    """ A system memo about a single resource/object. Usually stored in `Memories`. """
+
+    # For arbitrary user data to be stored in memory, passed as `memo` to all the handlers.
+    user_data: ObjectDict = dataclasses.field(default_factory=ObjectDict)
+
+    # For resuming handlers tracking and deciding on should they be called or not.
     noticed_by_listing: bool = False
     fully_handled_once: bool = False
 

tests/basic-structs/test_causes.py

@@ -25,20 +25,23 @@ def test_resource_watching_cause(mocker):
     resource = mocker.Mock()
     body = mocker.Mock()
     patch = mocker.Mock()
+    memo = mocker.Mock()
     type = mocker.Mock()
     raw = mocker.Mock()
     cause = ResourceWatchingCause(
         resource=resource,
         logger=logger,
         body=body,
         patch=patch,
+        memo=memo,
         type=type,
         raw=raw,
     )
     assert cause.resource is resource
     assert cause.logger is logger
     assert cause.body is body
     assert cause.patch is patch
+    assert cause.memo is memo
     assert cause.type is type
     assert cause.raw is raw
 
@@ -50,6 +53,7 @@ def test_resource_changing_cause_with_all_args(mocker):
     initial = mocker.Mock()
     body = mocker.Mock()
     patch = mocker.Mock()
+    memo = mocker.Mock()
     diff = mocker.Mock()
     old = mocker.Mock()
     new = mocker.Mock()
@@ -60,6 +64,7 @@ def test_resource_changing_cause_with_all_args(mocker):
         initial=initial,
         body=body,
         patch=patch,
+        memo=memo,
         diff=diff,
         old=old,
         new=new,
@@ -71,6 +76,7 @@ def test_resource_changing_cause_with_all_args(mocker):
     assert cause.initial is initial
     assert cause.body is body
     assert cause.patch is patch
+    assert cause.memo is memo
     assert cause.diff is diff
     assert cause.old is old
     assert cause.new is new
@@ -83,13 +89,15 @@ def test_resource_changing_cause_with_only_required_args(mocker):
     initial = mocker.Mock()
     body = mocker.Mock()
     patch = mocker.Mock()
+    memo = mocker.Mock()
     cause = ResourceChangingCause(
         resource=resource,
         logger=logger,
         reason=reason,
         initial=initial,
         body=body,
         patch=patch,
+        memo=memo,
     )
     assert cause.resource is resource
     assert cause.logger is logger
@@ -98,6 +106,7 @@ def test_resource_changing_cause_with_only_required_args(mocker):
     assert cause.initial is initial
     assert cause.body is body
     assert cause.patch is patch
+    assert cause.memo is memo
     assert cause.diff is not None
     assert not cause.diff
     assert cause.old is None

tests/basic-structs/test_containers.py

@@ -1,5 +1,9 @@
+import collections.abc
+
+import pytest
+
 from kopf.structs.bodies import Body
-from kopf.structs.containers import ResourceMemory, ResourceMemories
+from kopf.structs.containers import ResourceMemory, ResourceMemories, ObjectDict
 
 BODY: Body = {
     'metadata': {
@@ -38,3 +42,58 @@ async def test_forgetting_deletes_when_present():
 async def test_forgetting_ignores_when_absent():
     memories = ResourceMemories()
     await memories.forget(BODY)
+
+
+def test_object_dict_creation():
+    obj = ObjectDict()
+    assert isinstance(obj, collections.abc.MutableMapping)
+
+
+def test_object_dict_fields_are_keys():
+    obj = ObjectDict()
+    obj.xyz = 100
+    assert obj['xyz'] == 100
+
+
+def test_object_dict_keys_are_fields():
+    obj = ObjectDict()
+    obj['xyz'] = 100
+    assert obj.xyz == 100
+
+
+def test_object_dict_keys_deleted():
+    obj = ObjectDict()
+    obj['xyz'] = 100
+    del obj['xyz']
+    assert obj == {}
+
+
+def test_object_dict_fields_deleted():
+    obj = ObjectDict()
+    obj.xyz = 100
+    del obj.xyz
+    assert obj == {}
+
+
+def test_object_dict_raises_key_errors_on_get():
+    obj = ObjectDict()
+    with pytest.raises(KeyError):
+        obj['unexistent']
+
+
+def test_object_dict_raises_attribute_errors_on_get():
+    obj = ObjectDict()
+    with pytest.raises(AttributeError):
+        obj.unexistent
+
+
+def test_object_dict_raises_key_errors_on_del():
+    obj = ObjectDict()
+    with pytest.raises(KeyError):
+        del obj['unexistent']
+
+
+def test_object_dict_raises_attribute_errors_on_del():
+    obj = ObjectDict()
+    with pytest.raises(AttributeError):
+        del obj.unexistent

tests/causation/test_detection.py

@@ -116,13 +116,16 @@ def kwargs():
         resource=object(),
         logger=object(),
         patch=object(),
+        memo=object(),
     )
 
+
 def check_kwargs(cause, kwargs):
     __traceback_hide__ = True
     assert cause.resource is kwargs['resource']
     assert cause.logger is kwargs['logger']
     assert cause.patch is kwargs['patch']
+    assert cause.memo is kwargs['memo']
 
 
 #

tests/handling/conftest.py

@@ -178,11 +178,13 @@ def new_detect_fn(**kwargs):
         # Avoid collision of our mocked values with the passed kwargs.
         original_event = kwargs.pop('event', None)
         original_reason = kwargs.pop('reason', None)
+        original_memo = kwargs.pop('memo', None)
         original_body = kwargs.pop('body', None)
         original_diff = kwargs.pop('diff', None)
         original_new = kwargs.pop('new', None)
         original_old = kwargs.pop('old', None)
         reason = mock.reason if mock.reason is not None else original_reason
+        memo = copy.deepcopy(mock.memo) if mock.memo is not None else original_memo
         body = copy.deepcopy(mock.body) if mock.body is not None else original_body
         diff = copy.deepcopy(mock.diff) if mock.diff is not None else original_diff
         new = copy.deepcopy(mock.new) if mock.new is not None else original_new
@@ -195,6 +197,7 @@ def new_detect_fn(**kwargs):
         # I.e. everything except what we mock: reason & body.
         cause = ResourceChangingCause(
             reason=reason,
+            memo=memo,
             body=body,
             diff=diff,
             new=new,
@@ -214,8 +217,9 @@ def new_detect_fn(**kwargs):
     mocker.patch('kopf.reactor.causation.detect_resource_changing_cause', new=new_detect_fn)
 
     # The mock object stores some values later used by the factory substitute.
-    mock = mocker.Mock(spec_set=['reason', 'body', 'diff', 'new', 'old'])
+    mock = mocker.Mock(spec_set=['reason', 'memo', 'body', 'diff', 'new', 'old'])
     mock.reason = None
+    mock.memo = None
     mock.body = {'metadata': {'namespace': 'ns1', 'name': 'name1'}}
     mock.diff = None
     mock.new = None

tests/hierarchies/test_contextual_owner.py

@@ -7,6 +7,7 @@
 from kopf.reactor.handling import cause_var
 from kopf.reactor.invocation import context
 from kopf.structs.bodies import Body, Meta, Labels, Event
+from kopf.structs.containers import ObjectDict
 from kopf.structs.patches import Patch
 
 OWNER_API_VERSION = 'owner-api-version'
@@ -34,6 +35,7 @@ def owner(request, resource):
             logger=logging.getLogger('kopf.test.fake.logger'),
             resource=resource,
             patch=Patch(),
+            memo=ObjectDict(),
             body=OWNER,
             initial=False,
             reason=Reason.NOOP,
@@ -45,6 +47,7 @@ def owner(request, resource):
             logger=logging.getLogger('kopf.test.fake.logger'),
             resource=resource,
             patch=Patch(),
+            memo=ObjectDict(),
             body=OWNER,
             type='irrelevant',
             raw=Event(type='irrelevant', object=OWNER),

tests/invocations/test_callbacks.py

@@ -173,6 +173,7 @@ async def test_special_kwargs_added(fn, resource):
         patch=Patch(),
         initial=False,
         reason=Reason.NOOP,
+        memo=object(),
         body=body,
         diff=object(),
         old=object(),

tests/lifecycles/test_real_invocation.py

@@ -7,6 +7,7 @@
 from kopf.reactor.invocation import invoke
 from kopf.reactor.states import State
 from kopf.structs.bodies import Body
+from kopf.structs.containers import ObjectDict
 from kopf.structs.patches import Patch
 
 
@@ -28,6 +29,7 @@ async def test_protocol_invocation(lifecycle, resource):
         logger=logging.getLogger('kopf.test.fake.logger'),
         resource=resource,
         patch=Patch(),
+        memo=ObjectDict(),
         body=Body(),
         initial=False,
         reason=Reason.NOOP,