From 7ab78c821201acd1017bc14cd720afd645c55c61 Mon Sep 17 00:00:00 2001
From: Sergey Vasilyev <nolar@nolar.info>
Date: Thu, 14 Nov 2019 13:41:09 +0100
Subject: [PATCH 1/3] Use `memo` for arbitrary per-resource payload during
 operator lifetime

---
 docs/handlers.rst                          |  2 ++
 kopf/reactor/causation.py                  |  2 ++
 kopf/reactor/handling.py                   |  2 ++
 kopf/reactor/invocation.py                 |  1 +
 kopf/structs/containers.py                 | 26 +++++++++++++++-
 tests/basic-structs/test_causes.py         |  9 ++++++
 tests/basic-structs/test_containers.py     | 35 +++++++++++++++++++++-
 tests/causation/test_detection.py          |  3 ++
 tests/handling/conftest.py                 |  6 +++-
 tests/hierarchies/test_contextual_owner.py |  3 ++
 tests/invocations/test_callbacks.py        |  1 +
 tests/lifecycles/test_real_invocation.py   |  2 ++
 12 files changed, 89 insertions(+), 3 deletions(-)

diff --git a/docs/handlers.rst b/docs/handlers.rst
index 57f7842e..ce95ce68 100644
--- a/docs/handlers.rst
+++ b/docs/handlers.rst
@@ -56,6 +56,8 @@ Arguments
 The following keyword arguments are available to the handlers
 (though some handlers may have some of them empty):
 
+* ``memo`` for arbitrary in-memory runtime-only keys/fields and values stored
+  during the operator lifetime, per-object; they are lost on operator restarts.
 * ``body`` for the whole body of the handled objects.
 * ``spec`` as an alias for ``body['spec']``.
 * ``meta`` as an alias for ``body['metadata']``.
diff --git a/kopf/reactor/causation.py b/kopf/reactor/causation.py
index 3343361a..f928e4e0 100644
--- a/kopf/reactor/causation.py
+++ b/kopf/reactor/causation.py
@@ -26,6 +26,7 @@
 from typing import Any, Optional, Union, TypeVar
 
 from kopf.structs import bodies
+from kopf.structs import containers
 from kopf.structs import diffs
 from kopf.structs import finalizers
 from kopf.structs import lastseen
@@ -98,6 +99,7 @@ class ResourceCause(BaseCause):
     resource: resources.Resource
     patch: patches.Patch
     body: bodies.Body
+    memo: containers.ObjectDict
 
 
 @dataclasses.dataclass
diff --git a/kopf/reactor/handling.py b/kopf/reactor/handling.py
index a71808e1..1f0eff3f 100644
--- a/kopf/reactor/handling.py
+++ b/kopf/reactor/handling.py
@@ -189,6 +189,7 @@ async def resource_handler(
             resource=resource,
             logger=logger,
             patch=patch,
+            memo=memory.user_data,
         )
         await handle_resource_watching_cause(
             lifecycle=lifecycles.all_at_once,
@@ -210,6 +211,7 @@ async def resource_handler(
             old=old,
             new=new,
             diff=diff,
+            memo=memory.user_data,
             initial=memory.noticed_by_listing and not memory.fully_handled_once,
             requires_finalizer=registry.requires_finalizer(resource=resource, body=body),
         )
diff --git a/kopf/reactor/invocation.py b/kopf/reactor/invocation.py
index eaa10302..83d83107 100644
--- a/kopf/reactor/invocation.py
+++ b/kopf/reactor/invocation.py
@@ -75,6 +75,7 @@ async def invoke(
     if isinstance(cause, causation.ResourceCause):
         kwargs.update(
             patch=cause.patch,
+            memo=cause.memo,
             body=cause.body,
             spec=dicts.DictView(cause.body, 'spec'),
             meta=dicts.DictView(cause.body, 'metadata'),
diff --git a/kopf/structs/containers.py b/kopf/structs/containers.py
index 072f3d87..69f0c7dd 100644
--- a/kopf/structs/containers.py
+++ b/kopf/structs/containers.py
@@ -13,9 +13,33 @@
 from kopf.structs import bodies
 
 
+class ObjectDict(dict):
+    """ A container to hold arbitrary keys-fields assigned by the users. """
+
+    def __setattr__(self, key, value):
+        self[key] = value
+
+    def __delitem__(self, key):
+        try:
+            del self[key]
+        except KeyError as e:
+            raise AttributeError(str(e))
+
+    def __getattr__(self, key):
+        try:
+            return self[key]
+        except KeyError as e:
+            raise AttributeError(str(e))
+
+
 @dataclasses.dataclass(frozen=False)
 class ResourceMemory:
-    """ A memo about a single resource/object. Usually stored in `Memories`. """
+    """ A system memo about a single resource/object. Usually stored in `Memories`. """
+
+    # For arbitrary user data to be stored in memory, passed as `memo` to all the handlers.
+    user_data: ObjectDict = dataclasses.field(default_factory=ObjectDict)
+
+    # For resuming handlers tracking and deciding on should they be called or not.
     noticed_by_listing: bool = False
     fully_handled_once: bool = False
 
diff --git a/tests/basic-structs/test_causes.py b/tests/basic-structs/test_causes.py
index d7d9875b..6b057dbc 100644
--- a/tests/basic-structs/test_causes.py
+++ b/tests/basic-structs/test_causes.py
@@ -25,6 +25,7 @@ def test_resource_watching_cause(mocker):
     resource = mocker.Mock()
     body = mocker.Mock()
     patch = mocker.Mock()
+    memo = mocker.Mock()
     type = mocker.Mock()
     raw = mocker.Mock()
     cause = ResourceWatchingCause(
@@ -32,6 +33,7 @@ def test_resource_watching_cause(mocker):
         logger=logger,
         body=body,
         patch=patch,
+        memo=memo,
         type=type,
         raw=raw,
     )
@@ -39,6 +41,7 @@ def test_resource_watching_cause(mocker):
     assert cause.logger is logger
     assert cause.body is body
     assert cause.patch is patch
+    assert cause.memo is memo
     assert cause.type is type
     assert cause.raw is raw
 
@@ -50,6 +53,7 @@ def test_resource_changing_cause_with_all_args(mocker):
     initial = mocker.Mock()
     body = mocker.Mock()
     patch = mocker.Mock()
+    memo = mocker.Mock()
     diff = mocker.Mock()
     old = mocker.Mock()
     new = mocker.Mock()
@@ -60,6 +64,7 @@ def test_resource_changing_cause_with_all_args(mocker):
         initial=initial,
         body=body,
         patch=patch,
+        memo=memo,
         diff=diff,
         old=old,
         new=new,
@@ -71,6 +76,7 @@ def test_resource_changing_cause_with_all_args(mocker):
     assert cause.initial is initial
     assert cause.body is body
     assert cause.patch is patch
+    assert cause.memo is memo
     assert cause.diff is diff
     assert cause.old is old
     assert cause.new is new
@@ -83,6 +89,7 @@ def test_resource_changing_cause_with_only_required_args(mocker):
     initial = mocker.Mock()
     body = mocker.Mock()
     patch = mocker.Mock()
+    memo = mocker.Mock()
     cause = ResourceChangingCause(
         resource=resource,
         logger=logger,
@@ -90,6 +97,7 @@ def test_resource_changing_cause_with_only_required_args(mocker):
         initial=initial,
         body=body,
         patch=patch,
+        memo=memo,
     )
     assert cause.resource is resource
     assert cause.logger is logger
@@ -98,6 +106,7 @@ def test_resource_changing_cause_with_only_required_args(mocker):
     assert cause.initial is initial
     assert cause.body is body
     assert cause.patch is patch
+    assert cause.memo is memo
     assert cause.diff is not None
     assert not cause.diff
     assert cause.old is None
diff --git a/tests/basic-structs/test_containers.py b/tests/basic-structs/test_containers.py
index b36e752d..0235ffbe 100644
--- a/tests/basic-structs/test_containers.py
+++ b/tests/basic-structs/test_containers.py
@@ -1,5 +1,9 @@
+import collections.abc
+
+import pytest
+
 from kopf.structs.bodies import Body
-from kopf.structs.containers import ResourceMemory, ResourceMemories
+from kopf.structs.containers import ResourceMemory, ResourceMemories, ObjectDict
 
 BODY: Body = {
     'metadata': {
@@ -38,3 +42,32 @@ async def test_forgetting_deletes_when_present():
 async def test_forgetting_ignores_when_absent():
     memories = ResourceMemories()
     await memories.forget(BODY)
+
+
+def test_object_dict_creation():
+    obj = ObjectDict()
+    assert isinstance(obj, collections.abc.MutableMapping)
+
+
+def test_object_dict_fields_are_keys():
+    obj = ObjectDict()
+    obj.xyz = 100
+    assert obj['xyz'] == 100
+
+
+def test_object_dict_keys_are_fields():
+    obj = ObjectDict()
+    obj['xyz'] = 100
+    assert obj.xyz == 100
+
+
+def test_object_dict_raises_key_errors():
+    obj = ObjectDict()
+    with pytest.raises(KeyError):
+        obj['unexistent']
+
+
+def test_object_dict_raises_attribute_errors():
+    obj = ObjectDict()
+    with pytest.raises(AttributeError):
+        obj.unexistent
diff --git a/tests/causation/test_detection.py b/tests/causation/test_detection.py
index e16e6310..d9a687db 100644
--- a/tests/causation/test_detection.py
+++ b/tests/causation/test_detection.py
@@ -116,13 +116,16 @@ def kwargs():
         resource=object(),
         logger=object(),
         patch=object(),
+        memo=object(),
     )
 
+
 def check_kwargs(cause, kwargs):
     __traceback_hide__ = True
     assert cause.resource is kwargs['resource']
     assert cause.logger is kwargs['logger']
     assert cause.patch is kwargs['patch']
+    assert cause.memo is kwargs['memo']
 
 
 #
diff --git a/tests/handling/conftest.py b/tests/handling/conftest.py
index f1542c62..3a4bfd7a 100644
--- a/tests/handling/conftest.py
+++ b/tests/handling/conftest.py
@@ -178,11 +178,13 @@ def new_detect_fn(**kwargs):
         # Avoid collision of our mocked values with the passed kwargs.
         original_event = kwargs.pop('event', None)
         original_reason = kwargs.pop('reason', None)
+        original_memo = kwargs.pop('memo', None)
         original_body = kwargs.pop('body', None)
         original_diff = kwargs.pop('diff', None)
         original_new = kwargs.pop('new', None)
         original_old = kwargs.pop('old', None)
         reason = mock.reason if mock.reason is not None else original_reason
+        memo = copy.deepcopy(mock.memo) if mock.memo is not None else original_memo
         body = copy.deepcopy(mock.body) if mock.body is not None else original_body
         diff = copy.deepcopy(mock.diff) if mock.diff is not None else original_diff
         new = copy.deepcopy(mock.new) if mock.new is not None else original_new
@@ -195,6 +197,7 @@ def new_detect_fn(**kwargs):
         # I.e. everything except what we mock: reason & body.
         cause = ResourceChangingCause(
             reason=reason,
+            memo=memo,
             body=body,
             diff=diff,
             new=new,
@@ -214,8 +217,9 @@ def new_detect_fn(**kwargs):
     mocker.patch('kopf.reactor.causation.detect_resource_changing_cause', new=new_detect_fn)
 
     # The mock object stores some values later used by the factory substitute.
-    mock = mocker.Mock(spec_set=['reason', 'body', 'diff', 'new', 'old'])
+    mock = mocker.Mock(spec_set=['reason', 'memo', 'body', 'diff', 'new', 'old'])
     mock.reason = None
+    mock.memo = None
     mock.body = {'metadata': {'namespace': 'ns1', 'name': 'name1'}}
     mock.diff = None
     mock.new = None
diff --git a/tests/hierarchies/test_contextual_owner.py b/tests/hierarchies/test_contextual_owner.py
index b57181fb..9362c307 100644
--- a/tests/hierarchies/test_contextual_owner.py
+++ b/tests/hierarchies/test_contextual_owner.py
@@ -7,6 +7,7 @@
 from kopf.reactor.handling import cause_var
 from kopf.reactor.invocation import context
 from kopf.structs.bodies import Body, Meta, Labels, Event
+from kopf.structs.containers import ObjectDict
 from kopf.structs.patches import Patch
 
 OWNER_API_VERSION = 'owner-api-version'
@@ -34,6 +35,7 @@ def owner(request, resource):
             logger=logging.getLogger('kopf.test.fake.logger'),
             resource=resource,
             patch=Patch(),
+            memo=ObjectDict(),
             body=OWNER,
             initial=False,
             reason=Reason.NOOP,
@@ -45,6 +47,7 @@ def owner(request, resource):
             logger=logging.getLogger('kopf.test.fake.logger'),
             resource=resource,
             patch=Patch(),
+            memo=ObjectDict(),
             body=OWNER,
             type='irrelevant',
             raw=Event(type='irrelevant', object=OWNER),
diff --git a/tests/invocations/test_callbacks.py b/tests/invocations/test_callbacks.py
index bda7b1a7..f7bd90f0 100644
--- a/tests/invocations/test_callbacks.py
+++ b/tests/invocations/test_callbacks.py
@@ -173,6 +173,7 @@ async def test_special_kwargs_added(fn, resource):
         patch=Patch(),
         initial=False,
         reason=Reason.NOOP,
+        memo=object(),
         body=body,
         diff=object(),
         old=object(),
diff --git a/tests/lifecycles/test_real_invocation.py b/tests/lifecycles/test_real_invocation.py
index 5ea84c30..48fc14b7 100644
--- a/tests/lifecycles/test_real_invocation.py
+++ b/tests/lifecycles/test_real_invocation.py
@@ -7,6 +7,7 @@
 from kopf.reactor.invocation import invoke
 from kopf.reactor.states import State
 from kopf.structs.bodies import Body
+from kopf.structs.containers import ObjectDict
 from kopf.structs.patches import Patch
 
 
@@ -28,6 +29,7 @@ async def test_protocol_invocation(lifecycle, resource):
         logger=logging.getLogger('kopf.test.fake.logger'),
         resource=resource,
         patch=Patch(),
+        memo=ObjectDict(),
         body=Body(),
         initial=False,
         reason=Reason.NOOP,

From 82bec1c4c7ff01ca989964c9bd878c71d19a7175 Mon Sep 17 00:00:00 2001
From: Sergey Vasilyev <nolar@nolar.info>
Date: Thu, 14 Nov 2019 14:24:49 +0100
Subject: [PATCH 2/3] Add forgotten type annotations for the ObjectDict
 container

---
 kopf/structs/containers.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/kopf/structs/containers.py b/kopf/structs/containers.py
index 69f0c7dd..6090092c 100644
--- a/kopf/structs/containers.py
+++ b/kopf/structs/containers.py
@@ -8,24 +8,24 @@
 object, even if that object does not show up in the event streams for long time.
 """
 import dataclasses
-from typing import MutableMapping
+from typing import MutableMapping, Dict, Any
 
 from kopf.structs import bodies
 
 
-class ObjectDict(dict):
+class ObjectDict(Dict[Any, Any]):
     """ A container to hold arbitrary keys-fields assigned by the users. """
 
-    def __setattr__(self, key, value):
+    def __setattr__(self, key: str, value: Any) -> None:
         self[key] = value
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str) -> None:
         try:
             del self[key]
         except KeyError as e:
             raise AttributeError(str(e))
 
-    def __getattr__(self, key):
+    def __getattr__(self, key: str) -> Any:
         try:
             return self[key]
         except KeyError as e:

From 490e54f776a643e0a9d1c1e5f64d68ce9833eed7 Mon Sep 17 00:00:00 2001
From: Sergey Vasilyev <nolar@nolar.info>
Date: Thu, 14 Nov 2019 14:28:06 +0100
Subject: [PATCH 3/3] Fix the attr deletion in ObjectDict

---
 kopf/structs/containers.py             |  2 +-
 tests/basic-structs/test_containers.py | 30 ++++++++++++++++++++++++--
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/kopf/structs/containers.py b/kopf/structs/containers.py
index 6090092c..6eebd5f2 100644
--- a/kopf/structs/containers.py
+++ b/kopf/structs/containers.py
@@ -19,7 +19,7 @@ class ObjectDict(Dict[Any, Any]):
     def __setattr__(self, key: str, value: Any) -> None:
         self[key] = value
 
-    def __delitem__(self, key: str) -> None:
+    def __delattr__(self, key: str) -> None:
         try:
             del self[key]
         except KeyError as e:
diff --git a/tests/basic-structs/test_containers.py b/tests/basic-structs/test_containers.py
index 0235ffbe..88c01103 100644
--- a/tests/basic-structs/test_containers.py
+++ b/tests/basic-structs/test_containers.py
@@ -61,13 +61,39 @@ def test_object_dict_keys_are_fields():
     assert obj.xyz == 100
 
 
-def test_object_dict_raises_key_errors():
+def test_object_dict_keys_deleted():
+    obj = ObjectDict()
+    obj['xyz'] = 100
+    del obj['xyz']
+    assert obj == {}
+
+
+def test_object_dict_fields_deleted():
+    obj = ObjectDict()
+    obj.xyz = 100
+    del obj.xyz
+    assert obj == {}
+
+
+def test_object_dict_raises_key_errors_on_get():
     obj = ObjectDict()
     with pytest.raises(KeyError):
         obj['unexistent']
 
 
-def test_object_dict_raises_attribute_errors():
+def test_object_dict_raises_attribute_errors_on_get():
     obj = ObjectDict()
     with pytest.raises(AttributeError):
         obj.unexistent
+
+
+def test_object_dict_raises_key_errors_on_del():
+    obj = ObjectDict()
+    with pytest.raises(KeyError):
+        del obj['unexistent']
+
+
+def test_object_dict_raises_attribute_errors_on_del():
+    obj = ObjectDict()
+    with pytest.raises(AttributeError):
+        del obj.unexistent