Document the preservation of Kopf's persistence state in "v1" CRDs

.travis.yml

@@ -19,10 +19,11 @@ env:
     - KUBERNETES_VERSION=latest CLIENT=yes  # only one "yes" is enough
     - KUBERNETES_VERSION=latest CLIENT=no
     - KUBERNETES_VERSION=v1.16.0 CLIENT=no
-    - KUBERNETES_VERSION=v1.15.0 CLIENT=no
-    - KUBERNETES_VERSION=v1.14.0 CLIENT=no
-    - KUBERNETES_VERSION=v1.13.0 CLIENT=no
-    - KUBERNETES_VERSION=v1.12.0 CLIENT=no
+    - KUBERNETES_VERSION=v1.16.0 CLIENT=no CRDAPI=v1beta1
+    - KUBERNETES_VERSION=v1.15.0 CLIENT=no CRDAPI=v1beta1
+    - KUBERNETES_VERSION=v1.14.0 CLIENT=no CRDAPI=v1beta1
+    - KUBERNETES_VERSION=v1.13.0 CLIENT=no CRDAPI=v1beta1
+    - KUBERNETES_VERSION=v1.12.0 CLIENT=no CRDAPI=v1beta1
 #    - KUBERNETES_VERSION=v1.11.10  # Minikube fails on CRI preflight checks
 #    - KUBERNETES_VERSION=v1.10.13  # CRDs require spec.version, which fails on 1.14
 

docs/configuration.rst

@@ -252,11 +252,44 @@ For this, inherit from `kopf.StateStorage`, and implement its abstract methods
 
     The legacy behavior is an equivalent of
     ``kopf.StatusStateStorage(field='status.kopf.progress')``.
-    However, the ``.status`` stanza is not always stored by the server
-    for built-in or improperly configured custom resources since Kubernetes 1.16
-    (see `#321 <https://github.com/zalando-incubator/kopf/issues/321>`_).
 
-    The new default "smart" engine is supposed to ensure a smooth upgrade
+    Starting with Kubernetes 1.16, both custom and built-in resources have
+    strict structural schemas with pruning of unknown fields
+    (more information is in `Future of CRDs: Structural Schemas`__).
+
+    __ https://kubernetes.io/blog/2019/06/20/crd-structural-schema/
+
+    Long story short, unknown fields are silently pruned by Kubernetes API.
+    As a result, Kopf's status storage will not be able to actually store
+    anything in the resource, as it will be instantly lost.
+    (See `#321 <https://github.com/zalando-incubator/kopf/issues/321>`_.)
+
+    To quickly fix this for custom resources, modify their definitions
+    with ``x-kubernetes-preserve-unknown-fields: true``. For example:
+
+    .. code-block:: yaml
+
+        apiVersion: apiextensions.k8s.io/v1
+        kind: CustomResourceDefinition
+        spec:
+          scope: ...
+          group: ...
+          names: ...
+          versions:
+            - name: v1
+              served: true
+              storage: true
+              schema:
+                openAPIV3Schema:
+                  type: object
+                  x-kubernetes-preserve-unknown-fields: true
+
+    See a more verbose example in ``examples/crd.yaml``.
+
+    For built-in resources, such as pods, namespaces, etc, the schemas cannot
+    be modified, so a full switch to annotations storage is advised.
+
+    The new default "smart" storage is supposed to ensure a smooth upgrade
     of Kopf-based operators to the new state location without special upgrade
     actions or conversions needed.
 

docs/walkthrough/resources.rst

@@ -7,6 +7,8 @@ Custom Resource Definition
 
 Let us define a CRD (custom resource definition) for our object.
 
+For Kubernetes 1.15 and below:
+
 .. code-block:: yaml
     :caption: crd.yaml
     :name: crd-yaml
@@ -18,17 +20,62 @@ Let us define a CRD (custom resource definition) for our object.
     spec:
       scope: Namespaced
       group: zalando.org
+      names:
+        kind: EphemeralVolumeClaim
+        plural: ephemeralvolumeclaims
+        singular: ephemeralvolumeclaim
+        shortNames:
+          - evcs
+          - evc
       versions:
         - name: v1
           served: true
           storage: true
+          schema:
+            openAPIV3Schema:
+              type: object
+              properties:
+                spec:
+                  type: object
+                  x-kubernetes-preserve-unknown-fields: true
+                status:
+                  type: object
+                  x-kubernetes-preserve-unknown-fields: true
+
+For Kubernetes 1.16 and above:
+
+.. code-block:: yaml
+    :caption: crd.yaml
+    :name: crd-yaml
+
+    apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    metadata:
+      name: ephemeralvolumeclaims.zalando.org
+    spec:
+      scope: Namespaced
+      group: zalando.org
       names:
         kind: EphemeralVolumeClaim
         plural: ephemeralvolumeclaims
         singular: ephemeralvolumeclaim
         shortNames:
           - evcs
           - evc
+      versions:
+        - name: v1
+          served: true
+          storage: true
+          schema:
+            openAPIV3Schema:
+              type: object
+              properties:
+                spec:
+                  type: object
+                  x-kubernetes-preserve-unknown-fields: true
+                status:
+                  type: object
+                  x-kubernetes-preserve-unknown-fields: true
 
 Note the short names: they can be used as the aliases on the command line,
 when getting a list or an object of that kind.

examples/09-testing/test_example_09.py

@@ -6,13 +6,19 @@
 
 import kopf.testing
 
-crd_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'crd.yaml'))
 obj_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'obj.yaml'))
 example_py = os.path.relpath(os.path.join(os.path.dirname(__file__), 'example.py'))
 
 
+@pytest.fixture(scope='session')
+def crd_yaml():
+    crd_api = os.environ.get('CRDAPI', 'v1')
+    crd_file = 'crd.yaml' if crd_api == 'v1' else f'crd-{crd_api}.yaml'
+    return os.path.relpath(os.path.join(os.path.dirname(__file__), '..', crd_file))
+
+
 @pytest.fixture(autouse=True)
-def crd_exists():
+def crd_exists(crd_yaml):
     subprocess.run(f"kubectl apply -f {crd_yaml}",
                    check=True, timeout=10, capture_output=True, shell=True)
 

examples/11-filtering-handlers/test_example_11.py

@@ -6,13 +6,19 @@
 
 import kopf.testing
 
-crd_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'crd.yaml'))
 obj_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'obj.yaml'))
 example_py = os.path.relpath(os.path.join(os.path.dirname(__file__), 'example.py'))
 
 
+@pytest.fixture(scope='session')
+def crd_yaml():
+    crd_api = os.environ.get('CRDAPI', 'v1')
+    crd_file = 'crd.yaml' if crd_api == 'v1' else f'crd-{crd_api}.yaml'
+    return os.path.relpath(os.path.join(os.path.dirname(__file__), '..', crd_file))
+
+
 @pytest.fixture(autouse=True)
-def crd_exists():
+def crd_exists(crd_yaml):
     subprocess.run(f"kubectl apply -f {crd_yaml}",
                    check=True, timeout=10, capture_output=True, shell=True)
 

examples/crd-v1beta1.yaml

@@ -0,0 +1,39 @@
+# A demo CRD for the Kopf example operators.
+# Use it with Kubernetes 1.15 and below.
+# For Kubernetes 1.16 and above, use crd.yaml.
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: kopfexamples.zalando.org
+spec:
+  scope: Namespaced
+  group: zalando.org
+  versions:
+    - name: v1
+      served: true
+      storage: true
+  names:
+    kind: KopfExample
+    plural: kopfexamples
+    singular: kopfexample
+    shortNames:
+      - kopfexes
+      - kopfex
+      - kexes
+      - kex
+  additionalPrinterColumns:
+    - name: Duration
+      type: string
+      priority: 0
+      JSONPath: .spec.duration
+      description: For how long the pod should sleep.
+    - name: Children
+      type: string
+      priority: 0
+      JSONPath: .status.create_fn.children
+      description: The children pods created.
+    - name: Message
+      type: string
+      priority: 0
+      JSONPath: .status.create_fn.message
+      description: As returned from the handler (sometimes).

examples/crd.yaml

@@ -1,15 +1,13 @@
 # A demo CRD for the Kopf example operators.
-apiVersion: apiextensions.k8s.io/v1beta1
+# Use it with Kubernetes 1.16 and above.
+# For Kubernetes 1.15 and below, use crd-v1beta1.yaml.
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: kopfexamples.zalando.org
 spec:
   scope: Namespaced
   group: zalando.org
-  versions:
-    - name: v1
-      served: true
-      storage: true
   names:
     kind: KopfExample
     plural: kopfexamples
@@ -19,19 +17,33 @@ spec:
       - kopfex
       - kexes
       - kex
-  additionalPrinterColumns:
-    - name: Duration
-      type: string
-      priority: 0
-      JSONPath: .spec.duration
-      description: For how long the pod should sleep.
-    - name: Children
-      type: string
-      priority: 0
-      JSONPath: .status.create_fn.children
-      description: The children pods created.
-    - name: Message
-      type: string
-      priority: 0
-      JSONPath: .status.create_fn.message
-      description: As returned from the handler (sometimes).
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+            status:
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+      additionalPrinterColumns:
+        - name: Duration
+          type: string
+          priority: 0
+          jsonPath: .spec.duration
+          description: For how long the pod should sleep.
+        - name: Children
+          type: string
+          priority: 0
+          jsonPath: .status.create_fn.children
+          description: The children pods created.
+        - name: Message
+          type: string
+          priority: 0
+          jsonPath: .status.create_fn.message
+          description: As returned from the handler (sometimes).

peering-v1beta1.yaml

@@ -0,0 +1,46 @@
+# This file is for Kubernetes <= 1.15.
+# For Kubernetes >= 1.16, use peering.yaml.
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: clusterkopfpeerings.zalando.org
+spec:
+  scope: Cluster
+  group: zalando.org
+  names:
+    kind: ClusterKopfPeering
+    plural: clusterkopfpeerings
+    singular: clusterkopfpeering
+  versions:
+    - name: v1
+      served: true
+      storage: true
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: kopfpeerings.zalando.org
+spec:
+  scope: Namespaced
+  group: zalando.org
+  names:
+    kind: KopfPeering
+    plural: kopfpeerings
+    singular: kopfpeering
+  versions:
+    - name: v1
+      served: true
+      storage: true
+---
+apiVersion: zalando.org/v1
+kind: ClusterKopfPeering
+metadata:
+  name: default
+---
+apiVersion: zalando.org/v1
+kind: KopfPeering
+metadata:
+  namespace: default
+  name: default
+---

peering.yaml

@@ -1,35 +1,51 @@
+# This file is for Kubernetes >= 1.16.
+# For Kubernetes <= 1.15, use peering-v1beta1.yaml.
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: clusterkopfpeerings.zalando.org
 spec:
   scope: Cluster
   group: zalando.org
-  versions:
-    - name: v1
-      served: true
-      storage: true
   names:
     kind: ClusterKopfPeering
     plural: clusterkopfpeerings
     singular: clusterkopfpeering
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            status:
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: kopfpeerings.zalando.org
 spec:
   scope: Namespaced
   group: zalando.org
-  versions:
-    - name: v1
-      served: true
-      storage: true
   names:
     kind: KopfPeering
     plural: kopfpeerings
     singular: kopfpeering
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            status:
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
 ---
 apiVersion: zalando.org/v1
 kind: ClusterKopfPeering