Unit tests for #999 (k8s cache PodProcessor infinite loop issue)

CHANGELOG.md

@@ -1,6 +1,14 @@
 Scalyr Agent 2 Changes By Release
 =================================
 
+## 2.1.37 "TBD" - November 15, 2022
+<!---
+Packaged by Joseph Makar <[email protected]> on Sep 17, 2022 12:31 -0800
+--->
+
+Kubernetes:
+* Fix a bug / edge case in the Kubernetes caching PodProcessor code which could cause an agent to get stuck in an infinite loop when processing controllers which have a custom Kind which is not supported by the agent defined. Contributed by #xdvpser #998 #999.
+
 ## 2.1.36 "Corrntos" - September 15, 2022
 <!---
 Packaged by Joseph Makar <[email protected]> on Sep 17, 2022 12:31 -0800

scalyr_agent/monitor_utils/k8s.py

@@ -878,14 +878,22 @@ def _get_controller_from_owners(self, k8s, owners, namespace, query_options=None
             if controller.parent_name is None:
                 global_log.log(
                     scalyr_logging.DEBUG_LEVEL_1,
-                    "controller %s has no parent name" % controller.name,
+                    "controller %s has no parent name, ignoring" % controller.name,
                 )
                 break
 
             if controller.parent_kind is None:
                 global_log.log(
                     scalyr_logging.DEBUG_LEVEL_1,
-                    "controller %s has no parent kind" % controller.name,
+                    "controller %s has no parent kind, ignoring" % controller.name,
+                )
+                break
+
+            if controller.parent_kind not in _OBJECT_ENDPOINTS:
+                global_log.log(
+                    scalyr_logging.DEBUG_LEVEL_1,
+                    "parent of controller %s is not standard k8s object (got=%s), ignoring"
+                    % (controller.name, controller.parent_kind),
                 )
                 break
 

tests/unit/monitor_utils/k8s_test.py

@@ -25,6 +25,7 @@
 from scalyr_agent.monitor_utils.k8s import (
     _K8sCache,
     _K8sProcessor,
+    PodProcessor,
     KubernetesApi,
     K8sApiNotFoundException,
     K8sApiTemporaryError,
@@ -991,6 +992,155 @@ def test_ne(self):
         self.assertFalse(blacklist_filter_a != blacklist_filter_c)
 
 
+class PodProcessorTestCase(ScalyrTestCase):
+    @patch("scalyr_agent.monitor_utils.k8s.global_log")
+    def test_get_controller_from_owners_edge_cases_are_handled_correctly(
+        self, mock_global_log
+    ):
+        mock_controllers = mock.Mock()
+
+        mock_k8s = None
+        mock_obj = {
+            "metadata": {
+                "ownerReferences": [
+                    # Valid kind, should be included
+                    {
+                        "name": "name1",
+                        "kind": "CronJob",
+                        "controller": "controller1",
+                    },
+                    # No name, should be ignored
+                    {
+                        "name": "name2",
+                        "kind": "DaemonSet",
+                        "controller": "controller2",
+                    },
+                    # No kind, should be ignored
+                    {
+                        "name": "name3",
+                        "kind": "DaemonSet",
+                        "controller": "controller3",
+                    },
+                    # Invalid kind, should be ignored
+                    {
+                        "name": "name4",
+                        "kind": "invalid",
+                        "controller": "controller4",
+                    },
+                ]
+            }
+        }
+
+        # 1. Valid controller
+        def mock_lookup_1(*args, **kwargs):
+            if mock_lookup_1.counter == 0:
+                # Valid controller
+                result = mock.Mock(
+                    id="1", parent_name="parent1", parent_kind="DaemonSet"
+                )
+            elif mock_lookup_1.counter == 1:
+                return None
+
+            mock_lookup_1.counter += 1
+            return result
+
+        mock_lookup_1.counter = 0
+
+        self.assertEqual(mock_global_log.log.call_count, 0)
+
+        mock_controllers.lookup = mock_lookup_1
+        processor = PodProcessor(controllers=mock_controllers)
+        result = processor.process_object(k8s=mock_k8s, obj=mock_obj)
+
+        self.assertEqual(result.controller.id, "1")
+        self.assertEqual(result.controller.parent_name, "parent1")
+        self.assertEqual(result.controller.parent_kind, "DaemonSet")
+        self.assertEqual(mock_global_log.log.call_count, 2)
+
+        # 2. Invalid controller, no parent_name, should be ignored
+        mock_global_log.reset_mock()
+
+        def mock_lookup_2(*args, **kwargs):
+            if mock_lookup_2.counter == 0:
+                # Valid controller
+                result = mock.Mock(id="2", parent_name=None, parent_kind="CronJob")
+            elif mock_lookup_2.counter == 1:
+                return None
+
+            mock_lookup_2.counter += 1
+            return result
+
+        mock_lookup_2.counter = 0
+
+        self.assertEqual(mock_global_log.log.call_count, 0)
+
+        mock_controllers.lookup = mock_lookup_2
+        processor = PodProcessor(controllers=mock_controllers)
+        result = processor.process_object(k8s=mock_k8s, obj=mock_obj)
+        self.assertEqual(result.controller.id, "2")
+        self.assertEqual(result.controller.parent_name, None)
+        self.assertEqual(result.controller.parent_kind, "CronJob")
+        self.assertEqual(mock_global_log.log.call_count, 2 + 1)
+        self.assertTrue(
+            "has no parent name, ignoring"
+            in mock_global_log.log.call_args_list[0][0][1]
+        )
+
+        # 3. Invalid controller, no parent_kind, should be ignored
+        mock_global_log.reset_mock()
+
+        def mock_lookup_3(*args, **kwargs):
+            if mock_lookup_3.counter == 0:
+                # Valid controller
+                result = mock.Mock(id="3", parent_name="parent3", parent_kind=None)
+            elif mock_lookup_3.counter == 1:
+                return None
+
+            mock_lookup_3.counter += 1
+            return result
+
+        mock_lookup_3.counter = 0
+
+        mock_controllers.lookup = mock_lookup_3
+        processor = PodProcessor(controllers=mock_controllers)
+        result = processor.process_object(k8s=mock_k8s, obj=mock_obj)
+        self.assertEqual(result.controller.id, "3")
+        self.assertEqual(result.controller.parent_name, "parent3")
+        self.assertEqual(result.controller.parent_kind, None)
+        self.assertEqual(mock_global_log.log.call_count, 2 + 1)
+        self.assertTrue(
+            "has no parent kind, ignoring"
+            in mock_global_log.log.call_args_list[0][0][1]
+        )
+
+        # 4. Invalid controller, invalid parent_kind, should be ignored
+        mock_global_log.reset_mock()
+
+        def mock_lookup_4(*args, **kwargs):
+            if mock_lookup_4.counter == 0:
+                # Valid controller
+                result = mock.Mock(id="4", parent_name="parent4", parent_kind="Invalid")
+            elif mock_lookup_4.counter == 1:
+                return None
+
+            mock_lookup_4.counter += 1
+            return result
+
+        mock_lookup_4.counter = 0
+
+        mock_controllers.lookup = mock_lookup_4
+        processor = PodProcessor(controllers=mock_controllers)
+        result = processor.process_object(k8s=mock_k8s, obj=mock_obj)
+        self.assertEqual(result.controller.id, "4")
+        self.assertEqual(result.controller.parent_name, "parent4")
+        self.assertEqual(result.controller.parent_kind, "Invalid")
+        self.assertEqual(mock_global_log.log.call_count, 2 + 1)
+        self.assertTrue(
+            "is not standard k8s object (got=Invalid), ignoring"
+            in mock_global_log.log.call_args_list[0][0][1]
+        )
+
+
 class DockerClientFaker(object):
     """A fake DockerClient that only supports the `stats` call.  Used for tests to control when a `stats` call
     should finish and what it should return.