Merge pull request #312 from nolar/no-unexpected-warnings-in-tests

Treat all warnings in tests as errors, reduce verbosity

.travis.yml

@@ -39,10 +39,10 @@ before_script:
   - tools/kubernetes-client.sh
 
 script:
-  - pytest -v --cov=kopf --cov-branch
+  - pytest --cov=kopf --cov-branch
   - coveralls
   - codecov --flags unit
-  - pytest -v --only-e2e  # NB: after the coverage uploads!
+  - pytest --only-e2e  # NB: after the coverage uploads!
   - mypy kopf --strict --pretty
 
 deploy:

examples/02-children/example.py

@@ -1,5 +1,5 @@
 import kopf
-import kubernetes.client
+import pykube
 import yaml
 
 
@@ -28,8 +28,10 @@ def create_fn(spec, **kwargs):
     kopf.adopt(doc)
 
     # Actually create an object by requesting the Kubernetes API.
-    api = kubernetes.client.CoreV1Api()
-    pod = api.create_namespaced_pod(namespace=doc['metadata']['namespace'], body=doc)
+    api = pykube.HTTPClient(pykube.KubeConfig.from_env())
+    pod = pykube.Pod(api, doc)
+    pod.create()
+    api.session.close()
 
     # Update the parent's status.
-    return {'children': [pod.metadata.uid]}
+    return {'children': [pod.metadata['uid']]}

examples/09-testing/test_example_09.py

@@ -35,9 +35,11 @@ def test_resource_lifecycle(mocker):
 
     # Run an operator and simulate some activity with the operated resource.
     with kopf.testing.KopfRunner(['run', '--verbose', '--standalone', example_py], timeout=60) as runner:
-        subprocess.run(f"kubectl create -f {obj_yaml}", shell=True, check=True)
+        subprocess.run(f"kubectl create -f {obj_yaml}",
+                       shell=True, check=True, timeout=10, capture_output=True)
         time.sleep(5)  # give it some time to react
-        subprocess.run(f"kubectl delete -f {obj_yaml}", shell=True, check=True)
+        subprocess.run(f"kubectl delete -f {obj_yaml}",
+                       shell=True, check=True, timeout=10, capture_output=True)
         time.sleep(1)  # give it some time to react
 
     # Ensure that the operator did not die on start, or during the operation.

examples/10-builtins/example.py

@@ -5,12 +5,6 @@
 
 tasks = {}  # dict{namespace: dict{name: asyncio.Task}}
 
-try:
-    cfg = pykube.KubeConfig.from_service_account()
-except FileNotFoundError:
-    cfg = pykube.KubeConfig.from_file()
-api = pykube.HTTPClient(cfg)
-
 
 @kopf.on.resume('', 'v1', 'pods')
 @kopf.on.create('', 'v1', 'pods')
@@ -36,8 +30,10 @@ async def pod_killer(namespace, name, logger, timeout=30):
         await asyncio.sleep(timeout)
         logger.info(f"=== Pod killing happens NOW!")
 
+        api = pykube.HTTPClient(pykube.KubeConfig.from_env())
         pod = pykube.Pod.objects(api, namespace=namespace).get_by_name(name)
         pod.delete()
+        api.session.close()
 
     except asyncio.CancelledError:
         logger.info(f"=== Pod killing is cancelled!")

examples/11-filtering-handlers/test_example_11.py

@@ -35,9 +35,11 @@ def test_handler_filtering(mocker):
 
     # Run an operator and simulate some activity with the operated resource.
     with kopf.testing.KopfRunner(['run', '--verbose', '--standalone', example_py]) as runner:
-        subprocess.run(f"kubectl create -f {obj_yaml}", shell=True, check=True)
+        subprocess.run(f"kubectl create -f {obj_yaml}",
+                       shell=True, check=True, timeout=10, capture_output=True)
         time.sleep(5)  # give it some time to react
-        subprocess.run(f"kubectl delete -f {obj_yaml}", shell=True, check=True)
+        subprocess.run(f"kubectl delete -f {obj_yaml}",
+                       shell=True, check=True, timeout=10, capture_output=True)
         time.sleep(1)  # give it some time to react
 
     # Ensure that the operator did not die on start, or during the operation.

examples/12-embedded/example.py

@@ -1,4 +1,5 @@
 import asyncio
+import contextlib
 import threading
 import time
 
@@ -22,13 +23,14 @@ def kopf_thread(
 ):
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
+    with contextlib.closing(loop):
 
-    kopf.configure(verbose=True)  # log formatting
+        kopf.configure(verbose=True)  # log formatting
 
-    loop.run_until_complete(kopf.operator(
-        ready_flag=ready_flag,
-        stop_flag=stop_flag,
-    ))
+        loop.run_until_complete(kopf.operator(
+            ready_flag=ready_flag,
+            stop_flag=stop_flag,
+        ))
 
 
 def main(steps=3):

examples/99-all-at-once/example.py

@@ -18,12 +18,6 @@
 E2E_FAILURE_COUNTS = {}
 E2E_TRACEBACKS = True
 
-try:
-    cfg = pykube.KubeConfig.from_service_account()
-except FileNotFoundError:
-    cfg = pykube.KubeConfig.from_file()
-api = pykube.HTTPClient(cfg)
-
 
 @kopf.on.startup()
 async def startup_fn_simple(logger, **kwargs):
@@ -112,8 +106,10 @@ def create_pod(**kwargs):
     kopf.label(pod_data, {'application': 'kopf-example-10'})
 
     # Actually create an object by requesting the Kubernetes API.
+    api = pykube.HTTPClient(pykube.KubeConfig.from_env())
     pod = pykube.Pod(api, pod_data)
     pod.create()
+    api.session.close()
 
 
 @kopf.on.event('', 'v1', 'pods', labels={'application': 'kopf-example-10'})

kopf/config.py

@@ -48,9 +48,12 @@ def configure(
         del logger.handlers[1:]  # everything except the default NullHandler
 
     # Prevent the low-level logging unless in the debug verbosity mode. Keep only the operator's messages.
-    logging.getLogger('urllib3').propagate = bool(debug)
-    logging.getLogger('asyncio').propagate = bool(debug)
-    logging.getLogger('kubernetes').propagate = bool(debug)
+    # For no-propagation loggers, add a dummy null handler to prevent printing the messages.
+    for name in ['urllib3', 'asyncio', 'kubernetes']:
+        logger = logging.getLogger(name)
+        logger.propagate = bool(debug)
+        if not debug:
+            logger.handlers[:] = [logging.NullHandler()]
 
     loop = asyncio.get_event_loop()
     loop.set_debug(bool(debug))

kopf/reactor/causation.py

@@ -125,7 +125,7 @@ class ResourceChangingCause(ResourceCause):
 
     @property
     def event(self) -> Reason:
-        warnings.warn("`cause.event` is deprecated; use `cause.reason`.", DeprecationWarning)
+        warnings.warn("cause.event is deprecated; use cause.reason.", DeprecationWarning)
         return self.reason
 
     @property

kopf/reactor/handlers.py

@@ -64,5 +64,5 @@ class ResourceHandler(BaseHandler):
 
     @property
     def event(self) -> Optional[causation.Reason]:
-        warnings.warn("`handler.event` is deprecated; use `handler.reason`.", DeprecationWarning)
+        warnings.warn("handler.event is deprecated; use handler.reason.", DeprecationWarning)
         return self.reason

kopf/toolkits/runner.py

@@ -67,7 +67,6 @@ def __init__(
         self.kwargs = kwargs
         self.reraise = reraise
         self.timeout = timeout
-        self._loop = asyncio.new_event_loop()
         self._stop = threading.Event()
         self._ready = threading.Event()  # NB: not asyncio.Event!
         self._thread = threading.Thread(target=self._target)
@@ -113,7 +112,8 @@ def _target(self) -> None:
 
         # Every thread must have its own loop. The parent thread (pytest)
         # needs to know when the loop is set up, to be able to shut it down.
-        asyncio.set_event_loop(self._loop)
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
         self._ready.set()
 
         # Execute the requested CLI command in the thread & thread's loop.
@@ -126,6 +126,17 @@ def _target(self) -> None:
             self._future.set_exception(e)
         else:
             self._future.set_result(result)
+        finally:
+
+            # Shut down the API-watching streams.
+            loop.run_until_complete(loop.shutdown_asyncgens())
+
+            # Shut down the transports and prevent ResourceWarning: unclosed transport.
+            # See: https://docs.aiohttp.org/en/stable/client_advanced.html#graceful-shutdown
+            # TODO: Try a hack: https://github.com/aio-libs/aiohttp/issues/1925#issuecomment-575754386
+            loop.run_until_complete(asyncio.sleep(1.0))
+
+            loop.close()
 
     @property
     def future(self) -> ResultFuture:

kopf/utilities/piggybacking.py

@@ -11,6 +11,7 @@
     :mod:`credentials` and :func:`authentication`.
 """
 import logging
+import warnings
 from typing import Any, Union, Optional, Sequence
 
 from kopf.clients import auth
@@ -89,7 +90,9 @@ def login_via_pykube(
     # to inject custom authentication methods. Support these hacks if possible.
     config: pykube.KubeConfig
     try:
-        config = auth.get_pykube_cfg()
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            config = auth.get_pykube_cfg()
         logger.debug("Pykube is configured via monkey-patched get_pykube_cfg().")
     except NotImplementedError:
         try:

setup.py

@@ -2,8 +2,10 @@
 
 from setuptools import setup, find_packages
 
-LONG_DESCRIPTION = open(os.path.join(os.path.dirname(__file__), 'README.md')).read()
-DESCRIPTION = LONG_DESCRIPTION.splitlines()[0].lstrip('#').strip()
+with open(os.path.join(os.path.dirname(__file__), 'README.md')) as f:
+    LONG_DESCRIPTION = f.read()
+    DESCRIPTION = LONG_DESCRIPTION.splitlines()[0].lstrip('#').strip()
+
 PROJECT_URLS = {
     'Documentation': 'https://kopf.readthedocs.io',
     'Bug Tracker': 'https://github.com/zalando-incubator/kopf/issues',

tests/authentication/test_credentials.py

@@ -141,9 +141,10 @@ async def test_basic_auth(vault):
     })
     session = await fn()
 
-    assert session._default_auth.login == 'username'
-    assert session._default_auth.password == 'password'
-    assert 'Authorization' not in session._default_headers
+    async with session:
+        assert session._default_auth.login == 'username'
+        assert session._default_auth.password == 'password'
+        assert 'Authorization' not in session._default_headers
 
 
 async def test_header_with_token_only(vault):
@@ -155,8 +156,9 @@ async def test_header_with_token_only(vault):
     })
     session = await fn()
 
-    assert session._default_auth is None
-    assert session._default_headers['Authorization'] == 'Bearer token'
+    async with session:
+        assert session._default_auth is None
+        assert session._default_headers['Authorization'] == 'Bearer token'
 
 
 async def test_header_with_schema_only(vault):
@@ -168,8 +170,9 @@ async def test_header_with_schema_only(vault):
     })
     session = await fn()
 
-    assert session._default_auth is None
-    assert session._default_headers['Authorization'] == 'Digest xyz'
+    async with session:
+        assert session._default_auth is None
+        assert session._default_headers['Authorization'] == 'Digest xyz'
 
 
 async def test_header_with_schema_and_token(vault):
@@ -182,8 +185,9 @@ async def test_header_with_schema_and_token(vault):
     })
     session = await fn()
 
-    assert session._default_auth is None
-    assert session._default_headers['Authorization'] == 'Digest xyz'
+    async with session:
+        assert session._default_auth is None
+        assert session._default_headers['Authorization'] == 'Digest xyz'
 
 
 async def test_ca_insecure(vault, cafile):
@@ -195,8 +199,9 @@ async def test_ca_insecure(vault, cafile):
     })
     session = await fn()
 
-    ctx = session.connector._ssl
-    assert ctx.verify_mode == ssl.CERT_NONE
+    async with session:
+        ctx = session.connector._ssl
+        assert ctx.verify_mode == ssl.CERT_NONE
 
 
 async def test_ca_as_path(vault, cafile):
@@ -208,10 +213,11 @@ async def test_ca_as_path(vault, cafile):
     })
     session = await fn()
 
-    ctx = session.connector._ssl
-    assert len(ctx.get_ca_certs()) == 1
-    assert ctx.cert_store_stats()['x509'] == 1
-    assert ctx.cert_store_stats()['x509_ca'] == 1
+    async with session:
+        ctx = session.connector._ssl
+        assert len(ctx.get_ca_certs()) == 1
+        assert ctx.cert_store_stats()['x509'] == 1
+        assert ctx.cert_store_stats()['x509_ca'] == 1
 
 
 async def test_ca_as_data(vault, cabase64):
@@ -223,10 +229,11 @@ async def test_ca_as_data(vault, cabase64):
     })
     session = await fn()
 
-    ctx = session.connector._ssl
-    assert len(ctx.get_ca_certs()) == 1
-    assert ctx.cert_store_stats()['x509'] == 1
-    assert ctx.cert_store_stats()['x509_ca'] == 1
+    async with session:
+        ctx = session.connector._ssl
+        assert len(ctx.get_ca_certs()) == 1
+        assert ctx.cert_store_stats()['x509'] == 1
+        assert ctx.cert_store_stats()['x509_ca'] == 1
 
 
 # TODO: find a way to test that the client certificates/pkeys are indeed loaded.
@@ -240,7 +247,10 @@ async def test_clientcert_as_path(vault, cafile, certfile, pkeyfile):
             private_key_path=pkeyfile,
         ),
     })
-    await fn()
+    session = await fn()
+
+    async with session:
+        pass
 
 
 async def test_clientcert_as_data(vault, cafile, certbase64, pkeybase64):
@@ -252,4 +262,7 @@ async def test_clientcert_as_data(vault, cafile, certbase64, pkeybase64):
             private_key_data=pkeybase64,
         ),
     })
-    await fn()
+    session = await fn()
+
+    async with session:
+        pass