Fail the integration tests on warnings or errors. (#581)

Fails the integration tests if any unknown warnings or errors are found in the logs of the Nighthawk test server.
Includes an ignore list that (for now) accepts known warnings found in the logs, the ignore list can have different content per each test case.

Also:
- moving the `Attributes:` docstring section on `IntegrationTestBase` into its class docstring. `__init__` is just a method and should only have the `Args:` section.

Fixes #577

Signed-off-by: Jakub Sobon <[email protected]>

benchmarks/envoy_proxy.py

@@ -28,13 +28,21 @@ class EnvoyProxyServer(NighthawkTestServer):
     See InjectHttpProxyIntegrationTestBase below for usage.
   """
 
-  def __init__(self, config_template_path, server_ip, ip_version, parameters=dict(), tag=""):
+  def __init__(self,
+               config_template_path,
+               server_ip,
+               ip_version,
+               request,
+               parameters=dict(),
+               tag=""):
     """Initialize an EnvoyProxyServer instance.
 
     Arguments:
       config_template_path: Configuration template for the proxy.
       server_ip: IP address for the proxy to use.
       ip_version: IP version that the proxy should use when listening.
+      request: The pytest `request` test fixture used to determine information
+        about the currently executing test case.
       parameters: Dictionary. Supply this to provide template parameter replacement values (optional).
       tag: String. Supply this to get recognizeable output locations (optional).
     """
@@ -43,6 +51,7 @@ def __init__(self, config_template_path, server_ip, ip_version, parameters=dict(
                                            config_template_path,
                                            server_ip,
                                            ip_version,
+                                           request,
                                            parameters=parameters,
                                            tag=tag)
     self.docker_image = os.getenv("ENVOY_DOCKER_IMAGE_TO_TEST", "")
@@ -61,15 +70,16 @@ class InjectHttpProxyIntegrationTestBase(HttpIntegrationTestBase):
   which directs traffic to that. Both will be listing for plain http traffic.
   """
 
-  def __init__(self, ip_version, server_config, proxy_config):
+  def __init__(self, request, server_config, proxy_config):
     """Initialize an InjectHttpProxyIntegrationTestBase.
 
     Arguments:
-      ip_version: Use ipv4 or ipv6
+      request: The pytest `request` test fixture used to determine information
+        about the currently executing test case.
       server_config: Path to the server configuration.
       proxy_config: Path to the proxy configuration.
     """
-    super(InjectHttpProxyIntegrationTestBase, self).__init__(ip_version, server_config)
+    super(InjectHttpProxyIntegrationTestBase, self).__init__(request, server_config)
     self._proxy_config = proxy_config
 
   def setUp(self):
@@ -85,16 +95,17 @@ def setUp(self):
     proxy_server = EnvoyProxyServer(self._proxy_config,
                                     self.server_ip,
                                     self.ip_version,
+                                    self.request,
                                     parameters=self.parameters,
                                     tag=self.tag)
     assert (proxy_server.start())
     logging.info("envoy proxy listening at {ip}:{port}".format(ip=proxy_server.server_ip,
                                                                port=proxy_server.server_port))
     self.proxy_server = proxy_server
 
-  def tearDown(self):
+  def tearDown(self, caplog):
     """Tear down the proxy and test server. Assert that both exit succesfully."""
-    super(InjectHttpProxyIntegrationTestBase, self).tearDown()
+    super(InjectHttpProxyIntegrationTestBase, self).tearDown(caplog)
     assert (self.proxy_server.stop() == 0)
 
   def getTestServerRootUri(self):
@@ -106,7 +117,7 @@ def getTestServerRootUri(self):
 
 
 @pytest.fixture(params=determineIpVersionsFromEnvironment())
-def inject_envoy_http_proxy_fixture(request, server_config, proxy_config):
+def inject_envoy_http_proxy_fixture(request, server_config, proxy_config, caplog):
   """Injects an Envoy proxy in front of the test server.
 
   NOTE: Depends on the proxy_config fixture, which must be explicitly imported
@@ -116,10 +127,11 @@ def inject_envoy_http_proxy_fixture(request, server_config, proxy_config):
     request: supplies the ip version.
     server_config: path to the server configuration template.
     proxy_config: path to the proxy configuration template.
+    caplog: The pytest `caplog` test fixture used to examine logged messages.
 
   Yields: a successfully set up InjectHttpProxyIntegrationTestBase instance.
   """
-  fixture = InjectHttpProxyIntegrationTestBase(request.param, server_config, proxy_config)
+  fixture = InjectHttpProxyIntegrationTestBase(request, server_config, proxy_config)
   fixture.setUp()
   yield fixture
-  fixture.tearDown()
+  fixture.tearDown(caplog)

test/integration/integration_test_fixtures.py

@@ -54,30 +54,35 @@ class IntegrationTestBase():
   work when there is only one test server.
 
   This class will be refactored (https://github.com/envoyproxy/nighthawk/issues/258).
+
+  Attributes:
+    ip_version: IP version that the proxy should use when listening.
+    server_ip: string containing the server ip that will be used to listen
+    tag: String. Supply this to get recognizeable output locations.
+    parameters: Dictionary. Supply this to provide template parameter replacement values.
+    grpc_service: NighthawkGrpcService instance or None. Set by startNighthawkGrpcService().
+    test_server: NighthawkTestServer instance, set during setUp().
+    nighthawk_client_path: String, path to the nighthawk_client binary.
+    request: The pytest `request` test fixture used to determine information
+      about the currently executing test case.
   """
 
-  def __init__(self, ip_version, server_config, backend_count=1, bootstrap_version_arg=None):
+  def __init__(self, request, server_config, backend_count=1, bootstrap_version_arg=None):
     """Initialize the IntegrationTestBase instance.
 
     Args:
       ip_version: a single IP mode that this instance will test: IpVersion.IPV4 or IpVersion.IPV6
+      request: The pytest `request` test fixture used to determine information
+        about the currently executing test case.
       server_config: path to the server configuration
       backend_count: number of Nighthawk Test Server backends to run, to allow testing MultiTarget mode
       bootstrap_version_arg: An optional int, specify a bootstrap cli argument value for the test server binary. If None is specified, no bootstrap cli argment will be passed.
-
-    Attributes:
-      ip_version: IP version that the proxy should use when listening.
-      server_ip: string containing the server ip that will be used to listen
-      tag: String. Supply this to get recognizeable output locations.
-      parameters: Dictionary. Supply this to provide template parameter replacement values.
-      grpc_service: NighthawkGrpcService instance or None. Set by startNighthawkGrpcService().
-      test_server: NighthawkTestServer instance, set during setUp().
-      nighthawk_client_path: String, path to the nighthawk_client binary.
     """
     super(IntegrationTestBase, self).__init__()
-    assert ip_version != IpVersion.UNKNOWN
-    self.ip_version = ip_version
-    self.server_ip = "::" if ip_version == IpVersion.IPV6 else "0.0.0.0"
+    self.request = request
+    self.ip_version = request.param
+    assert self.ip_version != IpVersion.UNKNOWN
+    self.server_ip = "::" if self.ip_version == IpVersion.IPV6 else "0.0.0.0"
     self.server_ip = os.getenv("TEST_SERVER_EXTERNAL_IP", self.server_ip)
     self.tag = ""
     self.parameters = {}
@@ -88,7 +93,7 @@ def __init__(self, ip_version, server_config, backend_count=1, bootstrap_version
     self._nighthawk_test_config_path = server_config
     self._nighthawk_service_path = "nighthawk_service"
     self._nighthawk_output_transform_path = "nighthawk_output_transform"
-    self._socket_type = socket.AF_INET6 if ip_version == IpVersion.IPV6 else socket.AF_INET
+    self._socket_type = socket.AF_INET6 if self.ip_version == IpVersion.IPV6 else socket.AF_INET
     self._test_servers = []
     self._backend_count = backend_count
     self._test_id = ""
@@ -126,8 +131,14 @@ def setUp(self):
     self.tag = "{timestamp}/{test_id}".format(timestamp=_TIMESTAMP, test_id=self._test_id)
     assert self._tryStartTestServers(), "Test server(s) failed to start"
 
-  def tearDown(self):
-    """Stop the server."""
+  def tearDown(self, caplog):
+    """Stop the server.
+
+    Fails the test if any warnings or errors were logged.
+
+    Args:
+      caplog: The pytest `caplog` test fixture used to examine logged messages.
+    """
     if self.grpc_service is not None:
       assert (self.grpc_service.stop() == 0)
 
@@ -137,12 +148,22 @@ def tearDown(self):
         any_failed = True
     assert (not any_failed)
 
+    warnings_and_errors = []
+    for when in ("setup", "call", "teardown"):
+      for record in caplog.get_records(when):
+        if record.levelno not in (logging.WARNING, logging.ERROR):
+          continue
+        warnings_and_errors.append(record.message)
+    if warnings_and_errors:
+      pytest.fail("warnings or errors encountered during testing:\n{}".format(warnings_and_errors))
+
   def _tryStartTestServers(self):
     for i in range(self._backend_count):
       test_server = NighthawkTestServer(self._nighthawk_test_server_path,
                                         self._nighthawk_test_config_path,
                                         self.server_ip,
                                         self.ip_version,
+                                        self.request,
                                         parameters=self.parameters,
                                         tag=self.tag,
                                         bootstrap_version_arg=self._bootstrap_version_arg)
@@ -292,9 +313,9 @@ class HttpIntegrationTestBase(IntegrationTestBase):
   by pytest.
   """
 
-  def __init__(self, ip_version, server_config):
+  def __init__(self, request, server_config):
     """See base class."""
-    super(HttpIntegrationTestBase, self).__init__(ip_version, server_config)
+    super(HttpIntegrationTestBase, self).__init__(request, server_config)
 
   def getTestServerRootUri(self):
     """See base class."""
@@ -309,10 +330,10 @@ class HttpIntegrationTestBaseWithEnvoyDeprecatedV2Bootstrap(IntegrationTestBase)
   by pytest.
   """
 
-  def __init__(self, ip_version, server_config):
+  def __init__(self, request, server_config):
     """See base class."""
     super(HttpIntegrationTestBaseWithEnvoyDeprecatedV2Bootstrap,
-          self).__init__(ip_version, server_config, bootstrap_version_arg=2)
+          self).__init__(request, server_config, bootstrap_version_arg=2)
 
   def getTestServerRootUri(self):
     """See base class."""
@@ -323,10 +344,9 @@ def getTestServerRootUri(self):
 class MultiServerHttpIntegrationTestBase(IntegrationTestBase):
   """Base for running plain http tests against multiple Nighthawk test servers."""
 
-  def __init__(self, ip_version, server_config, backend_count):
+  def __init__(self, request, server_config, backend_count):
     """See base class."""
-    super(MultiServerHttpIntegrationTestBase, self).__init__(ip_version, server_config,
-                                                             backend_count)
+    super(MultiServerHttpIntegrationTestBase, self).__init__(request, server_config, backend_count)
 
   def getTestServerRootUri(self):
     """See base class."""
@@ -340,9 +360,9 @@ def getAllTestServerRootUris(self):
 class HttpsIntegrationTestBase(IntegrationTestBase):
   """Base for https tests against the Nighthawk test server."""
 
-  def __init__(self, ip_version, server_config):
+  def __init__(self, request, server_config):
     """See base class."""
-    super(HttpsIntegrationTestBase, self).__init__(ip_version, server_config)
+    super(HttpsIntegrationTestBase, self).__init__(request, server_config)
 
   def getTestServerRootUri(self):
     """See base class."""
@@ -352,9 +372,9 @@ def getTestServerRootUri(self):
 class SniIntegrationTestBase(HttpsIntegrationTestBase):
   """Base for https/sni tests against the Nighthawk test server."""
 
-  def __init__(self, ip_version, server_config):
+  def __init__(self, request, server_config):
     """See base class."""
-    super(SniIntegrationTestBase, self).__init__(ip_version, server_config)
+    super(SniIntegrationTestBase, self).__init__(request, server_config)
 
   def getTestServerRootUri(self):
     """See base class."""
@@ -364,10 +384,9 @@ def getTestServerRootUri(self):
 class MultiServerHttpsIntegrationTestBase(IntegrationTestBase):
   """Base for https tests against multiple Nighthawk test servers."""
 
-  def __init__(self, ip_version, server_config, backend_count):
+  def __init__(self, request, server_config, backend_count):
     """See base class."""
-    super(MultiServerHttpsIntegrationTestBase, self).__init__(ip_version, server_config,
-                                                              backend_count)
+    super(MultiServerHttpsIntegrationTestBase, self).__init__(request, server_config, backend_count)
 
   def getTestServerRootUri(self):
     """See base class."""
@@ -389,65 +408,65 @@ def server_config():
 
 
 @pytest.fixture(params=determineIpVersionsFromEnvironment())
-def http_test_server_fixture(request, server_config):
+def http_test_server_fixture(request, server_config, caplog):
   """Fixture for setting up a test environment with the stock http server configuration.
 
   Yields:
       HttpIntegrationTestBase: A fully set up instance. Tear down will happen automatically.
   """
-  f = HttpIntegrationTestBase(request.param, server_config)
+  f = HttpIntegrationTestBase(request, server_config)
   f.setUp()
   yield f
-  f.tearDown()
+  f.tearDown(caplog)
 
 
 @pytest.fixture(params=determineIpVersionsFromEnvironment())
-def http_test_server_fixture_envoy_deprecated_v2_api(request, server_config):
+def http_test_server_fixture_envoy_deprecated_v2_api(request, server_config, caplog):
   """Fixture for setting up a test environment with http server configuration that uses v2 configuration.
 
   Yields:
       HttpIntegrationTestBaseWithEnvoyDeprecatedV2Bootstrap: A fully set up instance. Tear down will happen automatically.
   """
-  f = HttpIntegrationTestBaseWithEnvoyDeprecatedV2Bootstrap(request.param, server_config)
+  f = HttpIntegrationTestBaseWithEnvoyDeprecatedV2Bootstrap(request, server_config)
   f.setUp()
   yield f
-  f.tearDown()
+  f.tearDown(caplog)
 
 
 @pytest.fixture(params=determineIpVersionsFromEnvironment())
-def https_test_server_fixture(request, server_config):
+def https_test_server_fixture(request, server_config, caplog):
   """Fixture for setting up a test environment with the stock https server configuration.
 
   Yields:
       HttpsIntegrationTestBase: A fully set up instance. Tear down will happen automatically.
   """
-  f = HttpsIntegrationTestBase(request.param, server_config)
+  f = HttpsIntegrationTestBase(request, server_config)
   f.setUp()
   yield f
-  f.tearDown()
+  f.tearDown(caplog)
 
 
 @pytest.fixture(params=determineIpVersionsFromEnvironment())
-def multi_http_test_server_fixture(request, server_config):
+def multi_http_test_server_fixture(request, server_config, caplog):
   """Fixture for setting up a test environment with multiple servers, using the stock http server configuration.
 
   Yields:
       MultiServerHttpIntegrationTestBase: A fully set up instance. Tear down will happen automatically.
   """
-  f = MultiServerHttpIntegrationTestBase(request.param, server_config, backend_count=3)
+  f = MultiServerHttpIntegrationTestBase(request, server_config, backend_count=3)
   f.setUp()
   yield f
-  f.tearDown()
+  f.tearDown(caplog)
 
 
 @pytest.fixture(params=determineIpVersionsFromEnvironment())
-def multi_https_test_server_fixture(request, server_config):
+def multi_https_test_server_fixture(request, server_config, caplog):
   """Fixture for setting up a test environment with multiple servers, using the stock https server configuration.
 
   Yields:
       MultiServerHttpsIntegrationTestBase: A fully set up instance. Tear down will happen automatically.
   """
-  f = MultiServerHttpsIntegrationTestBase(request.param, server_config, backend_count=3)
+  f = MultiServerHttpsIntegrationTestBase(request, server_config, backend_count=3)
   f.setUp()
   yield f
-  f.tearDown()
+  f.tearDown(caplog)