From 5b24a51bf418cd4607215fd59a0ef504f9f23f29 Mon Sep 17 00:00:00 2001 From: Jeff Nickoloff Date: Tue, 29 Aug 2023 22:14:42 -0700 Subject: [PATCH] latency, exception, and custom behavior Signed-off-by: Jeff Nickoloff --- README.md | 212 +++++++++++++++++++++++++++++- src/failureflags/__init__.py | 130 ++++++++++++------ test/behaviors_test.py | 247 +++++++++++++++++++++++++++++++++++ test/demo_test.py | 43 ++++++ test/fetch_test.py | 73 +++++++++++ test/invoke_test.py | 122 +++++++++++++++++ 6 files changed, 787 insertions(+), 40 deletions(-) create mode 100644 test/behaviors_test.py create mode 100644 test/demo_test.py create mode 100644 test/fetch_test.py create mode 100644 test/invoke_test.py diff --git a/README.md b/README.md index 2dbd239..2041b06 100644 --- a/README.md +++ b/README.md @@ -1 +1,211 @@ -# failure-flags-python +# failure-flags + +Failure Flags is a python SDK for building application-level chaos experiments and reliability tests using the Gremlin Fault Injection platform. This library works in concert with Gremlin-Lambda, a Lambda Extension; or Gremlin-Sidecar, a container sidecar agent. This architecture minimizes the impact to your application code, simplifies configuration, and makes adoption painless. + +Just like feature flags, Failure Flags are safe to add to and leave in your application. Failure Flags will always fail safe if it cannot communicate with its sidecar or its sidecar is misconfigured. + +Take three steps to run an application-level experiment with Failure Flags: + +1. Instrument your code with this SDK +2. Configure and deploy your code along side one of the Failure Flag sidecars +3. Run an Experiment with the console, API, or command line + +## Instrumenting Your Code + +You can get started by adding failureflags to your package dependencies: + +```sh +pip install failureflags +``` + +Then instrument the part of your application where you want to inject faults. + +```python +from failureflags import FailureFlag + +... + +FailureFlag(name: 'flagname', labels: {}).invoke() + +... +``` + +The best spots to add a failure flag are just before or just after a call to one of your network dependencies like a database or other network service. Or you can instrument your request handler and affect the way your application responses to its callers. Here's a simple Lambda example: + +```python +# Change 1: Bring in the failureflags module +from failureflags import FailureFlag + +import os +import logging +import time +from aws_xray_sdk.core import xray_recorder +from aws_xray_sdk.core import patch_all + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +patch_all() + +def lambda_handler(event, context): + start = time.time() + + # Change 2: add a FailureFlag to your code + FailureFlag("http-ingress", {}, debug=True).invoke() + + end = time.time() + return { + 'statusCode': 200, + 'headers': { + 'Content-Type': 'application/json' + }, + 'body': { + 'processingTime': f"{start - end}", + 'isActive': active, + 'isImpacted': impacted + } + } +``` + +*Don't forget to enable the SDK by setting the FAILURE_FLAGS_ENABLED environment variable!* If this environment variable is not set then the SDK will short-circuit and no attempt to fetch experiments will be made. + +You can always bring your own behaviors and effects by providing a behavior function. Here's another Lambda example that writes the experiment data to the console instead of changing the application behavior: + +```python +# Change 1: Bring in the failureflags module +from failureflags import FailureFlag, defaultBehavior + +import os +import logging +import time +from aws_xray_sdk.core import xray_recorder +from aws_xray_sdk.core import patch_all + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +patch_all() + +def customBehavior(ff, experiments): + logger.debug(experiments) + return defaultBehavior(ff, experiments) + +def lambda_handler(event, context): + start = time.time() + + # Change 2: add a FailureFlag to your code + FailureFlag("http-ingress", {}, debug=True, behavior=customBehavior).invoke() + + end = time.time() + return { + 'statusCode': 200, + 'headers': { + 'Content-Type': 'application/json' + }, + 'body': { + 'processingTime': f"{start - end}", + 'isActive': active, + 'isImpacted': impacted + } + } +``` + +### Doing Something Different + +Sometimes you need even more manual control. For example, in the event of an experiment you might not want to make some API call or need to rollback some transaction. In most cases the Exception effect can help, but the `invoke` function also returns a boolean to indicate if there was an experiment. You can use that to create branches in your code like you would for any feature flag. + +```python +... +active, impacted, experiments = FailureFlag("myFlag", {}).invoke() +if active and impacted: + // if there is a running experiment then do this +else: + // if there is no experiment then do this +... +``` + +### Pulling the Experiment and Branching Manually + +If you want to work with lower-level Experiment data you can use `fetch` directly. + +## Targeting with Selectors + +Experiments match specific invocations of a Failure Flag based on its name, and the labels you provide. Experiments define Selectors that the Failure Flags engine uses to determine if an invocation matches. Selectors are simple key to list of values maps. The basic matching logic is every key in a selector must be present in the Failure Flag labels, and at least one of the values in the list for a selector key must match the value in the label. + +## Effects and Examples + +Once you've instrumented your code and deployed your application with the sidecar you're ready to run an Experiment. None of the work you've done so far describes the Effect during an experiment. You've only marked the spots in code where you want the opportunity to experiment. Gremlin Failure Flags Experiments take an Effect parameter. The Effect parameter is a simple JSON map. That map is provided to the Failure Flags SDK if the application is targeted by a running Experiment. The Failure Flags SDK will process the map according to the default behavior chain or the behaviors you've provided. Today the default chain provides both latency and error Effects. + +### Introduce Flat Latency + +This Effect will introduce a constant 2000 millisecond delay. + +```json +{ "latency": 2000 } +``` + +### Introduce Minimum Latency with Some Maximum Jitter + +This Effect will introduce between 2000 and 2200 milliseconds of latency where there is a pseudo-random uniform probability of any delay between 2000 and 2200. + +```json +{ + "latency": { + "ms": 2000, + "jitter": 200 + } +} +``` + +### Throw an Error + +This Effect will cause Failure Flags to throw a ValueError with the provided message. This is useful if your application uses Errors with well-known messages. + +```json +{ "exception": "this is a custom message" } +``` + +If your app uses custom error types or other error condition metadata then use the object form of exception. This Effect will cause the SDK to import http.client module and raise an http.client.ImproperConnectionState exception: + +```json +{ + "exception": { + "message": "this is a custom message", + "module": "http.client", + "className": "ImproperConnectionState" + } +} +``` + +If `module` is omitted the SDK will assume `builtins`. If `className` is omitted the SDK will assume `ValueError`. + +### Combining the Two for a "Delayed Exception" + +Many common failure modes eventually result in an exception being thrown, but there will be some delay before that happens. Examples include network connection failures, or degradation, or other timeout-based issues. + +This Effect Statement will cause a Failure Flag to pause for a full 2 seconds before throwing an exception/error a message, "Custom TCP Timeout Simulation" + +```json +{ + "latency": 2000, + "exception": { + "message": "Custom TCP Timeout Simulation", + "className": "TimeoutError" + } +} +``` + +### Advanced: Providing Metadata to Custom Behaviors + +The default effect chain included with the Failure Flags SDK is aware of well-known effect properties including, "latency" and "exception." The user can extend or replace that functionality and use the same properties, or provide their own. For example, suppose a user wants to use a "random jitter" effect that the Standard Chain does not provide. Suppose they wanted to inject a random amount of jitter up to some maximum. They could implement that small extension and make up their own Effect property called, "my-jitter" that specifies that maximum. The resulting Effect Statement would look like: + +```json +{ "my-jitter": 500 } +``` + +They might also combine this with parts of the default chain: + +```json +{ + "latency": 1000, + "my-jitter": 500 +} +``` diff --git a/src/failureflags/__init__.py b/src/failureflags/__init__.py index 9865aaa..c15408c 100644 --- a/src/failureflags/__init__.py +++ b/src/failureflags/__init__.py @@ -1,4 +1,5 @@ from urllib.request import urlopen, Request +from random import random import json import collections import os @@ -10,10 +11,28 @@ logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) +VERSION = "0.0.1" + class FailureFlag: - enabled = True if os.environ.get('FAILURE_FLAGS_ENABLED') != None else False + """FailureFlag represents a point in your code where you want to be able to inject failures dynamically. + + The FailureFlag object can be created anywhere and will only have an effect at the line where the + invoke() function is called. Instead of relying on the built-in behavior processing a user can call the + fetch() function to simply retrieve any active experiments targeting a FailureFlag. + + This package is inert if the FAILURE_FLAGS_ENABLED environment variable is unset. + """ def __init__(self, name, labels, behavior=None, data={}, debug=False): + """Create a new FailureFlag. + + Keyword arguments: + behavior -- a function to invoke for retrieved experiments instead of the default behavior chain. + debug -- True or False (default False) to control debug logging. + data -- Data to be mutated by behaviors and effect data. + """ + + self.enabled = 'FAILURE_FLAGS_ENABLED' in os.environ self.name = name self.labels = labels self.behavior = behavior if behavior != None else defaultBehavior @@ -29,6 +48,8 @@ def invoke(self): impacted = False experiments = [] if not self.enabled: + if self.debug: + logger.debug("SDK not enabled") return (active, impacted, experiments) if len(self.name) <= 0: if self.debug: @@ -38,40 +59,52 @@ def invoke(self): experiments = self.fetch() except Exception as err: if self.debug: - logger.debug("received error while fetching experiments") + logger.debug("received error while fetching experiments", err) return (active, impacted, experiments) if len(experiments) > 0: active = True - impacted = self.behavior(self, experiments) + # TODO dice roll + dice = random() + filteredExperiments = filter(lambda experiment: + (type(experiment["rate"]) is float + or type(experiment["rate"]) is int) + and experiment["rate"] >= 0 + and experiment["rate"] <= 1 + and dice < experiment["rate"], experiments) + impacted = self.behavior(self, list(filteredExperiments)) + else: + if self.debug: + logger.debug("no experiments retrieved") return (active, impacted, experiments) def fetch(self): global logger + global VERSION experiments = [] if not self.enabled: return experiments + self.labels["failure-flags-sdk-version"] = f"python-v{VERSION}" data = json.dumps({"name": self.name, "labels": self.labels}).encode("utf-8") - request = Request('http://localhost:5032/experiment', headers={"Content-Type": "application/json", "Content-Length": len(data)}, data=data) + request = Request('http://localhost:5032/experiment', + headers={"Content-Type": "application/json", "Content-Length": len(data)}, + data=data) with urlopen(request, timeout=.001) as response: - # validate status code code = 0 if hasattr(response, 'status'): code = response.status - elif hasattr(response, 'code'): - code = response.code if code < 200 or code >= 300: if self.debug: - logger.debug("bad status code ({}) while fetching experiments".format(code)) + logger.debug(f"bad status code ({code}) while fetching experiments") return [] - # validate Content-Type - # validate Content-Length + # TODO validate Content-Type + # TODO validate Content-Length body = response.read() response.close() experiments = json.loads(body) - if isinstance(experiments, collections.list) or type(experiments) is list: + if isinstance(experiments, list) or type(experiments) is list: return experiments - elif isinstance(experiments, collections.Mapping) or type(experiments) is dict: + elif isinstance(experiments, dict) or type(experiments) is dict: return [experiments] else: return [] @@ -84,58 +117,76 @@ def delayedDataOrError(failureflag, experiments): dataImpact = data(failureflag, experiments) return latencyImpact or exceptionImpact or dataImpact -def latency(failureflag, experiments): +def latency(ff, experiments): impacted = False if experiments == None or len(experiments) == 0: + if ff.debug: + logger.debug("experiments was empty") return impacted - for f in experiments: - if not isinstance(f, collections.Mapping) or type(f) is not dict: + for e in experiments: + if not isinstance(e, dict) or type(e) is not dict: + if ff.debug: + logger.debug("experiment is not a dict, skipping") continue - if "effect" not in f: + if "effect" not in e: + if ff.debug: + logger.debug("no effect in experiment, skipping") continue - if "latency" not in f["effect"]: + if "latency" not in e["effect"]: + if ff.debug: + logger.debug("no latency in experiment effect, skipping") continue - if type(f["effect"]["latency"]) is int: + if type(e["effect"]["latency"]) is int: impacted = True - time.sleep(f["effect"]["latency"]/1000) - elif isinstance(f["effect"]["latency"], collections.Mapping): + time.sleep(e["effect"]["latency"]/1000) + elif type(e["effect"]["latency"]) is str: + try: + ms = int(e["effect"]["latency"]) + time.sleep(ms/1000) + impacted = True + except ValueError as err: + if ff.debug: + logger.debug("experiment contained a non-number latency clause") + elif isinstance(e["effect"]["latency"], dict): impacted = True ms = 0 jitter = 0 - if "ms" in f["effect"]["latency"] and f["effect"]["latency"]["ms"] is int: - ms = f["effect"]["latency"]["ms"] - if "jitter" in f["effect"]["latency"] and f["effect"]["latency"]["jitter"] is int: - jitter = f["effect"]["latency"]["jitter"] + if "ms" in e["effect"]["latency"] and type(e["effect"]["latency"]["ms"]) is int: + ms = e["effect"]["latency"]["ms"] + if "jitter" in e["effect"]["latency"] and type(e["effect"]["latency"]["jitter"]) is int: + jitter = e["effect"]["latency"]["jitter"] # convert both ms and jitter to seconds - time.sleep(ms/1000 + jitter*random.random()/1000) + time.sleep(ms/1000 + jitter*random()/1000) return impacted -def exception(failureflag, experiments): +def exception(ff, experiments): global logger for f in experiments: - if not isinstance(f, collections.Mapping) or type(f) is not dict: + if not isinstance(f, dict) or type(f) is not dict: continue if "effect" not in f: continue if "exception" not in f["effect"]: continue if type(f["effect"]["exception"]) is str: - raise ValueException(f["effect"]["exception"]) - elif isinstance(f["effect"]["exception"], collections.Mapping): - module = None - class_name = "ValueException" + raise ValueError(f["effect"]["exception"]) + elif isinstance(f["effect"]["exception"], dict): + module = "builtins" + class_name = "ValueError" message = "Error injected via Gremlin Failure Flags (default message)" hasKnown = False - if "module" in f["effect"]["exception"] and f["effect"]["exception"]["module"] is str: + if "module" in f["effect"]["exception"] and type(f["effect"]["exception"]["module"]) is str: module = f["effect"]["exception"]["module"] hasKnown = True - if "className" in f["effect"]["exception"] and f["effect"]["exception"]["className"] is str: + if "className" in f["effect"]["exception"] and type(f["effect"]["exception"]["className"]) is str: class_name= f["effect"]["exception"]["className"] hasKnown = True - if "message" in f["effect"]["exception"] and f["effect"]["exception"]["message"] is str: + if "message" in f["effect"]["exception"] and type(f["effect"]["exception"]["message"]) is str: message = f["effect"]["exception"]["message"] hasKnown = True if not hasKnown: + if ff.debug: + logger.debug("exception clause was not populated") continue if len(class_name) == 0: # for some reason this was explicitly unset @@ -143,20 +194,21 @@ def exception(failureflag, experiments): error = None try: if module is not None: - module_ = __import__(module) + module_ = __import__(module, fromlist=[None]) class_ = getattr(module_, class_name) else: class_ = globals()[class_name] error = class_(message) - except: + except Exception as err: # unable to load the class - if failureflag.debug: - logger.debug("unable to load the named module: {}", module) + if ff.debug: + logger.debug(f"unable to load the named module: {module}, {err}") + return False if error is not None: raise error return False -def data(failureflag, experiments): +def data(ff, experiments): return False defaultBehavior = delayedDataOrError diff --git a/test/behaviors_test.py b/test/behaviors_test.py new file mode 100644 index 0000000..2765551 --- /dev/null +++ b/test/behaviors_test.py @@ -0,0 +1,247 @@ +import logging + +import failureflags +import unittest +from unittest.mock import patch, call + +debug = logging.getLogger("failureflags") +debug.addHandler(logging.StreamHandler()) +debug.setLevel(logging.DEBUG) + +class TestFailureFlagsBehaviors(unittest.TestCase): + + ##################################################3 + # Testing the latency behavior + ##################################################3 + + @patch('failureflags.time.sleep') + def test_latencyNoExperiments(self, mock_sleep): + impacted = failureflags.latency(failureflags.FailureFlag("name", {}), []) + mock_sleep.assert_not_called() + assert impacted == False, "impact reported when no experiments were provided" + + @patch('failureflags.time.sleep') + def test_latencyOneExperimentNoLatency(self, mock_sleep): + impacted = failureflags.latency(failureflags.FailureFlag("name", {}), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "custom", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "custom": "10", + }}]) + mock_sleep.assert_not_called() + assert impacted == False, "impact reported when no experiments were provided" + + @patch('failureflags.time.sleep') + def test_latencyOneExperimentWithLatencyNumber(self, mock_sleep): + impacted = failureflags.latency(failureflags.FailureFlag("name", {}), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "custom", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": 10000, + }}]) + mock_sleep.assert_called() + assert impacted == True, "No impact reported when latency experiments were provided" + + @patch('failureflags.time.sleep') + def test_latencyOneExperimentWithLatencyString(self, mock_sleep): + impacted = failureflags.latency(failureflags.FailureFlag("name", {}), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "custom", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": "10000", + }}]) + mock_sleep.assert_called() + assert impacted == True, "No impact reported when latency experiments were provided" + + @patch('failureflags.time.sleep') + def test_latencyOneExperimentWithBadLatencyString(self, mock_sleep): + impacted = failureflags.latency(failureflags.FailureFlag("name", {}), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "custom", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": "notanumber", + }}]) + mock_sleep.assert_not_called() + assert impacted == False, "Impact reported when bad latency experiments were provided" + + @patch('failureflags.time.sleep') + def test_latencyOneExperimentWithDictLatency(self, mock_sleep): + impacted = failureflags.latency(failureflags.FailureFlag("name", {}, debug=True), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "custom", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": { + "ms": 10000, + "jitter": 0 + }, + }}]) + mock_sleep.assert_called_with(10) + assert impacted == True, "No impact reported when latency experiments were provided" + + @patch('failureflags.time.sleep') + def test_latencyTwoExperimentsWithDictLatency(self, mock_sleep): + impacted = failureflags.latency(failureflags.FailureFlag("name", {}, debug=True), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "name", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": 10000 + }},{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a8", + "failureFlagName": "name", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": 20000 + }}]) + mock_sleep.assert_has_calls([call(10), call(20)]) + assert impacted == True, "No impact reported when latency experiments were provided" + + ##################################################3 + # Testing the exception behavior + ##################################################3 + + def test_exceptionNoExperiments(self): + try: + impacted = failureflags.exception(failureflags.FailureFlag("name", {}, debug=True), []) + except Exception as err: + assert false, "No exception should be raised when no experiments are provided" + + def test_exceptionNoExceptionEffect(self): + try: + impacted = failureflags.exception(failureflags.FailureFlag("name", {}, debug=True), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a8", + "failureFlagName": "name", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": 20000 + }}]) + except Exception as err: + assert false, "No exception should be raised when no experiments have exception effects" + + def test_exceptionSimpleExceptionEffect(self): + try: + impacted = failureflags.exception(failureflags.FailureFlag("name", {}, debug=True), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a8", + "failureFlagName": "name", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "exception": "this is a test message" + }}]) + except Exception as err: + assert err.args[0] == "this is a test message" + return + assert False, "An exception must be raised if the experiment provides a valid exception clause" + + def test_exceptionDictExceptionEffect(self): + try: + impacted = failureflags.exception(failureflags.FailureFlag("name", {}), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a8", + "failureFlagName": "name", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "exception": { + "module": "http.client", + "className": "ImproperConnectionState", + "message": "this is an improper connection state error" + } + }}]) + except Exception as err: + assert err.args[0] == "this is an improper connection state error" + assert err.__class__.__name__ == "ImproperConnectionState" + return + assert False, "An exception must be raised if the experiment provides a valid exception clause" + + def test_exceptionPartialDictExceptionEffect(self): + try: + impacted = failureflags.exception(failureflags.FailureFlag("name", {}), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a8", + "failureFlagName": "name", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "exception": { + "className": "TimeoutError", + "message": "this is an improper connection state error" + } + }}]) + except Exception as err: + assert err.args[0] == "this is an improper connection state error" + assert err.__class__.__name__ == "TimeoutError" + return + assert False, "An exception must be raised if the experiment provides a valid exception clause" + + ##################################################3 + # Testing the delayedDataOrError behavior + ##################################################3 + + @patch('failureflags.time.sleep') + def test_delayedException(self, mock_sleep): + try: + impacted = failureflags.delayedDataOrError(failureflags.FailureFlag("name", {}, debug=True), [{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a8", + "failureFlagName": "name", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": 10000, + "exception": "this is a test message" + }}]) + except Exception as err: + mock_sleep.assert_called_with(10) + assert err.args[0] == "this is a test message" + return + assert False, "An exception must be raised if the experiment provides a valid exception clause" + +if __name__ == '__main__': + unittest.main() diff --git a/test/demo_test.py b/test/demo_test.py new file mode 100644 index 0000000..61f3d15 --- /dev/null +++ b/test/demo_test.py @@ -0,0 +1,43 @@ +import logging + +from failureflags import FailureFlag +import urllib.request +import os +import unittest +from unittest.mock import patch, MagicMock + +debug = logging.getLogger("failureflags") +debug.addHandler(logging.StreamHandler()) +debug.setLevel(logging.DEBUG) + +class TestInvoke(unittest.TestCase): + + @patch('failureflags.urlopen') + @patch('failureflags.time.sleep') + @patch.dict(os.environ, {"FAILURE_FLAGS_ENABLED": "TRUE"}) + def test_e2eEnabledWithLatency(self, mock_sleep, mock_urlopen): + url_cm = MagicMock() + url_cm.status = 200 + url_cm.read = MagicMock(return_value="""[{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "targetLatencyNumber", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": 10000 + }}]""") + url_cm.__enter__.return_value = url_cm + mock_urlopen.return_value = url_cm + assert "FAILURE_FLAGS_ENABLED" in os.environ + + FailureFlag("targetLatencyNumber", {}, debug=True).invoke() + + url_cm.__enter__.assert_called() + url_cm.read.assert_called() + mock_sleep.assert_called_with(10) + +if __name__ == '__main__': + unittest.main() diff --git a/test/fetch_test.py b/test/fetch_test.py new file mode 100644 index 0000000..6dad6a2 --- /dev/null +++ b/test/fetch_test.py @@ -0,0 +1,73 @@ +import logging + +import failureflags +import urllib.request +import os +import unittest +from unittest.mock import patch, MagicMock + +debug = logging.getLogger("failureflags") +debug.addHandler(logging.StreamHandler()) +debug.setLevel(logging.DEBUG) + +class TestFetch(unittest.TestCase): + + @patch('failureflags.urlopen') + @patch.dict(os.environ, clear=True) + def test_inert(self, mock_urlopen): + url_cm = MagicMock() + url_cm.status = 200 + url_cm.read = MagicMock(side_effect=Exception("should not be used"), return_value="[{}]") + url_cm.__enter__.return_value = url_cm + mock_urlopen.return_value = url_cm + + assert "FAILURE_FLAGS_ENABLED" not in os.environ + + experiments = [] + try: + experiments = failureflags.FailureFlag("name", {}, debug=True).fetch() + except Exception as err: + assert False, "an inert SDK will not throw an Exception when running fetch" + + @patch('failureflags.urlopen') + @patch.dict(os.environ, {"FAILURE_FLAGS_ENABLED": "TRUE"}) + def test_fetchWrapsSingleExperimentWithList(self, mock_urlopen): + url_cm = MagicMock() + url_cm.status = 200 + url_cm.read = MagicMock(return_value="{}") + url_cm.__enter__.return_value = url_cm + mock_urlopen.return_value = url_cm + assert "FAILURE_FLAGS_ENABLED" in os.environ + + experiments = [] + try: + experiments = failureflags.FailureFlag("targetLatencyNumber", {}, debug=True).fetch() + except Exception as err: + assert False, "an inert SDK will not throw an Exception when running fetch" + + url_cm.__enter__.assert_called() + url_cm.read.assert_called() + assert len(experiments) == 1 + + @patch('failureflags.urlopen') + @patch.dict(os.environ, {"FAILURE_FLAGS_ENABLED": "TRUE"}) + def test_fetchHandlesNon200CodeSilently(self, mock_urlopen): + url_cm = MagicMock() + url_cm.status = 400 + url_cm.read = MagicMock(return_value="[{}]") + url_cm.__enter__.return_value = url_cm + mock_urlopen.return_value = url_cm + assert "FAILURE_FLAGS_ENABLED" in os.environ + + experiments = [] + try: + experiments = failureflags.FailureFlag("targetLatencyNumber", {}, debug=True).fetch() + except Exception as err: + assert False, "an inert SDK will not throw an Exception when running fetch" + + url_cm.__enter__.assert_called() + url_cm.read.assert_not_called() + assert len(experiments) == 0 + +if __name__ == '__main__': + unittest.main() diff --git a/test/invoke_test.py b/test/invoke_test.py new file mode 100644 index 0000000..878ed36 --- /dev/null +++ b/test/invoke_test.py @@ -0,0 +1,122 @@ +import logging + +import failureflags +import urllib.request +import os +import unittest +from unittest.mock import patch, MagicMock + +debug = logging.getLogger("failureflags") +debug.addHandler(logging.StreamHandler()) +debug.setLevel(logging.DEBUG) + +class TestInvoke(unittest.TestCase): + + @patch('failureflags.urlopen') + @patch('failureflags.time.sleep') + @patch.dict(os.environ, {"FAILURE_FLAGS_ENABLED": "TRUE"}) + def test_e2eEnabledWithLatency(self, mock_sleep, mock_urlopen): + url_cm = MagicMock() + url_cm.status = 200 + url_cm.read = MagicMock(return_value="""[{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "targetLatencyNumber", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": 10000 + }}]""") + url_cm.__enter__.return_value = url_cm + mock_urlopen.return_value = url_cm + assert "FAILURE_FLAGS_ENABLED" in os.environ + + flag = failureflags.FailureFlag("targetLatencyNumber", {}, debug=True) + print(flag) + active, impacted, experiments = flag.invoke() + print(active, impacted, experiments) + + print(mock_urlopen) + url_cm.__enter__.assert_called() + url_cm.read.assert_called() + mock_sleep.assert_called_with(10) + assert len(experiments) == 1 + assert experiments[0]["failureFlagName"] == "targetLatencyNumber" + assert active == True, "works should be Active" + assert impacted == True, "works should be Impacted" + + @patch('failureflags.urlopen') + @patch('failureflags.time.sleep') + @patch.dict(os.environ, clear=True) + def test_e2eInert(self, mock_sleep, mock_urlopen): + url_cm = MagicMock() + url_cm.status = 200 + url_cm.read = MagicMock(return_value="""[{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "works", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": 10 + }}]""") + url_cm.__enter__.return_value = url_cm + mock_urlopen.return_value = url_cm + + assert "FAILURE_FLAGS_ENABLED" not in os.environ + + flag = failureflags.FailureFlag("works", {}, debug=True) + active, impacted, experiments = flag.invoke() + + url_cm.__enter__.assert_not_called(); + url_cm.read.assert_not_called(); + mock_sleep.assert_not_called(); + assert len(experiments) == 0 + assert active == False, "works should be inactive because the SDK should be inert" + assert impacted == False , "works should not be impacted because the SDK should be inert" + + @patch('failureflags.urlopen') + @patch('failureflags.time.sleep') + @patch.dict(os.environ, {"FAILURE_FLAGS_ENABLED": "TRUE"}) + def test_customBehaviorWithDelegateToDefault(self, mock_sleep, mock_urlopen): + url_cm = MagicMock() + url_cm.status = 200 + url_cm.read = MagicMock(return_value="""[{ + "guid": "6884c0df-ed70-4bc8-84c0-dfed703bc8a7", + "failureFlagName": "works", + "rate": 1, + "selector": { + "a":"1", + "b":"2" + }, + "effect": { + "latency": 10000 + }}]""") + url_cm.__enter__.return_value = url_cm + mock_urlopen.return_value = url_cm + + evidence = MagicMock(return_value="invoked") + + assert "FAILURE_FLAGS_ENABLED" in os.environ + + def customBehavior(ff, experiments): + evidence() + return failureflags.defaultBehavior(ff, experiments) + + failureflags.FailureFlag( + "works", + {}, + debug=True, + behavior=customBehavior).invoke() + + url_cm.__enter__.assert_called(); + url_cm.read.assert_called(); + evidence.assert_called(); + mock_sleep.assert_called_with(10) + +if __name__ == '__main__': + unittest.main()