add cli doc

README.md

@@ -93,6 +93,22 @@ Monkeyble allow to check the states of a task
     should_failed: false
 ```
 
+### Cli 
+
+Monkeyble comes with a CLI that allow to execute all tests from a single command and return a summary of test executions.
+```bash
+monkeyble test
+
+Playbook   | Scenario        | Test passed
+-----------+-----------------+-------------
+ play1.yml | validate_test_1 | ✅
+ play1.yml | validate_test_2 | ✅
+ play2.yml | validate_this   | ✅
+ play2.yml | validate_that   | ✅
+ 
+ 🐵 Monkeyble test result - Tests passed: 4 of 4 tests
+```
+
 ### Monkey patching
 
 Monkey patching is a technique that allows you to intercept what a function would normally do, substituting its full execution with a return value of your own specification. 

docs/cli.md

@@ -0,0 +1,97 @@
+# Monkeyble CLI
+
+The monkeyble CLI provides a convenient way to execute all your tests from a single command.
+
+## Installation
+
+### From Pypi
+
+Install from pypi
+```bash
+pip3 install monkeyble
+```
+
+If you are using pipx, inject Monkeyble in your ansible installation
+```bash
+pipx inject ansible --include-apps monkeyble
+```
+
+### From sources
+
+Run the Python setup 
+```bash
+python3 setup.py install
+```
+
+## Configuration file
+
+The CLI expect to find monkeyble configuration file. The file will be searched for in the following order:
+
+- `-c CONFIG` as a cli argument
+- `MONKEYBLE_CONFIG` placed as an environment variable
+- `monkeyble.yml` from the current directory
+
+### monkeyble_test_suite
+
+The `monkeyble_test_suite` contains a list of Monkeyble test definition.
+A test definition contains information about the playbook to test with all scenario to validate.
+
+| Name       | Required | Description                       |
+|------------|----------|-----------------------------------|
+| playbook   | true     | path to the playbook to test      |
+| inventory  | false    | optional path to the inventory    |
+| extra_vars | false    | List of path to extra var file    |
+| scenarios  | true     | List of scenario name to validate |
+
+Configuration example:
+
+```yaml
+monkeyble_test_suite:
+  - playbook: "play1.yml"
+    inventory: "inventory"
+    extra_vars:
+      - "shared_mocks.yml"
+      - "play1_scenarios.yml"
+    scenarios:
+      - "validate_test_1"
+      - "validate_test_2"
+  - playbook: "play2.yml"
+    inventory: "inventory"
+    extra_vars:
+      - "shared_mocks.yml"
+      - "play2_scenarios.yml"
+    scenarios:
+      - "validate_this"
+      - "validate_that"
+```
+
+!!! note
+
+    As the monkeyble [scenario configuration](scenarios.md) is passed as `extra_vars` you should at least 
+    have one file declared in the `extra_vars` list and one scenario name placed in `scenarios`.
+
+## Commands
+
+### monkeyble test
+
+The monkeyble test command executes all the test declared in the `monkeyble_test_suite` configuration flag and provides
+a test result summary.
+
+Command:
+```
+ANSIBLE_CONFIG='monkeyble.cfg' monkeyble test 
+```
+
+Output example:
+```
+# TRUNCATED. Playbook executions output
+
+Playbook   | Scenario        | Test passed
+-----------+-----------------+-------------
+ play1.yml | validate_test_1 | ✅
+ play1.yml | validate_test_2 | ✅
+ play2.yml | validate_this   | ✅
+ play2.yml | validate_that   | ✅
+ 
+ 🐵 Monkeyble test result - Tests passed: 4 of 4 tests
+```

mkdocs.yml

@@ -36,5 +36,5 @@ nav:
   - Mock: mock.md
   - Task filters: task_filters.md
   - Task extra vars: extra_vars.md
-  - CI/CD usage: ci_cd_usage.md
+  - CLI: cli.md
   - Dev env: dev_env.md

monkeyble/cli/monkeyble.yml

@@ -1,5 +1,4 @@
-ansible_cmd: "ansible-playbook -v"
-monkeyble_tests:
+monkeyble_test_suite:
   - playbook: "../../tests/test_playbook.yml"
 #    inventory: "inventory"
     extra_vars:

monkeyble/cli/monkeyble_cli.py

@@ -15,7 +15,7 @@
 
 logger = logging.getLogger(MONKEYBLE)
 
-logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
 
 # actions available
 ACTION_LIST = ["test"]
@@ -51,14 +51,10 @@ def run_ansible(ansible_cmd, playbook, inventory, extra_vars, scenario):
 
 def run_monkeyble_test(monkeyble_config):
     ansible_cmd = MONKEYBLE_DEFAULT_ANSIBLE_CMD
-    if "ansible_cmd" in monkeyble_config:
-        ansible_cmd = monkeyble_config["ansible_cmd"]
-    if "monkeyble_tests" not in monkeyble_config:
-        raise MonkeybleCLIException(message="No 'monkeyble_tests' variable defined")
+    if "monkeyble_test_suite" not in monkeyble_config:
+        raise MonkeybleCLIException(message="No 'monkeyble_test_suite' variable defined")
     list_result = list()
-    for test_config in monkeyble_config["monkeyble_tests"]:
-        if "ansible_cmd" in test_config:
-            ansible_cmd = test_config["ansible_cmd"]
+    for test_config in monkeyble_config["monkeyble_test_suite"]:
         Utils.print_info(f"Monkeyble - ansible cmd: {ansible_cmd}")
         playbook = test_config.get("playbook", None)
         new_result = MonkeybleResult(playbook)

tests/units/test_cli.py

@@ -33,11 +33,10 @@ def test_load_monkeyble_config_from_args(self):
     @patch("monkeyble.cli.monkeyble_cli.MONKEYBLE_DEFAULT_CONFIG_PATH", "test_config/monkeyble.yml")
     def test_load_monkeyble_config(self):
         data = load_monkeyble_config(None)
-        expected = {'ansible_cmd': 'ansible-playbook -v',
-                    'monkeyble_tests': [{'playbook': 'test_playbook.yml',
-                                         'inventory': 'inventory',
-                                         'extra_vars': ['mocks.yml', 'monkeyble_scenarios.yml'],
-                                         'scenarios': ['validate_test_1', 'validate_test_2']}]}
+        expected = {'monkeyble_test_suite': [{'playbook': 'test_playbook.yml',
+                                              'inventory': 'inventory',
+                                              'extra_vars': ['mocks.yml', 'monkeyble_scenarios.yml'],
+                                              'scenarios': ['validate_test_1', 'validate_test_2']}]}
 
         self.assertDictEqual(data, expected)
 
@@ -69,7 +68,7 @@ def test_run_monkeyble_test_no_tests_defined(self, mock_exit):
     @patch('sys.exit')
     def test_run_monkeyble_test_no_playbook_defined(self, mock_exit):
         monkeyble_config = {
-            "monkeyble_tests": [
+            "monkeyble_test_suite": [
                 {"inventory": "test"}
             ]
         }
@@ -80,7 +79,7 @@ def test_run_monkeyble_test_no_playbook_defined(self, mock_exit):
     @patch('sys.exit')
     def test_run_monkeyble_test_no_scenario_defined(self, mock_exit):
         monkeyble_config = {
-            "monkeyble_tests": [
+            "monkeyble_test_suite": [
                 {"playbook": "playbook.yml"}
             ]
         }
@@ -90,7 +89,7 @@ def test_run_monkeyble_test_no_scenario_defined(self, mock_exit):
 
     def test_run_monkeyble_test_run_ansible_called(self):
         monkeyble_config = {
-            "monkeyble_tests": [
+            "monkeyble_test_suite": [
                 {
                     "playbook": "playbook.yml",
                     "inventory": "my_inventory",

tests/units/test_config/monkeyble.yml

@@ -1,10 +1,9 @@
-ansible_cmd: "ansible-playbook -v"
-monkeyble_tests:
+monkeyble_test_suite:
   - playbook: "test_playbook.yml"
     inventory: "inventory"
     extra_vars:
       - "mocks.yml"
       - "monkeyble_scenarios.yml"
     scenarios:
-      - validate_test_1
-      - validate_test_2
+      - "validate_test_1"
+      - "validate_test_2"