Merge pull request #317 from yoshi-1224/manifest-validation

Add kubernetes manifest validation

README.md

@@ -286,7 +286,7 @@ Generic templated configuration management for Kubernetes, Terraform and other
 things
 
 positional arguments:
-  {eval,compile,inventory,searchvar,secrets,lint,init}
+  {eval,compile,inventory,searchvar,secrets,lint,init,validate}
                         commands
     eval                evaluate jsonnet file
     compile             compile targets
@@ -296,6 +296,8 @@ positional arguments:
     lint                linter for inventory and secrets
     init                initialize a directory with the recommended kapitan
                         project skeleton.
+    validate            validate the compile output against schemas as
+                        specified in inventory
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -637,6 +639,41 @@ $ kapitan searchvar parameters.elasticsearch.replicas
 ./inventory/classes/component/elasticsearch.yml   1
 ```
 
+### kapitan validate
+
+Validates the schema of compiled output (currently supports Kubernetes manifests).
+
+Refer to the `minikube-es` inventory in [kapitan inventory](#kapitan-inventory). To validate the schema of the compiled StatefulSet manifest at `compiled/minikube-es/manifests/es-client.yml` (created by `components/elasticsearch/main.jsonnet`), add `kapitan.validate` parameters in `minikube-es` inventory.
+
+```yaml
+kapitan:
+  vars:
+    target: ${target_name}
+    namespace: ${target_name}
+  compile:
+  - output_path: manifests
+    input_type: jsonnet
+    input_paths:
+      - components/elasticsearch/main.jsonnet
+
+    ### other inputs abbreviated for clarity ###
+  validate:
+  - output_paths:
+      - manifests/es-client.yml
+    type: kubernetes
+    kind: statefulset
+    version: 1.14.0 # optional, defaults to 1.14.0
+```
+
+Then run:
+
+```
+$ kapitan validate -t minikube-es
+invalid 'statefulset' manifest at ./compiled/minikube-es/manifests/es-client.yml
+['spec'] 'selector' is a required property
+```
+
+
 # Kapitan feature proposals
 
 See [kapitan_proposals/](docs/kap_proposals/).

examples/kubernetes/inventory/classes/component/mysql.yml

@@ -15,6 +15,18 @@ parameters:
       input_type: jinja2
       input_paths:
         - docs/mysql/README.md
+    validate:
+    - type: kubernetes
+      output_paths:
+        - manifests/mysql_secret.yml
+      kind: secret # temporarily replaced with 'deployment' during test
+      version: 1.14.0 # optional, defaults to 1.14.0
+    - type: kubernetes
+      output_paths:
+        - manifests/mysql_service_jsonnet.yml
+        - manifests/mysql_service_simple.yml
+      kind: service
+      version: 1.14.0
   mysql:
     storage: 10G
     storage_class: standard

kapitan/cli.py

@@ -35,7 +35,7 @@
 from kapitan.refs.secrets.gpg import GPGSecret, lookup_fingerprints
 from kapitan.resources import (inventory_reclass, resource_callbacks,
                                search_imports)
-from kapitan.targets import compile_targets
+from kapitan.targets import compile_targets, schema_validate_compiled
 from kapitan.inputs.jinja2_filters import default_jinja2_filters_path
 from kapitan.utils import (PrettyDumper, check_version, deep_get, fatal_error,
                            flatten_dict, from_dot_kapitan, jsonnet_file,
@@ -95,6 +95,9 @@ def main():
     compile_parser.add_argument('--fetch',
                                 help='fetches external dependencies', action='store_true',
                                 default=from_dot_kapitan('compile', 'fetch', False))
+    compile_parser.add_argument('--validate',
+                                help='validate compile output against schemas as specified in inventory',
+                                action='store_true', default=from_dot_kapitan('compile', 'validate', False))
     compile_parser.add_argument('--targets', '-t', help='targets to compile, default is all',
                                 type=str, nargs='+',
                                 default=from_dot_kapitan('compile', 'targets', []),
@@ -128,6 +131,9 @@ def main():
                                 help='ignore the version from .kapitan',
                                 action='store_true',
                                 default=from_dot_kapitan('compile', 'ignore-version-check', False))
+    compile_parser.add_argument('--schemas-path',
+                                 default=from_dot_kapitan('validate', 'schemas-path', './schemas'),
+                                 help='set schema cache path, default is "./schemas"')
 
     inventory_parser = subparser.add_parser('inventory', help='show inventory')
     inventory_parser.add_argument('--target-name', '-t',
@@ -230,6 +236,24 @@ def main():
                              default=from_dot_kapitan('init', 'directory', '.'),
                              help='set path, in which to generate the project skeleton, assumes directory already exists. default is "./"')
 
+    validate_parser = subparser.add_parser('validate', help='validates the compile output against schemas as specified in inventory')
+    validate_parser.add_argument('--compiled-path',
+                                 default=from_dot_kapitan('compile', 'compiled-path', './compiled'),
+                                 help='set compiled path, default is "./compiled')
+    validate_parser.add_argument('--inventory-path',
+                                 default=from_dot_kapitan('compile', 'inventory-path', './inventory'),
+                                 help='set inventory path, default is "./inventory"')
+    validate_parser.add_argument('--targets', '-t', help='targets to validate, default is all',
+                                type=str, nargs='+',
+                                default=from_dot_kapitan('compile', 'targets', []),
+                                metavar='TARGET'),
+    validate_parser.add_argument('--schemas-path',
+                                 default=from_dot_kapitan('validate', 'schemas-path', './schemas'),
+                                 help='set schema cache path, default is "./schemas"')
+    validate_parser.add_argument('--parallelism', '-p', type=int,
+                                default=from_dot_kapitan('validate', 'parallelism', 4),
+                                metavar='INT',
+                                help='Number of concurrent validate processes, default is 4')
     args = parser.parse_args()
 
     logger.debug('Running with args: %s', args)
@@ -280,7 +304,9 @@ def _search_imports(cwd, imp):
                         args.parallelism, args.targets, ref_controller,
                         prune=(args.prune), indent=args.indent, reveal=args.reveal,
                         cache=args.cache, cache_paths=args.cache_paths,
-                        fetch_dependencies=args.fetch, jinja2_filters=args.jinja2_filters)
+                        fetch_dependencies=args.fetch, validate=args.validate,
+                        schemas_path=args.schemas_path,
+                        jinja2_filters=args.jinja2_filters)
 
     elif cmd == 'inventory':
         if args.pattern and args.target_name == '':
@@ -323,6 +349,10 @@ def _search_imports(cwd, imp):
         elif args.update_targets or args.validate_targets:
             secret_update_validate(args, ref_controller)
 
+    elif cmd == 'validate':
+        schema_validate_compiled(args.targets, inventory_path=args.inventory_path, compiled_path=args.compiled_path,
+                                 schema_cache_path=args.schemas_path, parallel=args.parallelism)
+
 
 def secret_write(args, ref_controller):
     "Write secret to ref_controller based on cli args"

kapitan/dependency_manager/base.py

@@ -12,6 +12,7 @@
 from git import Repo
 
 from kapitan.errors import GitSubdirNotFoundError
+from kapitan.utils import make_request
 
 logger = logging.getLogger(__name__)
 
@@ -134,7 +135,9 @@ def fetch_http_dependency(dep_mapping, save_dir):
 
 def fetch_http_source(source, save_dir):
     """downloads a http[s] file from source and saves into save_dir"""
-    content, content_type = _make_request(source)
+    logger.info("Dependency {} : fetching now".format(source))
+    content, content_type = make_request(source)
+    logger.info("Dependency {} : successfully fetched".format(source))
     if content is not None:
         basename = os.path.basename(source)
         # to avoid collisions between basename(source)
@@ -146,15 +149,3 @@ def fetch_http_source(source, save_dir):
     else:
         logger.warning("Dependency {} : failed to fetch".format(source))
         return None
-
-
-def _make_request(source):
-    """downloads the http file at source and returns it's content"""
-    logger.info("Dependency {} : fetching now".format(source))
-    r = requests.get(source)
-    if r.ok:
-        logger.info("Dependency {} : successfully fetched".format(source))
-        return r.content, r.headers['Content-Type']
-    else:
-        r.raise_for_status()
-    return None, None

kapitan/errors.py

@@ -58,3 +58,13 @@ class RefHashMismatchError(KapitanError):
 class GitSubdirNotFoundError(KapitanError):
     """git dependency subdir not found error"""
     pass
+
+
+class RequestUnsuccessfulError(KapitanError):
+    """request error"""
+    pass
+
+
+class KubernetesManifestValidationError(KapitanError):
+    """kubernetes manifest schema validation error"""
+    pass

kapitan/targets.py

@@ -22,6 +22,8 @@
 import sys
 import multiprocessing
 import tempfile
+from collections import defaultdict
+
 import jsonschema
 import yaml
 import time
@@ -39,6 +41,8 @@
 
 from reclass.errors import NotFoundError, ReclassException
 
+from kapitan.validator.kubernetes_validator import KubernetesManifestValidator, DEFAULT_KUBERNETES_VERSION
+
 logger = logging.getLogger(__name__)
 
 
@@ -103,6 +107,12 @@ def compile_targets(inventory_path, search_paths, output_path, parallel, targets
             shutil.copytree(temp_path, compile_path)
             logger.debug("Copied %s into %s", temp_path, compile_path)
 
+        # validate the compiled outputs
+        if kwargs.get('validate', False):
+            validate_map = create_validate_mapping(target_objs, compile_path)
+            worker = partial(schema_validate_kubernetes_output, cache_dir=kwargs.get('schemas_path', './schemas'))
+            [p.get() for p in pool.imap_unordered(worker, validate_map.items()) if p]
+
         # Save inventory and folders cache
         save_inv_cache(compile_path, targets)
         pool.close()
@@ -375,6 +385,44 @@ def valid_target_obj(target_obj):
                     ],
                 }
             },
+            "validate": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "output_paths": {"type": "array"},
+                        "type": {
+                            "type": "string",
+                            "enum": ["kubernetes"]
+                        },
+                        "kind": {"type": "string"},
+                        "version": {"type": "string"},
+                    },
+                    "required": ["output_paths", "type"],
+                    "minItems": 1,
+                    "allOf": [
+                        {
+                            "if": {
+                                "properties": {"type": {"const": "kubernetes"}}
+                            },
+                            "then": {
+                                "properties": {
+                                    "type": {
+                                    },
+                                    "kind": {
+                                    },
+                                    "output_paths": {
+                                    },
+                                    "version": {
+                                    }
+                                },
+                                "additionalProperties": False,
+                                "required": ["kind"],
+                            }
+                        },
+                    ]
+                }
+            },
             "dependencies": {
                 "type": "array",
                 "items": {
@@ -445,3 +493,86 @@ def validate_matching_target_name(target_filename, target_obj, inventory_path):
         error_message = "Target \"{}\" is missing the corresponding yml file in {}\n" \
                         "Target name should match the name of the target yml file in inventory"
         raise InventoryError(error_message.format(target_name, target_path))
+
+
+def schema_validate_compiled(targets, compiled_path, inventory_path, schema_cache_path, parallel):
+    """
+    validates compiled output according to schemas specified in the inventory
+    """
+    if not os.path.isdir(compiled_path):
+        logger.error("compiled-path {} not found".format(compiled_path))
+        sys.exit(1)
+
+    if not os.path.isdir(schema_cache_path):
+        os.makedirs(schema_cache_path)
+        logger.info("created schema-cache-path at {}".format(schema_cache_path))
+
+    worker = partial(schema_validate_kubernetes_output, cache_dir=schema_cache_path)
+    pool = multiprocessing.Pool(parallel)
+
+    try:
+        target_objs = load_target_inventory(inventory_path, targets)
+        validate_map = create_validate_mapping(target_objs, compiled_path)
+
+        [p.get() for p in pool.imap_unordered(worker, validate_map.items()) if p]
+        pool.close()
+
+    except ReclassException as e:
+        if isinstance(e, NotFoundError):
+            logger.error("Inventory reclass error: inventory not found")
+        else:
+            logger.error("Inventory reclass error: %s", e.message)
+        raise InventoryError(e.message)
+    except Exception as e:
+        pool.terminate()
+        logger.debug("Validate pool terminated")
+        # only print traceback for errors we don't know about
+        if not isinstance(e, KapitanError):
+            logger.exception("Unknown (Non-Kapitan) Error occured")
+
+        logger.error("\n")
+        logger.error(e)
+        sys.exit(1)
+    finally:
+        # always wait for other worker processes to terminate
+        pool.join()
+
+
+def create_validate_mapping(target_objs, compiled_path):
+    """
+    creates mapping of (kind, version) tuple to output_paths across different targets
+    this is required to avoid redundant schema fetch when multiple targets use the same schema for validation
+    """
+    validate_files_map = defaultdict(list)
+    for target_obj in target_objs:
+        target_name = target_obj['vars']['target']
+        if 'validate' not in target_obj:
+            logger.debug("target '{}' does not have 'validate' parameter. skipping".format(target_name))
+            continue
+
+        for validate_item in target_obj['validate']:
+            validate_type = validate_item['type']
+            if validate_type == 'kubernetes':
+                kind_version_pair = (validate_item['kind'],
+                                     validate_item.get('version', DEFAULT_KUBERNETES_VERSION))
+                for output_path in validate_item['output_paths']:
+                    full_output_path = os.path.join(compiled_path, target_name, output_path)
+                    if not os.path.isfile(full_output_path):
+                        logger.warning("{} does not exist for target '{}'. skipping".
+                                       format(output_path, target_name))
+                        continue
+                    validate_files_map[kind_version_pair].append(full_output_path)
+            else:
+                logger.warning('type {} is not supported for validation. skipping'.format(validate_type))
+
+    return validate_files_map
+
+
+def schema_validate_kubernetes_output(validate_data, cache_dir):
+    """
+    validates given files according to kubernetes manifest schemas
+    schemas are cached from/to cache_dir
+    validate_data must be of structure ((kind, version), validate_files)
+    """
+    (kind, version), validate_files = validate_data
+    KubernetesManifestValidator(cache_dir).validate(validate_files, kind=kind, version=version)

kapitan/utils.py

@@ -16,6 +16,8 @@
 
 from __future__ import print_function
 
+import requests
+
 "random utils"
 
 import logging
@@ -462,3 +464,13 @@ def search_target_token_paths(target_secrets_path, targets):
             logger.debug('search_target_token_paths: found %s', secret_path)
             target_files[target_name].append(secret_path)
     return target_files
+
+
+def make_request(source):
+    """downloads the http file at source and returns it's content"""
+    r = requests.get(source)
+    if r.ok:
+        return r.content, r.headers['Content-Type']
+    else:
+        r.raise_for_status()
+    return None, None