Add kubernetes manifest validation

kapitan/dependency_manager/base.py

@@ -12,6 +12,7 @@
 from git import Repo
 
 from kapitan.errors import GitSubdirNotFoundError
+from kapitan.utils import make_request
 
 logger = logging.getLogger(__name__)
 
@@ -134,7 +135,9 @@ def fetch_http_dependency(dep_mapping, save_dir):
 
 def fetch_http_source(source, save_dir):
     """downloads a http[s] file from source and saves into save_dir"""
-    content, content_type = _make_request(source)
+    logger.info("Dependency {} : fetching now".format(source))
+    content, content_type = make_request(source)
+    logger.info("Dependency {} : successfully fetched".format(source))
     if content is not None:
         basename = os.path.basename(source)
         # to avoid collisions between basename(source)
@@ -146,15 +149,3 @@ def fetch_http_source(source, save_dir):
     else:
         logger.warning("Dependency {} : failed to fetch".format(source))
         return None
-
-
-def _make_request(source):
-    """downloads the http file at source and returns it's content"""
-    logger.info("Dependency {} : fetching now".format(source))
-    r = requests.get(source)
-    if r.ok:
-        logger.info("Dependency {} : successfully fetched".format(source))
-        return r.content, r.headers['Content-Type']
-    else:
-        r.raise_for_status()
-    return None, None

kapitan/errors.py

@@ -58,3 +58,13 @@ class RefHashMismatchError(KapitanError):
 class GitSubdirNotFoundError(KapitanError):
     """git dependency subdir not found error"""
     pass
+
+
+class RequestUnsuccessfulError(KapitanError):
+    """request error"""
+    pass
+
+
+class KubernetesManifestValidationError(KapitanError):
+    """kubernetes manifest schema validation error"""
+    pass

kapitan/utils.py

@@ -16,6 +16,8 @@
 
 from __future__ import print_function
 
+import requests
+
 "random utils"
 
 import logging
@@ -462,3 +464,13 @@ def search_target_token_paths(target_secrets_path, targets):
             logger.debug('search_target_token_paths: found %s', secret_path)
             target_files[target_name].append(secret_path)
     return target_files
+
+
+def make_request(source):
+    """downloads the http file at source and returns it's content"""
+    r = requests.get(source)
+    if r.ok:
+        return r.content, r.headers['Content-Type']
+    else:
+        r.raise_for_status()
+    return None, None

kapitan/validator/base.py

@@ -0,0 +1,6 @@
+class Validator(object):
+    def __init__(self, cache_dir, **kwargs):
+        self.cache_dir = cache_dir
+
+    def validate(self, validate_obj, **kwargs):
+        raise NotImplementedError

kapitan/validator/kubernetes_validator.py

@@ -0,0 +1,81 @@
+import os
+from functools import lru_cache
+
+import yaml
+import logging
+import jsonschema
+
+from kapitan.errors import RequestUnsuccessfulError, KubernetesManifestValidationError
+from kapitan.validator.base import Validator
+from kapitan.utils import make_request
+
+DEFAULT_VERSION = '1.14.0'
+
+logger = logging.getLogger(__name__)
+
+
+class KubernetesManifestValidator(Validator):
+    def __init__(self, cache_dir, **kwargs):
+        super().__init__(cache_dir, **kwargs)
+        self.schema_type = 'standalone-strict'
+        self.file_path_format = 'v{}-%s/{}.json' % self.schema_type
+
+    def validate(self, validate_obj, **kwargs):
+        """
+        validates validate_obj against json schema as specified by
+        'kind' and 'version' in kwargs
+        raises KubernetesManifestValidationError if validation fails, listing all the errors
+        inside validate_obj
+        """
+        kind = kwargs.get('kind')
+        version = kwargs.get('version', DEFAULT_VERSION)
+        schema = self._get_schema(kind, version)
+        v = jsonschema.Draft4Validator(schema)
+        errors = sorted(v.iter_errors(validate_obj), key=lambda e: e.path)
+        if errors:
+            error_message = ''
+            if 'file_path' in kwargs and 'target_name' in kwargs:
+                error_message += 'invalid manifest for target "{}" at {}\n'.format(
+                    kwargs.get('target_name'), kwargs.get('file_path'))
+
+            error_message += '\n'.join(['{} {}'.format(list(error.path), error.message) for error in errors])
+            raise KubernetesManifestValidationError(error_message)
+
+    @lru_cache(maxsize=256)
+    def _get_schema(self, kind, version):
+        """gets json validation schema from web or cache"""
+        schema = self._get_cached_schema(kind, version)
+        if schema is None:
+            schema = self._get_schema_from_web(kind, version)
+            self._cache_schema(kind, version, schema)
+        return schema
+
+    def _cache_schema(self, kind, version, schema):
+        cache_path = self._get_cache_path(kind, version)
+        os.makedirs(os.path.dirname(cache_path), exist_ok=True)
+        with open(cache_path, 'w') as fp:
+            yaml.safe_dump(schema, stream=fp, default_flow_style=False)
+
+    def _get_cached_schema(self, kind, version):
+        cache_path = self._get_cache_path(kind, version)
+        if os.path.isfile(cache_path):
+            with open(cache_path, 'r') as fp:
+                return yaml.safe_load(fp.read())
+        else:
+            return None
+
+    def _get_schema_from_web(self, kind, version):
+        url = self._get_request_url(kind, version)
+        logger.debug("Fetching schema from {}".format(url))
+        content, _ = make_request(url)
+        if content is None:
+            raise RequestUnsuccessfulError("schema failed to fetch from {}"
+                                           "\nThe specified version or kind may not be supported".format(url))
+        logger.debug("schema fetched  from {}".format(url))
+        return yaml.safe_load(content)
+
+    def _get_request_url(self, kind, version):
+        return 'https://kubernetesjsonschema.dev/' + self.file_path_format.format(version, kind)
+
+    def _get_cache_path(self, kind, version):
+        return os.path.join(self.cache_dir, self.file_path_format.format(version, kind))

tests/test_kubernetes_validator.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 The Kapitan Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import unittest
+import tempfile
+import yaml
+
+from kapitan.cached import reset_cache
+from kapitan.cli import main
+from kapitan.errors import KubernetesManifestValidationError
+from kapitan.validator.kubernetes_validator import KubernetesManifestValidator
+
+
+class KubernetesValidatorTest(unittest.TestCase):
+    def setUp(self):
+        self.cache_dir = tempfile.mkdtemp()
+        self.validator = KubernetesManifestValidator(self.cache_dir)
+
+    def test_download_and_cache(self):
+        downloaded_schema = self.validator._get_schema_from_web('service', '1.14.0')
+        self.validator._cache_schema('service', '1.14.0', downloaded_schema)
+        self.assertTrue(os.path.isfile(os.path.join(self.cache_dir, 'v1.14.0-standalone-strict', 'service.json')))
+
+    def test_load_from_cache(self):
+        kind = 'deployment'
+        version = '1.11.0'
+        downloaded_schema = self.validator._get_schema_from_web(kind, version)
+        self.validator._cache_schema(kind, version, downloaded_schema)
+        self.assertIsInstance(self.validator._get_cached_schema(kind, version), dict)
+
+    def test_validate(self):
+        service_manifest_string = """
+            apiVersion: v1
+            kind: Service
+            metadata:
+              name: my-service
+            spec:
+              selector:
+                app: MyApp
+              ports:
+              - protocol: TCP
+                port: 80
+                targetPort: 9376
+        """
+
+        service_manifest = yaml.safe_load(service_manifest_string)
+        self.validator.validate(service_manifest, kind='service', version='1.14.0')
+
+        with self.assertRaises(KubernetesManifestValidationError):
+            self.validator.validate(service_manifest, kind='deployment', version='1.14.0',
+                                    file_path='service/manifest', target_name='example')
+
+    def test_compile_with_validate(self):
+        pass
+
+    def tearDown(self):
+        reset_cache()