From c121edd67c856f56bac0cf76799b6f016690b995 Mon Sep 17 00:00:00 2001 From: yoshi-1224 Date: Fri, 5 Jul 2019 16:59:58 +0800 Subject: [PATCH] Add kubernetes manifest validation class --- kapitan/dependency_manager/base.py | 17 ++--- kapitan/errors.py | 10 +++ kapitan/utils.py | 12 ++++ kapitan/validator/__init__.py | 0 kapitan/validator/base.py | 6 ++ kapitan/validator/kubernetes_validator.py | 77 +++++++++++++++++++++++ tests/test_kubernetes_validator.py | 73 +++++++++++++++++++++ 7 files changed, 182 insertions(+), 13 deletions(-) create mode 100644 kapitan/validator/__init__.py create mode 100644 kapitan/validator/base.py create mode 100644 kapitan/validator/kubernetes_validator.py create mode 100644 tests/test_kubernetes_validator.py diff --git a/kapitan/dependency_manager/base.py b/kapitan/dependency_manager/base.py index f015d7ace..2522c4c1b 100644 --- a/kapitan/dependency_manager/base.py +++ b/kapitan/dependency_manager/base.py @@ -12,6 +12,7 @@ from git import Repo from kapitan.errors import GitSubdirNotFoundError +from kapitan.utils import make_request logger = logging.getLogger(__name__) @@ -134,7 +135,9 @@ def fetch_http_dependency(dep_mapping, save_dir): def fetch_http_source(source, save_dir): """downloads a http[s] file from source and saves into save_dir""" - content, content_type = _make_request(source) + logger.info("Dependency {} : fetching now".format(source)) + content, content_type = make_request(source) + logger.info("Dependency {} : successfully fetched".format(source)) if content is not None: basename = os.path.basename(source) # to avoid collisions between basename(source) @@ -146,15 +149,3 @@ def fetch_http_source(source, save_dir): else: logger.warning("Dependency {} : failed to fetch".format(source)) return None - - -def _make_request(source): - """downloads the http file at source and returns it's content""" - logger.info("Dependency {} : fetching now".format(source)) - r = requests.get(source) - if r.ok: - logger.info("Dependency {} : successfully fetched".format(source)) - return r.content, r.headers['Content-Type'] - else: - r.raise_for_status() - return None, None diff --git a/kapitan/errors.py b/kapitan/errors.py index e1ce93330..526ff59ab 100644 --- a/kapitan/errors.py +++ b/kapitan/errors.py @@ -58,3 +58,13 @@ class RefHashMismatchError(KapitanError): class GitSubdirNotFoundError(KapitanError): """git dependency subdir not found error""" pass + + +class RequestUnsuccessfulError(KapitanError): + """request error""" + pass + + +class KubernetesManifestValidationError(KapitanError): + """kubernetes manifest schema validation error""" + pass diff --git a/kapitan/utils.py b/kapitan/utils.py index 257cf5e7d..b0fe77566 100644 --- a/kapitan/utils.py +++ b/kapitan/utils.py @@ -16,6 +16,8 @@ from __future__ import print_function +import requests + "random utils" import logging @@ -462,3 +464,13 @@ def search_target_token_paths(target_secrets_path, targets): logger.debug('search_target_token_paths: found %s', secret_path) target_files[target_name].append(secret_path) return target_files + + +def make_request(source): + """downloads the http file at source and returns it's content""" + r = requests.get(source) + if r.ok: + return r.content, r.headers['Content-Type'] + else: + r.raise_for_status() + return None, None diff --git a/kapitan/validator/__init__.py b/kapitan/validator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kapitan/validator/base.py b/kapitan/validator/base.py new file mode 100644 index 000000000..fc1517b37 --- /dev/null +++ b/kapitan/validator/base.py @@ -0,0 +1,6 @@ +class Validator(object): + def __init__(self, cache_dir, **kwargs): + self.cache_dir = cache_dir + + def validate(self, validate_obj, **kwargs): + raise NotImplementedError diff --git a/kapitan/validator/kubernetes_validator.py b/kapitan/validator/kubernetes_validator.py new file mode 100644 index 000000000..48a95718b --- /dev/null +++ b/kapitan/validator/kubernetes_validator.py @@ -0,0 +1,77 @@ +import os +from functools import lru_cache +from pprint import pprint + +import yaml +import logging +import jsonschema + +from kapitan.errors import RequestUnsuccessfulError, KubernetesManifestValidationError +from kapitan.validator.base import Validator +from kapitan.utils import make_request + +DEFAULT_VERSION = '1.14.0' + +logger = logging.getLogger(__name__) + + +class KubernetesValidator(Validator): + def __init__(self, cache_dir, **kwargs): + super().__init__(cache_dir, **kwargs) + self.cache_dir = cache_dir + self.schema_type = 'standalone-strict' + self.file_path_format = 'v{}-%s/{}.json' % self.schema_type + + def validate(self, validate_obj, **kwargs): + kind = kwargs.get('kind') + version = kwargs.get('version', DEFAULT_VERSION) + schema = self._get_schema(kind, version) + v = jsonschema.Draft4Validator(schema) + errors = sorted(v.iter_errors(validate_obj), key=lambda e: e.path) + if errors: + error_message = '' + if 'file_path' in kwargs and 'target_name' in kwargs: + error_message += 'invalid manifest for target "{}" at {}\n'.format( + kwargs.get('target_name'), kwargs.get('file_path')) + + error_message += '\n'.join(['{} {}'.format(list(error.path), error.message) for error in errors]) + raise KubernetesManifestValidationError(error_message) + + @lru_cache(maxsize=256) + def _get_schema(self, kind, version): + """gets json validation schema from web or cache""" + schema = self._get_cached_schema(kind, version) + if schema is None: + schema = self._get_schema_from_web(kind, version) + self._cache_schema(kind, version, schema) + return schema + + def _cache_schema(self, kind, version, schema): + cache_path = self._get_cache_path(kind, version) + os.makedirs(os.path.dirname(cache_path), exist_ok=True) + with open(cache_path, 'w') as fp: + yaml.safe_dump(schema, stream=fp, default_flow_style=False) + + def _get_cached_schema(self, kind, version): + cache_path = self._get_cache_path(kind, version) + if os.path.isfile(cache_path): + with open(cache_path, 'r') as fp: + return yaml.safe_load(fp.read()) + else: + return None + + def _get_schema_from_web(self, kind, version): + url = self._get_request_url(kind, version) + logger.debug("Fetching schema from {}".format(url)) + content, _ = make_request(url) + if content is None: + raise RequestUnsuccessfulError("schema failed to fetch from {}" + "\nThe specified version or kind may not be supported".format(url)) + logger.debug("schema fetched from {}".format(url)) + return yaml.safe_load(content) + + def _get_request_url(self, kind, version): + return 'https://kubernetesjsonschema.dev/' + self.file_path_format.format(version, kind) + + def _get_cache_path(self, kind, version): + return os.path.join(self.cache_dir, self.file_path_format.format(version, kind)) diff --git a/tests/test_kubernetes_validator.py b/tests/test_kubernetes_validator.py new file mode 100644 index 000000000..6ab36dcba --- /dev/null +++ b/tests/test_kubernetes_validator.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 The Kapitan Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +import tempfile + +import jsonschema +import yaml + +from kapitan.cached import reset_cache +from kapitan.cli import main +from kapitan.errors import KubernetesManifestValidationError +from kapitan.validator.kubernetes_validator import KubernetesValidator + + +class KubernetesValidatorTest(unittest.TestCase): + def setUp(self): + self.cache_dir = tempfile.mkdtemp() + self.validator = KubernetesValidator(self.cache_dir) + + def test_download_and_cache(self): + downloaded_schema = self.validator._get_schema_from_web('service', '1.14.0') + self.validator._cache_schema('service', '1.14.0', downloaded_schema) + self.assertTrue(os.path.isfile(os.path.join(self.cache_dir, 'v1.14.0-standalone-strict', 'service.json'))) + + def test_load_from_cache(self): + kind = 'deployment' + version = '1.11.0' + downloaded_schema = self.validator._get_schema_from_web(kind, version) + self.validator._cache_schema(kind, version, downloaded_schema) + self.assertIsInstance(self.validator._get_cached_schema(kind, version), dict) + + def test_validate(self): + service_manifest_string = """ + apiVersion: v1 + kind: Service + metadata: + name: my-service + spec: + selector: + app: MyApp + ports: + - protocol: TCP + port: 80 + targetPort: 9376 + """ + + service_manifest = yaml.safe_load(service_manifest_string) + self.validator.validate(service_manifest, kind='service', version='1.14.0', + file_path='service manifest', target_name='example') + + with self.assertRaises(KubernetesManifestValidationError): + self.validator.validate(service_manifest, kind='deployment', version='1.14.0', + file_path='service/manifest', target_name='example') + + def test_compile_with_validate(self): + pass + + def tearDown(self): + reset_cache()