Optimisations

Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.6-alpine
 
-RUN apk add --no-cache git g++ make libstdc++ gnupg musl-dev && \
+RUN apk add --no-cache git g++ make libstdc++ gnupg musl-dev yaml-dev && \
     mkdir /kapitan
 
 WORKDIR /kapitan

README.md

@@ -50,8 +50,8 @@ For CI/CD usage, check out [ci/](https://github.com/deepmind/kapitan/tree/master
 Kapitan needs Python 3.6.
 
 **Install Python 3:**
-<br>Linux: `sudo apt-get update && sudo apt-get install -y python3.6-dev python3-pip`
-<br>Mac: `brew install python3`
+<br>Linux: `sudo apt-get update && sudo apt-get install -y python3.6-dev python3-pip python3-yaml`
+<br>Mac: `brew install python3 libyaml`
 
 **Install Kapitan:**
 

ci/Dockerfile

@@ -1,6 +1,6 @@
 FROM google/cloud-sdk:alpine
 
-RUN apk add --no-cache python3-dev git g++ make libstdc++ gnupg musl-dev util-linux openssl coreutils && \
+RUN apk add --no-cache python3-dev git g++ make libstdc++ gnupg musl-dev util-linux openssl coreutils yaml-dev && \
     python3 -m ensurepip && \
     rm -rf /usr/lib/python*/ensurepip && \
     pip3 install --upgrade --no-cache-dir pip setuptools && \

kapitan/cli.py

@@ -19,7 +19,7 @@
 from __future__ import print_function
 
 import argparse
-import json
+import ujson as json
 import logging
 import os
 import sys

kapitan/resources.py

@@ -18,7 +18,7 @@
 
 import errno
 from functools import partial
-import json
+import ujson as json
 import logging
 import os
 import io
@@ -34,6 +34,11 @@
 
 logger = logging.getLogger(__name__)
 
+try:
+    from yaml import CSafeLoader as YamlLoader
+except ImportError:
+    from yaml import SafeLoader as YamlLoader
+
 
 def resource_callbacks(search_path):
     """
@@ -90,11 +95,6 @@ def read_file(search_path, name):
     raise IOError("Could not find file %s" % name)
 
 
-def kapitan_path():
-    "return kapitan install path"
-    return os.path.dirname(kapitan_install_path)
-
-
 def search_imports(cwd, import_str, search_path):
     """
     This is only to be used as a callback for the jsonnet API!
@@ -109,7 +109,7 @@ def search_imports(cwd, import_str, search_path):
 
     # if import_str not found, search in search_path
     if not os.path.exists(full_import_path):
-        install_path = kapitan_path()
+        install_path = os.path.dirname(kapitan_install_path)
         for path in (install_path, search_path):
             _full_import_path = os.path.join(path, import_str)
             # if found, set as full_import_path
@@ -167,7 +167,7 @@ def inventory_reclass(inventory_path):
         try:
             cfg_file = os.path.join(inventory_path, 'reclass-config.yml')
             with open(cfg_file) as reclass_cfg:
-                reclass_config = yaml.safe_load(reclass_cfg)
+                reclass_config = yaml.load(reclass_cfg, Loader=YamlLoader)
                 # normalise relative nodes_uri and classes_uri paths
                 for uri in ('nodes_uri', 'classes_uri'):
                     uri_val = reclass_config.get(uri)

kapitan/secrets.py

@@ -22,7 +22,7 @@
 import errno
 from functools import partial
 import hashlib
-import json
+import ujson as json
 import logging
 import os
 import re
@@ -40,6 +40,11 @@
 SECRET_TOKEN_ATTR_PATTERN = r"(\w+):([\w\.\-\/]+)"  # e.g. gpg:my/secret/token
 SECRET_TOKEN_COMPILED_ATTR_PATTERN = r"(\w+):([\w\.\-\/]+):(\w+)"  # e.g. gpg:my/secret/token:1deadbeef
 
+try:
+    from yaml import CSafeLoader as YamlLoader
+except ImportError:
+    from yaml import SafeLoader as YamlLoader
+
 
 class GPGError(Exception):
     "Generic GPG errors"
@@ -73,7 +78,7 @@ def secret_gpg_read(gpg_obj, secrets_path, token, **kwargs):
     full_secret_path = os.path.join(secrets_path, token_path)
     try:
         with open(full_secret_path) as fp:
-            secret_obj = yaml.safe_load(fp)
+            secret_obj = yaml.load(fp, Loader=YamlLoader)
             data_decoded = base64.b64decode(secret_obj['data'])
             dec = secret_gpg_decrypt(gpg_obj, data_decoded, **kwargs)
             logger.debug("Read secret %s at %s", token, full_secret_path)
@@ -194,7 +199,7 @@ def secret_gpg_raw_read(secrets_path, token):
     full_secret_path = os.path.join(secrets_path, token_path)
     try:
         with open(full_secret_path) as fp:
-            secret_obj = yaml.safe_load(fp)
+            secret_obj = yaml.load(fp, Loader=YamlLoader)
             logger.debug("Read raw secret %s at %s", token, full_secret_path)
             return secret_obj
     except IOError as ex:
@@ -219,6 +224,7 @@ def reveal_gpg_replace(gpg_obj, secrets_path, match_obj, verify=True, **kwargs):
     logger.debug("Revealing %s", token_tag)
     return secret_gpg_read(gpg_obj, secrets_path, token, **kwargs)
 
+
 def secret_gpg_update_recipients(gpg_obj, secrets_path, token_path, recipients, **kwargs):
     "updates the recipient list for secret in token_path"
     token = "gpg:%s" % token_path
@@ -233,6 +239,7 @@ def secret_gpg_update_recipients(gpg_obj, secrets_path, token_path, recipients,
     secret_gpg_write(gpg_obj, secrets_path, token_path, data_dec, encode_base64,
                      recipients, **kwargs)
 
+
 def secret_gpg_reveal_raw(gpg_obj, secrets_path, filename, verify=True, output=None, **kwargs):
     """
     read filename and reveal content (per line search and replace) with secrets to stdout
@@ -323,7 +330,7 @@ def secret_gpg_reveal_file(gpg_obj, secrets_path, filename, verify=True, **kwarg
     elif filename.endswith('.yml'):
         logger.debug("secret_gpg_reveal_file: revealing yml file: %s", filename)
         with open(filename) as fp:
-            obj = yaml.safe_load(fp)
+            obj = yaml.load(fp, Loader=YamlLoader)
             rev_obj = secret_gpg_reveal_obj(gpg_obj, secrets_path, obj,
                                             verify=verify, **kwargs)
             out = yaml.dump(rev_obj, Dumper=PrettyDumper,
@@ -335,6 +342,7 @@ def secret_gpg_reveal_file(gpg_obj, secrets_path, filename, verify=True, **kwarg
                                         verify=verify, **kwargs)
     return out
 
+
 def search_target_token_paths(target_secrets_path, targets):
     """
     returns dict of target and their secret token paths in target_secrets_path
@@ -351,6 +359,7 @@ def search_target_token_paths(target_secrets_path, targets):
                 target_files[target_name].append(full_path)
     return target_files
 
+
 def lookup_fingerprints(gpg_obj, recipients):
     "returns a list of fingerprints for recipients obj"
     lookedup = []
@@ -367,6 +376,7 @@ def lookup_fingerprints(gpg_obj, recipients):
     # Remove duplicates, sort and return fingerprints list
     return sorted(set(lookedup))
 
+
 def secret_gpg_raw_read_fingerprints(secrets_path, token_path):
     "returns fingerprint list in raw secret for token_path"
     token = "gpg:%s" % token_path

kapitan/targets.py

@@ -22,7 +22,7 @@
 import os
 import errno
 import hashlib
-import json
+import ujson as json
 import re
 import shutil
 import sys
@@ -42,6 +42,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 def compile_targets(inventory_path, search_path, output_path, parallel, targets, **kwargs):
     """
     Searches and loads target files, and runs compile_target_file() on a
@@ -303,30 +304,6 @@ def valid_target_obj(target_obj):
     return jsonschema.validate(target_obj, schema)
 
 
-def load_target_file(target_file):
-    """
-    Loads target_file and returns a target obj
-    Format of the target file is determined by its extention (.json, .yml, .yaml)
-    """
-
-    bname = os.path.basename(target_file)
-
-    if re.match(r".+\.json$", bname):
-        with open(target_file) as fp:
-            target_obj = json.load(fp)
-            valid_target_obj(target_obj)
-            logger.debug("Target file %s is valid", target_file)
-
-            return target_obj
-    if re.match(r".+\.(yaml|yml)$", bname):
-        with open(target_file) as fp:
-            target_obj = yaml.safe_load(fp)
-            valid_target_obj(target_obj)
-            logger.debug("Target file %s is valid", target_file)
-
-            return target_obj
-
-
 class CompilingFile(object):
     def __init__(self, context, fp, **kwargs):
         self.fp = fp

kapitan/utils.py

@@ -18,7 +18,7 @@
 
 "random utils"
 
-import functools
+from functools import reduce
 from hashlib import sha256
 import logging
 import os
@@ -33,9 +33,13 @@
 from kapitan.version import VERSION
 from kapitan.errors import CompileError
 
-
 logger = logging.getLogger(__name__)
 
+try:
+    from yaml import CSafeLoader as YamlLoader
+except ImportError:
+    from yaml import SafeLoader as YamlLoader
+
 
 class termcolor:
     HEADER = '\033[95m'
@@ -183,7 +187,7 @@ def deep_get(dictionary, keys):
     '''
     Return (keys) values from dictionary
     '''
-    return functools.reduce(lambda d, key: d.get(key, None) if isinstance(d, dict) else None, keys, dictionary)
+    return reduce(lambda d, key: d.get(key, None) if isinstance(d, dict) else None, keys, dictionary)
 
 
 def searchvar(flat_var, inventory_path):
@@ -199,7 +203,7 @@ def searchvar(flat_var, inventory_path):
             if file.endswith(".yml") or file.endswith(".yaml"):
                 filename = os.path.join(root, file)
                 with open(filename, 'r') as fd:
-                    data = yaml.safe_load(fd)
+                    data = yaml.load(fd, Loader=YamlLoader)
                     value = deep_get(data, keys)
                     if value is not None:
                         output.append((filename, value))
@@ -246,9 +250,9 @@ def check_version():
             dot_kapitan = yaml.safe_load(f)
             # If 'saved version is bigger than current version'
             if dot_kapitan['version'] and StrictVersion(dot_kapitan['version']) > StrictVersion(VERSION):
-                print(f'{termcolor.WARNING}Current version: {VERSION}')
-                print(f'Last used version (in .kapitan): {dot_kapitan["version"]}{termcolor.ENDC}\n')
-                print(f'Please upgrade kapitan to at least "{dot_kapitan["version"]}" in order to keep results consistent:\n')
+                print('{}Current version: {}'.format(termcolor.WARNING, VERSION))
+                print('Last used version (in .kapitan): {}{}\n'.format(dot_kapitan["version"], termcolor.ENDC))
+                print('Please upgrade kapitan to at least "{}" in order to keep results consistent:\n'.format(dot_kapitan["version"]))
                 print('Docker: docker pull deepmind/kapitan')
                 print('Pip (user): pip3 install --user --upgrade git+https://github.com/deepmind/kapitan.git --process-dependency-links\n')
                 print('Check https://github.com/deepmind/kapitan#quickstart for more info.')

requirements.txt

@@ -1,5 +1,6 @@
 jsonnet==0.10.0
 PyYAML==3.12
+ujson==1.35
 Jinja2>=2.10
 # Latest commit from salt-formulas/reclass - develop branch (python3 support)
 # TODO: Change commit hash to release tag once develop is merged into master and release is cut