Label selectors for compilation

docs/usage.md

@@ -34,13 +34,16 @@ Additional parameters are available for each positional argument. For example:
 
 ```
 $ kapitan compile -h
-usage: kapitan compile [-h] [--search-paths JPATH [JPATH ...]] [--verbose]
-                       [--prune] [--quiet] [--output-path PATH]
-                       [--targets TARGET [TARGET ...]] [--parallelism INT]
-                       [--indent INT] [--secrets-path SECRETS_PATH] [--reveal]
+usage: kapitan compile [-h] [--search-paths JPATH [JPATH ...]]
+                       [--jinja2-filters FPATH] [--verbose] [--prune]
+                       [--quiet] [--output-path PATH] [--fetch] [--validate]
+                       [--parallelism INT] [--indent INT]
+                       [--refs-path REFS_PATH] [--reveal]
                        [--inventory-path INVENTORY_PATH] [--cache]
                        [--cache-paths PATH [PATH ...]]
-                       [--ignore-version-check]
+                       [--ignore-version-check] [--schemas-path SCHEMAS_PATH]
+                       [--targets TARGET [TARGET ...] | --labels
+                       [key=value [key=value ...]]]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -53,15 +56,16 @@ optional arguments:
   --prune               prune jsonnet output
   --quiet               set quiet mode, only critical output
   --output-path PATH    set output path, default is "."
-  --targets TARGET [TARGET ...], -t TARGET [TARGET ...]
-                        targets to compile, default is all
+  --fetch               fetches external dependencies
+  --validate            validate compile output against schemas as specified
+                        in inventory
   --parallelism INT, -p INT
                         Number of concurrent compile processes, default is 4
   --indent INT, -i INT  Indentation spaces for YAML/JSON, default is 2
-  --secrets-path SECRETS_PATH
-                        set secrets path, default is "./secrets"
-  --reveal              reveal secrets (warning: this will write sensitive
-                        data)
+  --refs-path REFS_PATH
+                        set refs path, default is "./refs"
+  --reveal              reveal refs (warning: this will potentially write
+                        sensitive data)
   --inventory-path INVENTORY_PATH
                         set inventory path, default is "./inventory"
   --cache, -c           enable compilation caching to .kapitan_cache, default
@@ -71,6 +75,41 @@ optional arguments:
                         []
   --ignore-version-check
                         ignore the version from .kapitan
+  --schemas-path SCHEMAS_PATH
+                        set schema cache path, default is "./schemas"
+  --targets TARGET [TARGET ...], -t TARGET [TARGET ...]
+                        targets to compile, default is all
+  --labels [key=value [key=value ...]], -l [key=value [key=value ...]]
+                        compile targets matching the labels, default is all
+```
+
+## Selective target compilation
+
+If you only want to compile a subset or specific targets, you can use the two kapitan compile flags `--targets, -t` or `--labels, -l`.
+
+#### Specific target(s)
+
+```
+$ cd examples/kubernetes
+$ kapitan compile -t minikube-mysql
+Compiled minikube-mysql (0.43s)
+```
+
+#### Using labels
+
+```
+$ cd examples/kubernetes
+
+$ cat inventory/classes/component/nginx-kadet.yml  # Inherited by minikube-nginx-kadet target
+parameters:
+  ...
+  kapitan:
+    ...
+    labels:
+      type: kadet
+
+$ kapitan compile -l type=kadet
+Compiled minikube-nginx-kadet (0.14s)
 ```
 
 ## Using `.kapitan` config file
@@ -123,4 +162,4 @@ which would be equivalent to always running:
 
 ```
 kapitan inventory --inventory-path=./some_path
-```
+```

examples/kubernetes/inventory/classes/component/nginx-jsonnet.yml

@@ -7,4 +7,6 @@ parameters:
         output_type: yaml
         input_type: jsonnet
         input_paths:
-        - components/nginx-jsonnet/main.jsonnet
+        - components/nginx-jsonnet/main.jsonnet
+    labels:
+      type: jsonnet

examples/kubernetes/inventory/classes/component/nginx-kadet.yml

@@ -8,3 +8,5 @@ parameters:
       output_type: yaml
       input_paths:
         - components/nginx-kadet/
+    labels:
+      type: kadet

kapitan/cli.py

@@ -101,10 +101,6 @@ def main():
     compile_parser.add_argument('--validate',
                                 help='validate compile output against schemas as specified in inventory',
                                 action='store_true', default=from_dot_kapitan('compile', 'validate', False))
-    compile_parser.add_argument('--targets', '-t', help='targets to compile, default is all',
-                                type=str, nargs='+',
-                                default=from_dot_kapitan('compile', 'targets', []),
-                                metavar='TARGET')
     compile_parser.add_argument('--parallelism', '-p', type=int,
                                 default=from_dot_kapitan('compile', 'parallelism', 4),
                                 metavar='INT',
@@ -138,6 +134,16 @@ def main():
                                 default=from_dot_kapitan('validate', 'schemas-path', './schemas'),
                                 help='set schema cache path, default is "./schemas"')
 
+    compile_selector_parser = compile_parser.add_mutually_exclusive_group()
+    compile_selector_parser.add_argument('--targets', '-t', help='targets to compile, default is all',
+                                type=str, nargs='+',
+                                default=from_dot_kapitan('compile', 'targets', []),
+                                metavar='TARGET')
+    compile_selector_parser.add_argument('--labels', '-l', help='compile targets matching the labels, default is all',
+                                type=str, nargs='*',
+                                default=from_dot_kapitan('compile', 'labels', []),
+                                metavar='key=value')
+
     inventory_parser = subparser.add_parser('inventory', help='show inventory')
     inventory_parser.add_argument('--target-name', '-t',
                                   default=from_dot_kapitan('inventory', 'target-name', ''),
@@ -318,7 +324,7 @@ def _search_imports(cwd, imp):
         cached.revealer_obj = Revealer(ref_controller)
 
         compile_targets(args.inventory_path, search_paths, args.output_path,
-                        args.parallelism, args.targets, ref_controller,
+                        args.parallelism, args.targets, args.labels, ref_controller,
                         prune=(args.prune), indent=args.indent, reveal=args.reveal,
                         cache=args.cache, cache_paths=args.cache_paths,
                         fetch_dependencies=args.fetch, validate=args.validate,

kapitan/targets.py

@@ -45,15 +45,15 @@
 logger = logging.getLogger(__name__)
 
 
-def compile_targets(inventory_path, search_paths, output_path, parallel, targets, ref_controller, **kwargs):
+def compile_targets(inventory_path, search_paths, output_path, parallel, targets, labels, ref_controller, **kwargs):
     """
     Searches and loads target files, and runs compile_target() on a
     multiprocessing pool with parallel number of processes.
     kwargs are passed to compile_target()
     """
     # temp_path will hold compiled items
     temp_path = tempfile.mkdtemp(suffix='.kapitan')
-    updated_targets = targets
+    updated_targets = search_targets(inventory_path, targets, labels)
     # If --cache is set
     if kwargs.get('cache'):
         additional_cache_paths = kwargs.get('cache_paths')
@@ -306,6 +306,43 @@ def load_target_inventory(inventory_path, targets):
     return target_objs
 
 
+def search_targets(inventory_path, targets, labels):
+    """returns a list of targets where the labels match, otherwise just return the original targets"""
+    if not labels:
+        return targets
+
+    try:
+        labels_dict = dict(label.split('=') for label in labels)
+    except ValueError:
+        logger.error("Compile error: Failed to parse labels, should be formatted like: kapitan compile -l env=prod app=example")
+        sys.exit(1)
+
+    targets_found = []
+    inv = inventory_reclass(inventory_path)
+
+    for target_name in inv['nodes']:
+        matched_all_labels = False
+        for label, value in labels_dict.items():
+            try:
+                if inv['nodes'][target_name]['parameters']['kapitan']['labels'][label] == value:
+                    matched_all_labels = True
+                    continue
+            except KeyError:
+                logger.debug("search_targets: label %s=%s didn't match target %s", label, value, target_name)
+
+            matched_all_labels = False
+            break
+
+        if matched_all_labels:
+            targets_found.append(target_name)
+
+    if len(targets_found) == 0:
+        logger.error("No targets found with labels: {}".format(labels))
+        sys.exit(1)
+
+    return targets_found
+
+
 def compile_target(target_obj, search_paths, compile_path, ref_controller, **kwargs):
     """Compiles target_obj and writes to compile_path"""
     start = time.time()

tests/test_compile.py

@@ -21,6 +21,7 @@
 import sys
 import io
 import contextlib
+import shutil
 from kapitan.cli import main
 from kapitan.utils import directory_hash
 from kapitan.cached import reset_cache
@@ -82,6 +83,24 @@ def test_compile_vars_target_missing(self):
                         "This parameter should be set to the target name"
         self.assertTrue(error_message.format(target_filename), ie.exception.args[0])
 
+    def test_compile_specific_target(self):
+        shutil.rmtree("compiled")
+        sys.argv = ["kapitan", "compile", "-t", "minikube-mysql"]
+        main()
+        self.assertTrue(os.path.exists("compiled/minikube-mysql") and not os.path.exists("compiled/minikube-es"))
+        # Reset compiled dir
+        sys.argv = ["kapitan", "compile"]
+        main()
+
+    def test_compile_target_with_label(self):
+        shutil.rmtree("compiled")
+        sys.argv = ["kapitan", "compile", "-l", "type=kadet"]
+        main()
+        self.assertTrue(os.path.exists("compiled/minikube-nginx-kadet") and not os.path.exists("compiled/minikube-nginx-jsonnet"))
+        # Reset compiled dir
+        sys.argv = ["kapitan", "compile"]
+        main()
+
     def tearDown(self):
         os.chdir(os.getcwd() + '/../../')
         reset_cache()

tests/test_jinja2.py

@@ -176,7 +176,7 @@ def test_reveal_maybe_tag_no_reveal_flag(self):
             f.write("{{ my_ref_tag_var|reveal_maybe }}".encode("UTF-8"))
             f.seek(0)
 
-            # new argparse namespace with --reveal and --secrets-path values
+            # new argparse namespace with --reveal and --refs-path values
             namespace = namedtuple('Namespace', [])
             namespace.reveal = False
             namespace.refs_path = tempfile.mkdtemp()
@@ -200,7 +200,7 @@ def test_reveal_maybe_no_tag(self):
             f.write("{{ my_var|reveal_maybe }}".encode("UTF-8"))
             f.seek(0)
 
-            # new argparse namespace with --reveal and --secrets-path values
+            # new argparse namespace with --reveal and --refs-path values
             namespace = namedtuple('Namespace', [])
             namespace.reveal = True
             namespace.refs_path = tempfile.mkdtemp()