Vault secret backend

docs/kap_proposals/kap_6_hashicorp_vault.md

@@ -35,6 +35,8 @@ The entire string __"foo:hello"__ is base64 encoded and stored in the secret_ins
 data: Zm9vOmhlbGxvCg==  
 encoding: original  
 type: vaultkv  
+vault_client_param:  
+  auth: token
 ```  
 
 Encoding tells the type of data given to kapitan, if it is `original` then after decoding base64 we'll get the original secret and if it is `base64` then after decoding once we still have a base64 encoded secret and have to decode again.  
@@ -96,10 +98,10 @@ spec:
           name: cod
 ``` 
 
-Only the user with the required tokens/permissions can reveal the secrets. Please note that the roles and permissions will be handled at the Vault level. We need not worry about it within Kapitan. Also, it is mandatory to provide a target name since the parameters would be loaded from the inventory. Use the following command to reveal the secrets:  
+Only the user with the required tokens/permissions can reveal the secrets. Please note that the roles and permissions will be handled at the Vault level. We need not worry about it within Kapitan. Use the following command to reveal the secrets:  
 
 ```shell  
-$ kapitan secrets --reveal -f compile/file/containing/secret -t dev-sea
+$ kapitan secrets --reveal -f compile/file/containing/secret 
 ```  
 
 Following is the result of the cod-deployment.md file after Kapitan reveal.

kapitan/cli.py

@@ -202,6 +202,9 @@ def main():
     refs_parser.add_argument('--key', '-K', help='set KMS key',
                              default=from_dot_kapitan('refs', 'key', ''),
                              metavar='KEY')
+    refs_parser.add_argument('--auth', help='set authentication type for vaultkv secrets',
+                             default=from_dot_kapitan('refs', 'auth', ''),
+                             metavar='AUTH')
     refs_parser.add_argument('--refs-path', help='set refs path, default is "./refs"',
                              default=from_dot_kapitan('refs', 'refs-path', './refs'))
     refs_parser.add_argument('--verbose', '-v',
@@ -309,7 +312,7 @@ def _search_imports(cwd, imp):
         if not args.ignore_version_check:
             check_version()
 
-        ref_controller = RefController(args.refs_path,inventory_path=args.inventory_path)
+        ref_controller = RefController(args.refs_path)
         # cache controller for use in reveal_maybe jinja2 filter
         cached.ref_controller_obj = ref_controller
         cached.revealer_obj = Revealer(ref_controller)
@@ -357,8 +360,7 @@ def _search_imports(cwd, imp):
         sys.exit(1)
 
     elif cmd == 'refs':
-        ref_controller = RefController(args.refs_path,target_name=args.target_name,
-                                       inventory_path=args.inventory_path)
+        ref_controller = RefController(args.refs_path)
 
         if args.write is not None:
             ref_write(args, ref_controller)
@@ -465,11 +467,13 @@ def ref_write(args, ref_controller):
                 raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name))
 
             parameter = kap_inv_params['secrets']['vaultkv']
+        if args.auth:
+            parameter['auth'] = args.auth
         if parameter.get('auth') is None:
             raise KapitanError("No Authentication type parameter specified. Specify it"
-                               " in parameters.kapitan.secrets.vaultkv.auth and use --target-name or -t")
+                               " in parameters.kapitan.secrets.vaultkv.auth and use --target-name or use --auth")
 
-        secret_obj = VaultSecret(_data, parameter=parameter, encoding=encoding)
+        secret_obj = VaultSecret(_data, vault_client_param=parameter, encoding=encoding)
         tag = '?{{vaultkv:{}}}'.format(token_path)
         ref_controller[tag] = secret_obj
 

kapitan/refs/base.py

@@ -355,15 +355,13 @@ def __init__(self, content, content_type):
 
 
 class RefController(object):
-    def __init__(self, path,target_name = '',inventory_path = './inventory'):
+    def __init__(self, path):
         """
         gets and sets tags for ref type objects.
         auto registers backends
         """
         self.backends = {}
         self.path = path
-        self.target_name = target_name
-        self.inventory_path = inventory_path
 
     def register_backend(self, backend):
         "register backend type"
@@ -392,8 +390,7 @@ def _get_backend(self, type_name):
                 self.register_backend(AWSKMSBackend(self.path))
             elif type_name == 'vaultkv':
                 from kapitan.refs.secrets.vaultkv import VaultBackend
-                self.register_backend(VaultBackend(self.path,target_name=self.target_name,
-                                                   inventory_path=self.inventory_path))
+                self.register_backend(VaultBackend(self.path))
             else:
                 raise RefBackendError('no backend for ref type: {}'.format(type_name))
         return self.backends[type_name]

kapitan/refs/secrets/vaultkv.py

@@ -161,7 +161,7 @@ def __init__(self, data, **kwargs):
         Set vault parameter and encoding of data
         """
         self.data = data
-        self.parameter = kwargs.get('parameter')
+        self.vault_client_param = kwargs.get('vault_client_param')
         super().__init__(self.data,**kwargs)
         self.type_name = 'vaultkv'
 
@@ -180,37 +180,14 @@ def from_params(cls, data, ref_params):
             if target_inv is None:
                 raise ValueError('target_inv not set')
 
-            ref_params.kwargs['parameter'] = target_inv['parameters']['kapitan']['secrets']['vaultkv']
+            ref_params.kwargs['vault_client_param'] = target_inv['parameters']['kapitan']['secrets']['vaultkv']
             return cls(data, **ref_params.kwargs)
         except KeyError:
             raise RefError("Could not create VaultSecret: vaultkv parameters missing")
 
     @classmethod
     def from_path(cls, ref_full_path, **kwargs):
-        """
-        return a new Ref from file at ref_full_path
-        the data key in the file must be base64 encoded
-        """
-        try:
-            with open(ref_full_path) as fp:
-                obj = yaml.load(fp, Loader=YamlLoader)
-                _kwargs = {key: value for key, value in obj.items() if key not in ('data', 'from_base64')}
-                inv = inventory_reclass(kwargs['inventory_path'])
-                target_inv = inv['nodes'][kwargs['target_name']]
-                if target_inv is None:
-                    raise ValueError('target not set')
-
-                _kwargs['parameter'] = target_inv['parameters']['kapitan']['secrets']['vaultkv']
-                kwargs.update(_kwargs)
-                return cls(obj['data'], from_base64=True, **kwargs)
-
-        except IOError as ex:
-            if ex.errno == errno.ENOENT:
-                logger.debug("no such file or directory {}".format(kwargs['inventory_path']))
-                return None
-
-        except KeyError:
-            raise VaultError("Could not fetch VaultSecret: vaultkv parameters missing")
+        return super().from_path(ref_full_path, encrypt=False)
 
     def reveal(self):
         """
@@ -222,9 +199,7 @@ def reveal(self):
                 self.data = base64.b64decode(self.data, validate=True)
 
         except b_error:
-            exit(
-                "non-alphabet characters in the data"
-            )
+            exit("non-alphabet characters in the data")
 
         return self._decrypt()
 
@@ -235,17 +210,17 @@ def _decrypt(self):
         :returns: secret in plain text
         """
         try:
-            client = vault_obj(self.parameter)
+            client = vault_obj(self.vault_client_param)
             # token will comprise of two parts path_in_vault:key
             data = self.data.rstrip().decode('utf-8').split(':')
             return_data = ''
-            if self.parameter.get('engine') == 'kv':
+            if self.vault_client_param.get('engine') == 'kv':
                 response = client.secrets.kv.v1.read_secret(path=data[0],
-                                                            mount_point=self.parameter.get('mount','secret'))
+                                                            mount_point=self.vault_client_param.get('mount','secret'))
                 return_data = response['data'][data[1]]
             else:
                 response = client.secrets.kv.v2.read_secret_version(path=data[0],
-                                                                    mount_point=self.parameter.get('mount','secret'))
+                                                                    mount_point=self.vault_client_param.get('mount','secret'))
                 return_data = response['data']['data'][data[1]]
             client.adapter.close()
         except Forbidden:
@@ -269,26 +244,10 @@ def dump(self):
         Returns dict with keys/values to be serialised.
         """
         return {"data": self.data, "encoding": self.encoding,
-                "type": self.type_name}
+                "type": self.type_name, "vault_client_param": self.vault_client_param}
 
 class VaultBackend(Base64RefBackend):
-    def __init__(self, path, target_name, inventory_path, ref_type=VaultSecret):
+    def __init__(self, path, ref_type=VaultSecret):
         "init VaultBackend ref backend type"
         super().__init__(path, ref_type)
         self.type_name = 'vaultkv'
-        self.target_name = target_name
-        self.inventory_path = inventory_path
-
-    def __getitem__(self, ref_path):
-        full_ref_path = os.path.join(self.path, ref_path)
-        ref = self.ref_type.from_path(full_ref_path, target_name=self.target_name,
-                                      inventory_path=self.inventory_path)
-
-        if ref:
-            ref.path = ref_path
-            ref_path_data = "{}{}".format(ref.path, ref.data)
-            ref.hash = hashlib.sha256(ref_path_data.encode()).hexdigest()
-            ref.token = "{}:{}:{}".format(ref.type_name, ref.path, ref.hash[:8])
-            return ref
-
-        raise KeyError(ref_path)

kapitan/targets.py

@@ -314,7 +314,6 @@ def compile_target(target_obj, search_paths, compile_path, ref_controller, **kwa
     compile_objs = target_obj["compile"]
     ext_vars = target_obj["vars"]
     target_name = ext_vars["target"]
-    ref_controller.target_name = target_name
 
     jinja2_compiler = Jinja2(compile_path, search_paths, ref_controller)
     jsonnet_compiler = Jsonnet(compile_path, search_paths, ref_controller)

tests/test_cli.py

@@ -396,9 +396,7 @@ def test_cli_secret_write_vault(self,mock_reveal):
             fp.write(test_secret_content)
 
         sys.argv = ["kapitan", "refs", "--write", "vaultkv:test_secret",
-                    "-f", test_secret_file, "--refs-path", REFS_PATH,
-                    "--inventory-path","./example/kubernetes/inventory",
-                    "--target-name","minikube-mysql"]
+                    "-f", test_secret_file, "--refs-path", REFS_PATH, "--auth", "token"]
         main()
 
         test_tag_content = "revealing: ?{vaultkv:test_secret}"
@@ -408,9 +406,7 @@ def test_cli_secret_write_vault(self,mock_reveal):
 
         mock_reveal.return_value = test_secret_content_value
         sys.argv = ["kapitan", "refs", "--reveal",
-                    "-f", test_tag_file, "--refs-path", REFS_PATH,
-                    "--inventory-path","./example/kubernetes/inventory",
-                    "--target-name","minikube-mysql"]
+                    "-f", test_tag_file, "--refs-path", REFS_PATH]
 
          # set stdout as string
         stdout = io.StringIO()
@@ -435,9 +431,7 @@ def test_cli_secret_write_base64_vault(self,mock_reveal):
             fp.write(test_secret_content_b64)
 
         sys.argv = ["kapitan", "refs", "--write", "vaultkv:test_secret","--base64",
-                    "-f", test_secret_file, "--refs-path", REFS_PATH,
-                    "--inventory-path","./example/kubernetes/inventory",
-                    "--target-name","minikube-mysql"]
+                    "-f", test_secret_file, "--refs-path", REFS_PATH, "--auth", "token"]
         main()
 
         test_tag_content = "revealing: ?{vaultkv:test_secret}"
@@ -447,9 +441,7 @@ def test_cli_secret_write_base64_vault(self,mock_reveal):
 
         mock_reveal.return_value = test_secret_content_value
         sys.argv = ["kapitan", "refs", "--reveal",
-                    "-f", test_tag_file, "--refs-path", REFS_PATH,
-                    "--inventory-path","./example/kubernetes/inventory",
-                    "--target-name","minikube-mysql"]
+                    "-f", test_tag_file, "--refs-path", REFS_PATH]
 
          # set stdout as string
         stdout = io.StringIO()

tests/test_vault.py

@@ -31,8 +31,7 @@
 
 # Create temporary folder
 REFS_HOME = tempfile.mkdtemp()
-REF_CONTROLLER = RefController(REFS_HOME,target_name='minikube-mysql',
-                               inventory_path=os.path.join('.','examples','kubernetes','inventory'))
+REF_CONTROLLER = RefController(REFS_HOME)
 REVEALER = Revealer(REF_CONTROLLER)
 # Create Vault docker container
 client = docker.from_env()
@@ -127,7 +126,7 @@ def test_vault_write_reveal(self):
         )
         env = {'auth':'token'}
         file_data = "foo:some_random_value".encode()
-        REF_CONTROLLER[tag] = VaultSecret(file_data,parameter=env,encoding='original')
+        REF_CONTROLLER[tag] = VaultSecret(file_data,vault_client_param=env,encoding='original')
         # confirming secret file exists
         self.assertTrue(os.path.isfile(os.path.join(REFS_HOME,'secret/batman')),
                         msg="Secret file doesn't exist")
@@ -154,7 +153,7 @@ def test_vault_base64_write_reveal(self):
         )
         env = {'auth':'token'}
         file_data = "foo:some_random_value".encode()
-        REF_CONTROLLER[tag] = VaultSecret(file_data,parameter=env,encoding='base64')
+        REF_CONTROLLER[tag] = VaultSecret(file_data,vault_client_param=env,encoding='base64')
         # confirming secret file exists
         self.assertTrue(os.path.isfile(os.path.join(REFS_HOME,'secret/batman')),
                         msg="Secret file doesn't exist")