diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 581201c7..b9c6c143 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,13 +65,12 @@ builderv2-github: PYTEST_ARGS: -v --color=yes --showlocals --cov qubesbuilder --cov-report term --cov-report html:artifacts/htmlcov --cov-report xml:artifacts/coverage.xml --junitxml=artifacts/qubesbuilder.xml # https://gitlab.com/gitlab-org/gitlab/-/issues/15603 before_script: - - sudo dnf install -y python3-pip python3-pytest-cov python3-pytest python3-pytest-mock python3-lxml python3-wheel sequoia-sqv git + - sudo dnf install -y python3-pip python3-pytest-cov python3-pytest python3-pytest-mock python3-lxml python3-wheel sequoia-sqv - sudo rm -rf ~/pytest-of-user/ - docker pull registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest - docker tag registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest qubes-builder-fedora:latest - mkdir ~/gitlab ~/tmp - - git clone https://github.com/QubesOS/qubes-core-admin-client ~/core-admin-client - - export PYTHONPATH=".:~/core-admin-client:$PYTHONPATH" BASE_ARTIFACTS_DIR=~/gitlab TMPDIR=~/tmp + - export PYTHONPATH=".:$PYTHONPATH" BASE_ARTIFACTS_DIR=~/gitlab TMPDIR=~/tmp - env after_script: - ci/codecov-wrapper -f artifacts/coverage.xml @@ -251,10 +250,8 @@ mypy: tags: - docker before_script: - - sudo dnf install -y python3-mypy python3-pip git + - sudo dnf install -y python3-mypy python3-pip - sudo python3 -m pip install types-PyYAML types-python-dateutil - - git clone https://github.com/QubesOS/qubes-core-admin-client ~/core-admin-client - - export PYTHONPATH="~/core-admin-client:$PYTHONPATH" script: - mypy --install-types --non-interactive --junit-xml mypy.xml qubesbuilder artifacts: diff --git a/qubesbuilder/executors/qrexec.py b/qubesbuilder/executors/qrexec.py new file mode 100755 index 00000000..d383a0df --- /dev/null +++ b/qubesbuilder/executors/qrexec.py @@ -0,0 +1,133 @@ +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2021 Frédéric Pierret (fepitre) +# Copyright (C) 2025 Rafał Wojdyła +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later +import re +import subprocess +from typing import List, Optional + +from qubesbuilder.common import sanitize_line +from qubesbuilder.executors import ExecutorError + + +def qrexec_call( + log, + what: str, + vm: str, + service: str, + args: Optional[List[str]] = None, + capture_output: bool = False, + options: Optional[List[str]] = None, + stdin: bytes = b"", + ignore_errors: bool = False, +) -> Optional[bytes]: + cmd = [ + "/usr/lib/qubes/qrexec-client-vm", + ] + + if options: + cmd += options + + cmd += [ + "--", + vm, + service, + ] + + if args: + cmd += args + + admin = service.startswith("admin.") + capture_output = capture_output or admin + try: + log.debug(f"qrexec call ({what}): {' '.join(cmd)}") + proc = subprocess.run( + cmd, + check=not ignore_errors, + capture_output=capture_output, + input=stdin, + ) + except subprocess.CalledProcessError as e: + if e.stderr is not None: + content = sanitize_line(e.stderr.rstrip(b"\n")).rstrip() + else: + content = str(e) + msg = f"Failed to {what}: {content}" + raise ExecutorError(msg, name=vm) + + if capture_output: + stdout = proc.stdout + if admin: + if not stdout.startswith(b"0\x00"): + stdout = stdout[2:].replace(b"\x00", b"\n") + + if not ignore_errors: + raise ExecutorError( + f"Failed to {what}: qrexec call failed: {stdout.decode("ascii", "strict")}" + ) + else: + log.debug( + f"Failed to {what}: qrexec call failed: {stdout.decode("ascii", "strict")}" + ) + stdout = stdout[2:] + return stdout + return None + + +def create_dispvm(log, template: str) -> str: + stdout = qrexec_call( + log=log, + what="create disposable qube", + vm=template, + service="admin.vm.CreateDisposable", + ) + + assert stdout + if not re.match(rb"\Adisp(0|[1-9][0-9]{0,8})\Z", stdout): + raise ExecutorError("Failed to create disposable qube.") + try: + return stdout.decode("ascii", "strict") + except UnicodeDecodeError as e: + raise ExecutorError(f"Failed to obtain disposable qube name: {str(e)}") + + +def start_vm(log, vm: str): + qrexec_call( + log=log, + what="start vm", + vm=vm, + service="admin.vm.Start", + ) + + +def kill_vm(log, vm: str): + qrexec_call( + log=log, + what="kill vm", + vm=vm, + service="admin.vm.Kill", + ignore_errors=True, + ) + + qrexec_call( + log=log, + what="remove vm", + vm=vm, + service="admin.vm.Remove", + ignore_errors=True, + ) diff --git a/qubesbuilder/executors/qubes.py b/qubesbuilder/executors/qubes.py index f5c7181c..892c8fbf 100644 --- a/qubesbuilder/executors/qubes.py +++ b/qubesbuilder/executors/qubes.py @@ -25,11 +25,14 @@ from time import sleep from typing import List, Optional, Tuple, Union -from qubesadmin.devices import DeviceAssignment, UnknownDevice -from qubesadmin.exc import DeviceAlreadyAttached, QubesException -from qubesadmin.vm import QubesVM from qubesbuilder.common import sanitize_line, PROJECT_PATH from qubesbuilder.executors import Executor, ExecutorError +from qubesbuilder.executors.qrexec import ( + create_dispvm, + kill_vm, + qrexec_call, + start_vm, +) from qubesbuilder.executors.windows import BaseWindowsExecutor @@ -70,6 +73,7 @@ def __init__(self, dispvm: str = "dom0", **kwargs): self._dispvm_template = "dom0" else: self._dispvm_template = dispvm + self.name = os.uname().nodename self.dispvm: Optional[str] = None # actual dispvm name self.copy_in_service = "qubesbuilder.FileCopyIn" self.copy_out_service = "qubesbuilder.FileCopyOut" @@ -85,24 +89,13 @@ def copy_in(self, source_path: Path, destination_dir: PurePath): # type: ignore src = source_path.expanduser().resolve() dst = destination_dir encoded_dst_path = encode_for_vmexec(str((dst / src.name).as_posix())) - copy_in_cmd = [ - "/usr/lib/qubes/qrexec-client-vm", - "--", - self.dispvm, - f"{self.copy_in_service}+{encoded_dst_path}", - "/usr/lib/qubes/qfile-agent", - str(src), - ] - try: - self.log.debug(f"copy-in (cmd): {' '.join(copy_in_cmd)}") - subprocess.run(copy_in_cmd, check=True) - except subprocess.CalledProcessError as e: - if e.stderr is not None: - content = sanitize_line(e.stderr.rstrip(b"\n")).rstrip() - else: - content = str(e) - msg = f"Failed to copy-in: {content}" - raise ExecutorError(msg, name=self.dispvm) + qrexec_call( + log=self.log, + what="copy-in", + vm=self.dispvm, + service=f"{self.copy_in_service}+{encoded_dst_path}", + args=["/usr/lib/qubes/qfile-agent", str(src)], + ) def copy_out( self, @@ -131,96 +124,53 @@ def copy_out( else: unpacker_path = old_unpacker_path encoded_src_path = encode_for_vmexec(str(src)) - cmd = [ - "/usr/lib/qubes/qrexec-client-vm", - self.dispvm, - f"{self.copy_out_service}+{encoded_src_path}", - unpacker_path, - str(os.getuid()), - str(dst), - ] - try: - self.log.debug(f"copy-out (cmd): {' '.join(cmd)}") - subprocess.run(cmd, check=True) + qrexec_call( + log=self.log, + what="copy-out", + vm=self.dispvm, + service=f"{self.copy_out_service}+{encoded_src_path}", + args=[ + unpacker_path, + str(os.getuid()), + str(dst), + ], + ) - if dig_holes and not dst_path.is_dir(): + if dig_holes and not dst_path.is_dir(): + try: self.log.debug( "copy-out (detect zeroes and replace with holes)" ) subprocess.run( ["/usr/bin/fallocate", "-d", str(dst_path)], check=True ) - except subprocess.CalledProcessError as e: - if e.stderr is not None: - content = sanitize_line(e.stderr.rstrip(b"\n")).rstrip() - else: - content = str(e) - msg = f"Failed to copy-out: {content}" - raise ExecutorError(msg, name=self.dispvm) - - def create_dispvm(self): - result = subprocess.run( - [ - "qrexec-client-vm", - "--", - self._dispvm_template, - "admin.vm.CreateDisposable", - ], - capture_output=True, - stdin=subprocess.DEVNULL, - ) - stdout = result.stdout - if not stdout.startswith(b"0\x00"): - raise ExecutorError("Failed to create disposable qube.") - stdout = stdout[2:] - if not re.match(rb"\Adisp(0|[1-9][0-9]{0,8})\Z", stdout): - raise ExecutorError("Failed to create disposable qube.") - try: - self.dispvm = stdout.decode("ascii", "strict") - except UnicodeDecodeError as e: - raise ExecutorError( - f"Failed to obtain disposable qube name: {str(e)}" - ) - - def start_dispvm(self): - assert self.dispvm - subprocess.run( - [ - "/usr/lib/qubes/qrexec-client-vm", - self.dispvm, - "admin.vm.Start", - ], - stdin=subprocess.DEVNULL, - capture_output=True, - check=True, - ) + except subprocess.CalledProcessError as e: + if e.stderr is not None: + content = sanitize_line(e.stderr.rstrip(b"\n")).rstrip() + else: + content = str(e) + msg = f"Failed to dig holes in copy-out: {content}" + raise ExecutorError(msg, name=self.dispvm) def copy_rpc_services(self): assert self.dispvm - copy_rpc_cmd = [ - "/usr/lib/qubes/qrexec-client-vm", - "--filter-escape-chars-stderr", - self.dispvm, - "qubes.Filecopy", - "/usr/lib/qubes/qfile-agent", - str(PROJECT_PATH / "rpc" / self.copy_in_service), - str(PROJECT_PATH / "rpc" / self.copy_out_service), - ] - subprocess.run( - copy_rpc_cmd, - stdin=subprocess.DEVNULL, - capture_output=True, - check=True, + qrexec_call( + log=self.log, + what="copy builder rpc services", + vm=self.dispvm, + service="qubes.Filecopy", + args=[ + "/usr/lib/qubes/qfile-agent", + str(PROJECT_PATH / "rpc" / self.copy_in_service), + str(PROJECT_PATH / "rpc" / self.copy_out_service), + ], + options=["--filter-escape-chars-stderr"], ) - @staticmethod - def cleanup(dispvm: str): + def cleanup(self): # Kill the DispVM to prevent hanging for while - subprocess.run( - ["qrexec-client-vm", "--", dispvm, "admin.vm.Kill"], - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - ) + assert self.dispvm + kill_vm(self.log, self.dispvm) class LinuxQubesExecutor(QubesExecutor): @@ -240,8 +190,8 @@ def run( # type: ignore dig_holes: bool = False, ): try: - self.create_dispvm() - self.start_dispvm() + self.dispvm = create_dispvm(self.log, self._dispvm_template) + start_vm(self.log, self.dispvm) self.copy_rpc_services() assert self.dispvm @@ -264,8 +214,8 @@ def run( # type: ignore "mv", "-f", "--", - f"/home/{self.get_user()}/QubesIncoming/{os.uname().nodename}/qubesbuilder.FileCopyIn", - f"/home/{self.get_user()}/QubesIncoming/{os.uname().nodename}/qubesbuilder.FileCopyOut", + f"/home/{self.get_user()}/QubesIncoming/{self.name}/qubesbuilder.FileCopyIn", + f"/home/{self.get_user()}/QubesIncoming/{self.name}/qubesbuilder.FileCopyOut", "/usr/local/etc/qubes-rpc/", ], [ @@ -388,11 +338,11 @@ def run( # type: ignore raise e except (subprocess.CalledProcessError, ExecutorError) as e: if self.dispvm and self._clean_on_error: - self.cleanup(self.dispvm) + self.cleanup() raise e else: if self.dispvm and self._clean: - self.cleanup(self.dispvm) + self.cleanup() class WindowsQubesExecutor(BaseWindowsExecutor, QubesExecutor): @@ -431,7 +381,7 @@ def _get_ewdk_loop(self) -> Optional[str]: f"Failed to run losetup: {proc.stderr.decode()}" ) from e - def _get_ewdk_assignment(self) -> DeviceAssignment: + def attach_ewdk(self): if not Path(self.ewdk_path).is_file(): raise ExecutorError(f"EWDK image not found at '{self.ewdk_path}'") @@ -458,54 +408,55 @@ def _get_ewdk_assignment(self) -> DeviceAssignment: self.log.debug(f"attached EWDK as '{loop_id}'") # wait for device to appear - self_vm = QubesVM(self.app, self.app.local_name) timeout = 10 - while isinstance(self_vm.devices["block"][loop_id], UnknownDevice): - if timeout == 0: - raise ExecutorError( - f"Failed to attach EWDK ({self.ewdk_path}): " - f"wait for loopback device timed out" + while timeout > 0: + stdout = qrexec_call( + log=self.log, + what="ewdk loop device query", + vm=self.name, + service=f"admin.vm.device.block.Available+{loop_id}", + ) + + assert stdout is not None + assert self.dispvm + if stdout.decode("ascii", "strict").startswith(loop_id): + # loop device ready, attach to vm + qrexec_call( + log=self.log, + what="attach ewdk to dispvm", + vm=self.dispvm, + service=f"admin.vm.device.block.Attach+{self.name}+{loop_id}", + stdin=b"devtype=cdrom read-only=true persistent=true", ) + return + timeout -= 1 sleep(1) - return DeviceAssignment( - backend_domain=self.app.local_name, - ident=loop_id, - devclass="block", - options={ - "devtype": "cdrom", - "read-only": True, - }, - persistent=True, + raise ExecutorError( + f"Failed to attach EWDK ({self.ewdk_path}): " + f"wait for loopback device timed out" ) - def start_worker(self) -> QubesVM: + def start_worker(self): # we need the dispvm in a stopped state to attach EWDK block device - self.create_dispvm() - vm = QubesVM(self.app, self.dispvm) - self.log.debug(f"dispvm: {vm.name}") - ewdk_assignment = self._get_ewdk_assignment() - try: - vm.devices["block"].attach(ewdk_assignment) - except DeviceAlreadyAttached: - pass - except Exception as e: - msg = f"Failed to assign EWDK iso image to '{self.dispvm}'" - raise ExecutorError(msg, name=self.dispvm) from e - - vm.start() + self.dispvm = create_dispvm(self.log, self._dispvm_template) + self.log.debug(f"dispvm: {self.dispvm}") + self.attach_ewdk() + start_vm(self.log, self.dispvm) # wait for startup for _ in range(10): try: - proc = vm.run_service("qubes.VMShell") - proc.communicate(b"exit 0") - if proc.returncode == 0: - return vm # all good - else: - self.log.debug(f"VMShell failed: {proc.returncode}") - except QubesException as e: + qrexec_call( + log=self.log, + what="dispvm qrexec test", + vm=self.dispvm, + service="qubes.VMShell", + stdin=b"exit 0", + ) + return # all good + except ExecutorError as e: self.log.debug(f"VMShell failed: {e}") sleep(5) raise ExecutorError( @@ -520,26 +471,28 @@ def run( ): try: # copy the rpc handlers + self.start_worker() # TODO: don't require two scripts per service - files = [ - str(PROJECT_PATH / "rpc" / "qubesbuilder.WinFileCopyIn"), - str(PROJECT_PATH / "rpc" / "qubesbuilder.WinFileCopyOut"), - str(PROJECT_PATH / "rpc" / "qubesbuilder-file-copy-in.ps1"), - str(PROJECT_PATH / "rpc" / "qubesbuilder-file-copy-out.ps1"), - ] - - vm = self.start_worker() - proc = vm.run_service( - "qubes.Filecopy", - localcmd=f"/usr/lib/qubes/qfile-agent {' '.join(files)}", + assert self.dispvm + qrexec_call( + log=self.log, + what="copy RPC services to dispvm", + vm=self.dispvm, + service="qubes.Filecopy", + args=[ + "/usr/lib/qubes/qfile-agent", + str(PROJECT_PATH / "rpc" / "qubesbuilder.WinFileCopyIn"), + str(PROJECT_PATH / "rpc" / "qubesbuilder.WinFileCopyOut"), + str(PROJECT_PATH / "rpc" / "qubesbuilder-file-copy-in.ps1"), + str( + PROJECT_PATH / "rpc" / "qubesbuilder-file-copy-out.ps1" + ), + ], ) - _, stderr = proc.communicate() - if proc.returncode != 0: - raise ExecutorError( - f"Failed to copy builder RPC services to qube '{self.dispvm}': {stderr.decode('utf-8')}" - ) - inc_dir = f"c:\\users\\{self.user}\\Documents\\QubesIncoming\\{os.uname().nodename}" + inc_dir = ( + f"c:\\users\\{self.user}\\Documents\\QubesIncoming\\{self.name}" + ) prep_cmd = [ f'move /y "{inc_dir}\\qubesbuilder.WinFileCopyIn" "%QUBES_TOOLS%\\qubes-rpc\\"', @@ -548,16 +501,15 @@ def run( f'move /y "{inc_dir}\\qubesbuilder-file-copy-out.ps1" "%QUBES_TOOLS%\\qubes-rpc-services\\"', ] - proc = vm.run_service("qubes.VMShell") - _, stderr = proc.communicate( - (" & ".join(prep_cmd) + " & exit !errorlevel!" + "\r\n").encode( - "utf-8" - ) + qrexec_call( + log=self.log, + what="prepare RPC services in dispvm", + vm=self.dispvm, + service="qubes.VMShell", + stdin=( + " & ".join(prep_cmd) + " & exit !errorlevel!" + "\r\n" + ).encode("utf-8"), ) - if proc.returncode != 0: - raise ExecutorError( - f"Failed to copy builder RPC services to qube '{self.dispvm}': {stderr}" - ) for src_in, dst_in in copy_in or []: self.copy_in(src_in, dst_in) @@ -568,19 +520,24 @@ def run( self.log.debug(f"{bin_cmd=}") # TODO: this doesn't stream stdout, use execute() - proc = vm.run_service("qubes.VMShell") - stdout, stderr = proc.communicate(bin_cmd) - if proc.returncode != 0: - raise ExecutorError( - f"Failed to run command in qube {self.dispvm}: {stderr.decode('utf-8')}" - ) + stdout = qrexec_call( + log=self.log, + what="run command in dispvm", + vm=self.dispvm, + service="qubes.VMShell", + stdin=bin_cmd, + capture_output=True, + ) + assert stdout is not None self.log.debug(stdout.decode("utf-8")) for src_out, dst_out in copy_out or []: self.copy_out(src_out, dst_out) - except QubesException as e: + except ExecutorError as e: suffix = f" in qube {self.dispvm}" if self.dispvm else "" - raise ExecutorError(f"Failed to run command{suffix}: {str(e)}") + raise ExecutorError( + f"Failed to run command{suffix}: {str(e)}" + ) from e finally: if self.dispvm: - self.kill_vm(self.dispvm) + kill_vm(self.log, self.dispvm) diff --git a/qubesbuilder/executors/windows.py b/qubesbuilder/executors/windows.py index fddfa5cf..ccaf62dc 100644 --- a/qubesbuilder/executors/windows.py +++ b/qubesbuilder/executors/windows.py @@ -23,18 +23,20 @@ from pathlib import Path, PurePath, PureWindowsPath from typing import List, Tuple -from qubesadmin import Qubes -from qubesadmin.exc import QubesException -from qubesadmin.vm import DispVM, QubesVM from qubesbuilder.common import sanitize_line from qubesbuilder.executors import Executor, ExecutorError +from qubesbuilder.executors.qrexec import ( + create_dispvm, + kill_vm as qkill_vm, + qrexec_call, + start_vm, +) class BaseWindowsExecutor(Executor, ABC): def __init__(self, user: str = "user", **kwargs): super().__init__(**kwargs) self.user = user - self.app = Qubes() def get_builder_dir(self): return PureWindowsPath("c:\\builder") @@ -47,19 +49,13 @@ def get_threads(self) -> int: # starts dispvm from default template (not windows) def start_dispvm(self) -> str: - dvm = DispVM.from_appvm(self.app, None) - dvm.start() - self.log.debug(f"created dispvm {dvm.name}") - return dvm.name - - def kill_vm(self, vm_name: str): - vm = QubesVM(self.app, vm_name) - if vm: - # Don't check is_running() since that depends on qrexec connectivity - if vm.is_halted(): - del self.app.domains[vm_name] - else: - vm.kill() + name = create_dispvm(self.log, "dom0") + self.log.debug(f"created dispvm {name}") + start_vm(self.log, name) + return name + + def kill_vm(self, vm: str): + qkill_vm(self.log, vm) def run_rpc_service( self, @@ -67,24 +63,17 @@ def run_rpc_service( service: str, description: str, stdin: bytes = b"", - check_return: bool = True, ) -> bytes: - try: - proc = self.app.run_service( - target, - service, - ) - - stdout, stderr = proc.communicate(stdin) - if check_return and proc.returncode != 0: - msg = f"Failed to {description}.\n" - msg += stderr.decode("utf-8") - raise ExecutorError(msg, name=target) - except QubesException as e: - msg = f"Failed to {description}: failed to run service '{service}' in qube '{target}'." - raise ExecutorError(msg, name=target) from e - - return stdout + out = qrexec_call( + log=self.log, + what=description, + vm=target, + service=service, + capture_output=True, + stdin=stdin, + ) + assert out is not None + return out class SSHWindowsExecutor(BaseWindowsExecutor): diff --git a/qubesbuilder/plugins/build_windows/__init__.py b/qubesbuilder/plugins/build_windows/__init__.py index 2f78db8d..f265b270 100644 --- a/qubesbuilder/plugins/build_windows/__init__.py +++ b/qubesbuilder/plugins/build_windows/__init__.py @@ -198,7 +198,6 @@ def sign_prep( test_sign: bool, ) -> bytes: if test_sign: - self.log.debug(f"creating key '{key_name}' in qube '{qube}'") self.executor.run_rpc_service( target=qube, service=f"qubesbuilder.WinSign.CreateKey+{mangle_key_name(key_name)}", @@ -218,12 +217,11 @@ def sign_sign( key_name: str, file: Path, ) -> bytes: - self.log.debug(f"signing '{file}' with '{key_name}'") with open(file, "rb") as f: return self.executor.run_rpc_service( target=qube, service=f"qubesbuilder.WinSign.Sign+{mangle_key_name(key_name)}", - description=f"sign '{file}' with key '{key_name}'", + description=f"authenticode sign '{file}' with key '{key_name}'", stdin=f.read(), ) @@ -233,7 +231,6 @@ def sign_delete_key(self, qube: str, key_name: str): target=qube, service=f"qubesbuilder.WinSign.QueryKey+{mangle_key_name(key_name)}", description=f"query signing key '{key_name}'", - check_return=False, ) if f"Key '{mangle_key_name(key_name)}' exists" not in out.decode( @@ -242,7 +239,6 @@ def sign_delete_key(self, qube: str, key_name: str): self.log.debug(f"key '{key_name}' does not exist") return - self.log.debug(f"deleting key '{key_name}' in qube '{qube}'") self.executor.run_rpc_service( target=qube, service=f"qubesbuilder.WinSign.DeleteKey+{mangle_key_name(key_name)}", diff --git a/rpc/qubesbuilder.WinSign.QueryKey b/rpc/qubesbuilder.WinSign.QueryKey index 4681b11c..4b092dda 100755 --- a/rpc/qubesbuilder.WinSign.QueryKey +++ b/rpc/qubesbuilder.WinSign.QueryKey @@ -22,8 +22,6 @@ ensure_db set +e if check_key_exists "${1//__/ }"; then echo "Key '$1' exists" - exit 0 else echo "Key '$1' does not exist" - exit 2 fi