From 7917ef55f6279fa28519d2160c4ec49453c275ec Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Wed, 10 Apr 2019 14:09:58 +0530 Subject: [PATCH 1/3] targetclid: add daemonize component for targetcli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: ------- Overall creation time of a block using targetcli is raising linearly as the block count increase. This is because of the recurring issue involving refresh(reload) at multiple objects/places, as the LIO's configfs is deeply nested. Earlier discussion of the problem statement with stats and graphs about delays: http://bit.ly/targetcli-create-delay Solution: -------- Introduce a daemon component for targetcli[d] which will retain state of Configshell object in memory, so that any new requests can directly use it, instead of loading the storageObjects/targetObjects again. Details about "how to use it ?": ------------------------------- $ systemctl start targetclid $ systemctl status targetclid ● targetclid.service - Targetcli daemon Loaded: loaded (/usr/lib/systemd/system/targetclid.service; disabled; vendor preset: disabled) Active: active (running) since Wed 2019-04-10 12:19:51 IST; 2h 17min ago Main PID: 3950 (targetclid) Tasks: 3 (limit: 4915) CGroup: /system.slice/targetclid.service └─3950 /usr/bin/python /usr/bin/targetclid Apr 10 12:19:51 localhost.localdomain systemd[1]: Started Targetcli daemon. $ targetcli help Usage: /usr/bin/targetcli [--version|--help|CMD|--tcp] --version Print version --help Print this information CMD Run targetcli shell command and exit Enter configuration shell --tcp CMD Pass targetcli command to targetclid --tcp Enter multi-line command mode for targetclid See man page for more information. One line command usage: ---------------------- $ targetcli --tcp CMD Eg: $ targetcli --tcp pwd / Multiple line commands usage: ---------------------------- $ targetcli --tcp CMD1 CMD2 . . CMDN exit Eg: $ targetcli --tcp ^Tab / backstores/ iscsi/ loopback/ vhost/ xen-pvscsi/ cd clearconfig exit get help ls pwd refresh restoreconfig saveconfig set status pwd get global logfile get global auto_save_on_exit / saveconfig exit output follows: / logfile=/var/log/gluster-block/gluster-block-configshell.log auto_save_on_exit=false Configuration saved to /etc/target/saveconfig.json Stats with and without changes: ------------------------------ Running simple 'pwd' command after creating 1000 blocks on a node: Without this change: $ time targetcli pwd / real 0m8.963s user 0m7.775s sys 0m1.103s with daemonize changes: $ time targetcli --tcp "pwd" / real 0m0.126s user 0m0.099s sys 0m0.024s Thanks to Maurizio for hangingout with me for all the discussions involved. Signed-off-by: Prasanna Kumar Kalever --- daemon/targetclid | 299 +++++++++++++++++++++++++++++++++++++ scripts/targetcli | 102 ++++++++++++- setup.py | 5 +- systemd/targetclid.service | 10 ++ 4 files changed, 407 insertions(+), 9 deletions(-) create mode 100755 daemon/targetclid create mode 100644 systemd/targetclid.service diff --git a/daemon/targetclid b/daemon/targetclid new file mode 100755 index 0000000..b909c6e --- /dev/null +++ b/daemon/targetclid @@ -0,0 +1,299 @@ +#!/usr/bin/python + +''' +targetclid + +This file is part of targetcli-fb. +Copyright (c) 2019 by Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +''' + +from __future__ import print_function +from targetcli import UIRoot +from targetcli import __version__ as targetcli_version +from configshell_fb import ConfigShell +from os import getuid, getenv, unlink +from threading import Thread + +import sys +import socket +import struct +import fcntl +import signal + + +err = sys.stderr + +class TargetCLI: + def __init__(self): + ''' + initializer + ''' + # socket for unix communication + self.socket_path = '/var/run/targetclid.sock' + # pid file for defending on multiple daemon runs + self.pid_file = '/var/run/targetclid.pid' + # lockfile for serializing multiple client requests + self.lock_file = '/var/run/targetclid.lock' + + self.NoSignal = True + + # shell console methods + self.shell = ConfigShell(getenv("TARGETCLI_HOME", '~/.targetcli')) + self.con = self.shell.con + self.display = self.shell.con.display + self.render = self.shell.con.render_text + + # Handle SIGINT SIGTERM SIGHUP gracefully + signal.signal(signal.SIGINT, self.signal_handler) + signal.signal(signal.SIGTERM, self.signal_handler) + signal.signal(signal.SIGHUP, self.signal_handler) + + try: + self.pfd = open(self.pid_file, 'w+'); + except IOError as e: + self.display( + self.render( + "opening pidfile failed: %s" %str(e), + 'red')) + sys.exit(1) + + self.try_pidfile_lock() + + is_root = False + if getuid() == 0: + is_root = True + + try: + root_node = UIRoot(self.shell, as_root=is_root) + root_node.refresh() + except Exception as error: + self.display(self.render(str(error), 'red')) + if not is_root: + self.display(self.render("Retry as root.", 'red')) + self.pfd.close() + sys.exit(1) + + try: + self.lkfd = open(self.lock_file, 'w+'); + except IOError as e: + self.display( + self.render( + "opening lockfile failed: %s" %str(e), + 'red')) + self.pfd.close() + sys.exit(1) + + # Keep track, for later use + self.con_stdout_ = self.con._stdout + self.con_stderr_ = self.con._stderr + + + def __del__(self): + ''' + destructor + ''' + if not self.lkfd.closed: + self.lkfd.close() + + if not self.pfd.closed: + self.pfd.close() + + + def signal_handler(self, signum, frame): + ''' + signal handler + ''' + self.NoSignal = False + + + def try_pidfile_lock(self): + ''' + get lock on pidfile, which is to check if targetclid is running + ''' + # check if targetclid is already running + lock = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0) + try: + fcntl.fcntl(self.pfd, fcntl.F_SETLK, lock) + except Exception: + self.display(self.render("targetclid is already running...", 'red')) + self.pfd.close() + sys.exit(1) + + + def release_pidfile_lock(self): + ''' + release lock on pidfile + ''' + lock = struct.pack('hhllhh', fcntl.F_UNLCK, 0, 0, 0, 0, 0) + try: + fcntl.fcntl(self.pfd, fcntl.F_SETLK, lock) + except Exception, e: + self.display( + self.render( + "fcntl(UNLCK) on pidfile failed: %s" %str(e), + 'red')) + self.pfd.close() + sys.exit(1) + self.pfd.close() + + + def try_op_lock(self): + ''' + acquire a blocking lock on lockfile, to serialize multiple client requests + ''' + try: + fcntl.flock(self.lkfd, fcntl.LOCK_EX) # wait here until ongoing request is finished + except Exception, e: + self.display( + self.render( + "taking lock on lockfile failed: %s" %str(e), + 'red')) + sys.exit(1) + + + def release_op_lock(self): + ''' + release blocking lock on lockfile, which can allow other requests process + ''' + try: + fcntl.flock(self.lkfd, fcntl.LOCK_UN) # allow other requests now + except Exception, e: + self.display( + self.render( + "unlock on lockfile failed: %s" %str(e), + 'red')) + sys.exit(1) + + + def client_thread(self, connection): + ''' + Handle commands from client + ''' + self.try_op_lock() + + still_listen = True + # Receive the data in small chunks and retransmit it + while still_listen: + data = connection.recv(65535) + if "-END@OF@DATA-" in data: + connection.close() + still_listen = False + else: + self.con._stdout = self.con._stderr = f = open("/tmp/data.txt", "w") + try: + # extract multiple commands delimited with '%' + list_data = data.split('%') + for cmd in list_data: + self.shell.run_cmdline(cmd) + except Exception as e: + print(str(e), file=f) # push error to stream + + # Restore + self.con._stdout = self.con_stdout_ + self.con._stderr = self.con_stderr_ + f.close() + + with open('/tmp/data.txt', 'r') as f: + output = f.read() + + var = struct.pack('i', len(output)) + connection.sendall(var) # length of string + connection.sendall(output) # actual string + + self.release_op_lock() + + +def usage(): + print("Usage: %s [--version|--help]" % sys.argv[0], file=err) + print(" --version\t\tPrint version", file=err) + print(" --help\t\tPrint this information", file=err) + sys.exit(0) + + +def version(): + print("%s version %s" % (sys.argv[0], targetcli_version), file=err) + sys.exit(0) + + +def usage_version(cmd): + if cmd in ("help", "--help", "-h"): + usage() + + if cmd in ("version", "--version", "-v"): + version() + + +def main(): + ''' + start targetclid + ''' + if len(sys.argv) > 1: + usage_version(sys.argv[1]) + print("unrecognized option: %s" % (sys.argv[1])) + sys.exit(-1) + + to = TargetCLI() + + # Make sure file doesn't exist already + try: + unlink(to.socket_path) + except: + pass + + # Create a TCP/IP socket + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + except socket.error as err: + to.display(to.render(err, 'red')) + sys.exit(1) + + # Bind the socket path + try: + sock.bind(to.socket_path) + except socket.error as err: + to.display(to.render(err, 'red')) + sys.exit(1) + + # Listen for incoming connections + try: + sock.listen(1) + except socket.error as err: + to.display(to.render(err, 'red')) + sys.exit(1) + + while to.NoSignal: + try: + # Wait for a connection + connection, client_address = sock.accept() + except socket.error as err: + to.display(to.render(err, 'red')) + break; + + thread = Thread(target=to.client_thread, args=(connection,)) + thread.start() + try: + thread.join() + except: + to.display(to.render(str(error), 'red')) + + to.release_pidfile_lock() + + if not to.NoSignal: + to.display(to.render("Signal received, quiting gracefully!", 'green')) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/targetcli b/scripts/targetcli index b042ad9..92d3378 100755 --- a/scripts/targetcli +++ b/scripts/targetcli @@ -24,10 +24,19 @@ from os import getuid, getenv from targetcli import UIRoot from rtslib_fb import RTSLibError from configshell_fb import ConfigShell, ExecutionError -import sys from targetcli import __version__ as targetcli_version +import sys +import socket +import struct +import readline +import six + err = sys.stderr +socket_path = '/var/run/targetclid.sock' +hints = ['/', 'backstores/', 'iscsi/', 'loopback/', 'vhost/', 'xen-pvscsi/', + 'cd', 'pwd', 'ls', 'set', 'get', 'help', 'refresh', 'status', + 'clearconfig', 'restoreconfig', 'saveconfig', 'exit'] class TargetCLI(ConfigShell): default_prefs = {'color_path': 'magenta', @@ -54,11 +63,13 @@ class TargetCLI(ConfigShell): } def usage(): - print("Usage: %s [--version|--help|CMD]" % sys.argv[0], file=err) + print("Usage: %s [--version|--help|CMD|--tcp]" % sys.argv[0], file=err) print(" --version\t\tPrint version", file=err) print(" --help\t\tPrint this information", file=err) print(" CMD\t\t\tRun targetcli shell command and exit", file=err) print(" \t\tEnter configuration shell", file=err) + print(" --tcp CMD\t\tPass targetcli command to targetclid", file=err) + print(" --tcp \tEnter multi-line command mode for targetclid", file=err) print("See man page for more information.", file=err) sys.exit(-1) @@ -66,6 +77,76 @@ def version(): print("%s version %s" % (sys.argv[0], targetcli_version), file=err) sys.exit(0) +def usage_version(cmd): + if cmd in ("help", "--help", "-h"): + usage() + + if cmd in ("version", "--version", "-v"): + version() + +def completer(text, state): + options = [x for x in hints if x.startswith(text)] + try: + return options[state] + except IndexError: + return None + +def call_daemon(shell, req): + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + except socket.error as err: + shell.con.display(shell.con.render_text(err, 'red')) + sys.exit(1) + + try: + sock.connect(socket_path) + except socket.error as err: + shell.con.display(shell.con.render_text(err, 'red')) + sys.exit(1) + + try: + # send request + sock.sendall(req) + except socket.error as err: + shell.con.display(shell.con.render_text(err, 'red')) + sys.exit(1) + + var = sock.recv(4) # get length of data + sending = struct.unpack('i', var) + amount_expected = sending[0] + amount_received = 0 + + # get the actual data in chunks + while amount_received < amount_expected: + data = sock.recv(1024) + amount_received += len(data) + print(data, end ="") + + sock.send(b'-END@OF@DATA-') + sock.close() + sys.exit(0) + +def get_arguments(): + readline.set_completer(completer) + readline.set_completer_delims('') + + if 'libedit' in readline.__doc__: + readline.parse_and_bind("bind ^I rl_complete") + else: + readline.parse_and_bind("tab: complete") + + if len(sys.argv[1:]) > 1: + command = " ".join(sys.argv[2:]) + else: + inputs = [] + while True: + command = six.moves.input() + if command.lower() == "exit": + break + inputs.append(command) + command = '%'.join(inputs) # delimit multiple commands with '%' + return command + def main(): ''' Start the targetcli shell. @@ -77,6 +158,17 @@ def main(): shell = TargetCLI(getenv("TARGETCLI_HOME", '~/.targetcli')) + if len(sys.argv) > 1: + usage_version(sys.argv[1]) + if sys.argv[1] in ("tcp", "--tcp", "-t"): + if len(sys.argv) > 2: + usage_version(sys.argv[2]) + args = get_arguments() + if not args: + sys.exit(1) + usage_version(args); + call_daemon(shell, args) + try: root_node = UIRoot(shell, as_root=is_root) root_node.refresh() @@ -87,12 +179,6 @@ def main(): sys.exit(-1) if len(sys.argv) > 1: - if sys.argv[1] in ("--help", "-h"): - usage() - - if sys.argv[1] in ("--version", "-v"): - version() - try: shell.run_cmdline(" ".join(sys.argv[1:])) except Exception as e: diff --git a/setup.py b/setup.py index 7b44304..8dff55e 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,10 @@ maintainer_email = 'agrover@redhat.com', url = 'http://github.com/open-iscsi/targetcli-fb', packages = ['targetcli'], - scripts = ['scripts/targetcli'], + scripts = [ + 'scripts/targetcli', + 'daemon/targetclid' + ], classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", diff --git a/systemd/targetclid.service b/systemd/targetclid.service new file mode 100644 index 0000000..2de9100 --- /dev/null +++ b/systemd/targetclid.service @@ -0,0 +1,10 @@ +[Unit] +Description=Targetcli daemon + +[Service] +Type=simple +ExecStart=/usr/bin/targetclid +Restart=on-failure + +[Install] +WantedBy=multi-user.target From ad37f94ae72d0e3d5963ce182e2897c84af9c039 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 11 Apr 2019 14:31:27 +0530 Subject: [PATCH 2/3] targetclid: enable socket based activation Signed-off-by: Prasanna Kumar Kalever --- systemd/targetclid.service | 2 ++ systemd/targetclid.socket | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 systemd/targetclid.socket diff --git a/systemd/targetclid.service b/systemd/targetclid.service index 2de9100..7883998 100644 --- a/systemd/targetclid.service +++ b/systemd/targetclid.service @@ -1,5 +1,7 @@ [Unit] Description=Targetcli daemon +After=network.target targetclid.socket +Requires=targetclid.socket [Service] Type=simple diff --git a/systemd/targetclid.socket b/systemd/targetclid.socket new file mode 100644 index 0000000..809a7c8 --- /dev/null +++ b/systemd/targetclid.socket @@ -0,0 +1,9 @@ +[Unit] +Description=targetclid socket +PartOf=targetclid.service + +[Socket] +ListenStream=/var/run/targetclid.sock + +[Install] +WantedBy=sockets.target From 797778eeb2997d34af7fd3760195f7f8d08470f4 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 11 Apr 2019 15:12:14 +0530 Subject: [PATCH 3/3] targetcli: way to enable targetclid as default choice $ targetcli set global GLOBAL CONFIG GROUP [...] auto_use_daemon=bool If true, commands will be sent to targetclid. [...] $ targetcli set global auto_use_daemon=True Parameter auto_use_daemon is now 'true'. Signed-off-by: Prasanna Kumar Kalever --- scripts/targetcli | 35 +++++++++++++++++++++++------------ targetcli/ui_node.py | 3 +++ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/scripts/targetcli b/scripts/targetcli index 92d3378..66cb5bb 100755 --- a/scripts/targetcli +++ b/scripts/targetcli @@ -60,6 +60,7 @@ class TargetCLI(ConfigShell): 'auto_save_on_exit': True, 'max_backup_files': '10', 'auto_add_default_portal': True, + 'auto_use_daemon': False, } def usage(): @@ -135,8 +136,12 @@ def get_arguments(): else: readline.parse_and_bind("tab: complete") - if len(sys.argv[1:]) > 1: - command = " ".join(sys.argv[2:]) + argstart = 1 + if len(sys.argv) > 1 and sys.argv[1] in ("tcp", "--tcp", "-t"): + argstart = 2 + + if len(sys.argv[argstart - 1:]) > 1: + command = " ".join(sys.argv[argstart:]) else: inputs = [] while True: @@ -145,29 +150,35 @@ def get_arguments(): break inputs.append(command) command = '%'.join(inputs) # delimit multiple commands with '%' + + if not command: + sys.exit(1) + + usage_version(command); + return command def main(): ''' Start the targetcli shell. ''' + shell = TargetCLI(getenv("TARGETCLI_HOME", '~/.targetcli')) + + is_root = False if getuid() == 0: is_root = True - else: - is_root = False - shell = TargetCLI(getenv("TARGETCLI_HOME", '~/.targetcli')) + use_daemon = False + if shell.prefs['auto_use_daemon']: + use_daemon = True if len(sys.argv) > 1: usage_version(sys.argv[1]) if sys.argv[1] in ("tcp", "--tcp", "-t"): - if len(sys.argv) > 2: - usage_version(sys.argv[2]) - args = get_arguments() - if not args: - sys.exit(1) - usage_version(args); - call_daemon(shell, args) + use_daemon = True + + if use_daemon: + call_daemon(shell, get_arguments()) try: root_node = UIRoot(shell, as_root=is_root) diff --git a/targetcli/ui_node.py b/targetcli/ui_node.py index a6982f1..4cb09ed 100644 --- a/targetcli/ui_node.py +++ b/targetcli/ui_node.py @@ -49,6 +49,9 @@ def __init__(self, name, parent=None, shell=None): self.define_config_group_param( 'global', 'max_backup_files', 'string', 'Max no. of configurations to be backed up in /etc/target/backup/ directory.') + self.define_config_group_param( + 'global', 'auto_use_daemon', 'bool', + 'If true, commands will be sent to targetclid.') def assert_root(self): '''