From 70165bde458559c218581222ffe61ef828ba1a2f Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 12 Sep 2019 16:21:19 +0530 Subject: [PATCH] reload: add ability to reload a single block volume Problem: ------- Right now, if any block volume is failed to load as part of service bringup or node reboot, may be because of an issue from the backend, then there is no way to reload that single block volume, we need to reload/restart all the\ block volumes present in the node just to load one block volume, this interrupts ongoing I/O (via this path) for all the volumes hosted within the node. Solution: -------- Add ability to reload a single block volume without touching other block volumes in the node. $ gluster-block help usage: gluster-block [timeout ] [] [--json*] commands: [...] reload reload a block device. [...] Reviewed-by: Xiubo Li Signed-off-by: Prasanna Kumar Kalever --- cli/gluster-block.c | 53 ++++- rpc/Makefile.am | 2 +- rpc/block_common.h | 5 +- rpc/block_reload.c | 421 +++++++++++++++++++++++++++++++++++++++ rpc/block_svc_routines.c | 34 ++++ rpc/rpcl/block.x | 14 ++ utils/utils.h | 6 + 7 files changed, 532 insertions(+), 3 deletions(-) create mode 100644 rpc/block_reload.c diff --git a/cli/gluster-block.c b/cli/gluster-block.c index efa1377..42d432e 100644 --- a/cli/gluster-block.c +++ b/cli/gluster-block.c @@ -23,6 +23,7 @@ " [size] [--json*]" # define GB_DELETE_HELP_STR "gluster-block delete " \ "[unlink-storage ] [force] [--json*]" +# define GB_RELOAD_HELP_STR "gluster-block reload [--json*]" # define GB_MODIFY_HELP_STR "gluster-block modify " \ "[auth ] [size " \ "[force]] [--json*]" @@ -57,7 +58,8 @@ typedef enum clioperations { MODIFY_CLI = 5, MODIFY_SIZE_CLI = 6, REPLACE_CLI = 7, - GENCONF_CLI = 8 + GENCONF_CLI = 8, + RELOAD_CLI = 9 } clioperations; @@ -104,6 +106,7 @@ glusterBlockCliRPC_1(void *cobj, clioperations opt) struct sockaddr_un saun = {0,}; blockCreateCli *create_obj; blockDeleteCli *delete_obj; + blockReloadCli *reload_obj; blockInfoCli *info_obj; blockListCli *list_obj; blockModifyCli *modify_obj; @@ -193,6 +196,15 @@ glusterBlockCliRPC_1(void *cobj, clioperations opt) goto out; } break; + case RELOAD_CLI: + reload_obj = cobj; + if (block_reload_cli_1(reload_obj, &reply, clnt) != RPC_SUCCESS) { + LOG("cli", GB_LOG_ERROR, "%s block %s reload on volume %s failed", + clnt_sperror(clnt, "block_reload_cli_1"), + reload_obj->block_name, reload_obj->volume); + goto out; + } + break; case INFO_CLI: info_obj = cobj; if (block_info_cli_1(info_obj, &reply, clnt) != RPC_SUCCESS) { @@ -318,6 +330,9 @@ glusterBlockHelp(void) " replace [force]\n" " replace operations.\n" "\n" + " reload \n" + " reload a block device.\n" + "\n" " genconfig enable-tpg \n" " generate the block volumes target configuration.\n" "\n" @@ -896,6 +911,35 @@ glusterBlockDelete(int argcount, char **options, int json) return ret; } +static int +glusterBlockReload(int argcount, char **options, int json) +{ + blockReloadCli robj = {{0},}; + int ret = -1; + + + GB_ARGCHECK_OR_RETURN(argcount, 2, "reload", GB_RELOAD_HELP_STR); + robj.json_resp = json; + + if (glusterBlockParseVolumeBlock (options[1], robj.volume, robj.block_name, + sizeof(robj.volume), sizeof(robj.block_name), + GB_RELOAD_HELP_STR, "reload")) { + goto out; + } + + getCommandString(&robj.cmd, argcount, options); + + ret = glusterBlockCliRPC_1(&robj, RELOAD_CLI); + if (ret) { + LOG("cli", GB_LOG_ERROR, + "failed reload of block %s on volume %s", + robj.block_name, robj.volume); + } + + out: + + return ret; +} static int glusterBlockInfo(int argcount, char **options, int json) @@ -1089,6 +1133,13 @@ glusterBlockParseArgs(int count, char **options, size_t opt, int json) } goto out; + case GB_CLI_RELOAD: + ret = glusterBlockReload(count, options, json); + if (ret) { + LOG("cli", GB_LOG_ERROR, FAILED_RELOAD); + } + goto out; + case GB_CLI_HELP: case GB_CLI_HYPHEN_HELP: case GB_CLI_USAGE: diff --git a/rpc/Makefile.am b/rpc/Makefile.am index 44c73fa..2d2ed12 100644 --- a/rpc/Makefile.am +++ b/rpc/Makefile.am @@ -5,7 +5,7 @@ noinst_LTLIBRARIES = libgbrpc.la libgbrpc_la_SOURCES = block_svc_routines.c block_info.c block_list.c \ block_create.c block_delete.c block_modify.c \ block_replace.c block_version.c block_genconfig.c \ - block_common.h glfs-operations.c + block_reload.c block_common.h glfs-operations.c noinst_HEADERS = glfs-operations.h diff --git a/rpc/block_common.h b/rpc/block_common.h index 3746c76..87e36d0 100644 --- a/rpc/block_common.h +++ b/rpc/block_common.h @@ -31,6 +31,8 @@ # define GB_TGCLI_GLFS "targetcli " GB_TGCLI_GLFS_PATH # define GB_TGCLI_CHECK GB_TGCLI_GLFS " ls | grep ' %s ' | grep '/%s ' > " DEVNULLPATH # define GB_TGCLI_ISCSI_PATH "/iscsi" +# define GB_TGCLI_ISCSI "targetcli " GB_TGCLI_ISCSI_PATH +# define GB_TGCLI_ISCSI_CHECK GB_TGCLI_ISCSI " ls | grep ' %s%s ' > " DEVNULLPATH # define GB_TGCLI_GLFS_SAVE GB_TGCLI_GLFS_PATH "/%s saveconfig" # define GB_TGCLI_ATTRIBUTES "generate_node_acls=1 demo_mode_write_protect=0" # define GB_TGCLI_IQN_PREFIX "iqn.2016-12.org.gluster-block:" @@ -64,7 +66,8 @@ typedef enum operations { LIST_SRV, INFO_SRV, VERSION_SRV, - GENCONFIG_SRV + GENCONFIG_SRV, + RELOAD_SRV } operations; diff --git a/rpc/block_reload.c b/rpc/block_reload.c new file mode 100644 index 0000000..bf48553 --- /dev/null +++ b/rpc/block_reload.c @@ -0,0 +1,421 @@ +/* + Copyright (c) 2019 Red Hat, Inc. + This file is part of gluster-block. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + + +# include "block_common.h" + +# define GB_RELOAD "restoreconfig " GB_SAVECONFIG " clear_existing" + + +void * +glusterBlockReloadRemote(void *data) +{ + int ret; + int saveret; + blockRemoteObj *args = (blockRemoteObj *)data; + blockReload robj = *(blockReload *)args->obj; + char *errMsg = NULL; + bool rpc_sent = FALSE; + + + ret = glusterBlockCallRPC_1(args->addr, &robj, RELOAD_SRV, &rpc_sent, + &args->reply); + if (ret) { + saveret = ret; + if (!rpc_sent) { + GB_ASPRINTF(&errMsg, ": %s", strerror(errno)); + LOG("mgmt", GB_LOG_ERROR, "%s hence %s for block %s on " + "host %s volume %s", strerror(errno), FAILED_REMOTE_RELOAD, + robj.block_name, args->addr, args->volume); + goto out; + } else if (args->reply) { + errMsg = args->reply; + args->reply = NULL; + } + + LOG("mgmt", GB_LOG_ERROR, "%s for block %s on host %s volume %s", + FAILED_REMOTE_RELOAD, robj.block_name, args->addr, args->volume); + + ret = saveret;; + goto out; + } + + out: + if (!args->reply) { + if (GB_ASPRINTF(&args->reply, "failed to reload config on %s %s", + args->addr, errMsg?errMsg:"") == -1) { + ret = ret?ret:-1; + } + } + GB_FREE(errMsg); + args->exit = ret; + + return NULL; +} + + +static size_t +glusterBlockReloadFillArgs(MetaInfo *info, blockRemoteObj *args, + struct glfs *glfs, blockReload *robj) +{ + int i = 0; + size_t count = 0; + + for (i = 0, count = 0; i < info->nhosts; i++) { + if (blockhostIsValid(info->list[i]->status)) { + if (args) { + args[count].glfs = glfs; + args[count].obj = (void *)robj; + args[count].volume = info->volume; + args[count].addr = info->list[i]->addr; + } + count++; + } + } + return count; +} + + +static int +glusterBlockReloadRemoteAsync(MetaInfo *info, + struct glfs *glfs, + blockReload *robj, + blockRemoteDeleteResp **savereply) +{ + pthread_t *tid = NULL; + blockRemoteDeleteResp *local = *savereply; + blockRemoteObj *args = NULL; + char *d_attempt = NULL; + char *d_success = NULL; + char *a_tmp = NULL; + char *s_tmp = NULL; + int ret = -1; + size_t i; + size_t count; + + + count = glusterBlockReloadFillArgs(info, NULL, NULL, NULL); + if (GB_ALLOC_N(tid, count) < 0 || GB_ALLOC_N(args, count) < 0) { + goto out; + } + + count = glusterBlockReloadFillArgs(info, args, glfs, robj); + + for (i = 0; i < count; i++) { + pthread_create(&tid[i], NULL, glusterBlockReloadRemote, &args[i]); + } + + for (i = 0; i < count; i++) { + pthread_join(tid[i], NULL); + } + + ret = glusterBlockCollectAttemptSuccess(args, info, RELOAD_SRV, count, + &d_attempt, &d_success); + if (ret) { + goto out; + } + ret = -1; + + if (d_attempt) { + a_tmp = local->d_attempt; + if (GB_ASPRINTF(&local->d_attempt, "%s %s", + (a_tmp==NULL?"":a_tmp), d_attempt) == -1) { + goto out; + } + GB_FREE(a_tmp); + a_tmp = local->d_attempt; + } + + if (d_success) { + s_tmp = local->d_success; + if (GB_ASPRINTF(&local->d_success, "%s %s", + (s_tmp==NULL?"":s_tmp), d_success) == -1) { + goto out; + } + GB_FREE(s_tmp); + s_tmp = local->d_success; + } + + ret = 0; + for (i = 0; i < count; i++) { + if (args[i].exit) { + ret = -1; + break; + } + } + + out: + GB_FREE(d_attempt); + GB_FREE(d_success); + GB_FREE(args); + GB_FREE(tid); + + return ret; +} + + +static void +blockReloadCliFormatResponse(blockReloadCli *blk, int errCode, char *errMsg, + blockRemoteDeleteResp *savereply, + blockResponse *reply) +{ + json_object *json_obj = NULL; + char *tmp = NULL; + + if (!reply) { + return; + } + + if (errCode < 0) { + errCode = GB_DEFAULT_ERRCODE; + } + reply->exit = errCode; + + if (errMsg) { + blockFormatErrorResponse(RELOAD_SRV, blk->json_resp, errCode, + errMsg, reply); + return; + } + + if (blk->json_resp) { + json_obj = json_object_new_object(); + + blockStr2arrayAddToJsonObj (json_obj, savereply->d_attempt, "FAILED ON"); + blockStr2arrayAddToJsonObj (json_obj, savereply->d_success, "SUCCESSFUL ON"); + + json_object_object_add(json_obj, "RESULT", + errCode?GB_JSON_OBJ_TO_STR("FAIL"):GB_JSON_OBJ_TO_STR("SUCCESS")); + + GB_ASPRINTF(&reply->out, "%s\n", + json_object_to_json_string_ext(json_obj, + mapJsonFlagToJsonCstring(blk->json_resp))); + + json_object_put(json_obj); + } else { + /* save 'failed on'*/ + if (savereply->d_attempt) { + if (GB_ASPRINTF(&tmp, "FAILED ON: %s\n", savereply->d_attempt) == -1) + goto out; + } + + if (GB_ASPRINTF(&reply->out, + "%sSUCCESSFUL ON: %s\nRESULT: %s\n", tmp?tmp:"", + savereply->d_success?savereply->d_success:"None", + errCode?"FAIL":"SUCCESS") == -1) { + goto out; + } + } + out: + /*catch all*/ + if (!reply->out) { + blockFormatErrorResponse(RELOAD_SRV, blk->json_resp, errCode, + GB_DEFAULT_ERRMSG, reply); + } + + GB_FREE (tmp); + return; +} + + +static blockResponse * +block_reload_cli_1_svc_st(blockReloadCli *blk, struct svc_req *rqstp) +{ + blockRemoteDeleteResp *savereply = NULL; + MetaInfo *info = NULL; + blockResponse *reply = NULL; + struct glfs *glfs; + struct glfs_fd *lkfd = NULL; + char *errMsg = NULL; + int errCode = -1; + int ret; +// blockServerDefPtr list = NULL; + blockReload robj = {{0},}; + + LOG("mgmt", GB_LOG_INFO, "reload cli request, volume=%s blockname=%s", + blk->volume, blk->block_name); + + if (GB_ALLOC(reply) < 0) { + goto optfail; + } + + if (GB_ALLOC(savereply) < 0) { + GB_FREE(reply); + goto optfail; + } + + errCode = 0; + glfs = glusterBlockVolumeInit(blk->volume, &errCode, &errMsg); + if (!glfs) { + LOG("mgmt", GB_LOG_ERROR, + "glusterBlockVolumeInit(%s) for block %s failed", + blk->volume, blk->block_name); + goto optfail; + } + + lkfd = glusterBlockCreateMetaLockFile(glfs, blk->volume, &errCode, &errMsg); + if (!lkfd) { + LOG("mgmt", GB_LOG_ERROR, "%s %s for block %s", + FAILED_CREATING_META, blk->volume, blk->block_name); + goto optfail; + } + + GB_METALOCK_OR_GOTO(lkfd, blk->volume, errCode, errMsg, optfail); + LOG("cmdlog", GB_LOG_INFO, "%s", blk->cmd); + + if (glfs_access(glfs, blk->block_name, F_OK)) { + errCode = errno; + if (errCode == ENOENT) { + GB_ASPRINTF(&errMsg, "block %s/%s doesn't exist", + blk->volume, blk->block_name); + LOG("mgmt", GB_LOG_ERROR, + "block with name %s doesn't exist in the volume %s", + blk->block_name, blk->volume); + } else { + GB_ASPRINTF(&errMsg, "block %s/%s is not accessible (%s)", + blk->volume, blk->block_name, strerror(errCode)); + LOG("mgmt", GB_LOG_ERROR, "block %s/%s is not accessible (%s)", + blk->volume, blk->block_name, strerror(errCode)); + } + goto out; + } + + if (GB_ALLOC(info) < 0) { + errCode = ENOMEM; + goto out; + } + + ret = blockGetMetaInfo(glfs, blk->block_name, info, NULL); + if (ret) { + errCode = ret; + goto out; + } + +/* + if (!blk->force) { + list = glusterBlockGetListFromInfo(info); + if (!list) { + errCode = ENOMEM; + goto out; + } + + errCode = glusterBlockCheckCapabilities((void *)blk, DELETE_SRV, list, NULL, &errMsg); + if (errCode) { + LOG("mgmt", GB_LOG_ERROR, + "glusterBlockCheckCapabilities() for block %s on volume %s failed", + blk->block_name, blk->volume); + goto out; + } + } +*/ + + GB_STRCPYSTATIC(robj.block_name, blk->block_name); + GB_STRCPYSTATIC(robj.gbid, info->gbid); + + errCode = glusterBlockReloadRemoteAsync(info, glfs, &robj, &savereply); + if (errCode) { + LOG("mgmt", GB_LOG_WARNING, "glusterBlockReloadRemoteAsync: return %d " + "on block %s for volume %s", errCode, blk->block_name, blk->volume); + } + + out: + GB_METAUNLOCK(lkfd, blk->volume, errCode, errMsg); + blockFreeMetaInfo(info); +// blockServerDefFree(list); + + optfail: + LOG("mgmt", ((!!errCode) ? GB_LOG_ERROR : GB_LOG_INFO), + "reload cli return %s, volume=%s blockname=%s", + errCode ? "failure" : "success", blk->volume, blk->block_name); + + if (lkfd && glfs_close(lkfd) != 0) { + LOG("mgmt", GB_LOG_ERROR, + "glfs_close(%s): for block %s on volume %s failed[%s]", + GB_TXLOCKFILE, blk->block_name, blk->volume, strerror(errno)); + } + + blockReloadCliFormatResponse(blk, errCode, errMsg, savereply, reply); + LOG("cmdlog", ((!!errCode) ? GB_LOG_ERROR : GB_LOG_INFO), "%s", + reply ? reply->out : "*Nil*"); + + if (savereply) { + GB_FREE(savereply->d_attempt); + GB_FREE(savereply->d_success); + GB_FREE(savereply); + } + GB_FREE(errMsg); + + return reply; +} + + +static blockResponse * +block_reload_1_svc_st(blockReload *blk, struct svc_req *rqstp) +{ + char *exec = NULL; + blockResponse *reply = NULL; + + + LOG("mgmt", GB_LOG_INFO, + "reload request, blockname=%s filename=%s", blk->block_name, blk->gbid); + + if (GB_ALLOC(reply) < 0) { + goto out; + } + reply->exit = -1; + + if (GB_ASPRINTF(&exec, "targetcli %s target=%s%s storage_object=%s", GB_RELOAD, + GB_TGCLI_IQN_PREFIX, blk->gbid, blk->block_name) == -1) { + goto out; + } + + if (GB_ALLOC_N(reply->out, 8192) < 0) { + GB_FREE(reply); + goto out; + } + + /* currently targetcli throw few error msgs for keywords in saveconfig.json + * which are unknown and also return few warning msgs and error code, hence + * not using GB_CMD_EXEC_AND_VALIDATE() macro and not caring for return + * value for now. For now lets just run the restoreconfig command and use + * blockCheckBlockLoadedStatus() to check we get success. + */ + gbRunner(exec); + + if (blockCheckBlockLoadedStatus(blk->block_name, blk->gbid, reply)) { + goto out; + } + snprintf(reply->out, 8192, "reload success"); + reply->exit = 0; + + out: + GB_FREE(exec); + + return reply; +} + +bool_t +block_reload_1_svc(blockReload *blk, blockResponse *reply, struct svc_req *rqstp) +{ + int ret; + + GB_RPC_CALL(reload, blk, reply, rqstp, ret); + return ret; +} + + +bool_t +block_reload_cli_1_svc(blockReloadCli *blk, blockResponse *reply, + struct svc_req *rqstp) +{ + int ret; + + GB_RPC_CALL(reload_cli, blk, reply, rqstp, ret); + return ret; +} diff --git a/rpc/block_svc_routines.c b/rpc/block_svc_routines.c index 48d51ae..c32d19e 100644 --- a/rpc/block_svc_routines.c +++ b/rpc/block_svc_routines.c @@ -110,6 +110,24 @@ blockCheckBlockLoadedStatus(char *block_name, char *gbid, blockResponse *reply) GB_ASPRINTF(&reply->out, "Block '%s' may be not loaded.", block_name); LOG("mgmt", GB_LOG_ERROR, "%s", reply->out); } + GB_FREE(exec); + + if (!ret) { + if (GB_ASPRINTF(&exec, GB_TGCLI_ISCSI_CHECK, GB_TGCLI_IQN_PREFIX, gbid) == -1) { + goto out; + } + + ret = gbRunner(exec); + if (ret == -1) { + GB_ASPRINTF(&reply->out, "command exit abnormally for '%s'.", block_name); + LOG("mgmt", GB_LOG_ERROR, "%s", reply->out); + goto out; + } else if (ret == 1) { + is_loaded = false; + GB_ASPRINTF(&reply->out, "Target for '%s' may be not loaded.", block_name); + LOG("mgmt", GB_LOG_ERROR, "%s", reply->out); + } + } /* if block is loaded, skip the rest */ if (!ret) { @@ -283,6 +301,14 @@ glusterBlockCallRPC_1(char *host, void *cobj, goto out; } break; + case RELOAD_SRV: + *rpc_sent = TRUE; + if (block_reload_1((blockReload *)cobj, &reply, clnt) != RPC_SUCCESS) { + LOG("mgmt", GB_LOG_ERROR, "%son host %s", + clnt_sperror(clnt, "block remote reload call failed"), host); + goto out; + } + break; case MODIFY_TPGC_SRV: case LIST_SRV: case INFO_SRV: @@ -508,6 +534,7 @@ blockValidateCommandOutput(const char *out, int opt, void *data) blockModify *mblk = data; blockModifySize *msblk = data; blockReplace *rblk = data; + blockReload *rlblk = data; int ret = -1; @@ -641,6 +668,13 @@ blockValidateCommandOutput(const char *out, int opt, void *data) rblk, rblk->ripaddr, "tpg"); ret = 0; break; + + case RELOAD_SRV: + /* get tpg of the portal */ + GB_OUT_VALIDATE_OR_GOTO(out, out, "failed to reload block volume: %s", + rlblk, rlblk->block_name, "Configuration restored"); + ret = 0; + break; } out: diff --git a/rpc/rpcl/block.x b/rpc/rpcl/block.x index 0a3dd9f..3b666dd 100644 --- a/rpc/rpcl/block.x +++ b/rpc/rpcl/block.x @@ -91,6 +91,18 @@ struct blockDelete { char gbid[127]; }; +struct blockReloadCli { + char block_name[255]; + char volume[255]; + string cmd<>; + enum JsonResponseFormat json_resp; +}; + +struct blockReload { + char block_name[255]; + char gbid[127]; +}; + struct blockInfoCli { char block_name[255]; char volume[255]; @@ -153,6 +165,7 @@ program GLUSTER_BLOCK { blockResponse BLOCK_MODIFY_SIZE(blockModifySize) = 6; blockResponse BLOCK_CREATE_V2(blockCreate2) = 7; + blockResponse BLOCK_RELOAD(blockReload) = 8; } = 1; } = 21215311; /* B2 L12 O15 C3 K11 */ @@ -166,5 +179,6 @@ program GLUSTER_BLOCK_CLI { blockResponse BLOCK_REPLACE_CLI(blockReplaceCli) = 6; blockResponse BLOCK_MODIFY_SIZE_CLI(blockModifySizeCli) = 7; blockResponse BLOCK_GEN_CONFIG_CLI(blockGenConfigCli) = 8; + blockResponse BLOCK_RELOAD_CLI(blockReloadCli) = 9; } = 1; } = 212153113; /* B2 L12 O15 C3 K11 C3 */ diff --git a/utils/utils.h b/utils/utils.h index 17ab38b..752e6e9 100644 --- a/utils/utils.h +++ b/utils/utils.h @@ -101,6 +101,10 @@ # define FAILED_DELETING_FILE "failed while deleting block file from gluster volume" # define FAILED_DELETING_META "failed while deleting block meta file from volume" +/* Target Reload */ +# define FAILED_RELOAD "failed in reload" +# define FAILED_REMOTE_RELOAD "failed in remote reload" + /* Target Replace */ # define FAILED_REPLACE "failed in replace" # define FAILED_REMOTE_REPLACE "failed in remote replace portal" @@ -429,6 +433,7 @@ typedef enum gbCliCmdlineOption { GB_CLI_DELETE, GB_CLI_MODIFY, GB_CLI_REPLACE, + GB_CLI_RELOAD, GB_CLI_GENCONFIG, GB_CLI_HELP, GB_CLI_HYPHEN_HELP, @@ -449,6 +454,7 @@ static const char *const gbCliCmdlineOptLookup[] = { [GB_CLI_DELETE] = "delete", [GB_CLI_MODIFY] = "modify", [GB_CLI_REPLACE] = "replace", + [GB_CLI_RELOAD] = "reload", [GB_CLI_GENCONFIG] = "genconfig", [GB_CLI_HELP] = "help", [GB_CLI_HYPHEN_HELP] = "--help",