From 81301e1de9bffad0540144b85ac2189f1a142e60 Mon Sep 17 00:00:00 2001 From: Haitao Yue Date: Sun, 2 Sep 2018 18:46:06 +0800 Subject: [PATCH] [CE-458] Refine delete cluster api Move delete cluster task into celery task for quick handle. Use supervisor instead of celeryd can restart celery when process is crashed. Add more info in chain list in operator dashboard. Can auto refresh chain info when processing. Change-Id: If4ee7b13bc18ea95bbb8a63b31ba175d3eb9ce83 Signed-off-by: Haitao Yue --- docker/baseimage/install.sh | 4 +- docker/operator-dashboard/Dockerfile.in | 3 +- src/celery.conf | 8 ++ src/celery.sh | 12 --- src/exceptions.py | 6 ++ src/resources/cluster_api.py | 14 +-- src/static/dashboard/src/locales/en.json | 2 + src/static/dashboard/src/locales/zh.json | 2 + src/static/dashboard/src/models/chain.js | 85 ++++++++++++++++++- .../dashboard/src/routes/Chain/index.js | 59 ++++++++++++- src/static/dashboard/src/services/chain.js | 4 + src/static/dashboard/src/utils/utils.js | 4 + src/static/package.json | 3 +- src/tasks.py | 16 ---- src/tasks/__init__.py | 5 ++ src/tasks/cluster.py | 39 +++++++++ 16 files changed, 217 insertions(+), 49 deletions(-) create mode 100644 src/celery.conf delete mode 100644 src/celery.sh create mode 100644 src/exceptions.py delete mode 100644 src/tasks.py create mode 100644 src/tasks/__init__.py create mode 100644 src/tasks/cluster.py diff --git a/docker/baseimage/install.sh b/docker/baseimage/install.sh index 78375a0ad..a93265c60 100644 --- a/docker/baseimage/install.sh +++ b/docker/baseimage/install.sh @@ -8,5 +8,5 @@ set -x # Based thie file on https://github.com/docker-library/mongo/blob/master/3.4/Dockerfile & # https://docs.mongodb.com/manual/tutorial/install-mongodb-enterprise-on-ubuntu/#install-mongodb-enterprise -#set -x \ -# && apt-get update && apt-get install -y --no-install-recommends circus && rm -rf /var/lib/apt/lists/* +set -x \ + && apt-get update && apt-get install -y supervisor && rm -rf /var/lib/apt/lists/* diff --git a/docker/operator-dashboard/Dockerfile.in b/docker/operator-dashboard/Dockerfile.in index 6a629b098..e9ff464b1 100644 --- a/docker/operator-dashboard/Dockerfile.in +++ b/docker/operator-dashboard/Dockerfile.in @@ -1,8 +1,9 @@ FROM hyperledger/cello-baseimage:latest COPY src /app +COPY src/celery.conf /etc/supervisor/conf.d/ RUN cd /app/ && \ pip install -r requirements.txt && \ rm -rf /tmp/cello -CMD bash /app/celery.sh && if [ "$DEBUG" = "True" ]; then python dashboard.py ; else gunicorn -w 1 --worker-class eventlet -b 0.0.0.0:8080 dashboard:app ;fi +CMD /etc/init.d/supervisor start && if [ "$DEBUG" = "True" ]; then python dashboard.py ; else gunicorn -w 1 --worker-class eventlet -b 0.0.0.0:8080 dashboard:app ;fi diff --git a/src/celery.conf b/src/celery.conf new file mode 100644 index 000000000..a41261319 --- /dev/null +++ b/src/celery.conf @@ -0,0 +1,8 @@ +[program:celery] +environment=C_FORCE_ROOT="yes" +command=/usr/local/bin/celery worker --autoscale=20,5 -l info -A dashboard.celery +directory=/app +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/celery.log +redirect_stderr=true \ No newline at end of file diff --git a/src/celery.sh b/src/celery.sh deleted file mode 100644 index c023f7d07..000000000 --- a/src/celery.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2009-2017 SAP SE or an SAP affiliate company. -# All Rights Reserved. -# -# SPDX-License-Identifier: Apache-2.0 -# -holdup -t 120 tcp://rabbitmq:5672 - -export C_FORCE_ROOT=yes -cd /app -celery worker -A dashboard.celery --autoscale=20,3 -l info -f /var/log/celery.log -D \ No newline at end of file diff --git a/src/exceptions.py b/src/exceptions.py new file mode 100644 index 000000000..02ba12af6 --- /dev/null +++ b/src/exceptions.py @@ -0,0 +1,6 @@ +class ReleaseClusterException(Exception): + pass + + +class DeleteClusterException(Exception): + pass diff --git a/src/resources/cluster_api.py b/src/resources/cluster_api.py index 96ba2524f..48c736642 100644 --- a/src/resources/cluster_api.py +++ b/src/resources/cluster_api.py @@ -21,7 +21,7 @@ FabricPreNetworkConfig, FabricV1NetworkConfig from modules import cluster_handler, host_handler -from tasks import release_cluster +from tasks import release_cluster, delete_cluster logger = logging.getLogger(__name__) logger.setLevel(LOG_LEVEL) @@ -295,16 +295,8 @@ def cluster_delete(): else: logger.debug("cluster delete with id={0}, col_name={1}".format( cluster_id, col_name)) - if col_name == "active": - result = cluster_handler.delete(id=cluster_id) - else: - result = cluster_handler.delete_released(id=cluster_id) - if result: - return make_ok_resp() - else: - error_msg = "Failed to delete cluster {}".format(cluster_id) - logger.warning(error_msg) - return make_fail_resp(error=error_msg) + delete_cluster.delay(cluster_id, col_name) + return make_ok_resp() @bp_cluster_api.route('/clusters', methods=['GET', 'POST']) diff --git a/src/static/dashboard/src/locales/en.json b/src/static/dashboard/src/locales/en.json index e0d24d27d..2a22b6877 100755 --- a/src/static/dashboard/src/locales/en.json +++ b/src/static/dashboard/src/locales/en.json @@ -98,6 +98,8 @@ "Chain.Label.ConsensusPlugin": "Consensus Plugin", "Chain.Label.Owner": "Owner", "Chain.Label.CreateTime": "Create Time", + "Chain.Label.Status": "Status", + "Chain.Label.Health": "Health", "Chain.Create.Title": "Create New Chain", "Chain.Create.Validate.Required.Host": "Must select a host", "Chain.Create.Label.Host": "Host", diff --git a/src/static/dashboard/src/locales/zh.json b/src/static/dashboard/src/locales/zh.json index 1fd894636..ab1828106 100755 --- a/src/static/dashboard/src/locales/zh.json +++ b/src/static/dashboard/src/locales/zh.json @@ -98,6 +98,8 @@ "Chain.Label.ConsensusPlugin": "共识", "Chain.Label.Owner": "使用者", "Chain.Label.CreateTime": "创建时间", + "Chain.Label.Status": "状态", + "Chain.Label.Health": "健康", "Chain.Create.Title": "创建新的链", "Chain.Create.Validate.Required.Host": "必须选择一个主机", "Chain.Create.Label.Host": "主机", diff --git a/src/static/dashboard/src/models/chain.js b/src/static/dashboard/src/models/chain.js index 54fb520b9..41ce88937 100644 --- a/src/static/dashboard/src/models/chain.js +++ b/src/static/dashboard/src/models/chain.js @@ -4,8 +4,8 @@ import { routerRedux } from 'dva/router'; import { IntlProvider, defineMessages } from 'react-intl'; import { message } from 'antd'; -import { queryChains, operateChain, deleteChain, createChain } from '../services/chain'; -import { getLocale } from '../utils/utils'; +import { queryChains, operateChain, deleteChain, createChain, getChain } from '../services/chain'; +import { getLocale, sleep } from '../utils/utils'; const currentLocale = getLocale(); const intlProvider = new IntlProvider( @@ -42,15 +42,59 @@ export default { state: { chains: [], + canQueryChain: false, }, effects: { *fetchChains({ payload }, { call, put }) { const response = yield call(queryChains, payload); + const chains = response.data || []; yield put({ type: 'setChains', - payload: response.data, + payload: chains, }); + yield *chains.map((chain) => { + if (['creating', 'deleting'].indexOf(chain.status) >= 0 || ['', 'FAIL'].indexOf(chain.health) >= 0) { + return put({ + type: 'getChain', + payload: { + id: chain.id, + }, + }) + } else { + return true; + } + }) + }, + *getChain({ payload }, { select, call, put}) { + const response = yield call(getChain, payload.id); + const canQueryChain = yield select(state => state.chain.canQueryChain); + if (response.code === 200) { + yield put({ + type: 'updateChain', + payload: { + id: payload.id, + data: response.data, + }, + }); + const chain = response.data; + if (canQueryChain && (['creating', 'deleting'].indexOf(chain.status) >= 0 || ['', 'FAIL'].indexOf(chain.health) >= 0)) { + yield sleep(5000); + yield put({ + type: 'getChain', + payload: { + id: chain.id, + }, + }) + } + } else if (response.code === 404) { + yield put({ + type: 'removeChain', + payload: { + id: payload.id, + }, + }) + } }, *operateChain({ payload }, { call, put }) { const response = yield call(operateChain, payload); @@ -92,5 +136,40 @@ export default { chains: action.payload, }; }, + updateChain(state, action) { + const { id, data } = action.payload; + const { chains } = state; + chains.forEach((chain, index) => { + if (chain.id === id) { + chains[index] = data; + return false; + } + }); + return { + ...state, + chains, + } + }, + removeChain(state, action) { + const { id } = action.payload; + const { chains } = state; + chains.forEach((chain, index) => { + if (chain.id === id) { + chains.splice(index, 1); + return false; + } + }); + return { + ...state, + chains, + } + }, + setCanQuery(state, action) { + const { canQueryChain } = action.payload; + return { + ...state, + canQueryChain, + } + }, }, }; diff --git a/src/static/dashboard/src/routes/Chain/index.js b/src/static/dashboard/src/routes/Chain/index.js index a823159d8..7f001d5f4 100644 --- a/src/static/dashboard/src/routes/Chain/index.js +++ b/src/static/dashboard/src/routes/Chain/index.js @@ -6,6 +6,7 @@ import { connect } from 'dva'; import { defineMessages, injectIntl, FormattedMessage } from 'react-intl'; import { List, Card, Button, Dropdown, Menu, Icon, Badge, Modal, Radio } from 'antd'; import { routerRedux } from 'dva/router'; +import moment from 'moment'; import PageHeaderLayout from '../../layouts/PageHeaderLayout'; import styles from './style.less'; @@ -64,6 +65,18 @@ const messages = defineMessages({ id: 'Chain.Label.CreateTime', defaultMessage: 'Create Time', }, + status: { + id: 'Chain.Label.Status', + defaultMessage: 'Status', + }, + health: { + id: 'Chain.Label.Health', + defaultMessage: 'Health', + }, + host: { + id: 'Chain.Create.Label.Host', + defaultMessage: 'Host', + }, }, radio: { option: { @@ -93,8 +106,22 @@ const messages = defineMessages({ })) class Chain extends PureComponent { componentDidMount() { + this.props.dispatch({ + type: 'chain/setCanQuery', + payload: { + canQueryChain: true, + }, + }); this.loadChains(); } + componentWillUnmount() { + this.props.dispatch({ + type: 'chain/setCanQuery', + payload: { + canQueryChain: false, + }, + }); + }; onClickAddChain = () => { this.props.dispatch( routerRedux.push({ @@ -162,6 +189,18 @@ class Chain extends PureComponent { }; function badgeStatus(status) { switch (status) { + case 'running': + return 'success'; + case 'creating': + case 'deleting': + return 'processing'; + case 'stopped': + default: + return 'default'; + } + } + function badgeHealth(health) { + switch (health) { case 'OK': return 'success'; case 'FAIL': @@ -172,7 +211,7 @@ class Chain extends PureComponent { return 'default'; } } - const ListContent = ({ data: { user_id, create_ts, health } }) => ( + const ListContent = ({ data: { user_id, create_ts, health, status } }) => (
@@ -184,10 +223,23 @@ class Chain extends PureComponent { -

{create_ts}

+

{moment(create_ts).format("YYYY-MM-DD HH:mm:ss")}

- + + + +

+ +

+
+
+ + + +

+ +

); @@ -264,6 +316,7 @@ class Chain extends PureComponent { description={

+ : {item.host}    : {item.network_type}

diff --git a/src/static/dashboard/src/services/chain.js b/src/static/dashboard/src/services/chain.js index bbd5f3978..12bab5248 100644 --- a/src/static/dashboard/src/services/chain.js +++ b/src/static/dashboard/src/services/chain.js @@ -27,3 +27,7 @@ export async function createChain(params) { body: params, }); } + +export async function getChain(id) { + return request(`${urls.cluster.crud}/${id}`) +} diff --git a/src/static/dashboard/src/utils/utils.js b/src/static/dashboard/src/utils/utils.js index 2d5405ed7..71f4cce25 100644 --- a/src/static/dashboard/src/utils/utils.js +++ b/src/static/dashboard/src/utils/utils.js @@ -146,3 +146,7 @@ export function getLocale() { export function getLang() { return (window.localStorage && localStorage.getItem('language')) || (navigator.language || navigator.browserLanguage).toLowerCase(); } + +export async function sleep(sleep_time_ms) { + return new Promise(resolve => setTimeout(resolve, sleep_time_ms)); +} diff --git a/src/static/package.json b/src/static/package.json index 001a25d56..e10e702bc 100644 --- a/src/static/package.json +++ b/src/static/package.json @@ -18,7 +18,8 @@ "echarts-for-react": "^1.1.6", "react-draft-wysiwyg": "^1.8.1", "react-helmet": "^5.0.0", - "echarts": "^3.4.0" + "echarts": "^3.4.0", + "moment": "^2.22.2" }, "devDependencies": { "atool-build": "^0.7.6", diff --git a/src/tasks.py b/src/tasks.py deleted file mode 100644 index 459a1ab9d..000000000 --- a/src/tasks.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright IBM Corp, All Rights Reserved. -# -# SPDX-License-Identifier: Apache-2.0 -# -from extensions import celery -import logging -from modules import cluster_handler - -logger = logging.getLogger(__name__) - - -@celery.task() -def release_cluster(cluster_id): - if cluster_handler.release_cluster(cluster_id): - logger.info("release cluster successfully") - return True diff --git a/src/tasks/__init__.py b/src/tasks/__init__.py new file mode 100644 index 000000000..8a0bed7fe --- /dev/null +++ b/src/tasks/__init__.py @@ -0,0 +1,5 @@ +# Copyright IBM Corp, All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +from . cluster import release_cluster, delete_cluster diff --git a/src/tasks/cluster.py b/src/tasks/cluster.py new file mode 100644 index 000000000..50aeb022a --- /dev/null +++ b/src/tasks/cluster.py @@ -0,0 +1,39 @@ +# Copyright IBM Corp, All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +import logging +import sys +import os + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) + +from modules import cluster_handler +from extensions import celery +from exceptions import ReleaseClusterException, DeleteClusterException + +logger = logging.getLogger(__name__) + + +@celery.task(name="release_cluster", bind=True, + default_retry_delay=5, max_retries=3) +def release_cluster(self, cluster_id): + if cluster_handler.release_cluster(cluster_id): + logger.info("release cluster {} successfully".format(cluster_id)) + return True + else: + self.retry(exc=ReleaseClusterException) + + +@celery.task(name="delete_cluster", bind=True, + default_retry_delay=5, max_retries=3) +def delete_cluster(self, cluster_id, status): + if status == "active": + result = cluster_handler.delete(id=cluster_id) + else: + result = cluster_handler.delete_released(id=cluster_id) + + if result: + return True + + self.retry(exc=DeleteClusterException)