diff --git a/src/container.py b/src/container.py index eca3ff1d..57f471d1 100755 --- a/src/container.py +++ b/src/container.py @@ -5,6 +5,7 @@ from log import logger import env from lvmtool import sys_run, check_volume +from monitor import History_Manager class Container(object): def __init__(self, addr, etcdclient): @@ -20,6 +21,7 @@ def __init__(self, addr, etcdclient): self.lxcpath = "/var/lib/lxc" self.imgmgr = imagemgr.ImageMgr() + self.historymgr = History_Manager() def create_container(self, lxc_name, username, setting, clustername, clusterid, containerid, hostname, ip, gateway, vlanid, image): logger.info("create container %s of %s for %s" %(lxc_name, clustername, username)) @@ -130,11 +132,13 @@ def config_prepare(content): except Exception as e: logger.error(e) return [False, "create container failed"] + self.historymgr.log(lxc_name,"Create") return [True, "create container success"] def delete_container(self, lxc_name): logger.info ("delete container:%s" % lxc_name) if self.imgmgr.deleteFS(lxc_name): + self.historymgr.log(lxc_name,"Delete") logger.info("delete container %s success" % lxc_name) return [True, "delete container success"] else: @@ -164,6 +168,7 @@ def start_container(self, lxc_name): subprocess.run(["lxc-start -n %s" % lxc_name], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, check=True) logger.info ("start container %s success" % lxc_name) + self.historymgr.log(lxc_name,"Start") return [True, "start container success"] except subprocess.CalledProcessError as sube: logger.error('start container %s failed: %s' % (lxc_name, @@ -209,6 +214,7 @@ def recover_container(self, lxc_name): if status == 'stopped': logger.info("%s stopped, recover it to running" % lxc_name) if self.start_container(lxc_name)[0]: + self.historymgr.log(lxc_name,"Recover") if self.start_services(lxc_name)[0]: logger.info("%s recover success" % lxc_name) return [True, "recover success"] @@ -235,6 +241,7 @@ def stop_container(self, lxc_name): logger.error("stop container %s failed" % lxc_name) return [False, "stop container failed"] else: + self.historymgr.log(lxc_name,"Stop") logger.info("stop container %s success" % lxc_name) return [True, "stop container success"] #if int(status) == 1: diff --git a/src/httprest.py b/src/httprest.py index c3ea6b44..4047081d 100755 --- a/src/httprest.py +++ b/src/httprest.py @@ -423,6 +423,7 @@ def hosts_monitor(cur_user, user, form, com_id, issue): @login_required def vnodes_monitor(cur_user, user, form, con_id, issue): global G_clustername + global G_historymgr logger.info("handle request: monitor/vnodes") res = {} fetcher = monitor.Container_Fetcher(con_id) @@ -434,6 +435,8 @@ def vnodes_monitor(cur_user, user, form, con_id, issue): res['disk_use'] = fetcher.get_disk_use() elif issue == 'basic_info': res['basic_info'] = fetcher.get_basic_info() + elif issue == 'history': + res['history'] = G_historymgr.getHistory(con_id) elif issue == 'owner': names = con_id.split('-') result = G_usermgr.query(username = names[0], cur_user = cur_user) @@ -801,6 +804,7 @@ def internal_server_error(error): global G_networkmgr global G_clustername global G_sysmgr + global G_historymgr # move 'tools.loadenv' to the beginning of this file fs_path = env.getenv("FS_PREFIX") @@ -876,7 +880,7 @@ def internal_server_error(error): if etcdclient.isdir("_lock")[0]: etcdclient.deldir("_lock") - G_usermgr = userManager.userManager('root') + G_usermgr = userManager.userManager('root','unias1616') if mode == "new": G_usermgr.initUsage() G_notificationmgr = notificationmgr.NotificationMgr() @@ -896,8 +900,10 @@ def internal_server_error(error): logger.info("vclustermgr started") G_imagemgr = imagemgr.ImageMgr() logger.info("imagemgr started") + G_historymgr = monitor.History_Manager() master_collector = monitor.Master_Collector(G_nodemgr) master_collector.start() + logger.info("master_collector started") logger.info("startting to listen on: ") masterip = env.getenv('MASTER_IP') diff --git a/src/model.py b/src/model.py index 3cf0c0fa..edf7d4ff 100755 --- a/src/model.py +++ b/src/model.py @@ -40,6 +40,7 @@ app = Flask(__name__) app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+fsdir+'/global/sys/UserTable.db' +app.config['SQLALCHEMY_BINDS'] = {'history': 'sqlite:///'+fsdir+'/global/sys/HistoryTable.db'} try: secret_key_file = open(env.getenv('FS_PREFIX') + '/local/token_secret_key.txt') app.secret_key = secret_key_file.read() @@ -184,3 +185,40 @@ def __init__(self, notification_id, group_name): def __repr__(self): return '' % (self.notification_id, self.group_name) + +class VNode(db.Model): + __bind_key__ = 'history' + name = db.Column(db.String(100), primary_key=True) + laststopcpuval = db.Column(db.Float) + laststopruntime = db.Column(db.Integer) + billing = db.Column(db.Integer) + histories = db.relationship('History', backref='v_node', lazy='dynamic') + + def __init__(self, vnode_name): + self.name = vnode_name + self.laststopcpuval = 0 + self.billing = 0 + self.laststopruntime = 0 + + def __repr__(self): + return '' % (self.name) + +class History(db.Model): + __bind_key__ = 'history' + id = db.Column(db.Integer, primary_key=True) + vnode = db.Column(db.String(100), db.ForeignKey('v_node.name')) + action = db.Column(db.String(30)) + runningtime = db.Column(db.Integer) + cputime = db.Column(db.Float) + billing = db.Column(db.Integer) + actionTime = db.Column(db.DateTime) + + def __init__(self, action, runningtime, cputime, billing): + self.action = action + self.runningtime = runningtime + self.cputime = cputime + self.billing = billing + self.actionTime = datetime.now() + + def __repr__(self): + return "{\"id\":\"%d\",\"vnode\":\"%s\",\"action\":\"%s\",\"runningtime\":\"%d\",\"cputime\":\"%f\",\"billing\":\"%d\",\"actionTime\":\"%s\"}" % (self.id, self.vnode, self.action, self.runningtime, self.cputime, self.billing, self.actionTime.strftime("%Y-%m-%d %H:%M:%S")) diff --git a/src/monitor.py b/src/monitor.py index f08a828e..f5b1b09e 100755 --- a/src/monitor.py +++ b/src/monitor.py @@ -3,6 +3,7 @@ import subprocess,re,os,etcdlib,psutil,math,sys import time,threading,json,traceback,platform +from model import db,VNode,History from log import logger monitor_hosts = {} @@ -13,12 +14,16 @@ containerpids = [] pid2name = {} +laststopcpuval = {} +laststopruntime = {} lastbillingtime = {} increment = {} class Container_Collector(threading.Thread): def __init__(self,test=False): + global laststopcpuval + global workercinfo threading.Thread.__init__(self) self.thread_stop = False self.interval = 2 @@ -28,6 +33,20 @@ def __init__(self,test=False): self.cpu_quota = {} self.mem_quota = {} self.cores_num = int(subprocess.getoutput("grep processor /proc/cpuinfo | wc -l")) + containers = self.list_container() + for container in containers: + if not container == '': + try: + vnode = VNode.query.get(container) + laststopcpuval[container] = vnode.laststopcpuval + laststopruntime[container] = vnode.laststopruntime + workercinfo[container] = {} + workercinfo[container]['basic_info'] = {} + workercinfo[container]['basic_info']['billing'] = vnode.billing + workercinfo[container]['basic_info']['RunningTime'] = vnode.laststopruntime + except: + laststopcpuval[container] = 0 + laststopruntime[container] = 0 return def list_container(self): @@ -57,6 +76,8 @@ def collect_containerinfo(self,container_name): global lastbillingtime global containerpids global pid2name + global laststopcpuval + global laststopruntime output = subprocess.check_output("sudo lxc-info -n %s" % (container_name),shell=True) output = output.decode('utf-8') parts = re.split('\n',output) @@ -67,7 +88,6 @@ def collect_containerinfo(self,container_name): basic_info = workercinfo[container_name]['basic_info'] else: basic_info['RunningTime'] = 0 - basic_info['LastTime'] = 0 basic_info['billing'] = 0 for part in parts: if not part == '': @@ -87,20 +107,13 @@ def collect_containerinfo(self,container_name): containerpids.append(info['PID']) pid2name[info['PID']] = container_name running_time = self.get_proc_etime(int(info['PID'])) - if basic_exist and 'PID' in workercinfo[container_name]['basic_info'].keys(): - last_time = workercinfo[container_name]['basic_info']['LastTime'] - if not info['PID'] == workercinfo[container_name]['basic_info']['PID']: - last_time = workercinfo[container_name]['basic_info']['RunningTime'] - else: - last_time = 0 - basic_info['LastTime'] = last_time - running_time += last_time + running_time += laststopruntime[container_name] basic_info['PID'] = info['PID'] basic_info['IP'] = info['IP'] basic_info['RunningTime'] = running_time cpu_parts = re.split(' +',info['CPU use']) - cpu_val = cpu_parts[0].strip() + cpu_val = float(cpu_parts[0].strip()) cpu_unit = cpu_parts[1].strip() if not container_name in self.cpu_last.keys(): confpath = "/var/lib/lxc/%s/config"%(container_name) @@ -127,6 +140,12 @@ def collect_containerinfo(self,container_name): return False self.cpu_last[container_name] = 0 cpu_use = {} + lastval = 0 + try: + lastval = laststopcpuval[container_name] + except: + logger.warning(traceback.format_exc()) + cpu_val += lastval cpu_use['val'] = cpu_val cpu_use['unit'] = cpu_unit cpu_usedp = (float(cpu_val)-float(self.cpu_last[container_name]))/(self.cpu_quota[container_name]*self.interval*1.05) @@ -139,7 +158,7 @@ def collect_containerinfo(self,container_name): if container_name not in increment.keys(): increment[container_name] = {} - increment[container_name]['lastcputime'] = 0 + increment[container_name]['lastcputime'] = cpu_val increment[container_name]['memincrement'] = 0 mem_parts = re.split(' +',info['Memory use']) @@ -158,13 +177,10 @@ def collect_containerinfo(self,container_name): mem_use['usedp'] = mem_usedp workercinfo[container_name]['mem_use'] = mem_use - lasttime = 0 - if container_name in lastbillingtime.keys(): - lasttime = lastbillingtime[container_name] - else: - lasttime = 0 - lastbillingtime[container_name] = 0 - #logger.info(running_time) + if not container_name in lastbillingtime.keys(): + lastbillingtime[container_name] = int(running_time/self.billingtime) + lasttime = lastbillingtime[container_name] + #logger.info(lasttime) if not int(running_time/self.billingtime) == lasttime: #logger.info("billing:"+str(float(cpu_val))) lastbillingtime[container_name] = int(running_time/self.billingtime) @@ -183,6 +199,16 @@ def collect_containerinfo(self,container_name): #logger.info("cpu_increment:"+str(cpu_increment)+" avemem:"+str(avemem)+" disk:"+str(disk_quota)+"\n") billing = cpu_increment/1000.0 + avemem/500000.0 + float(disk_quota)/1024.0/1024.0/2000 basic_info['billing'] += math.ceil(billing) + try: + vnode = VNode.query.get(container_name) + vnode.billing = basic_info['billing'] + db.session.commit() + except Exception as err: + vnode = VNode(container_name) + vnode.billing = basic_info['billing'] + db.session.add(vnode) + db.session.commit() + logger.warning(err) workercinfo[container_name]['basic_info'] = basic_info #print(output) #print(parts) @@ -403,6 +429,8 @@ def run(self): logger.warning(traceback.format_exc()) logger.warning(err) time.sleep(2) + #logger.info(History.query.all()) + #logger.info(VNode.query.all()) return def stop(self): @@ -556,3 +584,56 @@ def get_containerslist(self): logger.warning(err) res = {} return res + +class History_Manager: + + def __init__(self): + try: + VNode.query.all() + History.query.all() + except: + db.create_all(bind='__all__') + + def getAll(self): + return History.query.all() + + def log(self,vnode_name,action): + global workercinfo + global laststopcpuval + res = VNode.query.filter_by(name=vnode_name).first() + if res is None: + vnode = VNode(vnode_name) + vnode.histories = [] + db.session.add(vnode) + db.session.commit() + vnode = VNode.query.get(vnode_name) + billing = 0 + cputime = 0 + runtime = 0 + try: + owner = get_owner(vnode_name) + billing = int(workercinfo[vnode_name]['basic_info']['billing']) + cputime = float(workercinfo[vnode_name]['cpu_use']['val']) + runtime = float(workercinfo[vnode_name]['basic_info']['RunningTime']) + except Exception as err: + #print(traceback.format_exc()) + billing = 0 + cputime = 0.0 + runtime = 0 + history = History(action,runtime,cputime,billing) + vnode.histories.append(history) + if action == 'Stop' or action == 'Create': + laststopcpuval[vnode_name] = cputime + vnode.laststopcpuval = cputime + laststopruntime[vnode_name] = runtime + vnode.laststopruntime = runtime + db.session.add(history) + db.session.commit() + + def getHistory(self,vnode_name): + vnode = VNode.query.filter_by(name=vnode_name).first() + if vnode is None: + return [] + else: + res = History.query.filter_by(vnode=vnode_name).all() + return list(eval(str(res))) diff --git a/web/static/js/plot_monitor.js b/web/static/js/plot_monitor.js index 82f45e70..f6bdbce9 100755 --- a/web/static/js/plot_monitor.js +++ b/web/static/js/plot_monitor.js @@ -29,7 +29,7 @@ function processCpuData(data) if(is_running) { cpu_usedp = data.monitor.cpu_use.usedp; - var val = data.monitor.cpu_use.val; + var val = (data.monitor.cpu_use.val).toFixed(2); var unit = data.monitor.cpu_use.unit; var quota = data.monitor.cpu_use.quota.cpu; var quotaout = "("+quota; diff --git a/web/templates/monitor/history.html b/web/templates/monitor/history.html new file mode 100644 index 00000000..4f03ffe1 --- /dev/null +++ b/web/templates/monitor/history.html @@ -0,0 +1,69 @@ +{% extends 'base_AdminLTE.html' %} + +{% block title %}Docklet | History{% endblock %} + +{% block panel_title %}History for
{{ vnode_name }}
{% endblock %} + +{% block panel_list %} +
    +
  1. + Home +
    2. +
  2. + VClusterStatus +
    3. +
  3. + History +
    4. +
+{% endblock %} + +{% block css_src %} + +{% endblock %} + +{% block content %} +
+
+
+
+

History of {{ vnode_name }}

+ +
+ + +
+
+
+ + + + + + + + + + + + + {% for record in history %} + + + + + + + + + {% endfor %} + +
History IDActionRunning TimeCpu TimeBillingAction Time
{{ record['id'] }}{{ record['action'] }}{{ record['runningtime'] }} seconds{{ record['cputime'] }} seconds{{ record['billing'] }} beans{{ record['actionTime'] }}
+ +
+
+
+
+{% endblock %} + diff --git a/web/templates/monitor/hostsConAll.html b/web/templates/monitor/hostsConAll.html index 4ad7c5ee..3280afeb 100644 --- a/web/templates/monitor/hostsConAll.html +++ b/web/templates/monitor/hostsConAll.html @@ -112,7 +112,7 @@

Total Nodes

} $.post(url+"/cpu_use/",{},function(data){ - var val = data.monitor.cpu_use.val; + var val = (data.monitor.cpu_use.val).toFixed(2); var unit = data.monitor.cpu_use.unit; var hostpercent = data.monitor.cpu_use.hostpercent; var percent = (hostpercent*100).toFixed(2); diff --git a/web/templates/monitor/status.html b/web/templates/monitor/status.html index 224d49d2..03c1140a 100644 --- a/web/templates/monitor/status.html +++ b/web/templates/monitor/status.html @@ -88,6 +88,7 @@

VCluster Name: {{ cluster }}

Disk Usage Billing Summary + History @@ -108,6 +109,7 @@

VCluster Name: {{ cluster }}

-- -- Realtime + History {% endfor %} diff --git a/web/web.py b/web/web.py index fcbf1242..952b1031 100755 --- a/web/web.py +++ b/web/web.py @@ -263,6 +263,12 @@ def statusRealtime(vcluster_name,node_name): statusRealtimeView.node_name = node_name return statusRealtimeView.as_view() +@app.route("/vclusters///history/", methods=['GET']) +@login_required +def history(vcluster_name,vnode_name): + historyView.vnode_name = vnode_name + return historyView.as_view() + @app.route("/monitor/hosts///", methods=['POST']) @app.route("/monitor/vnodes///", methods=['POST']) @login_required diff --git a/web/webViews/monitor.py b/web/webViews/monitor.py index e4a655c3..aa4e7c24 100755 --- a/web/webViews/monitor.py +++ b/web/webViews/monitor.py @@ -44,6 +44,19 @@ def get(self): basic_info = result.get('monitor').get('basic_info') return self.render(self.template_path, node_name = self.node_name, user = session['username'], container = basic_info) +class historyView(normalView): + template_path = "monitor/history.html" + vnode_name = "" + + @classmethod + def get(self): + data = { + "user": session['username'], + } + result = dockletRequest.post('/monitor/vnodes/%s/history/'%(self.vnode_name), data) + history = result.get('monitor').get('history') + return self.render(self.template_path, vnode_name = self.vnode_name, user = session['username'], history = history) + class hostsRealtimeView(normalView): template_path = "monitor/hostsRealtime.html" com_ip = ""