Merge pull request #177 from FirmlyReality/history

History

src/container.py

@@ -5,6 +5,7 @@
 from log import logger
 import env
 from lvmtool import sys_run, check_volume
+from monitor import History_Manager
 
 class Container(object):
     def __init__(self, addr, etcdclient):
@@ -20,6 +21,7 @@ def __init__(self, addr, etcdclient):
 
         self.lxcpath = "/var/lib/lxc"
         self.imgmgr = imagemgr.ImageMgr()
+        self.historymgr = History_Manager()
 
     def create_container(self, lxc_name, username, setting, clustername, clusterid, containerid, hostname, ip, gateway, vlanid, image):
         logger.info("create container %s of %s for %s" %(lxc_name, clustername, username))
@@ -130,11 +132,13 @@ def config_prepare(content):
         except Exception as e:
             logger.error(e)
             return [False, "create container failed"]
+        self.historymgr.log(lxc_name,"Create")
         return [True, "create container success"]
 
     def delete_container(self, lxc_name):
         logger.info ("delete container:%s" % lxc_name)
         if self.imgmgr.deleteFS(lxc_name):
+            self.historymgr.log(lxc_name,"Delete")
             logger.info("delete container %s success" % lxc_name)
             return [True, "delete container success"]
         else:
@@ -164,6 +168,7 @@ def start_container(self, lxc_name):
             subprocess.run(["lxc-start -n %s" % lxc_name],
                     stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, check=True)
             logger.info ("start container %s success" % lxc_name)
+            self.historymgr.log(lxc_name,"Start")
             return [True, "start container success"]
         except subprocess.CalledProcessError as sube:
             logger.error('start container %s failed: %s' % (lxc_name,
@@ -209,6 +214,7 @@ def recover_container(self, lxc_name):
         if status == 'stopped':
             logger.info("%s stopped, recover it to running" % lxc_name)
             if self.start_container(lxc_name)[0]:
+                self.historymgr.log(lxc_name,"Recover")
                 if self.start_services(lxc_name)[0]:
                     logger.info("%s recover success" % lxc_name)
                     return [True, "recover success"]
@@ -235,6 +241,7 @@ def stop_container(self, lxc_name):
             logger.error("stop container %s failed" % lxc_name)
             return [False, "stop container failed"]
         else:
+            self.historymgr.log(lxc_name,"Stop")
             logger.info("stop container %s success" % lxc_name)
             return [True, "stop container success"]
         #if int(status) == 1:

src/httprest.py

@@ -423,6 +423,7 @@ def hosts_monitor(cur_user, user, form, com_id, issue):
 @login_required
 def vnodes_monitor(cur_user, user, form, con_id, issue):
     global G_clustername
+    global G_historymgr
     logger.info("handle request: monitor/vnodes")
     res = {}
     fetcher = monitor.Container_Fetcher(con_id)
@@ -434,6 +435,8 @@ def vnodes_monitor(cur_user, user, form, con_id, issue):
         res['disk_use'] = fetcher.get_disk_use()
     elif issue == 'basic_info':
         res['basic_info'] = fetcher.get_basic_info()
+    elif issue == 'history':
+        res['history'] = G_historymgr.getHistory(con_id)
     elif issue == 'owner':
         names = con_id.split('-')
         result = G_usermgr.query(username = names[0], cur_user = cur_user)
@@ -801,6 +804,7 @@ def internal_server_error(error):
     global G_networkmgr
     global G_clustername
     global G_sysmgr
+    global G_historymgr
     # move 'tools.loadenv' to the beginning of this file
 
     fs_path = env.getenv("FS_PREFIX")
@@ -876,7 +880,7 @@ def internal_server_error(error):
         if etcdclient.isdir("_lock")[0]:
             etcdclient.deldir("_lock")
 
-    G_usermgr = userManager.userManager('root')
+    G_usermgr = userManager.userManager('root','unias1616')
     if mode == "new":
         G_usermgr.initUsage()
     G_notificationmgr = notificationmgr.NotificationMgr()
@@ -896,8 +900,10 @@ def internal_server_error(error):
     logger.info("vclustermgr started")
     G_imagemgr = imagemgr.ImageMgr()
     logger.info("imagemgr started")
+    G_historymgr = monitor.History_Manager()
     master_collector = monitor.Master_Collector(G_nodemgr)
     master_collector.start()
+    logger.info("master_collector started")
 
     logger.info("startting to listen on: ")
     masterip = env.getenv('MASTER_IP')

src/model.py

@@ -40,6 +40,7 @@
 
 app = Flask(__name__)
 app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+fsdir+'/global/sys/UserTable.db'
+app.config['SQLALCHEMY_BINDS'] = {'history': 'sqlite:///'+fsdir+'/global/sys/HistoryTable.db'}
 try:
     secret_key_file = open(env.getenv('FS_PREFIX') + '/local/token_secret_key.txt')
     app.secret_key = secret_key_file.read()
@@ -184,3 +185,40 @@ def __init__(self, notification_id, group_name):
 
     def __repr__(self):
         return '<Notification: %r, Group: %r>' % (self.notification_id, self.group_name)
+
+class VNode(db.Model):
+    __bind_key__ = 'history'
+    name = db.Column(db.String(100), primary_key=True)
+    laststopcpuval = db.Column(db.Float)
+    laststopruntime = db.Column(db.Integer)
+    billing = db.Column(db.Integer)
+    histories = db.relationship('History', backref='v_node', lazy='dynamic')
+
+    def __init__(self, vnode_name):
+        self.name = vnode_name
+        self.laststopcpuval = 0
+        self.billing = 0
+        self.laststopruntime = 0
+
+    def __repr__(self):
+        return '<Vnodes %s>' % (self.name)
+
+class History(db.Model):
+    __bind_key__ = 'history'
+    id = db.Column(db.Integer, primary_key=True)
+    vnode = db.Column(db.String(100), db.ForeignKey('v_node.name'))
+    action = db.Column(db.String(30))
+    runningtime = db.Column(db.Integer)
+    cputime = db.Column(db.Float)
+    billing = db.Column(db.Integer)
+    actionTime = db.Column(db.DateTime)
+
+    def __init__(self, action, runningtime, cputime, billing):
+        self.action = action
+        self.runningtime = runningtime
+        self.cputime = cputime
+        self.billing = billing
+        self.actionTime = datetime.now()
+
+    def __repr__(self):
+        return "{\"id\":\"%d\",\"vnode\":\"%s\",\"action\":\"%s\",\"runningtime\":\"%d\",\"cputime\":\"%f\",\"billing\":\"%d\",\"actionTime\":\"%s\"}" % (self.id, self.vnode, self.action, self.runningtime, self.cputime, self.billing, self.actionTime.strftime("%Y-%m-%d %H:%M:%S"))

src/monitor.py

@@ -3,6 +3,7 @@
 import subprocess,re,os,etcdlib,psutil,math,sys
 import time,threading,json,traceback,platform
 
+from model import db,VNode,History
 from log import logger
 
 monitor_hosts = {}
@@ -13,12 +14,16 @@
 containerpids = []
 pid2name = {}
 
+laststopcpuval = {}
+laststopruntime = {}
 lastbillingtime = {}
 increment = {}
 
 class Container_Collector(threading.Thread):
 
     def __init__(self,test=False):
+        global laststopcpuval
+        global workercinfo
         threading.Thread.__init__(self)
         self.thread_stop = False
         self.interval = 2
@@ -28,6 +33,20 @@ def __init__(self,test=False):
         self.cpu_quota = {}
         self.mem_quota = {}
         self.cores_num = int(subprocess.getoutput("grep processor /proc/cpuinfo | wc -l"))
+        containers = self.list_container()
+        for container in containers:
+            if not container == '':
+                try:
+                    vnode = VNode.query.get(container)
+                    laststopcpuval[container] = vnode.laststopcpuval
+                    laststopruntime[container] = vnode.laststopruntime
+                    workercinfo[container] = {}
+                    workercinfo[container]['basic_info'] = {}
+                    workercinfo[container]['basic_info']['billing'] = vnode.billing
+                    workercinfo[container]['basic_info']['RunningTime'] = vnode.laststopruntime
+                except:
+                    laststopcpuval[container] = 0
+                    laststopruntime[container] = 0
         return
 
     def list_container(self):
@@ -57,6 +76,8 @@ def collect_containerinfo(self,container_name):
         global lastbillingtime
         global containerpids
         global pid2name
+        global laststopcpuval
+        global laststopruntime
         output = subprocess.check_output("sudo lxc-info -n %s" % (container_name),shell=True)
         output = output.decode('utf-8')
         parts = re.split('\n',output)
@@ -67,7 +88,6 @@ def collect_containerinfo(self,container_name):
             basic_info = workercinfo[container_name]['basic_info']
         else:
             basic_info['RunningTime'] = 0
-            basic_info['LastTime'] = 0
             basic_info['billing'] = 0
         for part in parts:
             if not part == '':
@@ -87,20 +107,13 @@ def collect_containerinfo(self,container_name):
             containerpids.append(info['PID'])
             pid2name[info['PID']] = container_name
         running_time = self.get_proc_etime(int(info['PID']))
-        if basic_exist and 'PID' in workercinfo[container_name]['basic_info'].keys():
-            last_time = workercinfo[container_name]['basic_info']['LastTime']
-            if not info['PID'] == workercinfo[container_name]['basic_info']['PID']:
-                last_time = workercinfo[container_name]['basic_info']['RunningTime']
-        else:
-            last_time = 0
-        basic_info['LastTime'] = last_time
-        running_time += last_time
+        running_time += laststopruntime[container_name]
         basic_info['PID'] = info['PID']
         basic_info['IP'] = info['IP']
         basic_info['RunningTime'] = running_time
 
         cpu_parts = re.split(' +',info['CPU use'])
-        cpu_val = cpu_parts[0].strip()
+        cpu_val = float(cpu_parts[0].strip())
         cpu_unit = cpu_parts[1].strip()
         if not container_name in self.cpu_last.keys():
             confpath = "/var/lib/lxc/%s/config"%(container_name)
@@ -127,6 +140,12 @@ def collect_containerinfo(self,container_name):
                 return False
             self.cpu_last[container_name] = 0 
         cpu_use = {}
+        lastval = 0
+        try:
+            lastval = laststopcpuval[container_name]
+        except:
+            logger.warning(traceback.format_exc())   
+        cpu_val += lastval
         cpu_use['val'] = cpu_val
         cpu_use['unit'] = cpu_unit
         cpu_usedp = (float(cpu_val)-float(self.cpu_last[container_name]))/(self.cpu_quota[container_name]*self.interval*1.05)
@@ -139,7 +158,7 @@ def collect_containerinfo(self,container_name):
 
         if container_name not in increment.keys():
             increment[container_name] = {}
-            increment[container_name]['lastcputime'] = 0
+            increment[container_name]['lastcputime'] = cpu_val
             increment[container_name]['memincrement'] = 0
 
         mem_parts = re.split(' +',info['Memory use'])
@@ -158,13 +177,10 @@ def collect_containerinfo(self,container_name):
         mem_use['usedp'] = mem_usedp
         workercinfo[container_name]['mem_use'] = mem_use
 
-        lasttime = 0
-        if container_name in lastbillingtime.keys():
-            lasttime = lastbillingtime[container_name]
-        else:
-            lasttime = 0
-            lastbillingtime[container_name] = 0
-        #logger.info(running_time)
+        if not container_name in lastbillingtime.keys():
+            lastbillingtime[container_name] = int(running_time/self.billingtime)
+        lasttime = lastbillingtime[container_name]
+        #logger.info(lasttime)
         if not int(running_time/self.billingtime) == lasttime:
             #logger.info("billing:"+str(float(cpu_val)))
             lastbillingtime[container_name] = int(running_time/self.billingtime)
@@ -183,6 +199,16 @@ def collect_containerinfo(self,container_name):
             #logger.info("cpu_increment:"+str(cpu_increment)+" avemem:"+str(avemem)+" disk:"+str(disk_quota)+"\n")
             billing = cpu_increment/1000.0 + avemem/500000.0 + float(disk_quota)/1024.0/1024.0/2000
             basic_info['billing'] += math.ceil(billing)
+            try:
+                vnode = VNode.query.get(container_name)
+                vnode.billing = basic_info['billing']
+                db.session.commit()
+            except Exception as err:
+                vnode = VNode(container_name)
+                vnode.billing = basic_info['billing']
+                db.session.add(vnode)
+                db.session.commit()
+                logger.warning(err)
         workercinfo[container_name]['basic_info'] = basic_info
         #print(output)
         #print(parts)
@@ -403,6 +429,8 @@ def run(self):
                     logger.warning(traceback.format_exc())
                     logger.warning(err)
             time.sleep(2)
+            #logger.info(History.query.all())
+            #logger.info(VNode.query.all())
         return
 
     def stop(self):
@@ -556,3 +584,56 @@ def get_containerslist(self):
             logger.warning(err)
             res = {}
         return res
+
+class History_Manager:
+
+    def __init__(self):
+        try:
+            VNode.query.all()
+            History.query.all()
+        except:
+            db.create_all(bind='__all__')
+
+    def getAll(self):
+        return History.query.all()
+
+    def log(self,vnode_name,action):
+        global workercinfo
+        global laststopcpuval
+        res = VNode.query.filter_by(name=vnode_name).first()
+        if res is None:
+            vnode = VNode(vnode_name)
+            vnode.histories = []
+            db.session.add(vnode)
+            db.session.commit()
+        vnode = VNode.query.get(vnode_name)
+        billing = 0
+        cputime = 0
+        runtime = 0
+        try:
+            owner = get_owner(vnode_name)
+            billing = int(workercinfo[vnode_name]['basic_info']['billing'])
+            cputime = float(workercinfo[vnode_name]['cpu_use']['val'])
+            runtime = float(workercinfo[vnode_name]['basic_info']['RunningTime'])
+        except Exception as err:
+            #print(traceback.format_exc())
+            billing = 0
+            cputime = 0.0
+            runtime = 0
+        history = History(action,runtime,cputime,billing)
+        vnode.histories.append(history)
+        if action == 'Stop' or action == 'Create':
+            laststopcpuval[vnode_name] = cputime
+            vnode.laststopcpuval = cputime
+            laststopruntime[vnode_name] = runtime
+            vnode.laststopruntime = runtime
+        db.session.add(history)
+        db.session.commit()
+
+    def getHistory(self,vnode_name):
+        vnode = VNode.query.filter_by(name=vnode_name).first()
+        if vnode is None:
+            return []
+        else:
+            res = History.query.filter_by(vnode=vnode_name).all()
+            return list(eval(str(res)))

web/static/js/plot_monitor.js

@@ -29,7 +29,7 @@ function processCpuData(data)
     if(is_running)
     {
 	    cpu_usedp = data.monitor.cpu_use.usedp;
-	    var val = data.monitor.cpu_use.val;
+	    var val = (data.monitor.cpu_use.val).toFixed(2);
 	    var unit = data.monitor.cpu_use.unit;
         var quota = data.monitor.cpu_use.quota.cpu;
         var quotaout = "("+quota;