notebooks/Step 2 - Heterogeneous Network Regularization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameters\n",
    "\n",
    "# input\n",
    "network = 'dados-brutos-tomcat.csv.network'\n",
    "reg_data = 'dados-brutos-tomcat.csv.labels'\n",
    "\n",
    "# output\n",
    "reg_model = 'dados-brutos-tomcat.csv.reg.model'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import sys\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    " \n",
    "\n",
    "def network_labeling(graph, labels):\n",
    "    print('Reading '+labels)\n",
    "    with open(labels) as fileobj:\n",
    "        for line in fileobj:  \n",
    "            data = line.split('\\t')\n",
    "            nodeName = data[0]\n",
    "            if(len(data)!=2): continue\n",
    "            nodeKey = data[0].strip()\n",
    "            f=(list(map(float,data[1].split(','))))\n",
    "            if (nodeKey not in graph): continue\n",
    "            f = np.array(f)\n",
    "            graph[nodeKey].setLabeled(True)\n",
    "            graph[nodeKey].setF(f)\n",
    "    print('Reading '+labels+': OK')\n",
    "    \n",
    "def get_dimension(labels):\n",
    "    dim = -1\n",
    "    with open(labels) as fileobj:\n",
    "        for line in fileobj:  \n",
    "            data = line.split('\\t')\n",
    "            nodeName = data[0]\n",
    "            if(len(data)!=2): continue\n",
    "            nodeKey = data[0].strip()\n",
    "            f=(list(map(float,data[1].split(','))))\n",
    "            dim = len(f)\n",
    "            break\n",
    "    return dim\n",
    "\n",
    "def network_builder(network, labels, directed=False):\n",
    "    index = 0\n",
    "    graph = {}\n",
    "    numNodes = 0\n",
    "    numEdges = 0\n",
    "    dimensions = get_dimension(labels)\n",
    "    print('Reading '+network)\n",
    "    with open(network) as fileobj:\n",
    "        for line in fileobj:  \n",
    "\n",
    "            data = line.split('\\t')\n",
    "            if(len(data)!=3): continue\n",
    "            \n",
    "            ln = data[0].split(':')\n",
    "            nodeSourceName = ln[0].strip()\n",
    "            nodeSourceType = ln[1].strip()\n",
    "            \n",
    "            ln = data[1].split(':')\n",
    "            nodeTargetName = ln[0].strip()\n",
    "            nodeTargetType = ln[1].strip()\n",
    "            \n",
    "            edgeWeight = int(data[2].strip())\n",
    "            \n",
    "            nodeKeySource = nodeSourceName+\":\"+nodeSourceType\n",
    "            if nodeKeySource not in graph:\n",
    "                node = Node(nodeSourceName, nodeSourceType)\n",
    "                node.setF([0]*dimensions)\n",
    "                graph[nodeKeySource] = node\n",
    "                numNodes+=1\n",
    "            \n",
    "            nodeKeyTarget = nodeTargetName+\":\"+nodeTargetType\n",
    "            if nodeKeyTarget not in graph:\n",
    "                node = Node(nodeTargetName, nodeTargetType)\n",
    "                node.setF([0]*dimensions)\n",
    "                graph[nodeKeyTarget] = node\n",
    "                numNodes+=1\n",
    "            \n",
    "            if(directed):\n",
    "                graph[nodeKeySource].addOut(nodeKeyTarget, edgeWeight)\n",
    "                graph[nodeKeyTarget].addIn(nodeKeySource, edgeWeight)\n",
    "            else:\n",
    "                graph[nodeKeySource].addOut(nodeKeyTarget, edgeWeight)\n",
    "                graph[nodeKeyTarget].addIn(nodeKeySource, edgeWeight)\n",
    "                graph[nodeKeySource].addIn(nodeKeyTarget, edgeWeight)\n",
    "                graph[nodeKeyTarget].addOut(nodeKeySource, edgeWeight)\n",
    "            \n",
    "            numEdges+=1\n",
    "    \n",
    "    network_labeling(graph,labels)\n",
    "    \n",
    "    print('Reading '+network+': OK')\n",
    "    print('#Nodes: '+str(numNodes))\n",
    "    print('#Edges: '+str(numEdges))\n",
    "    \n",
    "    return graph\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "    \n",
    "import numpy as np\n",
    "class Node:\n",
    "    \n",
    "    def __init__(self, nodeName, nodeType):\n",
    "        self.nodeName = nodeName\n",
    "        self.nodeType = nodeType\n",
    "        self.inNeighbors = {}\n",
    "        self.outNeighbors = {}\n",
    "        self.labeled = False\n",
    "        self.f = None\n",
    "        self.stats = {}\n",
    "\n",
    "    def addIn(self, nodeKey, weight):\n",
    "        self.inNeighbors[nodeKey]=weight\n",
    "        \n",
    "    def addOut(self, nodeKey, weight):\n",
    "        self.outNeighbors[nodeKey]=weight\n",
    "        \n",
    "    def getIn(self):\n",
    "        return self.inNeighbors\n",
    "        \n",
    "    def getOut(self):\n",
    "        return self.outNeighbors\n",
    "    \n",
    "    def getInDegree(self):\n",
    "        if ('inDegree' not in self.stats):\n",
    "            self.stats['inDegree'] = float(len(self.getIn().keys()))\n",
    "        return self.stats['inDegree']\n",
    "        \n",
    "    def getOutDegree(self):\n",
    "        if ('outDegree' not in self.stats):\n",
    "            self.stats['outDegree'] = float(len(self.getOut().keys()))\n",
    "        return self.stats['outDegree']\n",
    "    \n",
    "    def getWeightedInDegree(self):\n",
    "        if ('weightedInDegree' not in self.stats):\n",
    "            self.stats['weightedInDegree'] = float(0)\n",
    "            for nodeKey in self.getIn():\n",
    "                self.stats['weightedInDegree'] += self.getIn()[nodeKey]\n",
    "        return self.stats['weightedInDegree']\n",
    "        \n",
    "    def getWeightedOutDegree(self):\n",
    "        if ('weightedOutDegree' not in self.stats):\n",
    "            self.stats['weightedOutDegree'] = float(0)\n",
    "            for nodeKey in self.getOut():\n",
    "                self.stats['weightedOutDegree'] += self.getOut()[nodeKey]\n",
    "        return self.stats['weightedOutDegree']\n",
    "    \n",
    "    def setLabeled(self,b):\n",
    "        self.labeled=b\n",
    "\n",
    "    def isLabeled(self):\n",
    "        return self.labeled\n",
    "        \n",
    "    def setF(self,f):\n",
    "        self.f=np.array(f)\n",
    "    \n",
    "    def getF(self):\n",
    "        return self.f\n",
    "\n",
    "\n",
    "class SimpleRegularizer:\n",
    "    \n",
    "    def __init__(self, graph, min_iterations, max_iterations, convergence):\n",
    "        self.graph = graph\n",
    "        self.min_iterations = min_iterations\n",
    "        self.max_iterations = max_iterations\n",
    "        self.convergence = convergence\n",
    "        \n",
    "\n",
    "    \n",
    "    def walking(self):\n",
    "        nodes = list(self.graph.keys())\n",
    "        iteration = 1\n",
    "        while(True):\n",
    "            loss = 0\n",
    "            random.shuffle(nodes)\n",
    "            for nodeKey in nodes:\n",
    "                loss += self.propagate(nodeKey)\n",
    "\n",
    "            print('Iteration '+str(iteration)+\" | Loss=\"+str(loss))\n",
    "            if(iteration >= self.max_iterations and iteration > self.min_iterations): break\n",
    "            if(loss < self.convergence and iteration > self.min_iterations): break\n",
    "            iteration+=1\n",
    "\n",
    "        # last iteration for labeled data\n",
    "        for nodeKey in nodes:\n",
    "            graph = self.graph\n",
    "            if(graph[nodeKey].isLabeled()):\n",
    "                self.propagate(nodeKey,lastIteration=True)\n",
    "\n",
    "    \n",
    "    def propagate(self,nodeKey,lastIteration=False):\n",
    "        graph = self.graph\n",
    "        loss = 0\n",
    "        \n",
    "        # regularizer\n",
    "        if(graph[nodeKey].isLabeled() and lastIteration==False): return 0\n",
    "        \n",
    "        f = graph[nodeKey].getF()\n",
    "        #print(f)\n",
    "        f_new = np.array([0]*len(f))\n",
    "        \n",
    "        counter = 0\n",
    "        for nodeIn in graph[nodeKey].getIn():\n",
    "            \n",
    "            penalty = graph[nodeIn].getOutDegree()\n",
    "            if(penalty < 1): penalty=1.0\n",
    "            if(graph[nodeIn].isLabeled()): penalty = 1.0\n",
    "            \n",
    "            f_new = f_new + (graph[nodeIn].getF()/penalty)\n",
    "            \n",
    "            counter += 1\n",
    "            \n",
    "        f_new = f_new/counter\n",
    "        #print(f_new)\n",
    "        #print(\"====\")\n",
    "        loss = np.sum(np.abs(f - f_new))\n",
    "        graph[nodeKey].setF(f_new)\n",
    "        \n",
    "        return loss\n",
    "    \n",
    "    def saveModel(self,output):\n",
    "        \n",
    "        print(\"Writing model \"+output)\n",
    "        nodes = list(self.graph.keys())\n",
    "        \n",
    "        num_items = len(nodes)\n",
    "        dimension = -1\n",
    "        for nodeKey in nodes:\n",
    "            f = graph[nodeKey].getF()\n",
    "            dimension = len(f)\n",
    "            break\n",
    "        \n",
    "        if os.path.exists(output): os.remove(output)\n",
    "        \n",
    "        file = open(output, 'a')\n",
    "        file.write(str(num_items)+\" \"+str(dimension)+\"\\n\")\n",
    "        \n",
    "        for nodeKey in nodes:\n",
    "            f = graph[nodeKey].getF()\n",
    "            s = (str(list(f)).replace('[','').replace(']','').replace(' ','').replace(',',' '))\n",
    "            file.write(nodeKey.replace(' ','_')+\" \"+s+\"\\n\")\n",
    "\n",
    "        file.close()\n",
    "        print(\"Writing model \"+output+\": OK\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading dados-brutos-tomcat.csv.network\n",
      "Reading dados-brutos-tomcat.csv.labels\n",
      "Reading dados-brutos-tomcat.csv.labels: OK\n",
      "Reading dados-brutos-tomcat.csv.network: OK\n",
      "#Nodes: 9804\n",
      "#Edges: 135671\n",
      "Iteration 1 | Loss=81235.51490616487\n",
      "Iteration 2 | Loss=1862.2250610408112\n",
      "Iteration 3 | Loss=51.385717052507324\n",
      "Iteration 4 | Loss=2.6259099638168926\n",
      "Iteration 5 | Loss=0.19900384025927495\n",
      "Iteration 6 | Loss=0.015595830114272275\n",
      "Iteration 7 | Loss=0.0007813593283762033\n",
      "Iteration 8 | Loss=3.8221035008144616e-05\n",
      "Iteration 9 | Loss=2.8801129771268035e-06\n",
      "Iteration 10 | Loss=2.864127489884507e-07\n",
      "Iteration 11 | Loss=1.1929061465632959e-08\n",
      "Writing model dados-brutos-tomcat.csv.reg.model\n",
      "Writing model dados-brutos-tomcat.csv.reg.model: OK\n"
     ]
    }
   ],
   "source": [
    "graph = network_builder(network,reg_data,directed=False)\n",
    "\n",
    "regularizer = SimpleRegularizer(graph,10,30,0.0001)\n",
    "regularizer.walking()\n",
    "regularizer.saveModel(reg_model)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}