examples/exploratory_data_analysis/stackoverflow/stackoverflow_graph_eda.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stackoverflow Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebooks demonstrates how to use Runway's EDA module with Neo4j's example dataset containing information on Stackoverflow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from neo4j_runway.database.neo4j import Neo4jGraph\n",
    "from neo4j_runway.graph_eda import GraphEDA "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a Neo4j Instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = Neo4jGraph(uri=os.environ.get(\"NEO4J_URI\"), username=os.environ.get(\"NEO4J_USERNAME\"), password=os.environ.get(\"NEO4J_PASSWORD\"), database=\"stackoverflow\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GraphEDA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "eda = GraphEDA(g)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Possible Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['database_indexes',\n",
       " 'database_constraints',\n",
       " 'node_count',\n",
       " 'node_label_counts',\n",
       " 'node_multi_label_counts',\n",
       " 'node_properties',\n",
       " 'relationship_count',\n",
       " 'relationship_type_counts',\n",
       " 'relationship_properties',\n",
       " 'unlabeled_node_count',\n",
       " 'unlabeled_node_ids',\n",
       " 'disconnected_node_count',\n",
       " 'disconnected_node_count_by_label',\n",
       " 'disconnected_node_ids',\n",
       " 'node_degrees']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eda.available_methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can run analytical queries individually via the `GraphEDA` class. For example let's retrieve information on the data constraints."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>type</th>\n",
       "      <th>entityType</th>\n",
       "      <th>labelsOrTypes</th>\n",
       "      <th>properties</th>\n",
       "      <th>ownedIndex</th>\n",
       "      <th>propertyType</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20</td>\n",
       "      <td>constraint_32ea8862</td>\n",
       "      <td>UNIQUENESS</td>\n",
       "      <td>NODE</td>\n",
       "      <td>[Comment]</td>\n",
       "      <td>[uuid]</td>\n",
       "      <td>constraint_32ea8862</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>constraint_401df8db</td>\n",
       "      <td>UNIQUENESS</td>\n",
       "      <td>NODE</td>\n",
       "      <td>[Question]</td>\n",
       "      <td>[uuid]</td>\n",
       "      <td>constraint_401df8db</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>22</td>\n",
       "      <td>constraint_64b1b1cf</td>\n",
       "      <td>UNIQUENESS</td>\n",
       "      <td>NODE</td>\n",
       "      <td>[Tag]</td>\n",
       "      <td>[name]</td>\n",
       "      <td>constraint_64b1b1cf</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19</td>\n",
       "      <td>constraint_7e29bbac</td>\n",
       "      <td>UNIQUENESS</td>\n",
       "      <td>NODE</td>\n",
       "      <td>[Answer]</td>\n",
       "      <td>[uuid]</td>\n",
       "      <td>constraint_7e29bbac</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21</td>\n",
       "      <td>constraint_b13a3b7d</td>\n",
       "      <td>UNIQUENESS</td>\n",
       "      <td>NODE</td>\n",
       "      <td>[User]</td>\n",
       "      <td>[uuid]</td>\n",
       "      <td>constraint_b13a3b7d</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                 name        type entityType labelsOrTypes properties  \\\n",
       "0  20  constraint_32ea8862  UNIQUENESS       NODE     [Comment]     [uuid]   \n",
       "1  18  constraint_401df8db  UNIQUENESS       NODE    [Question]     [uuid]   \n",
       "2  22  constraint_64b1b1cf  UNIQUENESS       NODE         [Tag]     [name]   \n",
       "3  19  constraint_7e29bbac  UNIQUENESS       NODE      [Answer]     [uuid]   \n",
       "4  21  constraint_b13a3b7d  UNIQUENESS       NODE        [User]     [uuid]   \n",
       "\n",
       "            ownedIndex propertyType  \n",
       "0  constraint_32ea8862         None  \n",
       "1  constraint_401df8db         None  \n",
       "2  constraint_64b1b1cf         None  \n",
       "3  constraint_7e29bbac         None  \n",
       "4  constraint_b13a3b7d         None  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eda.database_constraints()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When we run a quering method, the results are appended to an internal cache. By default we return the stored content, but we can choose to refresh the cache by providing `refresh=True`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collecting Insights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can run *all* the analytical queries in the `GraphEDA` class by calling the `run` method.\n",
    "\n",
    "**This can be computationally intensive!**\n",
    "\n",
    "WARNING: The methods in this module can be computationally expensive.\n",
    "It is not recommended to use this module on massive Neo4j databases\n",
    "(i.e., nodes and relationships in the hundreds of millions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "eda.run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have our cache filled, let's see if there are any isolated nodes in the database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eda.disconnected_node_count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No disconnected nodes is a good sign!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can generate a report containing all the information we've gathered from our queries by calling `create_eda_report`. \n",
    "\n",
    "Some of the sections can become quite lengthy, so there are arguments to control the data that is returned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "eda.create_eda_report(include_disconnected_node_ids=True, include_unlabeled_node_ids=True, include_node_degrees=True, view_report=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "\n",
       "# Runway EDA Report\n",
       "\n",
       "## Database Information\n",
       "|    | databaseName   | databaseVersion   | databaseEdition   | APOCVersion   | GDSVersion    |\n",
       "|---:|:---------------|:------------------|:------------------|:--------------|:--------------|\n",
       "|  0 | stackoverflow  | 5.15.0            | enterprise        | 5.15.1        | not installed |\n",
       "\n",
       "### Counts\n",
       "|    | nodeCount   |   unlabeledNodeCount |   disconnectedNodeCount | relationshipCount   |\n",
       "|---:|:------------|---------------------:|------------------------:|:--------------------|\n",
       "|  0 | 6,193       |                    0 |                       0 | 11,540              |\n",
       "\n",
       "### Indexes\n",
       "|    |   id | name                | state   |   populationPercent | type   | entityType   | labelsOrTypes   | properties   | indexProvider    | owningConstraint    | lastRead   | readCount   |\n",
       "|---:|-----:|:--------------------|:--------|--------------------:|:-------|:-------------|:----------------|:-------------|:-----------------|:--------------------|:-----------|:------------|\n",
       "|  0 |   17 | constraint_32ea8862 | ONLINE  |                 100 | RANGE  | NODE         | ['Comment']     | ['uuid']     | range-1.0        | constraint_32ea8862 |            |             |\n",
       "|  1 |   13 | constraint_401df8db | ONLINE  |                 100 | RANGE  | NODE         | ['Question']    | ['uuid']     | range-1.0        | constraint_401df8db |            |             |\n",
       "|  2 |   14 | constraint_64b1b1cf | ONLINE  |                 100 | RANGE  | NODE         | ['Tag']         | ['name']     | range-1.0        | constraint_64b1b1cf |            |             |\n",
       "|  3 |   16 | constraint_7e29bbac | ONLINE  |                 100 | RANGE  | NODE         | ['Answer']      | ['uuid']     | range-1.0        | constraint_7e29bbac |            |             |\n",
       "|  4 |   15 | constraint_b13a3b7d | ONLINE  |                 100 | RANGE  | NODE         | ['User']        | ['uuid']     | range-1.0        | constraint_b13a3b7d |            |             |\n",
       "|  5 |    1 | index_343aff4e      | ONLINE  |                 100 | LOOKUP | NODE         |                 |              | token-lookup-1.0 |                     |            |             |\n",
       "|  6 |    2 | index_f7700477      | ONLINE  |                 100 | LOOKUP | RELATIONSHIP |                 |              | token-lookup-1.0 |                     |            |             |\n",
       "\n",
       "### Constraints\n",
       "|    |   id | name                | type       | entityType   | labelsOrTypes   | properties   | ownedIndex          | propertyType   |\n",
       "|---:|-----:|:--------------------|:-----------|:-------------|:----------------|:-------------|:--------------------|:---------------|\n",
       "|  0 |   20 | constraint_32ea8862 | UNIQUENESS | NODE         | ['Comment']     | ['uuid']     | constraint_32ea8862 |                |\n",
       "|  1 |   18 | constraint_401df8db | UNIQUENESS | NODE         | ['Question']    | ['uuid']     | constraint_401df8db |                |\n",
       "|  2 |   22 | constraint_64b1b1cf | UNIQUENESS | NODE         | ['Tag']         | ['name']     | constraint_64b1b1cf |                |\n",
       "|  3 |   19 | constraint_7e29bbac | UNIQUENESS | NODE         | ['Answer']      | ['uuid']     | constraint_7e29bbac |                |\n",
       "|  4 |   21 | constraint_b13a3b7d | UNIQUENESS | NODE         | ['User']        | ['uuid']     | constraint_b13a3b7d |                |\n",
       "\n",
       "## Nodes Overview\n",
       "### Label Counts\n",
       "|    | label    | count   |\n",
       "|---:|:---------|:--------|\n",
       "|  0 | Question | 1,589   |\n",
       "|  1 | Comment  | 1,396   |\n",
       "|  2 | Answer   | 1,367   |\n",
       "|  3 | User     | 1,365   |\n",
       "|  4 | Tag      | 476     |\n",
       "### Properties\n",
       "|    | nodeLabels   | propertyName       | propertyTypes      | mandatory   |\n",
       "|---:|:-------------|:-------------------|:-------------------|:------------|\n",
       "|  0 | ['User']     | uuid               | ['Long', 'String'] | True        |\n",
       "|  1 | ['User']     | display_name       | ['String']         | True        |\n",
       "|  2 | ['Tag']      | name               | ['String']         | True        |\n",
       "|  3 | ['Tag']      | link               | ['String']         | True        |\n",
       "|  4 | ['Answer']   | uuid               | ['Long']           | True        |\n",
       "|  5 | ['Answer']   | title              | ['String']         | True        |\n",
       "|  6 | ['Answer']   | link               | ['String']         | True        |\n",
       "|  7 | ['Answer']   | is_accepted        | ['Boolean']        | True        |\n",
       "|  8 | ['Answer']   | body_markdown      | ['String']         | True        |\n",
       "|  9 | ['Answer']   | score              | ['Long']           | True        |\n",
       "| 10 | ['Comment']  | uuid               | ['Long']           | True        |\n",
       "| 11 | ['Comment']  | link               | ['String']         | True        |\n",
       "| 12 | ['Comment']  | score              | ['Long']           | True        |\n",
       "| 13 | ['Question'] | uuid               | ['Long']           | True        |\n",
       "| 14 | ['Question'] | title              | ['String']         | True        |\n",
       "| 15 | ['Question'] | creation_date      | ['Long']           | True        |\n",
       "| 16 | ['Question'] | accepted_answer_id | ['Long']           | False       |\n",
       "| 17 | ['Question'] | link               | ['String']         | True        |\n",
       "| 18 | ['Question'] | view_count         | ['Long']           | True        |\n",
       "| 19 | ['Question'] | answer_count       | ['Long']           | True        |\n",
       "| 20 | ['Question'] | body_markdown      | ['String']         | True        |\n",
       "\n",
       "\n",
       "## Relationships Overview\n",
       "### Type Counts\n",
       "|    | relType      | count   |\n",
       "|---:|:-------------|:--------|\n",
       "|  0 | TAGGED       | 4,425   |\n",
       "|  1 | ASKED        | 1,589   |\n",
       "|  2 | COMMENTED_ON | 1,396   |\n",
       "|  3 | COMMENTED    | 1,396   |\n",
       "|  4 | ANSWERED     | 1,367   |\n",
       "|  5 | PROVIDED     | 1,367   |\n",
       "### Properties\n",
       "no relationship properties\n",
       "\n",
       "\n",
       "## Unlabeled Nodes\n",
       "no unlabeled nodes data in cache\n",
       "## Disconnected Nodes\n",
       "no disconnected nodes data in cache\n",
       "## Node Degrees\n",
       "* Top 5 Ordered By outDegree\n",
       "\n",
       "|    |   nodeId | nodeLabel   |   inDegree |   outDegree |\n",
       "|---:|---------:|:------------|-----------:|------------:|\n",
       "|  0 |     5620 | ['User']    |          0 |         318 |\n",
       "|  1 |     2441 | ['User']    |          0 |         193 |\n",
       "|  2 |     2452 | ['User']    |          0 |         178 |\n",
       "|  3 |     2485 | ['User']    |          0 |         144 |\n",
       "|  4 |     2445 | ['User']    |          0 |         138 |\n",
       "---\n",
       "\n",
       "Runway v0.14.0\n",
       "\n",
       "Report Generated @ 2024-11-04 09:52:17.872722\n"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "eda.view_report(notebook=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also save the report to a Markdown file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "eda.save_report(file_name=\"outputs/stackoverflow_runway_report.md\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}