From 5f631a77eb20c2d2fffc6164f9db21cecda68348 Mon Sep 17 00:00:00 2001
From: mlv42 <109242388+mlv42@users.noreply.github.com>
Date: Mon, 4 Nov 2024 10:24:50 +0000
Subject: [PATCH] Add files via upload
---
nas_UPDATE.ipynb | 1324 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 1324 insertions(+)
create mode 100644 nas_UPDATE.ipynb
diff --git a/nas_UPDATE.ipynb b/nas_UPDATE.ipynb
new file mode 100644
index 0000000..4d8122a
--- /dev/null
+++ b/nas_UPDATE.ipynb
@@ -0,0 +1,1324 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "c0f84710-fa07-4ae7-a37f-f933992a5183",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "fea5d599-883a-41b7-9084-a2c54697cd6d",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "2024-11-04 10:19:49.248914: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
+ "2024-11-04 10:19:49.546504: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
+ "2024-11-04 10:19:49.548861: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+ "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
+ "2024-11-04 10:19:50.437993: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
+ ]
+ }
+ ],
+ "source": [
+ "import numpy as np\n",
+ "import numpy.random as rnd\n",
+ "import numpy.linalg as la\n",
+ "import polars as pl\n",
+ "import pandas as pd\n",
+ "import datetime as dt\n",
+ "import os\n",
+ "from pathlib import Path\n",
+ "from dask.distributed import Client, LocalCluster, as_completed\n",
+ "from dask import delayed\n",
+ "from runpy import run_path\n",
+ "import matplotlib.pyplot as plt\n",
+ "import seaborn as sns\n",
+ "from tqdm.notebook import tqdm\n",
+ "import networkx as nx\n",
+ "import raphtory as rp\n",
+ "import umap.umap_ as umap\n",
+ "import streamlit as st\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "\n",
+ "import local2global as l2g # ADDED\n",
+ "\n",
+ "import torch\n",
+ "import torch_geometric as tg\n",
+ "from torch_geometric.data import Data\n",
+ "from torch_geometric.utils.convert import from_networkx\n",
+ "from torch_geometric.transforms import LargestConnectedComponents\n",
+ "from torch_geometric.utils import to_networkx, from_networkx, one_hot\n",
+ "from torch_geometric.nn import Node2Vec, GCNConv, VGAE\n",
+ "import torch.nn.functional as F"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "41105a8c-205f-4c34-a8d9-142290acf71a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from datasets import DataLoader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "7acfe164-880a-4aa5-8ffb-db702e78898f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "PATH = \"./datasets/data/nas/\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "148e6778-b3bc-437a-b023-db0d538c05ff",
+ "metadata": {},
+ "source": [
+ "# $\\quad$ Example use: New Autonomous Systems dataset \n",
+ "\n",
+ "$\\newcommand{\\vct}[1]{\\mathbf{#1}}$\n",
+ "$\\newcommand{\\mtx}[1]{\\mathbf{#1}}$\n",
+ "$\\newcommand{\\e}{\\varepsilon}$\n",
+ "$\\newcommand{\\norm}[1]{\\|#1\\|}$\n",
+ "$\\newcommand{\\minimize}{\\mathrm{minimize}\\quad}$\n",
+ "$\\newcommand{\\maximize}{\\mathrm{maximize}\\quad}$\n",
+ "$\\newcommand{\\subjto}{\\quad\\text{subject to}\\quad}$\n",
+ "$\\newcommand{\\R}{\\mathbb{R}}$\n",
+ "$\\newcommand{\\C}{\\mathbb{C}}$\n",
+ "$\\newcommand{\\N}{\\mathbb{N}}$\n",
+ "$\\newcommand{\\Z}{\\mathbb{Z}}$\n",
+ "$\\newcommand{Prob}{\\mathbb{P}}$\n",
+ "$\\newcommand{Expect}{\\mathbb{E}}$\n",
+ "$\\newcommand{Cov}{\\mathrm{Cov}}$\n",
+ "$\\newcommand{Var}{\\mathrm{Var}}$\n",
+ "$\\newcommand{\\trans}{T}$\n",
+ "$\\newcommand{\\ip}[2]{\\langle {#1}, {#2} \\rangle}$\n",
+ "$\\newcommand{\\zerovct}{\\vct{0}}$\n",
+ "$\\newcommand{\\diff}[1]{\\mathrm{d}{#1}}$\n",
+ "$\\newcommand{\\conv}{\\operatorname{conv}}$\n",
+ "$\\newcommand{\\inter}{{\\operatorname{int}}}$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ffc356df-39f0-4f1d-b98d-d4dcfe4665db",
+ "metadata": {},
+ "source": [
+ "The purpose of this notebook is to walk through the graph embedding and alignment process in a self-contained way. The full existing Local2Global package is available [here](https://github.com/LJeub/Local2Global_embedding) and the expectation is to pick parts from it as a starting point. It is also available in on this repository in the Local2Global_embedding folder for reference."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0e4208fd-da21-4181-ae65-1fa8c152ea2c",
+ "metadata": {},
+ "source": [
+ "### Table of Contents\n",
+ "\n",
+ "1. #### Data\n",
+ "2. #### Embedding\n",
+ "3. #### Visualisation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6d903eb7-22ad-4900-8720-f79d3eb44790",
+ "metadata": {},
+ "source": [
+ "### 1. Data "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7d25865e-21a9-4e34-bf80-125bb3d22ca5",
+ "metadata": {},
+ "source": [
+ "The data can be accessed via the dataloader. It is saved in the datasets/data/nas directory in two parquet files. There are many alternative ways of doing this. One option to explore is to have the datasets available as in [torch_geometric datasets](https://pytorch-geometric.readthedocs.io/en/2.6.0/modules/datasets.html)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "38fcc940-390a-4039-aa4b-c136185c16d2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dl = DataLoader(source='nAS')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7dcfd3ee-0e5d-4a5e-aa09-30e0dc958132",
+ "metadata": {},
+ "source": [
+ "The data is stored in one dataframe for the nodes (including all the features) and one for the edges (including edge weights)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "f9273655-0c7c-4f2f-b40a-574d38773307",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "
shape: (5, 5)| timestamp | nodes | nodetype | country | asname |
|---|
| datetime[μs] | str | str | str | str |
| 2024-09-14 00:00:00 | "AS7029" | "asn" | "US" | "WINDSTREAM" |
| 2024-09-14 00:00:00 | "AS32984" | "asn" | "US" | "RUELALA-INC" |
| 2024-09-14 00:00:00 | "AS136106" | "asn" | "ID" | "FIBERSTAR-AS-I" |
| 2024-09-14 00:00:00 | "AS58495" | "asn" | "ID" | "HSPNET-AS-I" |
| 2024-09-14 00:00:00 | "AS3491" | "asn" | "US" | "BTN-ASN" |
\n",
+ "
shape: (5, 4)| timestamp | source | dest | weight |
|---|
| datetime[μs] | str | str | i64 |
| 2024-09-14 00:00:00 | "AS7029" | "AS32984" | 1 |
| 2024-09-14 00:00:00 | "AS7029" | "AS19692" | 1 |
| 2024-09-14 00:00:00 | "AS7029" | "AS55037" | 1 |
| 2024-09-14 00:00:00 | "AS7029" | "AS1820" | 1 |
| 2024-09-14 00:00:00 | "AS7029" | "AS16265" | 1 |