Stable serialise (#1683)

* serialise one node graph with proto and prost

* add minimal support for properties

* add better support for properties and start using ids

* can encode props and encode and decode edges

* added graph const and temporal properties

* add test for edge properties

* add test for edge deletions

* arrange dependencies

* add string interning and add protoc to the build

* add protoc to rust doctest

* try adding github token to binstall and protoc install

* add protoc to benchmarks

* attempt to fix the github token issues with protoc

* more protoc

* Graph and Persistend graph now work with pickle and the format is stable

* move protobuf in a separate feature

* fix the python workflow

* fmt

* add more protoc in yml

* attempt to fix the disk storage

* fix python failing

* fix graphql lib import issues

* change the proto format so it can be easily extended

* try ubuntu 20.04 to avoid 143 error

* some changes as per review

* refactor StableDecode trait

* use assert_graph_equals

* fix compile issues

.github/workflows/_release_python.yml

@@ -24,6 +24,10 @@ jobs:
           permission: "write"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - name: Set up cargo cache
         uses: actions/cache@v3
         continue-on-error: false
@@ -121,6 +125,10 @@ jobs:
           permission: "write"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: actions/checkout@v3
         with:
           ref: ${{ inputs.base }}
@@ -157,6 +165,10 @@ jobs:
           permission: "write"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - name: Set up cargo cache
         uses: actions/cache@v3
         continue-on-error: false

.github/workflows/_release_rust.yml

@@ -38,6 +38,10 @@ jobs:
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: actions/checkout@v3
       - run: |
           git config --global user.email "[email protected]"

.github/workflows/benchmark.yml

@@ -25,6 +25,10 @@ jobs:
         name: Checkout
       - name: Update Rust
         run: rustup update
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - name: Cargo cache
         uses: Swatinem/rust-cache@v2
         with:

.github/workflows/test_python_workflow.yml

@@ -46,6 +46,10 @@ jobs:
           toolchain: 1.77.0
           override: true
           components: rustfmt, clippy
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: webfactory/[email protected]
         name: Load raphtory-disk_graph key
         with:

.github/workflows/test_rust_disk_storage_workflow.yml

@@ -14,11 +14,12 @@ jobs:
     runs-on: '${{ matrix.os }}'
     env:
       CARGO_NET_GIT_FETCH_WITH_CLI: true
+      RUST_BACKTRACE: 1
     strategy:
       matrix:
         include:
           - os: macos-latest
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
           - os: windows-latest
     steps:
       - uses: maxim-lobanov/setup-xcode@v1
@@ -36,7 +37,7 @@ jobs:
           override: true
           components: rustfmt, clippy
       - name: Free up space (ubuntu)
-        if: matrix.os == 'ubuntu-latest'
+        if: "contains(matrix.os, 'ubuntu')"
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /usr/local/lib/android
@@ -52,10 +53,12 @@ jobs:
         name: Cargo cache
         with:
           cache-all-crates: true
-      - name: Install bininstall
-        uses: cargo-bins/cargo-binstall@main
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - name: Install nextest
-        run: cargo binstall -y --force cargo-nextest
+        uses: taiki-e/install-action@nextest
       - name: Activate pometry-storage in Cargo.toml
         run: make pull-storage
       - name: Run all Tests (disk_graph)

.github/workflows/test_rust_workflow.yml

@@ -14,11 +14,12 @@ jobs:
     runs-on: '${{ matrix.os }}'
     env:
       CARGO_NET_GIT_FETCH_WITH_CLI: true
+      RUST_BACKTRACE: 1
     strategy:
       matrix:
         include:
           - os: macos-latest
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
           - os: windows-latest
     steps:
       - uses: maxim-lobanov/setup-xcode@v1
@@ -36,12 +37,16 @@ jobs:
           override: true
           components: rustfmt, clippy
       - name: Free up space (ubuntu)
-        if: matrix.os == 'ubuntu-latest'
+        if: "contains(matrix.os, 'ubuntu')"
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /opt/ghc
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: webfactory/[email protected]
         name: Load pometry-storage key
         with:
@@ -52,10 +57,8 @@ jobs:
         name: Cargo cache
         with:
           cache-all-crates: true
-      - name: Install bininstall
-        uses: cargo-bins/cargo-binstall@main
       - name: Install nextest
-        run: cargo binstall -y --force cargo-nextest
+        uses: taiki-e/install-action@nextest
       - name: Run all Tests (no disk_graph)
         env:
           RUSTFLAGS: -Awarnings
@@ -80,6 +83,10 @@ jobs:
           toolchain: 1.77.0
           override: true
           components: rustfmt, clippy
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: Swatinem/rust-cache@v2
         name: Cargo cache
         with:

Cargo.toml

@@ -127,6 +127,11 @@ url = "2.2"
 base64-compat = { package = "base64-compat", version = "1.0.0" }
 time = "0.3.36"
 
+prost = "0.12"
+prost-types = "0.12"
+bytes = "1.6.0"
+prost-build = "0.12"
+
 lazy_static = "1.4.0"
 pest = "2.7.8"
 pest_derive = "2.7.8"

python/Cargo.toml

@@ -19,7 +19,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { workspace = true }
-raphtory_core = { path = "../raphtory", version = "0.9.3", features = ["python", "search", "vectors"], package = "raphtory" }
+raphtory_core = { path = "../raphtory", version = "0.9.3", features = ["python", "search", "vectors", "proto"], package = "raphtory" }
 raphtory-graphql = { path = "../raphtory-graphql", version = "0.9.3",features = ["python"] }
 serde_json = { workspace = true }
 reqwest = { workspace = true }

python/tests/test_graphdb.py

@@ -19,6 +19,7 @@
 import os
 import shutil
 import numpy as np
+import pickle
 
 base_dir = Path(__file__).parent
 edges = [(1, 1, 2), (2, 1, 3), (-1, 2, 1), (0, 1, 1), (7, 3, 2), (1, 1, 1)]
@@ -59,6 +60,29 @@ def test_graph_len_edge_len():
     assert g.count_edges() == 5
 
 
+def test_graph_pickle():
+    g = create_graph()
+    # pickle graph
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        pickle.dump(g, open(tmpdirname + "/graph.p", "wb"))
+        # unpickle graph
+        g2 = pickle.load(open(tmpdirname + "/graph.p", "rb"))
+
+        assert g2.count_nodes() == 3
+        assert g2.count_edges() == 5
+
+
+def test_persistent_graph_pickle():
+    g = create_graph_with_deletions()
+    # pickle graph
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        pickle.dump(g, open(tmpdirname + "/graph.p", "wb"))
+        # unpickle graph
+        g2 = pickle.load(open(tmpdirname + "/graph.p", "rb"))
+
+        assert g2.count_nodes() == 3
+        assert g2.count_edges() == 5
+
 def test_id_iterable():
     g = create_graph()
 

raphtory-graphql/src/lib.rs

@@ -53,7 +53,10 @@ mod graphql_test {
     };
     use serde_json::json;
     use std::collections::{HashMap, HashSet};
-    use tempfile::{tempdir, TempDir};
+    use tempfile::tempdir;
+
+    #[cfg(feature = "storage")]
+    use tempfile::TempDir;
 
     #[tokio::test]
     async fn search_for_gandalf_query() {
@@ -266,7 +269,6 @@ mod graphql_test {
 
         let req = Request::new(prop_has_key_filter);
         let res = schema.execute(req).await;
-        let data = res.data.into_json().unwrap();
         let expected = json!({
             "graph": {
               "properties": {