Disk merge

.github/workflows/benchmark.yml

@@ -47,8 +47,7 @@ jobs:
       - name: Run benchmark (Unix)
         run: |
           set -o pipefail
-          cargo bench --bench base -p raphtory-benchmark -- --output-format=bencher | tee benchmark-result.txt
-          cargo bench --bench algobench -p raphtory-benchmark -- --output-format=bencher | tee benchmark-result.txt
+          cargo bench --bench base --bench algobench -p raphtory-benchmark -- --output-format=bencher | tee benchmark-result.txt
       - name: Delete cargo.lock if it exists
         run: |
           rm -f Cargo.lock

.github/workflows/test_rust_disk_storage_workflow.yml

@@ -18,9 +18,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: macos-latest
-          - os: ubuntu-20.04
-          - os: windows-latest
+          - { os: macos-latest, flags: "" }
+          - { os: ubuntu-20.04, flags: "-C link-arg=-fuse-ld=lld" }
+          - { os: windows-latest, flags: "" }
     steps:
       - uses: maxim-lobanov/setup-xcode@v1
         name: Xcode version
@@ -43,6 +43,10 @@ jobs:
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /opt/ghc
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+      - name: Install LLD
+        if: "contains(matrix.os, 'ubuntu')"
+        run: |
+          sudo apt-get install lld
       - uses: webfactory/[email protected]
         name: Load pometry-storage key
         with:
@@ -56,14 +60,14 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}        
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install nextest
         uses: taiki-e/install-action@nextest
       - name: Activate pometry-storage in Cargo.toml
         run: make pull-storage
       - name: Run all Tests (disk_graph)
         env:
-          RUSTFLAGS: -Awarnings
+          RUSTFLAGS: -Awarnings ${{ matrix.flags }}
           TEMPDIR: ${{ runner.temp }}
         run: |
           cargo nextest run --all --no-default-features --features "storage"

Cargo.toml

@@ -84,7 +84,7 @@ display-error-chain = "0.2.0"
 polars-arrow = "0.39.2"
 polars-parquet = "0.39.2"
 polars-utils = "0.39.2"
-kdam = { version = "0.5.1", features = ["notebook"] }
+kdam = { version = "0.5.1" }
 pretty_assertions = "1.4.0"
 quickcheck = "1.0.3"
 quickcheck_macros = "1.0.0"

examples/rust/Cargo.toml

@@ -30,10 +30,3 @@ name = "crypto"
 
 [[bin]]
 name = "pokec"
-
-[target.x86_64-unknown-linux-gnu]
-linker = "/usr/bin/clang"
-rustflags = ["-Clink-arg=-fuse-ld=lld", "-Clink-arg=-Wl,--no-rosegment"]
-
-[profile.release]
-debug = true

python/src/lib.rs

@@ -1,5 +1,3 @@
-#![allow(non_local_definitions)]
-
 extern crate core;
 use pyo3::prelude::*;
 use raphtory_core::python::packages::base_modules::{

raphtory-cypher/examples/raphtory_cypher.rs

@@ -17,7 +17,7 @@ mod cypher {
     use arrow::util::pretty::print_batches;
     use clap::Parser;
     use futures::{stream, StreamExt};
-    use raphtory::disk_graph::graph_impl::{DiskGraph, ParquetLayerCols};
+    use raphtory::disk_graph::{graph_impl::ParquetLayerCols, DiskGraph};
     use raphtory_cypher::{run_cypher, run_cypher_to_streams, run_sql};
     use serde::{de::DeserializeOwned, Deserialize};
 

raphtory-cypher/src/executor/table_provider/edge.rs

@@ -28,7 +28,7 @@ use datafusion::{
 };
 use futures::Stream;
 use pometry_storage::prelude::*;
-use raphtory::disk_graph::graph_impl::DiskGraph;
+use raphtory::disk_graph::DiskGraph;
 
 use crate::executor::{arrow2_to_arrow_buf, ExecError};
 
@@ -99,7 +99,7 @@ impl EdgeListTableProvider {
 
 fn lift_nested_arrow_schema(graph: &DiskGraph, layer_id: usize) -> Result<Arc<Schema>, ExecError> {
     let arrow2_fields = graph.as_ref().layer(layer_id).edges_data_type();
-    let a2_dt = crate::arrow2::datatypes::ArrowDataType::Struct(arrow2_fields.clone());
+    let a2_dt = crate::arrow2::datatypes::ArrowDataType::Struct(arrow2_fields.to_vec());
     let a_dt: DataType = a2_dt.into();
     let schema = match a_dt {
         DataType::Struct(fields) => {

raphtory-cypher/src/executor/table_provider/node.rs

@@ -24,7 +24,7 @@ use futures::Stream;
 use pometry_storage::properties::ConstProps;
 use raphtory::{
     core::entities::VID,
-    disk_graph::{graph_impl::DiskGraph, prelude::*},
+    disk_graph::{prelude::*, DiskGraph},
 };
 use std::{any::Any, fmt::Formatter, sync::Arc};
 

raphtory-cypher/src/hop/execution.rs

@@ -38,17 +38,16 @@ use datafusion::{
 use datafusion::physical_expr::Partitioning;
 use futures::{Stream, StreamExt};
 
+use crate::take_record_batch;
 use pometry_storage::graph_fragment::TempColGraphFragment;
 use raphtory::{
     core::{entities::VID, Direction},
     disk_graph::{
-        graph_impl::DiskGraph,
         prelude::{ArrayOps, BaseArrayOps, PrimitiveCol},
+        DiskGraph,
     },
 };
 
-use crate::take_record_batch;
-
 use super::operator::HopPlan;
 
 #[derive(Debug)]

raphtory-cypher/src/hop/operator.rs

@@ -5,7 +5,7 @@ use datafusion::{
     logical_expr::{Expr, LogicalPlan, TableScan, UserDefinedLogicalNodeCore},
 };
 
-use raphtory::{core::Direction, disk_graph::graph_impl::DiskGraph};
+use raphtory::{core::Direction, disk_graph::DiskGraph};
 
 #[derive(Debug, PartialEq, Hash, Eq)]
 pub struct HopPlan {

raphtory-cypher/src/hop/rule.rs

@@ -1,5 +1,6 @@
 use std::sync::Arc;
 
+use crate::hop::operator::HopPlan;
 use async_trait::async_trait;
 use datafusion::{
     common::Column,
@@ -10,9 +11,7 @@ use datafusion::{
     physical_plan::ExecutionPlan,
     physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner},
 };
-use raphtory::{core::Direction, disk_graph::graph_impl::DiskGraph};
-
-use crate::hop::operator::HopPlan;
+use raphtory::{core::Direction, disk_graph::DiskGraph};
 
 use super::execution::HopExec;
 
@@ -141,7 +140,7 @@ impl ExtensionPlanner for HopPlanner {
 #[cfg(test)]
 mod test {
     use arrow::util::pretty::print_batches;
-    use raphtory::disk_graph::graph_impl::DiskGraph;
+    use raphtory::disk_graph::DiskGraph;
     use tempfile::tempdir;
 
     use crate::prepare_plan;

raphtory-cypher/src/lib.rs

@@ -34,7 +34,7 @@ mod cypher {
         parser::ast::*,
         *,
     };
-    use raphtory::disk_graph::graph_impl::DiskGraph;
+    use raphtory::disk_graph::DiskGraph;
 
     use crate::{
         executor::table_provider::node::NodeTableProvider,
@@ -185,9 +185,8 @@ mod cypher {
         use arrow_array::RecordBatch;
         use tempfile::tempdir;
 
-        use raphtory::{disk_graph::graph_impl::DiskGraph, prelude::*};
-
         use crate::run_cypher;
+        use raphtory::{disk_graph::DiskGraph, prelude::*};
 
         lazy_static::lazy_static! {
             static ref EDGES: Vec<(u64, u64, i64, f64)> = vec![
@@ -278,10 +277,9 @@ mod cypher {
                 datatypes::*,
             };
             use arrow::util::pretty::print_batches;
+            use raphtory::disk_graph::{graph_impl::ParquetLayerCols, DiskGraph};
             use tempfile::tempdir;
 
-            use raphtory::disk_graph::graph_impl::{DiskGraph, ParquetLayerCols};
-
             use crate::run_cypher;
 
             fn schema() -> ArrowSchema {

raphtory-cypher/src/transpiler/mod.rs

@@ -14,7 +14,7 @@ use raphtory::{
         Direction,
     },
     db::{api::properties::internal::ConstPropertiesOps, graph::node::NodeView},
-    disk_graph::graph_impl::DiskGraph,
+    disk_graph::DiskGraph,
     prelude::*,
 };
 use sqlparser::ast::{
@@ -1143,18 +1143,15 @@ fn sql_like(
 
 #[cfg(test)]
 mod test {
-
     use crate::{parser, transpiler};
-
-    use super::*;
+    use pretty_assertions::assert_eq;
     use raphtory::{
         db::{api::mutation::AdditionOps, graph::graph::Graph},
+        disk_graph::DiskGraph,
         prelude::NO_PROPS,
     };
     use tempfile::tempdir;
 
-    use pretty_assertions::assert_eq;
-
     #[test]
     fn count_all_nodes() {
         check_cypher_to_sql(

raphtory-graphql/src/data.rs

@@ -1,6 +1,6 @@
 use parking_lot::RwLock;
 #[cfg(feature = "storage")]
-use raphtory::disk_graph::graph_impl::DiskGraph;
+use raphtory::disk_graph::DiskGraph;
 use raphtory::{
     core::Prop,
     db::api::view::MaterializedGraph,

raphtory-graphql/src/lib.rs

@@ -46,7 +46,7 @@ mod graphql_test {
     use async_graphql::UploadValue;
     use dynamic_graphql::{Request, Variables};
     #[cfg(feature = "storage")]
-    use raphtory::disk_graph::graph_impl::DiskGraph;
+    use raphtory::disk_graph::DiskGraph;
     use raphtory::{
         db::{api::view::IntoDynamic, graph::views::deletion_graph::PersistentGraph},
         prelude::*,

raphtory/Cargo.toml

@@ -124,8 +124,8 @@ python = [
     "dep:display-error-chain",
     "polars-arrow?/compute",
     "raphtory-api/python",
-    "dep:kdam",
     "dep:rpds",
+    "kdam?/notebook"
 ]
 
 # storage
@@ -145,7 +145,8 @@ storage = [
 arrow = [
     "dep:polars-arrow",
     "dep:polars-parquet",
-    "polars-parquet?/compression"
+    "polars-parquet?/compression",
+    "dep:kdam",
 ]
 
 proto = [

raphtory/src/core/utils/errors.rs

@@ -95,11 +95,11 @@ pub enum GraphError {
         source: std::io::Error,
     },
 
-    #[cfg(feature = "python")]
+    #[cfg(feature = "arrow")]
     #[error("Failed to load graph: {0}")]
     LoadFailure(String),
 
-    #[cfg(feature = "python")]
+    #[cfg(feature = "arrow")]
     #[error(
         "Failed to load graph as the following columns are not present within the dataframe: {0}"
     )]

raphtory/src/db/api/properties/props.rs

@@ -8,7 +8,7 @@ use raphtory_api::core::storage::arc_str::ArcStr;
 use std::collections::HashMap;
 
 /// View of the properties of an entity (graph|node|edge)
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct Properties<P: PropertiesOps + Clone> {
     pub(crate) props: P,
 }

raphtory/src/db/api/view/internal/materialize.rs

@@ -42,7 +42,7 @@ use serde::{de::Error, Deserialize, Deserializer, Serialize};
 use std::path::Path;
 
 #[cfg(feature = "storage")]
-use crate::disk_graph::graph_impl::DiskGraph;
+use crate::disk_graph::DiskGraph;
 
 #[enum_dispatch(CoreGraphOps)]
 #[enum_dispatch(InternalLayerOps)]

raphtory/src/db/graph/node.rs

@@ -36,6 +36,7 @@ use chrono::{DateTime, Utc};
 use raphtory_api::core::storage::arc_str::ArcStr;
 use std::{
     fmt,
+    fmt::Debug,
     hash::{Hash, Hasher},
     sync::Arc,
 };
@@ -72,15 +73,12 @@ impl<G, GH> AsNodeRef for NodeView<G, GH> {
     }
 }
 
-impl<'graph, G, GH: GraphViewOps<'graph>> fmt::Debug for NodeView<G, GH> {
+impl<'graph, G, GH: GraphViewOps<'graph> + Debug> fmt::Debug for NodeView<G, GH> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "NodeView {{ graph: {}{}, node: {} }}",
-            self.graph.count_nodes(),
-            self.graph.count_edges(),
-            self.node.0
-        )
+        f.debug_struct("NodeView")
+            .field("node", &self.node)
+            .field("graph", &self.graph)
+            .finish()
     }
 }
 

raphtory/src/disk_graph/graph_impl/const_properties_ops.rs

@@ -1,7 +1,7 @@
 use crate::{db::api::properties::internal::ConstPropertiesOps, prelude::Prop};
 use raphtory_api::core::storage::arc_str::ArcStr;
 
-use super::DiskGraph;
+use crate::disk_graph::DiskGraph;
 
 impl ConstPropertiesOps for DiskGraph {
     #[doc = " Find id for property name (note this only checks the meta-data, not if the property actually exists for the entity)"]

raphtory/src/disk_graph/graph_impl/core_ops.rs

@@ -22,18 +22,18 @@ use crate::{
         view::{internal::CoreGraphOps, BoxedIter},
     },
     disk_graph::{
-        graph_impl::DiskGraph,
         storage_interface::{
             edge::DiskOwnedEdge,
             edges::DiskEdges,
             node::{DiskNode, DiskOwnedNode},
             nodes::DiskNodesOwned,
         },
+        DiskGraph,
     },
 };
 use itertools::Itertools;
 use polars_arrow::datatypes::ArrowDataType;
-use pometry_storage::{properties::ConstProps, GidRef, GID};
+use pometry_storage::{properties::ConstProps, GidRef};
 use raphtory_api::core::{input::input_node::InputNode, storage::arc_str::ArcStr};
 use rayon::prelude::*;
 
@@ -121,8 +121,8 @@ impl CoreGraphOps for DiskGraph {
     fn internalise_node(&self, v: NodeRef) -> Option<VID> {
         match v {
             NodeRef::Internal(vid) => Some(vid),
-            NodeRef::External(vid) => self.inner.find_node(&GID::U64(vid)),
-            NodeRef::ExternalStr(string) => self.inner.find_node(&GID::Str(string.into())),
+            NodeRef::External(vid) => self.inner.find_node(GidRef::U64(vid)),
+            NodeRef::ExternalStr(string) => self.inner.find_node(GidRef::Str(string)),
         }
     }
 

raphtory/src/disk_graph/graph_impl/edge_filter_ops.rs

@@ -1,7 +1,7 @@
-use super::DiskGraph;
 use crate::{
     core::entities::LayerIds,
     db::api::{storage::edges::edge_ref::EdgeStorageRef, view::internal::EdgeFilterOps},
+    disk_graph::DiskGraph,
 };
 
 impl EdgeFilterOps for DiskGraph {

raphtory/src/disk_graph/graph_impl/layer_ops.rs

@@ -1,7 +1,7 @@
-use super::DiskGraph;
 use crate::{
     core::{entities::LayerIds, utils::errors::GraphError},
     db::api::view::internal::InternalLayerOps,
+    disk_graph::DiskGraph,
     prelude::Layer,
 };
 use itertools::Itertools;

raphtory/src/disk_graph/graph_impl/list_ops.rs

@@ -1,6 +1,6 @@
 use crate::{
     db::api::view::internal::{CoreGraphOps, EdgeList, ListOps, NodeList},
-    disk_graph::graph_impl::DiskGraph,
+    disk_graph::DiskGraph,
 };
 use rayon::prelude::*;