Merge pull request #10 from nomadiz/feat/sled

chungquantin · Jan 25, 2023 · 4599b48 · 4599b48
2 parents e27e833 + 7a9a287
commit 4599b48
Show file tree

Hide file tree

Showing 24 changed files with 767 additions and 238 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -39,7 +39,8 @@ border-radius: 10px;" src="https://user-images.githubusercontent.com/56880684/20
 
 ## Features
 
--   Multi embedded database supported: `RocksDB`, `Redb`
+-   Multi embedded database supported: `RocksDB`, `Redb`, `Sled
+-   `
 -   Cross-platform supported: `Windows`, `Linux` and `MacOS`
 -   Custom byte layout deserialization
 -   Execute database command directly in terminal
@@ -48,10 +49,22 @@ border-radius: 10px;" src="https://user-images.githubusercontent.com/56880684/20
 
 ## Roadmap
 
-- [ ] NEW: Universal Key Value Storage support (UKV)
-- [ ] NEW: Sled support
-- [ ] NEW: LevelDB support
-- [ ] Adding consistent mode for editor view
+-   [ ] NEW: Universal Key Value Storage support (UKV)
+-   [x] NEW: Sled support
+-   [ ] NEW: LevelDB support
+-   [ ] Adding consistent mode for editor view
+
+## Supported Storages
+
+EDMA supports multiple databases with easy plug-to-play architecture. Please check below list for supported databases and its features:
+
+| Database name | Description                                                    | EDMA release                                                                | Pull request                                                |
+| ------------- | -------------------------------------------------------------- | --------------------------------------------------------------------------- | ----------------------------------------------------------- |
+| RocksDB       | Support both non-column and column byte data viewer (`COLUMN`) | [v0.1.0-beta.4](https://github.com/nomadiz/edma/releases/tag/v0.1.0-beta.4) | N/A                                                         |
+| ReDB          | Support default database (Will add `TABLE` view)               | [v0.1.0-beta.4](https://github.com/nomadiz/edma/releases/tag/v0.1.0-beta.4) | N/A                                                         |
+| Sled          | Support both non-tree and tree byte data viewer (`TREE`)       | [v0.1.0-beta.5](https://github.com/nomadiz/edma/releases/tag/v0.1.0-beta.5) | [#8 Sled support](https://github.com/nomadiz/edma/issues/8) |
+
+To create a PR for a database integration, please go to [`Issues > New Issue > Feature request`](https://github.com/nomadiz/edma/issues/new?assignees=&labels=&template=feature_request.md&title=)
 
 ## Getting Started
 
@@ -202,6 +215,7 @@ Database name should be these two below
 
 -   `rocksdb`: RocksDB
 -   `redb`: Redb
+-   `sled`: Sled
 
 Database path should be `String` type
 
@@ -246,6 +260,10 @@ Configuration file example
 		{
 			"name": "rocksdb",
 			"path": "/temp"
+		},
+		{
+			"name": "sled",
+			"path": "/temp/sled"
 		}
 	],
 	"templates": [

diff --git a/db/CARGO.md b/db/CARGO.md
@@ -1,67 +1,15 @@
-<p align="center">
-<img src="https://user-images.githubusercontent.com/56880684/201497081-40976107-ef47-4a12-bf6d-ceafc8da3464.png" width="40%"/>
-</p>
-<h3 align="center">A distributed Gremlin-compatible graph database</h3>
+## Embedded database library for EDMA
 
-<h3 align="center">Open for contribution 🚀</h3>
-<br/>
+**NOTE:** (SolomonDB Forked)
 
-<p align="center">
- <a href="https://github.com/nomadiz/solomon-db"><img src="https://img.shields.io/badge/built_with-Rust-dca282.svg?style=flat-square"></a>
-&nbsp;
-<a href="https://github.com/nomadiz/solomon-db"><img src="https://img.shields.io/badge/build%20with-gremlin-green"></a>
-&nbsp;
-<a href="https://github.com/nomadiz/solomon-db"><img src="https://img.shields.io/github/v/release/nomadiz/solomon-db?color=%23ff00a0&include_prereleases&label=version&sort=semver&style=flat-square"></a>
-&nbsp;
-<a href="https://github.com/nomadiz/solomon-db/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT License-00bfff.svg?style=flat-square"></a>
+### Description
 
-</p>
-<p align="center">
-    <a href="https://github.com/nomadiz/solomon-db/graphs/contributors" alt="Contributors">
-        <img src="https://img.shields.io/github/contributors/nomadiz/solomon-db" /></a>
-    <a href="https://github.com/nomadiz/solomon-db/pulse" alt="Activity">
-        &nbsp;
-        <img src="https://img.shields.io/github/commit-activity/m/nomadiz/solomon-db" /></a>
-    <a href="https://users.rust-lang.org/t/solomondb-in-development-gremlin-compatible-graph-database-update/84750" alt="Activity">
-        &nbsp;
-		<img src="https://img.shields.io/badge/Rust%20User%20Forum-follow-orange"/>
-	</a>
-</p>
+Embedded database library that can be installed as Rust crate. This can be used to run an embedded graph database on top of other multiple storage engines
 
-## What is SolomonDB?
+### Storage layer
 
-**SolomonDB** is an open source, distributed, easy-to-use, user friendly graph database built by nomadic engineers. SolomonDB enhances the experience of working with graph database using GQL (Gremlin Query Language). SolomonDB can run in an offline mode and acts as an embedded graph database on top of **RocksDB**. Last but not least, SolomonDB is a database for community. It supports multiple storage layer with a set of plugins for serialization.
-
-## Why is it named "Solomon"?
-
-Solomon is the name of a the wisest person who ever lived, King Solomon. If you have ever read Bible, you might know some stories of King Solomon. One of those stories is when King Solomon asks for wishdom. Based on that idea, Solomon DB is built as my personal side project to gain more knowledge about database internals and graph database architecture.
-
-## Roadmap
-
--   [x] Implement RocskDB storage layer
--   [x] Implement Redb storage layer
--   [ ] Database server
--   [ ] Embedded library
--   [ ] Support Gremlin query language
--   [ ] Multi-row, multi-table ACID transactions
--   [ ] Single-node, or highly-scalable distributed mode
--   [ ] Store structured and unstructured data
--   [ ] Client (JS / Rust / Go) library
-
-## Documentation
-
-For guidance on installation, development, deployment, and administration, see our [SolomonDB documentation](https://nomadiz.github.io/solomon-db/).
-
-The documentation page is built using Rust Mdbook. Shoutout to Rust ecosystem. ❤️
-
-## Community
-
-Join our growing community around the world, for help, ideas, and discussions regarding SurrealDB.
-
--   View SolomonDB [Blog](https://nomadiz.hashnode.dev/)
--   View weekly announcement [Rust language Forum](https://users.rust-lang.org/t/solomondb-gremlin-compatible-graph-database-weekly-update/84750)
-
-Support the creator
-
--   [Github](https://github.com/chungquantin)
--   [Twitter](https://twitter.com/chasechung111)
+| Name        | Type      | Concurrency     | Description                                                                                                        |
+| ----------- | --------- | --------------- | ------------------------------------------------------------------------------------------------------------------ |
+| **RocksDB** | key-value | Multi-threaded  | OptimisticTransactionDB of RocksDB is applied into SolomonDB to allow ACID transaction with multithreaded feature. |
+| **Redb**    | key-value | Single-threaded | Simple use case of Redb is efficient for simple on-disk store.                                                     |
+| **Sled**    | key-value | Single-threaded | The champagne of beta embedded databases                                                                           |
diff --git a/db/Cargo.toml b/db/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "edma_storage"
-version = "0.0.1"
+version = "0.0.2"
 publish = true
 edition = "2021"
 readme = "CARGO.md"
@@ -21,14 +21,16 @@ license = "MIT"
 
 
 [features]
-default = ["kv-redb", "kv-rocksdb", "test-suite"]
+default = ["kv-redb", "kv-rocksdb", "kv-sled", "test-suite"]
+kv-sled = ["dep:sled"]
 kv-rocksdb = ["dep:rocksdb"]
 kv-redb = ["dep:redb"]
 test-suite = []
 debug-suite = []
 
 [dependencies]
 redb = { version = "0.10.0", optional = true }
+sled = { version = "0.34.7", optional = true }
 rocksdb = { version = "0.19.0", optional = true, features = [
  "multi-threaded-cf",
 ] }
@@ -45,6 +47,5 @@ path-absolutize = "3.0.14"
 [dev-dependencies]
 tokio = { version = "1.21.2", features = ["macros", "rt"] }
 
-
 [lib]
 name = "db"
diff --git a/db/src/constant/cf.rs b/db/src/constant/cf.rs
@@ -5,19 +5,10 @@ use lazy_static::lazy_static;
 #[derive(Hash, PartialEq, Eq)]
 pub enum ColumnFamily {
 	TestSuite,
-	Edge,
-	VertexProperty,
-	Property,
-	Vertex,
 }
 
 lazy_static! {
-	pub static ref COLUMN_FAMILIES: HashMap<ColumnFamily, String> = HashMap::from([
-		(ColumnFamily::TestSuite, "test_suite:v1".to_string()),
-		(ColumnFamily::Edge, "edges:v1".to_string()),
-		(ColumnFamily::VertexProperty, "vertex-properties:v1".to_string()),
-		(ColumnFamily::Property, "properties:v1".to_string()),
-		(ColumnFamily::Vertex, "vertices:v1".to_string())
-	]);
-	pub static ref CF_NAMES: Vec<&'static String> = COLUMN_FAMILIES.values().collect();
+	pub static ref KEYSPACES: HashMap<ColumnFamily, String> =
+		HashMap::from([(ColumnFamily::TestSuite, "test_suite:v1".to_string()),]);
+	pub static ref CF_NAMES: Vec<&'static String> = KEYSPACES.values().collect();
 }
diff --git a/db/src/err/mod.rs b/db/src/err/mod.rs
@@ -67,3 +67,10 @@ impl From<redb::Error> for Error {
 		Error::Tx(e.to_string())
 	}
 }
+
+#[cfg(feature = "kv-sled")]
+impl From<sled::Error> for Error {
+	fn from(e: sled::Error) -> Error {
+		Error::Tx(e.to_string())
+	}
+}
diff --git a/db/src/model/tag.rs b/db/src/model/tag.rs
@@ -28,6 +28,10 @@ impl TagBucket {
 		self.0.get(key).cloned()
 	}
 
+	pub fn insert(&mut self, key: TagKey, val: TagValue) {
+		self.0.insert(key, val);
+	}
+
 	pub fn unchecked_get(&self, key: TagKey) -> TagValue {
 		self.0.get(key).unwrap().clone()
 	}

diff --git a/db/src/storage/ds.rs b/db/src/storage/ds.rs
@@ -7,6 +7,7 @@ use super::ReDBAdapter;
 
 #[cfg(feature = "kv-rocksdb")]
 use super::RocksDBAdapter;
+use super::SledAdapter;
 
 #[derive(Copy, Clone)]
 pub struct DatastoreRef<'a> {
@@ -27,6 +28,8 @@ pub enum Inner {
 	RocksDB(RocksDBAdapter),
 	#[cfg(feature = "kv-redb")]
 	ReDB(ReDBAdapter),
+	#[cfg(feature = "kv-sled")]
+	Sled(SledAdapter),
 }
 
 pub struct Datastore {
@@ -43,7 +46,7 @@ impl Datastore {
 	pub fn new(path: &str) -> Datastore {
 		match path {
 			#[cfg(feature = "kv-rocksdb")]
-			s if s.starts_with("default:") | s.starts_with("rocksdb:") | s.eq("default") => {
+			s if s.starts_with("rocksdb:") => {
 				let db = RocksDBAdapter::new(s, None).unwrap();
 
 				Datastore {
@@ -58,6 +61,14 @@ impl Datastore {
 					inner: Inner::ReDB(db),
 				}
 			}
+			#[cfg(feature = "kv-sled")]
+			s if s.starts_with("sled:") => {
+				let db = SledAdapter::new(s).unwrap();
+
+				Datastore {
+					inner: Inner::Sled(db),
+				}
+			}
 			_ => unimplemented!(),
 		}
 	}
@@ -81,7 +92,8 @@ impl Datastore {
 		}
 		impl_transaction_method!(
 			RocksDB feat "kv-rocksdb",
-			ReDB feat "kv-redb"
+			ReDB feat "kv-redb",
+			Sled feat "kv-sled"
 		)
 	}
 
@@ -103,23 +115,49 @@ impl Datastore {
 		}
 		impl_transaction_method!(
 			RocksDB feat "kv-rocksdb",
-			ReDB feat "kv-redb"
+			ReDB feat "kv-redb",
+			Sled feat "kv-sled"
 		)
 	}
 }
 
 #[cfg(test)]
 mod test {
 	use crate::{
-		constant::{ColumnFamily, COLUMN_FAMILIES},
+		constant::{ColumnFamily, KEYSPACES},
 		tag, SimpleTransaction,
 	};
 
 	use super::Datastore;
 
 	#[tokio::test]
-	async fn should_create() {
-		let db = Datastore::new("redb:../temp/v1.redb");
+	async fn should_rocksdb_create_with_cf() {
+		let db = Datastore::new("rocksdb:../temp/cf");
+		assert!(db.transaction(false).await.is_ok());
+
+		// Seeding database
+		let cf_name = KEYSPACES.get(&ColumnFamily::TestSuite).unwrap();
+		let key1 = i32::to_be_bytes(2100);
+		let key2 = "cf => hello world";
+		let key3 = "cf => this is a key";
+
+		let val1 = "cf => mock value";
+		let val2 = "cf => mock value 2";
+		let val3 = "cf => this is a new value";
+
+		let mut tx = db.transaction(true).await.unwrap();
+		let tags = tag!("column_family" => cf_name.clone());
+		tx.set(key1, val1, tags.clone()).await.unwrap();
+		tx.set(key2, val2, tags.clone()).await.unwrap();
+		tx.set(key3, val3, tags.clone()).await.unwrap();
+		let iter = tx.iterate(tags.clone()).await.unwrap();
+		assert!(iter.len() == 3);
+		tx.commit().await.unwrap();
+	}
+
+	#[tokio::test]
+	async fn should_redb_create() {
+		let db = Datastore::new("redb:../temp/redb");
 		assert!(db.transaction(false).await.is_ok());
 
 		let key1 = i32::to_be_bytes(2001);
@@ -134,29 +172,56 @@ mod test {
 		tx.set(key1, val1, tag!()).await.unwrap();
 		tx.set(key2, val2, tag!()).await.unwrap();
 		tx.set(key3, val3, tag!()).await.unwrap();
+		let iter = tx.iterate(tag!()).await.unwrap();
+		assert!(iter.len() == 3);
 		tx.commit().await.unwrap();
 	}
 
 	#[tokio::test]
-	async fn should_create_with_cf() {
-		let db = Datastore::new("rocksdb:../temp/cf");
+	async fn should_sled_create() {
+		let db = Datastore::new("sled:../temp");
 		assert!(db.transaction(false).await.is_ok());
 
 		// Seeding database
-		let cf_name = COLUMN_FAMILIES.get(&ColumnFamily::TestSuite).unwrap();
 		let key1 = i32::to_be_bytes(2100);
-		let key2 = "cf => hello world";
-		let key3 = "cf => this is a key";
+		let key2 = "hello world";
+		let key3 = "this is a key";
 
-		let val1 = "cf => mock value";
-		let val2 = "cf => mock value 2";
-		let val3 = "cf => this is a new value";
+		let val1 = "mock value";
+		let val2 = "mock value 2";
+		let val3 = "this is a new value";
 
 		let mut tx = db.transaction(true).await.unwrap();
-		let tags = tag!("column_family" => cf_name.clone());
+		tx.set(key1, val1, tag!()).await.unwrap();
+		tx.set(key2, val2, tag!()).await.unwrap();
+		tx.set(key3, val3, tag!()).await.unwrap();
+		let iter = tx.iterate(tag!()).await.unwrap();
+		assert!(iter.len() == 3);
+		tx.commit().await.unwrap();
+	}
+
+	#[tokio::test]
+	async fn should_sled_create_tree() {
+		let db = Datastore::new("sled:../temp/cf");
+		assert!(db.transaction(false).await.is_ok());
+
+		// Seeding database
+		let tree_name = KEYSPACES.get(&ColumnFamily::TestSuite).unwrap();
+		let key1 = i32::to_be_bytes(2100);
+		let key2 = "tree => hello world";
+		let key3 = "tree => this is a key";
+
+		let val1 = "tree => mock value";
+		let val2 = "tree => mock value 2";
+		let val3 = "tree => this is a new value";
+
+		let mut tx = db.transaction(true).await.unwrap();
+		let tags = tag!("tree" => tree_name.clone());
 		tx.set(key1, val1, tags.clone()).await.unwrap();
 		tx.set(key2, val2, tags.clone()).await.unwrap();
 		tx.set(key3, val3, tags.clone()).await.unwrap();
+		let iter = tx.iterate(tags.clone()).await.unwrap();
+		assert!(iter.len() == 3);
 		tx.commit().await.unwrap();
 	}
 }
diff --git a/db/src/storage/kvs/mod.rs b/db/src/storage/kvs/mod.rs
@@ -3,7 +3,12 @@ mod redb;
 #[cfg(feature = "kv-rocksdb")]
 mod rocksdb;
 
+#[cfg(feature = "kv-sled")]
+mod sled;
+
 #[cfg(feature = "kv-redb")]
 pub use self::redb::*;
 #[cfg(feature = "kv-rocksdb")]
 pub use self::rocksdb::*;
+#[cfg(feature = "kv-sled")]
+pub use self::sled::*;