RocksDb storage plugin

.github/workflows/dotnetcore.yml

@@ -20,5 +20,8 @@ jobs:
       run: |
         dotnet tool install --tool-path ./ dotnet-format
         ./dotnet-format --check --dry-run -v diagnostic
-    - name: Build
-      run: dotnet build
+    - name: Test
+      run: |
+        sudo apt-get --assume-yes install libleveldb-dev libsnappy-dev libc6-dev
+        find tests -name *.csproj | xargs -I % dotnet add % package coverlet.msbuild
+        dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=${GITHUB_WORKSPACE}/coverage/lcov

LevelDBStore/Plugins/Storage/LevelDBStore.cs

@@ -8,7 +8,7 @@ public class LevelDBStore : Plugin, IStoragePlugin
 
         protected override void Configure()
         {
-            path = string.Format(GetConfiguration().GetSection("Path").Value, ProtocolSettings.Default.Magic.ToString("X8"));
+            path = string.Format(GetConfiguration().GetSection("Path").Value ?? "Data_LevelDB_{0}", ProtocolSettings.Default.Magic.ToString("X8"));
         }
 
         public IStore GetStore()

LevelDBStore/Plugins/Storage/Store.cs

@@ -1,4 +1,4 @@
-using Neo.IO.Data.LevelDB;
+using Neo.IO.Data.LevelDB;
 using Neo.Persistence;
 using System;
 using System.Collections.Generic;
@@ -18,15 +18,23 @@ public Store(string path)
             byte[] value = db.Get(ReadOptions.Default, Helper.CreateKey(SYS_Version));
             if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0"))
                 return;
+
             WriteBatch batch = new WriteBatch();
-            ReadOptions options = new ReadOptions { FillCache = false };
-            using (Iterator it = db.NewIterator(options))
+
+            if (value != null)
             {
-                for (it.SeekToFirst(); it.Valid(); it.Next())
+                // Clean all entries only if the version are different
+
+                ReadOptions options = new ReadOptions { FillCache = false };
+                using (Iterator it = db.NewIterator(options))
                 {
-                    batch.Delete(it.Key());
+                    for (it.SeekToFirst(); it.Valid(); it.Next())
+                    {
+                        batch.Delete(it.Key());
+                    }
                 }
             }
+
             db.Put(WriteOptions.Default, Helper.CreateKey(SYS_Version), Encoding.ASCII.GetBytes(Assembly.GetExecutingAssembly().GetName().Version.ToString()));
             db.Write(WriteOptions.Default, batch);
         }

RocksDBStore/Plugins/Storage/Options.cs

@@ -0,0 +1,25 @@
+using RocksDbSharp;
+
+namespace Neo.Plugins.Storage
+{
+    public static class Options
+    {
+        public static readonly DbOptions Default = CreateDbOptions();
+        public static readonly ReadOptions ReadDefault = new ReadOptions();
+        public static readonly WriteOptions WriteDefault = new WriteOptions();
+        public static readonly WriteOptions WriteDefaultSync = new WriteOptions().SetSync(true);
+
+        public static DbOptions CreateDbOptions()
+        {
+            DbOptions options = new DbOptions();
+            options.SetCreateMissingColumnFamilies(true);
+            options.SetCreateIfMissing(true);
+            options.SetErrorIfExists(false);
+            options.SetMaxOpenFiles(1000);
+            options.SetParanoidChecks(false);
+            options.SetWriteBufferSize(4 << 20);
+            options.SetBlockBasedTableFactory(new BlockBasedTableOptions().SetBlockSize(4096));
+            return options;
+        }
+    }
+}

RocksDBStore/Plugins/Storage/RocksDBStore.cs

@@ -0,0 +1,21 @@
+using Neo.Persistence;
+
+namespace Neo.Plugins.Storage
+{
+    public class RocksDBStore : Plugin, IStoragePlugin
+    {
+        /// <summary>
+        /// Configure
+        /// </summary>
+        public override void Configure()
+        {
+            Settings.Load(GetConfiguration());
+        }
+
+        /// <summary>
+        /// Get store
+        /// </summary>
+        /// <returns>RocksDbStore</returns>
+        public IStore GetStore() => new Store(Settings.Default.Path);
+    }
+}

RocksDBStore/Plugins/Storage/Settings.cs

@@ -0,0 +1,21 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Neo.Plugins.Storage
+{
+    internal class Settings
+    {
+        public string Path { get; }
+
+        public static Settings Default { get; private set; }
+
+        private Settings(IConfigurationSection section)
+        {
+            this.Path = string.Format(section.GetSection("Path").Value ?? "Data_RocksDB_{0}", ProtocolSettings.Default.Magic.ToString("X8"));
+        }
+
+        public static void Load(IConfigurationSection section)
+        {
+            Default = new Settings(section);
+        }
+    }
+}

RocksDBStore/Plugins/Storage/Snapshot.cs

@@ -0,0 +1,67 @@
+using Neo.Persistence;
+using RocksDbSharp;
+using System;
+using System.Collections.Generic;
+
+namespace Neo.Plugins.Storage
+{
+    internal class Snapshot : ISnapshot
+    {
+        private readonly Store store;
+        private readonly RocksDb db;
+        private readonly RocksDbSharp.Snapshot snapshot;
+        private readonly WriteBatch batch;
+        private readonly ReadOptions options;
+
+        public Snapshot(Store store, RocksDb db)
+        {
+            this.store = store;
+            this.db = db;
+            this.snapshot = db.CreateSnapshot();
+            this.batch = new WriteBatch();
+
+            options = new ReadOptions();
+            options.SetFillCache(false);
+            options.SetSnapshot(snapshot);
+        }
+
+        public void Commit()
+        {
+            db.Write(batch, Options.WriteDefault);
+        }
+
+        public void Delete(byte table, byte[] key)
+        {
+            batch.Delete(key, store.GetFamily(table));
+        }
+
+        public void Put(byte table, byte[] key, byte[] value)
+        {
+            batch.Put(key, value, store.GetFamily(table));
+        }
+
+        public IEnumerable<(byte[] Key, byte[] Value)> Find(byte table, byte[] prefix)
+        {
+            using var it = db.NewIterator(store.GetFamily(table), options);
+            for (it.Seek(prefix); it.Valid(); it.Next())
+            {
+                var key = it.Key();
+                byte[] y = prefix;
+                if (key.Length < y.Length) break;
+                if (!key.AsSpan().StartsWith(y)) break;
+                yield return (key, it.Value());
+            }
+        }
+
+        public byte[] TryGet(byte table, byte[] key)
+        {
+            return db.Get(key, store.GetFamily(table), options);
+        }
+
+        public void Dispose()
+        {
+            snapshot.Dispose();
+            batch.Dispose();
+        }
+    }
+}

RocksDBStore/Plugins/Storage/Store.cs

@@ -0,0 +1,117 @@
+using Neo.Persistence;
+using RocksDbSharp;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Neo.Plugins.Storage
+{
+    internal class Store : IStore
+    {
+        private static readonly byte[] SYS_Version = { 0xf0 };
+        private readonly RocksDb db;
+        private readonly Dictionary<byte, ColumnFamilyHandle> _families = new Dictionary<byte, ColumnFamilyHandle>();
+
+        public Store(string path)
+        {
+            var families = new ColumnFamilies();
+            for (int x = 0; x <= byte.MaxValue; x++)
+                families.Add(new ColumnFamilies.Descriptor(x.ToString(), new ColumnFamilyOptions()));
+            db = RocksDb.Open(Options.Default, Path.GetFullPath(path), families);
+
+            ColumnFamilyHandle defaultFamily = db.GetDefaultColumnFamily();
+            byte[] value = db.Get(SYS_Version, defaultFamily, Options.ReadDefault);
+            if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0"))
+                return;
+
+            if (value != null)
+            {
+                // Clean all families only if the version are different
+
+                Parallel.For(0, byte.MaxValue + 1, (x) => db.DropColumnFamily(x.ToString()));
+                _families.Clear();
+            }
+
+            // Update version
+
+            db.Put(SYS_Version, Encoding.ASCII.GetBytes(Assembly.GetExecutingAssembly().GetName().Version.ToString()), defaultFamily, Options.WriteDefault);
+        }
+
+        public void Dispose()
+        {
+            db.Dispose();
+            _families.Clear();
+        }
+
+        /// <summary>
+        /// Get family
+        /// </summary>
+        /// <param name="table">Table</param>
+        /// <returns>Return column family</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public ColumnFamilyHandle GetFamily(byte table)
+        {
+            if (!_families.TryGetValue(table, out var family))
+            {
+                try
+                {
+                    // Try to find the family
+
+                    family = db.GetColumnFamily(table.ToString());
+                    _families.Add(table, family);
+                }
+                catch (KeyNotFoundException)
+                {
+                    // Try to create the family
+
+                    family = db.CreateColumnFamily(new ColumnFamilyOptions(), table.ToString());
+                    _families.Add(table, family);
+                }
+            }
+
+            return family;
+        }
+
+        public ISnapshot GetSnapshot()
+        {
+            return new Snapshot(this, db);
+        }
+
+        public IEnumerable<(byte[] Key, byte[] Value)> Find(byte table, byte[] prefix)
+        {
+            using var it = db.NewIterator(GetFamily(table), Options.ReadDefault);
+            for (it.Seek(prefix); it.Valid(); it.Next())
+            {
+                var key = it.Key();
+                byte[] y = prefix;
+                if (key.Length < y.Length) break;
+                if (!key.AsSpan().StartsWith(y)) break;
+                yield return (key, it.Value());
+            }
+        }
+
+        public byte[] TryGet(byte table, byte[] key)
+        {
+            return db.Get(key, GetFamily(table), Options.ReadDefault);
+        }
+
+        public void Delete(byte table, byte[] key)
+        {
+            db.Remove(key, GetFamily(table), Options.WriteDefault);
+        }
+
+        public void Put(byte table, byte[] key, byte[] value)
+        {
+            db.Put(key, value, GetFamily(table), Options.WriteDefault);
+        }
+
+        public void PutSync(byte table, byte[] key, byte[] value)
+        {
+            db.Put(key, value, GetFamily(table), Options.WriteDefaultSync);
+        }
+    }
+}

RocksDBStore/RocksDBStore.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <Version>3.0.0-CI00810</Version>
+    <TargetFramework>netstandard2.1</TargetFramework>
+    <RootNamespace>Neo.Plugins.Storage</RootNamespace>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Update="RocksDBStore\config.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
+    </None>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Neo" Version="3.0.0-CI00810" />
+    <PackageReference Include="RocksDbNative" Version="6.2.2" />
+    <PackageReference Include="RocksDbSharp" Version="6.2.2" />
+  </ItemGroup>
+
+</Project>

RocksDBStore/RocksDBStore/config.json

@@ -0,0 +1,5 @@
+{
+  "PluginConfiguration": {
+    "Path": "Data_RocksDB_{0}"
+  }
+}

neo-plugins.sln

@@ -21,6 +21,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SystemLog", "SystemLog\Syst
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LevelDBStore", "LevelDBStore\LevelDBStore.csproj", "{C66214CD-0B97-4EA5-B7A2-164F54346F19}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RocksDBStore", "RocksDBStore\RocksDBStore.csproj", "{0E2AAF05-C55A-4B36-8750-F55743FBE4B3}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{59D802AB-C552-422A-B9C3-64D329FBCDCC}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "neo-plugins.Tests", "tests\neo-plugins.Tests\neo-plugins.Tests.csproj", "{9E7EA895-302A-4C0C-BA9B-54F9A67AD75C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -63,10 +69,21 @@ Global
 		{C66214CD-0B97-4EA5-B7A2-164F54346F19}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C66214CD-0B97-4EA5-B7A2-164F54346F19}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C66214CD-0B97-4EA5-B7A2-164F54346F19}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0E2AAF05-C55A-4B36-8750-F55743FBE4B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0E2AAF05-C55A-4B36-8750-F55743FBE4B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0E2AAF05-C55A-4B36-8750-F55743FBE4B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0E2AAF05-C55A-4B36-8750-F55743FBE4B3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9E7EA895-302A-4C0C-BA9B-54F9A67AD75C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9E7EA895-302A-4C0C-BA9B-54F9A67AD75C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9E7EA895-302A-4C0C-BA9B-54F9A67AD75C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9E7EA895-302A-4C0C-BA9B-54F9A67AD75C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{9E7EA895-302A-4C0C-BA9B-54F9A67AD75C} = {59D802AB-C552-422A-B9C3-64D329FBCDCC}
+	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {61D3ADE6-BBFC-402D-AB42-1C71C9F9EDE3}
 	EndGlobalSection