Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: introduce txid->blockhash index on IStore #1328

Merged
merged 10 commits into from
Jun 14, 2021
Merged

feat: introduce txid->blockhash index on IStore #1328

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1c17540
build: ignore tests projects for codecov
kfangw Jun 4, 2021
55d652f
feat: introduce txid->blockhash index on IStore
kfangw Jun 3, 2021
88a28f6
fix: remove chain id for the APIs
kfangw Jun 7, 2021
bc9e9ed
fix: change key
kfangw Jun 8, 2021
dc30fc4
chore: update changelog
kfangw Jun 8, 2021
3db3827
test: improve code coverage
kfangw Jun 8, 2021
cdf2e8d
fix: introduce separated db for the index
kfangw Jun 9, 2021
5f51ee4
chore: codecov ignore
kfangw Jun 9, 2021
0783fe8
chore: ignore tests, benchmarks
kfangw Jun 10, 2021
a15bf0d
Merge branch 'main' into feat/txbindex
kfangw Jun 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,14 @@ To be released.
- Removed `Transaction<T>.EvaluateActions()` method.
- Parameters `action`, `inputContext`, and `outputStates` for
`ActionEvaluation()` constructor can no longer be `null`. [[#1305]]
- Added `IStore.PutTxIdBlockHashIndex(TxId, BlockHash)` method.
[[#1294], [#1328]]
- Added `IStore.GetFirstTxIdBlockHashIndex(TxId)` method.
[[#1294], [#1328]]
- Added `IStore.DeleteTxIdBlockHashIndex(TxId, BlockHash)` method.
[[#1294], [#1328]]
- Added `IStore.IterateTxIdBlockHashIndex(TxId)` method.
[[#1294], [#1328]]

### Backward-incompatible network protocol changes

Expand Down Expand Up @@ -278,10 +286,12 @@ To be released.
[#1285]: https://github.com/planetarium/libplanet/issues/1285
[#1287]: https://github.com/planetarium/libplanet/pull/1287
[#1289]: https://github.com/planetarium/libplanet/pull/1289
[#1294]: https://github.com/planetarium/libplanet/issues/1294
[#1298]: https://github.com/planetarium/libplanet/pull/1298
[#1301]: https://github.com/planetarium/libplanet/issues/1301
[#1305]: https://github.com/planetarium/libplanet/pull/1305
[#1325]: https://github.com/planetarium/libplanet/pull/1325
[#1328]: https://github.com/planetarium/libplanet/pull/1328


Version 0.11.1
Expand Down
12 changes: 12 additions & 0 deletions Libplanet.Explorer/Store/LiteDBRichStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,18 @@ public void PutTxExecution(TxFailure txFailure) =>
public TxExecution GetTxExecution(BlockHash blockHash, TxId txid) =>
_store.GetTxExecution(blockHash, txid);

public void PutTxIdBlockHashIndex(TxId txId, BlockHash blockHash) =>
_store.PutTxIdBlockHashIndex(txId, blockHash);

public BlockHash? GetFirstTxIdBlockHashIndex(TxId txId) =>
_store.GetFirstTxIdBlockHashIndex(txId);

public IEnumerable<BlockHash> IterateTxIdBlockHashIndex(TxId txId) =>
_store.IterateTxIdBlockHashIndex(txId);

public void DeleteTxIdBlockHashIndex(TxId txId, BlockHash blockHash) =>
_store.DeleteTxIdBlockHashIndex(txId, blockHash);

public DateTimeOffset? GetBlockPerceivedTime(BlockHash blockHash)
{
return _store.GetBlockPerceivedTime(blockHash);
Expand Down
12 changes: 12 additions & 0 deletions Libplanet.Explorer/Store/MySQLRichStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,18 @@ public void PutTxExecution(TxFailure txFailure) =>
public TxExecution GetTxExecution(BlockHash blockHash, TxId txid) =>
_store.GetTxExecution(blockHash, txid);

public void PutTxIdBlockHashIndex(TxId txId, BlockHash blockHash) =>
_store.PutTxIdBlockHashIndex(txId, blockHash);

public BlockHash? GetFirstTxIdBlockHashIndex(TxId txId) =>
_store.GetFirstTxIdBlockHashIndex(txId);

public IEnumerable<BlockHash> IterateTxIdBlockHashIndex(TxId txId) =>
_store.IterateTxIdBlockHashIndex(txId);

public void DeleteTxIdBlockHashIndex(TxId txId, BlockHash blockHash) =>
_store.DeleteTxIdBlockHashIndex(txId, blockHash);

public DateTimeOffset? GetBlockPerceivedTime(BlockHash blockHash) =>
_store.GetBlockPerceivedTime(blockHash);

Expand Down
34 changes: 34 additions & 0 deletions Libplanet.RocksDBStore/MonoRocksDBStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ public class MonoRocksDBStore : BaseStore
private static readonly byte[] TxNonceKeyPrefix = { (byte)'N' };
private static readonly byte[] StagedTxKeyPrefix = { (byte)'t' };
private static readonly byte[] TxExecutionKeyPrefix = { (byte)'e' };
private static readonly byte[] TxIdBlockHashIndexPrefix = { (byte)'i' };
private static readonly byte[] IndexCountKey = { (byte)'c' };
private static readonly byte[] CanonicalChainIdIdKey = { (byte)'C' };

Expand Down Expand Up @@ -543,6 +544,33 @@ public override TxExecution GetTxExecution(BlockHash blockHash, TxId txid)
return null;
}

/// <inheritdoc cref="BaseStore.PutTxIdBlockHashIndex(TxId, BlockHash)"/>
public override void PutTxIdBlockHashIndex(TxId txId, BlockHash blockHash)
{
_txExecutionDb.Put(
TxIdBlockHashIndexKey(txId, blockHash),
blockHash.ToByteArray()
);
}

/// <inheritdoc cref="BaseStore.DeleteTxIdBlockHashIndex(TxId, BlockHash)"/>
public override void DeleteTxIdBlockHashIndex(TxId txId, BlockHash blockHash)
{
_txExecutionDb.Remove(
TxIdBlockHashIndexKey(txId, blockHash)
);
}

/// <inheritdoc cref="BaseStore.IterateTxIdBlockHashIndex(TxId)"/>
public override IEnumerable<BlockHash> IterateTxIdBlockHashIndex(TxId txId)
{
var prefix = TxIdBlockHashIndexTxIdKey(txId);
foreach (var it in IterateDb(_txExecutionDb, prefix, null))
{
yield return new BlockHash(it.Value());
}
}

/// <inheritdoc cref="BaseStore.SetBlockPerceivedTime(BlockHash, DateTimeOffset)"/>
public override void SetBlockPerceivedTime(
BlockHash blockHash,
Expand Down Expand Up @@ -680,6 +708,12 @@ private byte[] TxExecutionKey(in BlockHash blockHash, in TxId txId) =>
private byte[] TxExecutionKey(TxExecution txExecution) =>
TxExecutionKey(txExecution.BlockHash, txExecution.TxId);

private byte[] TxIdBlockHashIndexKey(in TxId txId, in BlockHash blockHash) =>
TxIdBlockHashIndexTxIdKey(txId).Concat(blockHash.ByteArray).ToArray();

private byte[] TxIdBlockHashIndexTxIdKey(in TxId txId) =>
TxIdBlockHashIndexPrefix.Concat(txId.ByteArray).ToArray();

private IEnumerable<Iterator> IterateDb(RocksDb db, byte[] prefix, Guid? chainId = null)
{
ColumnFamilyHandle cf = GetColumnFamily(db, chainId);
Expand Down
34 changes: 34 additions & 0 deletions Libplanet.RocksDBStore/RocksDBStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public class RocksDBStore : BaseStore
private static readonly byte[] TxNonceKeyPrefix = { (byte)'N' };
private static readonly byte[] StagedTxKeyPrefix = { (byte)'t' };
private static readonly byte[] TxExecutionKeyPrefix = { (byte)'e' };
private static readonly byte[] TxIdBlockHashIndexPrefix = { (byte)'i' };
private static readonly byte[] IndexCountKey = { (byte)'c' };
private static readonly byte[] CanonicalChainIdIdKey = { (byte)'C' };
private static readonly byte[] PreviousChainIdKey = { (byte)'P' };
Expand Down Expand Up @@ -759,6 +760,33 @@ public override bool ContainsBlock(BlockHash blockHash)
return false;
}

/// <inheritdoc cref="BaseStore.PutTxIdBlockHashIndex(TxId, BlockHash)"/>
public override void PutTxIdBlockHashIndex(TxId txId, BlockHash blockHash)
{
_txExecutionDb.Put(
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it OK for you to share the same db with TxExecution? @dahlia
IMHO, txExecutionDB consists of transient and deterministic data. So they can share the same db. #1321

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cdf2e8d

TxIdBlockHashIndexKey(txId, blockHash),
blockHash.ToByteArray()
);
}

/// <inheritdoc cref="BaseStore.DeleteTxIdBlockHashIndex(TxId, BlockHash)"/>
public override void DeleteTxIdBlockHashIndex(TxId txId, BlockHash blockHash)
{
_txExecutionDb.Remove(
TxIdBlockHashIndexKey(txId, blockHash)
);
}

/// <inheritdoc cref="BaseStore.IterateTxIdBlockHashIndex(TxId)"/>
public override IEnumerable<BlockHash> IterateTxIdBlockHashIndex(TxId txId)
{
var prefix = TxIdBlockHashIndexTxIdKey(txId);
foreach (var it in IterateDb(_txExecutionDb, prefix, null))
{
yield return new BlockHash(it.Value());
}
}

/// <inheritdoc cref="BaseStore.PutTxExecution(Libplanet.Tx.TxSuccess)"/>
public override void PutTxExecution(TxSuccess txSuccess) =>
_txExecutionDb.Put(
Expand Down Expand Up @@ -976,6 +1004,12 @@ private byte[] TxExecutionKey(in BlockHash blockHash, in TxId txId) =>
private byte[] TxExecutionKey(TxExecution txExecution) =>
TxExecutionKey(txExecution.BlockHash, txExecution.TxId);

private byte[] TxIdBlockHashIndexKey(in TxId txId, in BlockHash blockHash) =>
TxIdBlockHashIndexTxIdKey(txId).Concat(blockHash.ByteArray).ToArray();

private byte[] TxIdBlockHashIndexTxIdKey(in TxId txId) =>
TxIdBlockHashIndexPrefix.Concat(txId.ByteArray).ToArray();

private byte[] ForkedChainsKey(Guid guid) =>
ForkedChainsKeyPrefix.Concat(guid.ToByteArray()).ToArray();

Expand Down
50 changes: 50 additions & 0 deletions Libplanet.Tests/Store/StoreTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,56 @@ void AssertTxFailuresEqual(TxFailure expected, TxExecution actual)
Assert.Null(Fx.Store.GetTxExecution(Fx.Hash2, Fx.TxId2));
}

[SkippableFact]
public void TxIdBlockHashIndex()
{
Assert.Null(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId1));
Assert.Null(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId2));
Assert.Null(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId3));

Fx.Store.PutTxIdBlockHashIndex(Fx.TxId1, Fx.Hash1);
Assert.Null(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId2));
Assert.Null(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId3));

Fx.Store.PutTxIdBlockHashIndex(Fx.TxId2, Fx.Hash2);
Fx.Store.PutTxIdBlockHashIndex(Fx.TxId3, Fx.Hash3);

Assert.True(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId1)?.Equals(Fx.Hash1));
Assert.True(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId2)?.Equals(Fx.Hash2));
Assert.True(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId3)?.Equals(Fx.Hash3));

Fx.Store.PutTxIdBlockHashIndex(Fx.TxId1, Fx.Hash3);
Fx.Store.PutTxIdBlockHashIndex(Fx.TxId2, Fx.Hash3);
Fx.Store.PutTxIdBlockHashIndex(Fx.TxId3, Fx.Hash1);
Assert.Equal(2, Fx.Store.IterateTxIdBlockHashIndex(Fx.TxId1).Count());
Assert.Equal(2, Fx.Store.IterateTxIdBlockHashIndex(Fx.TxId2).Count());
Assert.Equal(2, Fx.Store.IterateTxIdBlockHashIndex(Fx.TxId3).Count());

Fx.Store.DeleteTxIdBlockHashIndex(Fx.TxId1, Fx.Hash1);
Fx.Store.DeleteTxIdBlockHashIndex(Fx.TxId2, Fx.Hash2);
Fx.Store.DeleteTxIdBlockHashIndex(Fx.TxId3, Fx.Hash3);

Assert.True(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId1)?.Equals(Fx.Hash3));
Assert.True(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId2)?.Equals(Fx.Hash3));
Assert.True(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId3)?.Equals(Fx.Hash1));

Assert.Single(Fx.Store.IterateTxIdBlockHashIndex(Fx.TxId1));
Assert.Single(Fx.Store.IterateTxIdBlockHashIndex(Fx.TxId2));
Assert.Single(Fx.Store.IterateTxIdBlockHashIndex(Fx.TxId3));

Fx.Store.DeleteTxIdBlockHashIndex(Fx.TxId1, Fx.Hash1);
Fx.Store.DeleteTxIdBlockHashIndex(Fx.TxId2, Fx.Hash2);
Fx.Store.DeleteTxIdBlockHashIndex(Fx.TxId3, Fx.Hash3);

Fx.Store.DeleteTxIdBlockHashIndex(Fx.TxId1, Fx.Hash3);
Fx.Store.DeleteTxIdBlockHashIndex(Fx.TxId2, Fx.Hash3);
Fx.Store.DeleteTxIdBlockHashIndex(Fx.TxId3, Fx.Hash1);

Assert.Null(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId1));
Assert.Null(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId2));
Assert.Null(Fx.Store.GetFirstTxIdBlockHashIndex(Fx.TxId3));
}

[SkippableFact]
public void BlockPerceivedTime()
{
Expand Down
24 changes: 24 additions & 0 deletions Libplanet.Tests/Store/StoreTracker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,30 @@ public TxExecution GetTxExecution(BlockHash blockHash, TxId txid)
return _store.GetTxExecution(blockHash, txid);
}

public void PutTxIdBlockHashIndex(TxId txId, BlockHash blockHash)
{
Log(nameof(PutTxIdBlockHashIndex), txId, blockHash);
_store.PutTxIdBlockHashIndex(txId, blockHash);
}

public BlockHash? GetFirstTxIdBlockHashIndex(TxId txId)
{
Log(nameof(GetFirstTxIdBlockHashIndex), txId);
return _store.GetFirstTxIdBlockHashIndex(txId);
}

public IEnumerable<BlockHash> IterateTxIdBlockHashIndex(TxId txId)
{
Log(nameof(IterateTxIdBlockHashIndex), txId);
return _store.IterateTxIdBlockHashIndex(txId);
}

public void DeleteTxIdBlockHashIndex(TxId txId, BlockHash blockHash)
{
Log(nameof(DeleteTxIdBlockHashIndex), txId, blockHash);
_store.DeleteTxIdBlockHashIndex(txId, blockHash);
}

public void SetBlockPerceivedTime(BlockHash blockHash, DateTimeOffset perceivedTime)
{
Log(nameof(SetBlockPerceivedTime), blockHash, perceivedTime);
Expand Down
24 changes: 24 additions & 0 deletions Libplanet/Store/BaseStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,30 @@ public abstract void PutBlock<T>(Block<T> block)
/// <inheritdoc cref="IStore.GetTxExecution(BlockHash, TxId)"/>
public abstract TxExecution GetTxExecution(BlockHash blockHash, TxId txid);

/// <inheritdoc cref="IStore.PutTxIdBlockHashIndex(TxId, BlockHash)"/>
public abstract void PutTxIdBlockHashIndex(TxId txId, BlockHash blockHash);

public BlockHash? GetFirstTxIdBlockHashIndex(TxId txId)
{
BlockHash? blockHash;
try
{
blockHash = IterateTxIdBlockHashIndex(txId).First();
}
catch (InvalidOperationException)
{
blockHash = null;
}

return blockHash;
}

/// <inheritdoc cref="IStore.IterateTxIdBlockHashIndex(TxId)"/>
public abstract IEnumerable<BlockHash> IterateTxIdBlockHashIndex(TxId txId);

/// <inheritdoc cref="IStore.DeleteTxIdBlockHashIndex(TxId, BlockHash)"/>
public abstract void DeleteTxIdBlockHashIndex(TxId txId, BlockHash blockHash);

/// <inheritdoc cref="IStore.SetBlockPerceivedTime(BlockHash, DateTimeOffset)"/>
public abstract void SetBlockPerceivedTime(
BlockHash blockHash,
Expand Down
Loading