Skip to content

Commit

Permalink
Merge pull request #289
Browse files Browse the repository at this point in the history 
01076ae Check if LMDB needs resize every 1000 blocks (Thomas Winget)
b0d849e null out batch txn pointer as needed (BlockchainLMDB) (Thomas Winget)
7b7ef73 LMDB should now dynamically resize the mapsize (Thomas Winget)
ac79502 Move mdb_txn_safe implementation to cpp file (Thomas Winget)
  • Loading branch information
@fluffypony
fluffypony committed May 26, 2015
2 parents 6f15194 + 01076ae commit 634e367
Show file tree
Hide file tree
Showing 2 changed files with 210 additions and 75 deletions.
195 changes: 172 additions & 23 deletions src/blockchain_db/lmdb/db_lmdb.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,20 +148,149 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi

namespace cryptonote
{
std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0};
std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT;

// If m_batch_active is set, a batch transaction exists beyond this class, such
// as a batch import with verification enabled, or possibly (later) a batch
// network sync.
//
// For some of the lookup methods, such as get_block_timestamp(), tx_exists(),
// and get_tx(), when m_batch_active is set, the lookup uses the batch
// transaction. This isn't only because the transaction is available, but it's
// necessary so that lookups include the database updates only present in the
// current batch write.
//
// A regular network sync without batch writes is expected to open a new read
// transaction, as those lookups are part of the validation done prior to the
// write for block and tx data, so no write transaction is open at the time.
mdb_txn_safe::mdb_txn_safe() : m_txn(NULL)
{
while (creation_gate.test_and_set());
num_active_txns++;
creation_gate.clear();
}

mdb_txn_safe::~mdb_txn_safe()
{
LOG_PRINT_L3("mdb_txn_safe: destructor");
if (m_txn != NULL)
{
if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
{
LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
}
else
{
// Example of when this occurs: a lookup fails, so a read-only txn is
// aborted through this destructor. However, successful read-only txns
// ideally should have been committed when done and not end up here.
//
// NOTE: not sure if this is ever reached for a non-batch write
// transaction, but it's probably not ideal if it did.
LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
}
mdb_txn_abort(m_txn);
}
num_active_txns--;
}

void mdb_txn_safe::commit(std::string message)
{
if (message.size() == 0)
{
message = "Failed to commit a transaction to the db";
}

if (mdb_txn_commit(m_txn))
{
m_txn = NULL;
LOG_PRINT_L0(message);
throw DB_ERROR(message.c_str());
}
m_txn = NULL;
}

void mdb_txn_safe::abort()
{
LOG_PRINT_L3("mdb_txn_safe: abort()");
if(m_txn != NULL)
{
mdb_txn_abort(m_txn);
m_txn = NULL;
}
else
{
LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
}
}

uint64_t mdb_txn_safe::num_active_tx()
{
return num_active_txns;
}

void mdb_txn_safe::prevent_new_txns()
{
while (creation_gate.test_and_set());
}

void mdb_txn_safe::wait_no_active_txns()
{
while (num_active_txns > 0);
}

void mdb_txn_safe::allow_new_txns()
{
creation_gate.clear();
}



void BlockchainLMDB::do_resize()
{
MDB_envinfo mei;

mdb_env_info(m_env, &mei);

MDB_stat mst;

mdb_env_stat(m_env, &mst);

uint64_t new_mapsize = (double)mei.me_mapsize * RESIZE_FACTOR;

new_mapsize += (new_mapsize % mst.ms_psize);

mdb_txn_safe::prevent_new_txns();

if (m_write_txn != nullptr)
{
if (m_batch_active)
{
throw0(DB_ERROR("lmdb resizing not yet supported when batch transactions enabled!"));
}
else
{
throw0(DB_ERROR("attempting resize with write transaction in progress, this should not happen!"));
}
}

mdb_txn_safe::wait_no_active_txns();

mdb_env_set_mapsize(m_env, new_mapsize);

LOG_PRINT_L0("LMDB Mapsize increased."
<< " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB"
<< ", New: " << new_mapsize / (1024 * 1024) << "MiB");

mdb_txn_safe::allow_new_txns();
}

bool BlockchainLMDB::need_resize()
{
MDB_envinfo mei;

mdb_env_info(m_env, &mei);

MDB_stat mst;

mdb_env_stat(m_env, &mst);

uint64_t size_used = mst.ms_psize * mei.me_last_pgno;

if ((double)size_used / mei.me_mapsize > 0.8)
{
return true;
}
return false;
}

void BlockchainLMDB::add_block( const block& blk
, const size_t& block_size
Expand Down Expand Up @@ -631,6 +760,7 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)

m_batch_transactions = batch_transactions;
m_write_txn = nullptr;
m_write_batch_txn = nullptr;
m_batch_active = false;
m_height = 0;
}
Expand Down Expand Up @@ -672,7 +802,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
if (mdb_env_set_maxdbs(m_env, 20))
throw0(DB_ERROR("Failed to set max number of dbs"));

size_t mapsize = 1LL << 34;
size_t mapsize = DEFAULT_MAPSIZE;
if (auto result = mdb_env_set_mapsize(m_env, mapsize))
throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str()));
if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644))
Expand Down Expand Up @@ -1660,16 +1790,21 @@ void BlockchainLMDB::batch_start()
throw0(DB_ERROR("batch transactions not enabled"));
if (m_batch_active)
throw0(DB_ERROR("batch transaction already in progress"));
if (m_write_batch_txn != nullptr)
throw0(DB_ERROR("batch transaction already in progress"));
if (m_write_txn)
throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
check_open();

m_write_batch_txn = new mdb_txn_safe();

// NOTE: need to make sure it's destroyed properly when done
if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
if (mdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
// indicates this transaction is for batch transactions, but not whether it's
// active
m_write_batch_txn.m_batch_txn = true;
m_write_txn = &m_write_batch_txn;
m_write_batch_txn->m_batch_txn = true;
m_write_txn = m_write_batch_txn;
m_batch_active = true;
LOG_PRINT_L3("batch transaction: begin");
}
Expand All @@ -1681,19 +1816,19 @@ void BlockchainLMDB::batch_commit()
throw0(DB_ERROR("batch transactions not enabled"));
if (! m_batch_active)
throw0(DB_ERROR("batch transaction not in progress"));
if (m_write_batch_txn == nullptr)
throw0(DB_ERROR("batch transaction not in progress"));
check_open();

LOG_PRINT_L3("batch transaction: committing...");
TIME_MEASURE_START(time1);
m_write_txn->commit();
TIME_MEASURE_FINISH(time1);
time_commit1 += time1;
LOG_PRINT_L3("batch transaction: committed");

if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
if (! m_write_batch_txn.m_batch_txn)
throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction"));
m_write_txn = &m_write_batch_txn;
m_write_txn = nullptr;
delete m_write_batch_txn;
}

void BlockchainLMDB::batch_stop()
Expand All @@ -1703,6 +1838,8 @@ void BlockchainLMDB::batch_stop()
throw0(DB_ERROR("batch transactions not enabled"));
if (! m_batch_active)
throw0(DB_ERROR("batch transaction not in progress"));
if (m_write_batch_txn == nullptr)
throw0(DB_ERROR("batch transaction not in progress"));
check_open();
LOG_PRINT_L3("batch transaction: committing...");
TIME_MEASURE_START(time1);
Expand All @@ -1711,6 +1848,8 @@ void BlockchainLMDB::batch_stop()
time_commit1 += time1;
// for destruction of batch transaction
m_write_txn = nullptr;
delete m_write_batch_txn;
m_write_batch_txn = nullptr;
m_batch_active = false;
LOG_PRINT_L3("batch transaction: end");
}
Expand All @@ -1726,8 +1865,9 @@ void BlockchainLMDB::batch_abort()
// for destruction of batch transaction
m_write_txn = nullptr;
// explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called.
m_write_batch_txn.abort();
m_write_batch_txn->abort();
m_batch_active = false;
m_write_batch_txn = nullptr;
LOG_PRINT_L3("batch transaction: aborted");
}

Expand All @@ -1748,6 +1888,15 @@ uint64_t BlockchainLMDB::add_block( const block& blk
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();

if (m_height % 1000 == 0)
{
if (need_resize())
{
LOG_PRINT_L0("LMDB memory map needs resized, doing that now.");
do_resize();
}
}

mdb_txn_safe txn;
if (! m_batch_active)
{
Expand Down
Loading

0 comments on commit 634e367

Please sign in to comment.