From 5f3c548f4b283b02087296885ef47bdfb5359382 Mon Sep 17 00:00:00 2001 From: Wenbo Li Date: Tue, 12 Mar 2024 23:33:33 +0800 Subject: [PATCH] Add flush command support - Add flush in lab2/30-aries.test - Fix a bug in LogManager::Flush: avoid writing next_lsn file when lsn less than existing next_lsn - Use next_lsn file instead of control file in log parser Signed-off-by: Wenbo Li --- src/bin/huadb-parser.cpp | 13 ++++----- src/bin/shell.cpp | 6 ++++ src/database/database_engine.cpp | 2 ++ src/database/database_engine.h | 1 + src/log/log_manager.cpp | 22 +++++++++++---- src/storage/disk.cpp | 3 +- test/lab2/30-aries.test | 48 +++++++++++++++++--------------- test/sqllogictest.cpp | 2 ++ 8 files changed, 60 insertions(+), 37 deletions(-) diff --git a/src/bin/huadb-parser.cpp b/src/bin/huadb-parser.cpp index ebc4d22..c499aa1 100644 --- a/src/bin/huadb-parser.cpp +++ b/src/bin/huadb-parser.cpp @@ -65,20 +65,19 @@ void parse_data(const fs::path &path) { } void parse_log(const fs::path &path) { - auto control_name = path / huadb::CONTROL_NAME; + auto next_lsn_name = path / huadb::NEXT_LSN_NAME; auto log_name = path / huadb::LOG_NAME; - if (!fs::is_regular_file(control_name)) { - std::cerr << "File not found: " << control_name << std::endl; + if (!fs::is_regular_file(next_lsn_name)) { + std::cerr << "File not found: " << next_lsn_name << std::endl; std::exit(1); } - std::ifstream file(control_name); + std::ifstream file(next_lsn_name); if (file.fail()) { - std::cerr << "Failed to open file: " << control_name << std::endl; + std::cerr << "Failed to open file: " << next_lsn_name << std::endl; std::exit(1); } - huadb::xid_t xid; huadb::lsn_t next_lsn; - file >> xid >> next_lsn; + file >> next_lsn; file.close(); file.clear(); diff --git a/src/bin/shell.cpp b/src/bin/shell.cpp index 23be19a..34347c7 100644 --- a/src/bin/shell.cpp +++ b/src/bin/shell.cpp @@ -21,6 +21,9 @@ void PlainShell() { if (query.substr(0, 5) == "crash") { database->Crash(); std::cout << "CRASH" << std::endl; + } else if (query.substr(0, 5) == "flush") { + database->Flush(); + std::cout << "FLUSH" << std::endl; } else if (query.substr(0, 7) == "restart") { database.reset(); database = std::make_unique(); @@ -82,6 +85,9 @@ void LinenoiseShell() { if (query.substr(0, 5) == "crash") { database->Crash(); std::cout << "CRASH" << std::endl; + } else if (query.substr(0, 5) == "flush") { + database->Flush(); + std::cout << "FLUSH" << std::endl; } else if (query.substr(0, 7) == "restart") { database.reset(); database = std::make_unique(); diff --git a/src/database/database_engine.cpp b/src/database/database_engine.cpp index 99279ad..a91e8f7 100644 --- a/src/database/database_engine.cpp +++ b/src/database/database_engine.cpp @@ -287,6 +287,8 @@ void DatabaseEngine::Crash() { crashed_ = true; } +void DatabaseEngine::Flush() { buffer_pool_->Flush(); } + void DatabaseEngine::Help(ResultWriter &writer) const { std::string help = R"( \? or \h show help message diff --git a/src/database/database_engine.h b/src/database/database_engine.h index b4125fa..d84bd43 100644 --- a/src/database/database_engine.h +++ b/src/database/database_engine.h @@ -37,6 +37,7 @@ class DatabaseEngine { void ExecuteSql(const std::string &sql, ResultWriter &writer, const Connection &connection); void Crash(); + void Flush(); private: void Help(ResultWriter &writer) const; diff --git a/src/log/log_manager.cpp b/src/log/log_manager.cpp index 7ff3578..b7319d8 100644 --- a/src/log/log_manager.cpp +++ b/src/log/log_manager.cpp @@ -190,12 +190,13 @@ uint32_t LogManager::GetRedoCount() const { return redo_count_; } void LogManager::Flush(lsn_t lsn) { size_t max_log_size = 0; - lsn_t max_lsn = 0; + lsn_t max_lsn = NULL_LSN; { std::unique_lock lock(log_buffer_mutex_); for (auto iterator = log_buffer_.cbegin(); iterator != log_buffer_.cend();) { const auto &log_record = *iterator; - if (log_record->GetLSN() > lsn && lsn != NULL_LSN) { + // 如果 lsn 为 NULL_LSN，表示 log_buffer_ 中所有日志都需要刷盘 + if (lsn != NULL_LSN && log_record->GetLSN() > lsn) { iterator++; continue; } @@ -204,18 +205,27 @@ void LogManager::Flush(lsn_t lsn) { log_record->SerializeTo(log); disk_.WriteLog(log_record->GetLSN(), log_size, log); delete[] log; - if (log_record->GetLSN() > max_lsn) { + if (max_lsn == NULL_LSN || log_record->GetLSN() > max_lsn) { max_lsn = log_record->GetLSN(); max_log_size = log_size; } iterator = log_buffer_.erase(iterator); } } - if (flushed_lsn_ == NULL_LSN || max_lsn > flushed_lsn_) { + // 如果 max_lsn 为 NULL_LSN，表示没有日志刷盘 + // 如果 flushed_lsn_ 为 NULL_LSN，表示还没有日志刷过盘 + if (max_lsn != NULL_LSN && (flushed_lsn_ == NULL_LSN || max_lsn > flushed_lsn_)) { flushed_lsn_ = max_lsn; + lsn_t next_lsn = FIRST_LSN; + if (disk_.FileExists(NEXT_LSN_NAME)) { + std::ifstream in(NEXT_LSN_NAME); + in >> next_lsn; + } + if (flushed_lsn_ + max_log_size > next_lsn) { + std::ofstream out(NEXT_LSN_NAME); + out << (flushed_lsn_ + max_log_size); + } } - std::ofstream out(NEXT_LSN_NAME); - out << (flushed_lsn_ + max_log_size); } void LogManager::Analyze() { diff --git a/src/storage/disk.cpp b/src/storage/disk.cpp index 84ecf9e..edc7716 100644 --- a/src/storage/disk.cpp +++ b/src/storage/disk.cpp @@ -17,17 +17,16 @@ Disk::Disk() { if (!FileExists(LOG_NAME)) { CreateFile(LOG_NAME); - log_fs_ = std::fstream(LOG_NAME, std::fstream::in | std::fstream::out | std::fstream::binary | std::fstream::trunc); std::filesystem::resize_file(LOG_NAME, LOG_SEGMENT_SIZE); log_segments = 1; } else { - log_fs_ = std::fstream(LOG_NAME, std::fstream::in | std::fstream::out | std::fstream::binary); auto log_file_size = std::filesystem::file_size(LOG_NAME); if (log_file_size / LOG_SEGMENT_SIZE == 0 || log_file_size % LOG_SEGMENT_SIZE != 0) { throw DbException("log file size is not a multiple of segment size"); } log_segments = log_file_size / LOG_SEGMENT_SIZE; } + log_fs_.open(LOG_NAME, std::fstream::in | std::fstream::out | std::fstream::binary); if (log_fs_.fail()) { throw DbException("fstream failed in Disk::Disk"); } diff --git a/test/lab2/30-aries.test b/test/lab2/30-aries.test index 1cf991e..69a92c6 100644 --- a/test/lab2/30-aries.test +++ b/test/lab2/30-aries.test @@ -1,5 +1,5 @@ statement ok -create table aries(id int, name varchar(200)); +create table aries(id varchar(1), name varchar(200)); statement ok restart; @@ -7,62 +7,66 @@ restart; statement ok C1 begin; +# Write one page of data query C1 -insert into aries values(1, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); +insert into aries values('A', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); ---- 1 query C1 -insert into aries values(2, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); +insert into aries values('B', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); ---- 1 statement ok checkpoint; -statement ok C2 +statement ok C3 begin; query C1 -insert into aries values(3, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); +insert into aries values('C', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); ---- 1 -statement ok C3 +statement ok C2 begin; -query C3 -insert into aries values(4, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); +statement ok +flush; + +query C2 +insert into aries values('D', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); ---- 1 statement ok C1 commit; -query C2 -delete from aries where id = 2; +query C3 +delete from aries where id = 'B'; ---- 1 -query C3 -delete from aries where id = 1; +query C2 +delete from aries where id = 'A'; ---- 1 -statement ok C3 +statement ok C2 commit; -query C2 -insert into aries values(5, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); +query C3 +insert into aries values('E', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz'); ---- 1 -query rowsort C2 +query rowsort C3 select * from aries; ---- -3 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz -4 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz -5 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz +C 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz +D 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz +E 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz statement ok crash; @@ -73,9 +77,9 @@ restart; query rowsort select * from aries; ---- -2 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz -3 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz -4 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz +B 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz +C 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz +D 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz statement ok drop table aries; diff --git a/test/sqllogictest.cpp b/test/sqllogictest.cpp index 3bd98a0..784ec77 100644 --- a/test/sqllogictest.cpp +++ b/test/sqllogictest.cpp @@ -67,6 +67,8 @@ bool Run(const fs::path &path) { } if (statement.sql_.substr(0, 5) == "crash") { database->Crash(); + } else if (statement.sql_.substr(0, 5) == "flush") { + database->Flush(); } else if (statement.sql_.substr(0, 7) == "restart") { database.reset(); database = std::make_unique();