Add flush command support

- Add flush in lab2/30-aries.test - Fix a bug in LogManager::Flush: avoid writing next_lsn file when lsn less than existing next_lsn - Use next_lsn file instead of control file in log parser Signed-off-by: Wenbo Li <[email protected]>
thu-db · Mar 12, 2024 · 5f3c548 · 5f3c548
1 parent 73f109c
commit 5f3c548
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 37 deletions.
diff --git a/src/bin/huadb-parser.cpp b/src/bin/huadb-parser.cpp
@@ -65,20 +65,19 @@ void parse_data(const fs::path &path) {
 }
 
 void parse_log(const fs::path &path) {
-  auto control_name = path / huadb::CONTROL_NAME;
+  auto next_lsn_name = path / huadb::NEXT_LSN_NAME;
   auto log_name = path / huadb::LOG_NAME;
-  if (!fs::is_regular_file(control_name)) {
-    std::cerr << "File not found: " << control_name << std::endl;
+  if (!fs::is_regular_file(next_lsn_name)) {
+    std::cerr << "File not found: " << next_lsn_name << std::endl;
     std::exit(1);
   }
-  std::ifstream file(control_name);
+  std::ifstream file(next_lsn_name);
   if (file.fail()) {
-    std::cerr << "Failed to open file: " << control_name << std::endl;
+    std::cerr << "Failed to open file: " << next_lsn_name << std::endl;
     std::exit(1);
   }
-  huadb::xid_t xid;
   huadb::lsn_t next_lsn;
-  file >> xid >> next_lsn;
+  file >> next_lsn;
   file.close();
   file.clear();
 

diff --git a/src/bin/shell.cpp b/src/bin/shell.cpp
@@ -21,6 +21,9 @@ void PlainShell() {
       if (query.substr(0, 5) == "crash") {
         database->Crash();
         std::cout << "CRASH" << std::endl;
+      } else if (query.substr(0, 5) == "flush") {
+        database->Flush();
+        std::cout << "FLUSH" << std::endl;
       } else if (query.substr(0, 7) == "restart") {
         database.reset();
         database = std::make_unique<huadb::DatabaseEngine>();
@@ -82,6 +85,9 @@ void LinenoiseShell() {
       if (query.substr(0, 5) == "crash") {
         database->Crash();
         std::cout << "CRASH" << std::endl;
+      } else if (query.substr(0, 5) == "flush") {
+        database->Flush();
+        std::cout << "FLUSH" << std::endl;
       } else if (query.substr(0, 7) == "restart") {
         database.reset();
         database = std::make_unique<huadb::DatabaseEngine>();

diff --git a/src/database/database_engine.cpp b/src/database/database_engine.cpp
@@ -287,6 +287,8 @@ void DatabaseEngine::Crash() {
   crashed_ = true;
 }
 
+void DatabaseEngine::Flush() { buffer_pool_->Flush(); }
+
 void DatabaseEngine::Help(ResultWriter &writer) const {
   std::string help = R"(
   \? or \h              show help message

diff --git a/src/database/database_engine.h b/src/database/database_engine.h
@@ -37,6 +37,7 @@ class DatabaseEngine {
   void ExecuteSql(const std::string &sql, ResultWriter &writer, const Connection &connection);
 
   void Crash();
+  void Flush();
 
  private:
   void Help(ResultWriter &writer) const;

diff --git a/src/log/log_manager.cpp b/src/log/log_manager.cpp
@@ -190,12 +190,13 @@ uint32_t LogManager::GetRedoCount() const { return redo_count_; }
 
 void LogManager::Flush(lsn_t lsn) {
   size_t max_log_size = 0;
-  lsn_t max_lsn = 0;
+  lsn_t max_lsn = NULL_LSN;
   {
     std::unique_lock lock(log_buffer_mutex_);
     for (auto iterator = log_buffer_.cbegin(); iterator != log_buffer_.cend();) {
       const auto &log_record = *iterator;
-      if (log_record->GetLSN() > lsn && lsn != NULL_LSN) {
+      // 如果 lsn 为 NULL_LSN，表示 log_buffer_ 中所有日志都需要刷盘
+      if (lsn != NULL_LSN && log_record->GetLSN() > lsn) {
         iterator++;
         continue;
       }
@@ -204,18 +205,27 @@ void LogManager::Flush(lsn_t lsn) {
       log_record->SerializeTo(log);
       disk_.WriteLog(log_record->GetLSN(), log_size, log);
       delete[] log;
-      if (log_record->GetLSN() > max_lsn) {
+      if (max_lsn == NULL_LSN || log_record->GetLSN() > max_lsn) {
         max_lsn = log_record->GetLSN();
         max_log_size = log_size;
       }
       iterator = log_buffer_.erase(iterator);
     }
   }
-  if (flushed_lsn_ == NULL_LSN || max_lsn > flushed_lsn_) {
+  // 如果 max_lsn 为 NULL_LSN，表示没有日志刷盘
+  // 如果 flushed_lsn_ 为 NULL_LSN，表示还没有日志刷过盘
+  if (max_lsn != NULL_LSN && (flushed_lsn_ == NULL_LSN || max_lsn > flushed_lsn_)) {
     flushed_lsn_ = max_lsn;
+    lsn_t next_lsn = FIRST_LSN;
+    if (disk_.FileExists(NEXT_LSN_NAME)) {
+      std::ifstream in(NEXT_LSN_NAME);
+      in >> next_lsn;
+    }
+    if (flushed_lsn_ + max_log_size > next_lsn) {
+      std::ofstream out(NEXT_LSN_NAME);
+      out << (flushed_lsn_ + max_log_size);
+    }
   }
-  std::ofstream out(NEXT_LSN_NAME);
-  out << (flushed_lsn_ + max_log_size);
 }
 
 void LogManager::Analyze() {

diff --git a/src/storage/disk.cpp b/src/storage/disk.cpp
@@ -17,17 +17,16 @@ Disk::Disk() {
 
   if (!FileExists(LOG_NAME)) {
     CreateFile(LOG_NAME);
-    log_fs_ = std::fstream(LOG_NAME, std::fstream::in | std::fstream::out | std::fstream::binary | std::fstream::trunc);
     std::filesystem::resize_file(LOG_NAME, LOG_SEGMENT_SIZE);
     log_segments = 1;
   } else {
-    log_fs_ = std::fstream(LOG_NAME, std::fstream::in | std::fstream::out | std::fstream::binary);
     auto log_file_size = std::filesystem::file_size(LOG_NAME);
     if (log_file_size / LOG_SEGMENT_SIZE == 0 || log_file_size % LOG_SEGMENT_SIZE != 0) {
       throw DbException("log file size is not a multiple of segment size");
     }
     log_segments = log_file_size / LOG_SEGMENT_SIZE;
   }
+  log_fs_.open(LOG_NAME, std::fstream::in | std::fstream::out | std::fstream::binary);
   if (log_fs_.fail()) {
     throw DbException("fstream failed in Disk::Disk");
   }

diff --git a/test/lab2/30-aries.test b/test/lab2/30-aries.test
@@ -1,68 +1,72 @@
 statement ok
-create table aries(id int, name varchar(200));
+create table aries(id varchar(1), name varchar(200));
 
 statement ok
 restart;
 
 statement ok C1
 begin;
 
+# Write one page of data
 query C1
-insert into aries values(1, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
+insert into aries values('A', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
 ----
 1
 
 query C1
-insert into aries values(2, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
+insert into aries values('B', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
 ----
 1
 
 statement ok
 checkpoint;
 
-statement ok C2
+statement ok C3
 begin;
 
 query C1
-insert into aries values(3, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
+insert into aries values('C', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
 ----
 1
 
-statement ok C3
+statement ok C2
 begin;
 
-query C3
-insert into aries values(4, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
+statement ok
+flush;
+
+query C2
+insert into aries values('D', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
 ----
 1
 
 statement ok C1
 commit;
 
-query C2
-delete from aries where id = 2;
+query C3
+delete from aries where id = 'B';
 ----
 1
 
-query C3
-delete from aries where id = 1;
+query C2
+delete from aries where id = 'A';
 ----
 1
 
-statement ok C3
+statement ok C2
 commit;
 
-query C2
-insert into aries values(5, '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
+query C3
+insert into aries values('E', '000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz');
 ----
 1
 
-query rowsort C2
+query rowsort C3
 select * from aries;
 ----
-3 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
-4 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
-5 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
+C 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
+D 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
+E 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
 
 statement ok
 crash;
@@ -73,9 +77,9 @@ restart;
 query rowsort
 select * from aries;
 ----
-2 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
-3 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
-4 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
+B 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
+C 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
+D 000111222333444555666777888999AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZaaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz
 
 statement ok
 drop table aries;
diff --git a/test/sqllogictest.cpp b/test/sqllogictest.cpp
@@ -67,6 +67,8 @@ bool Run(const fs::path &path) {
           }
           if (statement.sql_.substr(0, 5) == "crash") {
             database->Crash();
+          } else if (statement.sql_.substr(0, 5) == "flush") {
+            database->Flush();
           } else if (statement.sql_.substr(0, 7) == "restart") {
             database.reset();
             database = std::make_unique<huadb::DatabaseEngine>();