From a47baef85ab601489980061b6c5d4fe1c36d5896 Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Mon, 9 Dec 2024 21:15:05 -0500 Subject: [PATCH] make disk manager actually reclaim disk space on delete (#785) --- src/include/storage/disk/disk_manager.h | 80 +++++---- .../storage/disk/disk_manager_memory.h | 157 ++---------------- src/include/storage/disk/disk_scheduler.h | 9 - src/storage/disk/disk_manager.cpp | 132 +++++++++------ src/storage/disk/disk_manager_memory.cpp | 137 ++++++++++++++- test/storage/disk_manager_test.cpp | 37 +++++ 6 files changed, 298 insertions(+), 254 deletions(-) diff --git a/src/include/storage/disk/disk_manager.h b/src/include/storage/disk/disk_manager.h index 8fd28d488..60abcc18c 100644 --- a/src/include/storage/disk/disk_manager.h +++ b/src/include/storage/disk/disk_manager.h @@ -18,21 +18,24 @@ #include // NOLINT #include // NOLINT #include +#include +#include #include "common/config.h" +#include "common/logger.h" namespace bustub { /** * DiskManager takes care of the allocation and deallocation of pages within a database. It performs the reading and * writing of pages to and from disk, providing a logical file layer within the context of a database management system. + * + * DiskManager uses lazy allocation, meaning that it only allocates space on disk when it is first accessed. It + * maintains a mapping of page ids to their corresponding offsets in the database file. When a page is deleted, it is + * marked as free and can be reused by future allocations. */ class DiskManager { public: - /** - * Creates a new disk manager that writes to the specified database file. - * @param db_file the file name of the database file to write to - */ explicit DiskManager(const std::filesystem::path &db_file); /** FOR TEST / LEADERBOARD ONLY, used by DiskManagerMemory */ @@ -40,20 +43,8 @@ class DiskManager { virtual ~DiskManager() = default; - /** - * Shut down the disk manager and close all the file resources. - */ void ShutDown(); - /** - * @brief Increases the size of the database file. - * - * This function works like a dynamic array, where the capacity is doubled until all pages can fit. - * - * @param pages The number of pages the caller wants the file used for storage to support. - */ - virtual void IncreaseDiskSpace(size_t pages); - /** * Write a page to the database file. * @param page_id id of the page @@ -74,32 +65,16 @@ class DiskManager { */ virtual void DeletePage(page_id_t page_id); - /** - * Flush the entire log buffer into disk. - * @param log_data raw log data - * @param size size of log entry - */ void WriteLog(char *log_data, int size); - /** - * Read a log entry from the log file. - * @param[out] log_data output buffer - * @param size size of the log entry - * @param offset offset of the log entry in the file - * @return true if the read was successful, false otherwise - */ auto ReadLog(char *log_data, int size, int offset) -> bool; - /** @return the number of disk flushes */ auto GetNumFlushes() const -> int; - /** @return true iff the in-memory content has not been flushed yet */ auto GetFlushState() const -> bool; - /** @return the number of disk writes */ auto GetNumWrites() const -> int; - /** @return the number of deletions */ auto GetNumDeletes() const -> int; /** @@ -112,28 +87,47 @@ class DiskManager { inline auto HasFlushLogFuture() -> bool { return flush_log_f_ != nullptr; } /** @brief returns the log file name */ - inline auto GetLogFileName() const -> std::filesystem::path { return log_name_; } + inline auto GetLogFileName() const -> std::filesystem::path { return log_file_name_; } + + /** @brief returns the size of disk space in use */ + auto GetDbFileSize() -> size_t { + auto file_size = GetFileSize(db_file_name_); + if (file_size < 0) { + LOG_DEBUG("I/O error: Fail to get db file size"); + return -1; + } + return static_cast(file_size); + } protected: + int num_flushes_{0}; + int num_writes_{0}; + int num_deletes_{0}; + + /** @brief The capacity of the file used for storage on disk. */ + size_t page_capacity_{DEFAULT_DB_IO_SIZE}; + + private: auto GetFileSize(const std::string &file_name) -> int; + + auto AllocatePage() -> size_t; + // stream to write log file std::fstream log_io_; - std::filesystem::path log_name_; + std::filesystem::path log_file_name_; // stream to write db file std::fstream db_io_; - std::filesystem::path file_name_; - int num_flushes_{0}; - int num_writes_{0}; - int num_deletes_{0}; + std::filesystem::path db_file_name_; + + // Records the offset of each page in the db file. + std::unordered_map pages_; + // Records the free slots in the db file if pages are deleted, indicated by offset. + std::vector free_slots_; + bool flush_log_{false}; std::future *flush_log_f_{nullptr}; // With multiple buffer pool instances, need to protect file access std::mutex db_io_latch_; - - /** @brief The number of pages allocated to the DBMS on disk. */ - size_t pages_{0}; - /** @brief The capacity of the file used for storage on disk. */ - size_t page_capacity_{DEFAULT_DB_IO_SIZE}; }; } // namespace bustub diff --git a/src/include/storage/disk/disk_manager_memory.h b/src/include/storage/disk/disk_manager_memory.h index 3dd691f05..fe810f86d 100644 --- a/src/include/storage/disk/disk_manager_memory.h +++ b/src/include/storage/disk/disk_manager_memory.h @@ -40,32 +40,15 @@ namespace bustub { */ class DiskManagerMemory : public DiskManager { public: - explicit DiskManagerMemory(size_t pages); + explicit DiskManagerMemory(size_t capacity); ~DiskManagerMemory() override { delete[] memory_; } - /** - * This function should increase the disk space, but since we have a fixed amount of memory we just check that the - * pages are in bounds. - */ - void IncreaseDiskSpace(size_t pages) override; - - /** - * Write a page to the database file. - * @param page_id id of the page - * @param page_data raw page data - */ void WritePage(page_id_t page_id, const char *page_data) override; - /** - * Read a page from the database file. - * @param page_id id of the page - * @param[out] page_data output buffer - */ void ReadPage(page_id_t page_id, char *page_data) override; private: - size_t pages_; char *memory_; }; @@ -75,133 +58,21 @@ class DiskManagerMemory : public DiskManager { */ class DiskManagerUnlimitedMemory : public DiskManager { public: - DiskManagerUnlimitedMemory() { - std::scoped_lock l(mutex_); - while (data_.size() < pages_ + 1) { - data_.push_back(std::make_shared()); - } - std::fill(recent_access_.begin(), recent_access_.end(), -1); - } - - /** - * This function should increase the disk space, but since this is memory we just resize the vector. - */ - void IncreaseDiskSpace(size_t pages) override { - std::scoped_lock l(mutex_); - - if (pages < pages_) { - return; - } - - while (data_.size() < pages + 1) { - data_.push_back(std::make_shared()); - } - - pages_ = pages; - } - - /** - * Write a page to the database file. - * @param page_id id of the page - * @param page_data raw page data - */ - void WritePage(page_id_t page_id, const char *page_data) override { - ProcessLatency(page_id); - - std::unique_lock l(mutex_); - if (!thread_id_.has_value()) { - thread_id_ = std::this_thread::get_id(); - } - if (page_id >= static_cast(data_.size())) { - data_.resize(page_id + 1); - } - if (data_[page_id] == nullptr) { - data_[page_id] = std::make_shared(); - } - std::shared_ptr ptr = data_[page_id]; - std::unique_lock l_page(ptr->second); - l.unlock(); - - memcpy(ptr->first.data(), page_data, BUSTUB_PAGE_SIZE); - num_writes_ += 1; - - PostProcessLatency(page_id); - } - - /** - * Read a page from the database file. - * @param page_id id of the page - * @param[out] page_data output buffer - */ - void ReadPage(page_id_t page_id, char *page_data) override { - ProcessLatency(page_id); - - std::unique_lock l(mutex_); - if (!thread_id_.has_value()) { - thread_id_ = std::this_thread::get_id(); - } - if (page_id >= static_cast(data_.size()) || page_id < 0) { - fmt::println(stderr, "page {} not in range", page_id); - std::terminate(); - return; - } - if (data_[page_id] == nullptr) { - fmt::println(stderr, "page {} not exist", page_id, pages_); - std::terminate(); - return; - } - std::shared_ptr ptr = data_[page_id]; - std::shared_lock l_page(ptr->second); - l.unlock(); - - memcpy(page_data, ptr->first.data(), BUSTUB_PAGE_SIZE); - - PostProcessLatency(page_id); - } - - /** - * Delete a page from the database file. Reclaim the disk space. - * Note: This is a no-op for now without a more complex data structure to - * track deallocated pages. - * @param page_id id of the page - */ - void DeletePage(page_id_t page_id) override { num_deletes_ += 1; } - - void ProcessLatency(page_id_t page_id) { - uint64_t sleep_micro_sec = 1000; // for random access, 1ms latency - if (latency_simulator_enabled_) { - std::unique_lock lck(latency_processor_mutex_); - for (auto &recent_page_id : recent_access_) { - if ((recent_page_id & (~0x3)) == (page_id & (~0x3))) { - sleep_micro_sec = 100; // for access in the same "block", 0.1ms latency - break; - } - if (page_id >= recent_page_id && page_id <= recent_page_id + 3) { - sleep_micro_sec = 100; // for sequential access, 0.1ms latency - break; - } - } - lck.unlock(); - std::this_thread::sleep_for(std::chrono::microseconds(sleep_micro_sec)); - } - } - - void PostProcessLatency(page_id_t page_id) { - if (latency_simulator_enabled_) { - std::scoped_lock lck(latency_processor_mutex_); - recent_access_[access_ptr_] = page_id; - access_ptr_ = (access_ptr_ + 1) % recent_access_.size(); - } - } + DiskManagerUnlimitedMemory(); + + void WritePage(page_id_t page_id, const char *page_data) override; + + void ReadPage(page_id_t page_id, char *page_data) override; + + void DeletePage(page_id_t page_id) override; + + void ProcessLatency(page_id_t page_id); + + void PostProcessLatency(page_id_t page_id); void EnableLatencySimulator(bool enabled) { latency_simulator_enabled_ = enabled; } - auto GetLastReadThreadAndClear() -> std::optional { - std::unique_lock lck(mutex_); - auto t = thread_id_; - thread_id_ = std::nullopt; - return t; - } + auto GetLastReadThreadAndClear() -> std::optional; private: bool latency_simulator_enabled_{false}; @@ -216,8 +87,6 @@ class DiskManagerUnlimitedMemory : public DiskManager { std::mutex mutex_; std::optional thread_id_; std::vector> data_; - - size_t pages_{DEFAULT_DB_IO_SIZE}; }; } // namespace bustub diff --git a/src/include/storage/disk/disk_scheduler.h b/src/include/storage/disk/disk_scheduler.h index 3b6408da6..d20d9683b 100644 --- a/src/include/storage/disk/disk_scheduler.h +++ b/src/include/storage/disk/disk_scheduler.h @@ -68,15 +68,6 @@ class DiskScheduler { */ auto CreatePromise() -> DiskSchedulerPromise { return {}; }; - /** - * @brief Increases the size of the database file to fit the specified number of pages. - * - * This function works like a dynamic array, where the capacity is doubled until all pages can fit. - * - * @param pages The number of pages the caller wants the file used for storage to support. - */ - void IncreaseDiskSpace(size_t pages) { disk_manager_->IncreaseDiskSpace(pages); } - /** * @brief Deallocates a page on disk. * diff --git a/src/storage/disk/disk_manager.cpp b/src/storage/disk/disk_manager.cpp index 3a03e545e..7981ff755 100644 --- a/src/storage/disk/disk_manager.cpp +++ b/src/storage/disk/disk_manager.cpp @@ -28,18 +28,18 @@ namespace bustub { static char *buffer_used; /** - * Constructor: open/create a single database file & log file - * @input db_file: database file name + * Creates a new disk manager that writes to the specified database file. + * @param db_file the file name of the database file to write to */ -DiskManager::DiskManager(const std::filesystem::path &db_file) : file_name_(db_file) { - log_name_ = file_name_.filename().stem().string() + ".log"; +DiskManager::DiskManager(const std::filesystem::path &db_file) : db_file_name_(db_file) { + log_file_name_ = db_file_name_.filename().stem().string() + ".log"; - log_io_.open(log_name_, std::ios::binary | std::ios::in | std::ios::app | std::ios::out); + log_io_.open(log_file_name_, std::ios::binary | std::ios::in | std::ios::app | std::ios::out); // directory or file does not exist if (!log_io_.is_open()) { log_io_.clear(); // create a new file - log_io_.open(log_name_, std::ios::binary | std::ios::trunc | std::ios::out | std::ios::in); + log_io_.open(log_file_name_, std::ios::binary | std::ios::trunc | std::ios::out | std::ios::in); if (!log_io_.is_open()) { throw Exception("can't open dblog file"); } @@ -59,13 +59,13 @@ DiskManager::DiskManager(const std::filesystem::path &db_file) : file_name_(db_f // Initialize the database file. std::filesystem::resize_file(db_file, (page_capacity_ + 1) * BUSTUB_PAGE_SIZE); - assert(static_cast(GetFileSize(file_name_)) >= page_capacity_ * BUSTUB_PAGE_SIZE); + assert(static_cast(GetFileSize(db_file_name_)) >= page_capacity_ * BUSTUB_PAGE_SIZE); buffer_used = nullptr; } /** - * Close all file streams + * Shut down the disk manager and close all the file resources. */ void DiskManager::ShutDown() { { @@ -75,43 +75,31 @@ void DiskManager::ShutDown() { log_io_.close(); } -/** - * @brief Increases the size of the file to fit the specified number of pages. - */ -void DiskManager::IncreaseDiskSpace(size_t pages) { - std::scoped_lock scoped_db_io_latch(db_io_latch_); - - if (pages < pages_) { - return; - } - - pages_ = pages; - while (page_capacity_ < pages_) { - page_capacity_ *= 2; - } - - std::filesystem::resize_file(file_name_, (page_capacity_ + 1) * BUSTUB_PAGE_SIZE); - - assert(static_cast(GetFileSize(file_name_)) >= page_capacity_ * BUSTUB_PAGE_SIZE); -} - /** * Write the contents of the specified page into disk file */ void DiskManager::WritePage(page_id_t page_id, const char *page_data) { std::scoped_lock scoped_db_io_latch(db_io_latch_); - size_t offset = static_cast(page_id) * BUSTUB_PAGE_SIZE; + size_t offset; + if (pages_.find(page_id) != pages_.end()) { + // Page already exists, overwrite it. + offset = pages_[page_id]; + } else { + // Page does not exist, allocate a new page. We use a free slot if available. + offset = AllocatePage(); + } // Set the write cursor to the page offset. - num_writes_ += 1; db_io_.seekp(offset); db_io_.write(page_data, BUSTUB_PAGE_SIZE); - if (db_io_.bad()) { - LOG_DEBUG("I/O error while writing"); + LOG_DEBUG("I/O error while writing page %d", page_id); return; } + num_writes_ += 1; + pages_[page_id] = offset; + // Flush the write to disk. db_io_.flush(); } @@ -121,27 +109,40 @@ void DiskManager::WritePage(page_id_t page_id, const char *page_data) { */ void DiskManager::ReadPage(page_id_t page_id, char *page_data) { std::scoped_lock scoped_db_io_latch(db_io_latch_); - int offset = page_id * BUSTUB_PAGE_SIZE; + size_t offset; + if (pages_.find(page_id) != pages_.end()) { + offset = pages_[page_id]; + } else { + // Page does not exist, allocate a new page. We use a free slot if available. + offset = AllocatePage(); + } // Check if we have read beyond the file length. - if (offset > GetFileSize(file_name_)) { - LOG_DEBUG("I/O error: Read past the end of file at offset %d", offset); + int file_size = GetFileSize(db_file_name_); + if (file_size < 0) { + LOG_DEBUG("I/O error: Fail to get db file size"); return; } + if (offset > static_cast(file_size)) { + LOG_DEBUG("I/O error: Read page %d past the end of file at offset %lu", page_id, offset); + return; + } + + pages_[page_id] = offset; // Set the read cursor to the page offset. db_io_.seekg(offset); db_io_.read(page_data, BUSTUB_PAGE_SIZE); if (db_io_.bad()) { - LOG_DEBUG("I/O error while reading"); + LOG_DEBUG("I/O error while reading page %d", page_id); return; } // Check if the file ended before we could read a full page. int read_count = db_io_.gcount(); if (read_count < BUSTUB_PAGE_SIZE) { - LOG_DEBUG("I/O error: Read hit the end of file at offset %d, missing %d bytes", offset, + LOG_DEBUG("I/O error: Read page %d hit the end of file at offset %lu, missing %d bytes", page_id, offset, BUSTUB_PAGE_SIZE - read_count); db_io_.clear(); memset(page_data + read_count, 0, BUSTUB_PAGE_SIZE - read_count); @@ -152,11 +153,23 @@ void DiskManager::ReadPage(page_id_t page_id, char *page_data) { * Note: This is a no-op for now without a more complex data structure to * track deallocated pages. */ -void DiskManager::DeletePage(page_id_t page_id) { num_deletes_ += 1; } +void DiskManager::DeletePage(page_id_t page_id) { + std::scoped_lock scoped_db_io_latch(db_io_latch_); + if (pages_.find(page_id) == pages_.end()) { + return; + } + + size_t offset = pages_[page_id]; + free_slots_.push_back(offset); + pages_.erase(page_id); + num_deletes_ += 1; +} /** * Write the contents of the log into disk file * Only return when sync is done, and only perform sequence write + * @param log_data raw log data + * @param size size of log entry */ void DiskManager::WriteLog(char *log_data, int size) { // enforce swap log buffer @@ -191,10 +204,13 @@ void DiskManager::WriteLog(char *log_data, int size) { /** * Read the contents of the log into the given memory area * Always read from the beginning and perform sequence read - * @return: false means already reach the end + * @param[out] log_data output buffer + * @param size size of the log entry + * @param offset offset of the log entry in the file + * @return true if the read was successful, false otherwise */ auto DiskManager::ReadLog(char *log_data, int size, int offset) -> bool { - if (offset >= GetFileSize(log_name_)) { + if (offset >= GetFileSize(log_file_name_)) { // LOG_DEBUG("end of log file"); // LOG_DEBUG("file size is %d", GetFileSize(log_name_)); return false; @@ -216,24 +232,16 @@ auto DiskManager::ReadLog(char *log_data, int size, int offset) -> bool { return true; } -/** - * Returns number of flushes made so far - */ +/** @return the number of disk flushes */ auto DiskManager::GetNumFlushes() const -> int { return num_flushes_; } -/** - * Returns true if the log is currently being flushed - */ +/** @return true iff the in-memory content has not been flushed yet */ auto DiskManager::GetFlushState() const -> bool { return flush_log_; } -/** - * Returns number of Writes made so far - */ +/** @return the number of disk writes */ auto DiskManager::GetNumWrites() const -> int { return num_writes_; } -/** - * Returns number of deletions made so far - */ +/** @return the number of deletions */ auto DiskManager::GetNumDeletes() const -> int { return num_deletes_; } /** @@ -245,4 +253,24 @@ auto DiskManager::GetFileSize(const std::string &file_name) -> int { return rc == 0 ? static_cast(stat_buf.st_size) : -1; } +/** + * Allocate a page in a free slot. If no free slot is available, append to the end of the file. + * @return the offset of the allocated page + */ +auto DiskManager::AllocatePage() -> size_t { + // Find if there is a free slot in the file. + if (!free_slots_.empty()) { + auto offset = free_slots_.back(); + free_slots_.pop_back(); + return offset; + } + + // Increase the file size if necessary. + if (pages_.size() + 1 >= page_capacity_) { + page_capacity_ *= 2; + std::filesystem::resize_file(db_file_name_, (page_capacity_ + 1) * BUSTUB_PAGE_SIZE); + } + return pages_.size() * BUSTUB_PAGE_SIZE; +} + } // namespace bustub diff --git a/src/storage/disk/disk_manager_memory.cpp b/src/storage/disk/disk_manager_memory.cpp index cbbf87bc1..430d2dfc9 100644 --- a/src/storage/disk/disk_manager_memory.cpp +++ b/src/storage/disk/disk_manager_memory.cpp @@ -28,16 +28,19 @@ namespace bustub { /** * Constructor: used for memory based manager */ -DiskManagerMemory::DiskManagerMemory(size_t pages) : pages_(pages) { memory_ = new char[pages * BUSTUB_PAGE_SIZE]; } - -void DiskManagerMemory::IncreaseDiskSpace(size_t pages) { - BUSTUB_ASSERT(pages < pages_, "Ran out of disk space for limited memory disk manager implementation"); +DiskManagerMemory::DiskManagerMemory(size_t capacity) { + page_capacity_ = capacity; + memory_ = new char[capacity * BUSTUB_PAGE_SIZE]; } /** - * Write the contents of the specified page into disk file + * Write a page to the database file. + * @param page_id id of the page + * @param page_data raw page data */ void DiskManagerMemory::WritePage(page_id_t page_id, const char *page_data) { + BUSTUB_ASSERT(static_cast(page_id) < page_capacity_, + "Ran out of disk space for limited memory disk manager implementation"); size_t offset = static_cast(page_id) * BUSTUB_PAGE_SIZE; // set write cursor to offset num_writes_ += 1; @@ -45,11 +48,133 @@ void DiskManagerMemory::WritePage(page_id_t page_id, const char *page_data) { } /** - * Read the contents of the specified page into the given memory area + * Read a page from the database file. + * @param page_id id of the page + * @param[out] page_data output buffer */ void DiskManagerMemory::ReadPage(page_id_t page_id, char *page_data) { int64_t offset = static_cast(page_id) * BUSTUB_PAGE_SIZE; memcpy(page_data, memory_ + offset, BUSTUB_PAGE_SIZE); } +DiskManagerUnlimitedMemory::DiskManagerUnlimitedMemory() { + std::scoped_lock l(mutex_); + while (data_.size() < page_capacity_ + 1) { + data_.push_back(std::make_shared()); + } + std::fill(recent_access_.begin(), recent_access_.end(), -1); +} + +/** + * Write a page to the database file. + * @param page_id id of the page + * @param page_data raw page data + */ +void DiskManagerUnlimitedMemory::WritePage(page_id_t page_id, const char *page_data) { + if (page_id < 0) { + fmt::println(stderr, "read invalid page {}", page_id); + std::terminate(); + return; + } + + ProcessLatency(page_id); + + std::unique_lock l(mutex_); + if (!thread_id_.has_value()) { + thread_id_ = std::this_thread::get_id(); + } + if (page_id >= static_cast(data_.size())) { + data_.resize(page_id + 1); + } + if (data_[page_id] == nullptr) { + data_[page_id] = std::make_shared(); + } + std::shared_ptr ptr = data_[page_id]; + std::unique_lock l_page(ptr->second); + l.unlock(); + + memcpy(ptr->first.data(), page_data, BUSTUB_PAGE_SIZE); + num_writes_ += 1; + + PostProcessLatency(page_id); +} + +/** + * Read a page from the database file. + * @param page_id id of the page + * @param[out] page_data output buffer + */ +void DiskManagerUnlimitedMemory::ReadPage(page_id_t page_id, char *page_data) { + if (page_id < 0) { + fmt::println(stderr, "read invalid page {}", page_id); + std::terminate(); + return; + } + + ProcessLatency(page_id); + + std::unique_lock l(mutex_); + if (!thread_id_.has_value()) { + thread_id_ = std::this_thread::get_id(); + } + + if (page_id >= static_cast(data_.size())) { + data_.resize(page_id + 1); + } + if (data_[page_id] == nullptr) { + data_[page_id] = std::make_shared(); + } + + std::shared_ptr ptr = data_[page_id]; + std::shared_lock l_page(ptr->second); + l.unlock(); + + memcpy(page_data, ptr->first.data(), BUSTUB_PAGE_SIZE); + + PostProcessLatency(page_id); +} + +/** + * Delete a page from the database file. Reclaim the disk space. + * Since we are using memory, this is a no-op. + * @param page_id id of the page + */ +void DiskManagerUnlimitedMemory::DeletePage(page_id_t page_id) { + // no-op since we are using memory +} + +void DiskManagerUnlimitedMemory::ProcessLatency(page_id_t page_id) { + uint64_t sleep_micro_sec = 1000; // for random access, 1ms latency + if (latency_simulator_enabled_) { + std::unique_lock lck(latency_processor_mutex_); + for (auto &recent_page_id : recent_access_) { + if ((recent_page_id & (~0x3)) == (page_id & (~0x3))) { + sleep_micro_sec = 100; // for access in the same "block", 0.1ms latency + break; + } + if (page_id >= recent_page_id && page_id <= recent_page_id + 3) { + sleep_micro_sec = 100; // for sequential access, 0.1ms latency + break; + } + } + lck.unlock(); + std::this_thread::sleep_for(std::chrono::microseconds(sleep_micro_sec)); + } +} + +void DiskManagerUnlimitedMemory::PostProcessLatency(page_id_t page_id) { + if (latency_simulator_enabled_) { + std::scoped_lock lck(latency_processor_mutex_); + recent_access_[access_ptr_] = page_id; + access_ptr_ = (access_ptr_ + 1) % recent_access_.size(); + } +} + +auto DiskManagerUnlimitedMemory::GetLastReadThreadAndClear() -> std::optional { + std::unique_lock lck(mutex_); + auto t = thread_id_; + thread_id_ = std::nullopt; + return t; +} + } // namespace bustub diff --git a/test/storage/disk_manager_test.cpp b/test/storage/disk_manager_test.cpp index 8e61b3b35..ca2cf953c 100644 --- a/test/storage/disk_manager_test.cpp +++ b/test/storage/disk_manager_test.cpp @@ -12,6 +12,7 @@ #include +#include "common/config.h" #include "common/exception.h" #include "gtest/gtest.h" #include "storage/disk/disk_manager.h" @@ -73,6 +74,42 @@ TEST_F(DiskManagerTest, ReadWriteLogTest) { dm.ShutDown(); } +TEST_F(DiskManagerTest, DeletePageTest) { + char buf[BUSTUB_PAGE_SIZE] = {0}; + char data[BUSTUB_PAGE_SIZE] = {0}; + auto dm = DiskManager(db_fname); + auto initial_size = dm.GetDbFileSize(); + + dm.ReadPage(0, buf); // tolerate empty read + + std::strncpy(data, "A test string.", sizeof(data)); + size_t pages_to_write = 100; + for (page_id_t page_id = 0; page_id < static_cast(pages_to_write); page_id++) { + dm.WritePage(page_id, data); + dm.ReadPage(page_id, buf); + EXPECT_EQ(std::memcmp(buf, data, sizeof(buf)), 0); + } + + auto size_after_write = dm.GetDbFileSize(); + EXPECT_GE(size_after_write, initial_size); + + pages_to_write *= 2; + std::strncpy(data, "test string version 2", sizeof(data)); + for (page_id_t page_id = 0; page_id < static_cast(pages_to_write); page_id++) { + dm.WritePage(page_id, data); + dm.ReadPage(page_id, buf); + EXPECT_EQ(std::memcmp(buf, data, sizeof(buf)), 0); + + dm.DeletePage(page_id); + } + + // expect no change in file size after delete because we're reclaiming space + auto size_after_delete = dm.GetDbFileSize(); + EXPECT_EQ(size_after_delete, size_after_write); + + dm.ShutDown(); +} + // NOLINTNEXTLINE TEST_F(DiskManagerTest, ThrowBadFileTest) { EXPECT_THROW(DiskManager("dev/null\\/foo/bar/baz/test.bustub"), Exception);