diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index be417c0a26e..14376f1e2e3 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -398,4 +398,4 @@ add_target_pch("pch-stl.h" libprotobuf kvproto tipb libprotoc) add_target_pch("pch-stl.h" Net Crypto Util Data NetSSL) add_target_pch("$<$:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" XML Foundation JSON) -message (STATUS "Will build ${VERSION_FULL} (TiFlash ${TIFLASH_RELEASE_VERSION})") +message (STATUS "Will build TiFlash ${TIFLASH_RELEASE_VERSION}") diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index 09836adef23..addd9268b8f 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -262,6 +262,7 @@ namespace DB F(type_merged_task, {{"type", "merged_task"}}, ExpBuckets{0.001, 2, 20})) \ M(tiflash_mpp_task_manager, "The gauge of mpp task manager", Gauge, \ F(type_mpp_query_count, {"type", "mpp_query_count"})) \ + // clang-format on /// Buckets with boundaries [start * base^0, start * base^1, ..., start * base^(size-1)] diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 252118472a1..f74fea1e9a9 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1519,18 +1520,14 @@ SegmentReadTasks DeltaMergeStore::getReadTasksByRanges( bool try_split_task) { SegmentReadTasks tasks; + Stopwatch watch; std::shared_lock lock(read_write_mutex); auto range_it = sorted_ranges.begin(); auto seg_it = segments.upper_bound(range_it->getStart()); - if (seg_it == segments.end()) - { - throw Exception( - fmt::format("Failed to locate segment begin with start in range: {}", range_it->toDebugString()), - ErrorCodes::LOGICAL_ERROR); - } + RUNTIME_CHECK_MSG(seg_it != segments.end(), "Failed to locate segment begin with start in range: {}", range_it->toDebugString()); while (range_it != sorted_ranges.end() && seg_it != segments.end()) { @@ -1542,8 +1539,7 @@ SegmentReadTasks DeltaMergeStore::getReadTasksByRanges( { auto segment = seg_it->second; auto segment_snap = segment->createSnapshot(dm_context, false, CurrentMetrics::DT_SnapshotOfRead); - if (unlikely(!segment_snap)) - throw Exception("Failed to get segment snap", ErrorCodes::LOGICAL_ERROR); + RUNTIME_CHECK_MSG(segment_snap, "Failed to get segment snap"); tasks.push_back(std::make_shared(segment, segment_snap)); } @@ -1571,7 +1567,7 @@ SegmentReadTasks DeltaMergeStore::getReadTasksByRanges( ++seg_it; } } - auto tasks_before_split = tasks.size(); + const auto tasks_before_split = tasks.size(); if (try_split_task) { /// Try to make task number larger or equal to expected_tasks_count. @@ -1589,7 +1585,8 @@ SegmentReadTasks DeltaMergeStore::getReadTasksByRanges( auto tracing_logger = log->getChild(getLogTracingId(dm_context)); LOG_DEBUG( tracing_logger, - "[sorted_ranges: {}] [tasks before split: {}] [tasks final: {}] [ranges final: {}]", + "Segment read tasks build done, cost={}ms sorted_ranges={} n_tasks_before_split={} n_tasks_final={} n_ranges_final={}", + watch.elapsedMilliseconds(), sorted_ranges.size(), tasks_before_split, tasks.size(), diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp index e794913152c..94f15f89cfd 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp @@ -24,62 +24,63 @@ namespace DM StoreStats DeltaMergeStore::getStoreStats() { - std::shared_lock lock(read_write_mutex); - StoreStats stat; if (shutdown_called.load(std::memory_order_relaxed)) return stat; - stat.segment_count = segments.size(); - Int64 total_placed_rows = 0; Int64 total_delta_cache_rows = 0; Float64 total_delta_cache_size = 0; Int64 total_delta_valid_cache_rows = 0; - for (const auto & [handle, segment] : segments) { - UNUSED(handle); - const auto & delta = segment->getDelta(); - const auto & stable = segment->getStable(); - - total_placed_rows += delta->getPlacedDeltaRows(); + std::shared_lock lock(read_write_mutex); + stat.segment_count = segments.size(); - if (delta->getColumnFileCount()) + for (const auto & [handle, segment] : segments) { - stat.total_rows += delta->getRows(); - stat.total_size += delta->getBytes(); + UNUSED(handle); + const auto & delta = segment->getDelta(); + const auto & stable = segment->getStable(); - stat.total_delete_ranges += delta->getDeletes(); + total_placed_rows += delta->getPlacedDeltaRows(); - stat.delta_count += 1; - const auto num_delta_column_file = delta->getColumnFileCount(); - stat.total_pack_count_in_delta += num_delta_column_file; - stat.max_pack_count_in_delta = std::max(stat.max_pack_count_in_delta, num_delta_column_file); + if (delta->getColumnFileCount()) + { + stat.total_rows += delta->getRows(); + stat.total_size += delta->getBytes(); - stat.total_delta_rows += delta->getRows(); - stat.total_delta_size += delta->getBytes(); + stat.total_delete_ranges += delta->getDeletes(); - stat.delta_index_size += delta->getDeltaIndexBytes(); + stat.delta_count += 1; + const auto num_delta_column_file = delta->getColumnFileCount(); + stat.total_pack_count_in_delta += num_delta_column_file; + stat.max_pack_count_in_delta = std::max(stat.max_pack_count_in_delta, num_delta_column_file); - total_delta_cache_rows += delta->getTotalCacheRows(); - total_delta_cache_size += delta->getTotalCacheBytes(); - total_delta_valid_cache_rows += delta->getValidCacheRows(); - } + stat.total_delta_rows += delta->getRows(); + stat.total_delta_size += delta->getBytes(); - if (stable->getDMFilesPacks()) - { - stat.total_rows += stable->getRows(); - stat.total_size += stable->getBytes(); + stat.delta_index_size += delta->getDeltaIndexBytes(); + + total_delta_cache_rows += delta->getTotalCacheRows(); + total_delta_cache_size += delta->getTotalCacheBytes(); + total_delta_valid_cache_rows += delta->getValidCacheRows(); + } + + if (stable->getDMFilesPacks()) + { + stat.total_rows += stable->getRows(); + stat.total_size += stable->getBytes(); - stat.stable_count += 1; - stat.total_pack_count_in_stable += stable->getDMFilesPacks(); + stat.stable_count += 1; + stat.total_pack_count_in_stable += stable->getDMFilesPacks(); - stat.total_stable_rows += stable->getRows(); - stat.total_stable_size += stable->getBytes(); - stat.total_stable_size_on_disk += stable->getDMFilesBytesOnDisk(); + stat.total_stable_rows += stable->getRows(); + stat.total_stable_size += stable->getBytes(); + stat.total_stable_size_on_disk += stable->getDMFilesBytesOnDisk(); + } } - } + } // access to `segments` end stat.delta_rate_rows = static_cast(stat.total_delta_rows) / stat.total_rows; stat.delta_rate_segments = static_cast(stat.delta_count) / stat.segment_count; @@ -107,26 +108,32 @@ StoreStats DeltaMergeStore::getStoreStats() stat.avg_pack_rows_in_stable = static_cast(stat.total_stable_rows) / stat.total_pack_count_in_stable; stat.avg_pack_size_in_stable = static_cast(stat.total_stable_size) / stat.total_pack_count_in_stable; + // Only collect the snapshot stats for each table when PageStorage V2 is enabled. + // Collecting snapshot stats on the global PageStorage V3 for each table will cause too many + // waste on CPU and lock contention. Which cause slow queries. + if (storage_pool->getPageStorageRunMode() == PageStorageRunMode::ONLY_V2) { - auto snaps_stat = storage_pool->dataReader()->getSnapshotsStat(); - stat.storage_stable_num_snapshots = snaps_stat.num_snapshots; - stat.storage_stable_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; - stat.storage_stable_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; - stat.storage_stable_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; - } - { - auto snaps_stat = storage_pool->logReader()->getSnapshotsStat(); - stat.storage_delta_num_snapshots = snaps_stat.num_snapshots; - stat.storage_delta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; - stat.storage_delta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; - stat.storage_delta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; - } - { - auto snaps_stat = storage_pool->metaReader()->getSnapshotsStat(); - stat.storage_meta_num_snapshots = snaps_stat.num_snapshots; - stat.storage_meta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; - stat.storage_meta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; - stat.storage_meta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; + { + auto snaps_stat = storage_pool->dataReader()->getSnapshotsStat(); + stat.storage_stable_num_snapshots = snaps_stat.num_snapshots; + stat.storage_stable_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; + stat.storage_stable_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; + stat.storage_stable_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; + } + { + auto snaps_stat = storage_pool->logReader()->getSnapshotsStat(); + stat.storage_delta_num_snapshots = snaps_stat.num_snapshots; + stat.storage_delta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; + stat.storage_delta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; + stat.storage_delta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; + } + { + auto snaps_stat = storage_pool->metaReader()->getSnapshotsStat(); + stat.storage_meta_num_snapshots = snaps_stat.num_snapshots; + stat.storage_meta_oldest_snapshot_lifetime = snaps_stat.longest_living_seconds; + stat.storage_meta_oldest_snapshot_thread_id = snaps_stat.longest_living_from_thread_id; + stat.storage_meta_oldest_snapshot_tracing_id = snaps_stat.longest_living_from_tracing_id; + } } stat.background_tasks_length = background_tasks.length(); diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index b663a807d05..da37f2c77d7 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -561,8 +561,9 @@ SegmentPtr Segment::ingestDataForTest(DMContext & dm_context, SegmentSnapshotPtr Segment::createSnapshot(const DMContext & dm_context, bool for_update, CurrentMetrics::Metric metric) const { Stopwatch watch; - SCOPE_EXIT( - dm_context.scan_context->total_create_snapshot_time_ms += watch.elapsedMilliseconds();); + SCOPE_EXIT({ + dm_context.scan_context->total_create_snapshot_time_ms += watch.elapsedMilliseconds(); + }); auto delta_snap = delta->createSnapshot(dm_context, for_update, metric); auto stable_snap = stable->createSnapshot(); if (!delta_snap || !stable_snap)