Add compaction statics in log

apache · May 15, 2019 · 5d9e185 · 5d9e185
1 parent bdef591
commit 5d9e185
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 6 deletions.
diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp
@@ -310,6 +310,7 @@ bool BaseCompaction::_check_whether_satisfy_policy(bool is_manual_trigger,
 OLAPStatus BaseCompaction::_do_base_compaction(VersionHash new_base_version_hash,
                                                vector<ColumnData*>* base_data_sources,
                                                uint64_t* row_count) {
+    OlapStopWatch watch;
     // 1. 生成新base文件对应的olap index
     SegmentGroup* new_base = new (std::nothrow) SegmentGroup(_table.get(),
                                                        _new_base_version,
@@ -399,7 +400,9 @@ OLAPStatus BaseCompaction::_do_base_compaction(VersionHash new_base_version_hash
             << "source_rows=" << source_rows
             << ", merged_rows=" << merged_rows
             << ", filted_rows=" << filted_rows
-            << ", new_index_rows=" << new_base->num_rows();
+            << ", new_index_rows=" << new_base->num_rows()
+            << ", merged_version_num=" << _need_merged_versions.size()
+            << ", time=" << watch.get_elapse_time_us();
     }
 
     LOG(INFO) << "succeed to do base compaction. table=" << _table->full_name()

diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp
@@ -369,6 +369,7 @@ bool CumulativeCompaction::_find_previous_version(const Version current_version,
 
 OLAPStatus CumulativeCompaction::_do_cumulative_compaction() {
     OLAPStatus res = OLAP_SUCCESS;
+    OlapStopWatch watch;
     Merger merger(_table, _new_segment_group, READER_CUMULATIVE_COMPACTION);
 
     // 1. merge delta files into new cumulative file
@@ -412,7 +413,9 @@ OLAPStatus CumulativeCompaction::_do_cumulative_compaction() {
         LOG(INFO) << "all row nums. source_rows=" << source_rows
                   << ", merged_rows=" << merged_rows
                   << ", filted_rows=" << filted_rows
-                  << ", new_index_rows=" << _new_segment_group->num_rows();
+                  << ", new_index_rows=" << _new_segment_group->num_rows()
+                  << ", merged_version_num=" << _need_merged_versions.size()
+                  << ", time=" << watch.get_elapse_time_us();;
     }
 
     // 3. add new cumulative file into table

diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
@@ -57,16 +57,14 @@ OLAPStatus OLAPEngine::_start_bg_worker() {
         [this] {
             _unused_index_thread_callback(nullptr);
         });
-
-    uint32_t file_system_num = get_file_system_count();
     // convert store map to vector
     std::vector<OlapStore*> store_vec;
     for (auto& tmp_store : _store_map) {
         store_vec.push_back(tmp_store.second);
     }
     int32_t store_num = store_vec.size();
     // start be and ce threads for merge data
-    int32_t base_compaction_num_threads = config::base_compaction_num_threads_per_disk * file_system_num;
+    int32_t base_compaction_num_threads = config::base_compaction_num_threads_per_disk * store_num;
     _base_compaction_threads.reserve(base_compaction_num_threads);
     for (uint32_t i = 0; i < base_compaction_num_threads; ++i) {
         _base_compaction_threads.emplace_back(
@@ -75,7 +73,7 @@ OLAPStatus OLAPEngine::_start_bg_worker() {
             });
     }
 
-    int32_t cumulative_compaction_num_threads = config::cumulative_compaction_num_threads_per_disk * file_system_num;
+    int32_t cumulative_compaction_num_threads = config::cumulative_compaction_num_threads_per_disk * store_num;
     _cumulative_compaction_threads.reserve(cumulative_compaction_num_threads);
     for (uint32_t i = 0; i < cumulative_compaction_num_threads; ++i) {
         _cumulative_compaction_threads.emplace_back(