diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index 2d3458ae..7f4253e8 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -193,7 +193,14 @@ def flush_at_interval def handle_association(sincedb_value, watched_file) watched_file.update_bytes_read(sincedb_value.position) - sincedb_value.set_watched_file(watched_file) + if watched_file.all_read? + # avoid the last_changed_at of sincedb_value which record the watched file has bean read over being changed when reload sincedb + sincedb_value.set_watched_file_without_touch(watched_file) + logger.trace("handle_association call set_watched_file_without_touch") + else + sincedb_value.set_watched_file(watched_file) + logger.trace("handle_association call set_watched_file") + end watched_file.initial_completed if watched_file.all_read? watched_file.ignore diff --git a/lib/filewatch/sincedb_value.rb b/lib/filewatch/sincedb_value.rb index 56ac4840..bd2f3992 100644 --- a/lib/filewatch/sincedb_value.rb +++ b/lib/filewatch/sincedb_value.rb @@ -47,6 +47,10 @@ def set_watched_file(watched_file) @watched_file = watched_file end + def set_watched_file_without_touch(watched_file) + @watched_file = watched_file + end + def touch @last_changed_at = Time.now.to_f end diff --git a/lib/filewatch/tail_mode/processor.rb b/lib/filewatch/tail_mode/processor.rb index 6634dc91..60c5b292 100644 --- a/lib/filewatch/tail_mode/processor.rb +++ b/lib/filewatch/tail_mode/processor.rb @@ -187,7 +187,10 @@ def process_rotation_in_progress(watched_files) potential_sdb_value.set_watched_file(watched_file) end end - logger.trace("---------- >>>> Rotation In Progress: after handling rotation", :this_watched_file => watched_file.details, :sincedb_value => (potential_sdb_value || sdb_value)) + # Clean the path_in_sincedb because a file with the same name appears, this file must have been renamed + # Fix the duplicating collection after reloading sincedb(logstash restart or change logstash.conf) + sdb_value.add_path_in_sincedb(nil) unless sdb_value.nil? + logger.trace("---------- >>>> Rotation In Progress: after handling rotation", "this watched_file details" => watched_file.details, "sincedb_value" => (potential_sdb_value || sdb_value)) end end diff --git a/lib/filewatch/watched_file.rb b/lib/filewatch/watched_file.rb index 5d6876f3..ca3db693 100644 --- a/lib/filewatch/watched_file.rb +++ b/lib/filewatch/watched_file.rb @@ -104,6 +104,12 @@ def restat! if rotation_detected? # switch to new state now rotation_in_progress + # update the original file's size, there may be new data written after last read + if file_open? + original_stat = @file.stat + @size = original_stat.size + update_bytes_unread + end return true else @size = @stat.size