diff --git a/lua/presence/init.lua b/lua/presence/init.lua
index 1d2aefe..f3bc45e 100644
--- a/lua/presence/init.lua
+++ b/lua/presence/init.lua
@@ -767,7 +767,7 @@ function Presence:get_buttons(buffer, parent_dirpath)
 end
 
 -- Update Rich Presence for the provided vim buffer
-function Presence:update_for_buffer(buffer, should_debounce)
+function Presence:update_for_buffer(buffer)
     -- Avoid unnecessary updates if the previous activity was for the current buffer
     -- (allow same-buffer updates when line numbers are enabled)
     if self.options.enable_line_number == 0 and self.last_activity.file == buffer then
@@ -801,9 +801,8 @@ function Presence:update_for_buffer(buffer, should_debounce)
     end
 
     local activity_set_at = os.time()
-    -- If we shouldn't debounce and we trigger an activity, keep this value the same.
-    -- Otherwise set it to the current time.
-    local relative_activity_set_at = should_debounce and self.last_activity.relative_set_at or os.time()
+    -- Set the relative time if it does not already exist
+    local relative_activity_set_at = self.last_activity.relative_set_at or os.time()
 
     self.log:debug(string.format("Setting activity for %s...", buffer and #buffer > 0 and buffer or "unnamed buffer"))
 
@@ -881,15 +880,15 @@ function Presence:update_for_buffer(buffer, should_debounce)
 
             if self.workspaces[project_path] then
                 self.workspaces[project_path].updated_at = activity_set_at
-                activity.timestamps = self.options.show_time == 1 and {
-                    start = self.workspaces[project_path].started_at,
-                } or nil
             else
                 self.workspaces[project_path] = {
                     started_at = activity_set_at,
                     updated_at = activity_set_at,
                 }
             end
+            activity.timestamps = self.options.show_time == 1 and {
+                start = self.workspaces[project_path].started_at,
+            } or nil
         else
             self.log:debug("No project detected")
 
@@ -954,10 +953,10 @@ Presence.update = Presence.discord_event(function(self, buffer, should_debounce)
     end
 
     if buffer then
-        self:update_for_buffer(buffer, should_debounce)
+        self:update_for_buffer(buffer)
     else
         vim.schedule(function()
-            self:update_for_buffer(self.get_current_buffer(), should_debounce)
+            self:update_for_buffer(self.get_current_buffer())
         end)
     end
 end)