diff --git a/panel/models/plotly.ts b/panel/models/plotly.ts
index 917db24a8e..f18c4e0875 100644
--- a/panel/models/plotly.ts
+++ b/panel/models/plotly.ts
@@ -56,6 +56,23 @@ const filterEventData = (gd: any, eventData: any, event: string) => {
       return null
     }
 
+    const event_obj = eventData["event"];
+    if (event_obj !== undefined) {
+      /** @type {InputDeviceState} */
+      var inputDeviceState = {
+        // Keyboard modifiers
+        alt: event_obj["altKey"],
+        ctrl: event_obj["ctrlKey"],
+        meta: event_obj["metaKey"],
+        shift: event_obj["shiftKey"],
+
+        // Mouse buttons
+        button: event_obj["button"],
+        buttons: event_obj["buttons"],
+      };
+      filteredEventData.device_state = inputDeviceState
+    }
+
     let selectorObject
     if (eventData.hasOwnProperty("range")) {
       // Box selection
diff --git a/panel/pane/plotly.py b/panel/pane/plotly.py
index ad992059e3..82121b6e0d 100644
--- a/panel/pane/plotly.py
+++ b/panel/pane/plotly.py
@@ -329,15 +329,16 @@ def _get_model(
 
     def _process_event(self, event):
         etype = event.data['type']
+        data = event.data['data']
         pname = f'{etype}_data'
-        if getattr(self, pname) == event.data['data']:
+        if getattr(self, pname) == data:
             self.param.trigger(pname)
         else:
-            self.param.update(**{pname: event.data['data']})
-        if event.data['data'] is None or not hasattr(self.object, '_handler_js2py_pointsCallback'):
+            self.param.update(**{pname: data})
+        if data is None or not hasattr(self.object, '_handler_js2py_pointsCallback'):
             return
 
-        points = event.data['data']['points']
+        points = data['points']
         num_points = len(points)
 
         has_nested_point_objects = True
@@ -396,7 +397,8 @@ def _process_event(self, event):
                 "new": dict(
                     event_type=f'plotly_{etype}',
                     points=points_object,
-                    selector=event.data['data']['selector']
+                    selector=data.get('selector', None),
+                    device_state=data.get('device_state', None)
                 )
             }
         )
diff --git a/panel/tests/ui/pane/test_plotly.py b/panel/tests/ui/pane/test_plotly.py
index 9ee09fbeb2..40fcf7673c 100644
--- a/panel/tests/ui/pane/test_plotly.py
+++ b/panel/tests/ui/pane/test_plotly.py
@@ -203,10 +203,14 @@ def test_plotly_click_data_figure_widget(page, plotly_2d_figure_widget):
         def check_click(i=i):
             if len(events) < (i+1):
                 return False
-            click_trace, points, _ = events[i]
+            click_trace, points, device_state = events[i]
             assert click_trace is trace
             assert points.xs == [0+i]
             assert points.ys == [2+i]
+            assert not device_state.ctrl
+            assert not device_state.alt
+            assert not device_state.shift
+            assert not device_state.meta
         wait_until(check_click, page)
         time.sleep(0.2)