diff --git a/Packages/com.unity.inputsystem/InputSystem/Plugins/UI/InputSystemUIInputModule.cs b/Packages/com.unity.inputsystem/InputSystem/Plugins/UI/InputSystemUIInputModule.cs
index c72e043c71..b922c52aa8 100644
--- a/Packages/com.unity.inputsystem/InputSystem/Plugins/UI/InputSystemUIInputModule.cs
+++ b/Packages/com.unity.inputsystem/InputSystem/Plugins/UI/InputSystemUIInputModule.cs
@@ -343,9 +343,12 @@ private void ProcessPointer(ref PointerModel state)
 
             ////REVIEW: for touch, we only need the left button; should we skip right and middle button processing? then we also don't need to copy to/from the event
 
+            var pointerId = eventData.pointerId;
+
             // Left mouse button. Movement and scrolling is processed with event set left button.
             eventData.button = PointerEventData.InputButton.Left;
             state.leftButton.CopyPressStateTo(eventData);
+            eventData.pointerId = PointerInputModule.kMouseLeftId;
 
             // Unlike StandaloneInputModule, we process moves before processing buttons. This way
             // UI elements get pointer enters/exits before they get button ups/downs and clicks.
@@ -357,7 +360,11 @@ private void ProcessPointer(ref PointerModel state)
             // However, after that, early out at this point when there's no changes to the pointer state (except
             // for tracked pointers as the tracking origin may have moved).
             if (!state.changedThisFrame && (xrTrackingOrigin == null || state.pointerType != UIPointerType.Tracked))
+            {
+                // Restore the original pointerId
+                eventData.pointerId = pointerId;
                 return;
+            }
 
             ProcessPointerButton(ref state.leftButton, eventData);
             ProcessPointerButtonDrag(ref state.leftButton, eventData);
@@ -366,6 +373,7 @@ private void ProcessPointer(ref PointerModel state)
             // Right mouse button.
             eventData.button = PointerEventData.InputButton.Right;
             state.rightButton.CopyPressStateTo(eventData);
+            eventData.pointerId = PointerInputModule.kMouseRightId;
 
             ProcessPointerButton(ref state.rightButton, eventData);
             ProcessPointerButtonDrag(ref state.rightButton, eventData);
@@ -373,9 +381,13 @@ private void ProcessPointer(ref PointerModel state)
             // Middle mouse button.
             eventData.button = PointerEventData.InputButton.Middle;
             state.middleButton.CopyPressStateTo(eventData);
+            eventData.pointerId = PointerInputModule.kMouseMiddleId;
 
             ProcessPointerButton(ref state.middleButton, eventData);
             ProcessPointerButtonDrag(ref state.middleButton, eventData);
+
+            // Restore the original pointerId
+            eventData.pointerId = pointerId;
         }
 
         // if we are using a MultiplayerEventSystem, ignore any transforms