Add VMC toggle to mirror tracking (#1043)

gui/public/i18n/en/translation.ftl

@@ -553,6 +553,9 @@ settings-osc-vmc-vrm-file_select = Drag & drop a model to use, or <u>browse</u>
 settings-osc-vmc-anchor_hip = Anchor at hips
 settings-osc-vmc-anchor_hip-description = Anchor the tracking at the hips, useful for seated VTubing. If disabling, load a VRM model.
 settings-osc-vmc-anchor_hip-label = Anchor at hips
+settings-osc-vmc-mirror_tracking = Mirror tracking
+settings-osc-vmc-mirror_tracking-description = Mirror the tracking horizontally.
+settings-osc-vmc-mirror_tracking-label = Mirror tracking
 
 ## Setup/onboarding menu
 onboarding-skip = Skip setup

gui/src/components/settings/pages/VMCSettings.tsx

@@ -32,6 +32,7 @@ interface VMCSettingsForm {
     };
     vrmJson?: FileList;
     anchorHip: boolean;
+    mirrorTracking: boolean;
   };
 }
 
@@ -44,6 +45,7 @@ const defaultValues = {
       address: '127.0.0.1',
     },
     anchorHip: true,
+    mirrorTracking: true,
   },
 };
 
@@ -80,6 +82,7 @@ export function VMCSettings() {
         }
       }
       vmcOsc.anchorHip = values.vmc.anchorHip;
+      vmcOsc.mirrorTracking = values.vmc.mirrorTracking;
 
       settings.vmcOsc = vmcOsc;
     }
@@ -115,6 +118,7 @@ export function VMCSettings() {
       }
 
       formData.vmc.anchorHip = settings.vmcOsc.anchorHip;
+      formData.vmc.mirrorTracking = settings.vmcOsc.mirrorTracking;
     }
 
     reset(formData);
@@ -272,6 +276,23 @@ export function VMCSettings() {
                 label={l10n.getString('settings-osc-vmc-anchor_hip-label')}
               />
             </div>
+            <Typography bold>
+              {l10n.getString('settings-osc-vmc-mirror_tracking')}
+            </Typography>
+            <div className="flex flex-col pb-2">
+              <Typography color="secondary">
+                {l10n.getString('settings-osc-vmc-mirror_tracking-description')}
+              </Typography>
+            </div>
+            <div className="grid grid-cols-2 gap-3 pb-5">
+              <CheckBox
+                variant="toggle"
+                outlined
+                control={control}
+                name="vmc.mirrorTracking"
+                label={l10n.getString('settings-osc-vmc-mirror_tracking-label')}
+              />
+            </div>
           </>
         </SettingsPagePaneLayout>
       </form>
@@ -281,6 +302,7 @@ export function VMCSettings() {
 
 const gltfHeaderStart = 0;
 const gltfHeaderEnd = 20;
+
 async function parseVRMFile(vrm: File): Promise<string | null> {
   const headerView = new DataView(
     await vrm.slice(gltfHeaderStart, gltfHeaderEnd).arrayBuffer()

server/core/src/main/java/dev/slimevr/config/VMCConfig.kt

@@ -7,4 +7,7 @@ class VMCConfig : OSCConfig() {
 
 	// JSON part of the VRM to be used
 	var vrmJson: String? = null
+
+	// Mirror the tracking before sending it (turn left <=> turn right, left leg <=> right leg)
+	var mirrorTracking = false
 }

server/core/src/main/java/dev/slimevr/osc/VMCHandler.kt

@@ -57,6 +57,7 @@ class VMCHandler(
 	private var timeAtLastError: Long = 0
 	private var timeAtLastSend: Long = 0
 	private var anchorHip = false
+	private var mirrorTracking = false
 	private var lastPortIn = 0
 	private var lastPortOut = 0
 	private var lastAddress: InetAddress? = null
@@ -67,6 +68,7 @@ class VMCHandler(
 
 	override fun refreshSettings(refreshRouterSettings: Boolean) {
 		anchorHip = config.anchorHip
+		mirrorTracking = config.mirrorTracking
 
 		// Stops listening and closes OSC port
 		val wasListening = oscReceiver != null && oscReceiver!!.isListening
@@ -330,16 +332,23 @@ class VMCHandler(
 					oscBundle.addPacket(OSCMessage("/VMC/Ext/Root/Pos", oscArgs.clone()))
 
 					for (unityBone in UnityBone.entries) {
-						val boneType = unityBone.boneType ?: continue
+						if (unityBone.boneType == null) continue
+
+						// Get opposite bone if tracking must be mirrored
+						val boneType = (if (mirrorTracking) tryGetOppositeArmBone(unityBone) else unityBone).boneType
+
 						// Get SlimeVR bone
-						val bone = humanPoseManager.getBone(boneType)
+						val bone = humanPoseManager.getBone(boneType!!)
 
 						// Update unity hierarchy from bone's global rotation
-						outputUnityArmature
-							?.setGlobalRotationForBone(
-								unityBone,
-								bone.getGlobalRotation() * bone.rotationOffset.inv(),
-							)
+						val boneRotation = if (mirrorTracking) {
+							// Mirror tracking horizontally
+							val rotBuf = bone.getGlobalRotation() * bone.rotationOffset.inv()
+							Quaternion(rotBuf.w, rotBuf.x, -rotBuf.y, -rotBuf.z)
+						} else {
+							bone.getGlobalRotation() * bone.rotationOffset.inv()
+						}
+						outputUnityArmature?.setGlobalRotationForBone(unityBone, boneRotation)
 					}
 
 					if (!anchorHip) {
@@ -366,28 +375,17 @@ class VMCHandler(
 
 					// Add Unity humanoid bones transforms
 					for (bone in UnityBone.entries) {
-						if (bone.boneType != null && !(
-								humanPoseManager.isTrackingLeftArmFromController &&
-									isLeftArmUnityBone(bone)
-								) &&
-							!(
-								humanPoseManager.isTrackingRightArmFromController &&
-									isRightArmUnityBone(bone)
-								)
+						if (bone.boneType != null &&
+							!(humanPoseManager.isTrackingLeftArmFromController && isLeftArmUnityBone(bone)) &&
+							!(humanPoseManager.isTrackingRightArmFromController && isRightArmUnityBone(bone))
 						) {
 							oscArgs.clear()
 							oscArgs.add(bone.stringVal)
 							addTransformToArgs(
 								outputUnityArmature!!.getLocalTranslationForBone(bone),
 								outputUnityArmature!!.getLocalRotationForBone(bone),
 							)
-							oscBundle
-								.addPacket(
-									OSCMessage(
-										"/VMC/Ext/Bone/Pos",
-										oscArgs.clone(),
-									),
-								)
+							oscBundle.addPacket(OSCMessage("/VMC/Ext/Bone/Pos", oscArgs.clone()))
 						}
 					}
 				}
@@ -480,9 +478,31 @@ class VMCHandler(
 		oscArgs.add(-rot.w)
 	}
 
-	private fun isLeftArmUnityBone(bone: UnityBone): Boolean = bone == UnityBone.LEFT_UPPER_ARM || bone == UnityBone.LEFT_LOWER_ARM || bone == UnityBone.LEFT_HAND
+	/**
+	 * Returns the bone on the opposite limb, or the original bone if
+	 * it not a limb bone.
+	 */
+	private fun tryGetOppositeArmBone(bone: UnityBone): UnityBone = when (bone) {
+		UnityBone.LEFT_SHOULDER -> UnityBone.RIGHT_SHOULDER
+		UnityBone.LEFT_UPPER_ARM -> UnityBone.RIGHT_UPPER_ARM
+		UnityBone.LEFT_LOWER_ARM -> UnityBone.RIGHT_LOWER_ARM
+		UnityBone.LEFT_HAND -> UnityBone.RIGHT_HAND
+		UnityBone.RIGHT_SHOULDER -> UnityBone.LEFT_SHOULDER
+		UnityBone.RIGHT_UPPER_ARM -> UnityBone.LEFT_UPPER_ARM
+		UnityBone.RIGHT_LOWER_ARM -> UnityBone.LEFT_LOWER_ARM
+		UnityBone.RIGHT_HAND -> UnityBone.LEFT_HAND
+		UnityBone.LEFT_UPPER_LEG -> UnityBone.RIGHT_UPPER_LEG
+		UnityBone.LEFT_LOWER_LEG -> UnityBone.RIGHT_LOWER_LEG
+		UnityBone.LEFT_FOOT -> UnityBone.RIGHT_FOOT
+		UnityBone.RIGHT_UPPER_LEG -> UnityBone.LEFT_UPPER_LEG
+		UnityBone.RIGHT_LOWER_LEG -> UnityBone.LEFT_LOWER_LEG
+		UnityBone.RIGHT_FOOT -> UnityBone.LEFT_FOOT
+		else -> bone
+	}
+
+	private fun isLeftArmUnityBone(bone: UnityBone): Boolean = bone == UnityBone.LEFT_SHOULDER || bone == UnityBone.LEFT_UPPER_ARM || bone == UnityBone.LEFT_LOWER_ARM || bone == UnityBone.LEFT_HAND
 
-	private fun isRightArmUnityBone(bone: UnityBone): Boolean = bone == UnityBone.RIGHT_UPPER_ARM || bone == UnityBone.RIGHT_LOWER_ARM || bone == UnityBone.RIGHT_HAND
+	private fun isRightArmUnityBone(bone: UnityBone): Boolean = bone == UnityBone.RIGHT_SHOULDER || bone == UnityBone.RIGHT_UPPER_ARM || bone == UnityBone.RIGHT_LOWER_ARM || bone == UnityBone.RIGHT_HAND
 
 	override fun getOscSender(): OSCPortOut = oscSender!!
 

server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsBuilder.java

@@ -98,6 +98,7 @@ public static int createVMCOSCSettings(
 		if (vrmJson != null)
 			VMCOSCSettings.addVrmJson(fbb, vrmJsonOffset);
 		VMCOSCSettings.addAnchorHip(fbb, config.getAnchorHip());
+		VMCOSCSettings.addMirrorTracking(fbb, config.getMirrorTracking());
 
 		return VMCOSCSettings.endVMCOSCSettings(fbb);
 	}

server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsHandler.kt

@@ -151,6 +151,7 @@ class RPCSettingsHandler(var rpcHandler: RPCHandler, var api: ProtocolAPI) {
 			}
 			if (req.vmcOsc().vrmJson() != null) vmcConfig.vrmJson = req.vmcOsc().vrmJson()
 			vmcConfig.anchorHip = req.vmcOsc().anchorHip()
+			vmcConfig.mirrorTracking = req.vmcOsc().mirrorTracking()
 
 			vmcHandler.refreshSettings(true)
 		}