feat: ✨ Add chroma key support; make passthrough settings real time (#…

…2662) * feat: ✨ Add chroma key support; make passthrough settings real time * Use 4-point masking per channel * Enable/disabled passthrough layer at runtime * fix lints * Fix typo Co-authored-by: Daniël van Adrichem <[email protected]> --------- Co-authored-by: Daniël van Adrichem <[email protected]>
alvr-org · Feb 1, 2025 · 69e5e56 · 69e5e56
1 parent 626c6e6
commit 69e5e56
Show file tree

Hide file tree

Showing 14 changed files with 356 additions and 61 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/alvr/client_core/src/c_api.rs b/alvr/client_core/src/c_api.rs
@@ -81,6 +81,8 @@ pub enum AlvrEvent {
     DecoderConfig {
         codec: AlvrCodec,
     },
+    // Unimplemented
+    RealTimeConfig {},
 }
 
 #[repr(C)]
@@ -373,6 +375,7 @@ pub extern "C" fn alvr_poll_event(out_event: *mut AlvrEvent) -> bool {
                         },
                     }
                 }
+                ClientCoreEvent::RealTimeConfig(_) => AlvrEvent::RealTimeConfig {},
             };
 
             unsafe { *out_event = event };
@@ -792,7 +795,6 @@ pub unsafe extern "C" fn alvr_start_stream_opengl(config: AlvrStreamConfig) {
         true,
         false, // TODO: limited range fix config
         1.0,   // TODO: encoding gamma config
-        None,  // TODO: passthrough config
     )));
 }
 
@@ -852,6 +854,7 @@ pub unsafe extern "C" fn alvr_render_stream_opengl(
                         fov: from_capi_fov(right_params.fov),
                     },
                 ],
+                None,
             );
         }
     });

diff --git a/alvr/client_core/src/connection.rs b/alvr/client_core/src/connection.rs
@@ -492,6 +492,15 @@ fn connection_pipeline(
                         set_hud_message(&event_queue, SERVER_RESTART_MESSAGE);
                         disconnect_notif.notify_one();
                     }
+                    Ok(ServerControlPacket::ReservedBuffer(buffer)) => {
+                        // NB: it's normal for deserialization to fail if server has different
+                        // version
+                        if let Ok(config) = alvr_packets::decode_real_time_config(&buffer) {
+                            event_queue
+                                .lock()
+                                .push_back(ClientCoreEvent::RealTimeConfig(config));
+                        }
+                    }
                     Ok(_) => (),
                     Err(ConnectionError::TryAgain(_)) => {
                         if Instant::now() > disconnection_deadline {

diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs
@@ -25,8 +25,8 @@ use alvr_common::{
     HEAD_ID,
 };
 use alvr_packets::{
-    BatteryInfo, ButtonEntry, ClientControlPacket, FaceData, ReservedClientControlPacket,
-    StreamConfig, Tracking, ViewParams, ViewsConfig,
+    BatteryInfo, ButtonEntry, ClientControlPacket, FaceData, RealTimeConfig,
+    ReservedClientControlPacket, StreamConfig, Tracking, ViewParams, ViewsConfig,
 };
 use alvr_session::CodecType;
 use connection::{ConnectionContext, DecoderCallback};
@@ -55,6 +55,7 @@ pub enum ClientCoreEvent {
         codec: CodecType,
         config_nal: Vec<u8>,
     },
+    RealTimeConfig(RealTimeConfig),
 }
 
 // Note: this struct may change without breaking network protocol changes

diff --git a/alvr/client_mock/src/main.rs b/alvr/client_mock/src/main.rs
@@ -273,6 +273,7 @@ fn client_thread(
 
                     window_output.decoder_codec = Some(codec);
                 }
+                ClientCoreEvent::RealTimeConfig(_) => (),
             }
 
             output_sender.send(window_output.clone()).ok();

diff --git a/alvr/client_openxr/src/lib.rs b/alvr/client_openxr/src/lib.rs
@@ -427,6 +427,17 @@ pub fn entry_point() {
                             stream.maybe_initialize_decoder(codec, config_nal);
                         }
                     }
+                    ClientCoreEvent::RealTimeConfig(config) => {
+                        if config.passthrough.is_some() && passthrough_layer.is_none() {
+                            passthrough_layer = PassthroughLayer::new(&xr_session).ok();
+                        } else if config.passthrough.is_none() && passthrough_layer.is_some() {
+                            passthrough_layer = None;
+                        }
+
+                        if let Some(stream) = &mut stream_context {
+                            stream.update_real_time_config(&config);
+                        }
+                    }
                 }
             }
 

diff --git a/alvr/client_openxr/src/stream.rs b/alvr/client_openxr/src/stream.rs
@@ -14,7 +14,7 @@ use alvr_common::{
     Pose, RelaxedAtomic, HAND_LEFT_ID, HAND_RIGHT_ID, HEAD_ID,
 };
 use alvr_graphics::{GraphicsContext, StreamRenderer, StreamViewParams};
-use alvr_packets::{FaceData, StreamConfig, ViewParams};
+use alvr_packets::{FaceData, RealTimeConfig, StreamConfig, ViewParams};
 use alvr_session::{
     ClientsideFoveationConfig, ClientsideFoveationMode, CodecType, FoveatedEncodingConfig,
     MediacodecProperty, PassthroughMode,
@@ -185,7 +185,6 @@ impl StreamContext {
             platform != Platform::Lynx && !((platform.is_pico()) && config.enable_hdr),
             config.use_full_range && !config.enable_hdr, // TODO: figure out why HDR doesn't need the limited range hackfix in staging?
             config.encoding_gamma,
-            config.passthrough.clone(),
         );
 
         core_ctx.send_active_interaction_profile(
@@ -311,6 +310,10 @@ impl StreamContext {
         }
     }
 
+    pub fn update_real_time_config(&mut self, config: &RealTimeConfig) {
+        self.config.passthrough = config.passthrough.clone();
+    }
+
     pub fn render(
         &mut self,
         frame_interval: Duration,
@@ -368,6 +371,7 @@ impl StreamContext {
                         fov: view_params[1].fov,
                     },
                 ],
+                self.config.passthrough.as_ref(),
             )
         };
 
@@ -417,7 +421,10 @@ impl StreamContext {
                 .passthrough
                 .clone()
                 .map(|mode| ProjectionLayerAlphaConfig {
-                    premultiplied: !matches!(mode, PassthroughMode::Blend { .. }),
+                    premultiplied: matches!(
+                        mode,
+                        PassthroughMode::AugmentedReality { .. } | PassthroughMode::ChromaKey(_)
+                    ),
                 }),
         );
 

diff --git a/alvr/graphics/resources/stream.wgsl b/alvr/graphics/resources/stream.wgsl
@@ -9,37 +9,41 @@ override ENCODING_GAMMA: f32;
 
 override ENABLE_FFE: bool = false;
 
-override VIEW_WIDTH_RATIO: f32 = 0.;
-override VIEW_HEIGHT_RATIO: f32 = 0.;
-override EDGE_X_RATIO: f32 = 0.;
-override EDGE_Y_RATIO: f32 = 0.;
-
-override C1_X: f32 = 0.;
-override C1_Y: f32 = 0.;
-override C2_X: f32 = 0.;
-override C2_Y: f32 = 0.;
-override LO_BOUND_X: f32 = 0.;
-override LO_BOUND_Y: f32 = 0.;
-override HI_BOUND_X: f32 = 0.;
-override HI_BOUND_Y: f32 = 0.;
-
-override A_LEFT_X: f32 = 0.;
-override A_LEFT_Y: f32 = 0.;
-override B_LEFT_X: f32 = 0.;
-override B_LEFT_Y: f32 = 0.;
-
-override A_RIGHT_X: f32 = 0.;
-override A_RIGHT_Y: f32 = 0.;
-override B_RIGHT_X: f32 = 0.;
-override B_RIGHT_Y: f32 = 0.;
-override C_RIGHT_X: f32 = 0.;
-override C_RIGHT_Y: f32 = 0.;
-
-override COLOR_ALPHA: f32 = 1.0;
+override VIEW_WIDTH_RATIO: f32 = 0.0;
+override VIEW_HEIGHT_RATIO: f32 = 0.0;
+override EDGE_X_RATIO: f32 = 0.0;
+override EDGE_Y_RATIO: f32 = 0.0;
+
+override C1_X: f32 = 0.0;
+override C1_Y: f32 = 0.0;
+override C2_X: f32 = 0.0;
+override C2_Y: f32 = 0.0;
+override LO_BOUND_X: f32 = 0.0;
+override LO_BOUND_Y: f32 = 0.0;
+override HI_BOUND_X: f32 = 0.0;
+override HI_BOUND_Y: f32 = 0.0;
+
+override A_LEFT_X: f32 = 0.0;
+override A_LEFT_Y: f32 = 0.0;
+override B_LEFT_X: f32 = 0.0;
+override B_LEFT_Y: f32 = 0.0;
+
+override A_RIGHT_X: f32 = 0.0;
+override A_RIGHT_Y: f32 = 0.0;
+override B_RIGHT_X: f32 = 0.0;
+override B_RIGHT_Y: f32 = 0.0;
+override C_RIGHT_X: f32 = 0.0;
+override C_RIGHT_Y: f32 = 0.0;
 
 struct PushConstant {
     reprojection_transform: mat4x4f,
     view_idx: u32,
+    alpha: f32,
+    enable_chroma_key: u32,
+    _align: u32,
+    ck_hue: vec4f,
+    ck_saturation: vec4f,
+    ck_value: vec4f,
 }
 var<push_constant> pc: PushConstant;
 
@@ -127,5 +131,51 @@ fn fragment_main(@location(0) uv: vec2f) -> @location(0) vec4f {
         color = enc_condition * enc_lowValues + (1.0 - enc_condition) * enc_highValues;
     }
 
-    return vec4f(color, COLOR_ALPHA);
+    var alpha = pc.alpha;
+    if pc.enable_chroma_key == 1 {
+        let mask = chroma_key_mask(rgb_to_hsv(color));
+
+        // Note: because of this calculation, we require premultiplied alpha option in the XR layer
+        color = max(color * mask, vec3f(0.0));
+        alpha = mask;
+    }
+
+    return vec4f(color, alpha);
+}
+
+fn chroma_key_mask(hsv: vec3f) -> f32 {
+    let start_max = vec3f(pc.ck_hue.x, pc.ck_saturation.x, pc.ck_value.x);
+    let start_min = vec3f(pc.ck_hue.y, pc.ck_saturation.y, pc.ck_value.y);
+    let end_min = vec3f(pc.ck_hue.z, pc.ck_saturation.z, pc.ck_value.z);
+    let end_max = vec3f(pc.ck_hue.w, pc.ck_saturation.w, pc.ck_value.w);
+
+    let start_mask = smoothstep(start_min, start_max, hsv);
+    let end_mask = smoothstep(end_min, end_max, hsv);
+
+    return max(start_mask.x, max(start_mask.y, max(start_mask.z, max(end_mask.x, max(end_mask.y, end_mask.z)))));
+}
+
+fn rgb_to_hsv(rgb: vec3f) -> vec3f {
+    let cmax = max(rgb.r, max(rgb.g, rgb.b));
+    let cmin = min(rgb.r, min(rgb.g, rgb.b));
+    let delta = cmax - cmin;
+
+    var h = 0.0;
+    var s = 0.0;
+    let v = cmax;
+
+    if cmax > cmin {
+        s = delta / cmax;
+
+        if rgb.r == cmax {
+            h = (rgb.g - rgb.b) / delta;
+        } else if rgb.g == cmax {
+            h = 2.0 + (rgb.b - rgb.r) / delta;
+        } else {
+            h = 4.0 + (rgb.r - rgb.g) / delta;
+        }
+        h = fract(h / 6.0);
+    }
+
+    return vec3f(h, s, v);
 }
diff --git a/alvr/graphics/src/lib.rs b/alvr/graphics/src/lib.rs
@@ -21,7 +21,7 @@ use wgpu::{
 pub const SDR_FORMAT: TextureFormat = TextureFormat::Rgba8Unorm;
 pub const SDR_FORMAT_GL: u32 = gl::RGBA8;
 pub const GL_TEXTURE_EXTERNAL_OES: u32 = 0x8D65;
-pub const MAX_PUSH_CONSTANTS_SIZE: u32 = 72;
+pub const MAX_PUSH_CONSTANTS_SIZE: u32 = 128;
 
 type CreateImageFn = unsafe extern "C" fn(
     egl::EGLDisplay,
-Original file line number
+Diff line change
@@ Expand Up / @@ -273,6 +273,7 @@ fn client_thread( @@
                         window_output.decoder_codec = Some(codec);
                     }
+                    ClientCoreEvent::RealTimeConfig(_) => (),
                 }
                 output_sender.send(window_output.clone()).ok();
@@ Expand Down @@