Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: improve focus of captured windows #1096

Closed
wants to merge 1 commit into from
Closed

refactor: improve focus of captured windows #1096

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions screenpipe-vision/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ serde = "1.0.200"

once_cell = { workspace = true }

xcap = "0.2.1"


[dev-dependencies]
tempfile = "3.3.0"
criterion = { workspace = true }
Expand Down Expand Up @@ -88,13 +91,10 @@ path = "examples/websocket.rs"

[target.'cfg(target_os = "windows")'.dependencies]
windows = { version = "0.58", features = ["Graphics_Imaging", "Media_Ocr", "Storage", "Storage_Streams"] }
xcap = "0.0.12"

[target.'cfg(target_os = "macos")'.dependencies]
libc = "=0.2.164"
xcap-macos = { package = "xcap", git = "https://github.com/mediar-ai/xcap", rev = "965bc99" }
cidre = { git = "https://github.com/yury/cidre.git", version = "0.5.0" }

[target.'cfg(target_os = "linux")'.dependencies]
libc = "=0.2.164"
xcap = "0.0.14"
169 changes: 77 additions & 92 deletions screenpipe-vision/src/capture_screenshot_by_window.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,6 @@ use std::fmt;
use std::time::Duration;
use tokio::time;

#[cfg(target_os = "macos")]
use xcap_macos::{Monitor, Window, XCapError};

#[cfg(not(target_os = "macos"))]
use xcap::{Monitor, Window, XCapError};

#[derive(Debug)]
Expand Down Expand Up @@ -39,102 +35,102 @@ impl From<XCapError> for CaptureError {

// Platform specific skip lists
#[cfg(target_os = "macos")]
static SKIP_APPS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
static SKIP_APPS: Lazy<HashSet<String>> = Lazy::new(|| {
HashSet::from([
"Window Server",
"SystemUIServer",
"ControlCenter",
"Dock",
"NotificationCenter",
"loginwindow",
"WindowManager",
"Contexts",
"Screenshot",
String::from("window server"),
String::from("systemuiserver"),
String::from("controlcenter"),
String::from("dock"),
String::from("notificationcenter"),
String::from("loginwindow"),
String::from("windowmanager"),
String::from("contexts"),
String::from("screenshot"),
])
});

#[cfg(target_os = "windows")]
static SKIP_APPS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
static SKIP_APPS: Lazy<HashSet<String>> = Lazy::new(|| {
HashSet::from([
"Windows Shell Experience Host",
"Microsoft Text Input Application",
"Windows Explorer",
"Program Manager",
"Microsoft Store",
"Search",
"TaskBar",
String::from("windows shell experience host"),
String::from("microsoft text input application"),
String::from("windows explorer"),
String::from("program manager"),
String::from("microsoft store"),
String::from("search"),
String::from("taskbar"),
])
});

#[cfg(target_os = "linux")]
static SKIP_APPS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
static SKIP_APPS: Lazy<HashSet<String>> = Lazy::new(|| {
HashSet::from([
"Gnome-shell",
"Plasma",
"Xfdesktop",
"Polybar",
"i3bar",
"Plank",
"Dock",
String::from("gnome-shell"),
String::from("plasma"),
String::from("xfdesktop"),
String::from("polybar"),
String::from("i3bar"),
String::from("plank"),
String::from("dock"),
])
});

#[cfg(target_os = "macos")]
static SKIP_TITLES: Lazy<HashSet<&'static str>> = Lazy::new(|| {
static SKIP_TITLES: Lazy<HashSet<String>> = Lazy::new(|| {
HashSet::from([
"Item-0",
"App Icon Window",
"Dock",
"NowPlaying",
"FocusModes",
"Shortcuts",
"AudioVideoModule",
"Clock",
"WiFi",
"Battery",
"BentoBox",
"Menu Bar",
"Notification Center",
"Control Center",
"Spotlight",
"Mission Control",
"Desktop",
"Screen Sharing",
"Touch Bar",
"Status Bar",
"Menu Extra",
"System Settings",
String::from("item-0"),
String::from("app icon window"),
String::from("dock"),
String::from("nowplaying"),
String::from("focusmodes"),
String::from("shortcuts"),
String::from("audiovideomodule"),
String::from("clock"),
String::from("wifi"),
String::from("battery"),
String::from("bentobox"),
String::from("menubar"),
String::from("notification center"),
String::from("control center"),
String::from("spotlight"),
String::from("mission control"),
String::from("desktop"),
String::from("screen sharing"),
String::from("touch bar"),
String::from("status bar"),
String::from("menu extra"),
String::from("system settings"),
])
});

#[cfg(target_os = "windows")]
static SKIP_TITLES: Lazy<HashSet<&'static str>> = Lazy::new(|| {
static SKIP_TITLES: Lazy<HashSet<String>> = Lazy::new(|| {
HashSet::from([
"Program Manager",
"Windows Input Experience",
"Microsoft Text Input Application",
"Task View",
"Start",
"System Tray",
"Notification Area",
"Action Center",
"Task Bar",
"Desktop",
String::from("program manager"),
String::from("windows input experience"),
String::from("microsoft text input application"),
String::from("task view"),
String::from("start"),
String::from("system tray"),
String::from("notification area"),
String::from("action center"),
String::from("task bar"),
String::from("desktop"),
])
});

#[cfg(target_os = "linux")]
static SKIP_TITLES: Lazy<HashSet<&'static str>> = Lazy::new(|| {
static SKIP_TITLES: Lazy<HashSet<String>> = Lazy::new(|| {
HashSet::from([
"Desktop",
"Panel",
"Top Bar",
"Status Bar",
"Dock",
"Dashboard",
"Activities",
"System Tray",
"Notification Area",
String::from("desktop"),
String::from("panel"),
String::from("top bar"),
String::from("status bar"),
String::from("dock"),
String::from("dashboard"),
String::from("activities"),
String::from("system tray"),
String::from("notification area"),
])
});

Expand All @@ -161,19 +157,11 @@ impl WindowFilters {

// O(n) - we could figure out a better way to do this
pub fn is_valid(&self, app_name: &str, title: &str) -> bool {
let app_name_lower = app_name.to_lowercase();
let title_lower = title.to_lowercase();

// If include list is empty, we're done
if self.include_set.is_empty() {
return true;
}

// Check include list
if self
.include_set
.iter()
.any(|include| app_name_lower.contains(include) || title_lower.contains(include))
.any(|include| app_name.contains(include) || title == include)
{
return true;
}
Expand All @@ -183,7 +171,7 @@ impl WindowFilters {
&& self
.ignore_set
.iter()
.any(|ignore| app_name_lower.contains(ignore) || title_lower.contains(ignore))
.any(|ignore| app_name.contains(ignore) || title == ignore)
{
return false;
}
Expand Down Expand Up @@ -261,26 +249,23 @@ pub fn is_valid_window(
) -> bool {
if !capture_unfocused_windows {
// Early returns for simple checks
#[cfg(target_os = "macos")]
let is_focused = window.current_monitor().id() == monitor.id() && window.is_focused();

#[cfg(not(target_os = "macos"))]
let is_focused = window.current_monitor().id() == monitor.id() && !window.is_minimized();
let is_focused = window.current_monitor().id() == monitor.id() && window.z() == 0;

if !is_focused {
return false;
}
}

// Fast O(1) lookups using HashSet
let app_name = window.app_name();
let title = window.title();
let app_name = window.app_name().to_lowercase();
let title = window.title().to_lowercase();

if SKIP_APPS.contains(app_name) || SKIP_TITLES.contains(title) {
if SKIP_APPS.contains(&app_name) || SKIP_TITLES.contains(&title) {
return false;
}

filters.is_valid(app_name, title)
filters.is_valid(&app_name, &title)
}

async fn retry_with_backoff<F, T, E>(
Expand Down
5 changes: 0 additions & 5 deletions screenpipe-vision/src/core.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,10 @@
use std::{
collections::HashMap,
time::{Duration, Instant},
sync::OnceLock,

Check warning on line 23 in screenpipe-vision/src/core.rs

View workflow job for this annotation

GitHub Actions / test-windows 

unused import: `sync::OnceLock`

Check warning on line 23 in screenpipe-vision/src/core.rs

View workflow job for this annotation

GitHub Actions / test-ubuntu 

unused import: `sync::OnceLock`

Check warning on line 23 in screenpipe-vision/src/core.rs

View workflow job for this annotation

GitHub Actions / test-ubuntu 

unused import: `sync::OnceLock`

Check warning on line 23 in screenpipe-vision/src/core.rs

View workflow job for this annotation

GitHub Actions / test-linux 

unused import: `sync::OnceLock`

Check warning on line 23 in screenpipe-vision/src/core.rs

View workflow job for this annotation

GitHub Actions / test-windows 

unused import: `sync::OnceLock`

Check warning on line 23 in screenpipe-vision/src/core.rs

View workflow job for this annotation

GitHub Actions / test-windows 

unused import: `sync::OnceLock`
};
use tokio::sync::mpsc::Sender;
use tokio::time::sleep;

#[cfg(target_os = "macos")]
use xcap_macos::Monitor;

#[cfg(not(target_os = "macos"))]
use xcap::Monitor;

#[cfg(target_os = "macos")]
Expand Down
4 changes: 0 additions & 4 deletions screenpipe-vision/src/monitor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
#[cfg(target_os = "macos")]
use xcap_macos::Monitor;

#[cfg(not(target_os = "macos"))]
use xcap::Monitor;

pub async fn list_monitors() -> Vec<Monitor> {
Expand Down
4 changes: 0 additions & 4 deletions screenpipe-vision/src/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,6 @@ use log::{debug, error, warn};
use std::hash::{DefaultHasher, Hash, Hasher};
use std::time::{Duration, Instant};

#[cfg(target_os = "macos")]
use xcap_macos::Monitor;

#[cfg(not(target_os = "macos"))]
use xcap::Monitor;

#[derive(Clone, Debug, Copy)]
Expand Down
Loading