use std::mem::size_of; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; use arboard::SetExtWindows; use color_eyre::eyre::{Result, WrapErr, eyre}; use windows_sys::Win32::Foundation::HWND; use windows_sys::Win32::System::DataExchange::{CountClipboardFormats, GetClipboardSequenceNumber}; use windows_sys::Win32::System::Threading::GetCurrentProcessId; use windows_sys::Win32::UI::Input::KeyboardAndMouse::{ GetAsyncKeyState, GetKeyboardLayout, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYEVENTF_KEYUP, MAPVK_VK_TO_VSC, MapVirtualKeyExW, SendInput, VK_CONTROL, VK_LWIN, VK_MENU, VK_RETURN, VK_RWIN, VK_SHIFT, VK_V, }; use windows_sys::Win32::UI::WindowsAndMessaging::{ GUITHREADINFO, GetClassNameW, GetForegroundWindow, GetGUIThreadInfo, GetWindowThreadProcessId, IsIconic, IsWindow, IsWindowVisible, SetForegroundWindow, }; use crate::continuation::{Continuation, InputActivity}; use crate::dictation::paste_target::is_shell_target; const CLIPBOARD_RESTORE_DELAY: Duration = Duration::from_millis(500); /// How long a paste waits for the window its capture started in to come back /// to the front, and how often it asks. const RETURN_TIMEOUT: Duration = Duration::from_millis(200); const RETURN_POLL: Duration = Duration::from_millis(10); /// Remote desktop clients (#866) forward a clipboard change to the remote /// session asynchronously, and the remote fetches the text only when it /// pastes. Ctrl+V sent at once can reach the remote before the new /// clipboard does, and a restore after 500 ms can replace the text before /// the remote asked for it, so both wait longer there. const REMOTE_DESKTOP_STEMS: &[&str] = &["mstsc", "msrdc"]; const REMOTE_DESKTOP_PASTE_DELAY: Duration = Duration::from_millis(400); const REMOTE_DESKTOP_RESTORE_DELAY: Duration = Duration::from_secs(3); fn is_remote_desktop(stem: Option<&str>) -> bool { stem.is_some_and(|stem| REMOTE_DESKTOP_STEMS.contains(&stem)) } /// Java games such as Minecraft (#1362; GLFW and LWJGL windows run inside /// `javaw`/`java`) take only Ctrl+V as paste and ignore Ctrl+Shift+V, so /// Terminal paste sends them plain Ctrl+V. const PLAIN_PASTE_STEMS: &[&str] = &["javaw", "java"]; fn paste_with_shift_into(stem: Option<&str>, terminal_paste: bool) -> bool { terminal_paste && !stem.is_some_and(|stem| PLAIN_PASTE_STEMS.contains(&stem)) } /// A paste with Ctrl+Shift+V when this capture asks for it (Labs › Terminal /// paste shortcut) or the saved Terminal paste setting is on. fn shift_paste(for_this_capture: bool, terminal_paste: bool) -> bool { for_this_capture || terminal_paste } /// What a paste did with the text. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Paste { /// The chord was sent to the window in front. Sent, /// Nothing was sent: the text stayed on the clipboard, because the window /// in front was the shell or Labs › Copy dictation only is on. Copied, } enum ClipboardSnapshot { Empty, Files(Vec), Html { html: String, text: Option }, Image(arboard::ImageData<'static>), Text(String), } impl ClipboardSnapshot { fn read(clipboard: &mut arboard::Clipboard) -> Result { if let Ok(html) = clipboard.get().html() { return Ok(Self::Html { html, text: clipboard.get_text().ok(), }); } if let Ok(text) = clipboard.get_text() { return Ok(Self::Text(text)); } if let Ok(files) = clipboard.get().file_list() && !files.is_empty() { return Ok(Self::Files(files)); } if let Ok(image) = clipboard.get().image() { return Ok(Self::Image(image)); } if unsafe { CountClipboardFormats() } == 0 { return Ok(Self::Empty); } Err(eyre!( "the current clipboard format cannot be preserved safely" )) } fn restore(self, clipboard: &mut arboard::Clipboard) -> Result<()> { match self { Self::Empty => clipboard.clear()?, Self::Files(files) => clipboard.set().file_list(&files)?, Self::Html { html, text } => clipboard.set().html(html, text)?, Self::Image(image) => clipboard.set_image(image)?, Self::Text(text) => clipboard.set_text(text)?, } Ok(()) } } /// The window in front as a paste sees it: its class and process, and the /// child of the foreground thread that holds the keyboard focus. #[derive(Default)] struct Foreground { window: isize, class: String, application: Option, focus_class: Option, } impl Foreground { /// Reads the window in front now, for one paste's decision and log line. fn read() -> Self { // SAFETY: plain Win32 queries; a null foreground window owns nothing. unsafe { let window = GetForegroundWindow(); if window.is_null() { return Self::default(); } Self { window: window as isize, class: window_class(window), application: crate::windows::input::window_process_path(window).and_then(|path| { path.file_stem() .map(|stem| stem.to_string_lossy().to_lowercase()) }), focus_class: focused_window_class(), } } } /// Whether a paste here would be lost in the shell. fn is_shell(&self) -> bool { is_shell_target( (self.window != 0).then_some(self.class.as_str()), self.application.as_deref(), self.focus_class.as_deref(), ) } } /// The window a capture started in, with the process that owned it then, so /// a reused handle is not mistaken for the same window. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct CaptureWindow { window: isize, pid: u32, } impl CaptureWindow { /// The window in front now, or `None` when nothing is. pub fn read() -> Option { // SAFETY: plain queries; a null foreground window owns no process. unsafe { let window = GetForegroundWindow(); if window.is_null() { return None; } let mut pid = 0_u32; GetWindowThreadProcessId(window, &mut pid); (pid != 0).then_some(Self { window: window as isize, pid, }) } } } /// The window's class name, empty when Windows has none to give. fn window_class(window: HWND) -> String { let mut buffer = [0_u16; 256]; // SAFETY: `buffer` is writable for the length it states. let length = unsafe { GetClassNameW(window, buffer.as_mut_ptr(), buffer.len() as i32) }; if length <= 0 { return String::new(); } String::from_utf16_lossy(&buffer[..length as usize]) } /// The keyboard-focused child of the foreground thread, if any. fn focused_window_class() -> Option { let mut info = GUITHREADINFO { cbSize: size_of::() as u32, ..Default::default() }; // SAFETY: `info` states its own size; a null thread asks for the one in front. let filled = unsafe { GetGUIThreadInfo(0, &mut info) } != 0; (filled && !info.hwndFocus.is_null()).then(|| window_class(info.hwndFocus)) } pub struct WindowsPaster { clipboard: arboard::Clipboard, clipboard_restore: Arc>, /// Terminal paste: send Ctrl+Shift+V instead of Ctrl+V. Shared so a /// settings change applies to the next paste without a new paster. paste_with_shift: Arc, /// The user's own input since the listener started: a paste joins the /// one before it only while this revision is unchanged. activity: InputActivity, continuation: Continuation, } #[derive(Default)] struct ClipboardRestore { generation: u64, original: Option, last_sequence: Option, } impl ClipboardRestore { fn register( &mut self, previous: Option, previous_sequence: u32, inserted_sequence: u32, ) -> u64 { self.generation = self.generation.wrapping_add(1); if self.last_sequence != Some(previous_sequence) { self.original = previous; } self.last_sequence = Some(inserted_sequence); self.generation } /// Labs › Keep dictation on clipboard: the paste stays on the clipboard, /// so no earlier snapshot may come back and any pending restore stands /// down. fn keep(&mut self) { self.generation = self.generation.wrapping_add(1); self.original = None; self.last_sequence = None; } } impl WindowsPaster { pub fn new(paste_with_shift: Arc, activity: InputActivity) -> Result { Ok(Self { clipboard: arboard::Clipboard::new() .wrap_err("could not open the Windows clipboard")?, clipboard_restore: Arc::new(Mutex::new(ClipboardRestore::default())), paste_with_shift, activity, continuation: Continuation::default(), }) } /// Pastes `text`, with Ctrl+Shift+V when `shift` asks for it or the /// saved Terminal paste setting is on. Java games still take Ctrl+V. /// `return_to` is the window the capture started in: the desktop or a /// taskbar takes no paste, so Hex returns there first and, when it can't, /// leaves the transcript on the clipboard instead of sending it nowhere. /// `standalone` pastes the text without joining the previous dictation /// and records no continuation for the next one (Labs › Signature). pub fn paste( &mut self, text: &str, shift: bool, return_to: Option, standalone: bool, ) -> Result { if text.trim().is_empty() { return Err(eyre!("refusing to paste an empty transcript")); } let revision = self.activity.revision(); let plain = text; let mut restore = self .clipboard_restore .lock() .unwrap_or_else(|error| error.into_inner()); let original_sequence = clipboard_sequence(); // Read as the paste commits, so the toggle applies at once. let keep = crate::labs::keep_on_clipboard(); // A clipboard Hex can't snapshot (an app's private format) never // blocks the paste: the transcript still goes in, and that // clipboard just isn't restored afterwards. Keeping the paste needs // no snapshot at all. let snapshot = if keep { None } else { match ClipboardSnapshot::read(&mut self.clipboard) { Ok(snapshot) => Some(snapshot), Err(error) => { tracing::warn!(%error, "pasting without restoring the previous Windows clipboard"); None } } }; wait_for_conflicting_modifiers(); // Resolve the target before the clipboard write: a copy for the shell // leaves the plain transcript, while a paste carries the // continuation's join. let mut target = Foreground::read(); let returned = target.is_shell() && return_to.is_some_and(bring_to_front); if returned { target = Foreground::read(); } tracing::info!( hwnd = target.window, class = %target.class, application = target.application.as_deref().unwrap_or(""), focus = target.focus_class.as_deref().unwrap_or(""), returned, "Windows paste target" ); if clipboard_sequence() != original_sequence { return Err(eyre!( "the clipboard changed while the transcript was being prepared" )); } let copied = target.is_shell(); let text = if copied || standalone { plain.to_string() } else { text_to_paste(&self.continuation, revision, plain) }; if keep { // Labs › Keep dictation on clipboard: an ordinary entry, so // Windows keeps it in clipboard history (Win+V) and cloud sync, // as macOS keeps its kept entry ordinary too. self.clipboard .set() .text(text.clone()) .wrap_err("could not place the transcript on the Windows clipboard")?; restore.keep(); } else { // Marked so Windows keeps the transcript out of clipboard history // (Win+V) and cloud sync, as macOS marks it transient (#851). // A copy for the shell keeps the same marking: the fallback is // still a dictation, not a kept one. self.clipboard .set() .exclude_from_history() .exclude_from_cloud() .text(text.clone()) .wrap_err("could not place the transcript on the Windows clipboard")?; } if copied { // Nothing in front takes a paste: the words stay on the // clipboard for Paste Last or a manual Ctrl+V instead of being // sent to the desktop or taskbar and lost. restore.keep(); drop(restore); tracing::warn!( class = %target.class, application = target.application.as_deref().unwrap_or(""), "the paste target is the Windows shell; the dictation stays on the clipboard" ); self.continuation.clear(); return Ok(Paste::Copied); } let generation = if keep { None } else { let transcript_sequence = clipboard_sequence(); Some(restore.register(snapshot, original_sequence, transcript_sequence)) }; drop(restore); let remote_desktop = is_remote_desktop(target.application.as_deref()); if remote_desktop { thread::sleep(REMOTE_DESKTOP_PASTE_DELAY); } let shift = paste_with_shift_into( target.application.as_deref(), shift_paste(shift, self.paste_with_shift.load(Ordering::Relaxed)), ); if let Err(error) = send_paste_shortcut(shift) { if let Some(generation) = generation { restore_clipboard_generation(&self.clipboard_restore, generation); } return Err(error); } // Only a paste that reached the app is continued by the next one, and // a standalone paste ends the continuation instead. self.continuation.record(revision, !standalone, text); let Some(generation) = generation else { return Ok(Paste::Sent); }; let clipboard_restore = self.clipboard_restore.clone(); let restore_delay = if remote_desktop { REMOTE_DESKTOP_RESTORE_DELAY } else { CLIPBOARD_RESTORE_DELAY }; thread::spawn(move || { thread::sleep(restore_delay); restore_clipboard_generation(&clipboard_restore, generation); }); Ok(Paste::Sent) } /// Labs › YOLO mode: one press of Return after a paste, once /// the paste has had [`crate::dictation::yolo_mode::SETTLE`] to land. pub fn press_return(&mut self) -> Result<()> { // Return sends the message, so the next dictation starts a new one, // as macOS's Send mode ends its continuation. self.continuation.clear(); thread::sleep(crate::dictation::yolo_mode::SETTLE); wait_for_conflicting_modifiers(); let scan = foreground_scan_code(VK_RETURN); let inputs = [ keyboard_input(VK_RETURN, scan, 0), keyboard_input(VK_RETURN, scan, KEYEVENTF_KEYUP), ]; let sent = send_inputs(&inputs); if sent != inputs.len() as u32 { send_inputs(&inputs[1..]); return Err(eyre!( "Windows accepted only {sent} of 2 synthetic Return events; an elevated target window may reject input" )); } Ok(()) } /// Labs › Copy dictation only: leaves `text` on the clipboard for the /// user to paste, with no keystroke and no restore. A pending restore /// stands down and no earlier snapshot comes back. pub fn copy_only(&mut self, text: &str) -> Result<()> { if text.trim().is_empty() { return Err(eyre!("refusing to copy an empty transcript")); } // Nothing was inserted, so the next paste starts a new // continuation. self.continuation.clear(); let mut restore = self .clipboard_restore .lock() .unwrap_or_else(|error| error.into_inner()); restore.keep(); self.clipboard .set() .text(text.to_string()) .wrap_err("could not place the transcript on the Windows clipboard") } } /// What a dictation pastes: `text`, joined onto the previous dictation when /// nothing in between ended the input session. Dictation and Paste Last /// both continue their dictation. fn text_to_paste(continuation: &Continuation, revision: u64, text: &str) -> String { continuation .join(revision, true, text) .unwrap_or_else(|| text.to_string()) } fn clipboard_sequence() -> u32 { unsafe { GetClipboardSequenceNumber() } } fn restore_clipboard_generation(clipboard_restore: &Arc>, generation: u64) { let mut restore = clipboard_restore .lock() .unwrap_or_else(|error| error.into_inner()); if restore.generation != generation { return; } if restore.last_sequence != Some(clipboard_sequence()) { restore.original = None; restore.last_sequence = None; return; } let Some(snapshot) = restore.original.take() else { restore.last_sequence = None; return; }; restore.last_sequence = None; let result = arboard::Clipboard::new() .wrap_err("could not reopen the Windows clipboard") .and_then(|mut clipboard| snapshot.restore(&mut clipboard)); if let Err(error) = result { tracing::warn!(%error, "could not restore the previous Windows clipboard"); } } /// Waits, without a deadline, until Shift, Alt, and Win are up: held, they /// would turn the synthesized Ctrl+V (or Ctrl+Shift+V, whose Shift Hex /// presses and releases itself) into a different shortcut. A held Ctrl /// is harmless, so the user may keep it down after finishing with a chord. fn wait_for_conflicting_modifiers() { while [VK_SHIFT, VK_MENU, VK_LWIN, VK_RWIN] .into_iter() .any(key_down) { thread::sleep(Duration::from_millis(10)); } } /// Ctrl plus `key` (Ctrl+Shift plus `key` with `shift`), leaving a Ctrl the /// user physically holds (`control_held`) untouched: synthesizing its /// release would leave Windows thinking it is up while the user holds it. /// Shift goes down after Ctrl and comes up before it. Each key carries its /// scan code (`scan`), as a physical press does. fn control_chord( key: u16, shift: bool, control_held: bool, scan: impl Fn(u16) -> u16, ) -> Vec { let input = |key, flags| keyboard_input(key, scan(key), flags); let mut inputs = Vec::with_capacity(6); if !control_held { inputs.push(input(VK_CONTROL, 0)); } if shift { inputs.push(input(VK_SHIFT, 0)); } inputs.push(input(key, 0)); inputs.push(input(key, KEYEVENTF_KEYUP)); if shift { inputs.push(input(VK_SHIFT, KEYEVENTF_KEYUP)); } if !control_held { inputs.push(input(VK_CONTROL, KEYEVENTF_KEYUP)); } inputs } /// The scan code of `key` in the foreground window's keyboard layout. /// Chromium apps derive `KeyboardEvent.code` from it, and VS Code matches /// its keybindings by that code: without one, its terminal never saw Ctrl+V /// as Paste and sent ^V to the program inside it (#618). fn foreground_scan_code(key: u16) -> u16 { // SAFETY: plain Win32 queries; a null process-ID pointer is allowed. unsafe { let thread = GetWindowThreadProcessId(GetForegroundWindow(), std::ptr::null_mut()); MapVirtualKeyExW(u32::from(key), MAPVK_VK_TO_VSC, GetKeyboardLayout(thread)) as u16 } } fn key_down(key: u16) -> bool { unsafe { GetAsyncKeyState(i32::from(key)) < 0 } } /// Whether `window` is the desktop or a taskbar rather than a window a paste /// can land in. The window is not in front, so the foreground thread's focus /// says nothing about it: its class and process alone decide. pub(crate) fn is_shell_window(window: HWND) -> bool { if window.is_null() { return true; } let class = window_class(window); if !crate::dictation::paste_target::is_shell_class(&class) { return false; } let application = crate::windows::input::window_process_path(window).and_then(|path| { path.file_stem() .map(|stem| stem.to_string_lossy().to_lowercase()) }); is_shell_target(Some(&class), application.as_deref(), None) } /// Brings the capture's window back to the front when it is still the same /// process's window and another process can paste into it, and says whether /// it arrived within [`RETURN_TIMEOUT`]. The shell, Hex's own window, a /// minimized window, and a handle whose process changed are left alone. fn bring_to_front(capture: CaptureWindow) -> bool { let window = capture.window as HWND; // SAFETY: plain queries on a window that may have closed or been reused // since it was remembered; `IsWindow`, the pid it had at capture start, // and `IsIconic` vet the handle before the foreground request. unsafe { if IsWindow(window) == 0 || IsWindowVisible(window) == 0 || IsIconic(window) != 0 { return false; } let mut pid = 0_u32; let thread = GetWindowThreadProcessId(window, &mut pid); if thread == 0 || pid != capture.pid || pid == GetCurrentProcessId() || is_shell_window(window) { return false; } SetForegroundWindow(window); } let started = Instant::now(); loop { // SAFETY: plain query. if unsafe { GetForegroundWindow() } == window { return true; } if started.elapsed() >= RETURN_TIMEOUT { return false; } thread::sleep(RETURN_POLL); } } fn send_paste_shortcut(shift: bool) -> Result<()> { let inputs = control_chord(VK_V, shift, key_down(VK_CONTROL), foreground_scan_code); let sent = send_inputs(&inputs); if sent != inputs.len() as u32 { // Release only what this chord pressed; never a key the user holds. let cleanup = inputs .iter() .filter(|input| unsafe { input.Anonymous.ki.dwFlags } & KEYEVENTF_KEYUP != 0) .copied() .collect::>(); send_inputs(&cleanup); return Err(eyre!( "Windows accepted only {sent} of {} synthetic paste events; an elevated target window may reject input", inputs.len() )); } Ok(()) } fn send_inputs(inputs: &[INPUT]) -> u32 { unsafe { SendInput( inputs.len() as u32, inputs.as_ptr(), size_of::() as i32, ) } } fn keyboard_input(key: u16, scan: u16, flags: u32) -> INPUT { INPUT { r#type: INPUT_KEYBOARD, Anonymous: INPUT_0 { ki: KEYBDINPUT { wVk: key, wScan: scan, dwFlags: flags, time: 0, dwExtraInfo: crate::windows::input::HEX_INJECTED, }, }, } } #[cfg(test)] mod tests { use super::*; #[test] fn back_to_back_dictations_join_until_input_breaks_the_session() { let activity = InputActivity::default(); let mut continuation = Continuation::default(); let revision = activity.revision(); continuation.record(revision, true, "day?".into()); assert_eq!( text_to_paste(&continuation, activity.revision(), "Oh yeah"), " Oh yeah" ); activity.invalidate(); assert_eq!( text_to_paste(&continuation, activity.revision(), "Oh yeah"), "Oh yeah" ); } #[test] fn rapid_pastes_keep_the_original_clipboard_for_the_latest_restore() { let mut restore = ClipboardRestore::default(); let first = restore.register(Some(ClipboardSnapshot::Text("original".into())), 10, 11); let second = restore.register(Some(ClipboardSnapshot::Text("first paste".into())), 11, 12); assert_ne!(first, second); assert_eq!(restore.last_sequence, Some(12)); assert!(matches!( restore.original.as_ref(), Some(ClipboardSnapshot::Text(text)) if text == "original" )); } #[test] fn keeping_the_paste_cancels_a_pending_restore() { let mut restore = ClipboardRestore::default(); let generation = restore.register(Some(ClipboardSnapshot::Text("original".into())), 10, 11); restore.keep(); assert_ne!(restore.generation, generation); assert!(restore.original.is_none()); assert_eq!(restore.last_sequence, None); } #[test] fn an_unreadable_clipboard_still_pastes_and_restores_nothing() { let mut restore = ClipboardRestore::default(); restore.register(Some(ClipboardSnapshot::Text("old".into())), 1, 2); restore.register(None, 10, 11); assert_eq!(restore.last_sequence, Some(11)); assert!(restore.original.is_none()); } #[test] fn remote_desktop_clients_get_the_slower_paste() { assert!(is_remote_desktop(Some("mstsc"))); assert!(is_remote_desktop(Some("msrdc"))); assert!(!is_remote_desktop(Some("notepad"))); assert!(!is_remote_desktop(None)); } #[test] fn terminal_paste_sends_plain_control_v_to_java_games() { assert!(paste_with_shift_into(Some("windowsterminal"), true)); assert!(paste_with_shift_into(None, true)); assert!(!paste_with_shift_into(Some("javaw"), true)); assert!(!paste_with_shift_into(Some("java"), true)); assert!(!paste_with_shift_into(Some("notepad"), false)); } #[test] fn a_terminal_paste_capture_overrides_the_saved_setting() { assert!(shift_paste(true, false)); assert!(shift_paste(false, true)); assert!(shift_paste(true, true)); assert!(!shift_paste(false, false)); } /// US-layout scan codes for the chord's keys. fn scan(key: u16) -> u16 { match key { VK_CONTROL => 0x1D, VK_SHIFT => 0x2A, VK_V => 0x2F, _ => 0, } } #[test] fn paste_chords_carry_each_keys_scan_code() { let inputs = control_chord(VK_V, true, false, scan); assert!(inputs.iter().all(|input| { let ki = unsafe { input.Anonymous.ki }; ki.wScan == scan(ki.wVk) && ki.wScan != 0 })); } fn keys(inputs: &[INPUT]) -> Vec<(u16, bool)> { inputs .iter() .map(|input| { let ki = unsafe { input.Anonymous.ki }; (ki.wVk, ki.dwFlags & KEYEVENTF_KEYUP != 0) }) .collect() } #[test] fn paste_chords_press_shift_inside_control_and_leave_a_held_control_alone() { assert_eq!( keys(&control_chord(VK_V, false, false, scan)), [ (VK_CONTROL, false), (VK_V, false), (VK_V, true), (VK_CONTROL, true) ] ); assert_eq!( keys(&control_chord(VK_V, false, true, scan)), [(VK_V, false), (VK_V, true)] ); assert_eq!( keys(&control_chord(VK_V, true, false, scan)), [ (VK_CONTROL, false), (VK_SHIFT, false), (VK_V, false), (VK_V, true), (VK_SHIFT, true), (VK_CONTROL, true) ] ); assert_eq!( keys(&control_chord(VK_V, true, true, scan)), [ (VK_SHIFT, false), (VK_V, false), (VK_V, true), (VK_SHIFT, true) ] ); } }