diff --git a/hive-screen-mcp/src/main.rs b/hive-screen-mcp/src/main.rs index 9cf3d53f..7f5fa485 100644 --- a/hive-screen-mcp/src/main.rs +++ b/hive-screen-mcp/src/main.rs @@ -1,17 +1,24 @@ //! `hive-screen-mcp` — stdio MCP bridge for GUI agents. //! -//! Provides screenshot and keyboard-input tools for agents running a Weston -//! Wayland compositor (`hyperhive.gui.enable = true`). Each tool shells out -//! to the appropriate utility: +//! Provides screenshot, keyboard-input, and mouse tools for agents running a +//! Weston Wayland compositor (`hyperhive.gui.enable = true`). Each tool: //! //! - `screenshot` → `grim` (always available with gui enabled) -//! - `type_text` → `wtype` (Wayland-native text input, no daemon) -//! - `key_press` → `wtype -k` (Wayland virtual-keyboard protocol, no daemon) +//! - `type_text` → `wtype` (Wayland virtual-keyboard protocol, no daemon) +//! - `key_press` → `wtype -k` (same protocol as `type_text`) +//! - `mouse_move` → RFB `PointerEvent` to the local neatvnc server +//! (`HIVE_GUI_VNC_PORT`, default 5900) +//! - `mouse_click` → RFB `PointerEvent` sequence: move → button-down → +//! button-up (all in one TCP connection) +//! +//! All tools operate in userspace. The keyboard tools use the Wayland +//! virtual-keyboard protocol; the mouse tools speak RFB directly to the +//! Weston VNC backend — the compositor's native remote-input path, no +//! `/dev/uinput` or kernel bypass. //! -//! All three tools use Wayland protocols mediated by the compositor — -//! no `/dev/uinput` or kernel-level injection. //! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the -//! `weston-vnc` module so all tools find the compositor automatically. +//! `weston-vnc` module. `HIVE_GUI_VNC_PORT` is set by the harness service +//! when `hyperhive.gui.enable = true` (defaults to 5900 if absent). use anyhow::Result; use rmcp::{ @@ -22,6 +29,8 @@ use rmcp::{ transport::stdio, }; use serde::Deserialize; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; use tokio::process::Command; /// Run a command and return its result. @@ -56,6 +65,131 @@ fn cmd_result(r: Result) -> String { } } +/// Read `HIVE_GUI_VNC_PORT` from the environment, falling back to 5900. +fn vnc_port() -> u16 { + std::env::var("HIVE_GUI_VNC_PORT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(5900) +} + +/// Perform the RFB 3.8 handshake on `stream` and return on success. +/// +/// Assumes the server advertises security type 1 (None) as set by +/// `auth-method=none` in `weston.ini`. Consumes the full `ServerInit` +/// frame so the caller is ready to send client messages. +async fn rfb_handshake(stream: &mut TcpStream) -> Result<(), String> { + // Protocol version — server sends 12 bytes ("RFB 003.xxx\n"), we + // reply with 3.8. + let mut ver = [0u8; 12]; + stream + .read_exact(&mut ver) + .await + .map_err(|e| format!("VNC: read version: {e}"))?; + stream + .write_all(b"RFB 003.008\n") + .await + .map_err(|e| format!("VNC: send version: {e}"))?; + + // Security type negotiation (RFB 3.8): server sends count + list. + let count = stream + .read_u8() + .await + .map_err(|e| format!("VNC: read sec-type count: {e}"))?; + let mut types = vec![0u8; count as usize]; + stream + .read_exact(&mut types) + .await + .map_err(|e| format!("VNC: read sec-types: {e}"))?; + if !types.contains(&1) { + return Err(format!( + "VNC: server did not offer security type None (got: {types:?})" + )); + } + stream + .write_all(&[1]) + .await + .map_err(|e| format!("VNC: send sec-type: {e}"))?; + + // SecurityResult: 4 bytes big-endian; 0 = OK. + let result = stream + .read_u32() + .await + .map_err(|e| format!("VNC: read sec-result: {e}"))?; + if result != 0 { + return Err(format!( + "VNC: security handshake rejected (result={result})" + )); + } + + // ClientInit: 1 byte; 1 = shared session (don't disconnect other clients). + stream + .write_all(&[1]) + .await + .map_err(|e| format!("VNC: send client-init: {e}"))?; + + // ServerInit: width(2) + height(2) + pixel-format(16) + + // name-length(4) + name. Read and discard. + stream + .read_u16() + .await + .map_err(|e| format!("VNC: read width: {e}"))?; + stream + .read_u16() + .await + .map_err(|e| format!("VNC: read height: {e}"))?; + let mut pf = [0u8; 16]; + stream + .read_exact(&mut pf) + .await + .map_err(|e| format!("VNC: read pixel-format: {e}"))?; + let name_len = stream + .read_u32() + .await + .map_err(|e| format!("VNC: read name-len: {e}"))?; + let mut name = vec![0u8; name_len as usize]; + stream + .read_exact(&mut name) + .await + .map_err(|e| format!("VNC: read name: {e}"))?; + + Ok(()) +} + +/// Encode an RFB `PointerEvent` message (6 bytes). +/// +/// - `button_mask`: bitmask — bit 0 = left button, bit 1 = middle, bit 2 = right +/// - `x`, `y`: absolute pixel position (big-endian in the wire format) +fn rfb_pointer_event(button_mask: u8, x: u16, y: u16) -> [u8; 6] { + let xb = x.to_be_bytes(); + let yb = y.to_be_bytes(); + [5, button_mask, xb[0], xb[1], yb[0], yb[1]] +} + +/// Connect to the local neatvnc server, perform the RFB handshake, and +/// send a sequence of `PointerEvent` messages in one connection. +/// +/// `events` is a slice of `(button_mask, x, y)` tuples. Each is encoded +/// as an RFB `PointerEvent` and written in order; the connection is then +/// closed. Errors at any step short-circuit and return a `String`. +async fn rfb_send_pointer_events(port: u16, events: &[(u8, u16, u16)]) -> Result<(), String> { + let mut stream = TcpStream::connect(("127.0.0.1", port)) + .await + .map_err(|e| format!("VNC: connect to localhost:{port}: {e}"))?; + rfb_handshake(&mut stream).await?; + for &(mask, x, y) in events { + stream + .write_all(&rfb_pointer_event(mask, x, y)) + .await + .map_err(|e| format!("VNC: send pointer event: {e}"))?; + } + stream + .flush() + .await + .map_err(|e| format!("VNC: flush: {e}"))?; + Ok(()) +} + struct ScreenMcp; #[tool_router] @@ -118,6 +252,47 @@ impl ScreenMcp { let arg_refs: Vec<&str> = wtype_args.iter().map(String::as_str).collect(); cmd_result(run_cmd("wtype", &arg_refs).await) } + + #[tool( + description = "Move the mouse cursor to an absolute pixel position on the display. \ + `x` is measured from the left edge, `y` from the top edge. The display is \ + 1280×720 pixels when `hyperhive.gui.enable = true`. Injects a RFB \ + `PointerEvent` into the Weston VNC backend — compositor-mediated, \ + no kernel bypass." + )] + async fn mouse_move(&self, Parameters(args): Parameters) -> String { + let (Ok(x), Ok(y)) = (u16::try_from(args.x), u16::try_from(args.y)) else { + return "mouse_move: x and y must be in range 0–65535".to_owned(); + }; + match rfb_send_pointer_events(vnc_port(), &[(0, x, y)]).await { + Ok(()) => "ok".to_owned(), + Err(e) => e, + } + } + + #[tool( + description = "Click a mouse button at an absolute pixel position. `x` and `y` are \ + the target coordinates (0-based from the top-left, display is 1280×720). \ + `button` is `\"left\"` (default), `\"right\"`, or `\"middle\"`. Sends: move \ + to position → button-down → button-up as RFB `PointerEvent` messages to the \ + Weston VNC backend — compositor-mediated, no kernel bypass." + )] + async fn mouse_click(&self, Parameters(args): Parameters) -> String { + let (Ok(x), Ok(y)) = (u16::try_from(args.x), u16::try_from(args.y)) else { + return "mouse_click: x and y must be in range 0–65535".to_owned(); + }; + let btn_mask: u8 = match args.button.as_deref().unwrap_or("left") { + "right" => 0b0000_0100, + "middle" => 0b0000_0010, + _ => 0b0000_0001, // left + }; + // Move to position, press, release — all in one VNC connection. + let events = [(0, x, y), (btn_mask, x, y), (0, x, y)]; + match rfb_send_pointer_events(vnc_port(), &events).await { + Ok(()) => "ok".to_owned(), + Err(e) => e, + } + } } #[derive(Debug, Deserialize, JsonSchema)] @@ -133,6 +308,24 @@ struct KeyPressArgs { keys: String, } +#[derive(Debug, Deserialize, JsonSchema)] +struct MouseMoveArgs { + /// Horizontal pixel coordinate from the left edge of the display (0-based). + x: i32, + /// Vertical pixel coordinate from the top edge of the display (0-based). + y: i32, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct MouseClickArgs { + /// Horizontal pixel coordinate from the left edge of the display (0-based). + x: i32, + /// Vertical pixel coordinate from the top edge of the display (0-based). + y: i32, + /// Button to click: `"left"` (default), `"right"`, or `"middle"`. + button: Option, +} + #[tool_handler] impl ServerHandler for ScreenMcp {} diff --git a/nix/agent-modules/screen.nix b/nix/agent-modules/screen.nix index 0ba0f16f..88858e2d 100644 --- a/nix/agent-modules/screen.nix +++ b/nix/agent-modules/screen.nix @@ -1,13 +1,15 @@ -# Screen MCP — screenshot + keyboard input for GUI agents. +# Screen MCP — screenshot, keyboard, and mouse for GUI agents. # # Auto-activated when `hyperhive.gui.enable = true`. Wires the # `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude -# gets three tools: `screenshot`, `type_text`, and `key_press`. +# gets five tools: `screenshot`, `type_text`, `key_press`, +# `mouse_move`, and `mouse_click`. # -# All tools use Wayland protocols mediated by the Weston compositor — -# no `/dev/uinput` or kernel-level injection needed. `grim` takes -# screenshots; `wtype` handles both text input and key/modifier combos -# via the `zwp-virtual-keyboard-unstable-v1` protocol. +# All tools operate in userspace without `/dev/uinput`: +# - `grim` takes screenshots via Wayland screencopy +# - `wtype` handles text + key combos via `zwp-virtual-keyboard-unstable-v1` +# - mouse tools speak RFB `PointerEvent` directly to the local neatvnc +# server (port `HIVE_GUI_VNC_PORT`) — the VNC backend's native input path { pkgs, lib,