diff --git a/hive-screen-mcp/src/main.rs b/hive-screen-mcp/src/main.rs index 1cdd7727..9cf3d53f 100644 --- a/hive-screen-mcp/src/main.rs +++ b/hive-screen-mcp/src/main.rs @@ -1,24 +1,17 @@ //! `hive-screen-mcp` — stdio MCP bridge for GUI agents. //! -//! Provides screenshot, keyboard-input, and mouse tools for agents running a -//! Weston Wayland compositor (`hyperhive.gui.enable = true`). Each tool: +//! Provides screenshot and keyboard-input tools for agents running a Weston +//! Wayland compositor (`hyperhive.gui.enable = true`). Each tool shells out +//! to the appropriate utility: //! //! - `screenshot` → `grim` (always available with gui enabled) -//! - `type_text` → `wtype` (Wayland virtual-keyboard protocol, no daemon) -//! - `key_press` → `wtype -k` (same protocol as `type_text`) -//! - `mouse_move` → RFB `PointerEvent` to the local neatvnc server -//! (`HIVE_GUI_VNC_PORT`, default 5900) -//! - `mouse_click` → RFB `PointerEvent` sequence: move → button-down → -//! button-up (all in one TCP connection) -//! -//! All tools operate in userspace. The keyboard tools use the Wayland -//! virtual-keyboard protocol; the mouse tools speak RFB directly to the -//! Weston VNC backend — the compositor's native remote-input path, no -//! `/dev/uinput` or kernel bypass. +//! - `type_text` → `wtype` (Wayland-native text input, no daemon) +//! - `key_press` → `wtype -k` (Wayland virtual-keyboard protocol, no daemon) //! +//! All three tools use Wayland protocols mediated by the compositor — +//! no `/dev/uinput` or kernel-level injection. //! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the -//! `weston-vnc` module. `HIVE_GUI_VNC_PORT` is set by the harness service -//! when `hyperhive.gui.enable = true` (defaults to 5900 if absent). +//! `weston-vnc` module so all tools find the compositor automatically. use anyhow::Result; use rmcp::{ @@ -29,8 +22,6 @@ use rmcp::{ transport::stdio, }; use serde::Deserialize; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpStream; use tokio::process::Command; /// Run a command and return its result. @@ -65,136 +56,6 @@ fn cmd_result(r: Result) -> String { } } -/// Read `HIVE_GUI_VNC_PORT` from the environment, falling back to 5900. -fn vnc_port() -> u16 { - std::env::var("HIVE_GUI_VNC_PORT") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or(5900) -} - -/// Perform the RFB 3.8 handshake on `stream` and return on success. -/// -/// Assumes the server advertises security type 1 (None) as set by -/// `auth-method=none` in `weston.ini`. Consumes the full `ServerInit` -/// frame so the caller is ready to send client messages. -async fn rfb_handshake(stream: &mut TcpStream) -> Result<(), String> { - // Protocol version — server sends 12 bytes ("RFB 003.xxx\n"), we - // reply with 3.8. - let mut ver = [0u8; 12]; - stream - .read_exact(&mut ver) - .await - .map_err(|e| format!("VNC: read version: {e}"))?; - stream - .write_all(b"RFB 003.008\n") - .await - .map_err(|e| format!("VNC: send version: {e}"))?; - - // Security type negotiation (RFB 3.8): server sends count + list. - let count = stream - .read_u8() - .await - .map_err(|e| format!("VNC: read sec-type count: {e}"))?; - let mut types = vec![0u8; count as usize]; - stream - .read_exact(&mut types) - .await - .map_err(|e| format!("VNC: read sec-types: {e}"))?; - if !types.contains(&1) { - return Err(format!( - "VNC: server did not offer security type None (got: {types:?})" - )); - } - stream - .write_all(&[1]) - .await - .map_err(|e| format!("VNC: send sec-type: {e}"))?; - - // SecurityResult: 4 bytes big-endian; 0 = OK. - let result = stream - .read_u32() - .await - .map_err(|e| format!("VNC: read sec-result: {e}"))?; - if result != 0 { - return Err(format!( - "VNC: security handshake rejected (result={result})" - )); - } - - // ClientInit: 1 byte; 1 = shared session (don't disconnect other clients). - stream - .write_all(&[1]) - .await - .map_err(|e| format!("VNC: send client-init: {e}"))?; - - // ServerInit: width(2) + height(2) + pixel-format(16) + - // name-length(4) + name. Read and discard. - stream - .read_u16() - .await - .map_err(|e| format!("VNC: read width: {e}"))?; - stream - .read_u16() - .await - .map_err(|e| format!("VNC: read height: {e}"))?; - let mut pf = [0u8; 16]; - stream - .read_exact(&mut pf) - .await - .map_err(|e| format!("VNC: read pixel-format: {e}"))?; - let name_len = stream - .read_u32() - .await - .map_err(|e| format!("VNC: read name-len: {e}"))?; - if name_len > 256 { - return Err(format!( - "VNC: server-init name length {name_len} exceeds cap (256)" - )); - } - let mut name = vec![0u8; name_len as usize]; - stream - .read_exact(&mut name) - .await - .map_err(|e| format!("VNC: read name: {e}"))?; - - Ok(()) -} - -/// Encode an RFB `PointerEvent` message (6 bytes). -/// -/// - `button_mask`: bitmask — bit 0 = left button, bit 1 = middle, bit 2 = right -/// - `x`, `y`: absolute pixel position (big-endian in the wire format) -fn rfb_pointer_event(button_mask: u8, x: u16, y: u16) -> [u8; 6] { - let xb = x.to_be_bytes(); - let yb = y.to_be_bytes(); - [5, button_mask, xb[0], xb[1], yb[0], yb[1]] -} - -/// Connect to the local neatvnc server, perform the RFB handshake, and -/// send a sequence of `PointerEvent` messages in one connection. -/// -/// `events` is a slice of `(button_mask, x, y)` tuples. Each is encoded -/// as an RFB `PointerEvent` and written in order; the connection is then -/// closed. Errors at any step short-circuit and return a `String`. -async fn rfb_send_pointer_events(port: u16, events: &[(u8, u16, u16)]) -> Result<(), String> { - let mut stream = TcpStream::connect(("127.0.0.1", port)) - .await - .map_err(|e| format!("VNC: connect to localhost:{port}: {e}"))?; - rfb_handshake(&mut stream).await?; - for &(mask, x, y) in events { - stream - .write_all(&rfb_pointer_event(mask, x, y)) - .await - .map_err(|e| format!("VNC: send pointer event: {e}"))?; - } - stream - .flush() - .await - .map_err(|e| format!("VNC: flush: {e}"))?; - Ok(()) -} - struct ScreenMcp; #[tool_router] @@ -257,53 +118,6 @@ impl ScreenMcp { let arg_refs: Vec<&str> = wtype_args.iter().map(String::as_str).collect(); cmd_result(run_cmd("wtype", &arg_refs).await) } - - #[tool( - description = "Move the mouse cursor to an absolute pixel position on the display. \ - `x` is measured from the left edge, `y` from the top edge. Coordinates are in \ - display pixels — use `screenshot` first to see the current resolution (the \ - display size can vary). Injects a RFB `PointerEvent` into the Weston VNC \ - backend — compositor-mediated, no kernel bypass." - )] - async fn mouse_move(&self, Parameters(args): Parameters) -> String { - let (Ok(x), Ok(y)) = (u16::try_from(args.x), u16::try_from(args.y)) else { - return "mouse_move: x and y must be in range 0–65535".to_owned(); - }; - match rfb_send_pointer_events(vnc_port(), &[(0, x, y)]).await { - Ok(()) => "ok".to_owned(), - Err(e) => e, - } - } - - #[tool( - description = "Click a mouse button at an absolute pixel position. `x` and `y` are \ - the target coordinates (0-based from the top-left) in display pixels — use \ - `screenshot` first to determine the current resolution (display size can vary). \ - `button` is `\"left\"` (default), `\"right\"`, or `\"middle\"`. Sends: move to \ - position → button-down → button-up as RFB `PointerEvent` messages to the Weston \ - VNC backend — compositor-mediated, no kernel bypass." - )] - async fn mouse_click(&self, Parameters(args): Parameters) -> String { - let (Ok(x), Ok(y)) = (u16::try_from(args.x), u16::try_from(args.y)) else { - return "mouse_click: x and y must be in range 0–65535".to_owned(); - }; - let btn_mask: u8 = match args.button.as_deref().unwrap_or("left") { - "left" => 0b0000_0001, - "right" => 0b0000_0100, - "middle" => 0b0000_0010, - other => { - return format!( - "mouse_click: unknown button {other:?} — use \"left\", \"right\", or \"middle\"" - ); - } - }; - // Move to position, press, release — all in one VNC connection. - let events = [(0, x, y), (btn_mask, x, y), (0, x, y)]; - match rfb_send_pointer_events(vnc_port(), &events).await { - Ok(()) => "ok".to_owned(), - Err(e) => e, - } - } } #[derive(Debug, Deserialize, JsonSchema)] @@ -319,24 +133,6 @@ struct KeyPressArgs { keys: String, } -#[derive(Debug, Deserialize, JsonSchema)] -struct MouseMoveArgs { - /// Horizontal pixel coordinate from the left edge of the display (0-based). - x: i32, - /// Vertical pixel coordinate from the top edge of the display (0-based). - y: i32, -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct MouseClickArgs { - /// Horizontal pixel coordinate from the left edge of the display (0-based). - x: i32, - /// Vertical pixel coordinate from the top edge of the display (0-based). - y: i32, - /// Button to click: `"left"` (default), `"right"`, or `"middle"`. - button: Option, -} - #[tool_handler] impl ServerHandler for ScreenMcp {} diff --git a/nix/agent-modules/screen.nix b/nix/agent-modules/screen.nix index 88858e2d..0ba0f16f 100644 --- a/nix/agent-modules/screen.nix +++ b/nix/agent-modules/screen.nix @@ -1,15 +1,13 @@ -# Screen MCP — screenshot, keyboard, and mouse for GUI agents. +# Screen MCP — screenshot + keyboard input for GUI agents. # # Auto-activated when `hyperhive.gui.enable = true`. Wires the # `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude -# gets five tools: `screenshot`, `type_text`, `key_press`, -# `mouse_move`, and `mouse_click`. +# gets three tools: `screenshot`, `type_text`, and `key_press`. # -# All tools operate in userspace without `/dev/uinput`: -# - `grim` takes screenshots via Wayland screencopy -# - `wtype` handles text + key combos via `zwp-virtual-keyboard-unstable-v1` -# - mouse tools speak RFB `PointerEvent` directly to the local neatvnc -# server (port `HIVE_GUI_VNC_PORT`) — the VNC backend's native input path +# All tools use Wayland protocols mediated by the Weston compositor — +# no `/dev/uinput` or kernel-level injection needed. `grim` takes +# screenshots; `wtype` handles both text input and key/modifier combos +# via the `zwp-virtual-keyboard-unstable-v1` protocol. { pkgs, lib,