feat(#2618): add mouse_move + mouse_click via RFB PointerEvent
Implements mouse input by speaking the RFB protocol directly to Weston's neatvnc server (localhost:HIVE_GUI_VNC_PORT, default 5900) — the VNC backend's native remote-input path. No /dev/uinput, no kernel bypass; the compositor mediates all input just as it does for the browser VNC viewer. Changes: - rfb_handshake(): RFB 3.8 handshake with security type None (auth-method=none in weston.ini); shared-session ClientInit keeps the browser viewer connected - rfb_pointer_event(): encodes a 6-byte RFB PointerEvent (type=5, button-mask, x/y big-endian) - rfb_send_pointer_events(): connects, handshakes, sends an event slice, flushes — all in one TCP connection - mouse_move(x, y): sends a single PointerEvent(mask=0, x, y) - mouse_click(x, y, button): sends move → button-down → button-up sequence (left/middle/right via RFB button-mask bits 0/1/2) - vnc_port(): reads HIVE_GUI_VNC_PORT from env, falls back to 5900 No new packages or nix options — HIVE_GUI_VNC_PORT is already set by the harness when gui.enable = true; grim/wtype are the only runtime deps. Closes #2618.
This commit is contained in:
iris
parent
804c11c404
commit
228a5bacca
2 changed files with 209 additions and 14 deletions
|
|
@ -1,17 +1,24 @@
|
|||
//! `hive-screen-mcp` — stdio MCP bridge for GUI agents.
|
||||
//!
|
||||
//! Provides screenshot and keyboard-input tools for agents running a Weston
|
||||
//! Wayland compositor (`hyperhive.gui.enable = true`). Each tool shells out
|
||||
//! to the appropriate utility:
|
||||
//! Provides screenshot, keyboard-input, and mouse tools for agents running a
|
||||
//! Weston Wayland compositor (`hyperhive.gui.enable = true`). Each tool:
|
||||
//!
|
||||
//! - `screenshot` → `grim` (always available with gui enabled)
|
||||
//! - `type_text` → `wtype` (Wayland-native text input, no daemon)
|
||||
//! - `key_press` → `wtype -k` (Wayland virtual-keyboard protocol, no daemon)
|
||||
//! - `type_text` → `wtype` (Wayland virtual-keyboard protocol, no daemon)
|
||||
//! - `key_press` → `wtype -k` (same protocol as `type_text`)
|
||||
//! - `mouse_move` → RFB `PointerEvent` to the local neatvnc server
|
||||
//! (`HIVE_GUI_VNC_PORT`, default 5900)
|
||||
//! - `mouse_click` → RFB `PointerEvent` sequence: move → button-down →
|
||||
//! button-up (all in one TCP connection)
|
||||
//!
|
||||
//! All tools operate in userspace. The keyboard tools use the Wayland
|
||||
//! virtual-keyboard protocol; the mouse tools speak RFB directly to the
|
||||
//! Weston VNC backend — the compositor's native remote-input path, no
|
||||
//! `/dev/uinput` or kernel bypass.
|
||||
//!
|
||||
//! All three tools use Wayland protocols mediated by the compositor —
|
||||
//! no `/dev/uinput` or kernel-level injection.
|
||||
//! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the
|
||||
//! `weston-vnc` module so all tools find the compositor automatically.
|
||||
//! `weston-vnc` module. `HIVE_GUI_VNC_PORT` is set by the harness service
|
||||
//! when `hyperhive.gui.enable = true` (defaults to 5900 if absent).
|
||||
|
||||
use anyhow::Result;
|
||||
use rmcp::{
|
||||
|
|
@ -22,6 +29,8 @@ use rmcp::{
|
|||
transport::stdio,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::process::Command;
|
||||
|
||||
/// Run a command and return its result.
|
||||
|
|
@ -56,6 +65,131 @@ fn cmd_result(r: Result<String, String>) -> String {
|
|||
}
|
||||
}
|
||||
|
||||
/// Read `HIVE_GUI_VNC_PORT` from the environment, falling back to 5900.
|
||||
fn vnc_port() -> u16 {
|
||||
std::env::var("HIVE_GUI_VNC_PORT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(5900)
|
||||
}
|
||||
|
||||
/// Perform the RFB 3.8 handshake on `stream` and return on success.
|
||||
///
|
||||
/// Assumes the server advertises security type 1 (None) as set by
|
||||
/// `auth-method=none` in `weston.ini`. Consumes the full `ServerInit`
|
||||
/// frame so the caller is ready to send client messages.
|
||||
async fn rfb_handshake(stream: &mut TcpStream) -> Result<(), String> {
|
||||
// Protocol version — server sends 12 bytes ("RFB 003.xxx\n"), we
|
||||
// reply with 3.8.
|
||||
let mut ver = [0u8; 12];
|
||||
stream
|
||||
.read_exact(&mut ver)
|
||||
.await
|
||||
.map_err(|e| format!("VNC: read version: {e}"))?;
|
||||
stream
|
||||
.write_all(b"RFB 003.008\n")
|
||||
.await
|
||||
.map_err(|e| format!("VNC: send version: {e}"))?;
|
||||
|
||||
// Security type negotiation (RFB 3.8): server sends count + list.
|
||||
let count = stream
|
||||
.read_u8()
|
||||
.await
|
||||
.map_err(|e| format!("VNC: read sec-type count: {e}"))?;
|
||||
let mut types = vec![0u8; count as usize];
|
||||
stream
|
||||
.read_exact(&mut types)
|
||||
.await
|
||||
.map_err(|e| format!("VNC: read sec-types: {e}"))?;
|
||||
if !types.contains(&1) {
|
||||
return Err(format!(
|
||||
"VNC: server did not offer security type None (got: {types:?})"
|
||||
));
|
||||
}
|
||||
stream
|
||||
.write_all(&[1])
|
||||
.await
|
||||
.map_err(|e| format!("VNC: send sec-type: {e}"))?;
|
||||
|
||||
// SecurityResult: 4 bytes big-endian; 0 = OK.
|
||||
let result = stream
|
||||
.read_u32()
|
||||
.await
|
||||
.map_err(|e| format!("VNC: read sec-result: {e}"))?;
|
||||
if result != 0 {
|
||||
return Err(format!(
|
||||
"VNC: security handshake rejected (result={result})"
|
||||
));
|
||||
}
|
||||
|
||||
// ClientInit: 1 byte; 1 = shared session (don't disconnect other clients).
|
||||
stream
|
||||
.write_all(&[1])
|
||||
.await
|
||||
.map_err(|e| format!("VNC: send client-init: {e}"))?;
|
||||
|
||||
// ServerInit: width(2) + height(2) + pixel-format(16) +
|
||||
// name-length(4) + name. Read and discard.
|
||||
stream
|
||||
.read_u16()
|
||||
.await
|
||||
.map_err(|e| format!("VNC: read width: {e}"))?;
|
||||
stream
|
||||
.read_u16()
|
||||
.await
|
||||
.map_err(|e| format!("VNC: read height: {e}"))?;
|
||||
let mut pf = [0u8; 16];
|
||||
stream
|
||||
.read_exact(&mut pf)
|
||||
.await
|
||||
.map_err(|e| format!("VNC: read pixel-format: {e}"))?;
|
||||
let name_len = stream
|
||||
.read_u32()
|
||||
.await
|
||||
.map_err(|e| format!("VNC: read name-len: {e}"))?;
|
||||
let mut name = vec![0u8; name_len as usize];
|
||||
stream
|
||||
.read_exact(&mut name)
|
||||
.await
|
||||
.map_err(|e| format!("VNC: read name: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encode an RFB `PointerEvent` message (6 bytes).
|
||||
///
|
||||
/// - `button_mask`: bitmask — bit 0 = left button, bit 1 = middle, bit 2 = right
|
||||
/// - `x`, `y`: absolute pixel position (big-endian in the wire format)
|
||||
fn rfb_pointer_event(button_mask: u8, x: u16, y: u16) -> [u8; 6] {
|
||||
let xb = x.to_be_bytes();
|
||||
let yb = y.to_be_bytes();
|
||||
[5, button_mask, xb[0], xb[1], yb[0], yb[1]]
|
||||
}
|
||||
|
||||
/// Connect to the local neatvnc server, perform the RFB handshake, and
|
||||
/// send a sequence of `PointerEvent` messages in one connection.
|
||||
///
|
||||
/// `events` is a slice of `(button_mask, x, y)` tuples. Each is encoded
|
||||
/// as an RFB `PointerEvent` and written in order; the connection is then
|
||||
/// closed. Errors at any step short-circuit and return a `String`.
|
||||
async fn rfb_send_pointer_events(port: u16, events: &[(u8, u16, u16)]) -> Result<(), String> {
|
||||
let mut stream = TcpStream::connect(("127.0.0.1", port))
|
||||
.await
|
||||
.map_err(|e| format!("VNC: connect to localhost:{port}: {e}"))?;
|
||||
rfb_handshake(&mut stream).await?;
|
||||
for &(mask, x, y) in events {
|
||||
stream
|
||||
.write_all(&rfb_pointer_event(mask, x, y))
|
||||
.await
|
||||
.map_err(|e| format!("VNC: send pointer event: {e}"))?;
|
||||
}
|
||||
stream
|
||||
.flush()
|
||||
.await
|
||||
.map_err(|e| format!("VNC: flush: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct ScreenMcp;
|
||||
|
||||
#[tool_router]
|
||||
|
|
@ -118,6 +252,47 @@ impl ScreenMcp {
|
|||
let arg_refs: Vec<&str> = wtype_args.iter().map(String::as_str).collect();
|
||||
cmd_result(run_cmd("wtype", &arg_refs).await)
|
||||
}
|
||||
|
||||
#[tool(
|
||||
description = "Move the mouse cursor to an absolute pixel position on the display. \
|
||||
`x` is measured from the left edge, `y` from the top edge. The display is \
|
||||
1280×720 pixels when `hyperhive.gui.enable = true`. Injects a RFB \
|
||||
`PointerEvent` into the Weston VNC backend — compositor-mediated, \
|
||||
no kernel bypass."
|
||||
)]
|
||||
async fn mouse_move(&self, Parameters(args): Parameters<MouseMoveArgs>) -> String {
|
||||
let (Ok(x), Ok(y)) = (u16::try_from(args.x), u16::try_from(args.y)) else {
|
||||
return "mouse_move: x and y must be in range 0–65535".to_owned();
|
||||
};
|
||||
match rfb_send_pointer_events(vnc_port(), &[(0, x, y)]).await {
|
||||
Ok(()) => "ok".to_owned(),
|
||||
Err(e) => e,
|
||||
}
|
||||
}
|
||||
|
||||
#[tool(
|
||||
description = "Click a mouse button at an absolute pixel position. `x` and `y` are \
|
||||
the target coordinates (0-based from the top-left, display is 1280×720). \
|
||||
`button` is `\"left\"` (default), `\"right\"`, or `\"middle\"`. Sends: move \
|
||||
to position → button-down → button-up as RFB `PointerEvent` messages to the \
|
||||
Weston VNC backend — compositor-mediated, no kernel bypass."
|
||||
)]
|
||||
async fn mouse_click(&self, Parameters(args): Parameters<MouseClickArgs>) -> String {
|
||||
let (Ok(x), Ok(y)) = (u16::try_from(args.x), u16::try_from(args.y)) else {
|
||||
return "mouse_click: x and y must be in range 0–65535".to_owned();
|
||||
};
|
||||
let btn_mask: u8 = match args.button.as_deref().unwrap_or("left") {
|
||||
"right" => 0b0000_0100,
|
||||
"middle" => 0b0000_0010,
|
||||
_ => 0b0000_0001, // left
|
||||
};
|
||||
// Move to position, press, release — all in one VNC connection.
|
||||
let events = [(0, x, y), (btn_mask, x, y), (0, x, y)];
|
||||
match rfb_send_pointer_events(vnc_port(), &events).await {
|
||||
Ok(()) => "ok".to_owned(),
|
||||
Err(e) => e,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
|
|
@ -133,6 +308,24 @@ struct KeyPressArgs {
|
|||
keys: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
struct MouseMoveArgs {
|
||||
/// Horizontal pixel coordinate from the left edge of the display (0-based).
|
||||
x: i32,
|
||||
/// Vertical pixel coordinate from the top edge of the display (0-based).
|
||||
y: i32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
struct MouseClickArgs {
|
||||
/// Horizontal pixel coordinate from the left edge of the display (0-based).
|
||||
x: i32,
|
||||
/// Vertical pixel coordinate from the top edge of the display (0-based).
|
||||
y: i32,
|
||||
/// Button to click: `"left"` (default), `"right"`, or `"middle"`.
|
||||
button: Option<String>,
|
||||
}
|
||||
|
||||
#[tool_handler]
|
||||
impl ServerHandler for ScreenMcp {}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
# Screen MCP — screenshot + keyboard input for GUI agents.
|
||||
# Screen MCP — screenshot, keyboard, and mouse for GUI agents.
|
||||
#
|
||||
# Auto-activated when `hyperhive.gui.enable = true`. Wires the
|
||||
# `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude
|
||||
# gets three tools: `screenshot`, `type_text`, and `key_press`.
|
||||
# gets five tools: `screenshot`, `type_text`, `key_press`,
|
||||
# `mouse_move`, and `mouse_click`.
|
||||
#
|
||||
# All tools use Wayland protocols mediated by the Weston compositor —
|
||||
# no `/dev/uinput` or kernel-level injection needed. `grim` takes
|
||||
# screenshots; `wtype` handles both text input and key/modifier combos
|
||||
# via the `zwp-virtual-keyboard-unstable-v1` protocol.
|
||||
# All tools operate in userspace without `/dev/uinput`:
|
||||
# - `grim` takes screenshots via Wayland screencopy
|
||||
# - `wtype` handles text + key combos via `zwp-virtual-keyboard-unstable-v1`
|
||||
# - mouse tools speak RFB `PointerEvent` directly to the local neatvnc
|
||||
# server (port `HIVE_GUI_VNC_PORT`) — the VNC backend's native input path
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
|
|
|
|||
Loading…
Reference in a new issue