From 0b3268feae8c987af14e0af42e8e9d7490fd8e96 Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 20 Jul 2026 20:21:38 +0200 Subject: [PATCH] =?UTF-8?q?feat(#2305):=20hive-screen-mcp=20=E2=80=94=20sc?= =?UTF-8?q?reenshot=20+=20input=20MCP=20for=20GUI=20agents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New crate hive-screen-mcp: a stdio MCP bridge activated automatically when an agent has hyperhive.gui.enable = true. Provides five tools: - screenshot — grim → saves PNG, returns path for Read tool - type_text — wtype → Unicode text input (no daemon) - key_press — ydotool key → combos like ctrl+c, super+l - mouse_move — ydotool mousemove --absolute - mouse_click — ydotool click, optionally with prior move New nix/agent-modules/screen.nix: wires the MCP bridge into extraMcpServers.screen; adds grim + wtype to systemPackages. Adds hyperhive.gui.screenInput option (default false) which enables the ydotoold daemon + ydotool for mouse/keyboard injection via /dev/uinput. screenshot and type_text work without screenInput. key_press, mouse_move, and mouse_click return a ydotool error until ydotoold is running and /dev/uinput is accessible in the container. --- Cargo.lock | 13 ++ Cargo.toml | 1 + hive-screen-mcp/Cargo.toml | 23 ++++ hive-screen-mcp/src/main.rs | 209 +++++++++++++++++++++++++++++++++ nix/agent-modules/default.nix | 1 + nix/agent-modules/packages.nix | 2 +- nix/agent-modules/screen.nix | 62 ++++++++++ nix/packages/default.nix | 1 + 8 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 hive-screen-mcp/Cargo.toml create mode 100644 hive-screen-mcp/src/main.rs create mode 100644 nix/agent-modules/screen.nix diff --git a/Cargo.lock b/Cargo.lock index 22ddacbc..db46ddd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1734,6 +1734,19 @@ dependencies = [ "serde", ] +[[package]] +name = "hive-screen-mcp" +version = "0.1.0" +dependencies = [ + "anyhow", + "rmcp", + "schemars", + "serde", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "hive-sh4re" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4e4decc1..95993f7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "hive-bash-mcp", "hive-c0re", "hive-claude", + "hive-screen-mcp", "hive-forge", "hive-host-sock", "hive-jobq", diff --git a/hive-screen-mcp/Cargo.toml b/hive-screen-mcp/Cargo.toml new file mode 100644 index 00000000..8fa82d0a --- /dev/null +++ b/hive-screen-mcp/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "hive-screen-mcp" +edition.workspace = true +version.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +rmcp.workspace = true +schemars.workspace = true +serde.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +# `hive-screen-mcp` — stdio MCP bridge for GUI agents. +# Provides screenshot (grim), text input (wtype), key press and mouse +# actions (ydotool) for agents running a Weston compositor. +[[bin]] +name = "hive-screen-mcp" +path = "src/main.rs" diff --git a/hive-screen-mcp/src/main.rs b/hive-screen-mcp/src/main.rs new file mode 100644 index 00000000..fe8b18fd --- /dev/null +++ b/hive-screen-mcp/src/main.rs @@ -0,0 +1,209 @@ +//! `hive-screen-mcp` — stdio MCP bridge for GUI agents. +//! +//! Provides screenshot, text input, and mouse/keyboard tools for agents +//! running a Weston Wayland compositor (`hyperhive.gui.enable = true`). +//! Each tool shells out to the appropriate utility: +//! +//! - `screenshot` → `grim` (always available with gui enabled) +//! - `type_text` → `wtype` (Wayland-native text input, no daemon) +//! - `key_press` → `ydotool key` (needs ydotoold + `/dev/uinput`) +//! - `mouse_move` → `ydotool mousemove` (same requirement) +//! - `mouse_click` → `ydotool click` (same requirement) +//! +//! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the +//! `weston-vnc` module so all tools find the compositor automatically. +//! `ydotool`-based tools require the ydotoold daemon; enable it by +//! setting `hyperhive.gui.screenInput = true` in the agent config. + +use anyhow::Result; +use rmcp::{ + ServerHandler, ServiceExt, + handler::server::wrapper::Parameters, + schemars::{self, JsonSchema}, + tool, tool_handler, tool_router, + transport::stdio, +}; +use serde::Deserialize; +use tokio::process::Command; + +/// Run a command and return a human-readable result string. +/// On success returns stdout (trimmed) or `"ok"` when stdout is empty. +/// On failure returns a `"command failed (exit N): "` string. +async fn run_cmd(program: &str, args: &[&str]) -> String { + match Command::new(program).args(args).output().await { + Ok(out) if out.status.success() => { + let s = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + if s.is_empty() { "ok".to_owned() } else { s } + } + Ok(out) => { + let msg = String::from_utf8_lossy(&out.stderr).trim().to_owned(); + format!( + "command failed (exit {}): {}", + out.status.code().unwrap_or(-1), + if msg.is_empty() { "(no stderr)" } else { &msg } + ) + } + Err(e) => format!("failed to launch `{program}`: {e}"), + } +} + +struct ScreenMcp; + +#[tool_router] +impl ScreenMcp { + #[tool( + description = "Take a screenshot of the agent's Wayland display and save it as a PNG \ + file. Returns the path to the saved file — pass that path to the `Read` tool to \ + view the image visually. `dest_path` is optional; when omitted a timestamped temp \ + file under `/tmp/` is used. Requires `grim` and a live `WAYLAND_DISPLAY` \ + (provided automatically when `hyperhive.gui.enable = true`)." + )] + async fn screenshot(&self, Parameters(args): Parameters) -> String { + let path = args.dest_path.unwrap_or_else(|| { + let ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_millis()); + format!("/tmp/hive-screenshot-{ms}.png") + }); + let result = run_cmd("grim", &["-t", "png", &path]).await; + if result == "ok" { + format!("screenshot saved to `{path}` — use the Read tool to view it") + } else { + result + } + } + + #[tool( + description = "Type text into the currently focused Wayland window. Uses `wtype`, \ + which supports arbitrary Unicode. For special keys (Enter, Tab, Escape, function \ + keys, arrow keys) or modifier combos, use `key_press` instead. Requires a live \ + `WAYLAND_DISPLAY` (provided automatically when `hyperhive.gui.enable = true`)." + )] + async fn type_text(&self, Parameters(args): Parameters) -> String { + run_cmd("wtype", &[&args.text]).await + } + + #[tool( + description = "Press a key or key combination. `keys` uses ydotool syntax: a single \ + key name (`Return`, `Tab`, `Escape`, `BackSpace`, `Delete`, `space`, `F1`–`F12`, \ + `Left`, `Right`, `Up`, `Down`) or a modifier+key combo (`ctrl+c`, \ + `ctrl+shift+t`, `super+l`, `alt+F4`). Requires the ydotoold daemon and \ + `/dev/uinput` device access in the container — set \ + `hyperhive.gui.screenInput = true` in the agent config to enable the daemon." + )] + async fn key_press(&self, Parameters(args): Parameters) -> String { + run_cmd("ydotool", &["key", &args.keys]).await + } + + #[tool( + description = "Move the mouse cursor to an absolute pixel position on the display. \ + `x` is measured from the left edge, `y` from the top edge. Requires the ydotoold \ + daemon and `/dev/uinput` device access — set `hyperhive.gui.screenInput = true` \ + in the agent config to enable the daemon." + )] + async fn mouse_move(&self, Parameters(args): Parameters) -> String { + run_cmd( + "ydotool", + &[ + "mousemove", + "--absolute", + "-x", + &args.x.to_string(), + "-y", + &args.y.to_string(), + ], + ) + .await + } + + #[tool( + description = "Click a mouse button, optionally moving to a position first. `button` \ + is `\"left\"` (default), `\"right\"`, or `\"middle\"`. When `x` and `y` are \ + provided the cursor is moved to that absolute pixel position before clicking. \ + Requires the ydotoold daemon and `/dev/uinput` device access — set \ + `hyperhive.gui.screenInput = true` in the agent config to enable the daemon." + )] + async fn mouse_click(&self, Parameters(args): Parameters) -> String { + // Move first when coordinates are supplied. + if let (Some(x), Some(y)) = (args.x, args.y) { + let mv = run_cmd( + "ydotool", + &[ + "mousemove", + "--absolute", + "-x", + &x.to_string(), + "-y", + &y.to_string(), + ], + ) + .await; + if mv != "ok" { + return mv; + } + } + // Map the button name to ydotool's button code. + let button = match args.button.as_deref().unwrap_or("left") { + "right" => "0xC1", + "middle" => "0xC2", + _ => "0xC0", // left + }; + run_cmd("ydotool", &["click", button]).await + } +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct ScreenshotArgs { + /// Optional destination path for the PNG file. When omitted a + /// timestamped path under `/tmp/` is generated automatically. + dest_path: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct TypeTextArgs { + /// Text to type into the focused window. Supports arbitrary Unicode. + text: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct KeyPressArgs { + /// Key or key combination in ydotool syntax — e.g. `"Return"`, + /// `"Tab"`, `"ctrl+c"`, `"ctrl+shift+t"`, `"super+l"`, `"alt+F4"`. + keys: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct MouseMoveArgs { + /// Horizontal pixel coordinate from the left edge of the display. + x: i32, + /// Vertical pixel coordinate from the top edge of the display. + y: i32, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct MouseClickArgs { + /// Button to click: `"left"` (default), `"right"`, or `"middle"`. + button: Option, + /// Optional X coordinate to move to before clicking (absolute pixels). + x: Option, + /// Optional Y coordinate to move to before clicking (absolute pixels). + y: Option, +} + +#[tool_handler] +impl ServerHandler for ScreenMcp {} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_env("RUST_LOG") + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")), + ) + .with_writer(std::io::stderr) + .init(); + + let service = ScreenMcp.serve(stdio()).await?; + service.waiting().await?; + Ok(()) +} diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index 687a1c06..30033624 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -33,6 +33,7 @@ ./network.nix ./packages.nix ./user.nix + ./screen.nix ./weston-vnc.nix (lib.mkRemovedOptionModule [ "hyperhive" "web" "useUnixSocket" ] '' Unix socket mode is always enabled for all agents. Remove the diff --git a/nix/agent-modules/packages.nix b/nix/agent-modules/packages.nix index f5a29df1..4c2f0b55 100644 --- a/nix/agent-modules/packages.nix +++ b/nix/agent-modules/packages.nix @@ -13,7 +13,7 @@ per-binary daemon/CLI packages (`hive-agent`, `hive-agent-mcp`, `hive-agent-wake`, `hive-bash-daemon`, `hive-bash-mcp`, `hive-forge`, `hive-matrix-daemon`, `hive-matrix-mcp`, - `hive-metric`) plus the `assets`, `frontend` and + `hive-metric`, `hive-screen-mcp`) plus the `assets`, `frontend` and `reference-docs` trees. Wired by the flake's agent-base/ruth nixosModules to `hyperhive.packages..*`; override an individual key per-agent to swap in a patched binary. diff --git a/nix/agent-modules/screen.nix b/nix/agent-modules/screen.nix new file mode 100644 index 00000000..8bb0b806 --- /dev/null +++ b/nix/agent-modules/screen.nix @@ -0,0 +1,62 @@ +# Screen MCP — screenshot + input injection for GUI agents. +# +# Auto-activated when `hyperhive.gui.enable = true`. Wires the +# `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude +# gets five tools: `screenshot`, `type_text`, `key_press`, `mouse_move`, +# and `mouse_click`. +# +# `screenshot` and `type_text` work out of the box (grim + wtype, both +# pure Wayland clients). `key_press`, `mouse_move`, and `mouse_click` +# require the ydotoold daemon, which injects events via `/dev/uinput` +# at the kernel level — enable it by setting +# `hyperhive.gui.screenInput = true`. +{ + pkgs, + lib, + config, + ... +}: +{ + options.hyperhive.gui.screenInput = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Enable mouse and keyboard injection via ydotool + the ydotoold + daemon. Requires `/dev/uinput` device access inside the container + (the host must bind it in via `extraSystemdProperties` or + `systemd.nspawn..filesConfig.Bind`). When false, + `screenshot` and `type_text` still work; `key_press`, + `mouse_move`, and `mouse_click` return an error from ydotool + until ydotoold is running and `/dev/uinput` is accessible. + ''; + }; + + config = lib.mkIf config.hyperhive.gui.enable { + # Register the screen MCP bridge so claude gets the screen tools. + hyperhive.extraMcpServers.screen = { + command = "${config.hyperhive.packages.hive-screen-mcp}/bin/hive-screen-mcp"; + args = [ ]; + }; + + # grim: Wayland screenshot; wtype: text/key input (no daemon). + # ydotool: mouse + key injection via uinput (needs screenInput). + environment.systemPackages = + [ pkgs.grim pkgs.wtype ] + ++ lib.optional config.hyperhive.gui.screenInput pkgs.ydotool; + + # ydotoold — uinput event injection daemon. The socket lands at + # /tmp/.ydotool_socket by default; ydotool picks it up + # automatically. Only started when screenInput is enabled. + systemd.services.ydotoold = lib.mkIf config.hyperhive.gui.screenInput { + description = "ydotool input injection daemon"; + wantedBy = [ "multi-user.target" ]; + after = [ "local-fs.target" ]; + serviceConfig = { + ExecStart = "${pkgs.ydotool}/bin/ydotoold"; + Restart = "on-failure"; + RestartSec = "2s"; + SyslogIdentifier = "ydotoold"; + }; + }; + }; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index eaa8cfaa..d2bb19e4 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -32,6 +32,7 @@ let hive-matrix-daemon = "hyperhive per-agent matrix-sdk daemon"; hive-matrix-mcp = "hyperhive matrix MCP bridge"; hive-metric = "hyperhive agent-emitted custom metrics CLI"; + hive-screen-mcp = "hyperhive screen MCP bridge (screenshot + input for GUI agents)"; hive-forge = "hyperhive Forgejo CLI"; };