From 0b3268feae8c987af14e0af42e8e9d7490fd8e96 Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 20 Jul 2026 20:21:38 +0200 Subject: [PATCH 1/4] =?UTF-8?q?feat(#2305):=20hive-screen-mcp=20=E2=80=94?= =?UTF-8?q?=20screenshot=20+=20input=20MCP=20for=20GUI=20agents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New crate hive-screen-mcp: a stdio MCP bridge activated automatically when an agent has hyperhive.gui.enable = true. Provides five tools: - screenshot — grim → saves PNG, returns path for Read tool - type_text — wtype → Unicode text input (no daemon) - key_press — ydotool key → combos like ctrl+c, super+l - mouse_move — ydotool mousemove --absolute - mouse_click — ydotool click, optionally with prior move New nix/agent-modules/screen.nix: wires the MCP bridge into extraMcpServers.screen; adds grim + wtype to systemPackages. Adds hyperhive.gui.screenInput option (default false) which enables the ydotoold daemon + ydotool for mouse/keyboard injection via /dev/uinput. screenshot and type_text work without screenInput. key_press, mouse_move, and mouse_click return a ydotool error until ydotoold is running and /dev/uinput is accessible in the container. --- Cargo.lock | 13 ++ Cargo.toml | 1 + hive-screen-mcp/Cargo.toml | 23 ++++ hive-screen-mcp/src/main.rs | 209 +++++++++++++++++++++++++++++++++ nix/agent-modules/default.nix | 1 + nix/agent-modules/packages.nix | 2 +- nix/agent-modules/screen.nix | 62 ++++++++++ nix/packages/default.nix | 1 + 8 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 hive-screen-mcp/Cargo.toml create mode 100644 hive-screen-mcp/src/main.rs create mode 100644 nix/agent-modules/screen.nix diff --git a/Cargo.lock b/Cargo.lock index 22ddacbc..db46ddd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1734,6 +1734,19 @@ dependencies = [ "serde", ] +[[package]] +name = "hive-screen-mcp" +version = "0.1.0" +dependencies = [ + "anyhow", + "rmcp", + "schemars", + "serde", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "hive-sh4re" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4e4decc1..95993f7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "hive-bash-mcp", "hive-c0re", "hive-claude", + "hive-screen-mcp", "hive-forge", "hive-host-sock", "hive-jobq", diff --git a/hive-screen-mcp/Cargo.toml b/hive-screen-mcp/Cargo.toml new file mode 100644 index 00000000..8fa82d0a --- /dev/null +++ b/hive-screen-mcp/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "hive-screen-mcp" +edition.workspace = true +version.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +rmcp.workspace = true +schemars.workspace = true +serde.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +# `hive-screen-mcp` — stdio MCP bridge for GUI agents. +# Provides screenshot (grim), text input (wtype), key press and mouse +# actions (ydotool) for agents running a Weston compositor. +[[bin]] +name = "hive-screen-mcp" +path = "src/main.rs" diff --git a/hive-screen-mcp/src/main.rs b/hive-screen-mcp/src/main.rs new file mode 100644 index 00000000..fe8b18fd --- /dev/null +++ b/hive-screen-mcp/src/main.rs @@ -0,0 +1,209 @@ +//! `hive-screen-mcp` — stdio MCP bridge for GUI agents. +//! +//! Provides screenshot, text input, and mouse/keyboard tools for agents +//! running a Weston Wayland compositor (`hyperhive.gui.enable = true`). +//! Each tool shells out to the appropriate utility: +//! +//! - `screenshot` → `grim` (always available with gui enabled) +//! - `type_text` → `wtype` (Wayland-native text input, no daemon) +//! - `key_press` → `ydotool key` (needs ydotoold + `/dev/uinput`) +//! - `mouse_move` → `ydotool mousemove` (same requirement) +//! - `mouse_click` → `ydotool click` (same requirement) +//! +//! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the +//! `weston-vnc` module so all tools find the compositor automatically. +//! `ydotool`-based tools require the ydotoold daemon; enable it by +//! setting `hyperhive.gui.screenInput = true` in the agent config. + +use anyhow::Result; +use rmcp::{ + ServerHandler, ServiceExt, + handler::server::wrapper::Parameters, + schemars::{self, JsonSchema}, + tool, tool_handler, tool_router, + transport::stdio, +}; +use serde::Deserialize; +use tokio::process::Command; + +/// Run a command and return a human-readable result string. +/// On success returns stdout (trimmed) or `"ok"` when stdout is empty. +/// On failure returns a `"command failed (exit N): "` string. +async fn run_cmd(program: &str, args: &[&str]) -> String { + match Command::new(program).args(args).output().await { + Ok(out) if out.status.success() => { + let s = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + if s.is_empty() { "ok".to_owned() } else { s } + } + Ok(out) => { + let msg = String::from_utf8_lossy(&out.stderr).trim().to_owned(); + format!( + "command failed (exit {}): {}", + out.status.code().unwrap_or(-1), + if msg.is_empty() { "(no stderr)" } else { &msg } + ) + } + Err(e) => format!("failed to launch `{program}`: {e}"), + } +} + +struct ScreenMcp; + +#[tool_router] +impl ScreenMcp { + #[tool( + description = "Take a screenshot of the agent's Wayland display and save it as a PNG \ + file. Returns the path to the saved file — pass that path to the `Read` tool to \ + view the image visually. `dest_path` is optional; when omitted a timestamped temp \ + file under `/tmp/` is used. Requires `grim` and a live `WAYLAND_DISPLAY` \ + (provided automatically when `hyperhive.gui.enable = true`)." + )] + async fn screenshot(&self, Parameters(args): Parameters) -> String { + let path = args.dest_path.unwrap_or_else(|| { + let ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_millis()); + format!("/tmp/hive-screenshot-{ms}.png") + }); + let result = run_cmd("grim", &["-t", "png", &path]).await; + if result == "ok" { + format!("screenshot saved to `{path}` — use the Read tool to view it") + } else { + result + } + } + + #[tool( + description = "Type text into the currently focused Wayland window. Uses `wtype`, \ + which supports arbitrary Unicode. For special keys (Enter, Tab, Escape, function \ + keys, arrow keys) or modifier combos, use `key_press` instead. Requires a live \ + `WAYLAND_DISPLAY` (provided automatically when `hyperhive.gui.enable = true`)." + )] + async fn type_text(&self, Parameters(args): Parameters) -> String { + run_cmd("wtype", &[&args.text]).await + } + + #[tool( + description = "Press a key or key combination. `keys` uses ydotool syntax: a single \ + key name (`Return`, `Tab`, `Escape`, `BackSpace`, `Delete`, `space`, `F1`–`F12`, \ + `Left`, `Right`, `Up`, `Down`) or a modifier+key combo (`ctrl+c`, \ + `ctrl+shift+t`, `super+l`, `alt+F4`). Requires the ydotoold daemon and \ + `/dev/uinput` device access in the container — set \ + `hyperhive.gui.screenInput = true` in the agent config to enable the daemon." + )] + async fn key_press(&self, Parameters(args): Parameters) -> String { + run_cmd("ydotool", &["key", &args.keys]).await + } + + #[tool( + description = "Move the mouse cursor to an absolute pixel position on the display. \ + `x` is measured from the left edge, `y` from the top edge. Requires the ydotoold \ + daemon and `/dev/uinput` device access — set `hyperhive.gui.screenInput = true` \ + in the agent config to enable the daemon." + )] + async fn mouse_move(&self, Parameters(args): Parameters) -> String { + run_cmd( + "ydotool", + &[ + "mousemove", + "--absolute", + "-x", + &args.x.to_string(), + "-y", + &args.y.to_string(), + ], + ) + .await + } + + #[tool( + description = "Click a mouse button, optionally moving to a position first. `button` \ + is `\"left\"` (default), `\"right\"`, or `\"middle\"`. When `x` and `y` are \ + provided the cursor is moved to that absolute pixel position before clicking. \ + Requires the ydotoold daemon and `/dev/uinput` device access — set \ + `hyperhive.gui.screenInput = true` in the agent config to enable the daemon." + )] + async fn mouse_click(&self, Parameters(args): Parameters) -> String { + // Move first when coordinates are supplied. + if let (Some(x), Some(y)) = (args.x, args.y) { + let mv = run_cmd( + "ydotool", + &[ + "mousemove", + "--absolute", + "-x", + &x.to_string(), + "-y", + &y.to_string(), + ], + ) + .await; + if mv != "ok" { + return mv; + } + } + // Map the button name to ydotool's button code. + let button = match args.button.as_deref().unwrap_or("left") { + "right" => "0xC1", + "middle" => "0xC2", + _ => "0xC0", // left + }; + run_cmd("ydotool", &["click", button]).await + } +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct ScreenshotArgs { + /// Optional destination path for the PNG file. When omitted a + /// timestamped path under `/tmp/` is generated automatically. + dest_path: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct TypeTextArgs { + /// Text to type into the focused window. Supports arbitrary Unicode. + text: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct KeyPressArgs { + /// Key or key combination in ydotool syntax — e.g. `"Return"`, + /// `"Tab"`, `"ctrl+c"`, `"ctrl+shift+t"`, `"super+l"`, `"alt+F4"`. + keys: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct MouseMoveArgs { + /// Horizontal pixel coordinate from the left edge of the display. + x: i32, + /// Vertical pixel coordinate from the top edge of the display. + y: i32, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct MouseClickArgs { + /// Button to click: `"left"` (default), `"right"`, or `"middle"`. + button: Option, + /// Optional X coordinate to move to before clicking (absolute pixels). + x: Option, + /// Optional Y coordinate to move to before clicking (absolute pixels). + y: Option, +} + +#[tool_handler] +impl ServerHandler for ScreenMcp {} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_env("RUST_LOG") + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")), + ) + .with_writer(std::io::stderr) + .init(); + + let service = ScreenMcp.serve(stdio()).await?; + service.waiting().await?; + Ok(()) +} diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index 687a1c06..30033624 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -33,6 +33,7 @@ ./network.nix ./packages.nix ./user.nix + ./screen.nix ./weston-vnc.nix (lib.mkRemovedOptionModule [ "hyperhive" "web" "useUnixSocket" ] '' Unix socket mode is always enabled for all agents. Remove the diff --git a/nix/agent-modules/packages.nix b/nix/agent-modules/packages.nix index f5a29df1..4c2f0b55 100644 --- a/nix/agent-modules/packages.nix +++ b/nix/agent-modules/packages.nix @@ -13,7 +13,7 @@ per-binary daemon/CLI packages (`hive-agent`, `hive-agent-mcp`, `hive-agent-wake`, `hive-bash-daemon`, `hive-bash-mcp`, `hive-forge`, `hive-matrix-daemon`, `hive-matrix-mcp`, - `hive-metric`) plus the `assets`, `frontend` and + `hive-metric`, `hive-screen-mcp`) plus the `assets`, `frontend` and `reference-docs` trees. Wired by the flake's agent-base/ruth nixosModules to `hyperhive.packages..*`; override an individual key per-agent to swap in a patched binary. diff --git a/nix/agent-modules/screen.nix b/nix/agent-modules/screen.nix new file mode 100644 index 00000000..8bb0b806 --- /dev/null +++ b/nix/agent-modules/screen.nix @@ -0,0 +1,62 @@ +# Screen MCP — screenshot + input injection for GUI agents. +# +# Auto-activated when `hyperhive.gui.enable = true`. Wires the +# `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude +# gets five tools: `screenshot`, `type_text`, `key_press`, `mouse_move`, +# and `mouse_click`. +# +# `screenshot` and `type_text` work out of the box (grim + wtype, both +# pure Wayland clients). `key_press`, `mouse_move`, and `mouse_click` +# require the ydotoold daemon, which injects events via `/dev/uinput` +# at the kernel level — enable it by setting +# `hyperhive.gui.screenInput = true`. +{ + pkgs, + lib, + config, + ... +}: +{ + options.hyperhive.gui.screenInput = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Enable mouse and keyboard injection via ydotool + the ydotoold + daemon. Requires `/dev/uinput` device access inside the container + (the host must bind it in via `extraSystemdProperties` or + `systemd.nspawn..filesConfig.Bind`). When false, + `screenshot` and `type_text` still work; `key_press`, + `mouse_move`, and `mouse_click` return an error from ydotool + until ydotoold is running and `/dev/uinput` is accessible. + ''; + }; + + config = lib.mkIf config.hyperhive.gui.enable { + # Register the screen MCP bridge so claude gets the screen tools. + hyperhive.extraMcpServers.screen = { + command = "${config.hyperhive.packages.hive-screen-mcp}/bin/hive-screen-mcp"; + args = [ ]; + }; + + # grim: Wayland screenshot; wtype: text/key input (no daemon). + # ydotool: mouse + key injection via uinput (needs screenInput). + environment.systemPackages = + [ pkgs.grim pkgs.wtype ] + ++ lib.optional config.hyperhive.gui.screenInput pkgs.ydotool; + + # ydotoold — uinput event injection daemon. The socket lands at + # /tmp/.ydotool_socket by default; ydotool picks it up + # automatically. Only started when screenInput is enabled. + systemd.services.ydotoold = lib.mkIf config.hyperhive.gui.screenInput { + description = "ydotool input injection daemon"; + wantedBy = [ "multi-user.target" ]; + after = [ "local-fs.target" ]; + serviceConfig = { + ExecStart = "${pkgs.ydotool}/bin/ydotoold"; + Restart = "on-failure"; + RestartSec = "2s"; + SyslogIdentifier = "ydotoold"; + }; + }; + }; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index eaa8cfaa..d2bb19e4 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -32,6 +32,7 @@ let hive-matrix-daemon = "hyperhive per-agent matrix-sdk daemon"; hive-matrix-mcp = "hyperhive matrix MCP bridge"; hive-metric = "hyperhive agent-emitted custom metrics CLI"; + hive-screen-mcp = "hyperhive screen MCP bridge (screenshot + input for GUI agents)"; hive-forge = "hyperhive Forgejo CLI"; }; From 584dfed0c95a06b25aa0d9900912f55277d71121 Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 20 Jul 2026 20:30:02 +0200 Subject: [PATCH 2/4] fix(#2305): run_cmd returns Result, nix fmt, collapse nested if MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run_cmd now returns Result — callers pattern-match instead of comparing against an "ok" sentinel string - Add cmd_result() helper to format run_cmd results as tool strings - mouse_click: collapse nested if-let into let-chain (clippy collapsible_if) - nix fmt: reformat screen.nix package list --- hive-screen-mcp/src/main.rs | 78 ++++++++++++++++++++---------------- nix/agent-modules/screen.nix | 8 ++-- 2 files changed, 49 insertions(+), 37 deletions(-) diff --git a/hive-screen-mcp/src/main.rs b/hive-screen-mcp/src/main.rs index fe8b18fd..8de2acbe 100644 --- a/hive-screen-mcp/src/main.rs +++ b/hive-screen-mcp/src/main.rs @@ -26,24 +26,35 @@ use rmcp::{ use serde::Deserialize; use tokio::process::Command; -/// Run a command and return a human-readable result string. -/// On success returns stdout (trimmed) or `"ok"` when stdout is empty. -/// On failure returns a `"command failed (exit N): "` string. -async fn run_cmd(program: &str, args: &[&str]) -> String { +/// Run a command and return its result. +/// `Ok(stdout_trimmed)` on success (may be empty string when the tool +/// produces no output). `Err(human_readable_message)` on failure +/// (non-zero exit or failed spawn). +async fn run_cmd(program: &str, args: &[&str]) -> Result { match Command::new(program).args(args).output().await { Ok(out) if out.status.success() => { - let s = String::from_utf8_lossy(&out.stdout).trim().to_owned(); - if s.is_empty() { "ok".to_owned() } else { s } + Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) } Ok(out) => { let msg = String::from_utf8_lossy(&out.stderr).trim().to_owned(); - format!( + Err(format!( "command failed (exit {}): {}", out.status.code().unwrap_or(-1), if msg.is_empty() { "(no stderr)" } else { &msg } - ) + )) } - Err(e) => format!("failed to launch `{program}`: {e}"), + Err(e) => Err(format!("failed to launch `{program}`: {e}")), + } +} + +/// Format a `run_cmd` result as a tool-response string. `Ok("")` → +/// `"ok"` (most input tools produce no stdout); `Ok(s)` → `s`; +/// `Err(e)` → `e`. +fn cmd_result(r: Result) -> String { + match r { + Ok(s) if s.is_empty() => "ok".to_owned(), + Ok(s) => s, + Err(e) => e, } } @@ -65,11 +76,9 @@ impl ScreenMcp { .map_or(0, |d| d.as_millis()); format!("/tmp/hive-screenshot-{ms}.png") }); - let result = run_cmd("grim", &["-t", "png", &path]).await; - if result == "ok" { - format!("screenshot saved to `{path}` — use the Read tool to view it") - } else { - result + match run_cmd("grim", &["-t", "png", &path]).await { + Ok(_) => format!("screenshot saved to `{path}` — use the Read tool to view it"), + Err(e) => e, } } @@ -80,7 +89,7 @@ impl ScreenMcp { `WAYLAND_DISPLAY` (provided automatically when `hyperhive.gui.enable = true`)." )] async fn type_text(&self, Parameters(args): Parameters) -> String { - run_cmd("wtype", &[&args.text]).await + cmd_result(run_cmd("wtype", &[&args.text]).await) } #[tool( @@ -92,7 +101,7 @@ impl ScreenMcp { `hyperhive.gui.screenInput = true` in the agent config to enable the daemon." )] async fn key_press(&self, Parameters(args): Parameters) -> String { - run_cmd("ydotool", &["key", &args.keys]).await + cmd_result(run_cmd("ydotool", &["key", &args.keys]).await) } #[tool( @@ -102,18 +111,20 @@ impl ScreenMcp { in the agent config to enable the daemon." )] async fn mouse_move(&self, Parameters(args): Parameters) -> String { - run_cmd( - "ydotool", - &[ - "mousemove", - "--absolute", - "-x", - &args.x.to_string(), - "-y", - &args.y.to_string(), - ], + cmd_result( + run_cmd( + "ydotool", + &[ + "mousemove", + "--absolute", + "-x", + &args.x.to_string(), + "-y", + &args.y.to_string(), + ], + ) + .await, ) - .await } #[tool( @@ -125,8 +136,8 @@ impl ScreenMcp { )] async fn mouse_click(&self, Parameters(args): Parameters) -> String { // Move first when coordinates are supplied. - if let (Some(x), Some(y)) = (args.x, args.y) { - let mv = run_cmd( + if let (Some(x), Some(y)) = (args.x, args.y) + && let Err(e) = run_cmd( "ydotool", &[ "mousemove", @@ -137,10 +148,9 @@ impl ScreenMcp { &y.to_string(), ], ) - .await; - if mv != "ok" { - return mv; - } + .await + { + return e; } // Map the button name to ydotool's button code. let button = match args.button.as_deref().unwrap_or("left") { @@ -148,7 +158,7 @@ impl ScreenMcp { "middle" => "0xC2", _ => "0xC0", // left }; - run_cmd("ydotool", &["click", button]).await + cmd_result(run_cmd("ydotool", &["click", button]).await) } } diff --git a/nix/agent-modules/screen.nix b/nix/agent-modules/screen.nix index 8bb0b806..620bac4c 100644 --- a/nix/agent-modules/screen.nix +++ b/nix/agent-modules/screen.nix @@ -40,9 +40,11 @@ # grim: Wayland screenshot; wtype: text/key input (no daemon). # ydotool: mouse + key injection via uinput (needs screenInput). - environment.systemPackages = - [ pkgs.grim pkgs.wtype ] - ++ lib.optional config.hyperhive.gui.screenInput pkgs.ydotool; + environment.systemPackages = [ + pkgs.grim + pkgs.wtype + ] + ++ lib.optional config.hyperhive.gui.screenInput pkgs.ydotool; # ydotoold — uinput event injection daemon. The socket lands at # /tmp/.ydotool_socket by default; ydotool picks it up From 7fa7e2bdfd5fea83370554122b0b55507891987c Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 20 Jul 2026 20:41:58 +0200 Subject: [PATCH 3/4] refactor(#2305): drop mouse tools + ydotool, switch key_press to wtype - remove mouse_move and mouse_click (no Wayland-native alternative on Weston without /dev/uinput; follow-up filed for future investigation) - replace key_press from 'ydotool key' to 'wtype -k': parses mod1+mod2+key into -M mod1 ... -k key ... -m mod1 sequence via virtual-keyboard protocol - remove dest_path parameter from screenshot: always writes to /tmp/ (fixes arbitrary write-path concern from security review) - simplify screen.nix: drop screenInput option, ydotoold systemd unit, ydotool package; only grim + wtype remain (both compositor-mediated, no /dev/uinput) - update module header comment to reflect three-tool surface Addresses mara's /dev/uinput veto (PR #2617 comment #40524). --- hive-screen-mcp/src/main.rs | 159 ++++++++++------------------------- nix/agent-modules/screen.nix | 50 ++--------- 2 files changed, 55 insertions(+), 154 deletions(-) diff --git a/hive-screen-mcp/src/main.rs b/hive-screen-mcp/src/main.rs index 8de2acbe..9cf3d53f 100644 --- a/hive-screen-mcp/src/main.rs +++ b/hive-screen-mcp/src/main.rs @@ -1,19 +1,17 @@ //! `hive-screen-mcp` — stdio MCP bridge for GUI agents. //! -//! Provides screenshot, text input, and mouse/keyboard tools for agents -//! running a Weston Wayland compositor (`hyperhive.gui.enable = true`). -//! Each tool shells out to the appropriate utility: +//! Provides screenshot and keyboard-input tools for agents running a Weston +//! Wayland compositor (`hyperhive.gui.enable = true`). Each tool shells out +//! to the appropriate utility: //! //! - `screenshot` → `grim` (always available with gui enabled) //! - `type_text` → `wtype` (Wayland-native text input, no daemon) -//! - `key_press` → `ydotool key` (needs ydotoold + `/dev/uinput`) -//! - `mouse_move` → `ydotool mousemove` (same requirement) -//! - `mouse_click` → `ydotool click` (same requirement) +//! - `key_press` → `wtype -k` (Wayland virtual-keyboard protocol, no daemon) //! +//! All three tools use Wayland protocols mediated by the compositor — +//! no `/dev/uinput` or kernel-level injection. //! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the //! `weston-vnc` module so all tools find the compositor automatically. -//! `ydotool`-based tools require the ydotoold daemon; enable it by -//! setting `hyperhive.gui.screenInput = true` in the agent config. use anyhow::Result; use rmcp::{ @@ -64,18 +62,15 @@ struct ScreenMcp; impl ScreenMcp { #[tool( description = "Take a screenshot of the agent's Wayland display and save it as a PNG \ - file. Returns the path to the saved file — pass that path to the `Read` tool to \ - view the image visually. `dest_path` is optional; when omitted a timestamped temp \ - file under `/tmp/` is used. Requires `grim` and a live `WAYLAND_DISPLAY` \ - (provided automatically when `hyperhive.gui.enable = true`)." + file under `/tmp/`. Returns the path to the saved file — pass that path to the \ + `Read` tool to view the image visually. Requires `grim` and a live \ + `WAYLAND_DISPLAY` (provided automatically when `hyperhive.gui.enable = true`)." )] - async fn screenshot(&self, Parameters(args): Parameters) -> String { - let path = args.dest_path.unwrap_or_else(|| { - let ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_millis()); - format!("/tmp/hive-screenshot-{ms}.png") - }); + async fn screenshot(&self) -> String { + let ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_millis()); + let path = format!("/tmp/hive-screenshot-{ms}.png"); match run_cmd("grim", &["-t", "png", &path]).await { Ok(_) => format!("screenshot saved to `{path}` — use the Read tool to view it"), Err(e) => e, @@ -83,92 +78,48 @@ impl ScreenMcp { } #[tool( - description = "Type text into the currently focused Wayland window. Uses `wtype`, \ - which supports arbitrary Unicode. For special keys (Enter, Tab, Escape, function \ - keys, arrow keys) or modifier combos, use `key_press` instead. Requires a live \ - `WAYLAND_DISPLAY` (provided automatically when `hyperhive.gui.enable = true`)." + description = "Type text into the currently focused Wayland window. Uses `wtype` \ + via the Wayland virtual-keyboard protocol (compositor-mediated, no kernel \ + injection). Supports arbitrary Unicode. For special keys (Enter, Tab, Escape, \ + function keys, arrow keys) or modifier combos, use `key_press` instead. \ + Requires a live `WAYLAND_DISPLAY` (provided automatically when \ + `hyperhive.gui.enable = true`)." )] async fn type_text(&self, Parameters(args): Parameters) -> String { cmd_result(run_cmd("wtype", &[&args.text]).await) } #[tool( - description = "Press a key or key combination. `keys` uses ydotool syntax: a single \ - key name (`Return`, `Tab`, `Escape`, `BackSpace`, `Delete`, `space`, `F1`–`F12`, \ - `Left`, `Right`, `Up`, `Down`) or a modifier+key combo (`ctrl+c`, \ - `ctrl+shift+t`, `super+l`, `alt+F4`). Requires the ydotoold daemon and \ - `/dev/uinput` device access in the container — set \ - `hyperhive.gui.screenInput = true` in the agent config to enable the daemon." + description = "Press a key or key combination via the Wayland virtual-keyboard \ + protocol (compositor-mediated, no kernel injection). `keys` uses XKB keysym \ + syntax: a single key name (`Return`, `Tab`, `Escape`, `BackSpace`, `Delete`, \ + `space`, `F1`–`F12`, `Left`, `Right`, `Up`, `Down`) or a modifier+key combo \ + with `+`-separated parts where all but the last are modifiers (`ctrl+c`, \ + `ctrl+shift+t`, `super+l`, `alt+F4`). Modifier names: `ctrl`, `shift`, `alt`, \ + `super`. Requires a live `WAYLAND_DISPLAY` (provided automatically when \ + `hyperhive.gui.enable = true`)." )] async fn key_press(&self, Parameters(args): Parameters) -> String { - cmd_result(run_cmd("ydotool", &["key", &args.keys]).await) - } - - #[tool( - description = "Move the mouse cursor to an absolute pixel position on the display. \ - `x` is measured from the left edge, `y` from the top edge. Requires the ydotoold \ - daemon and `/dev/uinput` device access — set `hyperhive.gui.screenInput = true` \ - in the agent config to enable the daemon." - )] - async fn mouse_move(&self, Parameters(args): Parameters) -> String { - cmd_result( - run_cmd( - "ydotool", - &[ - "mousemove", - "--absolute", - "-x", - &args.x.to_string(), - "-y", - &args.y.to_string(), - ], - ) - .await, - ) - } - - #[tool( - description = "Click a mouse button, optionally moving to a position first. `button` \ - is `\"left\"` (default), `\"right\"`, or `\"middle\"`. When `x` and `y` are \ - provided the cursor is moved to that absolute pixel position before clicking. \ - Requires the ydotoold daemon and `/dev/uinput` device access — set \ - `hyperhive.gui.screenInput = true` in the agent config to enable the daemon." - )] - async fn mouse_click(&self, Parameters(args): Parameters) -> String { - // Move first when coordinates are supplied. - if let (Some(x), Some(y)) = (args.x, args.y) - && let Err(e) = run_cmd( - "ydotool", - &[ - "mousemove", - "--absolute", - "-x", - &x.to_string(), - "-y", - &y.to_string(), - ], - ) - .await - { - return e; + // Parse "mod1+mod2+key" into: -M mod1 -M mod2 -k key -m mod2 -m mod1 + // split('+') always yields at least one element, so split_last() is safe. + let parts: Vec<&str> = args.keys.split('+').collect(); + let (key, mods) = parts.split_last().expect("split yields ≥1 element"); + let mut wtype_args: Vec = Vec::new(); + for &m in mods { + wtype_args.push("-M".to_owned()); + wtype_args.push(m.to_owned()); } - // Map the button name to ydotool's button code. - let button = match args.button.as_deref().unwrap_or("left") { - "right" => "0xC1", - "middle" => "0xC2", - _ => "0xC0", // left - }; - cmd_result(run_cmd("ydotool", &["click", button]).await) + wtype_args.push("-k".to_owned()); + wtype_args.push((*key).to_owned()); + for &m in mods.iter().rev() { + wtype_args.push("-m".to_owned()); + wtype_args.push(m.to_owned()); + } + let arg_refs: Vec<&str> = wtype_args.iter().map(String::as_str).collect(); + cmd_result(run_cmd("wtype", &arg_refs).await) } } -#[derive(Debug, Deserialize, JsonSchema)] -struct ScreenshotArgs { - /// Optional destination path for the PNG file. When omitted a - /// timestamped path under `/tmp/` is generated automatically. - dest_path: Option, -} - #[derive(Debug, Deserialize, JsonSchema)] struct TypeTextArgs { /// Text to type into the focused window. Supports arbitrary Unicode. @@ -177,29 +128,11 @@ struct TypeTextArgs { #[derive(Debug, Deserialize, JsonSchema)] struct KeyPressArgs { - /// Key or key combination in ydotool syntax — e.g. `"Return"`, - /// `"Tab"`, `"ctrl+c"`, `"ctrl+shift+t"`, `"super+l"`, `"alt+F4"`. + /// Key or key combination in XKB keysym syntax — e.g. `"Return"`, `"Tab"`, + /// `"ctrl+c"`, `"ctrl+shift+t"`, `"super+l"`, `"alt+F4"`. keys: String, } -#[derive(Debug, Deserialize, JsonSchema)] -struct MouseMoveArgs { - /// Horizontal pixel coordinate from the left edge of the display. - x: i32, - /// Vertical pixel coordinate from the top edge of the display. - y: i32, -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct MouseClickArgs { - /// Button to click: `"left"` (default), `"right"`, or `"middle"`. - button: Option, - /// Optional X coordinate to move to before clicking (absolute pixels). - x: Option, - /// Optional Y coordinate to move to before clicking (absolute pixels). - y: Option, -} - #[tool_handler] impl ServerHandler for ScreenMcp {} diff --git a/nix/agent-modules/screen.nix b/nix/agent-modules/screen.nix index 620bac4c..0ba0f16f 100644 --- a/nix/agent-modules/screen.nix +++ b/nix/agent-modules/screen.nix @@ -1,15 +1,13 @@ -# Screen MCP — screenshot + input injection for GUI agents. +# Screen MCP — screenshot + keyboard input for GUI agents. # # Auto-activated when `hyperhive.gui.enable = true`. Wires the # `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude -# gets five tools: `screenshot`, `type_text`, `key_press`, `mouse_move`, -# and `mouse_click`. +# gets three tools: `screenshot`, `type_text`, and `key_press`. # -# `screenshot` and `type_text` work out of the box (grim + wtype, both -# pure Wayland clients). `key_press`, `mouse_move`, and `mouse_click` -# require the ydotoold daemon, which injects events via `/dev/uinput` -# at the kernel level — enable it by setting -# `hyperhive.gui.screenInput = true`. +# All tools use Wayland protocols mediated by the Weston compositor — +# no `/dev/uinput` or kernel-level injection needed. `grim` takes +# screenshots; `wtype` handles both text input and key/modifier combos +# via the `zwp-virtual-keyboard-unstable-v1` protocol. { pkgs, lib, @@ -17,20 +15,6 @@ ... }: { - options.hyperhive.gui.screenInput = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Enable mouse and keyboard injection via ydotool + the ydotoold - daemon. Requires `/dev/uinput` device access inside the container - (the host must bind it in via `extraSystemdProperties` or - `systemd.nspawn..filesConfig.Bind`). When false, - `screenshot` and `type_text` still work; `key_press`, - `mouse_move`, and `mouse_click` return an error from ydotool - until ydotoold is running and `/dev/uinput` is accessible. - ''; - }; - config = lib.mkIf config.hyperhive.gui.enable { # Register the screen MCP bridge so claude gets the screen tools. hyperhive.extraMcpServers.screen = { @@ -38,27 +22,11 @@ args = [ ]; }; - # grim: Wayland screenshot; wtype: text/key input (no daemon). - # ydotool: mouse + key injection via uinput (needs screenInput). + # grim: Wayland screenshot; wtype: text + key input via virtual-keyboard + # protocol (compositor-mediated, no /dev/uinput required). environment.systemPackages = [ pkgs.grim pkgs.wtype - ] - ++ lib.optional config.hyperhive.gui.screenInput pkgs.ydotool; - - # ydotoold — uinput event injection daemon. The socket lands at - # /tmp/.ydotool_socket by default; ydotool picks it up - # automatically. Only started when screenInput is enabled. - systemd.services.ydotoold = lib.mkIf config.hyperhive.gui.screenInput { - description = "ydotool input injection daemon"; - wantedBy = [ "multi-user.target" ]; - after = [ "local-fs.target" ]; - serviceConfig = { - ExecStart = "${pkgs.ydotool}/bin/ydotoold"; - Restart = "on-failure"; - RestartSec = "2s"; - SyslogIdentifier = "ydotoold"; - }; - }; + ]; }; } From 804c11c4044af98e0bea474988b3103009f0854e Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 20 Jul 2026 20:44:10 +0200 Subject: [PATCH 4/4] fix(#2305): drop redundant [[bin]] section from hive-screen-mcp/Cargo.toml name and path match cargo defaults; the explicit [[bin]] table is noise. --- hive-screen-mcp/Cargo.toml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/hive-screen-mcp/Cargo.toml b/hive-screen-mcp/Cargo.toml index 8fa82d0a..60239418 100644 --- a/hive-screen-mcp/Cargo.toml +++ b/hive-screen-mcp/Cargo.toml @@ -14,10 +14,3 @@ serde.workspace = true tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true - -# `hive-screen-mcp` — stdio MCP bridge for GUI agents. -# Provides screenshot (grim), text input (wtype), key press and mouse -# actions (ydotool) for agents running a Weston compositor. -[[bin]] -name = "hive-screen-mcp" -path = "src/main.rs"