diff --git a/Cargo.lock b/Cargo.lock index 22ddacbc..db46ddd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1734,6 +1734,19 @@ dependencies = [ "serde", ] +[[package]] +name = "hive-screen-mcp" +version = "0.1.0" +dependencies = [ + "anyhow", + "rmcp", + "schemars", + "serde", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "hive-sh4re" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4e4decc1..95993f7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "hive-bash-mcp", "hive-c0re", "hive-claude", + "hive-screen-mcp", "hive-forge", "hive-host-sock", "hive-jobq", diff --git a/hive-screen-mcp/Cargo.toml b/hive-screen-mcp/Cargo.toml new file mode 100644 index 00000000..60239418 --- /dev/null +++ b/hive-screen-mcp/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "hive-screen-mcp" +edition.workspace = true +version.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +rmcp.workspace = true +schemars.workspace = true +serde.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true diff --git a/hive-screen-mcp/src/main.rs b/hive-screen-mcp/src/main.rs new file mode 100644 index 00000000..9cf3d53f --- /dev/null +++ b/hive-screen-mcp/src/main.rs @@ -0,0 +1,152 @@ +//! `hive-screen-mcp` — stdio MCP bridge for GUI agents. +//! +//! Provides screenshot and keyboard-input tools for agents running a Weston +//! Wayland compositor (`hyperhive.gui.enable = true`). Each tool shells out +//! to the appropriate utility: +//! +//! - `screenshot` → `grim` (always available with gui enabled) +//! - `type_text` → `wtype` (Wayland-native text input, no daemon) +//! - `key_press` → `wtype -k` (Wayland virtual-keyboard protocol, no daemon) +//! +//! All three tools use Wayland protocols mediated by the compositor — +//! no `/dev/uinput` or kernel-level injection. +//! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the +//! `weston-vnc` module so all tools find the compositor automatically. + +use anyhow::Result; +use rmcp::{ + ServerHandler, ServiceExt, + handler::server::wrapper::Parameters, + schemars::{self, JsonSchema}, + tool, tool_handler, tool_router, + transport::stdio, +}; +use serde::Deserialize; +use tokio::process::Command; + +/// Run a command and return its result. +/// `Ok(stdout_trimmed)` on success (may be empty string when the tool +/// produces no output). `Err(human_readable_message)` on failure +/// (non-zero exit or failed spawn). +async fn run_cmd(program: &str, args: &[&str]) -> Result { + match Command::new(program).args(args).output().await { + Ok(out) if out.status.success() => { + Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) + } + Ok(out) => { + let msg = String::from_utf8_lossy(&out.stderr).trim().to_owned(); + Err(format!( + "command failed (exit {}): {}", + out.status.code().unwrap_or(-1), + if msg.is_empty() { "(no stderr)" } else { &msg } + )) + } + Err(e) => Err(format!("failed to launch `{program}`: {e}")), + } +} + +/// Format a `run_cmd` result as a tool-response string. `Ok("")` → +/// `"ok"` (most input tools produce no stdout); `Ok(s)` → `s`; +/// `Err(e)` → `e`. +fn cmd_result(r: Result) -> String { + match r { + Ok(s) if s.is_empty() => "ok".to_owned(), + Ok(s) => s, + Err(e) => e, + } +} + +struct ScreenMcp; + +#[tool_router] +impl ScreenMcp { + #[tool( + description = "Take a screenshot of the agent's Wayland display and save it as a PNG \ + file under `/tmp/`. Returns the path to the saved file — pass that path to the \ + `Read` tool to view the image visually. Requires `grim` and a live \ + `WAYLAND_DISPLAY` (provided automatically when `hyperhive.gui.enable = true`)." + )] + async fn screenshot(&self) -> String { + let ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_millis()); + let path = format!("/tmp/hive-screenshot-{ms}.png"); + match run_cmd("grim", &["-t", "png", &path]).await { + Ok(_) => format!("screenshot saved to `{path}` — use the Read tool to view it"), + Err(e) => e, + } + } + + #[tool( + description = "Type text into the currently focused Wayland window. Uses `wtype` \ + via the Wayland virtual-keyboard protocol (compositor-mediated, no kernel \ + injection). Supports arbitrary Unicode. For special keys (Enter, Tab, Escape, \ + function keys, arrow keys) or modifier combos, use `key_press` instead. \ + Requires a live `WAYLAND_DISPLAY` (provided automatically when \ + `hyperhive.gui.enable = true`)." + )] + async fn type_text(&self, Parameters(args): Parameters) -> String { + cmd_result(run_cmd("wtype", &[&args.text]).await) + } + + #[tool( + description = "Press a key or key combination via the Wayland virtual-keyboard \ + protocol (compositor-mediated, no kernel injection). `keys` uses XKB keysym \ + syntax: a single key name (`Return`, `Tab`, `Escape`, `BackSpace`, `Delete`, \ + `space`, `F1`–`F12`, `Left`, `Right`, `Up`, `Down`) or a modifier+key combo \ + with `+`-separated parts where all but the last are modifiers (`ctrl+c`, \ + `ctrl+shift+t`, `super+l`, `alt+F4`). Modifier names: `ctrl`, `shift`, `alt`, \ + `super`. Requires a live `WAYLAND_DISPLAY` (provided automatically when \ + `hyperhive.gui.enable = true`)." + )] + async fn key_press(&self, Parameters(args): Parameters) -> String { + // Parse "mod1+mod2+key" into: -M mod1 -M mod2 -k key -m mod2 -m mod1 + // split('+') always yields at least one element, so split_last() is safe. + let parts: Vec<&str> = args.keys.split('+').collect(); + let (key, mods) = parts.split_last().expect("split yields ≥1 element"); + let mut wtype_args: Vec = Vec::new(); + for &m in mods { + wtype_args.push("-M".to_owned()); + wtype_args.push(m.to_owned()); + } + wtype_args.push("-k".to_owned()); + wtype_args.push((*key).to_owned()); + for &m in mods.iter().rev() { + wtype_args.push("-m".to_owned()); + wtype_args.push(m.to_owned()); + } + let arg_refs: Vec<&str> = wtype_args.iter().map(String::as_str).collect(); + cmd_result(run_cmd("wtype", &arg_refs).await) + } +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct TypeTextArgs { + /// Text to type into the focused window. Supports arbitrary Unicode. + text: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct KeyPressArgs { + /// Key or key combination in XKB keysym syntax — e.g. `"Return"`, `"Tab"`, + /// `"ctrl+c"`, `"ctrl+shift+t"`, `"super+l"`, `"alt+F4"`. + keys: String, +} + +#[tool_handler] +impl ServerHandler for ScreenMcp {} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_env("RUST_LOG") + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")), + ) + .with_writer(std::io::stderr) + .init(); + + let service = ScreenMcp.serve(stdio()).await?; + service.waiting().await?; + Ok(()) +} diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index 687a1c06..30033624 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -33,6 +33,7 @@ ./network.nix ./packages.nix ./user.nix + ./screen.nix ./weston-vnc.nix (lib.mkRemovedOptionModule [ "hyperhive" "web" "useUnixSocket" ] '' Unix socket mode is always enabled for all agents. Remove the diff --git a/nix/agent-modules/packages.nix b/nix/agent-modules/packages.nix index f5a29df1..4c2f0b55 100644 --- a/nix/agent-modules/packages.nix +++ b/nix/agent-modules/packages.nix @@ -13,7 +13,7 @@ per-binary daemon/CLI packages (`hive-agent`, `hive-agent-mcp`, `hive-agent-wake`, `hive-bash-daemon`, `hive-bash-mcp`, `hive-forge`, `hive-matrix-daemon`, `hive-matrix-mcp`, - `hive-metric`) plus the `assets`, `frontend` and + `hive-metric`, `hive-screen-mcp`) plus the `assets`, `frontend` and `reference-docs` trees. Wired by the flake's agent-base/ruth nixosModules to `hyperhive.packages..*`; override an individual key per-agent to swap in a patched binary. diff --git a/nix/agent-modules/screen.nix b/nix/agent-modules/screen.nix new file mode 100644 index 00000000..0ba0f16f --- /dev/null +++ b/nix/agent-modules/screen.nix @@ -0,0 +1,32 @@ +# Screen MCP — screenshot + keyboard input for GUI agents. +# +# Auto-activated when `hyperhive.gui.enable = true`. Wires the +# `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude +# gets three tools: `screenshot`, `type_text`, and `key_press`. +# +# All tools use Wayland protocols mediated by the Weston compositor — +# no `/dev/uinput` or kernel-level injection needed. `grim` takes +# screenshots; `wtype` handles both text input and key/modifier combos +# via the `zwp-virtual-keyboard-unstable-v1` protocol. +{ + pkgs, + lib, + config, + ... +}: +{ + config = lib.mkIf config.hyperhive.gui.enable { + # Register the screen MCP bridge so claude gets the screen tools. + hyperhive.extraMcpServers.screen = { + command = "${config.hyperhive.packages.hive-screen-mcp}/bin/hive-screen-mcp"; + args = [ ]; + }; + + # grim: Wayland screenshot; wtype: text + key input via virtual-keyboard + # protocol (compositor-mediated, no /dev/uinput required). + environment.systemPackages = [ + pkgs.grim + pkgs.wtype + ]; + }; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index eaa8cfaa..d2bb19e4 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -32,6 +32,7 @@ let hive-matrix-daemon = "hyperhive per-agent matrix-sdk daemon"; hive-matrix-mcp = "hyperhive matrix MCP bridge"; hive-metric = "hyperhive agent-emitted custom metrics CLI"; + hive-screen-mcp = "hyperhive screen MCP bridge (screenshot + input for GUI agents)"; hive-forge = "hyperhive Forgejo CLI"; };