refactor(#2305): drop mouse tools + ydotool, switch key_press to wtype
- remove mouse_move and mouse_click (no Wayland-native alternative on Weston without /dev/uinput; follow-up filed for future investigation) - replace key_press from 'ydotool key' to 'wtype -k': parses mod1+mod2+key into -M mod1 ... -k key ... -m mod1 sequence via virtual-keyboard protocol - remove dest_path parameter from screenshot: always writes to /tmp/ (fixes arbitrary write-path concern from security review) - simplify screen.nix: drop screenInput option, ydotoold systemd unit, ydotool package; only grim + wtype remain (both compositor-mediated, no /dev/uinput) - update module header comment to reflect three-tool surface Addresses mara's /dev/uinput veto (PR #2617 comment #40524).
This commit is contained in:
iris
mara
parent
584dfed0c9
commit
7fa7e2bdfd
2 changed files with 55 additions and 154 deletions
|
|
@ -1,19 +1,17 @@
|
||||||
//! `hive-screen-mcp` — stdio MCP bridge for GUI agents.
|
//! `hive-screen-mcp` — stdio MCP bridge for GUI agents.
|
||||||
//!
|
//!
|
||||||
//! Provides screenshot, text input, and mouse/keyboard tools for agents
|
//! Provides screenshot and keyboard-input tools for agents running a Weston
|
||||||
//! running a Weston Wayland compositor (`hyperhive.gui.enable = true`).
|
//! Wayland compositor (`hyperhive.gui.enable = true`). Each tool shells out
|
||||||
//! Each tool shells out to the appropriate utility:
|
//! to the appropriate utility:
|
||||||
//!
|
//!
|
||||||
//! - `screenshot` → `grim` (always available with gui enabled)
|
//! - `screenshot` → `grim` (always available with gui enabled)
|
||||||
//! - `type_text` → `wtype` (Wayland-native text input, no daemon)
|
//! - `type_text` → `wtype` (Wayland-native text input, no daemon)
|
||||||
//! - `key_press` → `ydotool key` (needs ydotoold + `/dev/uinput`)
|
//! - `key_press` → `wtype -k` (Wayland virtual-keyboard protocol, no daemon)
|
||||||
//! - `mouse_move` → `ydotool mousemove` (same requirement)
|
|
||||||
//! - `mouse_click` → `ydotool click` (same requirement)
|
|
||||||
//!
|
//!
|
||||||
|
//! All three tools use Wayland protocols mediated by the compositor —
|
||||||
|
//! no `/dev/uinput` or kernel-level injection.
|
||||||
//! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the
|
//! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the
|
||||||
//! `weston-vnc` module so all tools find the compositor automatically.
|
//! `weston-vnc` module so all tools find the compositor automatically.
|
||||||
//! `ydotool`-based tools require the ydotoold daemon; enable it by
|
|
||||||
//! setting `hyperhive.gui.screenInput = true` in the agent config.
|
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use rmcp::{
|
use rmcp::{
|
||||||
|
|
@ -64,18 +62,15 @@ struct ScreenMcp;
|
||||||
impl ScreenMcp {
|
impl ScreenMcp {
|
||||||
#[tool(
|
#[tool(
|
||||||
description = "Take a screenshot of the agent's Wayland display and save it as a PNG \
|
description = "Take a screenshot of the agent's Wayland display and save it as a PNG \
|
||||||
file. Returns the path to the saved file — pass that path to the `Read` tool to \
|
file under `/tmp/`. Returns the path to the saved file — pass that path to the \
|
||||||
view the image visually. `dest_path` is optional; when omitted a timestamped temp \
|
`Read` tool to view the image visually. Requires `grim` and a live \
|
||||||
file under `/tmp/` is used. Requires `grim` and a live `WAYLAND_DISPLAY` \
|
`WAYLAND_DISPLAY` (provided automatically when `hyperhive.gui.enable = true`)."
|
||||||
(provided automatically when `hyperhive.gui.enable = true`)."
|
|
||||||
)]
|
)]
|
||||||
async fn screenshot(&self, Parameters(args): Parameters<ScreenshotArgs>) -> String {
|
async fn screenshot(&self) -> String {
|
||||||
let path = args.dest_path.unwrap_or_else(|| {
|
let ms = std::time::SystemTime::now()
|
||||||
let ms = std::time::SystemTime::now()
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
.map_or(0, |d| d.as_millis());
|
||||||
.map_or(0, |d| d.as_millis());
|
let path = format!("/tmp/hive-screenshot-{ms}.png");
|
||||||
format!("/tmp/hive-screenshot-{ms}.png")
|
|
||||||
});
|
|
||||||
match run_cmd("grim", &["-t", "png", &path]).await {
|
match run_cmd("grim", &["-t", "png", &path]).await {
|
||||||
Ok(_) => format!("screenshot saved to `{path}` — use the Read tool to view it"),
|
Ok(_) => format!("screenshot saved to `{path}` — use the Read tool to view it"),
|
||||||
Err(e) => e,
|
Err(e) => e,
|
||||||
|
|
@ -83,92 +78,48 @@ impl ScreenMcp {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tool(
|
#[tool(
|
||||||
description = "Type text into the currently focused Wayland window. Uses `wtype`, \
|
description = "Type text into the currently focused Wayland window. Uses `wtype` \
|
||||||
which supports arbitrary Unicode. For special keys (Enter, Tab, Escape, function \
|
via the Wayland virtual-keyboard protocol (compositor-mediated, no kernel \
|
||||||
keys, arrow keys) or modifier combos, use `key_press` instead. Requires a live \
|
injection). Supports arbitrary Unicode. For special keys (Enter, Tab, Escape, \
|
||||||
`WAYLAND_DISPLAY` (provided automatically when `hyperhive.gui.enable = true`)."
|
function keys, arrow keys) or modifier combos, use `key_press` instead. \
|
||||||
|
Requires a live `WAYLAND_DISPLAY` (provided automatically when \
|
||||||
|
`hyperhive.gui.enable = true`)."
|
||||||
)]
|
)]
|
||||||
async fn type_text(&self, Parameters(args): Parameters<TypeTextArgs>) -> String {
|
async fn type_text(&self, Parameters(args): Parameters<TypeTextArgs>) -> String {
|
||||||
cmd_result(run_cmd("wtype", &[&args.text]).await)
|
cmd_result(run_cmd("wtype", &[&args.text]).await)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tool(
|
#[tool(
|
||||||
description = "Press a key or key combination. `keys` uses ydotool syntax: a single \
|
description = "Press a key or key combination via the Wayland virtual-keyboard \
|
||||||
key name (`Return`, `Tab`, `Escape`, `BackSpace`, `Delete`, `space`, `F1`–`F12`, \
|
protocol (compositor-mediated, no kernel injection). `keys` uses XKB keysym \
|
||||||
`Left`, `Right`, `Up`, `Down`) or a modifier+key combo (`ctrl+c`, \
|
syntax: a single key name (`Return`, `Tab`, `Escape`, `BackSpace`, `Delete`, \
|
||||||
`ctrl+shift+t`, `super+l`, `alt+F4`). Requires the ydotoold daemon and \
|
`space`, `F1`–`F12`, `Left`, `Right`, `Up`, `Down`) or a modifier+key combo \
|
||||||
`/dev/uinput` device access in the container — set \
|
with `+`-separated parts where all but the last are modifiers (`ctrl+c`, \
|
||||||
`hyperhive.gui.screenInput = true` in the agent config to enable the daemon."
|
`ctrl+shift+t`, `super+l`, `alt+F4`). Modifier names: `ctrl`, `shift`, `alt`, \
|
||||||
|
`super`. Requires a live `WAYLAND_DISPLAY` (provided automatically when \
|
||||||
|
`hyperhive.gui.enable = true`)."
|
||||||
)]
|
)]
|
||||||
async fn key_press(&self, Parameters(args): Parameters<KeyPressArgs>) -> String {
|
async fn key_press(&self, Parameters(args): Parameters<KeyPressArgs>) -> String {
|
||||||
cmd_result(run_cmd("ydotool", &["key", &args.keys]).await)
|
// Parse "mod1+mod2+key" into: -M mod1 -M mod2 -k key -m mod2 -m mod1
|
||||||
}
|
// split('+') always yields at least one element, so split_last() is safe.
|
||||||
|
let parts: Vec<&str> = args.keys.split('+').collect();
|
||||||
#[tool(
|
let (key, mods) = parts.split_last().expect("split yields ≥1 element");
|
||||||
description = "Move the mouse cursor to an absolute pixel position on the display. \
|
let mut wtype_args: Vec<String> = Vec::new();
|
||||||
`x` is measured from the left edge, `y` from the top edge. Requires the ydotoold \
|
for &m in mods {
|
||||||
daemon and `/dev/uinput` device access — set `hyperhive.gui.screenInput = true` \
|
wtype_args.push("-M".to_owned());
|
||||||
in the agent config to enable the daemon."
|
wtype_args.push(m.to_owned());
|
||||||
)]
|
|
||||||
async fn mouse_move(&self, Parameters(args): Parameters<MouseMoveArgs>) -> String {
|
|
||||||
cmd_result(
|
|
||||||
run_cmd(
|
|
||||||
"ydotool",
|
|
||||||
&[
|
|
||||||
"mousemove",
|
|
||||||
"--absolute",
|
|
||||||
"-x",
|
|
||||||
&args.x.to_string(),
|
|
||||||
"-y",
|
|
||||||
&args.y.to_string(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
.await,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tool(
|
|
||||||
description = "Click a mouse button, optionally moving to a position first. `button` \
|
|
||||||
is `\"left\"` (default), `\"right\"`, or `\"middle\"`. When `x` and `y` are \
|
|
||||||
provided the cursor is moved to that absolute pixel position before clicking. \
|
|
||||||
Requires the ydotoold daemon and `/dev/uinput` device access — set \
|
|
||||||
`hyperhive.gui.screenInput = true` in the agent config to enable the daemon."
|
|
||||||
)]
|
|
||||||
async fn mouse_click(&self, Parameters(args): Parameters<MouseClickArgs>) -> String {
|
|
||||||
// Move first when coordinates are supplied.
|
|
||||||
if let (Some(x), Some(y)) = (args.x, args.y)
|
|
||||||
&& let Err(e) = run_cmd(
|
|
||||||
"ydotool",
|
|
||||||
&[
|
|
||||||
"mousemove",
|
|
||||||
"--absolute",
|
|
||||||
"-x",
|
|
||||||
&x.to_string(),
|
|
||||||
"-y",
|
|
||||||
&y.to_string(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
return e;
|
|
||||||
}
|
}
|
||||||
// Map the button name to ydotool's button code.
|
wtype_args.push("-k".to_owned());
|
||||||
let button = match args.button.as_deref().unwrap_or("left") {
|
wtype_args.push((*key).to_owned());
|
||||||
"right" => "0xC1",
|
for &m in mods.iter().rev() {
|
||||||
"middle" => "0xC2",
|
wtype_args.push("-m".to_owned());
|
||||||
_ => "0xC0", // left
|
wtype_args.push(m.to_owned());
|
||||||
};
|
}
|
||||||
cmd_result(run_cmd("ydotool", &["click", button]).await)
|
let arg_refs: Vec<&str> = wtype_args.iter().map(String::as_str).collect();
|
||||||
|
cmd_result(run_cmd("wtype", &arg_refs).await)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
|
||||||
struct ScreenshotArgs {
|
|
||||||
/// Optional destination path for the PNG file. When omitted a
|
|
||||||
/// timestamped path under `/tmp/` is generated automatically.
|
|
||||||
dest_path: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
struct TypeTextArgs {
|
struct TypeTextArgs {
|
||||||
/// Text to type into the focused window. Supports arbitrary Unicode.
|
/// Text to type into the focused window. Supports arbitrary Unicode.
|
||||||
|
|
@ -177,29 +128,11 @@ struct TypeTextArgs {
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
struct KeyPressArgs {
|
struct KeyPressArgs {
|
||||||
/// Key or key combination in ydotool syntax — e.g. `"Return"`,
|
/// Key or key combination in XKB keysym syntax — e.g. `"Return"`, `"Tab"`,
|
||||||
/// `"Tab"`, `"ctrl+c"`, `"ctrl+shift+t"`, `"super+l"`, `"alt+F4"`.
|
/// `"ctrl+c"`, `"ctrl+shift+t"`, `"super+l"`, `"alt+F4"`.
|
||||||
keys: String,
|
keys: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
|
||||||
struct MouseMoveArgs {
|
|
||||||
/// Horizontal pixel coordinate from the left edge of the display.
|
|
||||||
x: i32,
|
|
||||||
/// Vertical pixel coordinate from the top edge of the display.
|
|
||||||
y: i32,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
|
||||||
struct MouseClickArgs {
|
|
||||||
/// Button to click: `"left"` (default), `"right"`, or `"middle"`.
|
|
||||||
button: Option<String>,
|
|
||||||
/// Optional X coordinate to move to before clicking (absolute pixels).
|
|
||||||
x: Option<i32>,
|
|
||||||
/// Optional Y coordinate to move to before clicking (absolute pixels).
|
|
||||||
y: Option<i32>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tool_handler]
|
#[tool_handler]
|
||||||
impl ServerHandler for ScreenMcp {}
|
impl ServerHandler for ScreenMcp {}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,13 @@
|
||||||
# Screen MCP — screenshot + input injection for GUI agents.
|
# Screen MCP — screenshot + keyboard input for GUI agents.
|
||||||
#
|
#
|
||||||
# Auto-activated when `hyperhive.gui.enable = true`. Wires the
|
# Auto-activated when `hyperhive.gui.enable = true`. Wires the
|
||||||
# `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude
|
# `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude
|
||||||
# gets five tools: `screenshot`, `type_text`, `key_press`, `mouse_move`,
|
# gets three tools: `screenshot`, `type_text`, and `key_press`.
|
||||||
# and `mouse_click`.
|
|
||||||
#
|
#
|
||||||
# `screenshot` and `type_text` work out of the box (grim + wtype, both
|
# All tools use Wayland protocols mediated by the Weston compositor —
|
||||||
# pure Wayland clients). `key_press`, `mouse_move`, and `mouse_click`
|
# no `/dev/uinput` or kernel-level injection needed. `grim` takes
|
||||||
# require the ydotoold daemon, which injects events via `/dev/uinput`
|
# screenshots; `wtype` handles both text input and key/modifier combos
|
||||||
# at the kernel level — enable it by setting
|
# via the `zwp-virtual-keyboard-unstable-v1` protocol.
|
||||||
# `hyperhive.gui.screenInput = true`.
|
|
||||||
{
|
{
|
||||||
pkgs,
|
pkgs,
|
||||||
lib,
|
lib,
|
||||||
|
|
@ -17,20 +15,6 @@
|
||||||
...
|
...
|
||||||
}:
|
}:
|
||||||
{
|
{
|
||||||
options.hyperhive.gui.screenInput = lib.mkOption {
|
|
||||||
type = lib.types.bool;
|
|
||||||
default = false;
|
|
||||||
description = ''
|
|
||||||
Enable mouse and keyboard injection via ydotool + the ydotoold
|
|
||||||
daemon. Requires `/dev/uinput` device access inside the container
|
|
||||||
(the host must bind it in via `extraSystemdProperties` or
|
|
||||||
`systemd.nspawn.<name>.filesConfig.Bind`). When false,
|
|
||||||
`screenshot` and `type_text` still work; `key_press`,
|
|
||||||
`mouse_move`, and `mouse_click` return an error from ydotool
|
|
||||||
until ydotoold is running and `/dev/uinput` is accessible.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
config = lib.mkIf config.hyperhive.gui.enable {
|
config = lib.mkIf config.hyperhive.gui.enable {
|
||||||
# Register the screen MCP bridge so claude gets the screen tools.
|
# Register the screen MCP bridge so claude gets the screen tools.
|
||||||
hyperhive.extraMcpServers.screen = {
|
hyperhive.extraMcpServers.screen = {
|
||||||
|
|
@ -38,27 +22,11 @@
|
||||||
args = [ ];
|
args = [ ];
|
||||||
};
|
};
|
||||||
|
|
||||||
# grim: Wayland screenshot; wtype: text/key input (no daemon).
|
# grim: Wayland screenshot; wtype: text + key input via virtual-keyboard
|
||||||
# ydotool: mouse + key injection via uinput (needs screenInput).
|
# protocol (compositor-mediated, no /dev/uinput required).
|
||||||
environment.systemPackages = [
|
environment.systemPackages = [
|
||||||
pkgs.grim
|
pkgs.grim
|
||||||
pkgs.wtype
|
pkgs.wtype
|
||||||
]
|
];
|
||||||
++ lib.optional config.hyperhive.gui.screenInput pkgs.ydotool;
|
|
||||||
|
|
||||||
# ydotoold — uinput event injection daemon. The socket lands at
|
|
||||||
# /tmp/.ydotool_socket by default; ydotool picks it up
|
|
||||||
# automatically. Only started when screenInput is enabled.
|
|
||||||
systemd.services.ydotoold = lib.mkIf config.hyperhive.gui.screenInput {
|
|
||||||
description = "ydotool input injection daemon";
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
after = [ "local-fs.target" ];
|
|
||||||
serviceConfig = {
|
|
||||||
ExecStart = "${pkgs.ydotool}/bin/ydotoold";
|
|
||||||
Restart = "on-failure";
|
|
||||||
RestartSec = "2s";
|
|
||||||
SyslogIdentifier = "ydotoold";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue