feat(#2305): hive-screen-mcp — screenshot + input MCP for GUI agents
New crate hive-screen-mcp: a stdio MCP bridge activated automatically when an agent has hyperhive.gui.enable = true. Provides five tools: - screenshot — grim → saves PNG, returns path for Read tool - type_text — wtype → Unicode text input (no daemon) - key_press — ydotool key → combos like ctrl+c, super+l - mouse_move — ydotool mousemove --absolute - mouse_click — ydotool click, optionally with prior move New nix/agent-modules/screen.nix: wires the MCP bridge into extraMcpServers.screen; adds grim + wtype to systemPackages. Adds hyperhive.gui.screenInput option (default false) which enables the ydotoold daemon + ydotool for mouse/keyboard injection via /dev/uinput. screenshot and type_text work without screenInput. key_press, mouse_move, and mouse_click return a ydotool error until ydotoold is running and /dev/uinput is accessible in the container.
This commit is contained in:
parent
5906cc2f2b
commit
0b3268feae
8 changed files with 311 additions and 1 deletions
|
|
@ -33,6 +33,7 @@
|
|||
./network.nix
|
||||
./packages.nix
|
||||
./user.nix
|
||||
./screen.nix
|
||||
./weston-vnc.nix
|
||||
(lib.mkRemovedOptionModule [ "hyperhive" "web" "useUnixSocket" ] ''
|
||||
Unix socket mode is always enabled for all agents. Remove the
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
per-binary daemon/CLI packages (`hive-agent`, `hive-agent-mcp`,
|
||||
`hive-agent-wake`, `hive-bash-daemon`, `hive-bash-mcp`,
|
||||
`hive-forge`, `hive-matrix-daemon`, `hive-matrix-mcp`,
|
||||
`hive-metric`) plus the `assets`, `frontend` and
|
||||
`hive-metric`, `hive-screen-mcp`) plus the `assets`, `frontend` and
|
||||
`reference-docs` trees. Wired by the flake's agent-base/ruth
|
||||
nixosModules to `hyperhive.packages.<system>.*`; override an
|
||||
individual key per-agent to swap in a patched binary.
|
||||
|
|
|
|||
62
nix/agent-modules/screen.nix
Normal file
62
nix/agent-modules/screen.nix
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
# Screen MCP — screenshot + input injection for GUI agents.
|
||||
#
|
||||
# Auto-activated when `hyperhive.gui.enable = true`. Wires the
|
||||
# `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude
|
||||
# gets five tools: `screenshot`, `type_text`, `key_press`, `mouse_move`,
|
||||
# and `mouse_click`.
|
||||
#
|
||||
# `screenshot` and `type_text` work out of the box (grim + wtype, both
|
||||
# pure Wayland clients). `key_press`, `mouse_move`, and `mouse_click`
|
||||
# require the ydotoold daemon, which injects events via `/dev/uinput`
|
||||
# at the kernel level — enable it by setting
|
||||
# `hyperhive.gui.screenInput = true`.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
options.hyperhive.gui.screenInput = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Enable mouse and keyboard injection via ydotool + the ydotoold
|
||||
daemon. Requires `/dev/uinput` device access inside the container
|
||||
(the host must bind it in via `extraSystemdProperties` or
|
||||
`systemd.nspawn.<name>.filesConfig.Bind`). When false,
|
||||
`screenshot` and `type_text` still work; `key_press`,
|
||||
`mouse_move`, and `mouse_click` return an error from ydotool
|
||||
until ydotoold is running and `/dev/uinput` is accessible.
|
||||
'';
|
||||
};
|
||||
|
||||
config = lib.mkIf config.hyperhive.gui.enable {
|
||||
# Register the screen MCP bridge so claude gets the screen tools.
|
||||
hyperhive.extraMcpServers.screen = {
|
||||
command = "${config.hyperhive.packages.hive-screen-mcp}/bin/hive-screen-mcp";
|
||||
args = [ ];
|
||||
};
|
||||
|
||||
# grim: Wayland screenshot; wtype: text/key input (no daemon).
|
||||
# ydotool: mouse + key injection via uinput (needs screenInput).
|
||||
environment.systemPackages =
|
||||
[ pkgs.grim pkgs.wtype ]
|
||||
++ lib.optional config.hyperhive.gui.screenInput pkgs.ydotool;
|
||||
|
||||
# ydotoold — uinput event injection daemon. The socket lands at
|
||||
# /tmp/.ydotool_socket by default; ydotool picks it up
|
||||
# automatically. Only started when screenInput is enabled.
|
||||
systemd.services.ydotoold = lib.mkIf config.hyperhive.gui.screenInput {
|
||||
description = "ydotool input injection daemon";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "local-fs.target" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkgs.ydotool}/bin/ydotoold";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "2s";
|
||||
SyslogIdentifier = "ydotoold";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue