Compare commits
8 changed files with 1 additions and 217 deletions
13
Cargo.lock
generated
13
Cargo.lock
generated
|
|
@ -1734,19 +1734,6 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hive-screen-mcp"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"rmcp",
|
||||
"schemars",
|
||||
"serde",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hive-sh4re"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ members = [
|
|||
"hive-bash-mcp",
|
||||
"hive-c0re",
|
||||
"hive-claude",
|
||||
"hive-screen-mcp",
|
||||
"hive-forge",
|
||||
"hive-host-sock",
|
||||
"hive-jobq",
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
[package]
|
||||
name = "hive-screen-mcp"
|
||||
edition.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
rmcp.workspace = true
|
||||
schemars.workspace = true
|
||||
serde.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
|
@ -1,152 +0,0 @@
|
|||
//! `hive-screen-mcp` — stdio MCP bridge for GUI agents.
|
||||
//!
|
||||
//! Provides screenshot and keyboard-input tools for agents running a Weston
|
||||
//! Wayland compositor (`hyperhive.gui.enable = true`). Each tool shells out
|
||||
//! to the appropriate utility:
|
||||
//!
|
||||
//! - `screenshot` → `grim` (always available with gui enabled)
|
||||
//! - `type_text` → `wtype` (Wayland-native text input, no daemon)
|
||||
//! - `key_press` → `wtype -k` (Wayland virtual-keyboard protocol, no daemon)
|
||||
//!
|
||||
//! All three tools use Wayland protocols mediated by the compositor —
|
||||
//! no `/dev/uinput` or kernel-level injection.
|
||||
//! `WAYLAND_DISPLAY` and `XDG_RUNTIME_DIR` are injected globally by the
|
||||
//! `weston-vnc` module so all tools find the compositor automatically.
|
||||
|
||||
use anyhow::Result;
|
||||
use rmcp::{
|
||||
ServerHandler, ServiceExt,
|
||||
handler::server::wrapper::Parameters,
|
||||
schemars::{self, JsonSchema},
|
||||
tool, tool_handler, tool_router,
|
||||
transport::stdio,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use tokio::process::Command;
|
||||
|
||||
/// Run a command and return its result.
|
||||
/// `Ok(stdout_trimmed)` on success (may be empty string when the tool
|
||||
/// produces no output). `Err(human_readable_message)` on failure
|
||||
/// (non-zero exit or failed spawn).
|
||||
async fn run_cmd(program: &str, args: &[&str]) -> Result<String, String> {
|
||||
match Command::new(program).args(args).output().await {
|
||||
Ok(out) if out.status.success() => {
|
||||
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
|
||||
}
|
||||
Ok(out) => {
|
||||
let msg = String::from_utf8_lossy(&out.stderr).trim().to_owned();
|
||||
Err(format!(
|
||||
"command failed (exit {}): {}",
|
||||
out.status.code().unwrap_or(-1),
|
||||
if msg.is_empty() { "(no stderr)" } else { &msg }
|
||||
))
|
||||
}
|
||||
Err(e) => Err(format!("failed to launch `{program}`: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format a `run_cmd` result as a tool-response string. `Ok("")` →
|
||||
/// `"ok"` (most input tools produce no stdout); `Ok(s)` → `s`;
|
||||
/// `Err(e)` → `e`.
|
||||
fn cmd_result(r: Result<String, String>) -> String {
|
||||
match r {
|
||||
Ok(s) if s.is_empty() => "ok".to_owned(),
|
||||
Ok(s) => s,
|
||||
Err(e) => e,
|
||||
}
|
||||
}
|
||||
|
||||
struct ScreenMcp;
|
||||
|
||||
#[tool_router]
|
||||
impl ScreenMcp {
|
||||
#[tool(
|
||||
description = "Take a screenshot of the agent's Wayland display and save it as a PNG \
|
||||
file under `/tmp/`. Returns the path to the saved file — pass that path to the \
|
||||
`Read` tool to view the image visually. Requires `grim` and a live \
|
||||
`WAYLAND_DISPLAY` (provided automatically when `hyperhive.gui.enable = true`)."
|
||||
)]
|
||||
async fn screenshot(&self) -> String {
|
||||
let ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_millis());
|
||||
let path = format!("/tmp/hive-screenshot-{ms}.png");
|
||||
match run_cmd("grim", &["-t", "png", &path]).await {
|
||||
Ok(_) => format!("screenshot saved to `{path}` — use the Read tool to view it"),
|
||||
Err(e) => e,
|
||||
}
|
||||
}
|
||||
|
||||
#[tool(
|
||||
description = "Type text into the currently focused Wayland window. Uses `wtype` \
|
||||
via the Wayland virtual-keyboard protocol (compositor-mediated, no kernel \
|
||||
injection). Supports arbitrary Unicode. For special keys (Enter, Tab, Escape, \
|
||||
function keys, arrow keys) or modifier combos, use `key_press` instead. \
|
||||
Requires a live `WAYLAND_DISPLAY` (provided automatically when \
|
||||
`hyperhive.gui.enable = true`)."
|
||||
)]
|
||||
async fn type_text(&self, Parameters(args): Parameters<TypeTextArgs>) -> String {
|
||||
cmd_result(run_cmd("wtype", &[&args.text]).await)
|
||||
}
|
||||
|
||||
#[tool(
|
||||
description = "Press a key or key combination via the Wayland virtual-keyboard \
|
||||
protocol (compositor-mediated, no kernel injection). `keys` uses XKB keysym \
|
||||
syntax: a single key name (`Return`, `Tab`, `Escape`, `BackSpace`, `Delete`, \
|
||||
`space`, `F1`–`F12`, `Left`, `Right`, `Up`, `Down`) or a modifier+key combo \
|
||||
with `+`-separated parts where all but the last are modifiers (`ctrl+c`, \
|
||||
`ctrl+shift+t`, `super+l`, `alt+F4`). Modifier names: `ctrl`, `shift`, `alt`, \
|
||||
`super`. Requires a live `WAYLAND_DISPLAY` (provided automatically when \
|
||||
`hyperhive.gui.enable = true`)."
|
||||
)]
|
||||
async fn key_press(&self, Parameters(args): Parameters<KeyPressArgs>) -> String {
|
||||
// Parse "mod1+mod2+key" into: -M mod1 -M mod2 -k key -m mod2 -m mod1
|
||||
// split('+') always yields at least one element, so split_last() is safe.
|
||||
let parts: Vec<&str> = args.keys.split('+').collect();
|
||||
let (key, mods) = parts.split_last().expect("split yields ≥1 element");
|
||||
let mut wtype_args: Vec<String> = Vec::new();
|
||||
for &m in mods {
|
||||
wtype_args.push("-M".to_owned());
|
||||
wtype_args.push(m.to_owned());
|
||||
}
|
||||
wtype_args.push("-k".to_owned());
|
||||
wtype_args.push((*key).to_owned());
|
||||
for &m in mods.iter().rev() {
|
||||
wtype_args.push("-m".to_owned());
|
||||
wtype_args.push(m.to_owned());
|
||||
}
|
||||
let arg_refs: Vec<&str> = wtype_args.iter().map(String::as_str).collect();
|
||||
cmd_result(run_cmd("wtype", &arg_refs).await)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
struct TypeTextArgs {
|
||||
/// Text to type into the focused window. Supports arbitrary Unicode.
|
||||
text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
struct KeyPressArgs {
|
||||
/// Key or key combination in XKB keysym syntax — e.g. `"Return"`, `"Tab"`,
|
||||
/// `"ctrl+c"`, `"ctrl+shift+t"`, `"super+l"`, `"alt+F4"`.
|
||||
keys: String,
|
||||
}
|
||||
|
||||
#[tool_handler]
|
||||
impl ServerHandler for ScreenMcp {}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_env("RUST_LOG")
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")),
|
||||
)
|
||||
.with_writer(std::io::stderr)
|
||||
.init();
|
||||
|
||||
let service = ScreenMcp.serve(stdio()).await?;
|
||||
service.waiting().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -33,7 +33,6 @@
|
|||
./network.nix
|
||||
./packages.nix
|
||||
./user.nix
|
||||
./screen.nix
|
||||
./weston-vnc.nix
|
||||
(lib.mkRemovedOptionModule [ "hyperhive" "web" "useUnixSocket" ] ''
|
||||
Unix socket mode is always enabled for all agents. Remove the
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
per-binary daemon/CLI packages (`hive-agent`, `hive-agent-mcp`,
|
||||
`hive-agent-wake`, `hive-bash-daemon`, `hive-bash-mcp`,
|
||||
`hive-forge`, `hive-matrix-daemon`, `hive-matrix-mcp`,
|
||||
`hive-metric`, `hive-screen-mcp`) plus the `assets`, `frontend` and
|
||||
`hive-metric`) plus the `assets`, `frontend` and
|
||||
`reference-docs` trees. Wired by the flake's agent-base/ruth
|
||||
nixosModules to `hyperhive.packages.<system>.*`; override an
|
||||
individual key per-agent to swap in a patched binary.
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
# Screen MCP — screenshot + keyboard input for GUI agents.
|
||||
#
|
||||
# Auto-activated when `hyperhive.gui.enable = true`. Wires the
|
||||
# `hive-screen-mcp` stdio bridge as `extraMcpServers.screen` so claude
|
||||
# gets three tools: `screenshot`, `type_text`, and `key_press`.
|
||||
#
|
||||
# All tools use Wayland protocols mediated by the Weston compositor —
|
||||
# no `/dev/uinput` or kernel-level injection needed. `grim` takes
|
||||
# screenshots; `wtype` handles both text input and key/modifier combos
|
||||
# via the `zwp-virtual-keyboard-unstable-v1` protocol.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
config = lib.mkIf config.hyperhive.gui.enable {
|
||||
# Register the screen MCP bridge so claude gets the screen tools.
|
||||
hyperhive.extraMcpServers.screen = {
|
||||
command = "${config.hyperhive.packages.hive-screen-mcp}/bin/hive-screen-mcp";
|
||||
args = [ ];
|
||||
};
|
||||
|
||||
# grim: Wayland screenshot; wtype: text + key input via virtual-keyboard
|
||||
# protocol (compositor-mediated, no /dev/uinput required).
|
||||
environment.systemPackages = [
|
||||
pkgs.grim
|
||||
pkgs.wtype
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
@ -32,7 +32,6 @@ let
|
|||
hive-matrix-daemon = "hyperhive per-agent matrix-sdk daemon";
|
||||
hive-matrix-mcp = "hyperhive matrix MCP bridge";
|
||||
hive-metric = "hyperhive agent-emitted custom metrics CLI";
|
||||
hive-screen-mcp = "hyperhive screen MCP bridge (screenshot + input for GUI agents)";
|
||||
hive-forge = "hyperhive Forgejo CLI";
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue