agents can transcribe an audio file: the transcribe_audio tool, through the harness and the speech-to-text service

hive-agent-mcp/src/mcp/args.rs

@@ -13,6 +13,15 @@ pub struct MarkTodosDoneArgs {
     pub ids: Vec<i64>,
 }
 
+/// MCP tool args for `transcribe_audio`.
+#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
+pub struct TranscribeAudioArgs {
+    /// Absolute path of the audio file to transcribe (a regular file, at most
+    /// 6 MiB; Ogg, WAV, FLAC, MP3, AMR, `MP4`/`M4A` or `WebM`). For a matrix voice
+    /// memo, the `path` that `download_file` returned.
+    pub path: String,
+}
+
 /// MCP tool args for `remind`. Exactly one of `delay_seconds` or
 /// `at_unix_timestamp` must be set; both / neither is a tool-side error.
 /// Hides the tagged `ReminderTiming` enum behind a flatter schema so the

hive-agent-mcp/src/mcp/mod.rs

@@ -24,13 +24,14 @@ mod args;
 mod render;
 
 pub use args::{
-    CancelLooseEndArgs, CompactArgs, GetAgentMetaArgs, MarkTodosDoneArgs, RemindArgs, SetStatusArgs,
+    CancelLooseEndArgs, CompactArgs, GetAgentMetaArgs, MarkTodosDoneArgs, RemindArgs,
+    SetStatusArgs, TranscribeAudioArgs,
 };
 pub use render::{annotate_retries, format_ack, format_agent_meta};
 
 use render::{
     dial_agent_socket, format_matrix_summary, local_reminders, local_todos, mark_local_todo_done,
-    mark_local_todos_done, matrix_unread_summary, render_loose_ends, reply_err,
+    mark_local_todos_done, matrix_unread_summary, render_loose_ends, reply_err, transcribe_local,
 };
 
 /// Write (or remove) the status file in the agent's own `state/` directory.
@@ -329,6 +330,37 @@ impl AgentServer {
         .await
     }
 
+    #[tool(
+        description = "Get the text spoken in an audio file, from the swarm's speech-to-text \
+        service: a matrix voice memo (download it with the matrix `download_file` tool and pass \
+        the returned `path`), or any Ogg, WAV, FLAC, MP3, AMR, MP4/M4A or WebM file up to 6 MiB. \
+        The service keeps each transcript for a day, so the same memo heard by several agents is \
+        transcribed once. Can take minutes when other memos are queued ahead. Returns the text \
+        (empty for silence), or why it could not be transcribed."
+    )]
+    async fn transcribe_audio(&self, Parameters(args): Parameters<TranscribeAudioArgs>) -> String {
+        let log = format!("{args:?}");
+        run_tool_envelope("transcribe_audio", log, async move {
+            match transcribe_local(args.path).await {
+                Some(hive_agent_sock::Response::Transcript { text, .. }) => {
+                    if text.trim().is_empty() {
+                        "(no speech found)".to_owned()
+                    } else {
+                        text
+                    }
+                }
+                Some(hive_agent_sock::Response::Err { message }) => {
+                    format!("transcribe_audio failed: {message}")
+                }
+                Some(other) => format!("transcribe_audio unexpected response: {other:?}"),
+                None => "transcribe_audio: local agent socket unavailable \
+                         (HIVE_AGENT_SOCKET unset or harness unreachable)"
+                    .to_owned(),
+            }
+        })
+        .await
+    }
+
     #[tool(
         description = "Schedule a reminder that lands in this agent's own inbox at a future \
         time (sender will appear as `reminder`). Use for self-paced follow-ups: 'check task \

hive-agent-mcp/src/mcp/render.rs

@@ -302,6 +302,26 @@ pub(super) async fn mark_local_todos_done(ids: Vec<i64>) -> Option<hive_agent_so
     dial_agent_socket(&hive_agent_sock::Request::MarkTodosDone { ids }).await
 }
 
+/// Ask the harness for the text of the audio file at `path`. The harness
+/// waits on the swarm's speech-to-text service, which queues requests behind
+/// one engine slot, so this waits much longer than any other socket call.
+pub(super) async fn transcribe_local(path: String) -> Option<hive_agent_sock::Response> {
+    let socket = std::env::var_os("HIVE_AGENT_SOCKET").map(std::path::PathBuf::from)?;
+    if !socket.exists() {
+        return None;
+    }
+    dial_socket(
+        &socket,
+        &hive_agent_sock::Request::Transcribe { path },
+        TRANSCRIBE_DEADLINE,
+    )
+    .await
+}
+
+/// Longest a `transcribe_audio` call waits: the service's reply deadline (15
+/// minutes, `swarm_wire::stt::REPLY_DEADLINE`) plus a margin for the harness.
+const TRANSCRIBE_DEADLINE: std::time::Duration = std::time::Duration::from_mins(16);
+
 /// Format a `Vec<MatrixRoomUnread>` into a per-room summary string.
 /// Single room / single message collapses to one line; multi-room
 /// expands to a bulleted list. Returns an empty string for empty input.