From 512e9ff09f7678fd88d905932177c71d3e78c1e4 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 27 Jun 2026 13:59:57 +0200 Subject: [PATCH 1/9] fix(#2051): verify config-repo branch protection actually applied MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apply_config_repo_branch_protection treated 200/409/422 from the create-branch-protection POST all as success. But a 422 means Forgejo *rejected* the request and created no rule — so a rejected POST silently left the agent's config repo unprotected, with nothing logged (a new agent's config repo was found with no main-branch protection and no trace of why). Don't trust the status code: - On any non-201, GET the single .../branch_protections/main rule and only treat it as success if the rule is actually present. - Otherwise return Err carrying the POST's response body, so the real Forgejo rejection reason lands in the host journal. (forge_http discarded the body; added forge_http_full that returns it.) ensure_config_repo runs on every sync_agent sweep (startup + each rebuild), so a now-Err result is logged and retried next sweep — self-healing once a real cause is fixed. Net: the failure is loud + retried instead of silently swallowed. nix fmt clean. --- hive-c0re/src/forge.rs | 61 +++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/hive-c0re/src/forge.rs b/hive-c0re/src/forge.rs index e2bb8756..914121d5 100644 --- a/hive-c0re/src/forge.rs +++ b/hive-c0re/src/forge.rs @@ -157,6 +157,30 @@ async fn forge_http( Ok(resp.status()) } +/// Like [`forge_http`] but also returns the response body, so callers +/// can log *why* Forgejo rejected a request (e.g. the validation +/// message on a 422). Body read is best-effort — a read error yields an +/// empty string rather than failing the whole call. +async fn forge_http_full( + method: reqwest::Method, + url: &str, + token: &str, + body: &str, +) -> Result<(StatusCode, String)> { + let client = reqwest::Client::new(); + let resp = client + .request(method, url) + .header("Authorization", format!("token {token}")) + .header("Content-Type", "application/json") + .body(body.to_owned()) + .send() + .await + .with_context(|| format!("forge HTTP request to {url}"))?; + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + Ok((status, text)) +} + /// Ensure a forgejo user named `name` exists. Idempotent: forgejo /// returns a "user already exists" error which we treat as success. /// `admin` adds `--admin` (site admin) — used for the bootstrap @@ -1013,19 +1037,30 @@ async fn apply_config_repo_branch_protection(repo: &str, token: &str) -> Result< let body = format!( r#"{{"branch_name":"main","enable_push_whitelist":true,"push_whitelist_usernames":["core"],"enable_merge_whitelist":true,"merge_whitelist_usernames":["core"],"enable_approvals_whitelist":true,"approvals_whitelist_teams":["{OPERATORS_TEAM}"],"required_approvals":1,"block_on_official_review_requests":true,"allow_manual_merge":true,"enable_force_push":false}}"# ); - let status = forge_http(reqwest::Method::POST, &url, token, &body).await?; - match status.as_u16() { - 201 => { - tracing::info!(%repo, "forge: applied config-repo branch protection"); - Ok(()) - } - 200 | 409 | 422 => { - tracing::debug!(%repo, "forge: config-repo branch protection already present"); - Ok(()) - } - other => { - anyhow::bail!("POST {CONFIG_ORG}/{repo}/branch_protections returned HTTP {other}") - } + let (status, resp_body) = forge_http_full(reqwest::Method::POST, &url, token, &body).await?; + if status.as_u16() == 201 { + tracing::info!(%repo, "forge: applied config-repo branch protection"); + return Ok(()); + } + // Non-201 is ambiguous: it can mean "rule already exists" (idempotent + // success) OR a silent rejection — e.g. a 422 where Forgejo refused + // the request and created NO rule. The old code treated 200/409/422 + // all as success, so a rejected POST left the repo unprotected with + // no error (the reported case: a new agent's config repo had no + // `main` rule and nothing was logged). Don't trust the status code: + // verify the `main` rule actually exists, and on failure surface the + // POST's response body so the real reason is in the journal. + let main_url = format!("{url}/main"); + let check = forge_http(reqwest::Method::GET, &main_url, token, "").await?; + if check.as_u16() == 200 { + tracing::debug!(%repo, %status, "forge: config-repo branch protection already present"); + Ok(()) + } else { + anyhow::bail!( + "branch protection for {CONFIG_ORG}/{repo} not applied: POST -> HTTP {status} \ + (body: {body}); GET main -> HTTP {check}, no `main` rule present", + body = resp_body.trim(), + ) } } From 3fedc102cc5e48597c0a45dc568c6a83ae65795e Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 27 Jun 2026 14:09:22 +0200 Subject: [PATCH 2/9] refactor(#2051): fold body-return into forge_http instead of a near-copy Per review: rather than adding forge_http_full (a near-duplicate of forge_http), change forge_http itself to return (StatusCode, String). Status-only callers bind (status, _); the branch-protection verify path uses the body to log the real Forgejo rejection reason. Updates all call sites accordingly. --- hive-c0re/src/forge.rs | 64 ++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 40 deletions(-) diff --git a/hive-c0re/src/forge.rs b/hive-c0re/src/forge.rs index 914121d5..e2a868ae 100644 --- a/hive-c0re/src/forge.rs +++ b/hive-c0re/src/forge.rs @@ -139,33 +139,16 @@ fn agent_email(name: &str) -> String { /// All Forgejo API calls that don't shell out to `forgejo admin` go /// through here — one place for auth header, content-type, error /// propagation, and the shared reqwest Client. +/// Returns the response status **and body**. The body lets callers log +/// *why* Forgejo rejected a request (e.g. the validation message on a +/// 422); status-only callers just bind `(status, _)`. Body read is +/// best-effort — a read error yields an empty string rather than +/// failing the whole call. async fn forge_http( method: reqwest::Method, url: &str, token: &str, body: &str, -) -> Result { - let client = reqwest::Client::new(); - let resp = client - .request(method, url) - .header("Authorization", format!("token {token}")) - .header("Content-Type", "application/json") - .body(body.to_owned()) - .send() - .await - .with_context(|| format!("forge HTTP request to {url}"))?; - Ok(resp.status()) -} - -/// Like [`forge_http`] but also returns the response body, so callers -/// can log *why* Forgejo rejected a request (e.g. the validation -/// message on a 422). Body read is best-effort — a read error yields an -/// empty string rather than failing the whole call. -async fn forge_http_full( - method: reqwest::Method, - url: &str, - token: &str, - body: &str, ) -> Result<(StatusCode, String)> { let client = reqwest::Client::new(); let resp = client @@ -275,14 +258,14 @@ async fn ensure_user_email(name: &str) { let body = format!(r#"{{"email":"{email}","login_name":"{name}","source_id":0}}"#); let url = format!("{FORGE_HTTP}/api/v1/admin/users/{name}"); match forge_http(reqwest::Method::PATCH, &url, &token, &body).await { - Ok(status) if status.is_success() => { + Ok((status, _)) if status.is_success() => { if let Some(parent) = marker.parent() { std::fs::create_dir_all(parent).ok(); } std::fs::write(&marker, "").ok(); tracing::info!(%name, %email, "forge: user email aligned"); } - Ok(status) if status == reqwest::StatusCode::FORBIDDEN => { + Ok((status, _)) if status == reqwest::StatusCode::FORBIDDEN => { // Core token missing admin scope — see // `docs/forge.md::Token scopes` migration note. tracing::warn!( @@ -291,7 +274,7 @@ async fn ensure_user_email(name: &str) { Delete {CORE_TOKEN_PATH} and restart hive-c0re to re-mint with the new scopes." ); } - Ok(status) => { + Ok((status, _)) => { tracing::warn!(%name, %email, %status, "forge: PATCH user email returned non-success"); } Err(e) => tracing::warn!(%name, error = %e, "forge: PATCH user email transport error"), @@ -326,21 +309,21 @@ async fn ensure_repo_creation_disabled(name: &str) { let body = format!(r#"{{"login_name":"{name}","source_id":0,"max_repo_creation":0}}"#); let url = format!("{FORGE_HTTP}/api/v1/admin/users/{name}"); match forge_http(reqwest::Method::PATCH, &url, &token, &body).await { - Ok(status) if status.is_success() => { + Ok((status, _)) if status.is_success() => { if let Some(parent) = marker.parent() { std::fs::create_dir_all(parent).ok(); } std::fs::write(&marker, "").ok(); tracing::info!(%name, "forge: disabled direct repo creation (max_repo_creation=0)"); } - Ok(status) if status == reqwest::StatusCode::FORBIDDEN => { + Ok((status, _)) if status == reqwest::StatusCode::FORBIDDEN => { tracing::warn!( %name, %status, "forge: PATCH max_repo_creation forbidden — core token likely missing admin scope. \ Delete {CORE_TOKEN_PATH} and restart hive-c0re to re-mint with the new scopes." ); } - Ok(status) => { + Ok((status, _)) => { tracing::warn!(%name, %status, "forge: PATCH max_repo_creation returned non-success"); } Err(e) => { @@ -468,7 +451,7 @@ async fn ensure_core_avatar(token: &str) -> Result<()> { base64::engine::general_purpose::STANDARD.encode(&png_bytes), ); let url = format!("{FORGE_HTTP}/api/v1/admin/users/core/avatar"); - let status = forge_http(reqwest::Method::POST, &url, token, &body).await?; + let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; if !status.is_success() { anyhow::bail!("set core avatar: HTTP {status}"); } @@ -499,7 +482,7 @@ async fn ensure_config_org_avatar(token: &str) -> Result<()> { base64::engine::general_purpose::STANDARD.encode(&png_bytes), ); let url = format!("{FORGE_HTTP}/api/v1/orgs/{CONFIG_ORG}/avatar"); - let status = forge_http(reqwest::Method::POST, &url, token, &body).await?; + let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; if !status.is_success() { anyhow::bail!("set {CONFIG_ORG} avatar: HTTP {status}"); } @@ -552,7 +535,7 @@ fn classify_core_token_status(status: StatusCode) -> CoreTokenCheck { async fn check_core_token(token: &str) -> CoreTokenCheck { let url = format!("{FORGE_HTTP}/api/v1/user"); match forge_http(reqwest::Method::GET, &url, token, "").await { - Ok(status) => classify_core_token_status(status), + Ok((status, _)) => classify_core_token_status(status), Err(e) => { tracing::debug!( error = %e, @@ -614,7 +597,8 @@ fn repo_body_public(name: &str) -> String { /// created as private on an older deployment. async fn set_repo_public(owner: &str, repo: &str, token: &str) -> Result<()> { let url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}"); - let status = forge_http(reqwest::Method::PATCH, &url, token, r#"{"private":false}"#).await?; + let (status, _) = + forge_http(reqwest::Method::PATCH, &url, token, r#"{"private":false}"#).await?; match status.as_u16() { 200 => { tracing::debug!(%owner, %repo, "forge: repo set to public"); @@ -639,7 +623,7 @@ async fn ensure_org_repo_public(org: &str, name: &str, token: &str) -> Result<() /// (HTTP 409 / 422) into success. `label` is `/` — purely /// for log + error context. async fn create_repo(url: &str, body: &str, token: &str, label: &str) -> Result<()> { - let status = forge_http(reqwest::Method::POST, url, token, body).await?; + let (status, _) = forge_http(reqwest::Method::POST, url, token, body).await?; match status.as_u16() { 201 => { tracing::info!(%label, "forge: created repo"); @@ -915,7 +899,7 @@ pub async fn push_config(name: &str) -> Result<()> { async fn ensure_org(name: &str, admin_token: &str) -> Result<()> { let body = format!(r#"{{"username":"{name}"}}"#); let url = format!("{FORGE_HTTP}/api/v1/orgs"); - let status = forge_http(reqwest::Method::POST, &url, admin_token, &body).await?; + let (status, _) = forge_http(reqwest::Method::POST, &url, admin_token, &body).await?; match status.as_u16() { 201 => { tracing::info!(%name, "forge: created org"); @@ -950,7 +934,7 @@ async fn ensure_operators_team(token: &str) -> Result<()> { let body = format!( r#"{{"name":"{OPERATORS_TEAM}","description":"hyperhive operators — merge gate for agent repos","permission":"write","includes_all_repositories":true,"can_create_org_repo":false}}"# ); - let status = forge_http(reqwest::Method::POST, &url, token, &body).await?; + let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; match status.as_u16() { 201 => { tracing::info!("forge: created {OPERATORS_TEAM} team in {AGENTS_ORG}"); @@ -978,7 +962,7 @@ async fn add_collaborator( ) -> Result<()> { let url = format!("{FORGE_HTTP}/api/v1/repos/{owner}/{repo}/collaborators/{user}"); let body = format!(r#"{{"permission":"{permission}"}}"#); - let status = forge_http(reqwest::Method::PUT, &url, token, &body).await?; + let (status, _) = forge_http(reqwest::Method::PUT, &url, token, &body).await?; match status.as_u16() { 201 | 204 => { tracing::debug!(%owner, %repo, %user, %permission, "forge: collaborator set"); @@ -1000,7 +984,7 @@ async fn apply_operator_branch_protection(repo: &str, token: &str) -> Result<()> let body = format!( r#"{{"branch_name":"main","enable_merge_whitelist":true,"merge_whitelist_teams":["{OPERATORS_TEAM}"],"enable_approvals_whitelist":true,"approvals_whitelist_teams":["{OPERATORS_TEAM}"],"required_approvals":1,"block_on_official_review_requests":true}}"# ); - let status = forge_http(reqwest::Method::POST, &url, token, &body).await?; + let (status, _) = forge_http(reqwest::Method::POST, &url, token, &body).await?; match status.as_u16() { 201 => { tracing::info!(%repo, "forge: applied operator branch protection"); @@ -1037,7 +1021,7 @@ async fn apply_config_repo_branch_protection(repo: &str, token: &str) -> Result< let body = format!( r#"{{"branch_name":"main","enable_push_whitelist":true,"push_whitelist_usernames":["core"],"enable_merge_whitelist":true,"merge_whitelist_usernames":["core"],"enable_approvals_whitelist":true,"approvals_whitelist_teams":["{OPERATORS_TEAM}"],"required_approvals":1,"block_on_official_review_requests":true,"allow_manual_merge":true,"enable_force_push":false}}"# ); - let (status, resp_body) = forge_http_full(reqwest::Method::POST, &url, token, &body).await?; + let (status, resp_body) = forge_http(reqwest::Method::POST, &url, token, &body).await?; if status.as_u16() == 201 { tracing::info!(%repo, "forge: applied config-repo branch protection"); return Ok(()); @@ -1051,7 +1035,7 @@ async fn apply_config_repo_branch_protection(repo: &str, token: &str) -> Result< // verify the `main` rule actually exists, and on failure surface the // POST's response body so the real reason is in the journal. let main_url = format!("{url}/main"); - let check = forge_http(reqwest::Method::GET, &main_url, token, "").await?; + let (check, _) = forge_http(reqwest::Method::GET, &main_url, token, "").await?; if check.as_u16() == 200 { tracing::debug!(%repo, %status, "forge: config-repo branch protection already present"); Ok(()) @@ -1447,7 +1431,7 @@ pub async fn mark_pr_merged(repo: &str, pr: u64, sha: &str) -> Result<(), ForgeM .ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?; let url = format!("{FORGE_HTTP}/api/v1/repos/{repo}/pulls/{pr}/merge"); let body = format!(r#"{{"Do":"manually-merged","MergeCommitID":"{sha}"}}"#); - let status = forge_http(reqwest::Method::POST, &url, &token, &body) + let (status, _) = forge_http(reqwest::Method::POST, &url, &token, &body) .await .context("POST pulls//merge (manually-merged)")?; if status.is_success() { From 2bfa5bc1a8c0915bd3f7913f57410c565e523e4a Mon Sep 17 00:00:00 2001 From: iris Date: Sat, 27 Jun 2026 13:50:18 +0200 Subject: [PATCH 3/9] feat(schedules): make schedules pausable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds pause/resume support for scheduled prompts. Backend: - New paused_at_unix column on scheduled_prompts table (added via ALTER TABLE migration so existing databases are upgraded on first start). The due-rows index is dropped and recreated to also exclude paused rows so the worker never fires them while paused. - Worker's due() query gains AND paused_at_unix IS NULL filter. - New pause(id) and resume(id) methods on ScheduledPrompts; both are idempotent and refuse cancelled rows. - New POST /api/schedules/{id}/pause and /api/schedules/{id}/resume dashboard endpoints (operator-direct, no approval gate). Both emit a schedules snapshot on success so the tab updates live. - WireSchedule gains paused_at_unix: Option so the frontend can render the state without an extra fetch. Frontend: - Paused rows render with a distinct row class + muted opacity. - The next-fire cell shows a yellow pause glyph + tooltip with the paused-since timestamp and the would-have-fired time. - Actions column: pause/resume toggle button (⏸/▶) beside fire/edit/cancel. Fire-now is disabled while paused (resume first). - Sort order: active → paused → cancelled (paused slot keeps schedules visible without mixing them into the active top section). - pauseSchedule() / resumeSchedule() async functions POST to the new endpoints and refresh the table on success. --- frontend/packages/dashboard/src/dashboard.css | 4 + frontend/packages/dashboard/src/schedules.js | 80 ++++++++++++++++--- hive-c0re/src/dashboard.rs | 8 ++ hive-c0re/src/dashboard/schedules.rs | 46 +++++++++++ hive-c0re/src/scheduled_prompts.rs | 76 ++++++++++++++++-- hive-c0re/src/socket_server.rs | 2 + hive-sh4re/src/lib.rs | 6 ++ 7 files changed, 205 insertions(+), 17 deletions(-) diff --git a/frontend/packages/dashboard/src/dashboard.css b/frontend/packages/dashboard/src/dashboard.css index d3e6140d..28a68728 100644 --- a/frontend/packages/dashboard/src/dashboard.css +++ b/frontend/packages/dashboard/src/dashboard.css @@ -1100,6 +1100,10 @@ footer .banner-thin { opacity: 0.75; } .schedules-table-row-cancelled td { opacity: 0.55; } +.schedules-table-row-paused td { opacity: 0.75; } +.sched-paused-label { color: var(--yellow); font-size: 0.9em; } +.btn-pause-schedule { color: var(--teal); } +.btn-resume-schedule { color: var(--green); } .schedules-table-body-cell { max-width: 30em; overflow: hidden; diff --git a/frontend/packages/dashboard/src/schedules.js b/frontend/packages/dashboard/src/schedules.js index 0b196a0d..168e7d39 100644 --- a/frontend/packages/dashboard/src/schedules.js +++ b/frontend/packages/dashboard/src/schedules.js @@ -390,9 +390,10 @@ function renderSchedulesList() { table.append(renderSchedulesTableHead(agents)); const tbody = el('tbody', {}); const sorted = schedulesState.slice().sort((a, b) => { - const aDone = a.cancelled_at_unix ? 1 : 0; - const bDone = b.cancelled_at_unix ? 1 : 0; - if (aDone !== bDone) return aDone - bDone; + // Cancelled → bucket 2, paused → bucket 1, active → bucket 0. + const aOrd = a.cancelled_at_unix ? 2 : a.paused_at_unix ? 1 : 0; + const bOrd = b.cancelled_at_unix ? 2 : b.paused_at_unix ? 1 : 0; + if (aOrd !== bOrd) return aOrd - bOrd; return a.next_fire_at_unix - b.next_fire_at_unix; }); for (const s of sorted) { @@ -637,8 +638,11 @@ function renderSchedulesTableHead(agents) { } function renderScheduleRow(s, agents) { const cancelled = !!s.cancelled_at_unix; + const paused = !cancelled && !!s.paused_at_unix; const tr = el('tr', { - class: 'schedules-table-row' + (cancelled ? ' schedules-table-row-cancelled' : ''), + class: 'schedules-table-row' + + (cancelled ? ' schedules-table-row-cancelled' : '') + + (paused ? ' schedules-table-row-paused' : ''), }); tr.append(el('td', { class: 'meta schedules-table-id' }, '#' + s.id)); @@ -649,11 +653,20 @@ function renderScheduleRow(s, agents) { srcKind === 'approval' ? 'approval' : 'operator'))); // "next" cell — relative due-in for active schedules, "cancelled" - // for cancelled ones. Both carry the absolute ISO in the title. + // for cancelled ones, "paused" for paused ones. Both carry the + // absolute ISO in the title. const nextCell = el('td', { class: 'meta schedules-table-next-col' }); if (cancelled) { nextCell.title = 'cancelled ' + new Date(s.cancelled_at_unix * 1000).toISOString(); nextCell.textContent = 'cancelled'; + } else if (paused) { + nextCell.title = 'paused since ' + + new Date(s.paused_at_unix * 1000).toISOString() + + '\nwould fire at ' + + new Date(s.next_fire_at_unix * 1000).toISOString(); + nextCell.append( + el('span', { class: 'sched-paused-label' }, '⏸ paused'), + ); } else { const dueIn = s.next_fire_at_unix - Math.floor(Date.now() / 1000); nextCell.title = new Date(s.next_fire_at_unix * 1000).toISOString(); @@ -723,10 +736,10 @@ function renderScheduleRow(s, agents) { tr.append(td); } - // Actions cell — fire / edit / cancel-all. Glyph-only to fit a - // compact column; the buttons keep their existing colour classes - // so the visual cue (mauve = fire, yellow = edit, red = cancel) - // carries over from the card layout. + // Actions cell — fire / pause-toggle / edit / cancel-all. + // Glyph-only to fit a compact column; colour classes carry the + // visual cue: mauve = fire, teal = pause/resume, yellow = edit, + // red = cancel. const actionsCell = el('td', { class: 'schedules-table-actions' }); if (!cancelled) { const activeTargets = (s.targets || []).filter((t) => !t.cancelled_at_unix); @@ -738,14 +751,28 @@ function renderScheduleRow(s, agents) { fireBtn.title = isOneShot ? 'fire once — one-shot, consumed after the manual fire' : 'fire once now — recurring; choose whether to reset the next-fire timer'; - if (!activeTargets.length) { + if (!activeTargets.length || paused) { fireBtn.disabled = true; - fireBtn.title = 'every target is cancelled — nothing to fire'; + fireBtn.title = paused + ? 'resume the schedule first to use fire-now' + : 'every target is cancelled — nothing to fire'; } fireBtn.addEventListener('click', () => fireScheduleNow(s.id, isOneShot, activeTargets.map((t) => t.target), fireBtn)); actionsCell.append(fireBtn); + // Pause / resume toggle — only meaningful for recurring schedules. + // One-shots can still be paused (to delay a one-time fire), so we + // show the button for both cases. + const pauseBtn = el('button', { + type: 'button', + class: 'btn btn-pause-schedule btn-inline-small' + (paused ? ' btn-resume-schedule' : ''), + }, paused ? '▶' : '⏸'); + pauseBtn.title = paused ? 'resume schedule' : 'pause schedule'; + pauseBtn.addEventListener('click', () => + paused ? resumeSchedule(s.id) : pauseSchedule(s.id)); + actionsCell.append(pauseBtn); + const editingThis = editingSchedules.has(s.id); const editBtn = el('button', { type: 'button', @@ -1089,6 +1116,37 @@ async function postScheduleCancel(id, targets) { } } +async function pauseSchedule(id) { + try { + const resp = await fetch('/api/schedules/' + encodeURIComponent(id) + '/pause', { + method: 'POST', + }); + if (!resp.ok) { + const text = await resp.text().catch(() => ''); + themedToast('pause failed: http ' + resp.status + (text ? '\n\n' + text : ''), { type: 'error' }); + return; + } + await refreshSchedules(); + } catch (err) { + themedToast('pause failed: ' + err, { type: 'error' }); + } +} +async function resumeSchedule(id) { + try { + const resp = await fetch('/api/schedules/' + encodeURIComponent(id) + '/resume', { + method: 'POST', + }); + if (!resp.ok) { + const text = await resp.text().catch(() => ''); + themedToast('resume failed: http ' + resp.status + (text ? '\n\n' + text : ''), { type: 'error' }); + return; + } + await refreshSchedules(); + } catch (err) { + themedToast('resume failed: ' + err, { type: 'error' }); + } +} + export function applySchedulesChanged(ev) { schedulesState = (ev.schedules || []).slice(); renderSchedulesList(); diff --git a/hive-c0re/src/dashboard.rs b/hive-c0re/src/dashboard.rs index c977ec62..693c517a 100644 --- a/hive-c0re/src/dashboard.rs +++ b/hive-c0re/src/dashboard.rs @@ -131,6 +131,14 @@ pub async fn serve(port: u16, coord: Arc) -> Result<()> { "/api/schedules/{id}/cancel", post(schedules::post_schedule_cancel), ) + .route( + "/api/schedules/{id}/pause", + post(schedules::post_schedule_pause), + ) + .route( + "/api/schedules/{id}/resume", + post(schedules::post_schedule_resume), + ) .route( "/api/schedules/{id}/fire-now", post(schedules::post_schedule_fire_now), diff --git a/hive-c0re/src/dashboard/schedules.rs b/hive-c0re/src/dashboard/schedules.rs index ce0525d1..9777e3de 100644 --- a/hive-c0re/src/dashboard/schedules.rs +++ b/hive-c0re/src/dashboard/schedules.rs @@ -228,6 +228,52 @@ pub(super) async fn patch_schedule( } } +/// `POST /api/schedules/{id}/pause` — pause a schedule so the worker +/// skips it until explicitly resumed. Idempotent; no-op on an already- +/// paused row. Returns 404 when the schedule is cancelled or not found. +pub(super) async fn post_schedule_pause( + State(state): State, + AxumPath(id): AxumPath, +) -> Response { + match state.coord.scheduled_prompts.pause(id) { + Ok(()) => { + state.coord.emit_schedules_snapshot(); + (StatusCode::OK, "ok").into_response() + } + Err(e) => { + let msg = format!("{e:#}"); + if msg.contains("not found") || msg.contains("cancelled") { + (StatusCode::NOT_FOUND, msg).into_response() + } else { + error_response(&format!("pause schedule {id}: {msg}")) + } + } + } +} + +/// `POST /api/schedules/{id}/resume` — resume a paused schedule. +/// Idempotent; no-op on an already-active row. Returns 404 when the +/// schedule is cancelled or not found. +pub(super) async fn post_schedule_resume( + State(state): State, + AxumPath(id): AxumPath, +) -> Response { + match state.coord.scheduled_prompts.resume(id) { + Ok(()) => { + state.coord.emit_schedules_snapshot(); + (StatusCode::OK, "ok").into_response() + } + Err(e) => { + let msg = format!("{e:#}"); + if msg.contains("not found") || msg.contains("cancelled") { + (StatusCode::NOT_FOUND, msg).into_response() + } else { + error_response(&format!("resume schedule {id}: {msg}")) + } + } + } +} + /// `POST /api/schedules/{id}/cancel` — operator-side cancel /// (whole schedule when no `targets` field, partial when one is /// provided). Operator bypasses the topology check; the manager diff --git a/hive-c0re/src/scheduled_prompts.rs b/hive-c0re/src/scheduled_prompts.rs index 774bb540..764f8500 100644 --- a/hive-c0re/src/scheduled_prompts.rs +++ b/hive-c0re/src/scheduled_prompts.rs @@ -29,11 +29,12 @@ CREATE TABLE IF NOT EXISTS scheduled_prompts ( created_at_unix INTEGER NOT NULL, source TEXT NOT NULL, cancelled_at_unix INTEGER, - description TEXT + description TEXT, + paused_at_unix INTEGER ); CREATE INDEX IF NOT EXISTS idx_scheduled_due ON scheduled_prompts (next_fire_at_unix) - WHERE cancelled_at_unix IS NULL; + WHERE cancelled_at_unix IS NULL AND paused_at_unix IS NULL; CREATE TABLE IF NOT EXISTS scheduled_prompt_targets ( schedule_id INTEGER NOT NULL, @@ -68,6 +69,9 @@ pub struct Schedule { /// next pass. pub cancelled_at_unix: Option, pub description: Option, + /// Set while the schedule is paused. Worker skips rows where + /// `paused_at_unix IS NOT NULL`. Cleared by `resume()`. + pub paused_at_unix: Option, pub targets: Vec, } @@ -173,6 +177,24 @@ impl ScheduledPrompts { .context("enable foreign keys")?; conn.execute_batch(SCHEMA) .context("apply scheduled_prompts schema")?; + // Migration: add paused_at_unix to existing databases. + // Silently ignores "duplicate column name" errors so this is + // idempotent across daemon restarts on already-migrated DBs. + let _ = conn.execute( + "ALTER TABLE scheduled_prompts ADD COLUMN paused_at_unix INTEGER", + [], + ); + // Migration: recreate the due-rows index to also exclude paused + // rows. `CREATE INDEX IF NOT EXISTS` won't update an existing + // index's WHERE clause, so we drop + recreate on every open. + // The table is small and the op is cheap. + conn.execute_batch( + "DROP INDEX IF EXISTS idx_scheduled_due; + CREATE INDEX idx_scheduled_due + ON scheduled_prompts (next_fire_at_unix) + WHERE cancelled_at_unix IS NULL AND paused_at_unix IS NULL;", + ) + .context("scheduled_prompts: recreate due-index for paused support")?; Ok(Self { conn: Mutex::new(conn), }) @@ -221,7 +243,8 @@ impl ScheduledPrompts { let row = conn .query_row( "SELECT id, owner, body, interval_seconds, next_fire_at_unix, - created_at_unix, source, cancelled_at_unix, description + created_at_unix, source, cancelled_at_unix, description, + paused_at_unix FROM scheduled_prompts WHERE id = ?1", params![id], row_to_schedule_header, @@ -242,7 +265,8 @@ impl ScheduledPrompts { let conn = self.conn.lock().unwrap(); let mut stmt = conn.prepare( "SELECT id, owner, body, interval_seconds, next_fire_at_unix, - created_at_unix, source, cancelled_at_unix, description + created_at_unix, source, cancelled_at_unix, description, + paused_at_unix FROM scheduled_prompts ORDER BY next_fire_at_unix ASC, id ASC", )?; @@ -264,9 +288,11 @@ impl ScheduledPrompts { let conn = self.conn.lock().unwrap(); let mut stmt = conn.prepare( "SELECT id, owner, body, interval_seconds, next_fire_at_unix, - created_at_unix, source, cancelled_at_unix, description + created_at_unix, source, cancelled_at_unix, description, + paused_at_unix FROM scheduled_prompts - WHERE cancelled_at_unix IS NULL AND next_fire_at_unix <= ?1 + WHERE cancelled_at_unix IS NULL AND paused_at_unix IS NULL + AND next_fire_at_unix <= ?1 ORDER BY next_fire_at_unix ASC, id ASC LIMIT ?2", )?; @@ -546,6 +572,43 @@ impl ScheduledPrompts { Ok(()) } + /// Pause a schedule. Idempotent; no-op on an already-paused row. + /// The worker skips paused rows so no fires occur while paused; + /// `next_fire_at_unix` is preserved so the first resume fires at + /// the next intended cadence instant (no catch-up needed — a + /// paused schedule simply slips its upcoming fire). Returns + /// `Err` when the schedule is cancelled or does not exist. + pub fn pause(&self, id: i64) -> Result<()> { + let conn = self.conn.lock().unwrap(); + let now = now_unix(); + let n = conn.execute( + "UPDATE scheduled_prompts + SET paused_at_unix = COALESCE(paused_at_unix, ?1) + WHERE id = ?2 AND cancelled_at_unix IS NULL", + params![now, id], + )?; + if n == 0 { + bail!("schedule {id} not found or is cancelled"); + } + Ok(()) + } + + /// Resume a paused schedule. Idempotent; no-op on an active row. + /// Returns `Err` when the schedule is cancelled or does not exist. + pub fn resume(&self, id: i64) -> Result<()> { + let conn = self.conn.lock().unwrap(); + let n = conn.execute( + "UPDATE scheduled_prompts + SET paused_at_unix = NULL + WHERE id = ?1 AND cancelled_at_unix IS NULL", + params![id], + )?; + if n == 0 { + bail!("schedule {id} not found or is cancelled"); + } + Ok(()) + } + /// Reap cancelled rows older than `older_than_unix`. Returns /// the number of rows deleted. Called from the worker on each /// tick so cancellations clear out of the dashboard without an @@ -575,6 +638,7 @@ fn row_to_schedule_header(row: &rusqlite::Row) -> rusqlite::Result { source: ScheduleSource::from_db_string(&source_str), cancelled_at_unix: row.get(7)?, description: row.get(8)?, + paused_at_unix: row.get(9)?, targets: Vec::new(), }) } diff --git a/hive-c0re/src/socket_server.rs b/hive-c0re/src/socket_server.rs index 75bcf0be..3801da7a 100644 --- a/hive-c0re/src/socket_server.rs +++ b/hive-c0re/src/socket_server.rs @@ -2021,6 +2021,7 @@ fn schedule_to_wire(s: crate::scheduled_prompts::Schedule) -> hive_sh4re::WireSc } }, cancelled_at_unix: s.cancelled_at_unix, + paused_at_unix: s.paused_at_unix, description: s.description, targets: s .targets @@ -2126,6 +2127,7 @@ mod tests { created_at_unix: 0, source: hive_sh4re::WireScheduleSource::Operator, cancelled_at_unix: None, + paused_at_unix: None, description: None, targets: targets.iter().map(|t| target(t)).collect(), } diff --git a/hive-sh4re/src/lib.rs b/hive-sh4re/src/lib.rs index 80d44bc8..ebe397ef 100644 --- a/hive-sh4re/src/lib.rs +++ b/hive-sh4re/src/lib.rs @@ -1395,6 +1395,12 @@ pub struct WireSchedule { pub source: WireScheduleSource, #[serde(default, skip_serializing_if = "Option::is_none")] pub cancelled_at_unix: Option, + /// Set while the schedule is paused. Worker skips paused rows; + /// they keep their `next_fire_at_unix` so resuming at any time + /// fires at the next intended instant (no catch-up clamp needed + /// — a paused schedule simply slips its next fire). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub paused_at_unix: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, pub targets: Vec, From dbd4b7a15cf3d7a4f9f249fc8b0daf41915bf138 Mon Sep 17 00:00:00 2001 From: iris Date: Sat, 27 Jun 2026 13:57:50 +0200 Subject: [PATCH 4/9] fix(schedules): use typed error for pause/resume 404 discrimination Replace brittle msg.contains("not found") string matching in post_schedule_pause / post_schedule_resume with a typed ScheduleNotFoundOrCancelled error that handlers downcast on directly. pause() and resume() now return Err(ScheduleNotFoundOrCancelled(id).into()) instead of bail!("schedule {id} not found or is cancelled"); handlers call e.downcast_ref::().is_some() for the 404 branch, making the discrimination stable even if the error message wording changes. --- hive-c0re/src/dashboard/schedules.rs | 16 ++++++++-------- hive-c0re/src/scheduled_prompts.rs | 24 ++++++++++++++++++++---- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/hive-c0re/src/dashboard/schedules.rs b/hive-c0re/src/dashboard/schedules.rs index 9777e3de..967146f2 100644 --- a/hive-c0re/src/dashboard/schedules.rs +++ b/hive-c0re/src/dashboard/schedules.rs @@ -13,6 +13,8 @@ use axum::{ use problem_details::ProblemDetails; +use crate::scheduled_prompts::ScheduleNotFoundOrCancelled; + use super::{AppState, error_problem, error_response}; /// `GET /api/schedules` — snapshot of every schedule for the @@ -241,11 +243,10 @@ pub(super) async fn post_schedule_pause( (StatusCode::OK, "ok").into_response() } Err(e) => { - let msg = format!("{e:#}"); - if msg.contains("not found") || msg.contains("cancelled") { - (StatusCode::NOT_FOUND, msg).into_response() + if e.downcast_ref::().is_some() { + (StatusCode::NOT_FOUND, format!("{e}")).into_response() } else { - error_response(&format!("pause schedule {id}: {msg}")) + error_response(&format!("pause schedule {id}: {e:#}")) } } } @@ -264,11 +265,10 @@ pub(super) async fn post_schedule_resume( (StatusCode::OK, "ok").into_response() } Err(e) => { - let msg = format!("{e:#}"); - if msg.contains("not found") || msg.contains("cancelled") { - (StatusCode::NOT_FOUND, msg).into_response() + if e.downcast_ref::().is_some() { + (StatusCode::NOT_FOUND, format!("{e}")).into_response() } else { - error_response(&format!("resume schedule {id}: {msg}")) + error_response(&format!("resume schedule {id}: {e:#}")) } } } diff --git a/hive-c0re/src/scheduled_prompts.rs b/hive-c0re/src/scheduled_prompts.rs index 764f8500..9095e7b5 100644 --- a/hive-c0re/src/scheduled_prompts.rs +++ b/hive-c0re/src/scheduled_prompts.rs @@ -19,6 +19,20 @@ use anyhow::{Context, Result, bail}; use rusqlite::{Connection, OptionalExtension, params}; use serde::{Deserialize, Serialize}; +/// Typed error returned by [`ScheduledPrompts::pause`] and +/// [`ScheduledPrompts::resume`] when the target row does not exist or +/// is already cancelled. Handlers downcast on this type to emit 404 +/// rather than 500, avoiding brittle string-matching on the message. +#[derive(Debug)] +pub struct ScheduleNotFoundOrCancelled(pub i64); + +impl std::fmt::Display for ScheduleNotFoundOrCancelled { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "schedule {} not found or is cancelled", self.0) + } +} +impl std::error::Error for ScheduleNotFoundOrCancelled {} + const SCHEMA: &str = r" CREATE TABLE IF NOT EXISTS scheduled_prompts ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -577,7 +591,8 @@ impl ScheduledPrompts { /// `next_fire_at_unix` is preserved so the first resume fires at /// the next intended cadence instant (no catch-up needed — a /// paused schedule simply slips its upcoming fire). Returns - /// `Err` when the schedule is cancelled or does not exist. + /// `Err(ScheduleNotFoundOrCancelled)` when the schedule is + /// cancelled or does not exist (handlers downcast to emit 404). pub fn pause(&self, id: i64) -> Result<()> { let conn = self.conn.lock().unwrap(); let now = now_unix(); @@ -588,13 +603,14 @@ impl ScheduledPrompts { params![now, id], )?; if n == 0 { - bail!("schedule {id} not found or is cancelled"); + return Err(ScheduleNotFoundOrCancelled(id).into()); } Ok(()) } /// Resume a paused schedule. Idempotent; no-op on an active row. - /// Returns `Err` when the schedule is cancelled or does not exist. + /// Returns `Err(ScheduleNotFoundOrCancelled)` when the schedule is + /// cancelled or does not exist (handlers downcast to emit 404). pub fn resume(&self, id: i64) -> Result<()> { let conn = self.conn.lock().unwrap(); let n = conn.execute( @@ -604,7 +620,7 @@ impl ScheduledPrompts { params![id], )?; if n == 0 { - bail!("schedule {id} not found or is cancelled"); + return Err(ScheduleNotFoundOrCancelled(id).into()); } Ok(()) } From 42823a0b22778abbd85e5ccdf126105ea208d371 Mon Sep 17 00:00:00 2001 From: damocles Date: Sat, 27 Jun 2026 14:00:46 +0200 Subject: [PATCH 5/9] docs(#2012): add first-run setup guide + link from index --- CLAUDE.md | 2 + docs/setup.md | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 docs/setup.md diff --git a/CLAUDE.md b/CLAUDE.md index 21c14c64..b200aacb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,6 +68,8 @@ hand-maintained per-file tree drifts out of sync with the code. Pick the doc that matches your task. None depend on the others — read them à la carte. +- **"How do I bring a fresh hive online (first-run hivectl + bootstrap)?"** → [`docs/setup.md`](docs/setup.md). - **"What does the dashboard look like?"** → [`docs/web-ui.md`](docs/web-ui.md) (index; sub-pages: [`shape`](docs/web-ui/shape.md), diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 00000000..c288d99b --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,105 @@ +# First-run setup (fresh-deploy bootstrap) + +How to bring a fresh hyperhive hive online: provision accounts, open +the gateway, make matrix reachable, and spawn the first sub-agents. + +Aimed at `ruth` (the root/manager agent) on a fresh deploy, but it's a +plain reference doc — read it whenever you need the bootstrap command +sequence. All `hivectl` commands below run as **root on the host** (not +inside an agent container); the `request_*` steps run from ruth's own +turn via the MCP tools. + +## Step-by-step + +### 1 · Forge + +```bash +# Provision (or refresh) ruth's own forge account — do this first +hivectl forge create-user ruth + +# Create a human operator account (prints the token to stdout) +hivectl forge create-user mara --password hunter2 + +# Provision forge accounts for any sub-agents spawned later +hivectl forge create-user +``` + +### 2 · Gateway (HTTP Basic auth) + +```bash +# Add an operator login to the gateway (reads password from stdin) +echo "hunter2" | hivectl gateway create-user mara --password-stdin + +# List existing users +hivectl gateway list-users +``` + +### 3 · Matrix + +```bash +# 3a. Ensure the hive-internal admin account exists first +hivectl matrix sync-admin + +# 3b. Provision ruth's own matrix account +hivectl matrix create-user ruth + +# 3c. Create a human matrix account +hivectl matrix create-user mara --password hunter2 + +# 3d. Invite the operator to the hive Space (and optionally to rooms) +hivectl matrix invite mara +hivectl matrix invite @mara:yourserver --room '#hive-chat:yourserver' + +# 3e. Promote the operator to homeserver admin if needed +hivectl matrix promote-user mara +``` + +### 4 · Spawn sub-agents + +Sub-agent creation goes through the approval queue — ruth proposes, the +operator approves, the container builds. From ruth's own turn (inside +the container, via MCP tools): + +``` +# Step 1: initialise a new agent's config repo +request_init_config(name: "iris") +# → operator approves → config_ready event lands in the inbox + +# Step 2: edit /agents/iris/config/agent.nix, commit it, then: +request_apply_commit(agent: "iris", commit_ref: "") +# → operator approves → container built + started +``` + +See [`approvals.md`](approvals.md) for the full two-step flow. + +### 5 · Useful host commands + +```bash +# Roster: all agents, status, rev, parent, pending reminders +hivectl agents list + +# Restart a stuck container (no rebuild) +hivectl agents restart + +# Open a Claude session inside an agent's container +hivectl choom + +# Open hive web surfaces in a browser (or just print the URLs) +hivectl open # operator dashboard +hivectl open forge # Forgejo +hivectl open matrix # Matrix GUI (fluffychat) +``` + +See [`tools/hivectl.md`](tools/hivectl.md) for every `hivectl` verb. + +## Security notes + +- **No forge admin token is stored in any agent state dir.** Agents + hold a regular agent token in their `forge-token` file; sensitive + creds (the core token, the matrix admin token) live on the host. +- All config changes (`request_apply_commit`) go through operator + approval — agents can't unilaterally rebuild containers, by design. + See [`boundary.md`](boundary.md) and [`security.md`](security.md). + +Once the hive is running, ruth records anything it needs to remember +across restarts in `/agents/ruth/state/notes.md`. From 141764c6eb5178ffd7b93d312f26b8d46ab99c15 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 27 Jun 2026 15:05:54 +0200 Subject: [PATCH 6/9] feat(#2023): ship OTEL via managed claude settings json, drop the wrapper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per mara: configure OTEL in the generated claude settings json (what the Claude Code docs suggest), not a launch wrapper or /etc shell file. claude-code auto-discovers /etc/claude-code/managed-settings.json in every context — the harness turn loop AND hivectl choom — so putting the OTEL env there gives telemetry parity declaratively, with no wrapper and no --settings plumbing. - managed-settings.json: was a static shared .source; now, when OTEL is enabled, a per-agent build-time jq merge of the base asset + an env block (jq at build, not eval-time readFile, to avoid IFD). OTEL off = the static asset verbatim. - otelSettingsEnv carries the static OTEL knobs + OTEL_RESOURCE_ATTRIBUTES with the agent name (build-time) and the hive/swarm names forwarded by meta.rs into environment.variables (mara: forward host config into agent config where needed). - removed the hive-serve-otel ExecStart wrapper, the per-unit otelEnv, and the otel-headers LoadCredential from the harness service — the harness binary emits no OTEL itself; only claude does, and it now reads the settings json directly. Known follow-ups (noted in code): the auth header (otel.headersCredential, opt-in/default-null) is a secret and can't live in the world-readable settings file — authenticated collectors need a runtime mechanism; this PR covers the unauthenticated default. nix fmt clean. --- nix/templates/harness-base.nix | 121 ++++++++++++++++----------------- 1 file changed, 60 insertions(+), 61 deletions(-) diff --git a/nix/templates/harness-base.nix b/nix/templates/harness-base.nix index c081d777..13a2f415 100644 --- a/nix/templates/harness-base.nix +++ b/nix/templates/harness-base.nix @@ -17,6 +17,40 @@ let # from `userName` to keep them coupled. userName = config.hyperhive.user.name; homeDir = "/home/${userName}"; + # Hive-wide OpenTelemetry config (host-driven; baked in per-agent by + # meta.rs `otel_config`). + otelCfg = config.hyperhive.otel; + # Hive/swarm display names are forwarded into each agent's build by + # meta.rs as `environment.variables` (per-agent, build-time strings), + # so they can be baked into the resource attributes below without a + # runtime shell. Absent (option unset) → "unknown". + hiveDisplayName = config.environment.variables.HYPERHIVE_HIVE_NAME or "unknown"; + swarmDisplayName = config.environment.variables.HYPERHIVE_SWARM_NAME or "unknown"; + # OTEL environment Claude Code reads to export metrics/logs/traces. + # Shipped via the managed claude settings json (below), which claude + # auto-discovers for BOTH the harness turn-loop and `hivectl choom` — + # so telemetry parity is declarative, with no launch wrapper. The + # auth header (`otel.headersCredential`) is deliberately NOT included: + # it's a secret and the settings file is world-readable; authenticated + # collectors need a runtime mechanism (tracked as a follow-up). + otelSettingsEnv = { + CLAUDE_CODE_ENABLE_TELEMETRY = "1"; + OTEL_METRICS_EXPORTER = "otlp"; + OTEL_LOGS_EXPORTER = "otlp"; + OTEL_TRACES_EXPORTER = "otlp"; + OTEL_EXPORTER_OTLP_PROTOCOL = otelCfg.protocol; + OTEL_EXPORTER_OTLP_ENDPOINT = otelCfg.endpoint; + # Force CUMULATIVE temporality — Claude Code defaults to DELTA, + # which Prometheus/Mimir-family backends (incl. grafana-lgtm) + # silently drop without a deltatocumulative processor. + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; + OTEL_RESOURCE_ATTRIBUTES = + "service.name=hyperhive-agent,agent=${userName},hive=${hiveDisplayName},swarm=${swarmDisplayName}" + + lib.optionalString (otelCfg.extraResourceAttributes != "") ",${otelCfg.extraResourceAttributes}"; + } + // lib.optionalAttrs (otelCfg.metricIntervalMs != null) { + OTEL_METRIC_EXPORT_INTERVAL = toString otelCfg.metricIntervalMs; + }; # Single source of truth for the default matrix homeserver URL, shared # by the `hyperhive.matrix.url` option default and the daemon-unit guard # that decides whether to set a unit-level HIVE_MATRIX_URL (so the two @@ -1073,8 +1107,25 @@ in # deliberately NOT shipped here: effort is controlled live via the # `--effort` CLI flag (HIVE_DEFAULT_EFFORT / the per-agent UI slider), # which managed scope would otherwise override and lock. + # Base hive-enforced settings, plus — when OTEL is enabled — an `env` + # block so Claude Code exports telemetry natively from the settings + # file it reads in every context (harness turn-loop AND `hivectl + # choom`), with no launch wrapper. With OTEL off it's the shared + # static asset verbatim; with OTEL on it's a per-agent merge (the + # resource attributes carry the agent name) done at BUILD time via + # `jq` — not eval-time `readFile`, which would be import-from- + # derivation. environment.etc."claude-code/managed-settings.json".source = - "${pkgs.hyperhive-assets}/share/hyperhive/prompts/claude-settings.json"; + let + baseSettings = "${pkgs.hyperhive-assets}/share/hyperhive/prompts/claude-settings.json"; + in + if !otelCfg.enable then + baseSettings + else + pkgs.runCommand "managed-settings.json" { nativeBuildInputs = [ pkgs.jq ]; } '' + jq --argjson env ${lib.escapeShellArg (builtins.toJSON otelSettingsEnv)} \ + '. + { env: $env }' ${baseSettings} > "$out" + ''; # Merged frontend static tree. Base = `${frontend.dist}/agent/`, # then each `extraFiles` entry is laid on top at its `target` @@ -1707,49 +1758,11 @@ in systemd.services.hive-ag3nt = let binary = "hive"; - otel = config.hyperhive.otel; - # Claude Code's native OpenTelemetry is env-driven; the harness - # spawns `claude` as a child which inherits this unit's env, so - # setting these here is all it takes to export per-agent stats. - otelEnv = lib.optionalAttrs otel.enable ( - { - CLAUDE_CODE_ENABLE_TELEMETRY = "1"; - OTEL_METRICS_EXPORTER = "otlp"; - OTEL_LOGS_EXPORTER = "otlp"; - # Route traces to OTLP too so any spans Claude Code emits land - # at the configured collector rather than a default exporter. - OTEL_TRACES_EXPORTER = "otlp"; - OTEL_EXPORTER_OTLP_PROTOCOL = otel.protocol; - OTEL_EXPORTER_OTLP_ENDPOINT = otel.endpoint; - # Force CUMULATIVE metric temporality. Claude Code defaults to - # DELTA, which Prometheus/Mimir-family backends (the common case, - # incl. grafana-lgtm) silently drop unless a deltatocumulative - # processor is wired — so delta = "logs arrive, metrics vanish". - # Cumulative is what those backends ingest natively. Verified: - # a delta export never registers the metric name; cumulative does. - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; - } - // lib.optionalAttrs (otel.metricIntervalMs != null) { - OTEL_METRIC_EXPORT_INTERVAL = toString otel.metricIntervalMs; - } - ); - # When OTEL is on, wrap the harness launch so (1) the bearer-token - # header is read from the systemd credential at start (never in the - # nix store or argv) and (2) the resource attributes are assembled - # from this agent's name (known at build time) plus the hive/swarm - # names (inherited HYPERHIVE_HIVE_NAME / HYPERHIVE_SWARM_NAME env, - # the same vars `identity::hive_name`/`swarm_name` read at runtime). - otelExecStart = pkgs.writeShellScript "hive-serve-otel" '' - set -eu - if [ -n "''${CREDENTIALS_DIRECTORY:-}" ] && [ -r "$CREDENTIALS_DIRECTORY/otel-headers" ]; then - OTEL_EXPORTER_OTLP_HEADERS="$(cat "$CREDENTIALS_DIRECTORY/otel-headers")" - export OTEL_EXPORTER_OTLP_HEADERS - fi - export OTEL_RESOURCE_ATTRIBUTES="service.name=hyperhive-agent,agent=${userName},hive=''${HYPERHIVE_HIVE_NAME:-unknown},swarm=''${HYPERHIVE_SWARM_NAME:-unknown}${ - lib.optionalString (otel.extraResourceAttributes != "") ",${otel.extraResourceAttributes}" - }" - exec ${pkgs.hyperhive}/bin/${binary} serve - ''; + # OTEL is shipped declaratively via the managed claude settings + # json (`environment.etc."claude-code/managed-settings.json"`, + # `otelSettingsEnv` in the top-level let) — claude reads it for + # both the harness turn-loop and `hivectl choom`, so there's no + # launch wrapper or per-unit OTEL env here anymore. in { description = "${binary} harness"; @@ -1773,7 +1786,6 @@ in # bind-mounts and gateway upstream config stay in sync. HIVE_WEB_SOCKET = "/run/hive-agent/${userName}/web.sock"; } - // otelEnv // lib.optionalAttrs config.hyperhive.gui.enable { # Tells the harness which fixed VNC port weston bound, and (by # its presence) that gui is enabled — the harness `/screen/ws` @@ -1784,13 +1796,9 @@ in HIVE_GUI_VNC_PORT = toString config.hyperhive.gui.vncPort; }; serviceConfig = { - ExecStart = if otel.enable then "${otelExecStart}" else "${pkgs.hyperhive}/bin/${binary} serve"; - # Pin the journal identity to the binary name. Without this, - # systemd derives SyslogIdentifier from the ExecStart basename — - # which under OTEL is the wrapper script's store path - # (`-hive-serve-otel`), so every agent's harness logs showed - # that opaque name instead of `hive`. Set explicitly so the - # identity is stable across the otel / non-otel ExecStart branches. + ExecStart = "${pkgs.hyperhive}/bin/${binary} serve"; + # Pin the journal identity to the binary name (otherwise systemd + # derives SyslogIdentifier from the ExecStart basename). SyslogIdentifier = binary; Restart = "on-failure"; RestartSec = 2; @@ -1801,15 +1809,6 @@ in RuntimeDirectory = "hive-config"; User = userName; Group = userName; - } - // lib.optionalAttrs (otel.enable && otel.headersCredential != null) { - # Inherit form (no `:path`): hive-c0re forwards the host file at - # `headersCredential` into this container's credential store via - # nspawn `--load-credential=otel-headers:` (see - # lifecycle.rs::hive_load_credentials). The path isn't reachable - # from inside the container, so we inherit the already-loaded - # credential by name rather than re-reading the host path here. - LoadCredential = [ "otel-headers" ]; }; }; From b20dd3218922075a181438b94ae6278b8e66c96f Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 27 Jun 2026 16:05:03 +0200 Subject: [PATCH 7/9] docs(#2023): mark otel.headersCredential as not-yet-wired (argus review) The option description still claimed the credential is loaded via systemd LoadCredential, but this PR removed that path. Clarify that the option is currently inert (only the unauthenticated OTEL export is implemented) and that runtime header injection is a planned follow-up, so configuring it doesn't silently no-op without explanation. --- nix/templates/harness-base.nix | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nix/templates/harness-base.nix b/nix/templates/harness-base.nix index 13a2f415..ba88db05 100644 --- a/nix/templates/harness-base.nix +++ b/nix/templates/harness-base.nix @@ -264,12 +264,18 @@ in internal = true; description = '' Absolute path to an operator-provided secret file whose contents - become `OTEL_EXPORTER_OTLP_HEADERS` (e.g. - `Authorization=Bearer `). Loaded via systemd - `LoadCredential` into the unit-private credential store at - runtime, so the token is never copied into the nix store or - exposed in the process argv. Host-driven via + would become `OTEL_EXPORTER_OTLP_HEADERS` (e.g. + `Authorization=Bearer `). Host-driven via `services.hyperhive.otel.headersCredential`. + + **Not yet wired up.** OTEL config now ships through the managed + claude settings json (`/etc/claude-code/managed-settings.json`), + which is world-readable, so a secret auth header can't be baked + into it. Setting this option currently has no effect — the + unauthenticated export path is the only one implemented. A + follow-up will inject the header at runtime (e.g. the harness + writing it into the agent's `0600` `~/.claude/settings.json`), + keeping it out of the nix store and the world-readable file. ''; }; From cc962d0685954f5b679bc1236ec1551fb80e5dc5 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 27 Jun 2026 17:46:37 +0200 Subject: [PATCH 8/9] feat(#2023): inject OTEL auth header at runtime, never in the nix store (mara: b) Per mara: a secret in the nix store is not acceptable. The non-secret OTEL config (telemetry-enable, endpoint, protocol, resource attributes) stays in the world-readable managed settings json; the auth header is handled separately at runtime so it never touches the store. New hive-otel-header oneshot (only when otel.enable && headersCredential is set): inherits the forwarded otel-headers systemd credential via LoadCredential, reads it at start, and merges OTEL_EXPORTER_OTLP_HEADERS into the agent's 0600 ~/.claude/settings.json env block via jq. claude layers the user env on top of the managed settings, so both the harness turn-loop and hivectl choom (same agent user) export with auth. The token is read from disk at start and never copied into the nix store or the world-readable managed file. Ordering is best-effort (before=, not a hard dep): a failure leaves the harness running and telemetry exporting unauthenticated. headersCredential option description updated to reflect it's now wired. nix fmt clean. --- nix/templates/harness-base.nix | 71 ++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/nix/templates/harness-base.nix b/nix/templates/harness-base.nix index ba88db05..b355e20a 100644 --- a/nix/templates/harness-base.nix +++ b/nix/templates/harness-base.nix @@ -30,9 +30,13 @@ let # Shipped via the managed claude settings json (below), which claude # auto-discovers for BOTH the harness turn-loop and `hivectl choom` — # so telemetry parity is declarative, with no launch wrapper. The - # auth header (`otel.headersCredential`) is deliberately NOT included: - # it's a secret and the settings file is world-readable; authenticated - # collectors need a runtime mechanism (tracked as a follow-up). + # auth header (`otel.headersCredential`) is deliberately NOT included + # here: it's a secret and this file lives in the world-readable nix + # store. It's injected at *runtime* into the agent's `0600` + # `~/.claude/settings.json` by the `hive-otel-header` oneshot below + # (claude merges the `env` from the user settings on top of these + # managed ones), so the token is read from disk at start and never + # touches the store. otelSettingsEnv = { CLAUDE_CODE_ENABLE_TELEMETRY = "1"; OTEL_METRICS_EXPORTER = "otlp"; @@ -264,18 +268,18 @@ in internal = true; description = '' Absolute path to an operator-provided secret file whose contents - would become `OTEL_EXPORTER_OTLP_HEADERS` (e.g. + become `OTEL_EXPORTER_OTLP_HEADERS` (e.g. `Authorization=Bearer `). Host-driven via `services.hyperhive.otel.headersCredential`. - **Not yet wired up.** OTEL config now ships through the managed - claude settings json (`/etc/claude-code/managed-settings.json`), - which is world-readable, so a secret auth header can't be baked - into it. Setting this option currently has no effect — the - unauthenticated export path is the only one implemented. A - follow-up will inject the header at runtime (e.g. the harness - writing it into the agent's `0600` `~/.claude/settings.json`), - keeping it out of the nix store and the world-readable file. + The rest of the OTEL config ships in the world-readable managed + claude settings json, but the header is a secret, so it's handled + separately: hive-c0re forwards this file into the container's + systemd credential store, and the `hive-otel-header` oneshot + reads it at runtime (`LoadCredential`) and writes it into the + agent's `0600` `~/.claude/settings.json` `env` block. The token + is read from disk at start and never copied into the nix store or + the world-readable settings file. ''; }; @@ -1133,6 +1137,49 @@ in '. + { env: $env }' ${baseSettings} > "$out" ''; + # Inject the OTEL auth header (a secret) into the agent's *user* + # claude settings at runtime, keeping it out of the world-readable + # managed settings json above and out of the nix store entirely. + # hive-c0re forwards the operator's `headersCredential` file into + # this container's systemd credential store; this oneshot reads it + # via `LoadCredential` at start and merges `OTEL_EXPORTER_OTLP_HEADERS` + # into `~/.claude/settings.json` (0600, agent-owned). claude layers + # the user `env` on top of the managed one, so both the harness + # turn-loop and `hivectl choom` (same agent user) pick it up. Ordering + # is best-effort (`before`, not a hard dep): if it fails the harness + # still starts and telemetry just exports unauthenticated. + systemd.services.hive-otel-header = + lib.mkIf (config.hyperhive.otel.enable && config.hyperhive.otel.headersCredential != null) + { + description = "Inject the OTEL auth header into the agent's claude user settings"; + wantedBy = [ "multi-user.target" ]; + before = [ "hive-ag3nt.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = userName; + Group = userName; + LoadCredential = [ "otel-headers" ]; + ExecStart = pkgs.writeShellScript "hive-otel-header" '' + set -eu + umask 077 + hdr="$CREDENTIALS_DIRECTORY/otel-headers" + [ -r "$hdr" ] || exit 0 + dir=${homeDir}/.claude + settings="$dir/settings.json" + mkdir -p "$dir" + base='{}' + [ -s "$settings" ] && base="$(cat "$settings")" + printf '%s' "$base" | ${pkgs.jq}/bin/jq \ + --rawfile h "$hdr" \ + '.env = ((.env // {}) + { OTEL_EXPORTER_OTLP_HEADERS: ($h | rtrimstr("\n")) })' \ + > "$settings.tmp" + mv "$settings.tmp" "$settings" + chmod 0600 "$settings" + ''; + }; + }; + # Merged frontend static tree. Base = `${frontend.dist}/agent/`, # then each `extraFiles` entry is laid on top at its `target` # path. The runCommand derivation aborts on overwrite so a From c9115bdbf54684d82569c08479e7785d31a307af Mon Sep 17 00:00:00 2001 From: damocles Date: Sat, 27 Jun 2026 14:57:01 +0200 Subject: [PATCH 9/9] fix(#2019): pin harness claude session via --resume so choom can't clobber it --- hive-ag3nt/src/turn.rs | 90 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 11 deletions(-) diff --git a/hive-ag3nt/src/turn.rs b/hive-ag3nt/src/turn.rs index 8782c4dd..be80156b 100644 --- a/hive-ag3nt/src/turn.rs +++ b/hive-ag3nt/src/turn.rs @@ -63,6 +63,19 @@ const AUTH_FAIL_MARKERS: &[&str] = &[ "Failed to authenticate. API Error: 401", ]; +/// Substring claude-code emits when `--resume ` is handed a session id +/// that doesn't exist in this cwd's project (stale persisted id, or a crash +/// before the first turn ever completed a `.jsonl`). On a hit we clear the +/// persisted id so the NEXT turn starts a fresh session and re-captures — +/// the agent self-heals instead of failing `--resume` forever. +const SESSION_NOT_FOUND_MARKER: &str = "No conversation found with session ID"; + +/// Name of the harness-owned file under `paths::harness_dir()` that holds +/// the claude session id to resume. Written after every turn with the id +/// claude reported on its stream (the id can change across resume turns in +/// some claude-code versions, so we always rewrite with the last-seen value). +const CLAUDE_SESSION_ID_FILE: &str = "claude-session-id"; + /// How long to sleep after detecting a rate-limit before re-entering the /// serve loop. Overridable via `HIVE_RATE_LIMIT_SLEEP_SECS`. Default is /// 5 minutes — enough for most short-lived throttles; the operator can @@ -552,8 +565,11 @@ fn session_refreshed(prev: DirSnapshot, now: DirSnapshot) -> bool { /// Spawn `claude` for one turn and pump `stream-json` stdout into the /// live event bus. Prompt goes over stdin (variadic /// `--allowedTools`/`--tools` would otherwise eat a trailing positional -/// prompt). The session is persistent across turns via `--continue` and -/// claude's in-session auto-compact is disabled via the managed +/// prompt). The session is persistent across turns via `--resume ` +/// against the harness's own captured session id (NOT bare `--continue`, +/// which resumes the *latest* session in this cwd and so lets a `choom` +/// session hijack the live harness context). claude's in-session +/// auto-compact is disabled via the managed /// settings at `/etc/claude-code/managed-settings.json` so it doesn't /// stall mid-turn — hyperhive owns compaction. pub async fn run_turn(prompt: &str, files: &TurnFiles, bus: &Bus) -> TurnOutcome { @@ -624,16 +640,32 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, const STDERR_TAIL_LINES: usize = 20; let model = bus.model(); let effort = bus.effort(); - let resume = !bus.take_skip_continue(); - if !resume { - // Flag the fresh session so the bin loop mints a new `sessions` - // row + stamps its id onto this turn's stats (and subsequent - // turns until the next fresh start). + // Resolve which claude session to resume. We NEVER pass bare + // `--continue`: that resumes the latest session in this cwd, which a + // `choom` invocation (same cwd) can hijack, wiping the harness context. + // Instead we `--resume ` against the id claude reported on a prior + // turn, persisted under `harness_dir()/claude-session-id`. + let persist_path = crate::paths::harness_dir().join(CLAUDE_SESSION_ID_FILE); + let resume_id: Option = if bus.take_skip_continue() { + // Fresh session requested: mint a new one (no --resume). Flag it so + // the bin loop mints a new `sessions` row + stamps its id onto this + // turn's stats. Drop any stale persisted id — the new id claude + // reports this turn is captured + written below. bus.mark_fresh_session(); + let _ = std::fs::remove_file(&persist_path); bus.emit(LiveEvent::Note { - text: "fresh session (--continue suppressed for this turn)".into(), + text: "fresh session (continue suppressed for this turn)".into(), }); - } + None + } else { + // Continue: resume OUR captured id. Absent (first turn / just + // self-healed from a stale id) → fall through to a fresh session + // and capture the new id below. + match std::fs::read_to_string(&persist_path) { + Ok(s) if !s.trim().is_empty() => Some(s.trim().to_string()), + _ => None, + } + }; let mut cmd = Command::new("claude"); // Spawn inside the agent's state dir so relative paths in tool calls // (Read foo.md, Bash ls, Write notes.md) land in the durable dir @@ -651,8 +683,8 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, .arg(&model) .arg("--effort") .arg(&effort); - if resume { - cmd.arg("--continue"); + if let Some(id) = &resume_id { + cmd.arg("--resume").arg(id); } cmd.arg("--system-prompt-file").arg(&files.system_prompt); cmd.arg("--mcp-config") @@ -679,12 +711,21 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, let prompt_too_long = Arc::new(AtomicBool::new(false)); let rate_limited = Arc::new(AtomicBool::new(false)); let auth_failed = Arc::new(AtomicBool::new(false)); + // `--resume` against a stale/missing id: clear the persist file so the + // next turn self-heals into a fresh session. + let session_not_found = Arc::new(AtomicBool::new(false)); + // Last `session_id` claude reported on its stream this turn; persisted + // after the child exits so the next turn `--resume`s it. + let session_id_seen = Arc::new(Mutex::new(None::)); let flag_out = prompt_too_long.clone(); let flag_err = prompt_too_long.clone(); let rate_out = rate_limited.clone(); let rate_err = rate_limited.clone(); let auth_out = auth_failed.clone(); let auth_err = auth_failed.clone(); + let notfound_out = session_not_found.clone(); + let notfound_err = session_not_found.clone(); + let session_id_out = session_id_seen.clone(); let bus_out = bus.clone(); let bus_err = bus.clone(); let pump_stdout = tokio::spawn(async move { @@ -712,7 +753,19 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, if AUTH_FAIL_MARKERS.iter().any(|m| line.contains(m)) { auth_out.store(true, Ordering::Relaxed); } + if line.contains(SESSION_NOT_FOUND_MARKER) { + notfound_out.store(true, Ordering::Relaxed); + } if let Ok(v) = serde_json::from_str::(&line) { + // Track the session id claude reports (init + result events + // both carry it). Persisted after exit so the next turn + // `--resume`s it; re-captured each turn since the id can + // change across resumes in some claude-code versions. + if let Some(sid) = v.get("session_id").and_then(|s| s.as_str()) + && !sid.is_empty() + { + *session_id_out.lock().unwrap() = Some(sid.to_string()); + } // Rate-limit detection: only fire on JSON `error` events, // not on arbitrary text content. An agent discussing a past // rate limit in its response would otherwise trigger a false @@ -779,6 +832,9 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, if AUTH_FAIL_MARKERS.iter().any(|m| line.contains(m)) { auth_err.store(true, Ordering::Relaxed); } + if line.contains(SESSION_NOT_FOUND_MARKER) { + notfound_err.store(true, Ordering::Relaxed); + } // Mirror to journald so post-mortems work without the web UI // or the events sqlite. The bus event is what the dashboard // renders; the tracing line is what `journalctl -M -b` @@ -801,6 +857,18 @@ async fn run_claude(prompt: &str, files: &TurnFiles, bus: &Bus) -> Result<(bool, let too_long = prompt_too_long.load(Ordering::Relaxed); let is_rate_limited = rate_limited.load(Ordering::Relaxed); let is_auth_failed = auth_failed.load(Ordering::Relaxed); + // Session-id bookkeeping. On a stale/missing `--resume` id, drop the + // persist file so the next turn starts fresh and self-heals. Otherwise + // rewrite it with the id claude reported this turn (handles the id + // changing across resumes in some claude-code versions). + if session_not_found.load(Ordering::Relaxed) { + let _ = std::fs::remove_file(&persist_path); + } else if let Some(sid) = session_id_seen.lock().unwrap().clone() { + if let Some(parent) = persist_path.parent() { + let _ = std::fs::create_dir_all(parent); + } + let _ = std::fs::write(&persist_path, sid); + } if !status.success() && !too_long && !is_rate_limited && !is_auth_failed { let tail = stderr_tail.lock().unwrap(); if tail.is_empty() {