From 4573865745a45ef35d732899df55cf4151c15717 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 18 Aug 2026 10:07:49 +0200 Subject: [PATCH] feat(#3255): register the swarm-wide forge hooks against the controller The endpoint landed inert: nothing pointed at it, so the only way to see it work was to mint an HMAC by hand. Register the two swarm-wide hooks at startup so a real forge event produces a journal line. Registered ALONGSIDE the per-hive hooks, not instead of them. Every hive keeps receiving and acting on its own deliveries; the controller gets a copy and logs it. Moving the registration is a later step and has to be: fan-out swarm->hive does not exist yet, so a hook moved now would point at a receiver that forwards nowhere, silently on both sides. Deliberately no stale-hook deletion arm, unlike the two per-hive registrars this otherwise mirrors: theirs delete hooks matching their own path with a foreign base, and the hives' hooks are not stale. The route prefix is what keeps this safe. Both hive-side registrars delete any hook ending in /webhook/knowledge or /webhook/config-pr with a different base, so a swarm hook under those paths would be deleted by every hive on every boot. Serving them under /webhook/forge/ avoids it, and a test pins it -- there is nothing else that can. SWARM_CONTROLLER_PUBLIC_URL is set only where the swarm vhost is served, because a hook whose target_url nothing answers is worse than no hook. --- docs/knowledge.md | 4 + docs/swarm/README.md | 24 +++++ nix/host-modules/swarm-controller.nix | 15 +++ swarm-controller/src/forge.rs | 150 +++++++++++++++++++++++++- swarm-controller/src/main.rs | 45 +++++++- swarm-controller/src/webhook.rs | 100 +++++++++++++++-- 6 files changed, 325 insertions(+), 13 deletions(-) diff --git a/docs/knowledge.md b/docs/knowledge.md index a46c2a15..5f2b00d1 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -47,6 +47,10 @@ hive-c0re maintains the local clone at The endpoint is protected by an auto-generated HMAC secret that hive-c0re verifies on every delivery. + On a swarm, a **second** hook on the same repo points at the + swarm controller — see `docs/swarm/README.md` § Swarm-wide forge + webhooks. Both are expected; neither should be deleted. + 2. **Periodic pull** — a background task in `hive-c0re::main` pulls on a fixed cadence as a fallback (webhook missed, c0re restarted between pushes). The pull is best-effort — a failure diff --git a/docs/swarm/README.md b/docs/swarm/README.md index 14499b37..5f6e7fda 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -349,6 +349,30 @@ list, which would look like a silent swarm rather than a controller that cannot see. The body says which. Status survives a controller restart: it is stored in the queue, not in the daemon. +### Swarm-wide forge webhooks + +At startup the controller registers two Forgejo hooks pointing at +itself — a `push` hook on `internal/knowledge` and a `pull_request` hook +on the `agent-configs` org, both under +`https:///webhook/forge/`. + +**You will see two hooks where there used to be one**, and that is the +intended state for now: each hive still registers and receives its own, +and the controller's is an additional copy that it currently logs and +nothing more. **Do not delete either.** Removing a hive's hook stops that +hive acting on knowledge pushes and config PRs; removing the +controller's just gets recreated on its next start. + +Nothing to configure. The hooks are registered only when this host also +serves the swarm UI vhost — that is what publishes the endpoint, and a +hook the forge cannot reach would collect failed deliveries while +looking healthy. The HMAC secret is generated on first start and kept +(see [`docs/persistence.md`](../persistence.md)). + +To check it is working, push to `internal/knowledge` and look for +`webhook: verified delivery` in `journalctl -u swarm-controller`. A +refused delivery logs `webhook: refused delivery` with the reason. + ## Cross-references - `docs/snapshot-store.md` — the swarm's `btrfs receive` endpoint, and diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 44464fd4..c249d3ad 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -33,6 +33,7 @@ let natsCfg = config.services.hyperhive.swarm.nats; forgeCfg = config.services.hyperhive.swarm.forge; + uiCfg = config.services.hyperhive.swarm.ui; # The controller's own OAuth2 client. It is NOT a hive: the per-hive # clients the roster issues belong to hives, and the responder's client @@ -85,6 +86,19 @@ let SWARM_CONTROLLER_FORGE_TOKEN_FILE = "%d/forge-token"; }; + # How the forge must address this controller to deliver a swarm-wide + # webhook. Gated on the swarm UI being served *here*, because that module + # is what declares the vhost and the `/webhook/forge/` location inside it: + # absent, nothing outside this host can reach the endpoint. + # + # 🔑 The gate is the point, not a detail. The daemon registers hooks only + # when this is set, and a hook whose `target_url` nothing answers is worse + # than no hook at all — forgejo keeps the registration, marks every + # delivery failed, and the hook still reads as configured. + webhookEnv = lib.optionalAttrs uiCfg.enable { + SWARM_CONTROLLER_PUBLIC_URL = "https://${uiCfg.domain}"; + }; + # Not a secret to deliver — `swarm-authelia-bridge`'s own bearer check # is satisfied by THIS daemon's existing queue OIDC identity # (`queueEnv` above): "one identity per principal" already covers this, @@ -594,6 +608,7 @@ in } // queueEnv // forgeEnv + // webhookEnv // authBridgeEnv; }; diff --git a/swarm-controller/src/forge.rs b/swarm-controller/src/forge.rs index 29e5607e..d1bcf646 100644 --- a/swarm-controller/src/forge.rs +++ b/swarm-controller/src/forge.rs @@ -19,10 +19,14 @@ use anyhow::{Context, Result}; use forgejo_api::structs::{ AddCollaboratorOption, AddCollaboratorOptionPermission, ChangeFileOperation, ChangeFileOperationOperation, ChangeFilesOptions, CreateBranchProtectionOption, - CreateRepoOption, RepoGetContentsQuery, + CreateHookOption, CreateHookOptionConfig, CreateHookOptionType, CreateRepoOption, + RepoGetContentsQuery, }; use forgejo_api::{ApiErrorKind, Auth, Forgejo, ForgejoError}; use reqwest::StatusCode; +use std::collections::BTreeMap; + +use crate::webhook::DeliveryKind; /// The forge org that owns agent repos. Same org `hive-c0re::forge` /// already uses for its own single-hive `CreateRepo` path — this is @@ -38,6 +42,20 @@ pub const AGENTS_ORG: &str = "agents"; /// [`Client::create_repo`]'s doc comment. const OPERATORS_TEAM: &str = "operators"; +/// The org owning per-agent config repos, and where the `pull_request` hook +/// lives. Same value as `hive-c0re::forge::CONFIG_ORG` — one forge, one org. +/// +/// ⚠️ Duplicated across the crate boundary (this crate deliberately does not +/// depend on `hive-c0re`), so nothing makes the two fail together. The +/// failure mode if they drift is quiet: the hook is created on an org that +/// exists, forgejo reports it healthy, and it simply never fires. +const CONFIG_ORG: &str = "agent-configs"; + +/// The hive-wide knowledge repo, where the `push` hook lives. Same values as +/// `hive-c0re::workers::knowledge::{ORG, REPO}`. +const KNOWLEDGE_ORG: &str = "internal"; +const KNOWLEDGE_REPO: &str = "knowledge"; + /// The env vars `swarm-controller.nix`'s `forgeEnv` sets. Named here /// once so [`Client::from_env`] and the nix module can't drift silently /// — a rename on one side without the other fails loudly (env var @@ -325,6 +343,136 @@ impl Client { Err(e) => Err(e).with_context(|| format!("check for {path} in {AGENTS_ORG}/{repo}")), } } + + /// Register the swarm-wide hooks against this controller, so a real + /// forge event reaches [`crate::webhook`] instead of the endpoint only + /// being reachable by hand. + /// + /// **These are registered ALONGSIDE the per-hive hooks, not instead of + /// them.** Every hive keeps receiving and acting on its own deliveries + /// exactly as today; the controller receives a copy and (for now) logs + /// it. Taking the hive-side registration away is a later step, and it + /// has to be later: fan-out swarm→hive does not exist yet, so a hook + /// moved now would point at a receiver that forwards nowhere — silent on + /// both sides, indistinguishable from no activity. + /// + /// ⛔ **No stale-hook deletion arm, unlike the two per-hive registrars + /// this otherwise mirrors.** Their arm deletes hooks matching their own + /// path with a foreign base; copying it here would delete the hives' + /// live hooks, which are not stale — they are the path still in + /// production. The controller only ever adds its own. + /// + /// Idempotent: an existing hook with the same `target_url` is left + /// alone, so this is safe on every boot. + /// + /// # Errors + /// + /// Returns the first failure. A listing failure is not fatal — it falls + /// through to the create attempt, which is idempotent server-side by way + /// of the already-exists fold, the same best-effort shape `hive-c0re` + /// uses. + pub async fn ensure_swarm_webhooks(&self, public_base: &str, secret: &str) -> Result<()> { + for kind in DeliveryKind::ALL { + let target_url = kind.target_url(public_base); + let (event, scope) = match kind { + DeliveryKind::Knowledge => ( + "push", + HookScope::Repo { + org: KNOWLEDGE_ORG, + repo: KNOWLEDGE_REPO, + }, + ), + DeliveryKind::ConfigPr => ("pull_request", HookScope::Org { org: CONFIG_ORG }), + }; + self.ensure_hook(&scope, &target_url, event, secret) + .await + .with_context(|| format!("register swarm webhook {target_url}"))?; + } + Ok(()) + } + + /// One hook, idempotently. Split out of [`Self::ensure_swarm_webhooks`] + /// so the org-scoped and repo-scoped calls do not each grow their own + /// copy of the list-then-create logic. + async fn ensure_hook( + &self, + scope: &HookScope<'_>, + target_url: &str, + event: &str, + secret: &str, + ) -> Result<()> { + match scope.list_hook_urls(&self.api).await { + Ok(urls) => { + if urls.iter().any(|u| u == target_url) { + tracing::debug!(%target_url, "swarm forge: webhook already registered"); + return Ok(()); + } + } + Err(e) => { + // Best-effort, same as the per-hive registrars: a forge that + // cannot be listed may still accept a create, and a + // duplicate create is folded into success below. + tracing::debug!(error = %e, %target_url, "swarm forge: listing hooks failed; attempting create"); + } + } + + let mut additional = BTreeMap::new(); + additional.insert("secret".to_owned(), secret.to_owned()); + let hook = CreateHookOption { + active: Some(true), + authorization_header: None, + branch_filter: None, + config: CreateHookOptionConfig { + content_type: "json".to_owned(), + url: url::Url::parse(target_url) + .with_context(|| format!("parse webhook target url {target_url}"))?, + additional, + }, + events: Some(vec![event.to_owned()]), + r#type: CreateHookOptionType::Forgejo, + }; + match scope.create_hook(&self.api, hook).await { + Ok(()) => { + tracing::info!(%target_url, %event, "swarm forge: webhook registered"); + Ok(()) + } + Err(e) if is_already_exists(&e) => { + tracing::debug!(%target_url, "swarm forge: webhook already present"); + Ok(()) + } + Err(e) => Err(e.into()), + } + } +} + +/// Where a hook lives. The knowledge hook is repo-scoped and the config-PR +/// hook is org-scoped, mirroring exactly where the per-hive registrars put +/// theirs — a hook on the wrong scope would never fire, and forgejo would +/// report that as a perfectly healthy hook with no deliveries. +enum HookScope<'a> { + Repo { org: &'a str, repo: &'a str }, + Org { org: &'a str }, +} + +impl HookScope<'_> { + /// The `url` config value of every hook currently on this scope. + async fn list_hook_urls(&self, api: &Forgejo) -> Result, ForgejoError> { + let hooks = match self { + Self::Repo { org, repo } => api.repo_list_hooks(org, repo).all().await?, + Self::Org { org } => api.org_list_hooks(org).send().await?, + }; + Ok(hooks + .iter() + .filter_map(|h| h.config.as_ref()?.get("url").cloned()) + .collect()) + } + + async fn create_hook(&self, api: &Forgejo, hook: CreateHookOption) -> Result<(), ForgejoError> { + match self { + Self::Repo { org, repo } => api.repo_create_hook(org, repo, hook).await.map(drop), + Self::Org { org } => api.org_create_hook(org, hook).await.map(drop), + } + } } /// Base64-encode `content` for a forgejo content-API call — every file diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 1e172019..5b4539d0 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -661,6 +661,47 @@ async fn get_jobq_rollup(State(state): State) -> Json>, secret: Option>) { + let Some(forge) = forge else { return }; + let Some(secret) = secret else { return }; + let Ok(public_url) = std::env::var(PUBLIC_URL_ENV) else { + tracing::info!( + "{PUBLIC_URL_ENV} unset; not registering swarm-wide forge webhooks (the \ + endpoint is not published on this host)" + ); + return; + }; + tokio::spawn(async move { + if let Err(e) = forge.ensure_swarm_webhooks(&public_url, &secret).await { + tracing::warn!( + error = %format!("{e:#}"), + "registering swarm-wide forge webhooks failed; retrying on next start" + ); + } + }); +} + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -763,7 +804,7 @@ async fn main() -> Result<()> { }; let deps = WorkerDeps { auth, - forge: forge_client, + forge: forge_client.clone(), }; let jobq = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new( @@ -787,6 +828,8 @@ async fn main() -> Result<()> { } }; + register_swarm_webhooks(forge_client, webhook_secret.clone()); + let state = AppState { hives: Arc::new(load_hives()), links: Arc::new(load_links()), diff --git a/swarm-controller/src/webhook.rs b/swarm-controller/src/webhook.rs index e8b5da48..03efd5a5 100644 --- a/swarm-controller/src/webhook.rs +++ b/swarm-controller/src/webhook.rs @@ -14,18 +14,14 @@ //! //! **The HMAC code is not shared with `hive-c0re` because c0re's copy is //! leaving, not staying.** Once registration moves here, hives stop -//! registering *and* receiving; c0re's routes, its secret and the gateway's -//! `/webhook/` route all go with it, and what survives there is driven by a -//! queue event, authenticated by the queue. A shared crate is right when a -//! second consumer *arrives* — here it is departing, so extracting would -//! build an abstraction in order to unwind it. The two verifiers meanwhile -//! check different hooks with different secrets and never need to agree. +//! registering *and* receiving, and c0re's routes and secret go with it. A +//! shared crate is right when a second consumer *arrives*; here it is +//! departing. The two verifiers meanwhile check different hooks with +//! different secrets and never need to agree. //! -//! **Nothing is registered against these routes yet, on purpose.** The -//! replacement path is built and observable in full before anything takes -//! the old one away: delete a hive's registration first and the swarm's one -//! `target_url` points at a receiver that forwards nowhere — indistinguishable -//! from no activity, on both sides. +//! **These hooks are registered ALONGSIDE the per-hive ones** — see +//! [`crate::forge::Client::ensure_swarm_webhooks`], which carries why, and +//! why moving them cannot come before fan-out. use anyhow::{Context as _, Result}; use axum::{ @@ -173,7 +169,39 @@ pub(super) enum DeliveryKind { ConfigPr, } +/// The route prefix a registered `target_url` must point at. +/// +/// ⚠️ Deliberately **not** `/webhook/knowledge` or `/webhook/config-pr`, the +/// paths the per-hive receivers use. Both hive-side registrars delete any +/// hook whose URL ends with *their* path but has a different base — see +/// `hive-c0re`'s `forge::ensure_config_pr_webhook` and +/// `workers::knowledge::ensure_webhook`. A swarm-level hook under those +/// paths would therefore be deleted by every hive on every boot, and the +/// symptom is a hook that silently stops existing. `webhook_urls_survive_the_hive_side_reapers` +/// pins that. +const ROUTE_PREFIX: &str = "/webhook/forge/"; + impl DeliveryKind { + /// Every kind, for the registration sweep. An array rather than a + /// hand-written list at the call site, so adding a kind cannot leave one + /// hook unregistered. + pub(super) const ALL: [Self; 2] = [Self::Knowledge, Self::ConfigPr]; + + /// The `target_url` to register with Forgejo for this kind, given the + /// swarm's public base URL. + /// + /// Built here rather than at the registration call site so the URL that + /// is *registered* and the route that *serves* it are the same fact in + /// one place. A trailing slash on the base is tolerated — it arrives from + /// config, and `https://swarm//webhook/...` would be a hook that 404s. + pub(super) fn target_url(self, public_base: &str) -> String { + format!( + "{}{ROUTE_PREFIX}{}", + public_base.trim_end_matches('/'), + self.as_str() + ) + } + /// Parse the `{kind}` path segment. Unknown values are rejected rather /// than accepted-and-ignored: a typo in a registered `target_url` must /// be *observable*, and a 200 for an unrecognised path is exactly the @@ -509,4 +537,54 @@ mod tests { "an empty value must not resolve to a relative path" ); } + + /// The URL that gets registered must match the route that serves it, and + /// a trailing slash on the configured base must not produce `//webhook`. + #[test] + fn a_target_url_is_the_route_this_module_serves() { + assert_eq!( + DeliveryKind::Knowledge.target_url("https://swarm.example"), + "https://swarm.example/webhook/forge/knowledge" + ); + assert_eq!( + DeliveryKind::ConfigPr.target_url("https://swarm.example/"), + "https://swarm.example/webhook/forge/config-pr", + "a trailing slash on the base must not double up" + ); + for kind in DeliveryKind::ALL { + let url = kind.target_url("https://swarm.example"); + let segment = url.rsplit('/').next().expect("a last segment"); + assert_eq!( + DeliveryKind::parse(segment), + Some(kind), + "the last path segment of a registered URL must parse back \ + to the kind that built it" + ); + } + } + + /// A cross-daemon invariant with nothing else to enforce it: both + /// per-hive registrars in `hive-c0re` **delete** hooks whose URL ends + /// with their own path but carries a different base — see + /// `forge::ensure_config_pr_webhook` and `knowledge::ensure_webhook`. + /// + /// While the swarm-level hooks live alongside the per-hive ones, a + /// controller URL matching either suffix would be deleted by every hive + /// on every boot: the swarm hook would simply cease to exist, with the + /// cause in a different daemon's startup sweep. Serving these under + /// `/webhook/forge/` is what avoids it, and this is the only place that + /// says so in a form that fails. + #[test] + fn webhook_urls_survive_the_hive_side_reapers() { + for suffix in ["/webhook/knowledge", "/webhook/config-pr"] { + for kind in DeliveryKind::ALL { + let url = kind.target_url("https://swarm.example"); + assert!( + !url.ends_with(suffix), + "{url} ends with {suffix}, which every hive's startup \ + sweep deletes as a stale hook" + ); + } + } + } }