//! Forgejo webhook receipt at the swarm level. //! //! A Forgejo webhook has exactly **one** `target_url`, so every hive //! registering the same swarm-wide hooks is last-writer-wins rather than //! idempotent, and every hive but the most recent silently stops receiving //! deliveries. One owner is the only non-racing shape, and the only //! swarm-wide thing in the deployment is this daemon. //! //! **This daemon interprets a delivery; it does not forward it.** It parses //! the payload and emits a semantic message — *the knowledge repo changed*, //! *deploy agent X at rev Y* — addressed to the hives that need it. One place //! decides what a forge payload means, so no hive re-derives it. //! //! Receipt is all that is wired today: the payload is opaque bytes keyed by a //! [`DeliveryKind`] from the URL path, and nothing consumes it until the //! swarm→hive channel exists. //! //! The HMAC code is deliberately **not** shared with `hive-c0re`: that copy is //! leaving, and a shared crate is right only when a second consumer arrives. //! //! **These hooks are registered ALONGSIDE the per-hive ones** — see //! [`crate::forge::Client::ensure_swarm_webhooks`]. use anyhow::{Context as _, Result}; use axum::{ body::Bytes, extract::{Path, State}, http::{HeaderMap, StatusCode}, response::{IntoResponse as _, Response}, }; use hmac::{Hmac, KeyInit as _, Mac as _}; use sha2::Sha256; use super::AppState; /// Where the HMAC secret is kept, relative to the daemon's state directory. /// /// The secret is generated on first start and handed to Forgejo when the /// hook is registered, so it must survive a restart — a rotated secret /// would make every subsequent delivery fail verification with a correctly /// configured hook on both sides. const SECRET_FILE: &str = "webhook-secret"; /// Fallback state directory, used only when `STATE_DIRECTORY` is unset (a /// dev run outside systemd). Under the unit it is always set, because /// `StateDirectory = "swarm-controller"` is declared there. const DEFAULT_STATE_DIR: &str = "/var/lib/swarm-controller"; /// Path to the HMAC secret file. /// /// Reads systemd's `STATE_DIRECTORY` rather than introducing a /// `SWARM_CONTROLLER_*` env var for it: the unit already declares /// `StateDirectory=`, and a second env var would be a second declaration of /// the same fact, free to drift from the first. /// /// ⚠️ `StateDirectory=` accepts a *list*, in which case systemd exports the /// paths colon-separated. Only the first is taken — a bare `PathBuf::from` /// of the whole variable would silently produce a path containing a colon /// the day someone adds a second directory to the unit. fn secret_path() -> std::path::PathBuf { secret_path_from(std::env::var("STATE_DIRECTORY").ok().as_deref()) } /// The pure half of [`secret_path`], split out so the colon-list handling is /// testable **without touching the environment**. /// /// Not a stylistic split: `std::env::set_var` mutates process-global state, /// and cargo runs a crate's tests on parallel threads in one process, so two /// tests setting it race — which is not hypothetical here, it is how the /// first version of this module's tests failed. fn secret_path_from(raw: Option<&str>) -> std::path::PathBuf { let dir = raw .and_then(|raw| raw.split(':').next()) .filter(|first| !first.is_empty()) .unwrap_or(DEFAULT_STATE_DIR); std::path::PathBuf::from(dir).join(SECRET_FILE) } /// Load the swarm's webhook HMAC secret, generating and persisting it if the /// file is absent or malformed. /// /// Returns a hex-encoded 32-byte secret (64 hex chars). /// /// # Errors /// /// Returns an error when the secret cannot be generated (no `/dev/urandom`) /// or cannot be persisted. A malformed existing file is **not** an error: it /// is regenerated, because a secret that cannot be parsed is /// indistinguishable from one that was never written, and failing startup /// over it would strand the daemon with no path forward. pub(super) fn load_or_generate_secret() -> Result { load_or_generate_at(&secret_path()) } /// The path-taking half of [`load_or_generate_secret`], split out for the /// same reason as [`secret_path_from`]: a test can point it at a scratch /// file without mutating a process-global env var. fn load_or_generate_at(path: &std::path::Path) -> Result { if let Ok(raw) = std::fs::read_to_string(path) { let trimmed = raw.trim().to_owned(); if trimmed.len() == 64 && trimmed.chars().all(|c| c.is_ascii_hexdigit()) { return Ok(trimmed); } tracing::warn!( path = %path.display(), "webhook-secret file malformed (wrong length/chars); regenerating" ); } let secret = generate_hex_secret()?; std::fs::create_dir_all(path.parent().unwrap_or(path)) .with_context(|| format!("create dir for {}", path.display()))?; std::fs::write(path, format!("{secret}\n")) .with_context(|| format!("write webhook secret to {}", path.display()))?; tracing::info!(path = %path.display(), "webhook secret generated and persisted"); Ok(secret) } /// Read 32 random bytes from `/dev/urandom` and hex-encode them. /// /// `pub(super)` rather than private: `forge::Client::ensure_agent_user` /// reuses this for the throwaway account password Forgejo's admin /// create-user API requires (unlike the `forgejo admin` CLI's /// `--random-password`, the HTTP endpoint has no "generate one for me" /// option — see that function's doc comment). Same shape as the webhook /// secret above: a value nothing ever reads back, so 32 random bytes is /// as good as any other generator. pub(super) fn generate_hex_secret() -> Result { use std::io::Read as _; let mut buf = [0u8; 32]; let mut f = std::fs::File::open("/dev/urandom").context("open /dev/urandom for secret generation")?; f.read_exact(&mut buf) .context("read 32 bytes from /dev/urandom")?; Ok(hex_encode(&buf)) } /// Hex-encode `bytes` as a lowercase string. fn hex_encode(bytes: &[u8]) -> String { let mut out = String::with_capacity(bytes.len() * 2); for b in bytes { out.push(char::from_digit(u32::from(b >> 4), 16).unwrap_or('0')); out.push(char::from_digit(u32::from(b & 0xf), 16).unwrap_or('0')); } out } /// Decode a lowercase hex string into bytes; returns `None` on invalid input. fn hex_decode(s: &str) -> Option> { if !s.len().is_multiple_of(2) { return None; } let mut out = Vec::with_capacity(s.len() / 2); let mut chars = s.chars(); while let (Some(hi), Some(lo)) = (chars.next(), chars.next()) { let hi = u8::try_from(hi.to_digit(16)?).ok()?; let lo = u8::try_from(lo.to_digit(16)?).ok()?; out.push((hi << 4) | lo); } Some(out) } /// Which swarm-wide hook a delivery arrived on. /// /// The kind comes from the URL path rather than from Forgejo's /// `X-Forgejo-Event` header, so the routing key is decided by *the /// registration we made* instead of by a field the sender chooses. Each /// registered hook gets its own `target_url`, exactly as the per-hive hooks /// do today. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum DeliveryKind { /// Push events on the hive-wide knowledge repo. Knowledge, /// `pull_request` events on the agent-config repos. ConfigPr, /// `push` events on every repo in the instance — see /// `crate::vcs_metrics`'s doc comment for why this exists as its own /// kind rather than folding into [`Self::Knowledge`]'s already-`push` /// hook: that one is repo-scoped to the knowledge repo alone, and this /// one is instance-wide (registered via `admin_create_hook`, not /// `repo_create_hook`) — different scope, same event name. VcsActivity, } /// The route prefix a registered `target_url` must point at. /// /// ⚠️ Deliberately **not** `/webhook/knowledge` or `/webhook/config-pr`, the /// paths the per-hive receivers use. A hive-side registrar deletes any hook /// whose URL ends with *its* path but has a different base — see /// `hive-c0re`'s `forge::ensure_config_pr_webhook`. A swarm-level hook under /// such a path would therefore be deleted by every hive on every boot, and /// the symptom is a hook that silently stops existing. /// `webhook_urls_survive_the_hive_side_reapers` pins that, and its own doc /// records why `/webhook/knowledge` stays in the check even though the /// knowledge registrar no longer reaps. const ROUTE_PREFIX: &str = "/webhook/forge/"; impl DeliveryKind { /// Every kind, for the registration sweep. An array rather than a /// hand-written list at the call site, so adding a kind cannot leave one /// hook unregistered. pub(super) const ALL: [Self; 3] = [Self::Knowledge, Self::ConfigPr, Self::VcsActivity]; /// The `target_url` to register with Forgejo for this kind, given the /// swarm's public base URL. /// /// Built here rather than at the registration call site so the URL that /// is *registered* and the route that *serves* it are the same fact in /// one place. A trailing slash on the base is tolerated — it arrives from /// config, and `https://swarm//webhook/...` would be a hook that 404s. pub(super) fn target_url(self, public_base: &str) -> String { format!( "{}{ROUTE_PREFIX}{}", public_base.trim_end_matches('/'), self.as_str() ) } /// Parse the `{kind}` path segment. Unknown values are rejected rather /// than accepted-and-ignored: a typo in a registered `target_url` must /// be *observable*, and a 200 for an unrecognised path is exactly the /// silence this issue exists to remove. fn parse(segment: &str) -> Option { match segment { "knowledge" => Some(Self::Knowledge), "config-pr" => Some(Self::ConfigPr), "vcs-activity" => Some(Self::VcsActivity), _ => None, } } /// Stable string form, used for logging. Deliberately the same spelling /// as the path segment so a journal line can be matched against a /// registered URL. /// /// ⚠️ **Not the routing key for the swarm→hive message.** That message is /// semantic (*knowledge repo changed*, *deploy agent X at rev Y*) and is /// addressed to the hives that need it; which hook a delivery arrived on /// is an input to deriving it, not the thing sent. fn as_str(self) -> &'static str { match self { Self::Knowledge => "knowledge", Self::ConfigPr => "config-pr", Self::VcsActivity => "vcs-activity", } } } /// Why a delivery was refused. /// /// A value rather than a string, so the status code is chosen by matching on /// the reason instead of on message text — `hive-c0re`'s equivalent does /// `e.contains("unavailable")`, which couples an HTTP response to the exact /// wording of a log message. #[derive(Debug, PartialEq, Eq)] pub(super) enum Refusal { /// No secret was loaded at startup: the endpoint cannot verify anything, /// which is this daemon's fault and not the caller's. 503, so a retry is /// meaningful. SecretUnavailable, /// The signature was missing, malformed, or did not match. 401. BadSignature(String), } impl Refusal { fn status(&self) -> StatusCode { match self { Self::SecretUnavailable => StatusCode::SERVICE_UNAVAILABLE, Self::BadSignature(_) => StatusCode::UNAUTHORIZED, } } fn message(&self) -> String { match self { Self::SecretUnavailable => { "webhook HMAC secret unavailable; endpoint disabled".to_owned() } Self::BadSignature(detail) => detail.clone(), } } } /// Verify the `X-Hub-Signature-256` header Forgejo attaches to a delivery /// (`sha256=`). /// /// The body is the raw wire bytes, not a re-serialised structure: an HMAC is /// over exactly what was sent, and any parse-then-reserialise step would /// change whitespace or key order and fail every signature. /// /// The refusal messages are safe to log — they never contain the secret or /// the expected digest, so a caller can surface *why* a delivery was refused /// without handing an attacker the answer. pub(super) fn verify( secret: Option<&str>, headers: &HeaderMap, body: &Bytes, ) -> Result<(), Refusal> { let secret = secret.ok_or(Refusal::SecretUnavailable)?; let sig = headers .get("x-hub-signature-256") .and_then(|v| v.to_str().ok()) .unwrap_or(""); if sig.is_empty() { return Err(Refusal::BadSignature( "missing X-Hub-Signature-256 header".to_owned(), )); } let Some(sig_hex) = sig.strip_prefix("sha256=") else { return Err(Refusal::BadSignature( "X-Hub-Signature-256 missing 'sha256=' prefix".to_owned(), )); }; let Some(expected) = hex_decode(sig_hex) else { return Err(Refusal::BadSignature( "X-Hub-Signature-256 contains non-hex chars".to_owned(), )); }; let mut mac = Hmac::::new_from_slice(secret.as_bytes()) .map_err(|e| Refusal::BadSignature(format!("HMAC key error: {e}")))?; mac.update(body); // `verify_slice` is constant-time and also rejects a wrong-length digest, // which is why the comparison is not written by hand here. mac.verify_slice(&expected) .map_err(|_| Refusal::BadSignature("X-Hub-Signature-256 mismatch".to_owned())) } /// POST `/webhook/forge/{kind}` — a swarm-wide Forgejo delivery. /// /// Verifies the HMAC over the raw body, then acts per `kind`: a `Knowledge` /// delivery announces the change to every hive (see /// [`announce_knowledge_change`]); a `ConfigPr` delivery updates /// `crate::config_pr::ConfigPrCache` immediately (see that module's doc /// comment for why both this and a periodic poll write the same cache). /// /// Returns 200 on an accepted delivery so Forgejo does not retry. A refused /// one answers 401 (bad signature) or 503 (this daemon has no secret), and /// an unknown `{kind}` answers 404 rather than a cheerful 200. #[utoipa::path( post, path = "/webhook/forge/{kind}", params( ("kind" = String, Path, description = "which swarm-wide hook this delivery arrived on: \ `knowledge` or `config-pr`") ), request_body( content = String, content_type = "application/json", description = "Forgejo webhook payload, taken as raw bytes so HMAC \ verification runs over the exact wire bytes rather \ than over a reserialised copy" ), responses( (status = 200, description = "delivery accepted", body = String), (status = 401, description = "bad or missing HMAC signature"), (status = 404, description = "unknown hook kind in the path"), (status = 503, description = "HMAC secret unavailable at startup"), ), tag = "webhook" )] pub(super) async fn post_webhook_forge( State(state): State, Path(kind): Path, headers: HeaderMap, body: Bytes, ) -> Response { // Order matters: verify BEFORE looking at the path. An unauthenticated // caller learning which `{kind}` values exist, from the difference // between 404 and 401, is a small leak — but it is free to close. if let Err(refusal) = verify(state.webhook_secret.as_deref(), &headers, &body) { let message = refusal.message(); tracing::warn!(%kind, %message, "webhook: refused delivery"); return (refusal.status(), message).into_response(); } let Some(kind) = DeliveryKind::parse(&kind) else { tracing::warn!(%kind, "webhook: delivery on an unknown hook kind"); return (StatusCode::NOT_FOUND, "unknown hook kind").into_response(); }; tracing::info!( kind = kind.as_str(), bytes = body.len(), "webhook: verified delivery" ); match kind { DeliveryKind::Knowledge => announce_knowledge_change(&state).await, DeliveryKind::ConfigPr => { if let Some(cache) = state.config_prs.as_ref() { cache.apply_webhook_delivery(&body); } } DeliveryKind::VcsActivity => { if let Some(activity) = crate::vcs_metrics::parse_push(&body) { crate::vcs_metrics::record_push(&activity); } else { tracing::warn!("webhook: vcs-activity delivery did not parse as a push payload"); } } } (StatusCode::OK, "ok").into_response() } /// Tell every hive in the swarm that the knowledge repository changed. /// /// The event carries **no payload**, because there is nothing to carry: the /// hive-side handler this replaces read two fields from Forgejo's webhook and /// used neither — both were filters — and then ran `git pull`, which re-derives /// everything from the repository itself. So what crosses the queue is an edge /// trigger, and adding fields to it would invent a contract nobody reads. /// /// One publish to one shared subject, not one per hive: every subscriber gets /// the same empty event, so the roster is never consulted and this daemon does /// not need to know who the hives are in order to tell them. /// /// # Failure /// /// Returns nothing and fails soft. A missed announcement costs a hive stale /// knowledge until its next boot pull — the same cost as a webhook delivery to /// a hive that happened to be down, which is what this replaces. /// /// ⚠️ A **permission** failure cannot be observed here. `publish` hands the /// message to the connection's buffer, and a NATS authorization violation is /// reported asynchronously on the connection rather than as an error on this /// call — it reaches a client as a timeout, or as nothing at all. The `flush` /// below therefore proves the bytes left this process, and nothing more; if /// hives stop hearing events, the server log is the place that knows why. async fn announce_knowledge_change(state: &AppState) { let Some(status) = state.status.as_ref() else { // Verified, accepted, and dropped. Worth a warning rather than // silence: the forge will report a 200 and nobody would otherwise // learn that the delivery reached a controller with nowhere to put it. tracing::warn!( "webhook: knowledge delivery verified but no swarm queue is \ configured; no hive will be told" ); return; }; let client = status.queue_client(); let subject = swarm_queue_client::KNOWLEDGE_SUBJECT; // One publish, not one per hive: every subscriber gets the same empty // event, so the roster is not consulted at all. The controller does not // need to know who the hives are in order to say the repository moved. if let Err(e) = client.publish(subject, Vec::new().into()).await { tracing::warn!(%subject, error = %e, "webhook: publishing the knowledge event failed"); return; } // Logged after the flush rather than after the publish: `publish` only // hands the message to the client's write buffer, so a line printed there // would claim delivery this end cannot yet know about. if let Err(e) = client.flush().await { tracing::warn!(%subject, error = %e, "webhook: flushing the knowledge event failed"); return; } tracing::info!(%subject, "webhook: knowledge event published"); } #[cfg(test)] mod tests { use super::{DeliveryKind, Refusal, load_or_generate_at, secret_path_from, verify}; use axum::{body::Bytes, http::HeaderMap, http::StatusCode}; use hmac::{Hmac, KeyInit as _, Mac as _}; use sha2::Sha256; use std::fmt::Write as _; /// Mint the header Forgejo would send. Written out rather than reusing /// the verifier's internals so a test cannot pass by sharing a bug with /// the code under test. fn sign(secret: &str, body: &[u8]) -> String { let mut mac = Hmac::::new_from_slice(secret.as_bytes()).expect("key"); mac.update(body); let mut hex = String::new(); for byte in mac.finalize().into_bytes() { write!(hex, "{byte:02x}").expect("writing to a String cannot fail"); } format!("sha256={hex}") } fn headers_with(sig: &str) -> HeaderMap { let mut h = HeaderMap::new(); h.insert("x-hub-signature-256", sig.parse().expect("header value")); h } /// Each refusal reached separately, so a change that collapses them into /// one always-refuse path is visible — and the accept case alongside /// them, so a verifier that refuses *everything* cannot pass either. #[test] fn a_delivery_is_admitted_only_with_a_matching_signature() { let body = Bytes::from_static(b"{\"action\":\"opened\"}"); let good = sign("s3cret", &body); assert_eq!( verify(Some("s3cret"), &headers_with(&good), &body), Ok(()), "a correctly signed delivery must be admitted" ); assert!( matches!( verify(Some("other"), &headers_with(&good), &body), Err(Refusal::BadSignature(_)) ), "a signature minted with a different secret must be refused" ); assert!( matches!( verify( Some("s3cret"), &headers_with(&good), &Bytes::from_static(b"tampered") ), Err(Refusal::BadSignature(_)) ), "a body that does not match the digest must be refused" ); assert!( matches!( verify( Some("s3cret"), &headers_with(good.trim_start_matches("sha256=")), &body ), Err(Refusal::BadSignature(_)) ), "a bare hex digest with no 'sha256=' prefix is not what Forgejo sends" ); assert!( matches!( verify(Some("s3cret"), &HeaderMap::new(), &body), Err(Refusal::BadSignature(_)) ), "a delivery with no signature header at all must be refused" ); assert_eq!( verify(None, &headers_with(&good), &body), Err(Refusal::SecretUnavailable), "with no secret loaded the endpoint must refuse, not admit" ); } /// The status codes, asserted on the value rather than on message text. #[test] fn a_missing_secret_is_this_daemons_fault_and_a_bad_signature_is_the_callers() { assert_eq!( Refusal::SecretUnavailable.status(), StatusCode::SERVICE_UNAVAILABLE, "no secret is a server-side condition: a retry is meaningful" ); assert_eq!( Refusal::BadSignature(String::new()).status(), StatusCode::UNAUTHORIZED, "a bad signature is the caller's, and a retry will not help" ); } /// An unknown segment must not parse. The whole point of the 404 is that /// a mistyped `target_url` is observable rather than silently accepted. #[test] fn only_the_registered_hook_kinds_parse() { assert_eq!( DeliveryKind::parse("knowledge"), Some(DeliveryKind::Knowledge) ); assert_eq!( DeliveryKind::parse("config-pr"), Some(DeliveryKind::ConfigPr) ); assert_eq!( DeliveryKind::parse("config_pr"), None, "underscore is not the registered spelling; accepting both would make a journal line ambiguous" ); assert_eq!(DeliveryKind::parse(""), None); assert_eq!(DeliveryKind::parse("../knowledge"), None); } /// A malformed stored secret is replaced rather than fatal — and the /// replacement is a *usable* secret that survives a reload, because a /// version rotating on every call would pass a naive length check while /// silently breaking every registration made with the previous value. /// #[test] fn a_malformed_secret_file_is_regenerated_and_then_stable() { let dir = std::env::temp_dir().join(format!("swarm-hook-{}", std::process::id())); std::fs::create_dir_all(&dir).expect("mkdir"); let path = dir.join("webhook-secret"); std::fs::write(&path, "not-a-hex-secret\n").expect("seed"); let secret = load_or_generate_at(&path).expect("regenerates"); let again = load_or_generate_at(&path).expect("second load"); std::fs::remove_dir_all(&dir).ok(); assert_eq!(secret.len(), 64, "hex-encoded 32 bytes"); assert!(secret.chars().all(|c| c.is_ascii_hexdigit())); assert_eq!( again, secret, "a valid secret must be read back, not rotated" ); } /// `StateDirectory=` may name several directories, in which case systemd /// exports them colon-separated — take the first, never the raw value. /// /// Asserted against the pure half, so this never touches the process /// environment. The first version of these tests *did* use /// `std::env::set_var`, and it failed: cargo runs a crate's tests on /// parallel threads within **one** process, so the sibling env-setting /// test removed the variable mid-run and the other silently fell back to /// `/var/lib/swarm-controller`. #[test] fn the_secret_path_survives_a_multi_valued_state_directory() { assert_eq!( secret_path_from(Some("/var/lib/a:/var/lib/b")), std::path::Path::new("/var/lib/a/webhook-secret") ); assert_eq!( secret_path_from(Some("/var/lib/only")), std::path::Path::new("/var/lib/only/webhook-secret") ); assert_eq!( secret_path_from(None), std::path::Path::new("/var/lib/swarm-controller/webhook-secret"), "unset falls back to the directory the unit declares" ); assert_eq!( secret_path_from(Some("")), std::path::Path::new("/var/lib/swarm-controller/webhook-secret"), "an empty value must not resolve to a relative path" ); } /// The URL that gets registered must match the route that serves it, and /// a trailing slash on the configured base must not produce `//webhook`. #[test] fn a_target_url_is_the_route_this_module_serves() { assert_eq!( DeliveryKind::Knowledge.target_url("https://swarm.example"), "https://swarm.example/webhook/forge/knowledge" ); assert_eq!( DeliveryKind::ConfigPr.target_url("https://swarm.example/"), "https://swarm.example/webhook/forge/config-pr", "a trailing slash on the base must not double up" ); for kind in DeliveryKind::ALL { let url = kind.target_url("https://swarm.example"); let segment = url.rsplit('/').next().expect("a last segment"); assert_eq!( DeliveryKind::parse(segment), Some(kind), "the last path segment of a registered URL must parse back \ to the kind that built it" ); } } /// A cross-daemon invariant with nothing else to enforce it: a per-hive /// registrar in `hive-c0re` **deletes** hooks whose URL ends with its /// own path but carries a different base. A controller URL matching such /// a suffix would be deleted by every hive on every boot — the swarm hook /// would simply cease to exist, with the cause in a different daemon's /// startup sweep. Serving these under `/webhook/forge/` is what avoids /// it, and this is the only place that says so in a form that fails. /// /// `forge::ensure_config_pr_webhook` still reaps that way. The knowledge /// registrar no longer does — it was replaced by a removal that matches /// the full URL, so it cannot touch another hive's hook. **The /// `/webhook/knowledge` arm is kept anyway**, because the hazard is not /// this repository's current code: it is whatever is *deployed*, and a /// hive still running the previous version reaps by suffix until it is /// upgraded. Drop that arm once no such hive can exist, not when the /// source stops mentioning it. #[test] fn webhook_urls_survive_the_hive_side_reapers() { for suffix in ["/webhook/knowledge", "/webhook/config-pr"] { for kind in DeliveryKind::ALL { let url = kind.target_url("https://swarm.example"); assert!( !url.ends_with(suffix), "{url} ends with {suffix}, which every hive's startup \ sweep deletes as a stale hook" ); } } } }