//! Runtime nginx include-file generator for the gateway's per-agent //! `/agent//` location blocks. Writes //! `/var/lib/hive-gateway/conf/agents.conf` on every topology change. //! UDS upstream selection, the reload trigger, and idempotency: //! `docs/gateway.md::Per-agent unix-socket upstream`. use anyhow::{Context, Result}; use std::fmt::Write as _; use std::os::unix::fs::PermissionsExt as _; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; use crate::priv_client; use crate::stats::sweep_health::SweepHealth; use crate::agent_sockets; /// Set when `write` publishes a new agents.conf; cleared when /// `reload_gateway_nginx` submits the reload command successfully. /// Lets `spawn_poll` retry the reload on subsequent ticks when the /// previous attempt failed (e.g. gateway nginx temporarily down, priv /// helper unreachable) without re-writing the already-correct file. static RELOAD_PENDING: AtomicBool = AtomicBool::new(false); /// Unix timestamp (seconds) of the last failed reload attempt. /// `reload_if_pending` backs off to once per `RELOAD_RETRY_SECS` after a /// failure so a permanently broken gateway (bad config, nginx down) /// doesn't hammer `systemctl` on every 10-second `spawn_poll` tick. static LAST_FAILED_RELOAD: AtomicU64 = AtomicU64::new(0); /// Minimum gap between retry attempts after a reload failure (30 s). const RELOAD_RETRY_SECS: u64 = 30; /// Dashboard-banner health tracker for the reload sweep — raises a /// `gateway_nginx_reload` warning the first time a reload fails (routing is /// broken for whoever depends on the change *right now*, so no debounce /// window) and clears it the moment a reload succeeds again. fn health() -> &'static Mutex { static HEALTH: OnceLock> = OnceLock::new(); HEALTH.get_or_init(|| Mutex::new(SweepHealth::new("gateway_nginx_reload", "warn", 1))) } /// Nginx proxy headers present in every per-agent location block. /// `$connection_upgrade` is defined in the http context by the NixOS /// nginx module when `recommendedProxySettings = true` (which the /// gateway always sets). `X-Forwarded-Prefix` is per-location (set /// below in the render loop). Kept as a standalone const so it's /// visible to tests without repeating the text. const PROXY_HEADER_BLOCK: &str = " proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme;"; /// Render the nginx include body for `names`. Manager is included so /// `/agent/root/` is routable through the gateway. /// /// When `frontend_dir` is `Some(path)` each agent gets split location /// blocks: /// - Exact HTML pages (`/`, `/stats`, `/screen`) → `alias` from the nix /// store dist. These load even while the agent daemon is restarting. /// - Static assets (`/static/`) → `alias` from the nix store with cache /// headers. Served by nginx directly, no agent socket round-trip. /// - Everything else (all `/api/*`, `/events/*`, `/screen/ws`, `/icon`, /// `/send`, `/login/*`) → proxied to the agent daemon as before. /// /// When `frontend_dir` is `None` (legacy) each agent gets a single /// `location /agent//` proxy block. /// /// Output is deterministic for the same (sorted) set of names: /// no timestamps, no UUIDs. `BTreeMap` would give alphabetical order; /// we rely on the caller to pass sorted names if they care about diff /// stability, but the gateway treats the blocks as unordered by nginx's /// longest-prefix-match rules so ordering only affects human readability. fn render(names: &[String], frontend_dir: Option<&str>) -> String { let mut out = String::from( "# Generated by hive-c0re \u{2014} do not edit.\ \n# Refreshed on every topology change + when agents bind/drop their unix sockets.\ \n# Reload triggered by hive-c0re via hive-priv (systemctl reload nginx).\n", ); for name in names { // Two upstream forms because named locations (split mode's // `@_dynamic`) forbid a URI part on `proxy_pass`. The // legacy prefix-location path keeps the trailing `/` so nginx // strips `/agent//` automatically; the named-location // path strips the prefix via `rewrite` and uses a bare upstream. // When the socket is not yet bound, nginx returns 502, caught by // the error_page directive below. let sock = agent_sockets::socket_path_for(name).display().to_string(); let upstream_prefix = format!("http://unix:{sock}:/"); let upstream_bare = format!("http://unix:{sock}:"); if let Some(frontend) = frontend_dir { // Split mode: try to serve files from the nix-store dist first; // fall through to the agent daemon for anything not found there. // // `frontend` is a nix store path injected at build time by hive-c0re.nix // (`HIVE_AGENT_FRONTEND_DIR = "${cfg.frontend}/agent"`). Nix store paths // are of the form `/nix/store/-` — only [a-z0-9/._-], never // shell metacharacters — so interpolating directly into the nginx config // string is safe. We do NOT accept user-controlled input here. // // Location priority for a request to /agent//...: // ^~ /agent//static/ — highest priority; matches compiled assets. // Nix store paths are content-addressed and immutable — cache forever. // /agent// — try_files from dist, proxy fallback for the rest. // // try_files resolution (nginx applies the alias mapping before checking): // $uri — exact file match (/static/app.js → static/app.js) // $uri.html — bare-path .html fallback (/stats → stats.html) // $uri/index.html — directory index (/ → index.html) // @_dynamic — proxy catchall: api, events, icon, login, … // // Adding pages to the frontend dist works automatically — no generator // change needed. Per-agent extraFiles (in mergedDist, not in the base // dist path) still proxy to the agent daemon via @_dynamic. let _ = write!( out, "\n# Compiled JS/CSS assets — content-addressed nix store path, cache forever\n\ location ^~ /agent/{name}/static/ {{\n\ \n alias {frontend}/static/;\n\ \n expires 1y;\n\ \n add_header Cache-Control \"public, immutable, max-age=31536000\";\n\ }}\n\ \n# Static dist + proxy fallback for dynamic paths\n\ location /agent/{name}/ {{\n\ \n alias {frontend}/;\n\ \n try_files $uri $uri.html $uri/index.html @{name}_dynamic;\n\ }}\n\ \nlocation @{name}_dynamic {{\n\ {PROXY_HEADER_BLOCK}\n\ \n rewrite ^/agent/{name}/(.*)$ /$1 break;\n\ \n proxy_pass {upstream_bare};\n\ \n proxy_set_header X-Forwarded-Prefix /agent/{name};\n\ \n proxy_buffering off;\n\ \n proxy_read_timeout 1d;\n\ \n proxy_intercept_errors on;\n\ \n error_page 502 503 504 = /__hive_agent_unreachable;\n\ }}\n", ); } else { // Legacy mode: proxy everything to the agent daemon. let _ = write!( out, "\nlocation /agent/{name}/ {{\n\ {PROXY_HEADER_BLOCK}\n\ \n proxy_pass {upstream_prefix};\n\ \n proxy_set_header X-Forwarded-Prefix /agent/{name};\n\ \n proxy_buffering off;\n\ \n proxy_read_timeout 1d;\n\ \n proxy_intercept_errors on;\n\ \n error_page 502 503 504 = /__hive_agent_unreachable;\n\ }}\n", ); } } out } /// Atomically write the nginx include file for `names` to /// [`crate::paths::gateway_agents_conf()`]. Skips the write + reload when the rendered /// body matches what's already on disk (idempotent; avoids spurious /// gateway reloads on a quiet tick). /// /// After a successful write, triggers the appropriate nginx action via /// `hive-priv` (hive-c0re runs unprivileged and cannot act on a system /// unit): reload when nginx is active, reset-failed+start when in a /// failed state, plain start otherwise. Writer and nginx are on the same /// machine, so it is a plain unit action with no machine-bus hop. /// It stays an explicit trigger rather than a systemd path /// unit watching the file: the write already knows it changed something, /// and a watcher would turn one causal edge into a race with the writer's /// own rename (see `docs/gateway.md`). /// /// The priv call is best-effort — a failed sync is logged but not fatal. /// `reload_if_pending` retries on the next `spawn_poll` tick so a /// transient gateway-down situation converges without manual intervention. pub async fn write(names: &[String]) -> Result<()> { let frontend_dir = std::env::var("HIVE_AGENT_FRONTEND_DIR") .ok() .filter(|s| !s.is_empty()); let body = render(names, frontend_dir.as_deref()); let path = crate::paths::gateway_agents_conf(); if std::fs::read_to_string(&path).ok().as_deref() == Some(&body) { return Ok(()); } if let Some(parent) = path.parent() { std::fs::create_dir_all(parent) .with_context(|| format!("create dir {}", parent.display()))?; } let tmp = path.with_extension("conf.tmp"); std::fs::write(&tmp, &body).with_context(|| format!("write tmp {}", tmp.display()))?; // Mode set on the TMP file, before the rename, because a rename // carries the source's mode and owner and discards the destination's. // The tmpfiles rule declaring 0644 is `f` (create-if-absent), so it // never re-applies: from the first republish on, the published mode is // whatever this process's umask happened to produce. That is fine // today and becomes a dead gateway the day anything tightens c0re's // umask, since nginx reads this file as a different, unprivileged // user. Making it explicit means the mode is a property of the // publisher rather than of an unrelated unit's settings. std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o644)) .with_context(|| format!("chmod tmp {}", tmp.display()))?; std::fs::rename(&tmp, &path).with_context(|| { format!( "rename {} -> {} (atomic publish)", tmp.display(), path.display() ) })?; // Reset backoff so a fresh topology change gets an immediate attempt, // not a stale cooldown from a previous failure. LAST_FAILED_RELOAD.store(0, Ordering::Relaxed); // Mark reload pending before attempting so a failed attempt is // retried by the next spawn_poll tick (see `reload_if_pending`). RELOAD_PENDING.store(true, Ordering::Relaxed); reload_gateway_nginx().await; Ok(()) } /// Retry a pending nginx reload if a previous attempt failed. /// Called by `spawn_poll` on each tick so a transient failure /// (gateway nginx temporarily down, priv-helper error) is /// recovered automatically without requiring a new file write. /// /// Backs off to one retry per `RELOAD_RETRY_SECS` after a failure so a /// permanently broken gateway doesn't hammer `systemctl` on every tick. /// A fresh `write()` call always resets the backoff (new `RELOAD_PENDING` /// set to `true` + immediate attempt) so topology changes are still /// applied promptly. pub async fn reload_if_pending() { if !RELOAD_PENDING.load(Ordering::Relaxed) { return; } let last_failed = LAST_FAILED_RELOAD.load(Ordering::Relaxed); if last_failed > 0 { let now = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs(); if now.saturating_sub(last_failed) < RELOAD_RETRY_SECS { return; } } reload_gateway_nginx().await; } /// Synchronise the gateway nginx unit with the current agents.conf via /// `hive-priv` (privileged helper). The state-aware logic (active → /// reload; failed → reset-failed + start; inactive/unknown → start) /// runs inside hive-priv, which is root; hive-c0re runs as the /// unprivileged `hive-core` user and cannot act on a system unit. /// /// `RELOAD_PENDING` is cleared only after a successful operation so /// `reload_if_pending` keeps retrying on failure. async fn reload_gateway_nginx() { match priv_client::reload_gateway_nginx().await { Ok(()) => { tracing::debug!("gateway nginx sync succeeded"); RELOAD_PENDING.store(false, Ordering::Relaxed); LAST_FAILED_RELOAD.store(0, Ordering::Relaxed); if let Ok(mut h) = health().lock() { h.record_ok(); } } Err(e) => { tracing::warn!(error = %e, "gateway nginx sync failed — will retry"); let now = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs(); LAST_FAILED_RELOAD.store(now, Ordering::Relaxed); if let Ok(mut h) = health().lock() { let err = format!("{e:#}"); h.record_err(|ctx| { let age = ctx.since_last_ok.map_or_else( || "no success this process".to_owned(), |d| format!("last ok {} ago", crate::stats::sweep_health::fmt_age(d)), ); format!( "gateway nginx reload failing ({} consecutive, {age}) \ — routing changes are not taking effect: {err}", ctx.consecutive ) }); } } } } // ── Gateway HTTP-Basic (htpasswd) user management ────────────────────────── // The daemon owns the write (hivectl drives it over the host socket via the // `Gateway*User` requests, so hivectl never touches the credential file). // Keyed on the canonical `paths::GATEWAY_HTPASSWD` — a socket client doesn't // pick the path. fn htpasswd_path() -> std::path::PathBuf { std::path::PathBuf::from(crate::paths::GATEWAY_HTPASSWD) } /// Read the htpasswd file into lines, or an empty list if it doesn't exist. fn htpasswd_read() -> Result> { let path = htpasswd_path(); if !path.exists() { return Ok(vec![]); } let content = std::fs::read_to_string(&path) .with_context(|| format!("read htpasswd file {}", path.display()))?; Ok(content.lines().map(str::to_owned).collect()) } /// Write lines back atomically (`.tmp` then rename), with a trailing /// newline. fn htpasswd_write(lines: &[String]) -> Result<()> { let path = htpasswd_path(); let tmp = path.with_extension("htpasswd.tmp"); let content = if lines.is_empty() { String::new() } else { let mut s = lines.join("\n"); s.push('\n'); s }; std::fs::write(&tmp, &content) .with_context(|| format!("write htpasswd tmp {}", tmp.display()))?; std::fs::rename(&tmp, &path) .with_context(|| format!("rename {} → {}", tmp.display(), path.display()))?; Ok(()) } fn validate_htpasswd_username(username: &str) -> Result<()> { if username.is_empty() { anyhow::bail!("username must not be empty"); } if username.contains(':') { anyhow::bail!("username must not contain ':' (htpasswd field separator)"); } if username.chars().any(char::is_control) { anyhow::bail!("username must not contain control characters"); } Ok(()) } /// Add or update a gateway HTTP-Basic user, bcrypt-hashing `password` /// (cost 12). Returns the confirmation line for the operator. pub fn create_user(username: &str, password: &str) -> Result { validate_htpasswd_username(username)?; let raw_hash = bcrypt::hash(password, 12).context("bcrypt hash")?; // nginx auth_basic only recognises $2a$/$2x$/$2y$ — not $2b$. The prefixes // are algorithmically identical; remap so nginx accepts the hash. let hash = raw_hash.replacen("$2b$", "$2y$", 1); let entry = format!("{username}:{hash}"); let mut lines = htpasswd_read()?; let prefix = format!("{username}:"); let msg = if let Some(pos) = lines.iter().position(|l| l.starts_with(&prefix)) { lines[pos] = entry; format!("gateway: updated password for '{username}'") } else { lines.push(entry); format!("gateway: added user '{username}'") }; htpasswd_write(&lines)?; Ok(msg) } /// Remove a gateway HTTP-Basic user. Errors when the user isn't present so a /// no-op is detectable. pub fn delete_user(username: &str) -> Result { let mut lines = htpasswd_read()?; let prefix = format!("{username}:"); let before = lines.len(); lines.retain(|l| !l.starts_with(&prefix)); if lines.len() == before { anyhow::bail!("gateway: user '{username}' not found"); } htpasswd_write(&lines)?; Ok(format!("gateway: removed user '{username}'")) } /// List the gateway HTTP-Basic usernames (skips blank / comment lines). pub fn list_users() -> Result> { Ok(htpasswd_read()? .iter() .filter(|l| !l.is_empty() && !l.starts_with('#')) .filter_map(|l| l.split_once(':').map(|(name, _)| name.to_owned())) .collect()) } #[cfg(test)] mod tests { use super::*; use crate::lifecycle::MANAGER_NAME; // ── legacy mode (frontend_dir = None) ────────────────────────────── #[test] fn render_empty_is_header_only() { let body = render(&[], None); assert!(body.starts_with("# Generated by hive-c0re")); // No location blocks when no agents. assert!(!body.contains("location")); } #[test] fn render_includes_manager() { let names: Vec = [MANAGER_NAME, "iris"] .iter() .map(|s| (*s).to_owned()) .collect(); let body = render(&names, None); assert!(body.contains(&format!("/agent/{MANAGER_NAME}/"))); assert!(body.contains("/agent/iris/")); } #[test] fn render_uds_upstream_unconditional() { let names = vec!["iris".to_owned()]; let body = render(&names, None); // UDS form present assert!( body.contains("proxy_pass http://unix:"), "expected UDS upstream for iris, got:\n{body}" ); // No TCP loopback fallback assert!( !body.contains("127.0.0.1"), "unexpected TCP loopback in output:\n{body}" ); } #[test] fn render_includes_proxy_headers() { let names = vec!["iris".to_owned()]; let body = render(&names, None); // All required proxy headers present (WebSocket upgrade + forwarded-for). assert!(body.contains("proxy_http_version 1.1")); assert!(body.contains("proxy_set_header Upgrade $http_upgrade")); assert!(body.contains("proxy_set_header X-Forwarded-For")); } #[test] fn render_includes_forwarded_prefix() { let names = vec!["atlas".to_owned()]; let body = render(&names, None); assert!( body.contains("X-Forwarded-Prefix /agent/atlas"), "expected X-Forwarded-Prefix for atlas, got:\n{body}" ); } #[test] fn render_includes_error_pages() { let names = vec!["iris".to_owned()]; let body = render(&names, None); assert!(body.contains("proxy_intercept_errors on")); assert!(body.contains("__hive_agent_unreachable")); } #[test] fn render_includes_buffering_and_timeout() { let names = vec!["iris".to_owned()]; let body = render(&names, None); assert!(body.contains("proxy_buffering off")); assert!(body.contains("proxy_read_timeout 1d")); } // ── split mode (frontend_dir = Some) ─────────────────────────────── const FAKE_FRONTEND: &str = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-frontend/agent"; #[test] fn split_mode_has_immutable_cache_for_static_assets() { let names = vec!["iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); // ^~ prefix block for /static/ with immutable cache headers assert!( body.contains("location ^~ /agent/iris/static/ {"), "expected ^~ static location, got:\n{body}" ); assert!(body.contains(&format!("alias {FAKE_FRONTEND}/static/;"))); assert!(body.contains("expires 1y;")); assert!(body.contains("immutable")); } #[test] fn split_mode_has_alias_and_try_files() { let names = vec!["iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); // Plain prefix location with alias + try_files assert!( body.contains("location /agent/iris/ {"), "expected try_files location, got:\n{body}" ); assert!( body.contains(&format!("alias {FAKE_FRONTEND}/;")), "expected alias, got:\n{body}" ); // try_files handles exact file, bare-path .html, directory index assert!( body.contains("try_files $uri $uri.html $uri/index.html @iris_dynamic"), "expected try_files with .html fallback, got:\n{body}" ); } #[test] fn split_mode_has_named_proxy_location() { let names = vec!["iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); // Named location for dynamic proxy assert!( body.contains("location @iris_dynamic {"), "expected named dynamic location, got:\n{body}" ); assert!(body.contains("proxy_pass")); assert!(body.contains("proxy_intercept_errors on")); assert!(body.contains("__hive_agent_unreachable")); } #[test] fn split_mode_named_location_strips_prefix_without_uri_part() { // Named locations forbid a URI part on proxy_pass — nginx rejects // `proxy_pass http://host/` inside `location @name`. Must use // bare upstream (no trailing `/` or path) plus a `rewrite` to // strip the /agent// prefix. let names = vec!["iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); assert!( body.contains("rewrite ^/agent/iris/(.*)$ /$1 break;"), "expected prefix-strip rewrite, got:\n{body}" ); // The named-location proxy_pass must have no URI part (no // trailing slash or path). Extract the `@iris_dynamic { ... }` // block and check every proxy_pass directive in it. let block_start = body .find("location @iris_dynamic {") .expect("named location"); let block = &body[block_start..]; let block_end = block.find("\n}\n").expect("block close"); let block = &block[..block_end]; for line in block.lines() { let line = line.trim(); if let Some(rest) = line.strip_prefix("proxy_pass ") { let target = rest.trim_end_matches(';'); // No URI part means: TCP form `http://host:port` (no // trailing `/`), UDS form `http://unix:/path:` (trailing // colon, nothing after). Both cases: must not end in `/`. assert!( !target.ends_with('/'), "named-location proxy_pass must have no URI part, got: {target}" ); } } } #[test] fn split_mode_includes_manager() { let names: Vec = [MANAGER_NAME, "iris"] .iter() .map(|s| (*s).to_owned()) .collect(); let body = render(&names, Some(FAKE_FRONTEND)); assert!(body.contains(&format!("/agent/{MANAGER_NAME}/"))); assert!(body.contains("/agent/iris/")); } #[test] fn split_mode_proxy_includes_forwarded_prefix() { let names = vec!["atlas".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); assert!( body.contains("X-Forwarded-Prefix /agent/atlas"), "expected X-Forwarded-Prefix for atlas in split mode, got:\n{body}" ); } #[test] fn split_mode_named_location_uses_agent_name() { // Named location must be unique per agent to avoid nginx conflicts let names = vec!["damocles".to_owned(), "iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); assert!(body.contains("@damocles_dynamic")); assert!(body.contains("@iris_dynamic")); // Each agent gets exactly one named dynamic location block. let damocles_count = body.matches("@damocles_dynamic {").count(); let iris_count = body.matches("@iris_dynamic {").count(); assert_eq!( damocles_count, 1, "expected exactly one damocles_dynamic block" ); assert_eq!(iris_count, 1, "expected exactly one iris_dynamic block"); } }