//! Runtime nginx include-file generator for the gateway's per-agent //! `/agent//` location blocks. Writes //! `/var/lib/hyperhive/gateway/agents.conf` on every topology change. //! UDS vs TCP upstream selection, reload trigger (`systemd-run //! --machine=hive-gateway`), and idempotency: //! `docs/gateway.md::Per-agent unix-socket upstream`. use anyhow::{Context, Result}; use std::fmt::Write as _; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use crate::agent_sockets; use crate::lifecycle; /// Set when `write` publishes a new agents.conf; cleared when /// `reload_gateway_nginx` submits the reload command successfully. /// Lets `spawn_poll` retry the reload on subsequent ticks when the /// previous attempt failed (e.g. gateway container temporarily down, /// systemd-run not found) without re-writing the already-correct file. static RELOAD_PENDING: AtomicBool = AtomicBool::new(false); const HOST_CONF_PATH: &str = "/var/lib/hyperhive/gateway/agents.conf"; /// Host-side path where c0re writes the generated nginx include file. /// The gateway container bind-mounts `/var/lib/hyperhive/gateway/` /// (not the whole parent dir) at `/run/hive-state/` so nginx inside /// can read it at `/run/hive-state/agents.conf`. Subdirectory scoping /// avoids exposing the rest of `/var/lib/hyperhive/` (which may contain /// forge tokens or other credentials) to the gateway container. #[must_use] pub fn host_conf_path() -> PathBuf { PathBuf::from(HOST_CONF_PATH) } /// Nginx proxy headers present in every per-agent location block. /// `$connection_upgrade` is defined in the http context by the NixOS /// nginx module when `recommendedProxySettings = true` (which the /// gateway always sets). `X-Forwarded-Prefix` is per-location (set /// below in the render loop). Kept as a standalone const so it's /// visible to tests without repeating the text. const PROXY_HEADER_BLOCK: &str = " proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme;"; /// Render the nginx include body for `names`. Manager is included so /// `/agent/root/` is routable through the gateway. /// /// When `frontend_dir` is `Some(path)` each agent gets split location /// blocks: /// - Exact HTML pages (`/`, `/stats`, `/screen`) → `alias` from the nix /// store dist. These load even while the agent daemon is restarting. /// - Static assets (`/static/`) → `alias` from the nix store with cache /// headers. Served by nginx directly, no agent socket round-trip. /// - Everything else (all `/api/*`, `/events/*`, `/screen/ws`, `/icon`, /// `/send`, `/login/*`) → proxied to the agent daemon as before. /// /// When `frontend_dir` is `None` (legacy) each agent gets a single /// `location /agent//` proxy block. /// /// Output is deterministic for the same (sorted) set of names + /// `.bound` state: no timestamps, no UUIDs. `BTreeMap` would give /// alphabetical order; we rely on the caller to pass sorted names if /// they care about diff stability, but the gateway treats the blocks /// as unordered by nginx's longest-prefix-match rules so ordering only /// affects human readability. fn render(names: &[String], frontend_dir: Option<&str>) -> String { let mut out = String::from( "# Generated by hive-c0re \u{2014} do not edit.\ \n# Refreshed on every topology change + when agents bind/drop their unix sockets.\ \n# Reload triggered by hive-c0re via systemd-run --machine=hive-gateway.\n", ); for name in names { let port = lifecycle::agent_web_port(name); // Two upstream forms because named locations (split mode's // `@_dynamic`) forbid a URI part on `proxy_pass`. The // legacy prefix-location path keeps the trailing `/` so nginx // strips `/agent//` automatically; the named-location // path strips the prefix via `rewrite` and uses a bare upstream. let (upstream_prefix, upstream_bare) = if agent_sockets::ready_marker_for(name).exists() { let sock = agent_sockets::socket_path_for(name).display().to_string(); ( format!("http://unix:{sock}:/"), format!("http://unix:{sock}:"), ) } else { ( format!("http://127.0.0.1:{port}/"), format!("http://127.0.0.1:{port}"), ) }; if let Some(frontend) = frontend_dir { // Split mode: try to serve files from the nix-store dist first; // fall through to the agent daemon for anything not found there. // // `frontend` is a nix store path injected at build time by hive-c0re.nix // (`HIVE_AGENT_FRONTEND_DIR = "${cfg.frontend}/agent"`). Nix store paths // are of the form `/nix/store/-` — only [a-z0-9/._-], never // shell metacharacters — so interpolating directly into the nginx config // string is safe. We do NOT accept user-controlled input here. // // Location priority for a request to /agent//...: // ^~ /agent//static/ — highest priority; matches compiled assets. // Nix store paths are content-addressed and immutable — cache forever. // /agent// — try_files from dist, proxy fallback for the rest. // // try_files resolution (nginx applies the alias mapping before checking): // $uri — exact file match (/static/app.js → static/app.js) // $uri.html — bare-path .html fallback (/stats → stats.html) // $uri/index.html — directory index (/ → index.html) // @_dynamic — proxy catchall: api, events, icon, login, … // // Adding pages to the frontend dist works automatically — no generator // change needed. Per-agent extraFiles (in mergedDist, not in the base // dist path) still proxy to the agent daemon via @_dynamic. let _ = write!( out, "\n# Compiled JS/CSS assets — content-addressed nix store path, cache forever\n\ location ^~ /agent/{name}/static/ {{\n\ \n alias {frontend}/static/;\n\ \n expires 1y;\n\ \n add_header Cache-Control \"public, immutable, max-age=31536000\";\n\ }}\n\ \n# Static dist + proxy fallback for dynamic paths\n\ location /agent/{name}/ {{\n\ \n alias {frontend}/;\n\ \n try_files $uri $uri.html $uri/index.html @{name}_dynamic;\n\ }}\n\ \nlocation @{name}_dynamic {{\n\ {PROXY_HEADER_BLOCK}\n\ \n rewrite ^/agent/{name}/(.*)$ /$1 break;\n\ \n proxy_pass {upstream_bare};\n\ \n proxy_set_header X-Forwarded-Prefix /agent/{name};\n\ \n proxy_buffering off;\n\ \n proxy_read_timeout 1d;\n\ \n proxy_intercept_errors on;\n\ \n error_page 502 503 504 = /__hive_agent_unreachable;\n\ }}\n", ); } else { // Legacy mode: proxy everything to the agent daemon. let _ = write!( out, "\nlocation /agent/{name}/ {{\n\ {PROXY_HEADER_BLOCK}\n\ \n proxy_pass {upstream_prefix};\n\ \n proxy_set_header X-Forwarded-Prefix /agent/{name};\n\ \n proxy_buffering off;\n\ \n proxy_read_timeout 1d;\n\ \n proxy_intercept_errors on;\n\ \n error_page 502 503 504 = /__hive_agent_unreachable;\n\ }}\n", ); } } out } /// Atomically write the nginx include file for `names` to /// [`host_conf_path()`]. Skips the write + reload when the rendered /// body matches what's already on disk (idempotent; avoids spurious /// gateway reloads on a quiet tick). /// /// After a successful write, triggers an nginx reload inside the /// gateway container from the HOST side via /// `systemd-run --machine=hive-gateway nginx -s reload`. This is /// intentionally host-side rather than relying on a systemd path unit /// inside the container watching the bind-mounted file: `IN_MOVED_TO` /// (fired by the atomic rename) does not reliably propagate across the /// nspawn mount-namespace boundary, so the path-unit approach was /// silently broken (see `docs/gateway.md` for the failure analysis). /// /// The `systemd-run` call is best-effort — a failed reload is logged /// but not fatal. nginx will pick up the new include on its next /// housekeeping restart or the next manual reload; the host's agent /// topology has already been written correctly. pub fn write(names: &[String]) -> Result<()> { let frontend_dir = std::env::var("HIVE_AGENT_FRONTEND_DIR") .ok() .filter(|s| !s.is_empty()); let body = render(names, frontend_dir.as_deref()); let path = host_conf_path(); if std::fs::read_to_string(&path).ok().as_deref() == Some(&body) { return Ok(()); } if let Some(parent) = path.parent() { std::fs::create_dir_all(parent) .with_context(|| format!("create dir {}", parent.display()))?; } let tmp = path.with_extension("conf.tmp"); std::fs::write(&tmp, &body).with_context(|| format!("write tmp {}", tmp.display()))?; std::fs::rename(&tmp, &path).with_context(|| { format!( "rename {} -> {} (atomic publish)", tmp.display(), path.display() ) })?; // Mark reload pending before attempting so a failed attempt is // retried by the next spawn_poll tick (see `reload_if_pending`). RELOAD_PENDING.store(true, Ordering::Relaxed); reload_gateway_nginx(); Ok(()) } /// Retry a pending nginx reload if a previous attempt failed. /// Called by `spawn_poll` on each tick so a transient failure /// (gateway container temporarily down, systemd-run error) is /// recovered automatically without requiring a new file write. pub fn reload_if_pending() { if RELOAD_PENDING.load(Ordering::Relaxed) { reload_gateway_nginx(); } } /// Query the nginx unit's `ActiveState` inside the gateway container. /// Returns the raw state string from `systemctl show --property=ActiveState /// --value` (e.g. `"active"`, `"failed"`, `"inactive"`, `"activating"`). /// Returns `"unknown"` on any error so callers can branch safely. fn nginx_active_state() -> String { let out = std::process::Command::new("systemctl") .args([ "--machine=hive-gateway", "show", "--property=ActiveState", "--value", "nginx", ]) .output(); match out { Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).trim().to_owned(), Ok(o) => { tracing::warn!( exit_code = ?o.status.code(), "systemctl show ActiveState exited non-zero" ); "unknown".to_owned() } Err(e) => { tracing::warn!(error = %e, "systemctl show ActiveState failed"); "unknown".to_owned() } } } /// Send a `systemctl --machine=hive-gateway ` command and /// return whether it succeeded. Best-effort: errors are logged. fn gateway_systemctl(args: &[&str]) -> bool { let mut cmd = std::process::Command::new("systemctl"); cmd.arg("--machine=hive-gateway"); cmd.args(args); match cmd.status() { Ok(s) if s.success() => true, Ok(s) => { tracing::warn!( args = ?args, exit_code = ?s.code(), "gateway systemctl exited non-zero" ); false } Err(e) => { tracing::warn!(args = ?args, error = %e, "gateway systemctl invocation failed"); false } } } /// Synchronise the gateway nginx unit with the current agents.conf: /// /// - **active**: send `nginx -s reload` (SIGHUP to master, zero-downtime /// worker replacement). Keeps RELOAD_PENDING set on failure so the /// next poll tick retries. /// - **failed / start-limit-hit**: run `systemctl reset-failed nginx` /// then `systemctl start nginx`. This is the self-healing path: a /// transient bad agents.conf that causes five instant `nginx -t` /// failures trips systemd's start-limit. Once c0re publishes a correct /// config, the next reload attempt clears the failure and restarts. /// - **inactive / other**: run `systemctl start nginx` directly (no /// reset-failed needed when the unit isn't in a failed state). /// /// `RELOAD_PENDING` is cleared only after a successful operation so /// `reload_if_pending` keeps retrying on failure. fn reload_gateway_nginx() { let state = nginx_active_state(); let success = match state.as_str() { "active" => { // nginx master is running — SIGHUP is the zero-downtime path. // `systemd-run --machine=hive-gateway --quiet --wait -- nginx // -s reload` runs the signal inside the container and exits // with the nginx exit code. `--` separates systemd-run flags // from the command. let status = std::process::Command::new("systemd-run") .args([ "--machine=hive-gateway", "--quiet", "--wait", "--", "nginx", "-s", "reload", ]) .status(); match status { Ok(s) if s.success() => { tracing::debug!("gateway nginx reload signal sent"); true } Ok(s) => { tracing::warn!( exit_code = ?s.code(), "gateway nginx reload exited non-zero — will retry next poll tick" ); false } Err(e) => { tracing::warn!( error = %e, "failed to invoke systemd-run for gateway nginx reload — will retry" ); false } } } "failed" => { // Unit hit start-limit (e.g. repeated nginx -t failures from // a bad agents.conf). reset-failed clears the rate-limit so // start can proceed. tracing::info!("gateway nginx unit in failed state — resetting and starting"); gateway_systemctl(&["reset-failed", "nginx"]) && gateway_systemctl(&["start", "nginx"]) } other => { // inactive, deactivating, activating, unknown — just try start. tracing::info!(state = other, "gateway nginx unit not active — starting"); gateway_systemctl(&["start", "nginx"]) } }; if success { RELOAD_PENDING.store(false, Ordering::Relaxed); } } #[cfg(test)] mod tests { use super::*; use crate::lifecycle::MANAGER_NAME; // ── legacy mode (frontend_dir = None) ────────────────────────────── #[test] fn render_empty_is_header_only() { let body = render(&[], None); assert!(body.starts_with("# Generated by hive-c0re")); // No location blocks when no agents. assert!(!body.contains("location")); } #[test] fn render_includes_manager() { let names: Vec = [MANAGER_NAME, "iris"] .iter() .map(|s| (*s).to_owned()) .collect(); let body = render(&names, None); assert!(body.contains(&format!("/agent/{MANAGER_NAME}/"))); assert!(body.contains("/agent/iris/")); } #[test] fn render_tcp_upstream_when_no_bound_marker() { // No .bound file on disk → falls back to TCP loopback. let names = vec!["iris".to_owned()]; let body = render(&names, None); let port = lifecycle::agent_web_port("iris"); assert!( body.contains(&format!("proxy_pass http://127.0.0.1:{port}/")), "expected TCP upstream for iris, got:\n{body}" ); } #[test] fn render_includes_proxy_headers() { let names = vec!["iris".to_owned()]; let body = render(&names, None); // All required proxy headers present (WebSocket upgrade + forwarded-for). assert!(body.contains("proxy_http_version 1.1")); assert!(body.contains("proxy_set_header Upgrade $http_upgrade")); assert!(body.contains("proxy_set_header X-Forwarded-For")); } #[test] fn render_includes_forwarded_prefix() { let names = vec!["atlas".to_owned()]; let body = render(&names, None); assert!( body.contains("X-Forwarded-Prefix /agent/atlas"), "expected X-Forwarded-Prefix for atlas, got:\n{body}" ); } #[test] fn render_includes_error_pages() { let names = vec!["iris".to_owned()]; let body = render(&names, None); assert!(body.contains("proxy_intercept_errors on")); assert!(body.contains("__hive_agent_unreachable")); } #[test] fn render_includes_buffering_and_timeout() { let names = vec!["iris".to_owned()]; let body = render(&names, None); assert!(body.contains("proxy_buffering off")); assert!(body.contains("proxy_read_timeout 1d")); } // ── split mode (frontend_dir = Some) ─────────────────────────────── const FAKE_FRONTEND: &str = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-frontend/agent"; #[test] fn split_mode_has_immutable_cache_for_static_assets() { let names = vec!["iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); // ^~ prefix block for /static/ with immutable cache headers assert!( body.contains("location ^~ /agent/iris/static/ {"), "expected ^~ static location, got:\n{body}" ); assert!(body.contains(&format!("alias {FAKE_FRONTEND}/static/;"))); assert!(body.contains("expires 1y;")); assert!(body.contains("immutable")); } #[test] fn split_mode_has_alias_and_try_files() { let names = vec!["iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); // Plain prefix location with alias + try_files assert!( body.contains("location /agent/iris/ {"), "expected try_files location, got:\n{body}" ); assert!( body.contains(&format!("alias {FAKE_FRONTEND}/;")), "expected alias, got:\n{body}" ); // try_files handles exact file, bare-path .html, directory index assert!( body.contains("try_files $uri $uri.html $uri/index.html @iris_dynamic"), "expected try_files with .html fallback, got:\n{body}" ); } #[test] fn split_mode_has_named_proxy_location() { let names = vec!["iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); // Named location for dynamic proxy assert!( body.contains("location @iris_dynamic {"), "expected named dynamic location, got:\n{body}" ); assert!(body.contains("proxy_pass")); assert!(body.contains("proxy_intercept_errors on")); assert!(body.contains("__hive_agent_unreachable")); } #[test] fn split_mode_named_location_strips_prefix_without_uri_part() { // Named locations forbid a URI part on proxy_pass — nginx rejects // `proxy_pass http://host/` inside `location @name`. Must use // bare upstream (no trailing `/` or path) plus a `rewrite` to // strip the /agent// prefix. let names = vec!["iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); assert!( body.contains("rewrite ^/agent/iris/(.*)$ /$1 break;"), "expected prefix-strip rewrite, got:\n{body}" ); // The named-location proxy_pass must have no URI part (no // trailing slash or path). Extract the `@iris_dynamic { ... }` // block and check every proxy_pass directive in it. let block_start = body .find("location @iris_dynamic {") .expect("named location"); let block = &body[block_start..]; let block_end = block.find("\n}\n").expect("block close"); let block = &block[..block_end]; for line in block.lines() { let line = line.trim(); if let Some(rest) = line.strip_prefix("proxy_pass ") { let target = rest.trim_end_matches(';'); // No URI part means: TCP form `http://host:port` (no // trailing `/`), UDS form `http://unix:/path:` (trailing // colon, nothing after). Both cases: must not end in `/`. assert!( !target.ends_with('/'), "named-location proxy_pass must have no URI part, got: {target}" ); } } } #[test] fn split_mode_includes_manager() { let names: Vec = [MANAGER_NAME, "iris"] .iter() .map(|s| (*s).to_owned()) .collect(); let body = render(&names, Some(FAKE_FRONTEND)); assert!(body.contains(&format!("/agent/{MANAGER_NAME}/"))); assert!(body.contains("/agent/iris/")); } #[test] fn split_mode_proxy_includes_forwarded_prefix() { let names = vec!["atlas".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); assert!( body.contains("X-Forwarded-Prefix /agent/atlas"), "expected X-Forwarded-Prefix for atlas in split mode, got:\n{body}" ); } #[test] fn split_mode_named_location_uses_agent_name() { // Named location must be unique per agent to avoid nginx conflicts let names = vec!["damocles".to_owned(), "iris".to_owned()]; let body = render(&names, Some(FAKE_FRONTEND)); assert!(body.contains("@damocles_dynamic")); assert!(body.contains("@iris_dynamic")); // Each agent gets exactly one named dynamic location block. let damocles_count = body.matches("@damocles_dynamic {").count(); let iris_count = body.matches("@iris_dynamic {").count(); assert_eq!( damocles_count, 1, "expected exactly one damocles_dynamic block" ); assert_eq!(iris_count, 1, "expected exactly one iris_dynamic block"); } }