hyperhive/hive-c0re/src/gateway_nginx.rs
iris 9ea6160c94 sweep: wire gateway-nginx reload + config-repo branch-protection into warning banners
Extends the SweepHealth/warnings registry (already landed for
knowledge_pull) to two more background sweeps:

- gateway_nginx::reload_gateway_nginx: raises a warn-level banner
  immediately on the first failed reload (routing changes silently
  not taking effect is user-visible right now, so no debounce).
- forge::repos::ensure_config_repo: raises a crit-level banner
  listing every agent whose config-repo branch protection is
  currently unapplied (security-relevant — bypasses the deploy
  pipeline), clearing agents out of the message as they recover.

Journal warn!/error! logging is left in place; the registry adds a
dashboard-visible signal on top. forge::ensure_all() and
matrix::ensure_all() sweeps are deliberately left for a fast-follow.
2026-07-16 00:05:29 +02:00

585 lines
25 KiB
Rust

//! Runtime nginx include-file generator for the gateway's per-agent
//! `/agent/<name>/` location blocks. Writes
//! `/var/lib/hyperhive/gateway/agents.conf` on every topology change.
//! UDS upstream selection, reload trigger (`systemd-run
//! --machine=hive-gateway`), and idempotency:
//! `docs/gateway.md::Per-agent unix-socket upstream`.
use anyhow::{Context, Result};
use std::fmt::Write as _;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{SystemTime, UNIX_EPOCH};
use crate::priv_client;
use crate::stats::sweep_health::SweepHealth;
use crate::agent_sockets;
/// Set when `write` publishes a new agents.conf; cleared when
/// `reload_gateway_nginx` submits the reload command successfully.
/// Lets `spawn_poll` retry the reload on subsequent ticks when the
/// previous attempt failed (e.g. gateway container temporarily down,
/// systemd-run not found) without re-writing the already-correct file.
static RELOAD_PENDING: AtomicBool = AtomicBool::new(false);
/// Unix timestamp (seconds) of the last failed reload attempt.
/// `reload_if_pending` backs off to once per `RELOAD_RETRY_SECS` after a
/// failure so a permanently broken gateway (bad config, container down)
/// doesn't hammer `systemctl` on every 10-second `spawn_poll` tick.
static LAST_FAILED_RELOAD: AtomicU64 = AtomicU64::new(0);
/// Minimum gap between retry attempts after a reload failure (30 s).
const RELOAD_RETRY_SECS: u64 = 30;
/// Dashboard-banner health tracker for the reload sweep — raises a
/// `gateway_nginx_reload` warning the first time a reload fails (routing is
/// broken for whoever depends on the change *right now*, so no debounce
/// window) and clears it the moment a reload succeeds again.
fn health() -> &'static Mutex<SweepHealth> {
static HEALTH: OnceLock<Mutex<SweepHealth>> = OnceLock::new();
HEALTH.get_or_init(|| Mutex::new(SweepHealth::new("gateway_nginx_reload", "warn", 1)))
}
/// Nginx proxy headers present in every per-agent location block.
/// `$connection_upgrade` is defined in the http context by the NixOS
/// nginx module when `recommendedProxySettings = true` (which the
/// gateway always sets). `X-Forwarded-Prefix` is per-location (set
/// below in the render loop). Kept as a standalone const so it's
/// visible to tests without repeating the text.
const PROXY_HEADER_BLOCK: &str = " proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;";
/// Render the nginx include body for `names`. Manager is included so
/// `/agent/root/` is routable through the gateway.
///
/// When `frontend_dir` is `Some(path)` each agent gets split location
/// blocks:
/// - Exact HTML pages (`/`, `/stats`, `/screen`) → `alias` from the nix
/// store dist. These load even while the agent daemon is restarting.
/// - Static assets (`/static/`) → `alias` from the nix store with cache
/// headers. Served by nginx directly, no agent socket round-trip.
/// - Everything else (all `/api/*`, `/events/*`, `/screen/ws`, `/icon`,
/// `/send`, `/login/*`) → proxied to the agent daemon as before.
///
/// When `frontend_dir` is `None` (legacy) each agent gets a single
/// `location /agent/<name>/` proxy block.
///
/// Output is deterministic for the same (sorted) set of names:
/// no timestamps, no UUIDs. `BTreeMap` would give alphabetical order;
/// we rely on the caller to pass sorted names if they care about diff
/// stability, but the gateway treats the blocks as unordered by nginx's
/// longest-prefix-match rules so ordering only affects human readability.
fn render(names: &[String], frontend_dir: Option<&str>) -> String {
let mut out = String::from(
"# Generated by hive-c0re \u{2014} do not edit.\
\n# Refreshed on every topology change + when agents bind/drop their unix sockets.\
\n# Reload triggered by hive-c0re via systemd-run --machine=hive-gateway.\n",
);
for name in names {
// Two upstream forms because named locations (split mode's
// `@<name>_dynamic`) forbid a URI part on `proxy_pass`. The
// legacy prefix-location path keeps the trailing `/` so nginx
// strips `/agent/<name>/` automatically; the named-location
// path strips the prefix via `rewrite` and uses a bare upstream.
// When the socket is not yet bound, nginx returns 502, caught by
// the error_page directive below.
let sock = agent_sockets::socket_path_for(name).display().to_string();
let upstream_prefix = format!("http://unix:{sock}:/");
let upstream_bare = format!("http://unix:{sock}:");
if let Some(frontend) = frontend_dir {
// Split mode: try to serve files from the nix-store dist first;
// fall through to the agent daemon for anything not found there.
//
// `frontend` is a nix store path injected at build time by hive-c0re.nix
// (`HIVE_AGENT_FRONTEND_DIR = "${cfg.frontend}/agent"`). Nix store paths
// are of the form `/nix/store/<hash>-<name>` — only [a-z0-9/._-], never
// shell metacharacters — so interpolating directly into the nginx config
// string is safe. We do NOT accept user-controlled input here.
//
// Location priority for a request to /agent/<name>/...:
// ^~ /agent/<name>/static/ — highest priority; matches compiled assets.
// Nix store paths are content-addressed and immutable — cache forever.
// /agent/<name>/ — try_files from dist, proxy fallback for the rest.
//
// try_files resolution (nginx applies the alias mapping before checking):
// $uri — exact file match (/static/app.js → static/app.js)
// $uri.html — bare-path .html fallback (/stats → stats.html)
// $uri/index.html — directory index (/ → index.html)
// @<name>_dynamic — proxy catchall: api, events, icon, login, …
//
// Adding pages to the frontend dist works automatically — no generator
// change needed. Per-agent extraFiles (in mergedDist, not in the base
// dist path) still proxy to the agent daemon via @<name>_dynamic.
let _ = write!(
out,
"\n# Compiled JS/CSS assets — content-addressed nix store path, cache forever\n\
location ^~ /agent/{name}/static/ {{\n\
\n alias {frontend}/static/;\n\
\n expires 1y;\n\
\n add_header Cache-Control \"public, immutable, max-age=31536000\";\n\
}}\n\
\n# Static dist + proxy fallback for dynamic paths\n\
location /agent/{name}/ {{\n\
\n alias {frontend}/;\n\
\n try_files $uri $uri.html $uri/index.html @{name}_dynamic;\n\
}}\n\
\nlocation @{name}_dynamic {{\n\
{PROXY_HEADER_BLOCK}\n\
\n rewrite ^/agent/{name}/(.*)$ /$1 break;\n\
\n proxy_pass {upstream_bare};\n\
\n proxy_set_header X-Forwarded-Prefix /agent/{name};\n\
\n proxy_buffering off;\n\
\n proxy_read_timeout 1d;\n\
\n proxy_intercept_errors on;\n\
\n error_page 502 503 504 = /__hive_agent_unreachable;\n\
}}\n",
);
} else {
// Legacy mode: proxy everything to the agent daemon.
let _ = write!(
out,
"\nlocation /agent/{name}/ {{\n\
{PROXY_HEADER_BLOCK}\n\
\n proxy_pass {upstream_prefix};\n\
\n proxy_set_header X-Forwarded-Prefix /agent/{name};\n\
\n proxy_buffering off;\n\
\n proxy_read_timeout 1d;\n\
\n proxy_intercept_errors on;\n\
\n error_page 502 503 504 = /__hive_agent_unreachable;\n\
}}\n",
);
}
}
out
}
/// Atomically write the nginx include file for `names` to
/// [`crate::paths::gateway_agents_conf()`]. Skips the write + reload when the rendered
/// body matches what's already on disk (idempotent; avoids spurious
/// gateway reloads on a quiet tick).
///
/// After a successful write, triggers the appropriate nginx action inside
/// the gateway container via `hive-priv` (which has the
/// `--machine=hive-gateway` transport rights hive-c0re lacks):
/// reload when nginx is active, reset-failed+start when in a failed
/// state, plain start otherwise. This is intentionally host-side rather
/// than relying on a systemd path unit inside the container watching the
/// bind-mounted file: `IN_MOVED_TO` (fired by the atomic rename) does
/// not reliably propagate across the nspawn mount-namespace boundary, so
/// the path-unit approach was silently broken (see `docs/gateway.md`).
///
/// The priv call is best-effort — a failed sync is logged but not fatal.
/// `reload_if_pending` retries on the next `spawn_poll` tick so a
/// transient gateway-down situation converges without manual intervention.
pub async fn write(names: &[String]) -> Result<()> {
let frontend_dir = std::env::var("HIVE_AGENT_FRONTEND_DIR")
.ok()
.filter(|s| !s.is_empty());
let body = render(names, frontend_dir.as_deref());
let path = crate::paths::gateway_agents_conf();
if std::fs::read_to_string(&path).ok().as_deref() == Some(&body) {
return Ok(());
}
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("create dir {}", parent.display()))?;
}
let tmp = path.with_extension("conf.tmp");
std::fs::write(&tmp, &body).with_context(|| format!("write tmp {}", tmp.display()))?;
std::fs::rename(&tmp, &path).with_context(|| {
format!(
"rename {} -> {} (atomic publish)",
tmp.display(),
path.display()
)
})?;
// Reset backoff so a fresh topology change gets an immediate attempt,
// not a stale cooldown from a previous failure.
LAST_FAILED_RELOAD.store(0, Ordering::Relaxed);
// Mark reload pending before attempting so a failed attempt is
// retried by the next spawn_poll tick (see `reload_if_pending`).
RELOAD_PENDING.store(true, Ordering::Relaxed);
reload_gateway_nginx().await;
Ok(())
}
/// Retry a pending nginx reload if a previous attempt failed.
/// Called by `spawn_poll` on each tick so a transient failure
/// (gateway container temporarily down, systemd-run error) is
/// recovered automatically without requiring a new file write.
///
/// Backs off to one retry per `RELOAD_RETRY_SECS` after a failure so a
/// permanently broken gateway doesn't hammer `systemctl` on every tick.
/// A fresh `write()` call always resets the backoff (new `RELOAD_PENDING`
/// set to `true` + immediate attempt) so topology changes are still
/// applied promptly.
pub async fn reload_if_pending() {
if !RELOAD_PENDING.load(Ordering::Relaxed) {
return;
}
let last_failed = LAST_FAILED_RELOAD.load(Ordering::Relaxed);
if last_failed > 0 {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
if now.saturating_sub(last_failed) < RELOAD_RETRY_SECS {
return;
}
}
reload_gateway_nginx().await;
}
/// Synchronise the gateway nginx unit with the current agents.conf via
/// `hive-priv` (privileged helper). The state-aware logic (active →
/// reload; failed → reset-failed + start; inactive/unknown → start)
/// runs inside hive-priv where it has the `--machine=hive-gateway`
/// transport rights that hive-c0re (unprivileged) lacks.
///
/// `RELOAD_PENDING` is cleared only after a successful operation so
/// `reload_if_pending` keeps retrying on failure.
async fn reload_gateway_nginx() {
match priv_client::reload_gateway_nginx().await {
Ok(()) => {
tracing::debug!("gateway nginx sync succeeded");
RELOAD_PENDING.store(false, Ordering::Relaxed);
LAST_FAILED_RELOAD.store(0, Ordering::Relaxed);
if let Ok(mut h) = health().lock() {
h.record_ok();
}
}
Err(e) => {
tracing::warn!(error = %e, "gateway nginx sync failed — will retry");
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
LAST_FAILED_RELOAD.store(now, Ordering::Relaxed);
if let Ok(mut h) = health().lock() {
let err = format!("{e:#}");
h.record_err(|ctx| {
let age = ctx.since_last_ok.map_or_else(
|| "no success this process".to_owned(),
|d| format!("last ok {} ago", crate::stats::sweep_health::fmt_age(d)),
);
format!(
"gateway nginx reload failing ({} consecutive, {age}) \
— routing changes are not taking effect: {err}",
ctx.consecutive
)
});
}
}
}
}
// ── Gateway HTTP-Basic (htpasswd) user management ──────────────────────────
// The daemon owns the write (hivectl drives it over the host socket via the
// `Gateway*User` requests, so hivectl never touches the credential file).
// Keyed on the canonical `paths::GATEWAY_HTPASSWD` — a socket client doesn't
// pick the path.
fn htpasswd_path() -> std::path::PathBuf {
std::path::PathBuf::from(crate::paths::GATEWAY_HTPASSWD)
}
/// Read the htpasswd file into lines, or an empty list if it doesn't exist.
fn htpasswd_read() -> Result<Vec<String>> {
let path = htpasswd_path();
if !path.exists() {
return Ok(vec![]);
}
let content = std::fs::read_to_string(&path)
.with_context(|| format!("read htpasswd file {}", path.display()))?;
Ok(content.lines().map(str::to_owned).collect())
}
/// Write lines back atomically (`<path>.tmp` then rename), with a trailing
/// newline.
fn htpasswd_write(lines: &[String]) -> Result<()> {
let path = htpasswd_path();
let tmp = path.with_extension("htpasswd.tmp");
let content = if lines.is_empty() {
String::new()
} else {
let mut s = lines.join("\n");
s.push('\n');
s
};
std::fs::write(&tmp, &content)
.with_context(|| format!("write htpasswd tmp {}", tmp.display()))?;
std::fs::rename(&tmp, &path)
.with_context(|| format!("rename {}{}", tmp.display(), path.display()))?;
Ok(())
}
fn validate_htpasswd_username(username: &str) -> Result<()> {
if username.is_empty() {
anyhow::bail!("username must not be empty");
}
if username.contains(':') {
anyhow::bail!("username must not contain ':' (htpasswd field separator)");
}
if username.chars().any(char::is_control) {
anyhow::bail!("username must not contain control characters");
}
Ok(())
}
/// Add or update a gateway HTTP-Basic user, bcrypt-hashing `password`
/// (cost 12). Returns the confirmation line for the operator.
pub fn create_user(username: &str, password: &str) -> Result<String> {
validate_htpasswd_username(username)?;
let raw_hash = bcrypt::hash(password, 12).context("bcrypt hash")?;
// nginx auth_basic only recognises $2a$/$2x$/$2y$ — not $2b$. The prefixes
// are algorithmically identical; remap so nginx accepts the hash.
let hash = raw_hash.replacen("$2b$", "$2y$", 1);
let entry = format!("{username}:{hash}");
let mut lines = htpasswd_read()?;
let prefix = format!("{username}:");
let msg = if let Some(pos) = lines.iter().position(|l| l.starts_with(&prefix)) {
lines[pos] = entry;
format!("gateway: updated password for '{username}'")
} else {
lines.push(entry);
format!("gateway: added user '{username}'")
};
htpasswd_write(&lines)?;
Ok(msg)
}
/// Remove a gateway HTTP-Basic user. Errors when the user isn't present so a
/// no-op is detectable.
pub fn delete_user(username: &str) -> Result<String> {
let mut lines = htpasswd_read()?;
let prefix = format!("{username}:");
let before = lines.len();
lines.retain(|l| !l.starts_with(&prefix));
if lines.len() == before {
anyhow::bail!("gateway: user '{username}' not found");
}
htpasswd_write(&lines)?;
Ok(format!("gateway: removed user '{username}'"))
}
/// List the gateway HTTP-Basic usernames (skips blank / comment lines).
pub fn list_users() -> Result<Vec<String>> {
Ok(htpasswd_read()?
.iter()
.filter(|l| !l.is_empty() && !l.starts_with('#'))
.filter_map(|l| l.split_once(':').map(|(name, _)| name.to_owned()))
.collect())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::lifecycle::MANAGER_NAME;
// ── legacy mode (frontend_dir = None) ──────────────────────────────
#[test]
fn render_empty_is_header_only() {
let body = render(&[], None);
assert!(body.starts_with("# Generated by hive-c0re"));
// No location blocks when no agents.
assert!(!body.contains("location"));
}
#[test]
fn render_includes_manager() {
let names: Vec<String> = [MANAGER_NAME, "iris"]
.iter()
.map(|s| (*s).to_owned())
.collect();
let body = render(&names, None);
assert!(body.contains(&format!("/agent/{MANAGER_NAME}/")));
assert!(body.contains("/agent/iris/"));
}
#[test]
fn render_uds_upstream_unconditional() {
let names = vec!["iris".to_owned()];
let body = render(&names, None);
// UDS form present
assert!(
body.contains("proxy_pass http://unix:"),
"expected UDS upstream for iris, got:\n{body}"
);
// No TCP loopback fallback
assert!(
!body.contains("127.0.0.1"),
"unexpected TCP loopback in output:\n{body}"
);
}
#[test]
fn render_includes_proxy_headers() {
let names = vec!["iris".to_owned()];
let body = render(&names, None);
// All required proxy headers present (WebSocket upgrade + forwarded-for).
assert!(body.contains("proxy_http_version 1.1"));
assert!(body.contains("proxy_set_header Upgrade $http_upgrade"));
assert!(body.contains("proxy_set_header X-Forwarded-For"));
}
#[test]
fn render_includes_forwarded_prefix() {
let names = vec!["atlas".to_owned()];
let body = render(&names, None);
assert!(
body.contains("X-Forwarded-Prefix /agent/atlas"),
"expected X-Forwarded-Prefix for atlas, got:\n{body}"
);
}
#[test]
fn render_includes_error_pages() {
let names = vec!["iris".to_owned()];
let body = render(&names, None);
assert!(body.contains("proxy_intercept_errors on"));
assert!(body.contains("__hive_agent_unreachable"));
}
#[test]
fn render_includes_buffering_and_timeout() {
let names = vec!["iris".to_owned()];
let body = render(&names, None);
assert!(body.contains("proxy_buffering off"));
assert!(body.contains("proxy_read_timeout 1d"));
}
// ── split mode (frontend_dir = Some) ───────────────────────────────
const FAKE_FRONTEND: &str =
"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-frontend/agent";
#[test]
fn split_mode_has_immutable_cache_for_static_assets() {
let names = vec!["iris".to_owned()];
let body = render(&names, Some(FAKE_FRONTEND));
// ^~ prefix block for /static/ with immutable cache headers
assert!(
body.contains("location ^~ /agent/iris/static/ {"),
"expected ^~ static location, got:\n{body}"
);
assert!(body.contains(&format!("alias {FAKE_FRONTEND}/static/;")));
assert!(body.contains("expires 1y;"));
assert!(body.contains("immutable"));
}
#[test]
fn split_mode_has_alias_and_try_files() {
let names = vec!["iris".to_owned()];
let body = render(&names, Some(FAKE_FRONTEND));
// Plain prefix location with alias + try_files
assert!(
body.contains("location /agent/iris/ {"),
"expected try_files location, got:\n{body}"
);
assert!(
body.contains(&format!("alias {FAKE_FRONTEND}/;")),
"expected alias, got:\n{body}"
);
// try_files handles exact file, bare-path .html, directory index
assert!(
body.contains("try_files $uri $uri.html $uri/index.html @iris_dynamic"),
"expected try_files with .html fallback, got:\n{body}"
);
}
#[test]
fn split_mode_has_named_proxy_location() {
let names = vec!["iris".to_owned()];
let body = render(&names, Some(FAKE_FRONTEND));
// Named location for dynamic proxy
assert!(
body.contains("location @iris_dynamic {"),
"expected named dynamic location, got:\n{body}"
);
assert!(body.contains("proxy_pass"));
assert!(body.contains("proxy_intercept_errors on"));
assert!(body.contains("__hive_agent_unreachable"));
}
#[test]
fn split_mode_named_location_strips_prefix_without_uri_part() {
// Named locations forbid a URI part on proxy_pass — nginx rejects
// `proxy_pass http://host/` inside `location @name`. Must use
// bare upstream (no trailing `/` or path) plus a `rewrite` to
// strip the /agent/<name>/ prefix.
let names = vec!["iris".to_owned()];
let body = render(&names, Some(FAKE_FRONTEND));
assert!(
body.contains("rewrite ^/agent/iris/(.*)$ /$1 break;"),
"expected prefix-strip rewrite, got:\n{body}"
);
// The named-location proxy_pass must have no URI part (no
// trailing slash or path). Extract the `@iris_dynamic { ... }`
// block and check every proxy_pass directive in it.
let block_start = body
.find("location @iris_dynamic {")
.expect("named location");
let block = &body[block_start..];
let block_end = block.find("\n}\n").expect("block close");
let block = &block[..block_end];
for line in block.lines() {
let line = line.trim();
if let Some(rest) = line.strip_prefix("proxy_pass ") {
let target = rest.trim_end_matches(';');
// No URI part means: TCP form `http://host:port` (no
// trailing `/`), UDS form `http://unix:/path:` (trailing
// colon, nothing after). Both cases: must not end in `/`.
assert!(
!target.ends_with('/'),
"named-location proxy_pass must have no URI part, got: {target}"
);
}
}
}
#[test]
fn split_mode_includes_manager() {
let names: Vec<String> = [MANAGER_NAME, "iris"]
.iter()
.map(|s| (*s).to_owned())
.collect();
let body = render(&names, Some(FAKE_FRONTEND));
assert!(body.contains(&format!("/agent/{MANAGER_NAME}/")));
assert!(body.contains("/agent/iris/"));
}
#[test]
fn split_mode_proxy_includes_forwarded_prefix() {
let names = vec!["atlas".to_owned()];
let body = render(&names, Some(FAKE_FRONTEND));
assert!(
body.contains("X-Forwarded-Prefix /agent/atlas"),
"expected X-Forwarded-Prefix for atlas in split mode, got:\n{body}"
);
}
#[test]
fn split_mode_named_location_uses_agent_name() {
// Named location must be unique per agent to avoid nginx conflicts
let names = vec!["damocles".to_owned(), "iris".to_owned()];
let body = render(&names, Some(FAKE_FRONTEND));
assert!(body.contains("@damocles_dynamic"));
assert!(body.contains("@iris_dynamic"));
// Each agent gets exactly one named dynamic location block.
let damocles_count = body.matches("@damocles_dynamic {").count();
let iris_count = body.matches("@iris_dynamic {").count();
assert_eq!(
damocles_count, 1,
"expected exactly one damocles_dynamic block"
);
assert_eq!(iris_count, 1, "expected exactly one iris_dynamic block");
}
}