418 lines
18 KiB
Rust
418 lines
18 KiB
Rust
//! Runtime nginx include-file generator for the gateway's per-agent
|
|
//! `/agent/<name>/` location blocks. Writes
|
|
//! `/var/lib/hyperhive/gateway/agents.conf` on every topology change.
|
|
//! UDS vs TCP upstream selection, reload trigger (`systemd-run
|
|
//! --machine=hive-gateway`), and idempotency:
|
|
//! `docs/gateway.md::Per-agent unix-socket upstream`.
|
|
|
|
use anyhow::{Context, Result};
|
|
use std::fmt::Write as _;
|
|
use std::path::PathBuf;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
|
|
use crate::agent_sockets;
|
|
use crate::lifecycle;
|
|
|
|
/// Set when `write` publishes a new agents.conf; cleared when
|
|
/// `reload_gateway_nginx` submits the reload command successfully.
|
|
/// Lets `spawn_poll` retry the reload on subsequent ticks when the
|
|
/// previous attempt failed (e.g. gateway container temporarily down,
|
|
/// systemd-run not found) without re-writing the already-correct file.
|
|
static RELOAD_PENDING: AtomicBool = AtomicBool::new(false);
|
|
|
|
const HOST_CONF_PATH: &str = "/var/lib/hyperhive/gateway/agents.conf";
|
|
|
|
/// Host-side path where c0re writes the generated nginx include file.
|
|
/// The gateway container bind-mounts `/var/lib/hyperhive/gateway/`
|
|
/// (not the whole parent dir) at `/run/hive-state/` so nginx inside
|
|
/// can read it at `/run/hive-state/agents.conf`. Subdirectory scoping
|
|
/// avoids exposing the rest of `/var/lib/hyperhive/` (which may contain
|
|
/// forge tokens or other credentials) to the gateway container.
|
|
#[must_use]
|
|
pub fn host_conf_path() -> PathBuf {
|
|
PathBuf::from(HOST_CONF_PATH)
|
|
}
|
|
|
|
/// Nginx proxy headers present in every per-agent location block.
|
|
/// `$connection_upgrade` is defined in the http context by the NixOS
|
|
/// nginx module when `recommendedProxySettings = true` (which the
|
|
/// gateway always sets). `X-Forwarded-Prefix` is per-location (set
|
|
/// below in the render loop). Kept as a standalone const so it's
|
|
/// visible to tests without repeating the text.
|
|
const PROXY_HEADER_BLOCK: &str = " proxy_http_version 1.1;
|
|
proxy_set_header Upgrade $http_upgrade;
|
|
proxy_set_header Connection $connection_upgrade;
|
|
proxy_set_header Host $host;
|
|
proxy_set_header X-Real-IP $remote_addr;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
proxy_set_header X-Forwarded-Proto $scheme;";
|
|
|
|
/// Render the nginx include body for `names`. Manager is included so
|
|
/// `/agent/root/` is routable through the gateway.
|
|
///
|
|
/// When `frontend_dir` is `Some(path)` each agent gets split location
|
|
/// blocks:
|
|
/// - Exact HTML pages (`/`, `/stats`, `/screen`) → `alias` from the nix
|
|
/// store dist. These load even while the agent daemon is restarting.
|
|
/// - Static assets (`/static/`) → `alias` from the nix store with cache
|
|
/// headers. Served by nginx directly, no agent socket round-trip.
|
|
/// - Everything else (all `/api/*`, `/events/*`, `/screen/ws`, `/icon`,
|
|
/// `/send`, `/login/*`) → proxied to the agent daemon as before.
|
|
///
|
|
/// When `frontend_dir` is `None` (legacy) each agent gets a single
|
|
/// `location /agent/<name>/` proxy block.
|
|
///
|
|
/// Output is deterministic for the same (sorted) set of names +
|
|
/// `.bound` state: no timestamps, no UUIDs. `BTreeMap` would give
|
|
/// alphabetical order; we rely on the caller to pass sorted names if
|
|
/// they care about diff stability, but the gateway treats the blocks
|
|
/// as unordered by nginx's longest-prefix-match rules so ordering only
|
|
/// affects human readability.
|
|
fn render(names: &[String], frontend_dir: Option<&str>) -> String {
|
|
let mut out = String::from(
|
|
"# Generated by hive-c0re \u{2014} do not edit.\
|
|
\n# Refreshed on every topology change + when agents bind/drop their unix sockets.\
|
|
\n# Reload triggered by hive-c0re via systemd-run --machine=hive-gateway.\n",
|
|
);
|
|
for name in names {
|
|
let port = lifecycle::agent_web_port(name);
|
|
let upstream = if agent_sockets::ready_marker_for(name).exists() {
|
|
format!(
|
|
"http://unix:{}:/",
|
|
agent_sockets::socket_path_for(name).display()
|
|
)
|
|
} else {
|
|
format!("http://127.0.0.1:{port}/")
|
|
};
|
|
|
|
if let Some(frontend) = frontend_dir {
|
|
// Split mode: try to serve files from the nix-store dist first;
|
|
// fall through to the agent daemon for anything not found there.
|
|
//
|
|
// `frontend` is a nix store path injected at build time by hive-c0re.nix
|
|
// (`HIVE_AGENT_FRONTEND_DIR = "${cfg.frontend}/agent"`). Nix store paths
|
|
// are of the form `/nix/store/<hash>-<name>` — only [a-z0-9/._-], never
|
|
// shell metacharacters — so interpolating directly into the nginx config
|
|
// string is safe. We do NOT accept user-controlled input here.
|
|
//
|
|
// Location priority for a request to /agent/<name>/...:
|
|
// ^~ /agent/<name>/static/ — highest priority; matches compiled assets.
|
|
// Nix store paths are content-addressed and immutable — cache forever.
|
|
// /agent/<name>/ — try_files from dist, proxy fallback for the rest.
|
|
//
|
|
// try_files resolution (nginx applies the alias mapping before checking):
|
|
// $uri — exact file match (/static/app.js → static/app.js)
|
|
// $uri.html — bare-path .html fallback (/stats → stats.html)
|
|
// $uri/index.html — directory index (/ → index.html)
|
|
// @<name>_dynamic — proxy catchall: api, events, icon, login, …
|
|
//
|
|
// Adding pages to the frontend dist works automatically — no generator
|
|
// change needed. Per-agent extraFiles (in mergedDist, not in the base
|
|
// dist path) still proxy to the agent daemon via @<name>_dynamic.
|
|
let _ = write!(
|
|
out,
|
|
"\n# Compiled JS/CSS assets — content-addressed nix store path, cache forever\n\
|
|
location ^~ /agent/{name}/static/ {{\n\
|
|
\n alias {frontend}/static/;\n\
|
|
\n expires 1y;\n\
|
|
\n add_header Cache-Control \"public, immutable, max-age=31536000\";\n\
|
|
}}\n\
|
|
\n# Static dist + proxy fallback for dynamic paths\n\
|
|
location /agent/{name}/ {{\n\
|
|
\n alias {frontend}/;\n\
|
|
\n try_files $uri $uri.html $uri/index.html @{name}_dynamic;\n\
|
|
}}\n\
|
|
\nlocation @{name}_dynamic {{\n\
|
|
{PROXY_HEADER_BLOCK}\n\
|
|
\n proxy_pass {upstream};\n\
|
|
\n proxy_set_header X-Forwarded-Prefix /agent/{name};\n\
|
|
\n proxy_buffering off;\n\
|
|
\n proxy_read_timeout 1d;\n\
|
|
\n proxy_intercept_errors on;\n\
|
|
\n error_page 502 503 504 = /__hive_agent_unreachable;\n\
|
|
}}\n",
|
|
);
|
|
} else {
|
|
// Legacy mode: proxy everything to the agent daemon.
|
|
let _ = write!(
|
|
out,
|
|
"\nlocation /agent/{name}/ {{\n\
|
|
{PROXY_HEADER_BLOCK}\n\
|
|
\n proxy_pass {upstream};\n\
|
|
\n proxy_set_header X-Forwarded-Prefix /agent/{name};\n\
|
|
\n proxy_buffering off;\n\
|
|
\n proxy_read_timeout 1d;\n\
|
|
\n proxy_intercept_errors on;\n\
|
|
\n error_page 502 503 504 = /__hive_agent_unreachable;\n\
|
|
}}\n",
|
|
);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Atomically write the nginx include file for `names` to
|
|
/// [`host_conf_path()`]. Skips the write + reload when the rendered
|
|
/// body matches what's already on disk (idempotent; avoids spurious
|
|
/// gateway reloads on a quiet tick).
|
|
///
|
|
/// After a successful write, triggers an nginx reload inside the
|
|
/// gateway container from the HOST side via
|
|
/// `systemd-run --machine=hive-gateway nginx -s reload`. This is
|
|
/// intentionally host-side rather than relying on a systemd path unit
|
|
/// inside the container watching the bind-mounted file: `IN_MOVED_TO`
|
|
/// (fired by the atomic rename) does not reliably propagate across the
|
|
/// nspawn mount-namespace boundary, so the path-unit approach was
|
|
/// silently broken (see `docs/gateway.md` for the failure analysis).
|
|
///
|
|
/// The `systemd-run` call is best-effort — a failed reload is logged
|
|
/// but not fatal. nginx will pick up the new include on its next
|
|
/// housekeeping restart or the next manual reload; the host's agent
|
|
/// topology has already been written correctly.
|
|
pub fn write(names: &[String]) -> Result<()> {
|
|
let frontend_dir = std::env::var("HIVE_AGENT_FRONTEND_DIR")
|
|
.ok()
|
|
.filter(|s| !s.is_empty());
|
|
let body = render(names, frontend_dir.as_deref());
|
|
let path = host_conf_path();
|
|
if std::fs::read_to_string(&path).ok().as_deref() == Some(&body) {
|
|
return Ok(());
|
|
}
|
|
if let Some(parent) = path.parent() {
|
|
std::fs::create_dir_all(parent)
|
|
.with_context(|| format!("create dir {}", parent.display()))?;
|
|
}
|
|
let tmp = path.with_extension("conf.tmp");
|
|
std::fs::write(&tmp, &body)
|
|
.with_context(|| format!("write tmp {}", tmp.display()))?;
|
|
std::fs::rename(&tmp, &path).with_context(|| {
|
|
format!(
|
|
"rename {} -> {} (atomic publish)",
|
|
tmp.display(),
|
|
path.display()
|
|
)
|
|
})?;
|
|
// Mark reload pending before attempting so a failed attempt is
|
|
// retried by the next spawn_poll tick (see `reload_if_pending`).
|
|
RELOAD_PENDING.store(true, Ordering::Relaxed);
|
|
reload_gateway_nginx();
|
|
Ok(())
|
|
}
|
|
|
|
/// Retry a pending nginx reload if a previous attempt failed.
|
|
/// Called by `spawn_poll` on each tick so a transient failure
|
|
/// (gateway container temporarily down, systemd-run error) is
|
|
/// recovered automatically without requiring a new file write.
|
|
pub fn reload_if_pending() {
|
|
if RELOAD_PENDING.load(Ordering::Relaxed) {
|
|
reload_gateway_nginx();
|
|
}
|
|
}
|
|
|
|
/// Send `nginx -s reload` to the gateway container via systemd-run.
|
|
/// Uses `--wait` so the exit code reflects whether nginx received the
|
|
/// signal; clears `RELOAD_PENDING` on success so `reload_if_pending`
|
|
/// stops retrying. Best-effort: errors are logged, not bubbled.
|
|
fn reload_gateway_nginx() {
|
|
// `--machine=hive-gateway` targets the container by its nspawn
|
|
// machine name (same as the nixos-container name). `--quiet`
|
|
// suppresses the transient unit name echo. `--wait` blocks until
|
|
// the transient job exits so the exit code tells us whether
|
|
// `nginx -s reload` ran at all (RELOAD_PENDING is only cleared on
|
|
// success — a failed attempt is retried next tick). `--`
|
|
// separates systemd-run args from the command.
|
|
let status = std::process::Command::new("systemd-run")
|
|
.args([
|
|
"--machine=hive-gateway",
|
|
"--quiet",
|
|
"--wait",
|
|
"--",
|
|
"nginx",
|
|
"-s",
|
|
"reload",
|
|
])
|
|
.status();
|
|
match status {
|
|
Ok(s) if s.success() => {
|
|
// nginx -s reload ran successfully (SIGHUP sent to master).
|
|
// The actual worker replacement is async but the signal was
|
|
// delivered; clear the pending flag.
|
|
RELOAD_PENDING.store(false, Ordering::Relaxed);
|
|
tracing::debug!("gateway nginx reload signal sent");
|
|
}
|
|
Ok(s) => {
|
|
tracing::warn!(
|
|
exit_code = ?s.code(),
|
|
"gateway nginx reload exited non-zero — will retry next poll tick"
|
|
);
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
error = %e,
|
|
"failed to invoke systemd-run for gateway nginx reload — will retry next poll tick"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::lifecycle::MANAGER_NAME;
|
|
|
|
// ── legacy mode (frontend_dir = None) ──────────────────────────────
|
|
|
|
#[test]
|
|
fn render_empty_is_header_only() {
|
|
let body = render(&[], None);
|
|
assert!(body.starts_with("# Generated by hive-c0re"));
|
|
// No location blocks when no agents.
|
|
assert!(!body.contains("location"));
|
|
}
|
|
|
|
#[test]
|
|
fn render_includes_manager() {
|
|
let names: Vec<String> = [MANAGER_NAME, "iris"]
|
|
.iter()
|
|
.map(|s| (*s).to_owned())
|
|
.collect();
|
|
let body = render(&names, None);
|
|
assert!(body.contains(&format!("/agent/{MANAGER_NAME}/")));
|
|
assert!(body.contains("/agent/iris/"));
|
|
}
|
|
|
|
#[test]
|
|
fn render_tcp_upstream_when_no_bound_marker() {
|
|
// No .bound file on disk → falls back to TCP loopback.
|
|
let names = vec!["iris".to_owned()];
|
|
let body = render(&names, None);
|
|
let port = lifecycle::agent_web_port("iris");
|
|
assert!(
|
|
body.contains(&format!("proxy_pass http://127.0.0.1:{port}/")),
|
|
"expected TCP upstream for iris, got:\n{body}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn render_includes_proxy_headers() {
|
|
let names = vec!["iris".to_owned()];
|
|
let body = render(&names, None);
|
|
// All required proxy headers present (WebSocket upgrade + forwarded-for).
|
|
assert!(body.contains("proxy_http_version 1.1"));
|
|
assert!(body.contains("proxy_set_header Upgrade $http_upgrade"));
|
|
assert!(body.contains("proxy_set_header X-Forwarded-For"));
|
|
}
|
|
|
|
#[test]
|
|
fn render_includes_forwarded_prefix() {
|
|
let names = vec!["atlas".to_owned()];
|
|
let body = render(&names, None);
|
|
assert!(
|
|
body.contains("X-Forwarded-Prefix /agent/atlas"),
|
|
"expected X-Forwarded-Prefix for atlas, got:\n{body}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn render_includes_error_pages() {
|
|
let names = vec!["iris".to_owned()];
|
|
let body = render(&names, None);
|
|
assert!(body.contains("proxy_intercept_errors on"));
|
|
assert!(body.contains("__hive_agent_unreachable"));
|
|
}
|
|
|
|
#[test]
|
|
fn render_includes_buffering_and_timeout() {
|
|
let names = vec!["iris".to_owned()];
|
|
let body = render(&names, None);
|
|
assert!(body.contains("proxy_buffering off"));
|
|
assert!(body.contains("proxy_read_timeout 1d"));
|
|
}
|
|
|
|
// ── split mode (frontend_dir = Some) ───────────────────────────────
|
|
|
|
const FAKE_FRONTEND: &str = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-frontend/agent";
|
|
|
|
#[test]
|
|
fn split_mode_has_immutable_cache_for_static_assets() {
|
|
let names = vec!["iris".to_owned()];
|
|
let body = render(&names, Some(FAKE_FRONTEND));
|
|
// ^~ prefix block for /static/ with immutable cache headers
|
|
assert!(
|
|
body.contains("location ^~ /agent/iris/static/ {"),
|
|
"expected ^~ static location, got:\n{body}"
|
|
);
|
|
assert!(body.contains(&format!("alias {FAKE_FRONTEND}/static/;")));
|
|
assert!(body.contains("expires 1y;"));
|
|
assert!(body.contains("immutable"));
|
|
}
|
|
|
|
#[test]
|
|
fn split_mode_has_alias_and_try_files() {
|
|
let names = vec!["iris".to_owned()];
|
|
let body = render(&names, Some(FAKE_FRONTEND));
|
|
// Plain prefix location with alias + try_files
|
|
assert!(
|
|
body.contains("location /agent/iris/ {"),
|
|
"expected try_files location, got:\n{body}"
|
|
);
|
|
assert!(
|
|
body.contains(&format!("alias {FAKE_FRONTEND}/;")),
|
|
"expected alias, got:\n{body}"
|
|
);
|
|
// try_files handles exact file, bare-path .html, directory index
|
|
assert!(
|
|
body.contains("try_files $uri $uri.html $uri/index.html @iris_dynamic"),
|
|
"expected try_files with .html fallback, got:\n{body}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn split_mode_has_named_proxy_location() {
|
|
let names = vec!["iris".to_owned()];
|
|
let body = render(&names, Some(FAKE_FRONTEND));
|
|
// Named location for dynamic proxy
|
|
assert!(
|
|
body.contains("location @iris_dynamic {"),
|
|
"expected named dynamic location, got:\n{body}"
|
|
);
|
|
assert!(body.contains("proxy_pass"));
|
|
assert!(body.contains("proxy_intercept_errors on"));
|
|
assert!(body.contains("__hive_agent_unreachable"));
|
|
}
|
|
|
|
#[test]
|
|
fn split_mode_includes_manager() {
|
|
let names: Vec<String> = [MANAGER_NAME, "iris"]
|
|
.iter()
|
|
.map(|s| (*s).to_owned())
|
|
.collect();
|
|
let body = render(&names, Some(FAKE_FRONTEND));
|
|
assert!(body.contains(&format!("/agent/{MANAGER_NAME}/")));
|
|
assert!(body.contains("/agent/iris/"));
|
|
}
|
|
|
|
#[test]
|
|
fn split_mode_proxy_includes_forwarded_prefix() {
|
|
let names = vec!["atlas".to_owned()];
|
|
let body = render(&names, Some(FAKE_FRONTEND));
|
|
assert!(
|
|
body.contains("X-Forwarded-Prefix /agent/atlas"),
|
|
"expected X-Forwarded-Prefix for atlas in split mode, got:\n{body}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn split_mode_named_location_uses_agent_name() {
|
|
// Named location must be unique per agent to avoid nginx conflicts
|
|
let names = vec!["damocles".to_owned(), "iris".to_owned()];
|
|
let body = render(&names, Some(FAKE_FRONTEND));
|
|
assert!(body.contains("@damocles_dynamic"));
|
|
assert!(body.contains("@iris_dynamic"));
|
|
// Each agent gets exactly one named dynamic location block.
|
|
let damocles_count = body.matches("@damocles_dynamic {").count();
|
|
let iris_count = body.matches("@iris_dynamic {").count();
|
|
assert_eq!(damocles_count, 1, "expected exactly one damocles_dynamic block");
|
|
assert_eq!(iris_count, 1, "expected exactly one iris_dynamic block");
|
|
}
|
|
}
|