gateway: hot-reload agents.conf at runtime (#869)

Replace eval-time per-agent nginx location baking with a runtime
include file. c0re writes /var/lib/hyperhive/agents.conf (nginx
location blocks, UDS or TCP per agent) on every topology change and
on the 10s marker poll. The gateway container bind-mounts
/var/lib/hyperhive/ at /run/hive-state/ and nginx includes
/run/hive-state/agents.conf. A systemd path unit inside the container
watches the file for changes and fires `nginx -s reload` on each
atomic rename from c0re — no nixos-rebuild switch needed when agents
start, stop, or flip useUnixSocket.

  - new hive-c0re/src/gateway_nginx.rs: write() + render()
  - lib.rs + meta.rs + agent_sockets::spawn_poll: hook in write()
  - hive-gateway.nix: drop agentPortsTable/agentSocketsTable/
    agentUpstreamFor/lib.mapAttrs', add /run/hive-state bind-mount,
    include directive, systemd path unit + reload service, tmpfiles
    for /var/lib/hyperhive + agents.conf seed
  - docs/gateway.md: update vhost table + Per-agent UDS section

hive-c0re/src/agent_ports.rs

@@ -1,9 +1,11 @@
-//! `/var/lib/hyperhive/agent-ports.json` writer. Legacy TCP map for
-//! per-agent `/agent/<name>/` routing — the unix-socket replacement
-//! lives in `agent_sockets.rs`. The gateway reads this JSON at
-//! request-handling time rather than at gateway build time, so a
-//! `nixos-container update` of the gateway isn't needed every time
-//! an agent spawns / moves / destroys.
+//! `/var/lib/hyperhive/agent-ports.json` writer. Port map for
+//! per-agent `/agent/<name>/` TCP routing. Written alongside
+//! `agents.conf` (see `gateway_nginx.rs`) on every topology change;
+//! `gateway_nginx::render` reads it indirectly via
+//! `lifecycle::agent_web_port` to populate TCP upstreams for agents
+//! that haven't opted in to unix-socket mode yet. Also kept as a
+//! human-readable audit file — `cat agent-ports.json` shows every
+//! registered sub-agent and its deterministic port assignment.
 //!
 //! Shape (flat object keyed by logical agent name → web port):
 //!

hive-c0re/src/agent_sockets.rs

@@ -181,6 +181,14 @@ pub fn spawn_poll() {
                     if let Err(e) = write(&names) {
                         tracing::debug!(error = ?e, "agent_sockets poll write failed");
                     }
+                    // Regenerate the gateway nginx include whenever
+                    // socket readiness changes — the upstream
+                    // selection (UDS vs TCP) depends on .bound markers
+                    // which change independently of topology. Write is
+                    // idempotent; skips rename when nothing changed.
+                    if let Err(e) = crate::gateway_nginx::write(&names) {
+                        tracing::debug!(error = ?e, "gateway_nginx poll write failed");
+                    }
                 }
                 Err(e) => {
                     tracing::debug!(error = ?e, "agent_sockets poll: failed to list agents");

hive-c0re/src/gateway_nginx.rs

@@ -0,0 +1,205 @@
+//! Runtime nginx include-file generator for the gateway's per-agent
+//! `/agent/<name>/` location blocks (#869).
+//!
+//! Writes `/var/lib/hyperhive/agents.conf` on every topology change.
+//! The gateway container bind-mounts the whole `/var/lib/hyperhive/`
+//! directory at `/run/hive-state/` and nginx includes
+//! `/run/hive-state/agents.conf`. A systemd path unit inside the
+//! gateway container watches the file and triggers `nginx -s reload`
+//! on every atomic rename — no `nixos-rebuild switch` needed when
+//! agents start, stop, or flip `useUnixSocket`.
+//!
+//! Upstream selection mirrors `agent_sockets::build_map`: an agent
+//! gets a UDS upstream when its `.bound` marker exists (harness has
+//! bound the unix socket); otherwise falls back to the deterministic
+//! TCP port from `lifecycle::agent_web_port`. Proxy headers are
+//! emitted in full so the generated file is self-contained nginx
+//! config — no dependency on which `recommendedProxySettings` knobs
+//! the host config has on.
+//!
+//! `write()` is idempotent: if the rendered body equals what's already
+//! on disk, the rename is skipped and the path unit doesn't fire.
+//! Same atomic `<path>.tmp` + `rename()` shape as `agent_ports` /
+//! `agent_sockets` — a crashing c0re process never leaves a partial
+//! file the gateway's nginx would fail to parse.
+
+use anyhow::{Context, Result};
+use std::fmt::Write as _;
+use std::path::PathBuf;
+
+use crate::agent_sockets;
+use crate::lifecycle::{self, MANAGER_NAME};
+
+const HOST_CONF_PATH: &str = "/var/lib/hyperhive/agents.conf";
+
+/// Host-side path where c0re writes the generated nginx include file.
+/// The gateway container bind-mounts `/var/lib/hyperhive/` at
+/// `/run/hive-state/` so nginx inside can read it at
+/// `/run/hive-state/agents.conf`.
+#[must_use]
+pub fn host_conf_path() -> PathBuf {
+    PathBuf::from(HOST_CONF_PATH)
+}
+
+/// Nginx proxy headers present in every per-agent location block.
+/// `$connection_upgrade` is defined in the http context by the NixOS
+/// nginx module when `recommendedProxySettings = true` (which the
+/// gateway always sets). `X-Forwarded-Prefix` is per-location (set
+/// below in the render loop). Kept as a standalone const so it's
+/// visible to tests without repeating the text.
+const PROXY_HEADER_BLOCK: &str = "    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $connection_upgrade;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;";
+
+/// Render the nginx include body for `names`. Manager is filtered out.
+/// Each remaining agent gets a `location /agent/<name>/` block whose
+/// `proxy_pass` targets the unix socket when the `.bound` marker
+/// exists, or TCP loopback otherwise.
+///
+/// Output is deterministic for the same (sorted) set of names +
+/// `.bound` state: no timestamps, no UUIDs. `BTreeMap` would give
+/// alphabetical order; we rely on the caller to pass sorted names if
+/// they care about diff stability, but the gateway treats the blocks
+/// as unordered by nginx's longest-prefix-match rules so ordering only
+/// affects human readability.
+fn render(names: &[String]) -> String {
+    let mut out = String::from(
+        "# Generated by hive-c0re \u{2014} do not edit.\
+         \n# Refreshed on every topology change + when agents bind/drop their unix sockets.\
+         \n# Gateway reloads nginx automatically on each update (systemd path unit).\n",
+    );
+    for name in names {
+        if name == MANAGER_NAME {
+            continue;
+        }
+        let port = lifecycle::agent_web_port(name);
+        let upstream = if agent_sockets::ready_marker_for(name).exists() {
+            format!(
+                "http://unix:{}:/",
+                agent_sockets::socket_path_for(name).display()
+            )
+        } else {
+            format!("http://127.0.0.1:{port}/")
+        };
+        let _ = write!(
+            out,
+            "\nlocation /agent/{name}/ {{\n\
+             {PROXY_HEADER_BLOCK}\n\
+             \n    proxy_pass {upstream};\n\
+             \n    proxy_set_header X-Forwarded-Prefix /agent/{name};\n\
+             \n    proxy_buffering off;\n\
+             \n    proxy_read_timeout 1d;\n\
+             \n    proxy_intercept_errors on;\n\
+             \n    error_page 502 503 504 = /__hive_agent_unreachable;\n\
+             }}\n",
+        );
+    }
+    out
+}
+
+/// Atomically write the nginx include file for `names` to
+/// [`host_conf_path()`]. Skips the rename when the rendered body
+/// matches what's already on disk (idempotent; avoids spurious
+/// gateway reloads on a quiet tick). On a fresh install where the
+/// file doesn't exist yet, writes an empty-but-valid config so nginx
+/// can start before any agents have registered.
+pub fn write(names: &[String]) -> Result<()> {
+    let body = render(names);
+    let path = host_conf_path();
+    if std::fs::read_to_string(&path).ok().as_deref() == Some(&body) {
+        return Ok(());
+    }
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent)
+            .with_context(|| format!("create dir {}", parent.display()))?;
+    }
+    let tmp = path.with_extension("conf.tmp");
+    std::fs::write(&tmp, &body)
+        .with_context(|| format!("write tmp {}", tmp.display()))?;
+    std::fs::rename(&tmp, &path).with_context(|| {
+        format!(
+            "rename {} -> {} (atomic publish)",
+            tmp.display(),
+            path.display()
+        )
+    })?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn render_empty_is_header_only() {
+        let body = render(&[]);
+        assert!(body.starts_with("# Generated by hive-c0re"));
+        // No location blocks when no agents.
+        assert!(!body.contains("location"));
+    }
+
+    #[test]
+    fn render_filters_manager() {
+        let names: Vec<String> = [MANAGER_NAME, "iris"]
+            .iter()
+            .map(|s| (*s).to_owned())
+            .collect();
+        // Use build_map_with so we don't need real .bound files.
+        // For render, we call it directly with names.
+        let body = render(&names);
+        assert!(!body.contains(&format!("/agent/{MANAGER_NAME}/")));
+        assert!(body.contains("/agent/iris/"));
+    }
+
+    #[test]
+    fn render_tcp_upstream_when_no_bound_marker() {
+        // No .bound file on disk → falls back to TCP loopback.
+        let names = vec!["iris".to_owned()];
+        let body = render(&names);
+        let port = lifecycle::agent_web_port("iris");
+        assert!(
+            body.contains(&format!("proxy_pass http://127.0.0.1:{port}/")),
+            "expected TCP upstream for iris, got:\n{body}"
+        );
+    }
+
+    #[test]
+    fn render_includes_proxy_headers() {
+        let names = vec!["iris".to_owned()];
+        let body = render(&names);
+        // All required proxy headers present (WebSocket upgrade + forwarded-for).
+        assert!(body.contains("proxy_http_version 1.1"));
+        assert!(body.contains("proxy_set_header Upgrade $http_upgrade"));
+        assert!(body.contains("proxy_set_header X-Forwarded-For"));
+    }
+
+    #[test]
+    fn render_includes_forwarded_prefix() {
+        let names = vec!["atlas".to_owned()];
+        let body = render(&names);
+        assert!(
+            body.contains("X-Forwarded-Prefix /agent/atlas"),
+            "expected X-Forwarded-Prefix for atlas, got:\n{body}"
+        );
+    }
+
+    #[test]
+    fn render_includes_error_pages() {
+        let names = vec!["iris".to_owned()];
+        let body = render(&names);
+        assert!(body.contains("proxy_intercept_errors on"));
+        assert!(body.contains("__hive_agent_unreachable"));
+    }
+
+    #[test]
+    fn render_includes_buffering_and_timeout() {
+        let names = vec!["iris".to_owned()];
+        let body = render(&names);
+        assert!(body.contains("proxy_buffering off"));
+        assert!(body.contains("proxy_read_timeout 1d"));
+    }
+}

hive-c0re/src/lib.rs

@@ -16,6 +16,7 @@ pub mod actions;
 pub mod agent_ports;
 pub mod agent_server;
 pub mod agent_sockets;
+pub mod gateway_nginx;
 pub mod approvals;
 pub mod auto_update;
 pub mod broker;

hive-c0re/src/meta.rs

@@ -126,6 +126,16 @@ pub async fn sync_agents(
         tracing::warn!(error = ?e, "agent_sockets::write failed (non-fatal)");
     }
 
+    // Refresh /var/lib/hyperhive/agents.conf — the nginx include file
+    // the gateway picks up at runtime without needing a
+    // nixos-rebuild. The gateway container bind-mounts
+    // /var/lib/hyperhive/ and a systemd path unit fires
+    // `nginx -s reload` when this file changes (#869). Same
+    // best-effort + non-fatal shape.
+    if let Err(e) = crate::gateway_nginx::write(&agent_names) {
+        tracing::warn!(error = ?e, "gateway_nginx::write failed (non-fatal)");
+    }
+
     if initial {
         git(&dir, &["init", "--initial-branch=main"]).await?;
     }