From 01d7c37af2d679dd01027c3d946555e10fd1fb30 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 31 May 2026 22:35:06 +0200 Subject: [PATCH] =?UTF-8?q?gateway:=20fix=20nginx=20reload=20=E2=80=94=20t?= =?UTF-8?q?rigger=20from=20host=20via=20systemd-run=20(#889)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #872 path-unit approach was silently broken: IN_MOVED_TO from an atomic rename on the host does not propagate across the nspawn mount-namespace boundary into the container, so the watcher inside the gateway container never fired. Fix: after each agents.conf write, c0re calls `systemd-run --machine=hive-gateway -- nginx -s reload` from the host. The reload is best-effort (logged on failure, not fatal). Remove the now-unused `hive-gateway-agents-conf.path` + `hive-gateway-nginx-reload.service` from the gateway container config. Update docs/gateway.md + comments to reflect the host-side approach. --- docs/gateway.md | 26 +++++++------- hive-c0re/src/gateway_nginx.rs | 64 ++++++++++++++++++++++++++++++---- nix/modules/hive-gateway.nix | 59 ++++++++++--------------------- 3 files changed, 90 insertions(+), 59 deletions(-) diff --git a/docs/gateway.md b/docs/gateway.md index 7ecd39f3..b1971bff 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -91,21 +91,23 @@ unix-domain socket as each agent opts in. The mechanism: socket for every sub-agent that hasn't opted in yet. (Legacy name `.bound` also accepted during the transition window.) 4. **Gateway side**. `gateway_nginx::write` generates - `/var/lib/hyperhive/agents.conf` — a plain nginx include file with - one `location /agent//` block per agent. UDS upstream - (`http://unix:/run/hive-agent//web.sock:/`) when + `/var/lib/hyperhive/gateway/agents.conf` — a plain nginx include + file with one `location /agent//` block per agent. UDS + upstream (`http://unix:/run/hive-agent//web.sock:/`) when `hyperhive-socket-bound` marker present; TCP loopback otherwise. - The gateway container bind-mounts `/var/lib/hyperhive/` at + The gateway container bind-mounts `/var/lib/hyperhive/gateway/` at `/run/hive-state/`; nginx includes `/run/hive-state/agents.conf`. - A systemd path unit (`hive-gateway-agents-conf.path`) inside the - container watches the file and fires `nginx -s reload` on every - atomic rename from c0re — no `nixos-rebuild` needed. + After each write, c0re triggers `nginx -s reload` inside the + gateway container from the HOST via + `systemd-run --machine=hive-gateway nginx -s reload`. This is + intentionally host-side: `IN_MOVED_TO` from an atomic rename does + not propagate across the nspawn mount-namespace boundary, so a + path unit inside the container would never fire (#889). -c0re regenerates `agents.conf` (and fires the path unit → reload) on -two triggers: every topology change (new/removed agents) and every -10s marker poll tick (`agent_sockets::spawn_poll`). `write()` is -idempotent — skips the rename when content is unchanged so the path -unit doesn't fire spuriously. +c0re regenerates `agents.conf` (and triggers a reload) on two +triggers: every topology change (new/removed agents) and every 10s +marker poll tick (`agent_sockets::spawn_poll`). `write()` is +idempotent — skips the rename+reload when content is unchanged. Transition: agents that haven't flipped `useUnixSocket = true` get a TCP loopback upstream in `agents.conf` (deterministic port from diff --git a/hive-c0re/src/gateway_nginx.rs b/hive-c0re/src/gateway_nginx.rs index ef869cf5..9ca5abe1 100644 --- a/hive-c0re/src/gateway_nginx.rs +++ b/hive-c0re/src/gateway_nginx.rs @@ -1,7 +1,7 @@ //! Runtime nginx include-file generator for the gateway's per-agent //! `/agent//` location blocks (#869). //! -//! Writes `/var/lib/hyperhive/agents.conf` on every topology change. +//! Writes `/var/lib/hyperhive/gateway/agents.conf` on every topology change. //! The gateway container bind-mounts the whole `/var/lib/hyperhive/` //! directory at `/run/hive-state/` and nginx includes //! `/run/hive-state/agents.conf`. A systemd path unit inside the @@ -105,11 +105,23 @@ fn render(names: &[String]) -> String { } /// Atomically write the nginx include file for `names` to -/// [`host_conf_path()`]. Skips the rename when the rendered body -/// matches what's already on disk (idempotent; avoids spurious -/// gateway reloads on a quiet tick). On a fresh install where the -/// file doesn't exist yet, writes an empty-but-valid config so nginx -/// can start before any agents have registered. +/// [`host_conf_path()`]. Skips the write + reload when the rendered +/// body matches what's already on disk (idempotent; avoids spurious +/// gateway reloads on a quiet tick). +/// +/// After a successful write, triggers an nginx reload inside the +/// gateway container from the HOST side via +/// `systemd-run --machine=hive-gateway nginx -s reload`. This is +/// intentionally host-side rather than relying on a systemd path unit +/// inside the container watching the bind-mounted file: `IN_MOVED_TO` +/// (fired by the atomic rename) does not reliably propagate across the +/// nspawn mount-namespace boundary, so the path-unit approach was +/// silently broken after #872 merged (#889). +/// +/// The `systemd-run` call is best-effort — a failed reload is logged +/// but not fatal. nginx will pick up the new include on its next +/// housekeeping restart or the next manual reload; the host's agent +/// topology has already been written correctly. pub fn write(names: &[String]) -> Result<()> { let body = render(names); let path = host_conf_path(); @@ -130,9 +142,49 @@ pub fn write(names: &[String]) -> Result<()> { path.display() ) })?; + // Trigger nginx reload from the host. Ignore errors — a failed + // reload is recoverable (nginx keeps serving the previous config). + reload_gateway_nginx(); Ok(()) } +/// Send `nginx -s reload` to the gateway container via systemd-run. +/// Runs non-interactively in a transient scope so it doesn't block +/// c0re's polling loop. Best-effort: errors are logged, not bubbled. +fn reload_gateway_nginx() { + // `--machine=hive-gateway` targets the container by its nspawn + // machine name (same as the nixos-container name). `--quiet` + // suppresses the transient unit name echo. `--` separates + // systemd-run args from the command. + let status = std::process::Command::new("systemd-run") + .args([ + "--machine=hive-gateway", + "--quiet", + "--", + "nginx", + "-s", + "reload", + ]) + .status(); + match status { + Ok(s) if s.success() => { + tracing::debug!("gateway nginx reloaded successfully"); + } + Ok(s) => { + tracing::warn!( + exit_code = ?s.code(), + "gateway nginx reload exited non-zero; will pick up on next restart" + ); + } + Err(e) => { + tracing::warn!( + error = %e, + "failed to invoke systemd-run for gateway nginx reload" + ); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/nix/modules/hive-gateway.nix b/nix/modules/hive-gateway.nix index ac25f6bd..20653e71 100644 --- a/nix/modules/hive-gateway.nix +++ b/nix/modules/hive-gateway.nix @@ -252,10 +252,10 @@ in # Bind-mount ONLY the gateway-specific subdir of the hyperhive # state dir. Scoped to /var/lib/hyperhive/gateway/ rather than # the whole parent so the gateway container can't read forge - # tokens or other files that may live at the parent level (argus - # 🟡 on #872). c0re writes agents.conf under this subdir; - # the systemd path unit inside the container fires nginx -s reload - # on each atomic rename. Pre-created by a tmpfiles rule. + # tokens or other files that may live at the parent level. + # c0re writes agents.conf under this subdir and triggers an nginx + # reload from the host via systemd-run after each write. + # Pre-created by a tmpfiles rule. bindMounts."/run/hive-state" = { hostPath = "/var/lib/hyperhive/gateway"; isReadOnly = true; @@ -389,37 +389,14 @@ in ''; }; - # Watch /run/hive-state/agents.conf (bind-mounted from the - # host's /var/lib/hyperhive/agents.conf) for changes and - # trigger an nginx reload when c0re atomically renames a new - # version into place (#869). PathChanged fires on - # IN_CLOSE_WRITE + IN_MOVED_TO, so the atomic rename c0re - # uses (write .conf.tmp → rename) wakes the path unit. - # The reload is a no-op if the new config is identical — - # gateway_nginx::write skips the rename when content is - # unchanged, so the path unit doesn't fire at all on quiet - # ticks. - systemd.paths.hive-gateway-agents-conf = { - wantedBy = [ "nginx.service" ]; - after = [ "nginx.service" ]; - pathConfig = { - PathChanged = "/run/hive-state/agents.conf"; - Unit = "hive-gateway-nginx-reload.service"; - }; - }; - - systemd.services.hive-gateway-nginx-reload = { - description = "Reload nginx after agents.conf change"; - # Don't block any target — fires only when the path unit - # triggers it. - serviceConfig = { - Type = "oneshot"; - # nginx -s reload sends SIGHUP to the master process via - # the pid file. Runs as root inside the container (pid 1 - # is the nspawn init; nginx master starts as root). - ExecStart = "/run/current-system/sw/bin/nginx -s reload"; - }; - }; + # nginx reload is triggered from the HOST side by hive-c0re + # via `systemd-run --machine=hive-gateway nginx -s reload` + # after each agents.conf write. A path unit watching the + # bind-mounted file inside the container was tried first + # (in #872) but IN_MOVED_TO from an atomic rename on the host + # does not propagate across the nspawn mount-namespace boundary, + # so the watcher never fired (#889). Host-side trigger is the + # correct approach. services.nginx = { enable = true; @@ -546,14 +523,14 @@ in }; }; # Per-agent location blocks, generated at runtime by - # hive-c0re and written to /var/lib/hyperhive/agents.conf + # hive-c0re and written to /var/lib/hyperhive/gateway/agents.conf # on the host. The bind-mount at /run/hive-state/ exposes # that file here. nginx parses `include` at config-load - # time so a reload (triggered by the hive-gateway-nginx- - # reload path unit when agents.conf changes) picks up new - # or removed agents without a nixos-rebuild. nginx's - # longest-prefix-match rule ensures `/agent//` from - # this file beats the `/agent/` catch-all above (#869). + # time so a reload (triggered by c0re via systemd-run + # after each agents.conf write) picks up new or removed + # agents without a nixos-rebuild. nginx's longest-prefix- + # match rule ensures `/agent//` from this file beats + # the `/agent/` catch-all above. extraConfig = '' include /run/hive-state/agents.conf; '';