gateway: fix nginx reload — trigger from host via systemd-run (#889)

The #872 path-unit approach was silently broken: IN_MOVED_TO from an
atomic rename on the host does not propagate across the nspawn
mount-namespace boundary into the container, so the watcher inside
the gateway container never fired.

Fix: after each agents.conf write, c0re calls
`systemd-run --machine=hive-gateway -- nginx -s reload` from the host.
The reload is best-effort (logged on failure, not fatal).

Remove the now-unused `hive-gateway-agents-conf.path` +
`hive-gateway-nginx-reload.service` from the gateway container config.
Update docs/gateway.md + comments to reflect the host-side approach.

docs/gateway.md

@@ -91,21 +91,23 @@ unix-domain socket as each agent opts in. The mechanism:
    socket for every sub-agent that hasn't opted in yet. (Legacy name
    `.bound` also accepted during the transition window.)
 4. **Gateway side**. `gateway_nginx::write` generates
-   `/var/lib/hyperhive/agents.conf` — a plain nginx include file with
+   `/var/lib/hyperhive/gateway/agents.conf` — a plain nginx include
-   one `location /agent/<name>/` block per agent. UDS upstream
+   file with one `location /agent/<name>/` block per agent. UDS
-   (`http://unix:/run/hive-agent/<name>/web.sock:/`) when
+   upstream (`http://unix:/run/hive-agent/<name>/web.sock:/`) when
    `hyperhive-socket-bound` marker present; TCP loopback otherwise.
-   The gateway container bind-mounts `/var/lib/hyperhive/` at
+   The gateway container bind-mounts `/var/lib/hyperhive/gateway/` at
    `/run/hive-state/`; nginx includes `/run/hive-state/agents.conf`.
-   A systemd path unit (`hive-gateway-agents-conf.path`) inside the
+   After each write, c0re triggers `nginx -s reload` inside the
-   container watches the file and fires `nginx -s reload` on every
+   gateway container from the HOST via
-   atomic rename from c0re — no `nixos-rebuild` needed.
+   `systemd-run --machine=hive-gateway nginx -s reload`. This is
+   intentionally host-side: `IN_MOVED_TO` from an atomic rename does
+   not propagate across the nspawn mount-namespace boundary, so a
+   path unit inside the container would never fire (#889).
 
-c0re regenerates `agents.conf` (and fires the path unit → reload) on
+c0re regenerates `agents.conf` (and triggers a reload) on two
-two triggers: every topology change (new/removed agents) and every
+triggers: every topology change (new/removed agents) and every 10s
-10s marker poll tick (`agent_sockets::spawn_poll`). `write()` is
+marker poll tick (`agent_sockets::spawn_poll`). `write()` is
-idempotent — skips the rename when content is unchanged so the path
+idempotent — skips the rename+reload when content is unchanged.
-unit doesn't fire spuriously.
 
 Transition: agents that haven't flipped `useUnixSocket = true` get a
 TCP loopback upstream in `agents.conf` (deterministic port from

hive-c0re/src/gateway_nginx.rs

@@ -1,7 +1,7 @@
 //! Runtime nginx include-file generator for the gateway's per-agent
 //! `/agent/<name>/` location blocks (#869).
 //!
-//! Writes `/var/lib/hyperhive/agents.conf` on every topology change.
+//! Writes `/var/lib/hyperhive/gateway/agents.conf` on every topology change.
 //! The gateway container bind-mounts the whole `/var/lib/hyperhive/`
 //! directory at `/run/hive-state/` and nginx includes
 //! `/run/hive-state/agents.conf`. A systemd path unit inside the
@@ -105,11 +105,23 @@ fn render(names: &[String]) -> String {
 }
 
 /// Atomically write the nginx include file for `names` to
-/// [`host_conf_path()`]. Skips the rename when the rendered body
+/// [`host_conf_path()`]. Skips the write + reload when the rendered
-/// matches what's already on disk (idempotent; avoids spurious
+/// body matches what's already on disk (idempotent; avoids spurious
-/// gateway reloads on a quiet tick). On a fresh install where the
+/// gateway reloads on a quiet tick).
-/// file doesn't exist yet, writes an empty-but-valid config so nginx
+///
-/// can start before any agents have registered.
+/// After a successful write, triggers an nginx reload inside the
+/// gateway container from the HOST side via
+/// `systemd-run --machine=hive-gateway nginx -s reload`. This is
+/// intentionally host-side rather than relying on a systemd path unit
+/// inside the container watching the bind-mounted file: `IN_MOVED_TO`
+/// (fired by the atomic rename) does not reliably propagate across the
+/// nspawn mount-namespace boundary, so the path-unit approach was
+/// silently broken after #872 merged (#889).
+///
+/// The `systemd-run` call is best-effort — a failed reload is logged
+/// but not fatal. nginx will pick up the new include on its next
+/// housekeeping restart or the next manual reload; the host's agent
+/// topology has already been written correctly.
 pub fn write(names: &[String]) -> Result<()> {
     let body = render(names);
     let path = host_conf_path();
@@ -130,9 +142,49 @@ pub fn write(names: &[String]) -> Result<()> {
             path.display()
         )
     })?;
+    // Trigger nginx reload from the host. Ignore errors — a failed
+    // reload is recoverable (nginx keeps serving the previous config).
+    reload_gateway_nginx();
     Ok(())
 }
 
+/// Send `nginx -s reload` to the gateway container via systemd-run.
+/// Runs non-interactively in a transient scope so it doesn't block
+/// c0re's polling loop. Best-effort: errors are logged, not bubbled.
+fn reload_gateway_nginx() {
+    // `--machine=hive-gateway` targets the container by its nspawn
+    // machine name (same as the nixos-container name). `--quiet`
+    // suppresses the transient unit name echo. `--` separates
+    // systemd-run args from the command.
+    let status = std::process::Command::new("systemd-run")
+        .args([
+            "--machine=hive-gateway",
+            "--quiet",
+            "--",
+            "nginx",
+            "-s",
+            "reload",
+        ])
+        .status();
+    match status {
+        Ok(s) if s.success() => {
+            tracing::debug!("gateway nginx reloaded successfully");
+        }
+        Ok(s) => {
+            tracing::warn!(
+                exit_code = ?s.code(),
+                "gateway nginx reload exited non-zero; will pick up on next restart"
+            );
+        }
+        Err(e) => {
+            tracing::warn!(
+                error = %e,
+                "failed to invoke systemd-run for gateway nginx reload"
+            );
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

nix/modules/hive-gateway.nix

@@ -252,10 +252,10 @@ in
       # Bind-mount ONLY the gateway-specific subdir of the hyperhive
       # state dir. Scoped to /var/lib/hyperhive/gateway/ rather than
       # the whole parent so the gateway container can't read forge
-      # tokens or other files that may live at the parent level (argus
+      # tokens or other files that may live at the parent level.
-      # 🟡 on #872). c0re writes agents.conf under this subdir;
+      # c0re writes agents.conf under this subdir and triggers an nginx
-      # the systemd path unit inside the container fires nginx -s reload
+      # reload from the host via systemd-run after each write.
-      # on each atomic rename. Pre-created by a tmpfiles rule.
+      # Pre-created by a tmpfiles rule.
       bindMounts."/run/hive-state" = {
         hostPath = "/var/lib/hyperhive/gateway";
         isReadOnly = true;
@@ -389,37 +389,14 @@ in
               '';
           };
 
-          # Watch /run/hive-state/agents.conf (bind-mounted from the
+          # nginx reload is triggered from the HOST side by hive-c0re
-          # host's /var/lib/hyperhive/agents.conf) for changes and
+          # via `systemd-run --machine=hive-gateway nginx -s reload`
-          # trigger an nginx reload when c0re atomically renames a new
+          # after each agents.conf write. A path unit watching the
-          # version into place (#869). PathChanged fires on
+          # bind-mounted file inside the container was tried first
-          # IN_CLOSE_WRITE + IN_MOVED_TO, so the atomic rename c0re
+          # (in #872) but IN_MOVED_TO from an atomic rename on the host
-          # uses (write .conf.tmp → rename) wakes the path unit.
+          # does not propagate across the nspawn mount-namespace boundary,
-          # The reload is a no-op if the new config is identical —
+          # so the watcher never fired (#889). Host-side trigger is the
-          # gateway_nginx::write skips the rename when content is
+          # correct approach.
-          # unchanged, so the path unit doesn't fire at all on quiet
-          # ticks.
-          systemd.paths.hive-gateway-agents-conf = {
-            wantedBy = [ "nginx.service" ];
-            after = [ "nginx.service" ];
-            pathConfig = {
-              PathChanged = "/run/hive-state/agents.conf";
-              Unit = "hive-gateway-nginx-reload.service";
-            };
-          };
-
-          systemd.services.hive-gateway-nginx-reload = {
-            description = "Reload nginx after agents.conf change";
-            # Don't block any target — fires only when the path unit
-            # triggers it.
-            serviceConfig = {
-              Type = "oneshot";
-              # nginx -s reload sends SIGHUP to the master process via
-              # the pid file. Runs as root inside the container (pid 1
-              # is the nspawn init; nginx master starts as root).
-              ExecStart = "/run/current-system/sw/bin/nginx -s reload";
-            };
-          };
 
           services.nginx = {
             enable = true;
@@ -546,14 +523,14 @@ in
                     };
                   };
                 # Per-agent location blocks, generated at runtime by
-                # hive-c0re and written to /var/lib/hyperhive/agents.conf
+                # hive-c0re and written to /var/lib/hyperhive/gateway/agents.conf
                 # on the host. The bind-mount at /run/hive-state/ exposes
                 # that file here. nginx parses `include` at config-load
-                # time so a reload (triggered by the hive-gateway-nginx-
+                # time so a reload (triggered by c0re via systemd-run
-                # reload path unit when agents.conf changes) picks up new
+                # after each agents.conf write) picks up new or removed
-                # or removed agents without a nixos-rebuild. nginx's
+                # agents without a nixos-rebuild. nginx's longest-prefix-
-                # longest-prefix-match rule ensures `/agent/<name>/` from
+                # match rule ensures `/agent/<name>/` from this file beats
-                # this file beats the `/agent/` catch-all above (#869).
+                # the `/agent/` catch-all above.
                 extraConfig = ''
                   include /run/hive-state/agents.conf;
                 '';