diff --git a/docs/gateway.md b/docs/gateway.md index e8614a5b..d8336fce 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -71,13 +71,14 @@ SSH for forge stays direct on `cfg.sshPort` — separate listener protocol, not ## Per-agent unix-socket upstream -Sub-agent `/agent//` upstreams flip from TCP loopback to a -unix-domain socket as each agent opts in. The mechanism: +All agents bind their web UI on a unix-domain socket at +`/run/hive-agent//web.sock` — the `HIVE_WEB_SOCKET` env var is +now set unconditionally for every agent. The mechanism: -1. **Agent side** (`hyperhive.web.useUnixSocket = true` in - `agent.nix`). Sets `HIVE_WEB_SOCKET=/run/hive-agent//web.sock` - on the harness service env; `web_ui::serve` binds a `UnixListener` - at that path instead of TCP. +1. **Agent side**. `HIVE_WEB_SOCKET=/run/hive-agent//web.sock` + is set on every harness service env; `web_ui::serve` binds a + `UnixListener` at that path. The deprecated + `hyperhive.web.useUnixSocket` option is now a no-op. 2. **Host side**. `hive-c0re` bind-mounts the per-agent subdir (`/run/hive-agent//`) into the agent's container. Dir bind, not file bind — file bind-mounts don't survive the @@ -87,32 +88,28 @@ unix-domain socket as each agent opts in. The mechanism: `/hyperhive-socket-bound` next to the socket. c0re's `agent_sockets::write` filters its JSON map by marker presence — only agents whose harness has actually bound the socket appear there. - Without this filter, the gateway would `proxy_pass` to a non-existent - socket for every sub-agent that hasn't opted in yet. (Legacy name - `.bound` also accepted during the transition window.) + (Legacy name `.bound` also accepted during the transition window.) 4. **Gateway side**. `gateway_nginx::write` generates `/var/lib/hyperhive/gateway/agents.conf` — a plain nginx include file with one `location /agent//` block per agent. UDS upstream (`http://unix:/run/hive-agent//web.sock:/`) when - `hyperhive-socket-bound` marker present; TCP loopback otherwise. + `hyperhive-socket-bound` marker present; TCP loopback for agents + that haven't yet been rebuilt under the new config. The gateway container bind-mounts `/var/lib/hyperhive/gateway/` at `/run/hive-state/`; nginx includes `/run/hive-state/agents.conf`. After each write, c0re triggers `nginx -s reload` inside the gateway container from the HOST via - `systemd-run --machine=hive-gateway nginx -s reload`. This is + `systemd-run --machine=hive-gateway --wait nginx -s reload`. This is intentionally host-side: `IN_MOVED_TO` from an atomic rename does not propagate across the nspawn mount-namespace boundary, so a - path unit inside the container would never fire (#889). + path unit inside the container would never fire. c0re regenerates `agents.conf` (and triggers a reload) on two triggers: every topology change (new/removed agents) and every 10s marker poll tick (`agent_sockets::spawn_poll`). `write()` is -idempotent — skips the rename+reload when content is unchanged. - -Transition: agents that haven't flipped `useUnixSocket = true` get a -TCP loopback upstream in `agents.conf` (deterministic port from -`agent_web_port(name)`). A future cleanup will drop the TCP fallback -once every agent's flipped. +idempotent — skips the rename when content is unchanged. Failed reloads +are retried automatically on subsequent poll ticks via +`gateway_nginx::reload_if_pending`. ## Agent port map (`agent-ports.json`) diff --git a/hive-c0re/src/agent_sockets.rs b/hive-c0re/src/agent_sockets.rs index 9a715f36..f31f5b28 100644 --- a/hive-c0re/src/agent_sockets.rs +++ b/hive-c0re/src/agent_sockets.rs @@ -30,12 +30,11 @@ pub const AGENT_SOCKET_DIR: &str = "/run/hive-agent"; pub const SOCKET_FILENAME: &str = "web.sock"; /// Marker file the harness drops next to the socket after a -/// successful `bind_unix`. Presence = "this agent has opted in to -/// `hyperhive.web.useUnixSocket = true` and its harness has bound -/// the socket"; absence = "the harness is still on TCP, don't -/// publish the unix upstream for this agent yet". Without this gate -/// the gateway would `proxy_pass` to a non-existent socket for every -/// sub-agent that hasn't flipped the option yet. +/// successful `bind_unix`. Presence = "harness has bound the socket, +/// unix upstream is live"; absence = "harness hasn't started yet or +/// hasn't been rebuilt under the new config — keep TCP fallback". +/// Without this gate the gateway would `proxy_pass` to a non-existent +/// socket for an agent that's still starting up after a rebuild. /// /// Renamed from `.bound` (legacy) to match the `hyperhive-` prefix /// convention for all harness-written state files. `build_map` @@ -70,8 +69,8 @@ pub fn socket_path_for(name: &str) -> PathBuf { /// Includes manager and sub-agents. Filters by `READY_MARKER` /// presence: only agents whose harness has actually bound the unix /// socket appear in the map. Without this, the gateway would -/// `proxy_pass` to a non-existent socket for every sub-agent that -/// hasn't yet flipped `hyperhive.web.useUnixSocket = true`. +/// `proxy_pass` to a non-existent socket for agents that haven't +/// been rebuilt yet or are mid-restart. /// /// Accepts either the new `hyperhive-socket-bound` marker or the legacy /// `.bound` marker so existing containers keep their gateway routing @@ -166,12 +165,17 @@ pub fn write(names: &[String]) -> Result<()> { } /// Spawn the marker poll task. Periodically re-runs `write` so the -/// JSON map picks up newly-bound sockets (an agent flipping -/// `hyperhive.web.useUnixSocket = true`, rebuilding, then having its -/// harness drop a fresh `.bound` marker) without needing an explicit -/// hook on container start. `write` is idempotent (skips the rename -/// when content unchanged) so the steady-state cost is one directory -/// stat per agent per poll interval. +/// JSON map picks up newly-bound sockets after a rebuild (harness +/// drops a fresh `.bound` marker on start) without needing an explicit +/// hook on container +/// start. `write` is idempotent (skips the rename when content +/// unchanged) so the steady-state cost is one directory stat per +/// agent per poll interval. +/// +/// Also calls `gateway_nginx::reload_if_pending` on every tick to +/// retry a gateway nginx reload that may have failed on the previous +/// tick (e.g. gateway container temporarily down). This recovers +/// gateway routing without needing a manual gateway restart. /// /// Mirrors the spawn-loop shape used by `crash_watch`, /// `reminder_scheduler`, etc. — the existing background-task @@ -198,6 +202,9 @@ pub fn spawn_poll() { if let Err(e) = crate::gateway_nginx::write(&names) { tracing::debug!(error = ?e, "gateway_nginx poll write failed"); } + // Retry a pending nginx reload that failed on a + // previous tick (no-op if no reload is pending). + crate::gateway_nginx::reload_if_pending(); } Err(e) => { tracing::debug!(error = ?e, "agent_sockets poll: failed to list agents"); diff --git a/hive-c0re/src/gateway_nginx.rs b/hive-c0re/src/gateway_nginx.rs index 20a55936..e5cf30c6 100644 --- a/hive-c0re/src/gateway_nginx.rs +++ b/hive-c0re/src/gateway_nginx.rs @@ -8,10 +8,18 @@ use anyhow::{Context, Result}; use std::fmt::Write as _; use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; use crate::agent_sockets; use crate::lifecycle; +/// Set when `write` publishes a new agents.conf; cleared when +/// `reload_gateway_nginx` submits the reload command successfully. +/// Lets `spawn_poll` retry the reload on subsequent ticks when the +/// previous attempt failed (e.g. gateway container temporarily down, +/// systemd-run not found) without re-writing the already-correct file. +static RELOAD_PENDING: AtomicBool = AtomicBool::new(false); + const HOST_CONF_PATH: &str = "/var/lib/hyperhive/gateway/agents.conf"; /// Host-side path where c0re writes the generated nginx include file. @@ -64,7 +72,7 @@ fn render(names: &[String], frontend_dir: Option<&str>) -> String { let mut out = String::from( "# Generated by hive-c0re \u{2014} do not edit.\ \n# Refreshed on every topology change + when agents bind/drop their unix sockets.\ - \n# Gateway reloads nginx automatically on each update (systemd path unit).\n", + \n# Reload triggered by hive-c0re via systemd-run --machine=hive-gateway.\n", ); for name in names { let port = lifecycle::agent_web_port(name); @@ -184,24 +192,40 @@ pub fn write(names: &[String]) -> Result<()> { path.display() ) })?; - // Trigger nginx reload from the host. Ignore errors — a failed - // reload is recoverable (nginx keeps serving the previous config). + // Mark reload pending before attempting so a failed attempt is + // retried by the next spawn_poll tick (see `reload_if_pending`). + RELOAD_PENDING.store(true, Ordering::Relaxed); reload_gateway_nginx(); Ok(()) } +/// Retry a pending nginx reload if a previous attempt failed. +/// Called by `spawn_poll` on each tick so a transient failure +/// (gateway container temporarily down, systemd-run error) is +/// recovered automatically without requiring a new file write. +pub fn reload_if_pending() { + if RELOAD_PENDING.load(Ordering::Relaxed) { + reload_gateway_nginx(); + } +} + /// Send `nginx -s reload` to the gateway container via systemd-run. -/// Runs non-interactively in a transient scope so it doesn't block -/// c0re's polling loop. Best-effort: errors are logged, not bubbled. +/// Uses `--wait` so the exit code reflects whether nginx received the +/// signal; clears `RELOAD_PENDING` on success so `reload_if_pending` +/// stops retrying. Best-effort: errors are logged, not bubbled. fn reload_gateway_nginx() { // `--machine=hive-gateway` targets the container by its nspawn // machine name (same as the nixos-container name). `--quiet` - // suppresses the transient unit name echo. `--` separates - // systemd-run args from the command. + // suppresses the transient unit name echo. `--wait` blocks until + // the transient job exits so the exit code tells us whether + // `nginx -s reload` ran at all (RELOAD_PENDING is only cleared on + // success — a failed attempt is retried next tick). `--` + // separates systemd-run args from the command. let status = std::process::Command::new("systemd-run") .args([ "--machine=hive-gateway", "--quiet", + "--wait", "--", "nginx", "-s", @@ -210,21 +234,22 @@ fn reload_gateway_nginx() { .status(); match status { Ok(s) if s.success() => { - // systemd-run accepted the request; nginx reload runs - // asynchronously inside the container and may still fail - // silently, but that's acceptable given the best-effort contract. - tracing::debug!("systemd-run accepted gateway nginx reload request"); + // nginx -s reload ran successfully (SIGHUP sent to master). + // The actual worker replacement is async but the signal was + // delivered; clear the pending flag. + RELOAD_PENDING.store(false, Ordering::Relaxed); + tracing::debug!("gateway nginx reload signal sent"); } Ok(s) => { tracing::warn!( exit_code = ?s.code(), - "gateway nginx reload exited non-zero; will pick up on next restart" + "gateway nginx reload exited non-zero — will retry next poll tick" ); } Err(e) => { tracing::warn!( error = %e, - "failed to invoke systemd-run for gateway nginx reload" + "failed to invoke systemd-run for gateway nginx reload — will retry next poll tick" ); } } diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 762a26ff..4c2dedc4 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -261,12 +261,12 @@ async fn cmd_serve( // operator-initiated transient state. crash_watch::spawn(coord.clone()); // Agent-sockets marker poll: re-fires `agent_sockets::write` - // every 10s so the JSON picks up newly-bound `.bound` markers - // (a sub-agent flipping `hyperhive.web.useUnixSocket = true`, - // rebuilding, then having its harness bind the socket) without - // needing an explicit hook on each container start. write() is - // idempotent so steady-state cost is one stat per agent per - // tick. See `docs/gateway.md::Per-agent unix-socket upstream`. + // and `gateway_nginx::write` every 10s so the JSON and nginx + // config pick up newly-bound `.bound` markers after a rebuild. + // Also retries any pending gateway nginx reload that failed on + // the previous tick. write() is idempotent so steady-state cost + // is one stat per agent per tick. + // See `docs/gateway.md::Per-agent unix-socket upstream`. agent_sockets::spawn_poll(); // Reminder scheduler: drains due reminders + handles // file_path payload persistence. See reminder_scheduler.rs. diff --git a/nix/templates/harness-base.nix b/nix/templates/harness-base.nix index 1112e767..e157d2c5 100644 --- a/nix/templates/harness-base.nix +++ b/nix/templates/harness-base.nix @@ -80,24 +80,9 @@ in default = false; example = true; description = '' - When `true`, set `HIVE_WEB_SOCKET=/run/hive-agent/${userName}/web.sock` - on the harness service env, which makes `web_ui::serve` bind a - `UnixListener` at that path instead of the legacy TCP listener - on `HIVE_PORT`. - - Default `false` so an agent's web UI keeps binding TCP until - the per-agent flip is explicit. Rollout shape: - - 1. flip one canary agent to `true` via its `agent.nix`; - 2. validate the gateway's `proxy_pass http://unix:.../web.sock` - end-to-end against that canary; - 3. flip remaining agents per-agent as the gateway side soaks; - 4. eventually drop this option once every agent is on unix and - the TCP fallback is removed from the harness. - - Sub-agents only: the manager always has `HIVE_WEB_SOCKET` set - unconditionally in the `isManager` env block, so this toggle - has no effect when `hyperhive.role = "manager"`. + Deprecated. Unix socket mode is now always enabled for all agents. + Setting this option to `true` has no effect and the option will be + removed in a future version. Safe to drop from agent configs. ''; }; @@ -1304,23 +1289,15 @@ in HIVE_STATIC_DIR = "${config.hyperhive.frontend.mergedDist}"; HIVE_ASSETS_DIR = "${pkgs.hyperhive-assets}/share/hyperhive"; HIVE_ROLE = config.hyperhive.role; - } - // lib.optionalAttrs config.hyperhive.web.useUnixSocket { - # Per-agent unix-socket path for the web UI. When set, - # the harness's `web_ui::serve` binds a `UnixListener` - # at this path instead of TCP. Path matches - # `hive_c0re::agent_sockets::socket_path_for(name)` so - # the lifecycle bind-mount and the gateway's upstream - # config all derive from the same canonical - # `/run/hive-agent//web.sock` shape. + # Unix-socket path for the harness web UI. All agents (sub-agents + # and manager) always bind here; TCP fallback is removed. Path + # matches `hive_c0re::agent_sockets::socket_path_for(name)` so + # lifecycle bind-mounts and gateway upstream config stay in sync. HIVE_WEB_SOCKET = "/run/hive-agent/${userName}/web.sock"; } // lib.optionalAttrs isManager { # Standalone-eval fallback; meta.rs overrides at deploy time. HIVE_LABEL = "root"; - # Manager always uses a unix socket so the gateway can route - # /agent// to it the same way it routes sub-agents. - HIVE_WEB_SOCKET = "/run/hive-agent/${userName}/web.sock"; }; serviceConfig = { ExecStart = "${pkgs.hyperhive}/bin/${binary} serve";