From 3a29aee0011dab1f578abf91ff9419cc0a66e4f7 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 31 May 2026 16:00:45 +0200 Subject: [PATCH] nix/hive-gateway: UDS upstream for /agent// (#784 phase 2 step 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch per-agent gateway upstreams from TCP loopback to unix-domain socket when the agent has opted in via `hyperhive.web.useUnixSocket` (#822). Coexists with the TCP path during rollout. Changes: - New `agentSocketsFile` option (default `/var/lib/hyperhive/agent-sockets.json`) — c0re writes the map there via `hive_c0re::agent_sockets::write` (#809). - `agentSocketsTable = lib.importJSON ...` (graceful empty when file missing). - `agentUpstreamFor name port` picks `http://unix::/` when the socket has a JSON entry AND the file exists at eval time; else `http://127.0.0.1:/`. Path-exists gate guards against c0re's blanket-emit shape during the canary window (agents in `agent-sockets.json` who haven't actually flipped have no bound socket on disk → fall back to TCP). Damocles will ship a `.bound` marker filter on the c0re side (#784 step 2d follow-up); once that's in, the path-exists check is redundant but harmless. Step 4 drops it entirely along with the TCP fallback. - `containers.hive-gateway.bindMounts."/run/hive-agent"` — read-only, unconditional. Inert when no agents have opted in. Required so nginx inside the gateway container can `connect(2)` to the per-agent sockets damocles's #813 bind-mounts into agent containers at the same paths. Docs: - `docs/gateway.md::Per-agent UDS upstream (#784)` — full rollout flow, subdir-bind rationale (damocles #813), eval-time gate explainer, step 4 drop plan. `nix flake check` clean; `nix fmt` clean. Canary plan: once #822 (`useUnixSocket` option) lands + this PR merges, manager flips atlas's agent.nix to `useUnixSocket = true` via the config-update flow. End-to-end validation against atlas before broader rollout. --- docs/gateway.md | 54 +++++ nix/modules/hive-gateway.nix | 379 +++++++++++++++++++++-------------- 2 files changed, 278 insertions(+), 155 deletions(-) diff --git a/docs/gateway.md b/docs/gateway.md index 2dd53ef6..91f4bb74 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -165,3 +165,57 @@ Scope is intentionally narrow per mara on #755: "only for routes already special cased in the nginx config". Other gateway routes (forge / matrix / fluffychat) get nginx defaults — extending the custom-error pattern there is a separate follow-up. + +## Per-agent UDS upstream (#784) + +Per-agent `/agent//` upstreams default to TCP loopback +(`http://127.0.0.1:/`) but each agent can opt in to unix- +domain socket upstream by flipping `hyperhive.web.useUnixSocket = +true` in its `agent.nix`. Rollout flow: + +1. **Harness** binds a `UnixListener` at + `/run/hive-agent//web.sock` when `HIVE_WEB_SOCKET` is set + (PR #800). The env var is set by `harness-base.nix` from the + `useUnixSocket` option (#822). +2. **hive-c0re** writes a sibling `agent-sockets.json` next to + `agent-ports.json` (PR #809) and bind-mounts the per-agent + subdir `/run/hive-agent//` into each sub-agent container + via `set_nspawn_flags` (PR #813). Path-shape lives in + `hive_c0re::agent_sockets::socket_path_for(name)` — one canonical + derivation, no triangulation across the c0re / harness / gateway + boundaries. +3. **Gateway** reads both `agentPortsFile` + `agentSocketsFile` at + deploy time. Per agent: a socket entry beats the TCP port. The + gateway container bind-mounts `/run/hive-agent/` read-only so + nginx inside can `connect(2)` to the per-agent sockets. + +Mixed state during rollout: agents flip per-agent. Agents that +haven't opted in keep the TCP path; agents that have flipped use +the UDS path. The two coexist on the same gateway with zero +per-agent special-casing in the nginx config (`agentUpstreamFor` +resolves the right shape from the JSON maps). + +**Eval-time gate during the rollout window**: `agentUpstreamFor` +checks `builtins.pathExists` on the socket path before picking the +UDS upstream. c0re's `agent_sockets::write` emits an entry for every +sub-agent regardless of whether they've actually flipped, so the +gateway has no other signal that a given agent is or isn't actually +binding the socket. The path-exists check works because a flipped +agent's harness binds the socket on container start, and the +gateway-container rebuild (which re-runs nix eval) happens on every +topology change — so a freshly-flipped agent flips through TCP → +UDS over one rebuild cycle. Once c0re's `.bound` marker filter +ships (#784 step 2d follow-up), `agent-sockets.json` only contains +agents that have actually bound, and the path-exists check is +redundant but harmless. Step 4 drops it. + +**Why per-agent subdir** (not a flat `/run/hive-agent/.sock`): +the harness's `bind_unix` helper unlinks any stale socket before +calling `bind(2)`, and a file bind-mount loses its host-side anchor +on unlink. Dir bind-mount keeps the same dir inode visible on both +sides, so the new `web.sock` shows up on the host the moment the +harness binds it (damocles #813 design note). + +**Step 4 plan**: once every agent has flipped + soaked, the +`agentPortsFile` fallback drops + the harness's TCP bind goes away +entirely. Tracked at #784 step 4. diff --git a/nix/modules/hive-gateway.nix b/nix/modules/hive-gateway.nix index 26d322a6..6f63acbf 100644 --- a/nix/modules/hive-gateway.nix +++ b/nix/modules/hive-gateway.nix @@ -20,6 +20,33 @@ let else builtins.fromJSON (builtins.readFile cfg.agentPortsFile); + # Per-agent unix-socket table for `/agent//` UDS upstream + # (#784 phase 2 step 3). C0re writes this JSON alongside + # agent-ports.json; gateway reads at deploy time. Per-agent the + # entry wins over the TCP port. Missing entry (or missing file) + # → fall back to the TCP port. See + # `docs/gateway.md::Per-agent UDS upstream (#784)`. + agentSocketsTable = + if cfg.agentSocketsFile == null || !builtins.pathExists cfg.agentSocketsFile then + { } + else + builtins.fromJSON (builtins.readFile cfg.agentSocketsFile); + + # Resolve a per-agent upstream URL. Socket entry wins ONLY when the + # socket file actually exists at eval time — guards against agents + # that have an `agent-sockets.json` entry from c0re's blanket emit + # but haven't actually flipped `hyperhive.web.useUnixSocket = true` + # (their harness still binds TCP only, so a UDS upstream would 502). + # Falls back to the TCP loopback otherwise. Once c0re ships the + # `.bound` marker filter (#784 step 2d follow-up), the path-exists + # check becomes redundant but harmless; step 4 drops it entirely. + agentUpstreamFor = + name: port: + if agentSocketsTable ? ${name} && builtins.pathExists agentSocketsTable.${name} then + "http://unix:${agentSocketsTable.${name}}:/" + else + "http://127.0.0.1:${toString port}/"; + # Static error pages for `/agent//` mishaps (#755). Mara's # call: useful pages instead of nginx's default 404/502 for routes # we've already special-cased. See `docs/gateway.md::Per-agent @@ -204,6 +231,40 @@ in include + reload + eventual per-agent unix sockets. ''; }; + + agentSocketsFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = "/var/lib/hyperhive/agent-sockets.json"; + example = "/var/lib/hyperhive/agent-sockets.json"; + description = '' + Path to a JSON file mapping sub-agent names to their + per-agent unix-socket paths for `/agent//` UDS upstream + routing (#784 phase 2 step 3). Shape: + `{ "": "/run/hive-agent//web.sock", ... }`. + Written by hive-c0re alongside `agentPortsFile` on every + topology change (`hive_c0re::agent_sockets::write`; + path-shape derives from + `agent_sockets::socket_path_for(name)`). + + Per-agent, the socket entry wins over the TCP port: when an + agent appears in this map, the gateway's `proxy_pass` for + that agent's `/agent//` location targets + `http://unix::/` instead of `http://127.0.0.1:/`. + Agents that haven't opted in (no `HIVE_WEB_SOCKET` set, + no entry in the JSON, or both files unset) fall back to + TCP via `agentPortsFile`. Coexists with the TCP map during + the rollout — eventually drops `agentPortsFile` entirely + when every agent's flipped (#784 step 4). + + Set to `null` to skip UDS upstreams entirely (gateway uses + TCP for every agent regardless of what hive-c0re writes). + + **Bind-mount requirement**: when this is enabled the gateway + container needs `/run/hive-agent/` bind-mounted from the + host. Handled automatically by `containers.hive-gateway` + below when at least one socket entry exists. + ''; + }; }; config = lib.mkIf cfg.enable { @@ -226,6 +287,17 @@ in # port-forward dance, and the firewall config below is the only # layer that matters. privateNetwork = false; + # Bind-mount the per-agent socket dir so nginx inside the gateway + # container can `connect(2)` to the UDS upstreams hive-c0re + # publishes in `agent-sockets.json` (#784 phase 2 step 3). + # Read-only (we don't bind anything here; just connect). Mount + # is unconditional but inert when no agents have opted in: + # agent-sockets.json missing/empty → `agentSocketsTable = {}` + # → every per-agent location uses the TCP fallback. + bindMounts."/run/hive-agent" = { + hostPath = "/run/hive-agent"; + isReadOnly = true; + }; config = { pkgs, ... }: { @@ -247,143 +319,141 @@ in ''; virtualHosts = { "_" = { - listen = [ - { - addr = "0.0.0.0"; - port = cfg.port; - } - ]; - locations = - # `/matrix/*` → 301 → `matrix./$1` - # (fluffychat moved to sub-domain root in #772; this - # keeps bookmarks + deep-links working during the - # transition). See `docs/gateway.md` for the vhost - # map. - lib.optionalAttrs ( - matrixCfg.enable - && matrixCfg.gui.enable - && matrixCfg.gatewayHost != null - ) ( - let - portSuffix = if cfg.port == 80 then "" else ":${toString cfg.port}"; - target = "http://${matrixCfg.gatewayHost}${portSuffix}"; - in + listen = [ { - "/matrix/" = { - extraConfig = '' - rewrite ^/matrix/(.*)$ ${target}/$1 permanent; - ''; - }; + addr = "0.0.0.0"; + port = cfg.port; } - ) - // - # `.well-known/matrix/{client,server}` discovery JSON. - # Points clients at `matrixCfg.gatewayHost` (sub-domain - # vhost) when set; falls back to direct `:` - # when no gateway target. CORS `*` per matrix spec. - # See `docs/gateway.md` "Discovery flow" for the full - # client-bootstrap sequence. - lib.optionalAttrs (matrixCfg.enable && hyperhiveDomain != null) ( + ]; + locations = + # `/matrix/*` → 301 → `matrix./$1` + # (fluffychat moved to sub-domain root in #772; this + # keeps bookmarks + deep-links working during the + # transition). See `docs/gateway.md` for the vhost + # map. + lib.optionalAttrs (matrixCfg.enable && matrixCfg.gui.enable && matrixCfg.gatewayHost != null) ( let portSuffix = if cfg.port == 80 then "" else ":${toString cfg.port}"; - clientBaseUrl = - if matrixCfg.gatewayHost != null then - "http://${matrixCfg.gatewayHost}${portSuffix}" - else - "http://${hyperhiveDomain}:${toString matrixCfg.httpPort}"; - serverHostPort = - if matrixCfg.gatewayHost != null then - "${matrixCfg.gatewayHost}${portSuffix}" - else - "${hyperhiveDomain}:${toString matrixCfg.httpPort}"; + target = "http://${matrixCfg.gatewayHost}${portSuffix}"; in { - "= /.well-known/matrix/client" = { + "/matrix/" = { extraConfig = '' - default_type application/json; - add_header Access-Control-Allow-Origin *; - return 200 '{"m.homeserver":{"base_url":"${clientBaseUrl}"}}'; - ''; - }; - "= /.well-known/matrix/server" = { - extraConfig = '' - default_type application/json; - return 200 '{"m.server":"${serverHostPort}"}'; + rewrite ^/matrix/(.*)$ ${target}/$1 permanent; ''; }; } ) - // - # Per-agent UIs (#15 v0). One `/agent//` block - # per entry in `agentPortsTable`. Trailing-slash pair - # strips the prefix; `X-Forwarded-Prefix` lets the - # harness build absolute URLs when relative isn't - # enough. `proxy_intercept_errors` + `error_page` rewrite - # upstream 502/503/504 (container down / restarting) to - # the static `unreachable.html` instead of nginx's - # default Bad Gateway page (#755). See - # `docs/gateway.md` for the vhost map + tuning. - lib.mapAttrs' (name: port: { - name = "/agent/${name}/"; - value = { - proxyPass = "http://127.0.0.1:${toString port}/"; + // + # `.well-known/matrix/{client,server}` discovery JSON. + # Points clients at `matrixCfg.gatewayHost` (sub-domain + # vhost) when set; falls back to direct `:` + # when no gateway target. CORS `*` per matrix spec. + # See `docs/gateway.md` "Discovery flow" for the full + # client-bootstrap sequence. + lib.optionalAttrs (matrixCfg.enable && hyperhiveDomain != null) ( + let + portSuffix = if cfg.port == 80 then "" else ":${toString cfg.port}"; + clientBaseUrl = + if matrixCfg.gatewayHost != null then + "http://${matrixCfg.gatewayHost}${portSuffix}" + else + "http://${hyperhiveDomain}:${toString matrixCfg.httpPort}"; + serverHostPort = + if matrixCfg.gatewayHost != null then + "${matrixCfg.gatewayHost}${portSuffix}" + else + "${hyperhiveDomain}:${toString matrixCfg.httpPort}"; + in + { + "= /.well-known/matrix/client" = { + extraConfig = '' + default_type application/json; + add_header Access-Control-Allow-Origin *; + return 200 '{"m.homeserver":{"base_url":"${clientBaseUrl}"}}'; + ''; + }; + "= /.well-known/matrix/server" = { + extraConfig = '' + default_type application/json; + return 200 '{"m.server":"${serverHostPort}"}'; + ''; + }; + } + ) + // + # Per-agent UIs (#15 v0; UDS upstream #784 step 3). + # One `/agent//` block per entry in + # `agentPortsTable`. `agentUpstreamFor` resolves + # to `http://unix::/` when the agent has + # opted in via `hyperhive.web.useUnixSocket` (and + # appears in `agentSocketsTable`); otherwise + # `http://127.0.0.1:/`. Trailing-slash pair + # strips the prefix; `X-Forwarded-Prefix` lets the + # harness build absolute URLs when relative isn't + # enough. `proxy_intercept_errors` + `error_page` rewrite + # upstream 502/503/504 to `unreachable.html` (#755). + lib.mapAttrs' (name: port: { + name = "/agent/${name}/"; + value = { + proxyPass = agentUpstreamFor name port; + proxyWebsockets = true; + extraConfig = '' + proxy_set_header X-Forwarded-Prefix /agent/${name}; + proxy_buffering off; + proxy_read_timeout 1d; + proxy_intercept_errors on; + error_page 502 503 504 = /__hive_agent_unreachable; + ''; + }; + }) agentPortsTable + // + # `/agent/` catch-all (#755): hits when an operator + # requests `/agent//...` — a name not in + # `agentPortsTable`. Without this it falls through to + # `/` (c0re dashboard upstream) which returns 404 + # with no useful context. Custom 404 page instead. + { + "/agent/" = { + extraConfig = '' + error_page 404 = /__hive_agent_not_found; + return 404; + ''; + }; + # Internal static-file locations the error_page + # directives above point at. `internal` keeps + # operators from hitting the file directly (only + # nginx's error-handling can reach it); `alias` + # serves the exact file regardless of request URI. + "= /__hive_agent_not_found" = { + extraConfig = '' + internal; + alias ${agentErrorPagesDir}/not-found.html; + default_type text/html; + ''; + }; + "= /__hive_agent_unreachable" = { + extraConfig = '' + internal; + alias ${agentErrorPagesDir}/unreachable.html; + default_type text/html; + ''; + }; + } + // { + # Everything else proxies to hive-c0re. Upgrade + # headers stay set so SSE (`/dashboard/stream`, + # `/events/stream`) + websocket (`/screen/ws`) + # endpoints keep working transparently. + "/" = { + proxyPass = "http://${cfg.upstreamHost}:${toString cfg.upstreamPort}"; proxyWebsockets = true; extraConfig = '' - proxy_set_header X-Forwarded-Prefix /agent/${name}; proxy_buffering off; proxy_read_timeout 1d; - proxy_intercept_errors on; - error_page 502 503 504 = /__hive_agent_unreachable; ''; }; - }) agentPortsTable - // - # `/agent/` catch-all (#755): hits when an operator - # requests `/agent//...` — a name not in - # `agentPortsTable`. Without this it falls through to - # `/` (c0re dashboard upstream) which returns 404 - # with no useful context. Custom 404 page instead. - { - "/agent/" = { - extraConfig = '' - error_page 404 = /__hive_agent_not_found; - return 404; - ''; - }; - # Internal static-file locations the error_page - # directives above point at. `internal` keeps - # operators from hitting the file directly (only - # nginx's error-handling can reach it); `alias` - # serves the exact file regardless of request URI. - "= /__hive_agent_not_found" = { - extraConfig = '' - internal; - alias ${agentErrorPagesDir}/not-found.html; - default_type text/html; - ''; - }; - "= /__hive_agent_unreachable" = { - extraConfig = '' - internal; - alias ${agentErrorPagesDir}/unreachable.html; - default_type text/html; - ''; - }; - } - // { - # Everything else proxies to hive-c0re. Upgrade - # headers stay set so SSE (`/dashboard/stream`, - # `/events/stream`) + websocket (`/screen/ws`) - # endpoints keep working transparently. - "/" = { - proxyPass = "http://${cfg.upstreamHost}:${toString cfg.upstreamPort}"; - proxyWebsockets = true; - extraConfig = '' - proxy_buffering off; - proxy_read_timeout 1d; - ''; }; - }; }; } // @@ -412,7 +482,7 @@ in }; }; } - // + // # Matrix sub-domain vhost (#747 / #764). `server_name = # matrixCfg.gatewayHost`. `/_matrix/*` → tuwunel (CORS *, # 50M body cap, 1h long-poll timeout). `/` serves @@ -427,48 +497,47 @@ in port = cfg.port; } ]; - locations = + locations = { + "/_matrix/" = { + proxyPass = "http://127.0.0.1:${toString matrixCfg.httpPort}"; + proxyWebsockets = true; + extraConfig = '' + proxy_buffering off; + client_max_body_size 50M; + proxy_read_timeout 1h; + proxy_send_timeout 1h; + add_header Access-Control-Allow-Origin *; + ''; + }; + } + // lib.optionalAttrs (matrixCfg.gui.enable) ( { - "/_matrix/" = { - proxyPass = "http://127.0.0.1:${toString matrixCfg.httpPort}"; - proxyWebsockets = true; + # fluffychat at sub-domain root, SPA-fallback via + # the Accept-header `$matrix_spa_target` map. + "/" = { + alias = "${matrixCfg.gui.package}/"; extraConfig = '' - proxy_buffering off; - client_max_body_size 50M; - proxy_read_timeout 1h; - proxy_send_timeout 1h; - add_header Access-Control-Allow-Origin *; + try_files $uri $uri/ $matrix_spa_target =404; ''; }; } - // lib.optionalAttrs (matrixCfg.gui.enable) ( - { - # fluffychat at sub-domain root, SPA-fallback via - # the Accept-header `$matrix_spa_target` map. - "/" = { - alias = "${matrixCfg.gui.package}/"; - extraConfig = '' - try_files $uri $uri/ $matrix_spa_target =404; - ''; - }; - } - // lib.optionalAttrs (hyperhiveDomain != null) { - # FluffyChat boot-config pre-fill so the client's - # `.well-known/matrix/client` lookup hits the - # right delegation endpoint (#736). - "= /config.json" = { - extraConfig = '' - default_type application/json; - return 200 '{"defaultHomeserver":"${hyperhiveDomain}"}'; - ''; - }; - } - ) - // lib.optionalAttrs (!matrixCfg.gui.enable) { - "/" = { - return = "404"; + // lib.optionalAttrs (hyperhiveDomain != null) { + # FluffyChat boot-config pre-fill so the client's + # `.well-known/matrix/client` lookup hits the + # right delegation endpoint (#736). + "= /config.json" = { + extraConfig = '' + default_type application/json; + return 200 '{"defaultHomeserver":"${hyperhiveDomain}"}'; + ''; }; + } + ) + // lib.optionalAttrs (!matrixCfg.gui.enable) { + "/" = { + return = "404"; }; + }; }; }; };