diff --git a/docs/gateway.md b/docs/gateway.md index 91f4bb74..2dd53ef6 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -165,57 +165,3 @@ Scope is intentionally narrow per mara on #755: "only for routes already special cased in the nginx config". Other gateway routes (forge / matrix / fluffychat) get nginx defaults — extending the custom-error pattern there is a separate follow-up. - -## Per-agent UDS upstream (#784) - -Per-agent `/agent//` upstreams default to TCP loopback -(`http://127.0.0.1:/`) but each agent can opt in to unix- -domain socket upstream by flipping `hyperhive.web.useUnixSocket = -true` in its `agent.nix`. Rollout flow: - -1. **Harness** binds a `UnixListener` at - `/run/hive-agent//web.sock` when `HIVE_WEB_SOCKET` is set - (PR #800). The env var is set by `harness-base.nix` from the - `useUnixSocket` option (#822). -2. **hive-c0re** writes a sibling `agent-sockets.json` next to - `agent-ports.json` (PR #809) and bind-mounts the per-agent - subdir `/run/hive-agent//` into each sub-agent container - via `set_nspawn_flags` (PR #813). Path-shape lives in - `hive_c0re::agent_sockets::socket_path_for(name)` — one canonical - derivation, no triangulation across the c0re / harness / gateway - boundaries. -3. **Gateway** reads both `agentPortsFile` + `agentSocketsFile` at - deploy time. Per agent: a socket entry beats the TCP port. The - gateway container bind-mounts `/run/hive-agent/` read-only so - nginx inside can `connect(2)` to the per-agent sockets. - -Mixed state during rollout: agents flip per-agent. Agents that -haven't opted in keep the TCP path; agents that have flipped use -the UDS path. The two coexist on the same gateway with zero -per-agent special-casing in the nginx config (`agentUpstreamFor` -resolves the right shape from the JSON maps). - -**Eval-time gate during the rollout window**: `agentUpstreamFor` -checks `builtins.pathExists` on the socket path before picking the -UDS upstream. c0re's `agent_sockets::write` emits an entry for every -sub-agent regardless of whether they've actually flipped, so the -gateway has no other signal that a given agent is or isn't actually -binding the socket. The path-exists check works because a flipped -agent's harness binds the socket on container start, and the -gateway-container rebuild (which re-runs nix eval) happens on every -topology change — so a freshly-flipped agent flips through TCP → -UDS over one rebuild cycle. Once c0re's `.bound` marker filter -ships (#784 step 2d follow-up), `agent-sockets.json` only contains -agents that have actually bound, and the path-exists check is -redundant but harmless. Step 4 drops it. - -**Why per-agent subdir** (not a flat `/run/hive-agent/.sock`): -the harness's `bind_unix` helper unlinks any stale socket before -calling `bind(2)`, and a file bind-mount loses its host-side anchor -on unlink. Dir bind-mount keeps the same dir inode visible on both -sides, so the new `web.sock` shows up on the host the moment the -harness binds it (damocles #813 design note). - -**Step 4 plan**: once every agent has flipped + soaked, the -`agentPortsFile` fallback drops + the harness's TCP bind goes away -entirely. Tracked at #784 step 4. diff --git a/nix/modules/hive-gateway.nix b/nix/modules/hive-gateway.nix index ae787d37..26d322a6 100644 --- a/nix/modules/hive-gateway.nix +++ b/nix/modules/hive-gateway.nix @@ -20,33 +20,6 @@ let else builtins.fromJSON (builtins.readFile cfg.agentPortsFile); - # Per-agent unix-socket table for `/agent//` UDS upstream - # (#784 phase 2 step 3). C0re writes this JSON alongside - # agent-ports.json; gateway reads at deploy time. Per-agent the - # entry wins over the TCP port. Missing entry (or missing file) - # → fall back to the TCP port. See - # `docs/gateway.md::Per-agent UDS upstream (#784)`. - agentSocketsTable = - if cfg.agentSocketsFile == null || !builtins.pathExists cfg.agentSocketsFile then - { } - else - builtins.fromJSON (builtins.readFile cfg.agentSocketsFile); - - # Resolve a per-agent upstream URL. Socket entry wins ONLY when the - # socket file actually exists at eval time — guards against agents - # that have an `agent-sockets.json` entry from c0re's blanket emit - # but haven't actually flipped `hyperhive.web.useUnixSocket = true` - # (their harness still binds TCP only, so a UDS upstream would 502). - # Falls back to the TCP loopback otherwise. Once c0re ships the - # `.bound` marker filter (#784 step 2d follow-up), the path-exists - # check becomes redundant but harmless; step 4 drops it entirely. - agentUpstreamFor = - name: port: - if agentSocketsTable ? ${name} && builtins.pathExists agentSocketsTable.${name} then - "http://unix:${agentSocketsTable.${name}}:/" - else - "http://127.0.0.1:${toString port}/"; - # Static error pages for `/agent//` mishaps (#755). Mara's # call: useful pages instead of nginx's default 404/502 for routes # we've already special-cased. See `docs/gateway.md::Per-agent @@ -231,40 +204,6 @@ in include + reload + eventual per-agent unix sockets. ''; }; - - agentSocketsFile = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = "/var/lib/hyperhive/agent-sockets.json"; - example = "/var/lib/hyperhive/agent-sockets.json"; - description = '' - Path to a JSON file mapping sub-agent names to their - per-agent unix-socket paths for `/agent//` UDS upstream - routing (#784 phase 2 step 3). Shape: - `{ "": "/run/hive-agent//web.sock", ... }`. - Written by hive-c0re alongside `agentPortsFile` on every - topology change (`hive_c0re::agent_sockets::write`; - path-shape derives from - `agent_sockets::socket_path_for(name)`). - - Per-agent, the socket entry wins over the TCP port: when an - agent appears in this map, the gateway's `proxy_pass` for - that agent's `/agent//` location targets - `http://unix::/` instead of `http://127.0.0.1:/`. - Agents that haven't opted in (no `HIVE_WEB_SOCKET` set, - no entry in the JSON, or both files unset) fall back to - TCP via `agentPortsFile`. Coexists with the TCP map during - the rollout — eventually drops `agentPortsFile` entirely - when every agent's flipped (#784 step 4). - - Set to `null` to skip UDS upstreams entirely (gateway uses - TCP for every agent regardless of what hive-c0re writes). - - **Bind-mount requirement**: when this is enabled the gateway - container needs `/run/hive-agent/` bind-mounted from the - host. Handled automatically by `containers.hive-gateway` - below when at least one socket entry exists. - ''; - }; }; config = lib.mkIf cfg.enable { @@ -279,16 +218,6 @@ in } ]; - # Ensure the per-agent UDS bind-mount source exists at host boot, - # before the gateway container's first start. nspawn would - # auto-create an empty dir if missing (argus 🟡 on #829), but a - # tmpfiles rule makes the intent explicit and dodges the - # fresh-boot-before-any-agent-spawn window where the dir wouldn't - # exist yet from c0re's per-agent `set_nspawn_flags` mkdir chain. - systemd.tmpfiles.rules = [ - "d /run/hive-agent 0755 root root - -" - ]; - containers.hive-gateway = { autoStart = true; ephemeral = false; @@ -297,17 +226,6 @@ in # port-forward dance, and the firewall config below is the only # layer that matters. privateNetwork = false; - # Bind-mount the per-agent socket dir so nginx inside the gateway - # container can `connect(2)` to the UDS upstreams hive-c0re - # publishes in `agent-sockets.json` (#784 phase 2 step 3). - # Read-only (we don't bind anything here; just connect). Mount - # is unconditional but inert when no agents have opted in: - # agent-sockets.json missing/empty → `agentSocketsTable = {}` - # → every per-agent location uses the TCP fallback. - bindMounts."/run/hive-agent" = { - hostPath = "/run/hive-agent"; - isReadOnly = true; - }; config = { pkgs, ... }: { @@ -329,141 +247,143 @@ in ''; virtualHosts = { "_" = { - listen = [ + listen = [ + { + addr = "0.0.0.0"; + port = cfg.port; + } + ]; + locations = + # `/matrix/*` → 301 → `matrix./$1` + # (fluffychat moved to sub-domain root in #772; this + # keeps bookmarks + deep-links working during the + # transition). See `docs/gateway.md` for the vhost + # map. + lib.optionalAttrs ( + matrixCfg.enable + && matrixCfg.gui.enable + && matrixCfg.gatewayHost != null + ) ( + let + portSuffix = if cfg.port == 80 then "" else ":${toString cfg.port}"; + target = "http://${matrixCfg.gatewayHost}${portSuffix}"; + in { - addr = "0.0.0.0"; - port = cfg.port; + "/matrix/" = { + extraConfig = '' + rewrite ^/matrix/(.*)$ ${target}/$1 permanent; + ''; + }; } - ]; - locations = - # `/matrix/*` → 301 → `matrix./$1` - # (fluffychat moved to sub-domain root in #772; this - # keeps bookmarks + deep-links working during the - # transition). See `docs/gateway.md` for the vhost - # map. - lib.optionalAttrs (matrixCfg.enable && matrixCfg.gui.enable && matrixCfg.gatewayHost != null) ( + ) + // + # `.well-known/matrix/{client,server}` discovery JSON. + # Points clients at `matrixCfg.gatewayHost` (sub-domain + # vhost) when set; falls back to direct `:` + # when no gateway target. CORS `*` per matrix spec. + # See `docs/gateway.md` "Discovery flow" for the full + # client-bootstrap sequence. + lib.optionalAttrs (matrixCfg.enable && hyperhiveDomain != null) ( let portSuffix = if cfg.port == 80 then "" else ":${toString cfg.port}"; - target = "http://${matrixCfg.gatewayHost}${portSuffix}"; + clientBaseUrl = + if matrixCfg.gatewayHost != null then + "http://${matrixCfg.gatewayHost}${portSuffix}" + else + "http://${hyperhiveDomain}:${toString matrixCfg.httpPort}"; + serverHostPort = + if matrixCfg.gatewayHost != null then + "${matrixCfg.gatewayHost}${portSuffix}" + else + "${hyperhiveDomain}:${toString matrixCfg.httpPort}"; in { - "/matrix/" = { + "= /.well-known/matrix/client" = { extraConfig = '' - rewrite ^/matrix/(.*)$ ${target}/$1 permanent; + default_type application/json; + add_header Access-Control-Allow-Origin *; + return 200 '{"m.homeserver":{"base_url":"${clientBaseUrl}"}}'; + ''; + }; + "= /.well-known/matrix/server" = { + extraConfig = '' + default_type application/json; + return 200 '{"m.server":"${serverHostPort}"}'; ''; }; } ) - // - # `.well-known/matrix/{client,server}` discovery JSON. - # Points clients at `matrixCfg.gatewayHost` (sub-domain - # vhost) when set; falls back to direct `:` - # when no gateway target. CORS `*` per matrix spec. - # See `docs/gateway.md` "Discovery flow" for the full - # client-bootstrap sequence. - lib.optionalAttrs (matrixCfg.enable && hyperhiveDomain != null) ( - let - portSuffix = if cfg.port == 80 then "" else ":${toString cfg.port}"; - clientBaseUrl = - if matrixCfg.gatewayHost != null then - "http://${matrixCfg.gatewayHost}${portSuffix}" - else - "http://${hyperhiveDomain}:${toString matrixCfg.httpPort}"; - serverHostPort = - if matrixCfg.gatewayHost != null then - "${matrixCfg.gatewayHost}${portSuffix}" - else - "${hyperhiveDomain}:${toString matrixCfg.httpPort}"; - in - { - "= /.well-known/matrix/client" = { - extraConfig = '' - default_type application/json; - add_header Access-Control-Allow-Origin *; - return 200 '{"m.homeserver":{"base_url":"${clientBaseUrl}"}}'; - ''; - }; - "= /.well-known/matrix/server" = { - extraConfig = '' - default_type application/json; - return 200 '{"m.server":"${serverHostPort}"}'; - ''; - }; - } - ) - // - # Per-agent UIs (#15 v0; UDS upstream #784 step 3). - # One `/agent//` block per entry in - # `agentPortsTable`. `agentUpstreamFor` resolves - # to `http://unix::/` when the agent has - # opted in via `hyperhive.web.useUnixSocket` (and - # appears in `agentSocketsTable`); otherwise - # `http://127.0.0.1:/`. Trailing-slash pair - # strips the prefix; `X-Forwarded-Prefix` lets the - # harness build absolute URLs when relative isn't - # enough. `proxy_intercept_errors` + `error_page` rewrite - # upstream 502/503/504 to `unreachable.html` (#755). - lib.mapAttrs' (name: port: { - name = "/agent/${name}/"; - value = { - proxyPass = agentUpstreamFor name port; - proxyWebsockets = true; - extraConfig = '' - proxy_set_header X-Forwarded-Prefix /agent/${name}; - proxy_buffering off; - proxy_read_timeout 1d; - proxy_intercept_errors on; - error_page 502 503 504 = /__hive_agent_unreachable; - ''; - }; - }) agentPortsTable - // - # `/agent/` catch-all (#755): hits when an operator - # requests `/agent//...` — a name not in - # `agentPortsTable`. Without this it falls through to - # `/` (c0re dashboard upstream) which returns 404 - # with no useful context. Custom 404 page instead. - { - "/agent/" = { - extraConfig = '' - error_page 404 = /__hive_agent_not_found; - return 404; - ''; - }; - # Internal static-file locations the error_page - # directives above point at. `internal` keeps - # operators from hitting the file directly (only - # nginx's error-handling can reach it); `alias` - # serves the exact file regardless of request URI. - "= /__hive_agent_not_found" = { - extraConfig = '' - internal; - alias ${agentErrorPagesDir}/not-found.html; - default_type text/html; - ''; - }; - "= /__hive_agent_unreachable" = { - extraConfig = '' - internal; - alias ${agentErrorPagesDir}/unreachable.html; - default_type text/html; - ''; - }; - } - // { - # Everything else proxies to hive-c0re. Upgrade - # headers stay set so SSE (`/dashboard/stream`, - # `/events/stream`) + websocket (`/screen/ws`) - # endpoints keep working transparently. - "/" = { - proxyPass = "http://${cfg.upstreamHost}:${toString cfg.upstreamPort}"; + // + # Per-agent UIs (#15 v0). One `/agent//` block + # per entry in `agentPortsTable`. Trailing-slash pair + # strips the prefix; `X-Forwarded-Prefix` lets the + # harness build absolute URLs when relative isn't + # enough. `proxy_intercept_errors` + `error_page` rewrite + # upstream 502/503/504 (container down / restarting) to + # the static `unreachable.html` instead of nginx's + # default Bad Gateway page (#755). See + # `docs/gateway.md` for the vhost map + tuning. + lib.mapAttrs' (name: port: { + name = "/agent/${name}/"; + value = { + proxyPass = "http://127.0.0.1:${toString port}/"; proxyWebsockets = true; extraConfig = '' + proxy_set_header X-Forwarded-Prefix /agent/${name}; proxy_buffering off; proxy_read_timeout 1d; + proxy_intercept_errors on; + error_page 502 503 504 = /__hive_agent_unreachable; ''; }; + }) agentPortsTable + // + # `/agent/` catch-all (#755): hits when an operator + # requests `/agent//...` — a name not in + # `agentPortsTable`. Without this it falls through to + # `/` (c0re dashboard upstream) which returns 404 + # with no useful context. Custom 404 page instead. + { + "/agent/" = { + extraConfig = '' + error_page 404 = /__hive_agent_not_found; + return 404; + ''; + }; + # Internal static-file locations the error_page + # directives above point at. `internal` keeps + # operators from hitting the file directly (only + # nginx's error-handling can reach it); `alias` + # serves the exact file regardless of request URI. + "= /__hive_agent_not_found" = { + extraConfig = '' + internal; + alias ${agentErrorPagesDir}/not-found.html; + default_type text/html; + ''; + }; + "= /__hive_agent_unreachable" = { + extraConfig = '' + internal; + alias ${agentErrorPagesDir}/unreachable.html; + default_type text/html; + ''; + }; + } + // { + # Everything else proxies to hive-c0re. Upgrade + # headers stay set so SSE (`/dashboard/stream`, + # `/events/stream`) + websocket (`/screen/ws`) + # endpoints keep working transparently. + "/" = { + proxyPass = "http://${cfg.upstreamHost}:${toString cfg.upstreamPort}"; + proxyWebsockets = true; + extraConfig = '' + proxy_buffering off; + proxy_read_timeout 1d; + ''; }; + }; }; } // @@ -492,7 +412,7 @@ in }; }; } - // + // # Matrix sub-domain vhost (#747 / #764). `server_name = # matrixCfg.gatewayHost`. `/_matrix/*` → tuwunel (CORS *, # 50M body cap, 1h long-poll timeout). `/` serves @@ -507,47 +427,48 @@ in port = cfg.port; } ]; - locations = { - "/_matrix/" = { - proxyPass = "http://127.0.0.1:${toString matrixCfg.httpPort}"; - proxyWebsockets = true; - extraConfig = '' - proxy_buffering off; - client_max_body_size 50M; - proxy_read_timeout 1h; - proxy_send_timeout 1h; - add_header Access-Control-Allow-Origin *; - ''; - }; - } - // lib.optionalAttrs (matrixCfg.gui.enable) ( + locations = { - # fluffychat at sub-domain root, SPA-fallback via - # the Accept-header `$matrix_spa_target` map. + "/_matrix/" = { + proxyPass = "http://127.0.0.1:${toString matrixCfg.httpPort}"; + proxyWebsockets = true; + extraConfig = '' + proxy_buffering off; + client_max_body_size 50M; + proxy_read_timeout 1h; + proxy_send_timeout 1h; + add_header Access-Control-Allow-Origin *; + ''; + }; + } + // lib.optionalAttrs (matrixCfg.gui.enable) ( + { + # fluffychat at sub-domain root, SPA-fallback via + # the Accept-header `$matrix_spa_target` map. + "/" = { + alias = "${matrixCfg.gui.package}/"; + extraConfig = '' + try_files $uri $uri/ $matrix_spa_target =404; + ''; + }; + } + // lib.optionalAttrs (hyperhiveDomain != null) { + # FluffyChat boot-config pre-fill so the client's + # `.well-known/matrix/client` lookup hits the + # right delegation endpoint (#736). + "= /config.json" = { + extraConfig = '' + default_type application/json; + return 200 '{"defaultHomeserver":"${hyperhiveDomain}"}'; + ''; + }; + } + ) + // lib.optionalAttrs (!matrixCfg.gui.enable) { "/" = { - alias = "${matrixCfg.gui.package}/"; - extraConfig = '' - try_files $uri $uri/ $matrix_spa_target =404; - ''; + return = "404"; }; - } - // lib.optionalAttrs (hyperhiveDomain != null) { - # FluffyChat boot-config pre-fill so the client's - # `.well-known/matrix/client` lookup hits the - # right delegation endpoint (#736). - "= /config.json" = { - extraConfig = '' - default_type application/json; - return 200 '{"defaultHomeserver":"${hyperhiveDomain}"}'; - ''; - }; - } - ) - // lib.optionalAttrs (!matrixCfg.gui.enable) { - "/" = { - return = "404"; }; - }; }; }; };