swarm-otel: routes carry a route header and the readiness probe requires it

docs/swarm/services.md

@@ -339,9 +339,11 @@ receivers come from one list, so they name the same sockets. The
 `swarm-otel-sockets` unit connects to each socket as the gateway's user after
 the container starts, then requests every route through the gateway vhost. It
 fails if a socket refuses for 30 seconds, or if a route answers `000`, `502` or
-`504` for 30 seconds, so a collector that's up but not listening, or an nginx
+`504`, or lacks its own `X-Swarm-Otel-Route` response header, for 30 seconds.
-that doesn't route to it, shows as a failed unit. Any other status passes, the
+The gateway sets that header, named after the route, on every response from a
-collector's 4xx included. A switch leaves a window of about 20 seconds in which
+route, so a collector that's up but not listening, or an nginx that doesn't
+have the route, shows as a failed unit. Any status other than those passes,
+the collector's 4xx included. A switch leaves a window of about 20 seconds in which
 nginx and the container are out of step: the container stops before nginx
 restarts and starts about 18 seconds after it. Senders retry through that window.
 

nix/host-modules/swarm-otel.nix

@@ -312,6 +312,9 @@ let
   # different sockets.
   receiverSockets = hiveSockets // lib.genAttrs ([ swarmTierName ] ++ serviceProducerNames) socketOf;
   upstreamOf = sock: "http://unix:${sock}:/";
+  # Set by nginx on every response from a route, whatever its status, so the
+  # readiness check can tell a route the running nginx has from the catch-all.
+  routeHeader = "X-Swarm-Otel-Route";
   allSockets = lib.attrValues receiverSockets;
 
   # The gid the gateway's nginx runs as. Only the number crosses into the
@@ -496,8 +499,18 @@ in
         # serves `/v1/metrics` and knows nothing about hives. Without it
         # the receiver sees `/<hive>/v1/metrics` and answers 404 to a
         # request that authenticated perfectly.
+        #
+        # ⚠️ A location with an `add_header` of its own inherits none of the
+        # server-scope ones, so the security headers are repeated here.
         lib.mapAttrs' (
-          owner: sock: lib.nameValuePair "/${owner}/" { proxyPass = upstreamOf sock; }
+          owner: sock:
+          lib.nameValuePair "/${owner}/" {
+            proxyPass = upstreamOf sock;
+            extraConfig = ''
+              ${gatewayCfg.lib.securityHeaders}
+              add_header ${routeHeader} ${owner} always;
+            '';
+          }
         ) receiverSockets
         // {
           # There is no swarm-wide inbox, and a closed door is the honest
@@ -698,10 +711,11 @@ in
     # A collector that is up but not listening on a socket the gateway names
     # answers every producer with a 502 and fails no unit. This one fails when
     # any receiver socket refuses a connection for 30s, or when the running
-    # nginx cannot reach one, so the switch reports it. It runs as the
+    # nginx does not route to one, so the switch reports it. It runs as the
     # gateway's user: a socket nginx cannot open fails too. On the socket any
-    # HTTP status counts as accepted. Through nginx 000, 502 and 504 fail and
+    # HTTP status counts as accepted. Through nginx, 000, 502 and 504 fail, and so
-    # every other status passes, a collector's 4xx included.
+    # does a response without its route's own header; any other status passes, a
+    # collector's 4xx included.
     systemd.services.swarm-otel-sockets = {
       description = "wait for every swarm collector receiver socket to accept a connection";
       after = [
@@ -734,15 +748,17 @@ in
         done
 
         # The running nginx, not its config. `--resolve` pins the vhost to
-        # this host's gateway.
+        # this host's gateway. The route header is what separates a route nginx
+        # has from its catch-all 404.
         for route in ${lib.escapeShellArgs (lib.attrNames receiverSockets)}; do
           for _ in $(seq 30); do
-            code=$(curl -sk --max-time 5 --resolve ${cfg.domain}:443:127.0.0.1 -o /dev/null -w '%{http_code}' \
+            out=$(curl -sk --max-time 5 --resolve ${cfg.domain}:443:127.0.0.1 -o /dev/null \
-              "https://${cfg.domain}/$route/") || code=000
+              -w '%{http_code} %header{${lib.toLower routeHeader}}' "https://${cfg.domain}/$route/") || out=000
-            case $code in 000 | 502 | 504) ;; *) continue 2 ;; esac
+            read -r code seen <<< "$out"
+            case $code in 000 | 502 | 504) ;; *) [ "$seen" = "$route" ] && continue 2 ;; esac
             sleep 1
           done
-          echo "https://${cfg.domain}/$route/ answered $code after 30s" >&2
+          echo "https://${cfg.domain}/$route/ answered $code, route header '$seen', after 30s" >&2
           exit 1
         done
       '';

nix/module-eval/swarm-services-switch.nix

@@ -949,10 +949,74 @@ let
         && !(lib.any (writes (sc // { ReadWritePaths = [ ]; })) socks)
         && !(lib.any (writes (sc // { ReadWritePaths = [ "/var/lib/opentelemetry-collector" ]; })) socks);
     }
+    {
+      # The readiness unit tells a route nginx has from its catch-all 404 by
+      # this header, so every route carries its own name in it, `always` so a
+      # 4xx or 5xx from the collector carries it too. A location with an
+      # `add_header` inherits none of the server-scope ones, so the security
+      # headers have to be repeated. The catch-all carries no route header.
+      name = "every collector route sets its own route header and keeps the security headers";
+      ok =
+        let
+          host = serviceContainersWithCi;
+          otel = host.services.hyperhive.swarm.otel;
+          locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
+          sec = host.services.hyperhive.gateway.lib.securityHeaders;
+          proxied = lib.filter (l: (locs.${l}.proxyPass or null) != null) (lib.attrNames locs);
+          nameOf = l: lib.removeSuffix "/" (lib.removePrefix "/" l);
+          verdict =
+            ls:
+            lib.all (
+              l:
+              let
+                extra = ls.${l}.extraConfig or "";
+              in
+              lib.hasInfix "add_header X-Swarm-Otel-Route ${nameOf l} always;" extra
+              && lib.hasInfix "add_header X-Frame-Options" extra
+            ) proxied;
+        in
+        lib.length proxied > 2
+        && lib.hasInfix "add_header X-Frame-Options" sec
+        && !(lib.hasInfix "X-Swarm-Otel-Route" (locs."/".extraConfig or ""))
+        && verdict locs
+        && !verdict (
+          locs
+          // {
+            "/h1/" = locs."/h1/" // {
+              extraConfig = sec;
+            };
+          }
+        )
+        && !verdict (
+          locs
+          // {
+            "/h1/" = locs."/h1/" // {
+              extraConfig = "add_header X-Swarm-Otel-Route h1 always;";
+            };
+          }
+        )
+        && !verdict (
+          locs
+          // {
+            "/h1/" = locs."/h1/" // {
+              extraConfig = sec + "add_header X-Swarm-Otel-Route swarm always;";
+            };
+          }
+        )
+        && !verdict (
+          locs
+          // {
+            "/h1/" = locs."/h1/" // {
+              extraConfig = sec + "add_header X-Swarm-Otel-Route h1;";
+            };
+          }
+        );
+    }
     {
       # The probe loop names each gateway route, and the routes are read from
       # the vhost, not from the receivers the probe is built from. A script
-      # missing one route, or treating 502 as a pass, is rejected.
+      # missing one route, treating 502 as a pass, or passing on status alone
+      # without the route header, is rejected.
       name = "the readiness unit probes every collector route through the gateway";
       ok =
         let
@@ -980,7 +1044,9 @@ let
             probed script == routes
             && lib.hasInfix "--resolve ${otel.domain}:443:127.0.0.1" script
             && lib.hasInfix "https://${otel.domain}/$route/" script
-            && lib.hasInfix "case $code in 000 | 502 | 504) ;;" script;
+            && lib.hasInfix "case $code in 000 | 502 | 504) ;;" script
+            && lib.hasInfix "%header{x-swarm-otel-route}" script
+            && lib.hasInfix ''[ "$seen" = "$route" ] && continue 2'' script;
           script = host.systemd.services.swarm-otel-sockets.script;
         in
         lib.length routes > 2
@@ -989,7 +1055,9 @@ let
         && verdict script
         && !verdict (lib.replaceStrings [ " swarm-grafana" ] [ "" ] script)
         && !verdict (lib.replaceStrings [ "000 | 502 | 504" ] [ "000 | 504" ] script)
-        && !verdict (lib.replaceStrings [ "--resolve" ] [ "--noresolve" ] script);
+        && !verdict (lib.replaceStrings [ "--resolve" ] [ "--noresolve" ] script)
+        && !verdict (lib.replaceStrings [ "%header{x-swarm-otel-route}" ] [ "" ] script)
+        && !verdict (lib.replaceStrings [ ''[ "$seen" = "$route" ] && '' ] [ "" ] script);
     }
     {
       # The DynamicUser layout keeps the history under /var/lib/private; the