swarm-otel: let the collector write its socket dir, and probe every route through nginx

docs/swarm/services.md

@@ -337,8 +337,13 @@ the swarm tier's and one per service forwarder) listens on a unix socket in
 open, and the gateway forwards to them. The gateway's routes and the collector's
 receivers come from one list, so they name the same sockets. The
 `swarm-otel-sockets` unit connects to each socket as the gateway's user after
-the container starts and fails if one refuses for 30 seconds, so a collector
+the container starts, then requests every route through the gateway vhost. It
-that's up but not listening shows as a failed unit.
+fails if a socket refuses for 30 seconds, or if a route answers `000`, `502` or
+`504` for 30 seconds, so a collector that's up but not listening, or an nginx
+that doesn't route to it, shows as a failed unit. Any other status passes, the
+collector's 4xx included. A switch leaves a window of about 20 seconds in which
+nginx and the container are out of step: the container stops before nginx
+restarts and starts about 18 seconds after it. Senders retry through that window.
 
 Every hive's own collector reaches this one by its gateway name,
 `swarm.otel.domain` (default `otel.<swarm domain>`) — the same

nix/host-modules/swarm-otel.nix

@@ -697,12 +697,17 @@ in
 
     # A collector that is up but not listening on a socket the gateway names
     # answers every producer with a 502 and fails no unit. This one fails when
-    # any receiver socket refuses a connection for 30s, so the switch reports
+    # any receiver socket refuses a connection for 30s, or when the running
-    # it. It runs as the gateway's user: a socket nginx cannot open fails too.
+    # nginx cannot reach one, so the switch reports it. It runs as the
-    # Any HTTP status counts as accepted; only a failed connect is a failure.
+    # gateway's user: a socket nginx cannot open fails too. On the socket any
+    # HTTP status counts as accepted. Through nginx 000, 502 and 504 fail and
+    # every other status passes, a collector's 4xx included.
     systemd.services.swarm-otel-sockets = {
       description = "wait for every swarm collector receiver socket to accept a connection";
-      after = [ "container@${cfg.machine}.service" ];
+      after = [
+        "container@${cfg.machine}.service"
+        "nginx.service"
+      ];
       wants = [ "container@${cfg.machine}.service" ];
       # Restarted with the container, so every start of the collector is
       # checked and not only the first one after a boot.
@@ -727,6 +732,19 @@ in
           echo "no connection to $sock after 30s" >&2
           exit 1
         done
+
+        # The running nginx, not its config. `--resolve` pins the vhost to
+        # this host's gateway.
+        for route in ${lib.escapeShellArgs (lib.attrNames receiverSockets)}; do
+          for _ in $(seq 30); do
+            code=$(curl -sk --max-time 5 --resolve ${cfg.domain}:443:127.0.0.1 -o /dev/null -w '%{http_code}' \
+              "https://${cfg.domain}/$route/") || code=000
+            case $code in 000 | 502 | 504) ;; *) continue 2 ;; esac
+            sleep 1
+          done
+          echo "https://${cfg.domain}/$route/ answered $code after 30s" >&2
+          exit 1
+        done
       '';
     };
 
@@ -1666,6 +1684,10 @@ in
                 # `0007` makes each socket `0770`: group nginx can connect.
                 SupplementaryGroups = [ socketGroup ];
                 UMask = "0007";
+                # `DynamicUser` forces `ProtectSystem=strict` over upstream's
+                # `full`, so `/run` is read-only in the unit and the socket
+                # directory's writable bind mount does not reach it.
+                ReadWritePaths = [ socketDir ];
                 # A socket left by a killed process would fail the next `bind`.
                 ExecStartPre = [ "${pkgs.coreutils}/bin/rm -f ${lib.escapeShellArgs allSockets}" ];
               };

nix/module-eval/swarm-services-switch.nix

@@ -920,6 +920,77 @@ let
         && lib.elem "container@swarm-otel.service" u.partOf
         && lib.elem "multi-user.target" u.wantedBy;
     }
+    {
+      # `DynamicUser` forces `ProtectSystem=strict`, which leaves `/run`
+      # read-only unless the path is in `ReadWritePaths`. The controls: the
+      # unit really is DynamicUser, and a copy without the path, or with some
+      # other path, is read-only.
+      name = "the collector unit can write the directory its receiver sockets are created in";
+      ok =
+        let
+          host = serviceContainersWithCi;
+          sc = host.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig;
+          s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings;
+          socks = map (r: r.protocols.http.endpoint) (
+            lib.filter (r: (r.protocols.http.transport or null) == "unix") (lib.attrValues s.receivers)
+          );
+          writes =
+            c: sock:
+            let
+              strict = (c.DynamicUser or false) || (c.ProtectSystem or "no") == "strict";
+              dir = dirOf sock;
+              under = p: p == dir || lib.hasPrefix "${p}/" dir;
+            in
+            !strict || lib.any under (c.ReadWritePaths or [ ]);
+        in
+        socks != [ ]
+        && sc.DynamicUser
+        && lib.all (writes sc) socks
+        && !(lib.any (writes (sc // { ReadWritePaths = [ ]; })) socks)
+        && !(lib.any (writes (sc // { ReadWritePaths = [ "/var/lib/opentelemetry-collector" ]; })) socks);
+    }
+    {
+      # The probe loop names each gateway route, and the routes are read from
+      # the vhost, not from the receivers the probe is built from. A script
+      # missing one route, or treating 502 as a pass, is rejected.
+      name = "the readiness unit probes every collector route through the gateway";
+      ok =
+        let
+          host = serviceContainersWithCi;
+          otel = host.services.hyperhive.swarm.otel;
+          locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
+          routes = lib.sort lib.lessThan (
+            map (l: lib.removeSuffix "/" (lib.removePrefix "/" l)) (
+              lib.filter (l: (locs.${l}.proxyPass or null) != null) (lib.attrNames locs)
+            )
+          );
+          lines = script: map (lib.removePrefix "        ") (lib.splitString "\n" script);
+          probed =
+            script:
+            let
+              loop = lib.findFirst (l: lib.hasPrefix "for route in " l) "" (lines script);
+            in
+            lib.sort lib.lessThan (
+              lib.filter (w: w != "") (
+                lib.splitString " " (lib.removeSuffix "; do" (lib.removePrefix "for route in " loop))
+              )
+            );
+          verdict =
+            script:
+            probed script == routes
+            && lib.hasInfix "--resolve ${otel.domain}:443:127.0.0.1" script
+            && lib.hasInfix "https://${otel.domain}/$route/" script
+            && lib.hasInfix "case $code in 000 | 502 | 504) ;;" script;
+          script = host.systemd.services.swarm-otel-sockets.script;
+        in
+        lib.length routes > 2
+        && lib.elem "h1" routes
+        && lib.elem "swarm-grafana" routes
+        && verdict script
+        && !verdict (lib.replaceStrings [ " swarm-grafana" ] [ "" ] script)
+        && !verdict (lib.replaceStrings [ "000 | 502 | 504" ] [ "000 | 504" ] script)
+        && !verdict (lib.replaceStrings [ "--resolve" ] [ "--noresolve" ] script);
+    }
     {
       # The DynamicUser layout keeps the history under /var/lib/private; the
       # static user's StateDirectory is the plain path. The move runs as its