swarm-otel: route the gateway from the receiver list and check the sockets accept

Refs #4815

docs/swarm/services.md

@@ -334,7 +334,11 @@ It runs in a `swarm-otel` container on the hive bridge, in a network
 namespace of its own. It binds no OTLP port: every receiver (one per hive,
 the swarm tier's and one per service forwarder) listens on a unix socket in
 `/run/swarm-otel`, a directory only the gateway's nginx and the container can
-open, and the gateway forwards to them.
+open, and the gateway forwards to them. The gateway's routes and the collector's
+receivers come from one list, so they name the same sockets. The
+`swarm-otel-sockets` unit connects to each socket as the gateway's user after
+the container starts and fails if one refuses for 30 seconds, so a collector
+that's up but not listening shows as a failed unit.
 
 Every hive's own collector reaches this one by its gateway name,
 `swarm.otel.domain` (default `otel.<swarm domain>`) — the same

nix/host-modules/swarm-otel.nix

@@ -307,8 +307,12 @@ let
   # One socket per receiver, named after its owner (hive, swarm tier or
   # service): `otlp/<owner>` is already unique, so the socket is too.
   socketOf = owner: "${socketDir}/${owner}.sock";
-  upstreamOf = owner: "http://unix:${socketOf owner}:/";
+  # Every receiver's socket, by owner. The collector's receivers, the gateway's
-  allSockets = lib.attrValues hiveSockets ++ map socketOf ([ swarmTierName ] ++ serviceProducerNames);
+  # routes and the readiness check all read this attrset, so they cannot name
+  # different sockets.
+  receiverSockets = hiveSockets // lib.genAttrs ([ swarmTierName ] ++ serviceProducerNames) socketOf;
+  upstreamOf = sock: "http://unix:${sock}:/";
+  allSockets = lib.attrValues receiverSockets;
 
   # The gid the gateway's nginx runs as. Only the number crosses into the
   # container, which knows it as `gateway-nginx`.
@@ -492,24 +496,16 @@ in
         # serves `/v1/metrics` and knows nothing about hives. Without it
         # the receiver sees `/<hive>/v1/metrics` and answers 404 to a
         # request that authenticated perfectly.
-        lib.mapAttrs' (h: _: lib.nameValuePair "/${h}/" { proxyPass = upstreamOf h; }) hiveSockets
+        lib.mapAttrs' (
+          owner: sock: lib.nameValuePair "/${owner}/" { proxyPass = upstreamOf sock; }
+        ) receiverSockets
         // {
-          # The swarm-tier producer's own route — same trailing-slash
-          # shape as the per-hive locations above, and load-bearing for
-          # the same reason: it strips `/${producerName}` before the
-          # request reaches the receiver, which knows nothing about the
-          # path it was found at.
-          "/${swarmTierName}/".proxyPass = upstreamOf swarmTierName;
           # There is no swarm-wide inbox, and a closed door is the honest
           # description of that. Every other route into this collector
-          # belongs to exactly one hive or one swarm-tier producer.
+          # belongs to exactly one hive, swarm-tier producer or service
+          # forwarder, each on its own route for its own audience.
           "/".return = "404";
-        }
+        };
-        # Each service forwarder, on its own route for its own audience —
-        # same trailing slashes, same reason.
-        // lib.mapAttrs' (
-          name: _: lib.nameValuePair "/${name}/" { proxyPass = upstreamOf name; }
-        ) serviceProducers;
     };
 
     # The identities this tier authenticates against. Stated rather than
@@ -699,6 +695,41 @@ in
     # over a file that does not exist yet.
     systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
 
+    # A collector that is up but not listening on a socket the gateway names
+    # answers every producer with a 502 and fails no unit. This one fails when
+    # any receiver socket refuses a connection for 30s, so the switch reports
+    # it. It runs as the gateway's user: a socket nginx cannot open fails too.
+    # Any HTTP status counts as accepted; only a failed connect is a failure.
+    systemd.services.swarm-otel-sockets = {
+      description = "wait for every swarm collector receiver socket to accept a connection";
+      after = [ "container@${cfg.machine}.service" ];
+      wants = [ "container@${cfg.machine}.service" ];
+      # Restarted with the container, so every start of the collector is
+      # checked and not only the first one after a boot.
+      partOf = [ "container@${cfg.machine}.service" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [
+        pkgs.curl
+        pkgs.coreutils
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        User = config.services.nginx.user;
+        TimeoutStartSec = 90;
+      };
+      script = ''
+        for sock in ${lib.escapeShellArgs allSockets}; do
+          for _ in $(seq 30); do
+            curl -s -o /dev/null --max-time 5 --unix-socket "$sock" http://localhost/ && continue 2
+            sleep 1
+          done
+          echo "no connection to $sock after 30s" >&2
+          exit 1
+        done
+      '';
+    };
+
     assertions = [
       {
         # Shaped after ./swarm-grafana.nix's `haveClientIdentity` assertion —
@@ -1036,7 +1067,7 @@ in
                 # the per-hive `hiveClientPrefix` namespace.
                 // {
                   "otlp/${swarmTierName}".protocols.http = {
-                    endpoint = socketOf swarmTierName;
+                    endpoint = receiverSockets.${swarmTierName};
                     transport = "unix";
                     auth.authenticator = "oidc/${swarmTierName}";
                   };
@@ -1048,7 +1079,7 @@ in
                   name: _:
                   lib.nameValuePair "otlp/${name}" {
                     protocols.http = {
-                      endpoint = socketOf name;
+                      endpoint = receiverSockets.${name};
                       transport = "unix";
                       auth.authenticator = "oidc/${name}";
                     };

nix/module-eval/swarm-services-switch.nix

@@ -851,6 +851,75 @@ let
         && lib.elem "gateway-nginx" unit.SupplementaryGroups
         && unit.UMask == "0007";
     }
+    {
+      # The routes the gateway forwards to are exactly the sockets the
+      # collector listens on, and none is a TCP address: a route left on
+      # loopback after the collector moved into its own netns returns a 502 to
+      # every producer while the collector runs. `verdict` is applied to
+      # broken copies of the routes too, so a check that cannot fail is caught.
+      name = "the gateway's collector routes are the collector's unix receivers and none is loopback";
+      ok =
+        let
+          host = serviceContainersWithCi;
+          otel = host.services.hyperhive.swarm.otel;
+          s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings;
+          locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
+          sorted = xs: lib.sort lib.lessThan (lib.unique xs);
+          unixReceivers = lib.filter (r: (r.protocols.http.transport or null) == "unix") (
+            lib.attrValues s.receivers
+          );
+          receiverSocks = sorted (map (r: r.protocols.http.endpoint) unixReceivers);
+          socketOfRoute =
+            p:
+            if lib.hasPrefix "http://unix:" p then
+              lib.removeSuffix ":/" (lib.removePrefix "http://unix:" p)
+            else
+              p;
+          verdict =
+            routes:
+            let
+              passes = lib.filter (p: p != null) (map (l: l.proxyPass or null) (lib.attrValues routes));
+            in
+            sorted (map socketOfRoute passes) == receiverSocks
+            && !(lib.any (p: lib.hasInfix "127.0.0.1" p || lib.hasInfix "localhost" p) passes);
+        in
+        verdict locs
+        && lib.elem "/run/swarm-otel/h1.sock" receiverSocks
+        && lib.elem "/run/swarm-otel/${otel.producerName}.sock" receiverSocks
+        && lib.elem "/run/swarm-otel/swarm-grafana.sock" receiverSocks
+        &&
+          lib.length unixReceivers == lib.length (lib.attrNames host.services.hyperhive.swarm.hives)
+          + 1
+          + lib.length (lib.attrNames otel.serviceProducers)
+        # Each broken copy must fail: a TCP route, a missing route, a route to
+        # a socket no receiver has.
+        && !verdict (locs // { "/h1/".proxyPass = "http://127.0.0.1:4319/"; })
+        && !verdict (lib.removeAttrs locs [ "/h1/" ])
+        && !verdict (locs // { "/x/".proxyPass = "http://unix:/run/swarm-otel/x.sock:/"; });
+    }
+    {
+      # After the container, restarted with it, and run as the gateway's user,
+      # so a socket nginx cannot open fails the unit. Every receiver socket is
+      # in the script.
+      name = "the collector's receiver sockets are connect-checked after the container starts";
+      ok =
+        let
+          host = serviceContainersWithCi;
+          u = host.systemd.services.swarm-otel-sockets;
+          s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings;
+          socks = map (r: r.protocols.http.endpoint) (
+            lib.filter (r: (r.protocols.http.transport or null) == "unix") (lib.attrValues s.receivers)
+          );
+        in
+        socks != [ ]
+        && lib.all (sock: lib.hasInfix sock u.script) socks
+        && !(lib.hasInfix "/run/swarm-otel/absent.sock" u.script)
+        && u.serviceConfig.Type == "oneshot"
+        && u.serviceConfig.User == host.services.nginx.user
+        && lib.elem "container@swarm-otel.service" u.after
+        && lib.elem "container@swarm-otel.service" u.partOf
+        && lib.elem "multi-user.target" u.wantedBy;
+    }
     {
       # The DynamicUser layout keeps the history under /var/lib/private; the
       # static user's StateDirectory is the plain path. The move runs as its