swarm-otel: run in its own network namespace, receivers on unix sockets

Refs #4815

docs/swarm/services.md

@@ -330,10 +330,11 @@ go there is `deploy.swarm-otel.upstream.signals` (default `[ "metrics" "logs" ]`
 always get both. It holds
 the upstream credential, which is why no hive and no agent needs one.
 
-It runs in a `swarm-otel` container. Its `swarm.otel.port` defaults to
+It runs in a `swarm-otel` container on the hive bridge, in a network
-`4319` rather than OTLP's usual `4318`, which the hive tier uses — swarm
+namespace of its own. It binds no OTLP port: every receiver (one per hive,
-containers share the host's network namespace, so two collectors on one
+the swarm tier's and one per service forwarder) listens on a unix socket in
-port is a coin toss at runtime rather than an error at build time.
+`/run/swarm-otel`, a directory only the gateway's nginx and the container can
+open, and the gateway forwards to them.
 
 Every hive's own collector reaches this one by its gateway name,
 `swarm.otel.domain` (default `otel.<swarm domain>`) — the same
@@ -345,7 +346,6 @@ gateway either way.
 | Option              | When you'd touch it                                                                     |
 | ------------------- | --------------------------------------------------------------------------------------- |
 | `swarm.otel.domain` | Only to rename it — the default already resolves correctly for every hive in the swarm. |
-| `swarm.otel.port`   | Only if something else on the services host already claims `4319`.                      |
 
 The `swarm-bao`, `swarm-nats`, `swarm-builder`, `swarm-claude-login`,
 `swarm-notify-router`, `swarm-review-classifier`, `swarm-ci-runner`, `swarm-grafana`,

nix/host-modules/deploy.nix

@@ -69,6 +69,19 @@ in
       own, and the gateway reaches it there, so it binds no host port. Remove
       this definition.
     '')
+    (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "otel" "port" ] ''
+      The swarm collector's OTLP receivers listen on unix sockets in a network
+      namespace of its own, and the gateway reaches them there, so there is no
+      receiver port range. Remove this definition.
+    '')
+    (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "otel" "producerPort" ] ''
+      The swarm-tier receiver listens on a unix socket, so it binds no port.
+      Remove this definition.
+    '')
+    (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "otel" "storeProducerPort" ] ''
+      The store forwarder's receiver listens on a unix socket, so it binds no
+      port. Remove this definition.
+    '')
     (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "authelia" "port" ] ''
       authelia listens on a unix socket in a network namespace of its own, and
       the gateway reaches it there, so it binds no port. Remove this

nix/host-modules/swarm-otel-service.nix

@@ -50,34 +50,6 @@ in
       '';
     };
 
-    port = lib.mkOption {
-      type = lib.types.port;
-      default = 4319;
-      description = ''
-        First port of this collector's receiver range. Every hive in
-        {option}`services.hyperhive.swarm.hives` gets its **own**
-        authenticated receiver — that is what makes the `hive` label
-        unforgeable — so the range is one port per hive, starting here, in
-        sorted-name order.
-
-        ⚠️ Internal. No client is ever told a port: a hive reaches its own
-        receiver as `https://''${domain}/<hive>`, and the gateway routes on
-        that path. So adding a hive, which renumbers the ones after it, is
-        harmless — nginx is rendered from this same evaluation and moves
-        with it.
-
-        ⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the
-        hive tier already uses it (`services.hyperhive.swarm.hiveOtel.collectorPort`)
-        and every swarm container shares the host's network namespace. Two
-        listeners claiming one port on one host is not a build failure —
-        it is a runtime coin toss over which one gets it, with nothing in
-        any log saying so. On the host running this collector, an
-        assertion rejects a derived range that overlaps any other port
-        this module or the hive tier declares, which is as far as a module
-        can see.
-      '';
-    };
-
     telemetryPort = lib.mkOption {
       type = lib.types.port;
       default = 8889;
@@ -87,73 +59,8 @@ in
         that telemetry is being lost, so it is worth keeping rather than
         switching off.
 
-        ⚠️ **Deliberately not 8889's neighbour 8888**, which is the
+        Bound on the container's own loopback, so no other collector
-        collector's built-in default and therefore what the hive tier
+        shares it; only this collector scrapes it.
-        already binds. Two collectors share a network namespace whenever
-        they are co-located, and unlike the OTLP port this one appears
-        nowhere in either config — it is a default inside the binary, so
-        nothing that compares configured ports can see the clash. The
-        second collector to start simply dies with
-        `bind: address already in use`.
-      '';
-    };
-
-    producerPort = lib.mkOption {
-      type = lib.types.port;
-      default = 4390;
-      description = ''
-        Port the swarm tier's own OTLP/HTTP receiver for a
-        **swarm-level** producer (`swarm-controller`'s vcs/jobq counters
-        today) listens on, at `127.0.0.1`. Reached through the gateway at
-        `https://''${domain}/''${producerName}/`, same shape as a hive's
-        own receiver — a swarm-level producer may not share a host with
-        this collector (the swarm services need not run on the same host
-        as the swarm controller), so loopback-only reachability is not a
-        supported shape here, as it is not for a hive.
-
-        Authenticated by `oidc/''${producerName}`, checking for the
-        audience `services.hyperhive.swarm.controller.queueClientId`
-        requests — not the per-hive `hiveClientPrefix` namespace, because
-        a swarm-level producer is explicitly not a hive (see that
-        option's own doc comment). The `swarm` label this receiver's
-        samples get is still the one the receiver's own *existence*
-        establishes, same as before — there is no hive identity being
-        forged or attributed here, just a caller proving it is the one
-        principal allowed to push into this pipeline.
-
-        ⚠️ Deliberately NOT derived from `port + (number of hives)`: that
-        range grows every time a hive is added, and a fixed offset from
-        it would silently start colliding once the hive count caught up.
-        Kept as its own reserved value instead, and the port-collision
-        assertion still rejects a real collision (including one hive
-        growth eventually causes) rather than starting a collector that
-        quietly drops one receiver's samples.
-      '';
-    };
-
-    storeProducerPort = lib.mkOption {
-      type = lib.types.port;
-      default = 4391;
-      description = ''
-        Port the receiver for the **secret store's own forwarder** listens
-        on, at `127.0.0.1`. Reached through the gateway at
-        `https://''${domain}/''${storeProducerName}/`, the same
-        path-per-producer shape a hive and the swarm-tier producer above
-        both use.
-
-        Its own receiver, and not a second sender into
-        {option}`services.hyperhive.swarm.otel.producerPort`, because an
-        `oidc` authenticator checks exactly ONE audience: that receiver
-        admits the audience `swarm-controller` registers for itself, and
-        the store's forwarder is a different principal with a client of its
-        own. Sharing the route would mean sharing that client — which is
-        the thing the store exists to make unnecessary.
-
-        ⚠️ A reserved value rather than an offset from
-        {option}`services.hyperhive.swarm.otel.port`, for the reason stated
-        on `producerPort`: that range grows with the hive count and would
-        eventually walk into any fixed offset from it. The port-collision
-        assertion covers this port too.
       '';
     };
 
@@ -170,14 +77,6 @@ in
                 the forwarder requests.
               '';
             };
-            port = lib.mkOption {
-              type = lib.types.port;
-              description = ''
-                Loopback port of the producer's receiver on the collector's
-                host. In the port-collision assertion with every other
-                receiver there.
-              '';
-            };
           };
         }
       );
@@ -256,13 +155,15 @@ in
     scrapeTargets = lib.mkOption {
       type = lib.types.attrsOf lib.types.str;
       default = { };
-      example = lib.literalExpression ''{ forgejo = "127.0.0.1:3000"; }'';
+      example = lib.literalExpression ''{ collector = "127.0.0.1:8889"; }'';
       description = ''
         Prometheus exposition endpoints this collector scrapes, as
-        `<job name> = "<host>:<port>"`.
+        `<job name> = "<host>:<port>"`. The collector runs in a network namespace of
+        its own, so `127.0.0.1` is the collector container's loopback: a target
+        here is a listener inside that container, such as its own telemetry.
 
         A path and query may follow the port —
-        `"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an
+        `"127.0.0.1:9100/stats?format=prometheus"` — for an
         exporter that does not serve `/metrics`. Both are optional and
         omitted when absent, so a bare `host:port` is scraped exactly as
         before.
@@ -270,11 +171,10 @@ in
         **A service declares its own entry, from its own module, under its
         own `enable`.** Do not assemble the list here: an entry then exists
         only where the service that named it runs, so a target is never
-        declared on a host that does not serve it. An entry that also needs
+        declared on a host that does not serve it. A swarm service on the
-        the collector on the same host (the loopback address is only honest
+        host is not reachable here, since its loopback is not the collector's:
-        then) is declared from a glue module instead, which reads both
+        it is a `publishedScrapeTargets` entry instead, which a collector on
-        switches. Every swarm service is a `publishedScrapeTargets` entry
+        any host scrapes.
-        instead, which a collector on any host scrapes.
 
         ⚠️ **That constrains the TARGET, not the SCRAPER, and the difference
         is a silent gap.** Nothing here puts the collector on the same host.
@@ -383,73 +283,59 @@ in
     # The forwarder in ./swarm-bao.nix.
     ${cfg.storeProducerName} = {
       clientId = baoCfg.otel.clientId;
-      port = cfg.storeProducerPort;
     };
     # The forwarder in ./swarm-nats.nix, which addresses this route by the
     # queue container's name.
     swarm-nats = {
       clientId = hyperhiveCfg.swarm.nats.clientId;
-      port = lib.mkDefault 4392;
     };
     # The forwarders in ./swarm-builder.nix, ./swarm-claude-login.nix,
     # ./swarm-notify-router.nix and ./swarm-review-classifier.nix, which
     # address their routes by container name.
     swarm-builder = {
       clientId = hyperhiveCfg.swarm.builder.clientId;
-      port = lib.mkDefault 4393;
     };
     swarm-claude-login = {
       clientId = hyperhiveCfg.swarm.claude-login.clientId;
-      port = lib.mkDefault 4394;
     };
     swarm-review-classifier = {
       clientId = hyperhiveCfg.swarm.review-classifier.clientId;
-      port = lib.mkDefault 4395;
     };
     swarm-notify-router = {
       clientId = hyperhiveCfg.swarm.notify-router.clientId;
-      port = lib.mkDefault 4404;
     };
     # The forwarders in ./swarm-ci-runner.nix, ./swarm-grafana.nix and
     # ./swarm-victorialogs.nix, each authenticated as a client of its own.
     swarm-ci-runner = {
       clientId = hyperhiveCfg.swarm.ci-runner.otel.clientId;
-      port = lib.mkDefault 4396;
     };
     swarm-grafana = {
       clientId = hyperhiveCfg.swarm.grafana.otel.clientId;
-      port = lib.mkDefault 4397;
     };
     swarm-victorialogs = {
       clientId = hyperhiveCfg.swarm.victorialogs.otel.clientId;
-      port = lib.mkDefault 4398;
     };
     # The forwarders in ./hive-forge/default.nix and ./hive-matrix.nix, each
     # authenticated as a client of its own.
     hive-forge = {
       clientId = hyperhiveCfg.swarm.forge.otel.clientId;
-      port = lib.mkDefault 4399;
     };
     hive-matrix = {
       clientId = hyperhiveCfg.swarm.matrix.otel.clientId;
-      port = lib.mkDefault 4400;
     };
     # The forwarder in ./swarm-victoriametrics.nix, which reaches this route
     # through its host's nginx.
     swarm-victoriametrics = {
       clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId;
-      port = lib.mkDefault 4401;
     };
     # The CI run poller in ./swarm-ci-metrics.nix, a host unit that pushes
     # its run samples here directly.
     swarm-ci-metrics = {
       clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
-      port = lib.mkDefault 4403;
     };
     # The forwarder in ./swarm-authelia.nix.
     swarm-authelia = {
       clientId = hyperhiveCfg.swarm.authelia.otel.clientId;
-      port = lib.mkDefault 4402;
     };
   };
 

nix/host-modules/swarm-otel.nix

@@ -168,16 +168,10 @@ let
   # `attrNames` is sorted, so this is a function of the hive SET and not of
   # the order anyone wrote it in.
   #
-  # These ports are internal and appear in no URL: a hive addresses its own
+  # A hive addresses its own receiver as a PATH on this collector's single
-  # receiver as a PATH on this collector's single gateway name, and nginx —
+  # gateway name; nginx, rendered from this same evaluation, is the only thing
-  # rendered from this same evaluation — is the only thing that ever names
+  # that ever names the socket.
-  # the port. That is what makes deriving them safe here and unsafe in the
+  hiveSockets = lib.genAttrs (lib.attrNames hyperhiveCfg.swarm.hives) socketOf;
-  # obvious other place: were a hive told a port, inserting a hive would
-  # renumber the ones after it and silently move a port a running hive was
-  # already sending to.
-  hivePorts = lib.listToAttrs (
-    lib.imap0 (i: h: lib.nameValuePair h (cfg.port + i)) (lib.attrNames hyperhiveCfg.swarm.hives)
-  );
 
   # The swarm's authelia is reached by its gateway name, whose leaf is
   # issued by the swarm services sub-CA — so this container needs the same
@@ -301,11 +295,25 @@ let
   # Now always satisfied, as a consequence of the log store always existing.
   collectLogs = logExporterNames != [ ];
 
-  # Shared host netns, like every sibling swarm service: the hive tier
+  # A netns of its own on the hive bridge: the container dials authelia, the
-  # reaches this collector, and this collector reaches the metrics
+  # stores and the upstream by name through the bridge, and the gateway reaches
-  # store, without either crossing a network boundary that would need
+  # every OTLP receiver over a socket, so no receiver binds a TCP port on any
-  # its own trust material.
+  # interface.
-  privateNetwork = false;
+  privateNetwork = true;
+
+  # Fixed, not an option: the directory's mode is what keeps the sockets
+  # private, so it holds the receivers' sockets and nothing else.
+  socketDir = "/run/swarm-otel";
+  # One socket per receiver, named after its owner (hive, swarm tier or
+  # service): `otlp/<owner>` is already unique, so the socket is too.
+  socketOf = owner: "${socketDir}/${owner}.sock";
+  upstreamOf = owner: "http://unix:${socketOf owner}:/";
+  allSockets = lib.attrValues hiveSockets ++ map socketOf ([ swarmTierName ] ++ serviceProducerNames);
+
+  # The gid the gateway's nginx runs as. Only the number crosses into the
+  # container, which knows it as `gateway-nginx`.
+  nginxGid = config.ids.gids.nginx;
+  socketGroup = "gateway-nginx";
   storeRetry = import ./lib/store-retry.nix { };
   storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
 in
@@ -455,23 +463,23 @@ in
     # vhost would answer for a service it does not have.
     services.hyperhive.gateway.localNames = [ cfg.domain ];
 
-    # This host serves the vhost, and the container behind it resolves
+    # This host serves the vhost, and the hive's dnsmasq answers its name.
-    # through the hive's dnsmasq.
     services.hyperhive.gateway.enable = lib.mkDefault true;
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
+    # The hive bridge carries the container's DHCP lease and its resolver.
+    services.hyperhive.network.enable = lib.mkDefault true;
+
     # The metrics counterpart to this collector's own journal, which its
     # receiver ships like any other unit's: the journal says the process is
     # alive, these counters say whether it is dropping what it receives.
-    # Loopback works here because `privateNetwork = false` — this container
+    # Loopback is the container's own: this collector scrapes `telemetryPort`
-    # shares the host's netns, so `127.0.0.1` is where `telemetryPort` is
+    # inside its netns, and nothing else reads it.
-    # bound.
     services.hyperhive.swarm.otel.scrapeTargets.collector = "127.0.0.1:${toString cfg.telemetryPort}";
 
     # OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
-    # every sibling swarm service: TLS terminates here, then plain http to
+    # every sibling swarm service: TLS terminates here, then plain http over
-    # the co-located container over loopback (shared netns, like the store
+    # the receiver's unix socket.
-    # this collector writes to).
     services.hyperhive.gateway.vhosts."${cfg.domain}" = {
       locations =
         # One name for the whole collector, and the hive is a path under
@@ -484,16 +492,14 @@ in
         # serves `/v1/metrics` and knows nothing about hives. Without it
         # the receiver sees `/<hive>/v1/metrics` and answers 404 to a
         # request that authenticated perfectly.
-        lib.mapAttrs' (
+        lib.mapAttrs' (h: _: lib.nameValuePair "/${h}/" { proxyPass = upstreamOf h; }) hiveSockets
-          h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; }
-        ) hivePorts
         // {
           # The swarm-tier producer's own route — same trailing-slash
           # shape as the per-hive locations above, and load-bearing for
           # the same reason: it strips `/${producerName}` before the
           # request reaches the receiver, which knows nothing about the
           # path it was found at.
-          "/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/";
+          "/${swarmTierName}/".proxyPass = upstreamOf swarmTierName;
           # There is no swarm-wide inbox, and a closed door is the honest
           # description of that. Every other route into this collector
           # belongs to exactly one hive or one swarm-tier producer.
@@ -502,7 +508,7 @@ in
         # Each service forwarder, on its own route for its own audience —
         # same trailing slashes, same reason.
         // lib.mapAttrs' (
-          name: p: lib.nameValuePair "/${name}/" { proxyPass = "http://127.0.0.1:${toString p.port}/"; }
+          name: _: lib.nameValuePair "/${name}/" { proxyPass = upstreamOf name; }
         ) serviceProducers;
     };
 
@@ -663,7 +669,18 @@ in
       lib.mkDefault collectorSecretInContainer
     );
 
-    systemd.tmpfiles.rules = lib.optionals haveClientIdentity (
+    systemd.tmpfiles.rules = [
+      # The socket directory, created host-side before the container starts:
+      # nixos-container refuses to start when a bind source is missing.
+      #
+      # Root-owned, group nginx, setgid `2770`: the collector creates its
+      # sockets here through the group (its uid is allocated inside the
+      # container, so the host cannot name it), and a socket takes the
+      # directory's group, so the gateway can connect. Everything else is
+      # kept out.
+      "d ${socketDir} 2770 root ${toString nginxGid} - -"
+    ]
+    ++ lib.optionals haveClientIdentity (
       [
         # The bind source must exist when the container starts.
         "d ${collectorSecretDir} 0700 root root -"
@@ -872,44 +889,11 @@ in
           anything, and the two are scraped by different rules with
           different trust.
 
-          Rename one. A target is either reachable on loopback because it
+          Rename one. A target is either reachable on the collector's own
-          shares this host, or published and reached with a credential; it
+          loopback, or published and reached with a credential; it
           should not be described as both.
         '';
       }
-      {
-        # A port collision between two listeners on one host is a runtime
-        # coin toss with nothing in any log — the failure this whole
-        # comment budget exists to prevent. Checked against every port
-        # reachable from here; a port some other module picks is not.
-        #
-        # ⚠️ `cfg.port` is deliberately absent from `others`: it is the
-        # FIRST element of the derived range, so listing it would make this
-        # assertion fire on every config.
-        assertion =
-          let
-            derived = lib.attrValues hivePorts;
-            others = [
-              cfg.telemetryPort
-              cfg.producerPort
-              config.services.hyperhive.swarm.hiveOtel.collectorPort
-            ]
-            ++ map (p: p.port) (lib.attrValues serviceProducers);
-            all = derived ++ others;
-          in
-          lib.length (lib.unique all) == lib.length all;
-        message = ''
-          services.hyperhive.swarm.otel: the receiver range starting at
-          port (${toString cfg.port}, one port per hive in
-          services.hyperhive.swarm.hives) overlaps another port on this
-          host.
-
-          Every swarm container shares the host's network namespace, so
-          two listeners claiming one port is not a build failure — it is
-          whichever process started first, silently. Move
-          services.hyperhive.swarm.otel.port to a free range.
-        '';
-      }
     ];
 
     containers.${cfg.machine} = {
@@ -918,19 +902,28 @@ in
       # No `--link-journal=host`: this container's journal stays in the guest,
       # where `journald/self` reads it.
       inherit privateNetwork;
+      hostBridge = config.services.hyperhive.network.bridgeName;
 
-      # The upstream credential is operator-provided and lives on the host.
-      # Read-only, and only when one is configured — binding a path that
-      # does not exist makes nixos-container refuse to start the container,
-      # which is a stall several layers from its cause.
       bindMounts =
+        # A bind mount rather than a path both sides happen to know: a
+        # container's `/run` is an nspawn tmpfs the host cannot see into.
+        {
+          ${socketDir} = {
+            hostPath = socketDir;
+            isReadOnly = false;
+          };
+        }
         # The collector's OIDC secret, where the delivery unit writes it.
-        lib.optionalAttrs haveClientIdentity {
+        // lib.optionalAttrs haveClientIdentity {
           ${collectorSecretDir} = {
             hostPath = collectorSecretDir;
             isReadOnly = true;
           };
         }
+        # The upstream credential is operator-provided and lives on the host.
+        # Read-only, and only when one is configured — binding a path that
+        # does not exist makes nixos-container refuse to start the container,
+        # which is a stall several layers from its cause.
         // lib.optionalAttrs (upstream.headersCredential != null) {
           ${upstream.headersCredential} = {
             hostPath = upstream.headersCredential;
@@ -996,6 +989,15 @@ in
           system.stateVersion = config.system.stateVersion;
           services.hyperhive.swarmContainer = { inherit privateNetwork; };
 
+          # The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
+          # owns resolv.conf, so dhcpcd must not write it over that unit.
+          networking.interfaces.eth0.useDHCP = true;
+          networking.dhcpcd.extraConfig = "nohook resolv.conf";
+
+          # The sockets' group is the directory's, so that group is nginx's gid.
+          # A static group, so the unit below can name it.
+          users.groups.${socketGroup}.gid = nginxGid;
+
           services.opentelemetry-collector = {
             enable = true;
             package = pkgs.opentelemetry-collector-contrib;
@@ -1018,22 +1020,24 @@ in
               # reveals which one matched.
               receivers =
                 lib.mapAttrs' (
-                  h: p:
+                  h: sock:
                   lib.nameValuePair "otlp/${h}" {
                     protocols.http = {
-                      endpoint = "127.0.0.1:${toString p}";
+                      endpoint = sock;
+                      transport = "unix";
                       auth.authenticator = "oidc/${h}";
                     };
                   }
-                ) hivePorts
+                ) hiveSockets
                 # The swarm tier's OWN receiver — authenticated exactly
                 # like a hive's, just against a different identity (the
-                # swarm-level producer is not a hive; see
+                # swarm-level producer is not a hive). Its authenticator checks
-                # `producerPort`'s description for what its audience
+                # the audience `swarm.controller.queueClientId` requests, not
-                # checks and why).
+                # the per-hive `hiveClientPrefix` namespace.
                 // {
                   "otlp/${swarmTierName}".protocols.http = {
-                    endpoint = "127.0.0.1:${toString cfg.producerPort}";
+                    endpoint = socketOf swarmTierName;
+                    transport = "unix";
                     auth.authenticator = "oidc/${swarmTierName}";
                   };
                 }
@@ -1041,10 +1045,11 @@ in
                 # principal of its own — a receiver per audience, because an
                 # `oidc` authenticator checks one.
                 // lib.mapAttrs' (
-                  name: p:
+                  name: _:
                   lib.nameValuePair "otlp/${name}" {
                     protocols.http = {
-                      endpoint = "127.0.0.1:${toString p.port}";
+                      endpoint = socketOf name;
+                      transport = "unix";
                       auth.authenticator = "oidc/${name}";
                     };
                   }
@@ -1256,7 +1261,7 @@ in
                 # Same rule, and the journald receiver's `storage:` above
                 # would name a component the collector never starts.
                 [ "file_storage" ]
-                ++ map (h: "oidc/${h}") (lib.attrNames hivePorts)
+                ++ map (h: "oidc/${h}") (lib.attrNames hiveSockets)
                 # The swarm-tier producer's own authenticator — unconditional,
                 # same reasoning as its receiver above: `otlp/${swarmTierName}`
                 # always exists, so the authenticator it names must too, or an
@@ -1285,7 +1290,7 @@ in
                     processors = [ "resource/${h}" ];
                     exporters = exporterNames;
                   }
-                ) hivePorts
+                ) hiveSockets
                 # Its OWN pipeline, and that separation is the ruling, not a
                 # tidiness choice: every `resource/<hive>` above UPSERTS a
                 # `hive` key, so a scraped swarm sample routed through any of
@@ -1343,7 +1348,7 @@ in
                       processors = [ "resource/${h}" ];
                       exporters = logExporterNames;
                     }
-                  ) hivePorts
+                  ) hiveSockets
                 )
                 # Logs fan out exactly as metrics do: the swarm's store when it
                 # runs, the operator's upstream when one is configured, both
@@ -1446,7 +1451,7 @@ in
                     # worth not re-deriving: `issuer_ca_file`, `ca_file` and
                     # `tls.ca_file` are INVALID KEYS for this extension.)
                   }
-                ) hivePorts
+                ) hiveSockets
                 # The swarm-tier producer's own authenticator. Same extension
                 # type and same `issuer_url`/no-`issuer_ca_path` shape as every
                 # `oidc/${h}` above — only the audience differs, and
@@ -1534,7 +1539,7 @@ in
                       }
                     ];
                   }
-                ) hivePorts
+                ) hiveSockets
                 # The swarm tier's own stamp: `swarm` and deliberately NO
                 # `hive`. A scraped swarm service belongs to the swarm, not to
                 # any one hive, so there is no honest value to put there — and
@@ -1623,6 +1628,18 @@ in
               serviceConfig.RestartSec = 5;
             }
 
+            {
+              serviceConfig = {
+                # The unit's user is allocated per start (`DynamicUser`), so
+                # it reaches the socket directory through this group, and
+                # `0007` makes each socket `0770`: group nginx can connect.
+                SupplementaryGroups = [ socketGroup ];
+                UMask = "0007";
+                # A socket left by a killed process would fail the next `bind`.
+                ExecStartPre = [ "${pkgs.coreutils}/bin/rm -f ${lib.escapeShellArgs allSockets}" ];
+              };
+            }
+
             # The credential file is already `NAME=value`, systemd's
             # EnvironmentFile format — so the secret reaches the process as
             # an environment variable without being read by nix, written to

nix/module-eval/nats-authelia.nix

@@ -795,7 +795,7 @@ let
           otel = h.services.hyperhive.swarm.otel;
           s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings;
           clientOf = svc: h.services.hyperhive.swarm.${svc.name}.clientId;
-          ports = map (svc: otel.serviceProducers.${svc.machine}.port) forwarderServices;
+          socks = map (svc: s.receivers."otlp/${svc.machine}".protocols.http.endpoint) forwarderServices;
         in
         !h.services.hyperhive.deploy.builder.enable
         && lib.all (
@@ -806,9 +806,9 @@ let
           && lib.elem "otlp/${svc.machine}" s.service.pipelines."logs/${otel.producerName}".receivers
           && h.services.nginx.virtualHosts.${otel.domain}.locations ? "/${svc.machine}/"
         ) forwarderServices
-        # Their receivers and the queue's on the collector's host, none sharing a port.
+        # Their receivers and the queue's on the collector's host, none sharing a socket.
         &&
-          lib.length (lib.unique (ports ++ [ otel.serviceProducers.swarm-nats.port ]))
+          lib.length (lib.unique (socks ++ [ s.receivers."otlp/swarm-nats".protocols.http.endpoint ]))
           == lib.length forwarderServices + 1;
     }
     {
@@ -837,7 +837,9 @@ let
           h = shared.baoWithCollector;
           otel = h.services.hyperhive.swarm.otel;
           s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings;
-          ports = map (p: p.port) (lib.attrValues otel.serviceProducers);
+          socks = map (n: s.receivers."otlp/${n}".protocols.http.endpoint) (
+            lib.attrNames otel.serviceProducers
+          );
         in
         lib.all (
           svc:
@@ -847,7 +849,7 @@ let
           && lib.elem "otlp/${svc.machine}" s.service.pipelines."logs/${otel.producerName}".receivers
           && h.services.nginx.virtualHosts.${otel.domain}.locations ? "/${svc.machine}/"
         ) ownClientForwarders
-        && lib.length (lib.unique ports) == lib.length ports;
+        && lib.length (lib.unique socks) == lib.length socks;
     }
     {
       # Grafana's forwarder is a second client beside its browser login, which

nix/module-eval/swarm-services-switch.nix

@@ -170,9 +170,10 @@ let
     lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links;
 
   # Every service container at once: the services-only host plus the CI
-  # runner. The CI runner, the review classifier, the builder, the claude login
+  # runner. The CI runner, the review classifier, the notification router, the
-  # minter, the metrics store, the homeserver, authelia and Grafana are the eight
+  # builder, the claude login minter, the metrics store, the homeserver,
-  # containers on a netns of their own.
+  # authelia, Grafana and the collector are the ten containers on a netns of
+  # their own.
   serviceContainersWithCi = hive {
     deploy.hive-controller.enable = false;
     deploy.allSwarmServices = true;
@@ -792,6 +793,62 @@ let
         && lib.all (lib.hasPrefix "http://unix:${sock}:") forwarded
         && c.bindMounts.${dir}.hostPath == dir;
     }
+    {
+      # On the bridge by DHCP, with every receiver on a socket: the hive's, the
+      # swarm tier's and each service forwarder's. Every gateway location that
+      # forwards goes through one of them, and each receiver's socket is the
+      # one its location names. The three kinds are named so an empty filter
+      # can't pass.
+      name = "the swarm collector runs on a netns of its own and the gateway reaches every receiver over a socket";
+      ok =
+        let
+          host = serviceContainersWithCi;
+          net = host.services.hyperhive.network;
+          c = host.containers.swarm-otel;
+          otel = host.services.hyperhive.swarm.otel;
+          dir = "/run/swarm-otel";
+          sock = n: "${dir}/${n}.sock";
+          locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
+          forwarded = lib.filter (p: p != null) (map (l: l.proxyPass) (lib.attrValues locs));
+          inner = c.config;
+          s = inner.services.opentelemetry-collector.settings;
+          unit = inner.systemd.services.opentelemetry-collector.serviceConfig;
+          nginxGid = host.ids.gids.nginx;
+          receivers = lib.filterAttrs (n: _: lib.hasPrefix "otlp/" n) s.receivers;
+          onSocket =
+            owner:
+            let
+              r = s.receivers."otlp/${owner}".protocols.http;
+            in
+            r.transport == "unix"
+            && r.endpoint == sock owner
+            && locs."/${owner}/".proxyPass == "http://unix:${sock owner}:/";
+        in
+        c.privateNetwork
+        && c.hostBridge == net.bridgeName
+        && c.hostAddress == null
+        && c.localAddress == null
+        && inner.networking.interfaces.eth0.useDHCP
+        && inner.networking.firewall.enable
+        && onSocket "h1"
+        && onSocket otel.producerName
+        && onSocket "swarm-grafana"
+        && lib.all (n: (receivers.${n}.protocols.http.transport or null) == "unix") (
+          lib.attrNames receivers
+        )
+        &&
+          lib.length (lib.attrNames receivers)
+          == lib.length (lib.attrNames host.services.hyperhive.swarm.hives)
+          + 1
+          + lib.length (lib.attrNames otel.serviceProducers)
+        && forwarded != [ ]
+        && lib.all (lib.hasPrefix "http://unix:${dir}/") forwarded
+        && c.bindMounts.${dir}.hostPath == dir
+        && builtins.elem "d ${dir} 2770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
+        && inner.users.groups.gateway-nginx.gid == nginxGid
+        && lib.elem "gateway-nginx" unit.SupplementaryGroups
+        && unit.UMask == "0007";
+    }
     {
       # The DynamicUser layout keeps the history under /var/lib/private; the
       # static user's StateDirectory is the plain path. The move runs as its