diff --git a/docs/swarm/services.md b/docs/swarm/services.md index e0c472cc..708fdf7a 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -334,7 +334,11 @@ It runs in a `swarm-otel` container on the hive bridge, in a network namespace of its own. It binds no OTLP port: every receiver (one per hive, the swarm tier's and one per service forwarder) listens on a unix socket in `/run/swarm-otel`, a directory only the gateway's nginx and the container can -open, and the gateway forwards to them. +open, and the gateway forwards to them. The gateway's routes and the collector's +receivers come from one list, so they name the same sockets. The +`swarm-otel-sockets` unit connects to each socket as the gateway's user after +the container starts and fails if one refuses for 30 seconds, so a collector +that's up but not listening shows as a failed unit. Every hive's own collector reaches this one by its gateway name, `swarm.otel.domain` (default `otel.`) — the same diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index db1d5557..9441b486 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -307,8 +307,12 @@ let # One socket per receiver, named after its owner (hive, swarm tier or # service): `otlp/` is already unique, so the socket is too. socketOf = owner: "${socketDir}/${owner}.sock"; - upstreamOf = owner: "http://unix:${socketOf owner}:/"; - allSockets = lib.attrValues hiveSockets ++ map socketOf ([ swarmTierName ] ++ serviceProducerNames); + # Every receiver's socket, by owner. The collector's receivers, the gateway's + # routes and the readiness check all read this attrset, so they cannot name + # different sockets. + receiverSockets = hiveSockets // lib.genAttrs ([ swarmTierName ] ++ serviceProducerNames) socketOf; + upstreamOf = sock: "http://unix:${sock}:/"; + allSockets = lib.attrValues receiverSockets; # The gid the gateway's nginx runs as. Only the number crosses into the # container, which knows it as `gateway-nginx`. @@ -492,24 +496,16 @@ in # serves `/v1/metrics` and knows nothing about hives. Without it # the receiver sees `//v1/metrics` and answers 404 to a # request that authenticated perfectly. - lib.mapAttrs' (h: _: lib.nameValuePair "/${h}/" { proxyPass = upstreamOf h; }) hiveSockets + lib.mapAttrs' ( + owner: sock: lib.nameValuePair "/${owner}/" { proxyPass = upstreamOf sock; } + ) receiverSockets // { - # The swarm-tier producer's own route — same trailing-slash - # shape as the per-hive locations above, and load-bearing for - # the same reason: it strips `/${producerName}` before the - # request reaches the receiver, which knows nothing about the - # path it was found at. - "/${swarmTierName}/".proxyPass = upstreamOf swarmTierName; # There is no swarm-wide inbox, and a closed door is the honest # description of that. Every other route into this collector - # belongs to exactly one hive or one swarm-tier producer. + # belongs to exactly one hive, swarm-tier producer or service + # forwarder, each on its own route for its own audience. "/".return = "404"; - } - # Each service forwarder, on its own route for its own audience — - # same trailing slashes, same reason. - // lib.mapAttrs' ( - name: _: lib.nameValuePair "/${name}/" { proxyPass = upstreamOf name; } - ) serviceProducers; + }; }; # The identities this tier authenticates against. Stated rather than @@ -699,6 +695,41 @@ in # over a file that does not exist yet. systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; + # A collector that is up but not listening on a socket the gateway names + # answers every producer with a 502 and fails no unit. This one fails when + # any receiver socket refuses a connection for 30s, so the switch reports + # it. It runs as the gateway's user: a socket nginx cannot open fails too. + # Any HTTP status counts as accepted; only a failed connect is a failure. + systemd.services.swarm-otel-sockets = { + description = "wait for every swarm collector receiver socket to accept a connection"; + after = [ "container@${cfg.machine}.service" ]; + wants = [ "container@${cfg.machine}.service" ]; + # Restarted with the container, so every start of the collector is + # checked and not only the first one after a boot. + partOf = [ "container@${cfg.machine}.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ + pkgs.curl + pkgs.coreutils + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = config.services.nginx.user; + TimeoutStartSec = 90; + }; + script = '' + for sock in ${lib.escapeShellArgs allSockets}; do + for _ in $(seq 30); do + curl -s -o /dev/null --max-time 5 --unix-socket "$sock" http://localhost/ && continue 2 + sleep 1 + done + echo "no connection to $sock after 30s" >&2 + exit 1 + done + ''; + }; + assertions = [ { # Shaped after ./swarm-grafana.nix's `haveClientIdentity` assertion — @@ -1036,7 +1067,7 @@ in # the per-hive `hiveClientPrefix` namespace. // { "otlp/${swarmTierName}".protocols.http = { - endpoint = socketOf swarmTierName; + endpoint = receiverSockets.${swarmTierName}; transport = "unix"; auth.authenticator = "oidc/${swarmTierName}"; }; @@ -1048,7 +1079,7 @@ in name: _: lib.nameValuePair "otlp/${name}" { protocols.http = { - endpoint = socketOf name; + endpoint = receiverSockets.${name}; transport = "unix"; auth.authenticator = "oidc/${name}"; }; diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 56fd0be9..3d1709de 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -851,6 +851,75 @@ let && lib.elem "gateway-nginx" unit.SupplementaryGroups && unit.UMask == "0007"; } + { + # The routes the gateway forwards to are exactly the sockets the + # collector listens on, and none is a TCP address: a route left on + # loopback after the collector moved into its own netns returns a 502 to + # every producer while the collector runs. `verdict` is applied to + # broken copies of the routes too, so a check that cannot fail is caught. + name = "the gateway's collector routes are the collector's unix receivers and none is loopback"; + ok = + let + host = serviceContainersWithCi; + otel = host.services.hyperhive.swarm.otel; + s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings; + locs = host.services.nginx.virtualHosts.${otel.domain}.locations; + sorted = xs: lib.sort lib.lessThan (lib.unique xs); + unixReceivers = lib.filter (r: (r.protocols.http.transport or null) == "unix") ( + lib.attrValues s.receivers + ); + receiverSocks = sorted (map (r: r.protocols.http.endpoint) unixReceivers); + socketOfRoute = + p: + if lib.hasPrefix "http://unix:" p then + lib.removeSuffix ":/" (lib.removePrefix "http://unix:" p) + else + p; + verdict = + routes: + let + passes = lib.filter (p: p != null) (map (l: l.proxyPass or null) (lib.attrValues routes)); + in + sorted (map socketOfRoute passes) == receiverSocks + && !(lib.any (p: lib.hasInfix "127.0.0.1" p || lib.hasInfix "localhost" p) passes); + in + verdict locs + && lib.elem "/run/swarm-otel/h1.sock" receiverSocks + && lib.elem "/run/swarm-otel/${otel.producerName}.sock" receiverSocks + && lib.elem "/run/swarm-otel/swarm-grafana.sock" receiverSocks + && + lib.length unixReceivers == lib.length (lib.attrNames host.services.hyperhive.swarm.hives) + + 1 + + lib.length (lib.attrNames otel.serviceProducers) + # Each broken copy must fail: a TCP route, a missing route, a route to + # a socket no receiver has. + && !verdict (locs // { "/h1/".proxyPass = "http://127.0.0.1:4319/"; }) + && !verdict (lib.removeAttrs locs [ "/h1/" ]) + && !verdict (locs // { "/x/".proxyPass = "http://unix:/run/swarm-otel/x.sock:/"; }); + } + { + # After the container, restarted with it, and run as the gateway's user, + # so a socket nginx cannot open fails the unit. Every receiver socket is + # in the script. + name = "the collector's receiver sockets are connect-checked after the container starts"; + ok = + let + host = serviceContainersWithCi; + u = host.systemd.services.swarm-otel-sockets; + s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings; + socks = map (r: r.protocols.http.endpoint) ( + lib.filter (r: (r.protocols.http.transport or null) == "unix") (lib.attrValues s.receivers) + ); + in + socks != [ ] + && lib.all (sock: lib.hasInfix sock u.script) socks + && !(lib.hasInfix "/run/swarm-otel/absent.sock" u.script) + && u.serviceConfig.Type == "oneshot" + && u.serviceConfig.User == host.services.nginx.user + && lib.elem "container@swarm-otel.service" u.after + && lib.elem "container@swarm-otel.service" u.partOf + && lib.elem "multi-user.target" u.wantedBy; + } { # The DynamicUser layout keeps the history under /var/lib/private; the # static user's StateDirectory is the plain path. The move runs as its