From 41bea3c6d9dba6d3de42ba71b0f1d71ef320c04a Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 24 Oct 2026 21:12:35 +0200 Subject: [PATCH] swarm-otel: let the collector write its socket dir, and probe every route through nginx --- docs/swarm/services.md | 9 ++- nix/host-modules/swarm-otel.nix | 30 ++++++++-- nix/module-eval/swarm-services-switch.nix | 71 +++++++++++++++++++++++ 3 files changed, 104 insertions(+), 6 deletions(-) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 708fdf7a..68c283cc 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -337,8 +337,13 @@ the swarm tier's and one per service forwarder) listens on a unix socket in open, and the gateway forwards to them. The gateway's routes and the collector's receivers come from one list, so they name the same sockets. The `swarm-otel-sockets` unit connects to each socket as the gateway's user after -the container starts and fails if one refuses for 30 seconds, so a collector -that's up but not listening shows as a failed unit. +the container starts, then requests every route through the gateway vhost. It +fails if a socket refuses for 30 seconds, or if a route answers `000`, `502` or +`504` for 30 seconds, so a collector that's up but not listening, or an nginx +that doesn't route to it, shows as a failed unit. Any other status passes, the +collector's 4xx included. A switch leaves a window of about 20 seconds in which +nginx and the container are out of step: the container stops before nginx +restarts and starts about 18 seconds after it. Senders retry through that window. Every hive's own collector reaches this one by its gateway name, `swarm.otel.domain` (default `otel.`) — the same diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 9441b486..1d0ffeb0 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -697,12 +697,17 @@ in # A collector that is up but not listening on a socket the gateway names # answers every producer with a 502 and fails no unit. This one fails when - # any receiver socket refuses a connection for 30s, so the switch reports - # it. It runs as the gateway's user: a socket nginx cannot open fails too. - # Any HTTP status counts as accepted; only a failed connect is a failure. + # any receiver socket refuses a connection for 30s, or when the running + # nginx cannot reach one, so the switch reports it. It runs as the + # gateway's user: a socket nginx cannot open fails too. On the socket any + # HTTP status counts as accepted. Through nginx 000, 502 and 504 fail and + # every other status passes, a collector's 4xx included. systemd.services.swarm-otel-sockets = { description = "wait for every swarm collector receiver socket to accept a connection"; - after = [ "container@${cfg.machine}.service" ]; + after = [ + "container@${cfg.machine}.service" + "nginx.service" + ]; wants = [ "container@${cfg.machine}.service" ]; # Restarted with the container, so every start of the collector is # checked and not only the first one after a boot. @@ -727,6 +732,19 @@ in echo "no connection to $sock after 30s" >&2 exit 1 done + + # The running nginx, not its config. `--resolve` pins the vhost to + # this host's gateway. + for route in ${lib.escapeShellArgs (lib.attrNames receiverSockets)}; do + for _ in $(seq 30); do + code=$(curl -sk --max-time 5 --resolve ${cfg.domain}:443:127.0.0.1 -o /dev/null -w '%{http_code}' \ + "https://${cfg.domain}/$route/") || code=000 + case $code in 000 | 502 | 504) ;; *) continue 2 ;; esac + sleep 1 + done + echo "https://${cfg.domain}/$route/ answered $code after 30s" >&2 + exit 1 + done ''; }; @@ -1666,6 +1684,10 @@ in # `0007` makes each socket `0770`: group nginx can connect. SupplementaryGroups = [ socketGroup ]; UMask = "0007"; + # `DynamicUser` forces `ProtectSystem=strict` over upstream's + # `full`, so `/run` is read-only in the unit and the socket + # directory's writable bind mount does not reach it. + ReadWritePaths = [ socketDir ]; # A socket left by a killed process would fail the next `bind`. ExecStartPre = [ "${pkgs.coreutils}/bin/rm -f ${lib.escapeShellArgs allSockets}" ]; }; diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 3d1709de..60dff61b 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -920,6 +920,77 @@ let && lib.elem "container@swarm-otel.service" u.partOf && lib.elem "multi-user.target" u.wantedBy; } + { + # `DynamicUser` forces `ProtectSystem=strict`, which leaves `/run` + # read-only unless the path is in `ReadWritePaths`. The controls: the + # unit really is DynamicUser, and a copy without the path, or with some + # other path, is read-only. + name = "the collector unit can write the directory its receiver sockets are created in"; + ok = + let + host = serviceContainersWithCi; + sc = host.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig; + s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings; + socks = map (r: r.protocols.http.endpoint) ( + lib.filter (r: (r.protocols.http.transport or null) == "unix") (lib.attrValues s.receivers) + ); + writes = + c: sock: + let + strict = (c.DynamicUser or false) || (c.ProtectSystem or "no") == "strict"; + dir = dirOf sock; + under = p: p == dir || lib.hasPrefix "${p}/" dir; + in + !strict || lib.any under (c.ReadWritePaths or [ ]); + in + socks != [ ] + && sc.DynamicUser + && lib.all (writes sc) socks + && !(lib.any (writes (sc // { ReadWritePaths = [ ]; })) socks) + && !(lib.any (writes (sc // { ReadWritePaths = [ "/var/lib/opentelemetry-collector" ]; })) socks); + } + { + # The probe loop names each gateway route, and the routes are read from + # the vhost, not from the receivers the probe is built from. A script + # missing one route, or treating 502 as a pass, is rejected. + name = "the readiness unit probes every collector route through the gateway"; + ok = + let + host = serviceContainersWithCi; + otel = host.services.hyperhive.swarm.otel; + locs = host.services.nginx.virtualHosts.${otel.domain}.locations; + routes = lib.sort lib.lessThan ( + map (l: lib.removeSuffix "/" (lib.removePrefix "/" l)) ( + lib.filter (l: (locs.${l}.proxyPass or null) != null) (lib.attrNames locs) + ) + ); + lines = script: map (lib.removePrefix " ") (lib.splitString "\n" script); + probed = + script: + let + loop = lib.findFirst (l: lib.hasPrefix "for route in " l) "" (lines script); + in + lib.sort lib.lessThan ( + lib.filter (w: w != "") ( + lib.splitString " " (lib.removeSuffix "; do" (lib.removePrefix "for route in " loop)) + ) + ); + verdict = + script: + probed script == routes + && lib.hasInfix "--resolve ${otel.domain}:443:127.0.0.1" script + && lib.hasInfix "https://${otel.domain}/$route/" script + && lib.hasInfix "case $code in 000 | 502 | 504) ;;" script; + script = host.systemd.services.swarm-otel-sockets.script; + in + lib.length routes > 2 + && lib.elem "h1" routes + && lib.elem "swarm-grafana" routes + && verdict script + && !verdict (lib.replaceStrings [ " swarm-grafana" ] [ "" ] script) + && !verdict (lib.replaceStrings [ "000 | 502 | 504" ] [ "000 | 504" ] script) + && !verdict (lib.replaceStrings [ "--resolve" ] [ "--noresolve" ] script); + } { # The DynamicUser layout keeps the history under /var/lib/private; the # static user's StateDirectory is the plain path. The move runs as its