vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: routes carry a route header and the readiness probe requires it

Browse source
This commit is contained in:
atlas 2026-10-24 21:19:39 +02:00 • committed by mara
commit 7680b07fcb
3 changed files with 101 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

8
docs/swarm/services.md
View file

@ -339,9 +339,11 @@ receivers come from one list, so they name the same sockets. The
`swarm-otel-sockets` unit connects to each socket as the gateway's user after
the container starts, then requests every route through the gateway vhost. It
fails if a socket refuses for 30 seconds, or if a route answers `000`, `502` or
`504` for 30 seconds, so a collector that's up but not listening, or an nginx
that doesn't route to it, shows as a failed unit. Any other status passes, the
collector's 4xx included. A switch leaves a window of about 20 seconds in which
`504`, or lacks its own `X-Swarm-Otel-Route` response header, for 30 seconds.
The gateway sets that header, named after the route, on every response from a
route, so a collector that's up but not listening, or an nginx that doesn't
have the route, shows as a failed unit. Any status other than those passes,
the collector's 4xx included. A switch leaves a window of about 20 seconds in which
nginx and the container are out of step: the container stops before nginx
restarts and starts about 18 seconds after it. Senders retry through that window.

34
nix/host-modules/swarm-otel.nix
View file

@ -312,6 +312,9 @@ let
# different sockets.
receiverSockets = hiveSockets // lib.genAttrs ([ swarmTierName ] ++ serviceProducerNames) socketOf;
upstreamOf = sock: "http://unix:${sock}:/";
# Set by nginx on every response from a route, whatever its status, so the
# readiness check can tell a route the running nginx has from the catch-all.
routeHeader = "X-Swarm-Otel-Route";
allSockets = lib.attrValues receiverSockets;
# The gid the gateway's nginx runs as. Only the number crosses into the
@ -496,8 +499,18 @@ in
# serves `/v1/metrics` and knows nothing about hives. Without it
# the receiver sees `/<hive>/v1/metrics` and answers 404 to a
# request that authenticated perfectly.
#
# ⚠️ A location with an `add_header` of its own inherits none of the
# server-scope ones, so the security headers are repeated here.
lib.mapAttrs' (
owner: sock: lib.nameValuePair "/${owner}/" { proxyPass = upstreamOf sock; }
owner: sock:
lib.nameValuePair "/${owner}/" {
proxyPass = upstreamOf sock;
extraConfig = ''
${gatewayCfg.lib.securityHeaders}
add_header ${routeHeader} ${owner} always;
'';
}
) receiverSockets
// {
# There is no swarm-wide inbox, and a closed door is the honest
@ -698,10 +711,11 @@ in
# A collector that is up but not listening on a socket the gateway names
# answers every producer with a 502 and fails no unit. This one fails when
# any receiver socket refuses a connection for 30s, or when the running
# nginx cannot reach one, so the switch reports it. It runs as the
# nginx does not route to one, so the switch reports it. It runs as the
# gateway's user: a socket nginx cannot open fails too. On the socket any
# HTTP status counts as accepted. Through nginx 000, 502 and 504 fail and
# every other status passes, a collector's 4xx included.
# HTTP status counts as accepted. Through nginx, 000, 502 and 504 fail, and so
# does a response without its route's own header; any other status passes, a
# collector's 4xx included.
systemd.services.swarm-otel-sockets = {
description = "wait for every swarm collector receiver socket to accept a connection";
after = [
@ -734,15 +748,17 @@ in
done
# The running nginx, not its config. `--resolve` pins the vhost to
# this host's gateway.
# this host's gateway. The route header is what separates a route nginx
# has from its catch-all 404.
for route in ${lib.escapeShellArgs (lib.attrNames receiverSockets)}; do
for _ in $(seq 30); do
code=$(curl -sk --max-time 5 --resolve ${cfg.domain}:443:127.0.0.1 -o /dev/null -w '%{http_code}' \
"https://${cfg.domain}/$route/") || code=000
case $code in 000 | 502 | 504) ;; *) continue 2 ;; esac
out=$(curl -sk --max-time 5 --resolve ${cfg.domain}:443:127.0.0.1 -o /dev/null \
-w '%{http_code} %header{${lib.toLower routeHeader}}' "https://${cfg.domain}/$route/") || out=000
read -r code seen <<< "$out"
case $code in 000 | 502 | 504) ;; *) [ "$seen" = "$route" ] && continue 2 ;; esac
sleep 1
done
echo "https://${cfg.domain}/$route/ answered $code after 30s" >&2
echo "https://${cfg.domain}/$route/ answered $code, route header '$seen', after 30s" >&2
exit 1
done
'';

74
nix/module-eval/swarm-services-switch.nix
View file

@ -949,10 +949,74 @@ let
&& !(lib.any (writes (sc // { ReadWritePaths = [ ]; })) socks)
&& !(lib.any (writes (sc // { ReadWritePaths = [ "/var/lib/opentelemetry-collector" ]; })) socks);
}
{
# The readiness unit tells a route nginx has from its catch-all 404 by
# this header, so every route carries its own name in it, `always` so a
# 4xx or 5xx from the collector carries it too. A location with an
# `add_header` inherits none of the server-scope ones, so the security
# headers have to be repeated. The catch-all carries no route header.
name = "every collector route sets its own route header and keeps the security headers";
ok =
let
host = serviceContainersWithCi;
otel = host.services.hyperhive.swarm.otel;
locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
sec = host.services.hyperhive.gateway.lib.securityHeaders;
proxied = lib.filter (l: (locs.${l}.proxyPass or null) != null) (lib.attrNames locs);
nameOf = l: lib.removeSuffix "/" (lib.removePrefix "/" l);
verdict =
ls:
lib.all (
l:
let
extra = ls.${l}.extraConfig or "";
in
lib.hasInfix "add_header X-Swarm-Otel-Route ${nameOf l} always;" extra
&& lib.hasInfix "add_header X-Frame-Options" extra
) proxied;
in
lib.length proxied > 2
&& lib.hasInfix "add_header X-Frame-Options" sec
&& !(lib.hasInfix "X-Swarm-Otel-Route" (locs."/".extraConfig or ""))
&& verdict locs
&& !verdict (
locs
// {
"/h1/" = locs."/h1/" // {
extraConfig = sec;
};
}
)
&& !verdict (
locs
// {
"/h1/" = locs."/h1/" // {
extraConfig = "add_header X-Swarm-Otel-Route h1 always;";
};
}
)
&& !verdict (
locs
// {
"/h1/" = locs."/h1/" // {
extraConfig = sec + "add_header X-Swarm-Otel-Route swarm always;";
};
}
)
&& !verdict (
locs
// {
"/h1/" = locs."/h1/" // {
extraConfig = sec + "add_header X-Swarm-Otel-Route h1;";
};
}
);
}
{
# The probe loop names each gateway route, and the routes are read from
# the vhost, not from the receivers the probe is built from. A script
# missing one route, or treating 502 as a pass, is rejected.
# missing one route, treating 502 as a pass, or passing on status alone
# without the route header, is rejected.
name = "the readiness unit probes every collector route through the gateway";
ok =
let
@ -980,7 +1044,9 @@ let
probed script == routes
&& lib.hasInfix "--resolve ${otel.domain}:443:127.0.0.1" script
&& lib.hasInfix "https://${otel.domain}/$route/" script
&& lib.hasInfix "case $code in 000 | 502 | 504) ;;" script;
&& lib.hasInfix "case $code in 000 | 502 | 504) ;;" script
&& lib.hasInfix "%header{x-swarm-otel-route}" script
&& lib.hasInfix ''[ "$seen" = "$route" ] && continue 2'' script;
script = host.systemd.services.swarm-otel-sockets.script;
in
lib.length routes > 2
@ -989,7 +1055,9 @@ let
&& verdict script
&& !verdict (lib.replaceStrings [ " swarm-grafana" ] [ "" ] script)
&& !verdict (lib.replaceStrings [ "000 | 502 | 504" ] [ "000 | 504" ] script)
&& !verdict (lib.replaceStrings [ "--resolve" ] [ "--noresolve" ] script);
&& !verdict (lib.replaceStrings [ "--resolve" ] [ "--noresolve" ] script)
&& !verdict (lib.replaceStrings [ "%header{x-swarm-otel-route}" ] [ "" ] script)
&& !verdict (lib.replaceStrings [ ''[ "$seen" = "$route" ] && '' ] [ "" ] script);
}
{
# The DynamicUser layout keeps the history under /var/lib/private; the