vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: route the gateway from the receiver list and check the sockets accept

Browse source 
Refs #4815
This commit is contained in:
atlas 2026-10-24 20:53:47 +02:00 • committed by mara
commit 1325f44707
3 changed files with 123 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

6
docs/swarm/services.md
View file

@ -334,7 +334,11 @@ It runs in a `swarm-otel` container on the hive bridge, in a network
namespace of its own. It binds no OTLP port: every receiver (one per hive,
the swarm tier's and one per service forwarder) listens on a unix socket in
`/run/swarm-otel`, a directory only the gateway's nginx and the container can
open, and the gateway forwards to them.
open, and the gateway forwards to them. The gateway's routes and the collector's
receivers come from one list, so they name the same sockets. The
`swarm-otel-sockets` unit connects to each socket as the gateway's user after
the container starts and fails if one refuses for 30 seconds, so a collector
that's up but not listening shows as a failed unit.
Every hive's own collector reaches this one by its gateway name,
`swarm.otel.domain` (default `otel.<swarm domain>`) — the same

67
nix/host-modules/swarm-otel.nix
View file

@ -307,8 +307,12 @@ let
# One socket per receiver, named after its owner (hive, swarm tier or
# service): `otlp/<owner>` is already unique, so the socket is too.
socketOf = owner: "${socketDir}/${owner}.sock";
upstreamOf = owner: "http://unix:${socketOf owner}:/";
allSockets = lib.attrValues hiveSockets ++ map socketOf ([ swarmTierName ] ++ serviceProducerNames);
# Every receiver's socket, by owner. The collector's receivers, the gateway's
# routes and the readiness check all read this attrset, so they cannot name
# different sockets.
receiverSockets = hiveSockets // lib.genAttrs ([ swarmTierName ] ++ serviceProducerNames) socketOf;
upstreamOf = sock: "http://unix:${sock}:/";
allSockets = lib.attrValues receiverSockets;
# The gid the gateway's nginx runs as. Only the number crosses into the
# container, which knows it as `gateway-nginx`.
@ -492,24 +496,16 @@ in
# serves `/v1/metrics` and knows nothing about hives. Without it
# the receiver sees `/<hive>/v1/metrics` and answers 404 to a
# request that authenticated perfectly.
lib.mapAttrs' (h: _: lib.nameValuePair "/${h}/" { proxyPass = upstreamOf h; }) hiveSockets
lib.mapAttrs' (
owner: sock: lib.nameValuePair "/${owner}/" { proxyPass = upstreamOf sock; }
) receiverSockets
// {
# The swarm-tier producer's own route — same trailing-slash
# shape as the per-hive locations above, and load-bearing for
# the same reason: it strips `/${producerName}` before the
# request reaches the receiver, which knows nothing about the
# path it was found at.
"/${swarmTierName}/".proxyPass = upstreamOf swarmTierName;
# There is no swarm-wide inbox, and a closed door is the honest
# description of that. Every other route into this collector
# belongs to exactly one hive or one swarm-tier producer.
# belongs to exactly one hive, swarm-tier producer or service
# forwarder, each on its own route for its own audience.
"/".return = "404";
}
# Each service forwarder, on its own route for its own audience —
# same trailing slashes, same reason.
// lib.mapAttrs' (
name: _: lib.nameValuePair "/${name}/" { proxyPass = upstreamOf name; }
) serviceProducers;
};
};
# The identities this tier authenticates against. Stated rather than
@ -699,6 +695,41 @@ in
# over a file that does not exist yet.
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
# A collector that is up but not listening on a socket the gateway names
# answers every producer with a 502 and fails no unit. This one fails when
# any receiver socket refuses a connection for 30s, so the switch reports
# it. It runs as the gateway's user: a socket nginx cannot open fails too.
# Any HTTP status counts as accepted; only a failed connect is a failure.
systemd.services.swarm-otel-sockets = {
description = "wait for every swarm collector receiver socket to accept a connection";
after = [ "container@${cfg.machine}.service" ];
wants = [ "container@${cfg.machine}.service" ];
# Restarted with the container, so every start of the collector is
# checked and not only the first one after a boot.
partOf = [ "container@${cfg.machine}.service" ];
wantedBy = [ "multi-user.target" ];
path = [
pkgs.curl
pkgs.coreutils
];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = config.services.nginx.user;
TimeoutStartSec = 90;
};
script = ''
for sock in ${lib.escapeShellArgs allSockets}; do
for _ in $(seq 30); do
curl -s -o /dev/null --max-time 5 --unix-socket "$sock" http://localhost/ && continue 2
sleep 1
done
echo "no connection to $sock after 30s" >&2
exit 1
done
'';
};
assertions = [
{
# Shaped after ./swarm-grafana.nix's `haveClientIdentity` assertion —
@ -1036,7 +1067,7 @@ in
# the per-hive `hiveClientPrefix` namespace.
// {
"otlp/${swarmTierName}".protocols.http = {
endpoint = socketOf swarmTierName;
endpoint = receiverSockets.${swarmTierName};
transport = "unix";
auth.authenticator = "oidc/${swarmTierName}";
};
@ -1048,7 +1079,7 @@ in
name: _:
lib.nameValuePair "otlp/${name}" {
protocols.http = {
endpoint = socketOf name;
endpoint = receiverSockets.${name};
transport = "unix";
auth.authenticator = "oidc/${name}";
};

69
nix/module-eval/swarm-services-switch.nix
View file

@ -851,6 +851,75 @@ let
&& lib.elem "gateway-nginx" unit.SupplementaryGroups
&& unit.UMask == "0007";
}
{
# The routes the gateway forwards to are exactly the sockets the
# collector listens on, and none is a TCP address: a route left on
# loopback after the collector moved into its own netns returns a 502 to
# every producer while the collector runs. `verdict` is applied to
# broken copies of the routes too, so a check that cannot fail is caught.
name = "the gateway's collector routes are the collector's unix receivers and none is loopback";
ok =
let
host = serviceContainersWithCi;
otel = host.services.hyperhive.swarm.otel;
s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings;
locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
sorted = xs: lib.sort lib.lessThan (lib.unique xs);
unixReceivers = lib.filter (r: (r.protocols.http.transport or null) == "unix") (
lib.attrValues s.receivers
);
receiverSocks = sorted (map (r: r.protocols.http.endpoint) unixReceivers);
socketOfRoute =
p:
if lib.hasPrefix "http://unix:" p then
lib.removeSuffix ":/" (lib.removePrefix "http://unix:" p)
else
p;
verdict =
routes:
let
passes = lib.filter (p: p != null) (map (l: l.proxyPass or null) (lib.attrValues routes));
in
sorted (map socketOfRoute passes) == receiverSocks
&& !(lib.any (p: lib.hasInfix "127.0.0.1" p || lib.hasInfix "localhost" p) passes);
in
verdict locs
&& lib.elem "/run/swarm-otel/h1.sock" receiverSocks
&& lib.elem "/run/swarm-otel/${otel.producerName}.sock" receiverSocks
&& lib.elem "/run/swarm-otel/swarm-grafana.sock" receiverSocks
&&
lib.length unixReceivers == lib.length (lib.attrNames host.services.hyperhive.swarm.hives)
+ 1
+ lib.length (lib.attrNames otel.serviceProducers)
# Each broken copy must fail: a TCP route, a missing route, a route to
# a socket no receiver has.
&& !verdict (locs // { "/h1/".proxyPass = "http://127.0.0.1:4319/"; })
&& !verdict (lib.removeAttrs locs [ "/h1/" ])
&& !verdict (locs // { "/x/".proxyPass = "http://unix:/run/swarm-otel/x.sock:/"; });
}
{
# After the container, restarted with it, and run as the gateway's user,
# so a socket nginx cannot open fails the unit. Every receiver socket is
# in the script.
name = "the collector's receiver sockets are connect-checked after the container starts";
ok =
let
host = serviceContainersWithCi;
u = host.systemd.services.swarm-otel-sockets;
s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings;
socks = map (r: r.protocols.http.endpoint) (
lib.filter (r: (r.protocols.http.transport or null) == "unix") (lib.attrValues s.receivers)
);
in
socks != [ ]
&& lib.all (sock: lib.hasInfix sock u.script) socks
&& !(lib.hasInfix "/run/swarm-otel/absent.sock" u.script)
&& u.serviceConfig.Type == "oneshot"
&& u.serviceConfig.User == host.services.nginx.user
&& lib.elem "container@swarm-otel.service" u.after
&& lib.elem "container@swarm-otel.service" u.partOf
&& lib.elem "multi-user.target" u.wantedBy;
}
{
# The DynamicUser layout keeps the history under /var/lib/private; the
# static user's StateDirectory is the plain path. The move runs as its