vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: route the gateway from the receiver list and check the sockets accept

Browse source 
Refs #4815
This commit is contained in:
atlas 2026-10-24 20:53:47 +02:00 • committed by mara
commit 1325f44707
3 changed files with 123 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

67
nix/host-modules/swarm-otel.nix
View file

@ -307,8 +307,12 @@ let
# One socket per receiver, named after its owner (hive, swarm tier or
# service): `otlp/<owner>` is already unique, so the socket is too.
socketOf = owner: "${socketDir}/${owner}.sock";
upstreamOf = owner: "http://unix:${socketOf owner}:/";
allSockets = lib.attrValues hiveSockets ++ map socketOf ([ swarmTierName ] ++ serviceProducerNames);
# Every receiver's socket, by owner. The collector's receivers, the gateway's
# routes and the readiness check all read this attrset, so they cannot name
# different sockets.
receiverSockets = hiveSockets // lib.genAttrs ([ swarmTierName ] ++ serviceProducerNames) socketOf;
upstreamOf = sock: "http://unix:${sock}:/";
allSockets = lib.attrValues receiverSockets;
# The gid the gateway's nginx runs as. Only the number crosses into the
# container, which knows it as `gateway-nginx`.
@ -492,24 +496,16 @@ in
# serves `/v1/metrics` and knows nothing about hives. Without it
# the receiver sees `/<hive>/v1/metrics` and answers 404 to a
# request that authenticated perfectly.
lib.mapAttrs' (h: _: lib.nameValuePair "/${h}/" { proxyPass = upstreamOf h; }) hiveSockets
lib.mapAttrs' (
owner: sock: lib.nameValuePair "/${owner}/" { proxyPass = upstreamOf sock; }
) receiverSockets
// {
# The swarm-tier producer's own route — same trailing-slash
# shape as the per-hive locations above, and load-bearing for
# the same reason: it strips `/${producerName}` before the
# request reaches the receiver, which knows nothing about the
# path it was found at.
"/${swarmTierName}/".proxyPass = upstreamOf swarmTierName;
# There is no swarm-wide inbox, and a closed door is the honest
# description of that. Every other route into this collector
# belongs to exactly one hive or one swarm-tier producer.
# belongs to exactly one hive, swarm-tier producer or service
# forwarder, each on its own route for its own audience.
"/".return = "404";
}
# Each service forwarder, on its own route for its own audience —
# same trailing slashes, same reason.
// lib.mapAttrs' (
name: _: lib.nameValuePair "/${name}/" { proxyPass = upstreamOf name; }
) serviceProducers;
};
};
# The identities this tier authenticates against. Stated rather than
@ -699,6 +695,41 @@ in
# over a file that does not exist yet.
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
# A collector that is up but not listening on a socket the gateway names
# answers every producer with a 502 and fails no unit. This one fails when
# any receiver socket refuses a connection for 30s, so the switch reports
# it. It runs as the gateway's user: a socket nginx cannot open fails too.
# Any HTTP status counts as accepted; only a failed connect is a failure.
systemd.services.swarm-otel-sockets = {
description = "wait for every swarm collector receiver socket to accept a connection";
after = [ "container@${cfg.machine}.service" ];
wants = [ "container@${cfg.machine}.service" ];
# Restarted with the container, so every start of the collector is
# checked and not only the first one after a boot.
partOf = [ "container@${cfg.machine}.service" ];
wantedBy = [ "multi-user.target" ];
path = [
pkgs.curl
pkgs.coreutils
];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = config.services.nginx.user;
TimeoutStartSec = 90;
};
script = ''
for sock in ${lib.escapeShellArgs allSockets}; do
for _ in $(seq 30); do
curl -s -o /dev/null --max-time 5 --unix-socket "$sock" http://localhost/ && continue 2
sleep 1
done
echo "no connection to $sock after 30s" >&2
exit 1
done
'';
};
assertions = [
{
# Shaped after ./swarm-grafana.nix's `haveClientIdentity` assertion —
@ -1036,7 +1067,7 @@ in
# the per-hive `hiveClientPrefix` namespace.
// {
"otlp/${swarmTierName}".protocols.http = {
endpoint = socketOf swarmTierName;
endpoint = receiverSockets.${swarmTierName};
transport = "unix";
auth.authenticator = "oidc/${swarmTierName}";
};
@ -1048,7 +1079,7 @@ in
name: _:
lib.nameValuePair "otlp/${name}" {
protocols.http = {
endpoint = socketOf name;
endpoint = receiverSockets.${name};
transport = "unix";
auth.authenticator = "oidc/${name}";
};