vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: run in its own network namespace, receivers on unix sockets

Browse source 
Refs #4815
This commit is contained in:
atlas 2026-10-22 03:22:49 +02:00 • committed by mara
commit e226e65954
6 changed files with 196 additions and 221 deletions
Download patch file Download diff file Expand all files Collapse all files

183
nix/host-modules/swarm-otel.nix
View file

@ -168,16 +168,10 @@ let
# `attrNames` is sorted, so this is a function of the hive SET and not of
# the order anyone wrote it in.
#
# These ports are internal and appear in no URL: a hive addresses its own
# receiver as a PATH on this collector's single gateway name, and nginx —
# rendered from this same evaluation — is the only thing that ever names
# the port. That is what makes deriving them safe here and unsafe in the
# obvious other place: were a hive told a port, inserting a hive would
# renumber the ones after it and silently move a port a running hive was
# already sending to.
hivePorts = lib.listToAttrs (
lib.imap0 (i: h: lib.nameValuePair h (cfg.port + i)) (lib.attrNames hyperhiveCfg.swarm.hives)
);
# A hive addresses its own receiver as a PATH on this collector's single
# gateway name; nginx, rendered from this same evaluation, is the only thing
# that ever names the socket.
hiveSockets = lib.genAttrs (lib.attrNames hyperhiveCfg.swarm.hives) socketOf;
# The swarm's authelia is reached by its gateway name, whose leaf is
# issued by the swarm services sub-CA — so this container needs the same
@ -301,11 +295,25 @@ let
# Now always satisfied, as a consequence of the log store always existing.
collectLogs = logExporterNames != [ ];
# Shared host netns, like every sibling swarm service: the hive tier
# reaches this collector, and this collector reaches the metrics
# store, without either crossing a network boundary that would need
# its own trust material.
privateNetwork = false;
# A netns of its own on the hive bridge: the container dials authelia, the
# stores and the upstream by name through the bridge, and the gateway reaches
# every OTLP receiver over a socket, so no receiver binds a TCP port on any
# interface.
privateNetwork = true;
# Fixed, not an option: the directory's mode is what keeps the sockets
# private, so it holds the receivers' sockets and nothing else.
socketDir = "/run/swarm-otel";
# One socket per receiver, named after its owner (hive, swarm tier or
# service): `otlp/<owner>` is already unique, so the socket is too.
socketOf = owner: "${socketDir}/${owner}.sock";
upstreamOf = owner: "http://unix:${socketOf owner}:/";
allSockets = lib.attrValues hiveSockets ++ map socketOf ([ swarmTierName ] ++ serviceProducerNames);
# The gid the gateway's nginx runs as. Only the number crosses into the
# container, which knows it as `gateway-nginx`.
nginxGid = config.ids.gids.nginx;
socketGroup = "gateway-nginx";
storeRetry = import ./lib/store-retry.nix { };
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
in
@ -455,23 +463,23 @@ in
# vhost would answer for a service it does not have.
services.hyperhive.gateway.localNames = [ cfg.domain ];
# This host serves the vhost, and the container behind it resolves
# through the hive's dnsmasq.
# This host serves the vhost, and the hive's dnsmasq answers its name.
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# The hive bridge carries the container's DHCP lease and its resolver.
services.hyperhive.network.enable = lib.mkDefault true;
# The metrics counterpart to this collector's own journal, which its
# receiver ships like any other unit's: the journal says the process is
# alive, these counters say whether it is dropping what it receives.
# Loopback works here because `privateNetwork = false` — this container
# shares the host's netns, so `127.0.0.1` is where `telemetryPort` is
# bound.
# Loopback is the container's own: this collector scrapes `telemetryPort`
# inside its netns, and nothing else reads it.
services.hyperhive.swarm.otel.scrapeTargets.collector = "127.0.0.1:${toString cfg.telemetryPort}";
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
# every sibling swarm service: TLS terminates here, then plain http to
# the co-located container over loopback (shared netns, like the store
# this collector writes to).
# every sibling swarm service: TLS terminates here, then plain http over
# the receiver's unix socket.
services.hyperhive.gateway.vhosts."${cfg.domain}" = {
locations =
# One name for the whole collector, and the hive is a path under
@ -484,16 +492,14 @@ in
# serves `/v1/metrics` and knows nothing about hives. Without it
# the receiver sees `/<hive>/v1/metrics` and answers 404 to a
# request that authenticated perfectly.
lib.mapAttrs' (
h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; }
) hivePorts
lib.mapAttrs' (h: _: lib.nameValuePair "/${h}/" { proxyPass = upstreamOf h; }) hiveSockets
// {
# The swarm-tier producer's own route — same trailing-slash
# shape as the per-hive locations above, and load-bearing for
# the same reason: it strips `/${producerName}` before the
# request reaches the receiver, which knows nothing about the
# path it was found at.
"/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/";
"/${swarmTierName}/".proxyPass = upstreamOf swarmTierName;
# There is no swarm-wide inbox, and a closed door is the honest
# description of that. Every other route into this collector
# belongs to exactly one hive or one swarm-tier producer.
@ -502,7 +508,7 @@ in
# Each service forwarder, on its own route for its own audience —
# same trailing slashes, same reason.
// lib.mapAttrs' (
name: p: lib.nameValuePair "/${name}/" { proxyPass = "http://127.0.0.1:${toString p.port}/"; }
name: _: lib.nameValuePair "/${name}/" { proxyPass = upstreamOf name; }
) serviceProducers;
};
@ -663,7 +669,18 @@ in
lib.mkDefault collectorSecretInContainer
);
systemd.tmpfiles.rules = lib.optionals haveClientIdentity (
systemd.tmpfiles.rules = [
# The socket directory, created host-side before the container starts:
# nixos-container refuses to start when a bind source is missing.
#
# Root-owned, group nginx, setgid `2770`: the collector creates its
# sockets here through the group (its uid is allocated inside the
# container, so the host cannot name it), and a socket takes the
# directory's group, so the gateway can connect. Everything else is
# kept out.
"d ${socketDir} 2770 root ${toString nginxGid} - -"
]
++ lib.optionals haveClientIdentity (
[
# The bind source must exist when the container starts.
"d ${collectorSecretDir} 0700 root root -"
@ -872,44 +889,11 @@ in
anything, and the two are scraped by different rules with
different trust.
Rename one. A target is either reachable on loopback because it
shares this host, or published and reached with a credential; it
Rename one. A target is either reachable on the collector's own
loopback, or published and reached with a credential; it
should not be described as both.
'';
}
{
# A port collision between two listeners on one host is a runtime
# coin toss with nothing in any log — the failure this whole
# comment budget exists to prevent. Checked against every port
# reachable from here; a port some other module picks is not.
#
# ⚠️ `cfg.port` is deliberately absent from `others`: it is the
# FIRST element of the derived range, so listing it would make this
# assertion fire on every config.
assertion =
let
derived = lib.attrValues hivePorts;
others = [
cfg.telemetryPort
cfg.producerPort
config.services.hyperhive.swarm.hiveOtel.collectorPort
]
++ map (p: p.port) (lib.attrValues serviceProducers);
all = derived ++ others;
in
lib.length (lib.unique all) == lib.length all;
message = ''
services.hyperhive.swarm.otel: the receiver range starting at
port (${toString cfg.port}, one port per hive in
services.hyperhive.swarm.hives) overlaps another port on this
host.
Every swarm container shares the host's network namespace, so
two listeners claiming one port is not a build failure — it is
whichever process started first, silently. Move
services.hyperhive.swarm.otel.port to a free range.
'';
}
];
containers.${cfg.machine} = {
@ -918,19 +902,28 @@ in
# No `--link-journal=host`: this container's journal stays in the guest,
# where `journald/self` reads it.
inherit privateNetwork;
hostBridge = config.services.hyperhive.network.bridgeName;
# The upstream credential is operator-provided and lives on the host.
# Read-only, and only when one is configured — binding a path that
# does not exist makes nixos-container refuse to start the container,
# which is a stall several layers from its cause.
bindMounts =
# A bind mount rather than a path both sides happen to know: a
# container's `/run` is an nspawn tmpfs the host cannot see into.
{
${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
}
# The collector's OIDC secret, where the delivery unit writes it.
lib.optionalAttrs haveClientIdentity {
// lib.optionalAttrs haveClientIdentity {
${collectorSecretDir} = {
hostPath = collectorSecretDir;
isReadOnly = true;
};
}
# The upstream credential is operator-provided and lives on the host.
# Read-only, and only when one is configured — binding a path that
# does not exist makes nixos-container refuse to start the container,
# which is a stall several layers from its cause.
// lib.optionalAttrs (upstream.headersCredential != null) {
${upstream.headersCredential} = {
hostPath = upstream.headersCredential;
@ -996,6 +989,15 @@ in
system.stateVersion = config.system.stateVersion;
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
# owns resolv.conf, so dhcpcd must not write it over that unit.
networking.interfaces.eth0.useDHCP = true;
networking.dhcpcd.extraConfig = "nohook resolv.conf";
# The sockets' group is the directory's, so that group is nginx's gid.
# A static group, so the unit below can name it.
users.groups.${socketGroup}.gid = nginxGid;
services.opentelemetry-collector = {
enable = true;
package = pkgs.opentelemetry-collector-contrib;
@ -1018,22 +1020,24 @@ in
# reveals which one matched.
receivers =
lib.mapAttrs' (
h: p:
h: sock:
lib.nameValuePair "otlp/${h}" {
protocols.http = {
endpoint = "127.0.0.1:${toString p}";
endpoint = sock;
transport = "unix";
auth.authenticator = "oidc/${h}";
};
}
) hivePorts
) hiveSockets
# The swarm tier's OWN receiver — authenticated exactly
# like a hive's, just against a different identity (the
# swarm-level producer is not a hive; see
# `producerPort`'s description for what its audience
# checks and why).
# swarm-level producer is not a hive). Its authenticator checks
# the audience `swarm.controller.queueClientId` requests, not
# the per-hive `hiveClientPrefix` namespace.
// {
"otlp/${swarmTierName}".protocols.http = {
endpoint = "127.0.0.1:${toString cfg.producerPort}";
endpoint = socketOf swarmTierName;
transport = "unix";
auth.authenticator = "oidc/${swarmTierName}";
};
}
@ -1041,10 +1045,11 @@ in
# principal of its own — a receiver per audience, because an
# `oidc` authenticator checks one.
// lib.mapAttrs' (
name: p:
name: _:
lib.nameValuePair "otlp/${name}" {
protocols.http = {
endpoint = "127.0.0.1:${toString p.port}";
endpoint = socketOf name;
transport = "unix";
auth.authenticator = "oidc/${name}";
};
}
@ -1256,7 +1261,7 @@ in
# Same rule, and the journald receiver's `storage:` above
# would name a component the collector never starts.
[ "file_storage" ]
++ map (h: "oidc/${h}") (lib.attrNames hivePorts)
++ map (h: "oidc/${h}") (lib.attrNames hiveSockets)
# The swarm-tier producer's own authenticator — unconditional,
# same reasoning as its receiver above: `otlp/${swarmTierName}`
# always exists, so the authenticator it names must too, or an
@ -1285,7 +1290,7 @@ in
processors = [ "resource/${h}" ];
exporters = exporterNames;
}
) hivePorts
) hiveSockets
# Its OWN pipeline, and that separation is the ruling, not a
# tidiness choice: every `resource/<hive>` above UPSERTS a
# `hive` key, so a scraped swarm sample routed through any of
@ -1343,7 +1348,7 @@ in
processors = [ "resource/${h}" ];
exporters = logExporterNames;
}
) hivePorts
) hiveSockets
)
# Logs fan out exactly as metrics do: the swarm's store when it
# runs, the operator's upstream when one is configured, both
@ -1446,7 +1451,7 @@ in
# worth not re-deriving: `issuer_ca_file`, `ca_file` and
# `tls.ca_file` are INVALID KEYS for this extension.)
}
) hivePorts
) hiveSockets
# The swarm-tier producer's own authenticator. Same extension
# type and same `issuer_url`/no-`issuer_ca_path` shape as every
# `oidc/${h}` above — only the audience differs, and
@ -1534,7 +1539,7 @@ in
}
];
}
) hivePorts
) hiveSockets
# The swarm tier's own stamp: `swarm` and deliberately NO
# `hive`. A scraped swarm service belongs to the swarm, not to
# any one hive, so there is no honest value to put there — and
@ -1623,6 +1628,18 @@ in
serviceConfig.RestartSec = 5;
}
{
serviceConfig = {
# The unit's user is allocated per start (`DynamicUser`), so
# it reaches the socket directory through this group, and
# `0007` makes each socket `0770`: group nginx can connect.
SupplementaryGroups = [ socketGroup ];
UMask = "0007";
# A socket left by a killed process would fail the next `bind`.
ExecStartPre = [ "${pkgs.coreutils}/bin/rm -f ${lib.escapeShellArgs allSockets}" ];
};
}
# The credential file is already `NAME=value`, systemd's
# EnvironmentFile format — so the secret reaches the process as
# an environment variable without being read by nix, written to