vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swarm-otel/swarm-controller: authenticate + route the swarm-tier OTLP producer through the gateway

Browse source
This commit is contained in:
damocles 2026-08-27 00:24:19 +02:00 • committed by mara
commit 350b4e9fb7
3 changed files with 170 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

65
nix/host-modules/swarm-controller.nix
View file

@ -155,27 +155,39 @@ let
# above, genuinely optional and gated on the option resolving rather
# than assumed.
#
# Gated on THIS HOST running `swarm-otel`, not merely on it existing
# somewhere in the swarm: its receiver for a swarm-tier producer binds
# `127.0.0.1` only (see `swarm-otel.nix::producerPort`), so reaching it
# needs co-location, and this checks that directly rather than assuming
# it — the unasserted-co-location gap flagged elsewhere in this codebase
# doesn't apply here because there is nothing to assert: a controller on
# a host that doesn't run the collector simply exports nothing, the same
# graceful absence `forgeEnv` already models.
# By NAME through the gateway, never a loopback literal or a bare
# `producerPort` — mara's ruling on this exact module: "the swarm
# services dont have to run on the same host as the swarm controller."
# A controller and the swarm's collector are two independently-placed
# swarm services, exactly like a hive and the collector already are
# (`otel.nix`'s own `endpoint`), so this reaches it the same way: by
# its swarm-service name, resolved locally by dnsmasq on a co-located
# host and over the real network otherwise — no loopback-vs-remote
# knob to get wrong, and nothing here to gate on co-location because
# there is no co-location requirement left to check.
#
# No trailing slash: `opentelemetry_otlp`'s http/json exporter appends
# `/v1/metrics` to this base itself, and the swarm-otel gateway route
# (`swarm-otel.nix`'s nginx `locations."/${producerName}/"`) strips the
# `/${producerName}` prefix before the request reaches the receiver —
# so the two ends agree on exactly one path, `/v1/metrics`, arrived at
# from opposite directions.
#
# Gated on `otelSwarmCfg.enable` alone (a swarm-wide fact, not "enabled
# HERE"): the collector is reachable by name wherever it runs, so a
# controller not co-located with it now exports over https instead of
# exporting nothing — the graceful-absence case left is "no swarm-otel
# anywhere in this swarm at all," which `forgeEnv` already models the
# same way.
otelSwarmCfg = config.services.hyperhive.swarm.otel;
otelEnv = lib.optionalAttrs otelSwarmCfg.enable {
OTEL_EXPORTER_OTLP_ENDPOINT = "http://${otelSwarmCfg.domain}:${toString otelSwarmCfg.producerPort}";
};
# Only when THIS host also runs `swarm-otel` — same reasoning and same
# shape as `hive-forge/default.nix`'s `ssoLocal`-gated entry: a raw host
# systemd unit does not get the bridge's dnsmasq resolution containers
# get, so the name that reaches a co-located collector over `otelEnv`
# above needs an explicit loopback alias here, or it resolves however
# (or however it fails to) off-host DNS says on this host.
otelHostsEntry = lib.optionalAttrs otelSwarmCfg.enable {
"127.0.0.1" = [ otelSwarmCfg.domain ];
OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}";
# Checked by `swarm-otel.nix`'s `oidc/${producerName}` authenticator
# against exactly this controller's own registered audience (see the
# `audience = [ queueClientId ]` client entry below) — the two have
# to name the same value or the exporter mints a token the receiver
# was never told to accept.
SWARM_CONTROLLER_OTEL_AUDIENCE = queueClientId;
};
# Wrapped rather than documented: every one of these values is derived
@ -479,10 +491,6 @@ in
"swarm-controller-credential"
];
# See `otelHostsEntry`'s own comment: only present, and only ever
# `{ "127.0.0.1" = [ ... ]; }`, when this host also runs `swarm-otel`.
networking.hosts = otelHostsEntry;
users.users.swarm-controller = {
isSystemUser = true;
group = "swarm-controller";
@ -509,6 +517,17 @@ in
# redirect list, because authelia permits only the grants a client
# names and an omitted `grant_types` means authorization-code alone.
kind = "machine";
# Its own id as its own permitted audience — same "client's own id
# is the value it may ask for" shape `swarm-authelia.nix`'s
# `hiveClients` uses, just self-referential instead of prefixed:
# this controller is not a hive (see `queueClientId`'s own doc
# comment), so it has no `hiveClientPrefix` name to reuse. This is
# what lets `vcs_metrics::authenticated_http_client()` mint a token
# for `SWARM_CONTROLLER_OTEL_AUDIENCE` and have `swarm-otel.nix`'s
# `oidc/swarm` authenticator (which checks for this same id) accept
# it — registering here without requesting there, or the reverse,
# both fail as a healthy-looking 401 several layers from this line.
audience = [ queueClientId ];
}
];

181
nix/host-modules/swarm-otel.nix
View file

@ -54,7 +54,11 @@ let
# that reserves it is checking the same string the config emits. A literal
# repeated at each site would let the guard and the config drift apart, which
# is the failure this guard exists to prevent.
swarmTierName = "swarm";
# Read-only option below, not a bare literal — `swarm-controller.nix` needs
# the identical string to build the same audience/endpoint, and a value
# bound once here (rather than copy-pasted at both sites) is the only way
# the two can't drift apart.
swarmTierName = cfg.producerName;
# Every `<owner>` no hive may take. Read from `nix/reserved-names.nix`, the
# same file the daemons are handed as `HIVE_RESERVED_NAMES`, because agent
# names and hive names are ONE namespace going forward — a locally-owned
@ -63,7 +67,8 @@ let
#
# The assertion below still checks the string this module emits: the file
# is asserted to CONTAIN `swarmTierName`, so a rename that dropped it from
# the file would be an eval error rather than a silently missing guard.
# the file (or a `producerName` override the file was never updated for)
# would be an eval error rather than a silently missing guard.
reservedOwners = import ../reserved-names.nix;
# A published target is declared as ONE url, because that url is also the
@ -253,14 +258,25 @@ in
type = lib.types.port;
default = 4390;
description = ''
Port the swarm tier's own unauthenticated OTLP/HTTP receiver
listens on, at `127.0.0.1` — for a **swarm-level** producer
(`swarm-controller`'s vcs/jobq counters today) to push to, on the
same host this collector runs on. No authenticator, deliberately:
unlike the per-hive receivers, the label this receiver's samples
get (`swarm`) is the one the receiver's own *existence*
establishes — there is no hive identity to forge or attribute,
so the per-hive attribution mechanism does not apply here.
Port the swarm tier's own OTLP/HTTP receiver for a
**swarm-level** producer (`swarm-controller`'s vcs/jobq counters
today) listens on, at `127.0.0.1`. Reached through the gateway at
`https://''${domain}/''${producerName}/`, same shape as a hive's
own receiver — a swarm-level producer may not share a host with
this collector (mara, on the swarm-controller topology: "the
swarm services dont have to run on the same host as the swarm
controller"), so loopback-only reachability is not a supported
shape here any more than it is for a hive.
Authenticated by `oidc/''${producerName}`, checking for the
audience `services.hyperhive.swarm.controller.queueClientId`
requests — not the per-hive `hiveClientPrefix` namespace, because
a swarm-level producer is explicitly not a hive (see that
option's own doc comment). The `swarm` label this receiver's
samples get is still the one the receiver's own *existence*
establishes, same as before — there is no hive identity being
forged or attributed here, just a caller proving it is the one
principal allowed to push into this pipeline.
⚠️ Deliberately NOT derived from `port + (number of hives)`: that
range grows every time a hive is added, and a fixed offset from
@ -272,6 +288,23 @@ in
'';
};
producerName = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm";
description = ''
The swarm-tier producer's own component/path/authenticator name
— the same reserved literal this module's collector components
(`otlp/swarm`, `oidc/swarm`, `resource/swarm`, `metrics/swarm`)
are already named after internally. Published as an option so a
sibling module (`swarm-controller.nix`) can address the receiver
by name — `https://''${domain}/''${producerName}/` — instead of
repeating the string. Read-only for the same reason
`clientId`/`machine`/`domain` are: two spellings of a name that
has to match on both ends is a mismatch waiting to happen.
'';
};
domain = lib.mkOption {
type = lib.types.str;
default = "otel.${domainBase}";
@ -446,9 +479,15 @@ in
h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; }
) hivePorts
// {
# The swarm-tier producer's own route — same trailing-slash
# shape as the per-hive locations above, and load-bearing for
# the same reason: it strips `/${producerName}` before the
# request reaches the receiver, which knows nothing about the
# path it was found at.
"/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/";
# There is no swarm-wide inbox, and a closed door is the honest
# description of that. Every route into this collector belongs to
# exactly one hive.
# description of that. Every other route into this collector
# belongs to exactly one hive or the swarm-tier producer above.
"/".return = "404";
};
};
@ -870,11 +909,16 @@ in
};
}
) hivePorts
# The swarm tier's OWN receiver, unauthenticated on
# purpose — see `producerPort`'s description for why no
# per-hive-shaped attribution applies here.
# The swarm tier's OWN receiver — authenticated exactly
# like a hive's, just against a different identity (the
# swarm-level producer is not a hive; see
# `producerPort`'s description for what its audience
# checks and why).
// {
"otlp/${swarmTierName}".protocols.http.endpoint = "127.0.0.1:${toString cfg.producerPort}";
"otlp/${swarmTierName}".protocols.http = {
endpoint = "127.0.0.1:${toString cfg.producerPort}";
auth.authenticator = "oidc/${swarmTierName}";
};
}
# MERGED with the per-hive receivers, never assigned over
# them. A plain assignment here would drop every hive's
@ -1050,7 +1094,13 @@ in
# INERT — the collector starts clean and the receiver
# naming it authenticates nothing. Derived from the same
# attrset as the receivers so the two cannot disagree.
service.extensions = map (h: "oidc/${h}") (lib.attrNames hivePorts);
service.extensions =
map (h: "oidc/${h}") (lib.attrNames hivePorts)
# The swarm-tier producer's own authenticator — unconditional,
# same reasoning as its receiver above: `otlp/${swarmTierName}`
# always exists, so the authenticator it names must too, or an
# extension-not-listed startup failure follows every deploy.
++ [ "oidc/${swarmTierName}" ];
# Fan-out, not a choice: with both configured the same
# samples go upstream AND into the swarm's store. The store
@ -1107,45 +1157,64 @@ in
};
}
// {
extensions = lib.mapAttrs' (
h: _:
lib.nameValuePair "oidc/${h}" {
issuer_url = autheliaCfg.url;
# The audience this hive's client is registered to
# request, and the reason one hive's token is refused by
# another hive's receiver. Same expression authelia
# registers it under — a second spelling here would deny
# every hive, as a 401 that blames the token.
audience = "${autheliaCfg.hiveClientPrefix}${h}";
# ⛔ DO NOT ADD `issuer_ca_path` HERE. It took this
# collector down for forty minutes once, and the failure
# is invisible to every check we have.
#
# It loads only the FIRST certificate in the file it
# names. The bundle assembled for this container is
# `system CAs ++ hive trust bundle`, so the anchor sits
# ~123rd and is never in the pool: the extension then
# cannot verify authelia and the whole collector exits
# `x509: certificate signed by unknown authority`, on
# every start, with 125 valid certificates in the file.
#
# Leaving it unset makes the extension use the process
# trust store, which `trustBundle` already populates via
# `SSL_CERT_FILE` — and *that* consumer reads every
# certificate regardless of order. One file, two
# consumers, opposite parsing: the fix is to stop naming
# it twice, not to reorder the bundle.
#
# ⚠️ Nor is pointing it at the hive trust bundle a fix:
# `hive-tls.nix` writes that leading with the *hive* CA
# (`nameConstraints` = this hive's domain), which cannot
# issue a swarm-level name at all.
#
# (Kept from the original note, still true and still
# worth not re-deriving: `issuer_ca_file`, `ca_file` and
# `tls.ca_file` are INVALID KEYS for this extension.)
}
) hivePorts;
extensions =
lib.mapAttrs' (
h: _:
lib.nameValuePair "oidc/${h}" {
issuer_url = autheliaCfg.url;
# The audience this hive's client is registered to
# request, and the reason one hive's token is refused by
# another hive's receiver. Same expression authelia
# registers it under — a second spelling here would deny
# every hive, as a 401 that blames the token.
audience = "${autheliaCfg.hiveClientPrefix}${h}";
# ⛔ DO NOT ADD `issuer_ca_path` HERE. It took this
# collector down for forty minutes once, and the failure
# is invisible to every check we have.
#
# It loads only the FIRST certificate in the file it
# names. The bundle assembled for this container is
# `system CAs ++ hive trust bundle`, so the anchor sits
# ~123rd and is never in the pool: the extension then
# cannot verify authelia and the whole collector exits
# `x509: certificate signed by unknown authority`, on
# every start, with 125 valid certificates in the file.
#
# Leaving it unset makes the extension use the process
# trust store, which `trustBundle` already populates via
# `SSL_CERT_FILE` — and *that* consumer reads every
# certificate regardless of order. One file, two
# consumers, opposite parsing: the fix is to stop naming
# it twice, not to reorder the bundle.
#
# ⚠️ Nor is pointing it at the hive trust bundle a fix:
# `hive-tls.nix` writes that leading with the *hive* CA
# (`nameConstraints` = this hive's domain), which cannot
# issue a swarm-level name at all.
#
# (Kept from the original note, still true and still
# worth not re-deriving: `issuer_ca_file`, `ca_file` and
# `tls.ca_file` are INVALID KEYS for this extension.)
}
) hivePorts
# The swarm-tier producer's own authenticator. Same extension
# type and same `issuer_url`/no-`issuer_ca_path` shape as every
# `oidc/${h}` above — only the audience differs, and
# deliberately does not reuse `hiveClientPrefix`:
# `swarm-controller` is explicitly not a hive (see its own
# module's `queueClientId` doc comment), so its audience is
# its own client id rather than a value from the per-hive
# namespace. Read via the option rather than a literal so a
# rename of `queueClientId`'s default cannot silently
# desync the two ends — this authenticator and the client
# requesting the audience it checks both have to agree, or
# a real swarm-controller push gets a healthy-looking 401.
// {
"oidc/${swarmTierName}" = {
issuer_url = autheliaCfg.url;
audience = config.services.hyperhive.swarm.controller.queueClientId;
};
};
# `upsert`, not `insert`: a sender that stamps its own
# `hive` must be OVERWRITTEN, not deferred to. This

5
swarm-queue-client/src/lib.rs
View file

@ -452,8 +452,9 @@ fn build_http_client(cfg: &QueueConfig) -> Result<reqwest::Client, Error> {
/// per call, same as this crate did before the reconnect-storm fix added the
/// cache.
///
/// `audience` is passed straight to [`token_request`] — see its doc for why
/// it is optional and when a caller needs it. `None` reproduces this
/// `audience` is passed straight to the private `token_request` helper —
/// see its doc for why it is optional and when a caller needs it. `None`
/// reproduces this
/// function's behaviour before the parameter existed, so every caller from
/// before that added it (the queue connect path, `auth.rs`'s bridge client)
/// is unaffected.