From 350b4e9fb77d0092c266231aaa04a8752a7c64ee Mon Sep 17 00:00:00 2001 From: damocles Date: Thu, 27 Aug 2026 00:24:19 +0200 Subject: [PATCH] swarm-otel/swarm-controller: authenticate + route the swarm-tier OTLP producer through the gateway --- nix/host-modules/swarm-controller.nix | 65 +++++---- nix/host-modules/swarm-otel.nix | 181 ++++++++++++++++++-------- swarm-queue-client/src/lib.rs | 5 +- 3 files changed, 170 insertions(+), 81 deletions(-) diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 672481ff..cd100f3e 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -155,27 +155,39 @@ let # above, genuinely optional and gated on the option resolving rather # than assumed. # - # Gated on THIS HOST running `swarm-otel`, not merely on it existing - # somewhere in the swarm: its receiver for a swarm-tier producer binds - # `127.0.0.1` only (see `swarm-otel.nix::producerPort`), so reaching it - # needs co-location, and this checks that directly rather than assuming - # it — the unasserted-co-location gap flagged elsewhere in this codebase - # doesn't apply here because there is nothing to assert: a controller on - # a host that doesn't run the collector simply exports nothing, the same - # graceful absence `forgeEnv` already models. + # By NAME through the gateway, never a loopback literal or a bare + # `producerPort` — mara's ruling on this exact module: "the swarm + # services dont have to run on the same host as the swarm controller." + # A controller and the swarm's collector are two independently-placed + # swarm services, exactly like a hive and the collector already are + # (`otel.nix`'s own `endpoint`), so this reaches it the same way: by + # its swarm-service name, resolved locally by dnsmasq on a co-located + # host and over the real network otherwise — no loopback-vs-remote + # knob to get wrong, and nothing here to gate on co-location because + # there is no co-location requirement left to check. + # + # No trailing slash: `opentelemetry_otlp`'s http/json exporter appends + # `/v1/metrics` to this base itself, and the swarm-otel gateway route + # (`swarm-otel.nix`'s nginx `locations."/${producerName}/"`) strips the + # `/${producerName}` prefix before the request reaches the receiver — + # so the two ends agree on exactly one path, `/v1/metrics`, arrived at + # from opposite directions. + # + # Gated on `otelSwarmCfg.enable` alone (a swarm-wide fact, not "enabled + # HERE"): the collector is reachable by name wherever it runs, so a + # controller not co-located with it now exports over https instead of + # exporting nothing — the graceful-absence case left is "no swarm-otel + # anywhere in this swarm at all," which `forgeEnv` already models the + # same way. otelSwarmCfg = config.services.hyperhive.swarm.otel; otelEnv = lib.optionalAttrs otelSwarmCfg.enable { - OTEL_EXPORTER_OTLP_ENDPOINT = "http://${otelSwarmCfg.domain}:${toString otelSwarmCfg.producerPort}"; - }; - - # Only when THIS host also runs `swarm-otel` — same reasoning and same - # shape as `hive-forge/default.nix`'s `ssoLocal`-gated entry: a raw host - # systemd unit does not get the bridge's dnsmasq resolution containers - # get, so the name that reaches a co-located collector over `otelEnv` - # above needs an explicit loopback alias here, or it resolves however - # (or however it fails to) off-host DNS says on this host. - otelHostsEntry = lib.optionalAttrs otelSwarmCfg.enable { - "127.0.0.1" = [ otelSwarmCfg.domain ]; + OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}"; + # Checked by `swarm-otel.nix`'s `oidc/${producerName}` authenticator + # against exactly this controller's own registered audience (see the + # `audience = [ queueClientId ]` client entry below) — the two have + # to name the same value or the exporter mints a token the receiver + # was never told to accept. + SWARM_CONTROLLER_OTEL_AUDIENCE = queueClientId; }; # Wrapped rather than documented: every one of these values is derived @@ -479,10 +491,6 @@ in "swarm-controller-credential" ]; - # See `otelHostsEntry`'s own comment: only present, and only ever - # `{ "127.0.0.1" = [ ... ]; }`, when this host also runs `swarm-otel`. - networking.hosts = otelHostsEntry; - users.users.swarm-controller = { isSystemUser = true; group = "swarm-controller"; @@ -509,6 +517,17 @@ in # redirect list, because authelia permits only the grants a client # names and an omitted `grant_types` means authorization-code alone. kind = "machine"; + # Its own id as its own permitted audience — same "client's own id + # is the value it may ask for" shape `swarm-authelia.nix`'s + # `hiveClients` uses, just self-referential instead of prefixed: + # this controller is not a hive (see `queueClientId`'s own doc + # comment), so it has no `hiveClientPrefix` name to reuse. This is + # what lets `vcs_metrics::authenticated_http_client()` mint a token + # for `SWARM_CONTROLLER_OTEL_AUDIENCE` and have `swarm-otel.nix`'s + # `oidc/swarm` authenticator (which checks for this same id) accept + # it — registering here without requesting there, or the reverse, + # both fail as a healthy-looking 401 several layers from this line. + audience = [ queueClientId ]; } ]; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index e037c448..b39e66bd 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -54,7 +54,11 @@ let # that reserves it is checking the same string the config emits. A literal # repeated at each site would let the guard and the config drift apart, which # is the failure this guard exists to prevent. - swarmTierName = "swarm"; + # Read-only option below, not a bare literal — `swarm-controller.nix` needs + # the identical string to build the same audience/endpoint, and a value + # bound once here (rather than copy-pasted at both sites) is the only way + # the two can't drift apart. + swarmTierName = cfg.producerName; # Every `` no hive may take. Read from `nix/reserved-names.nix`, the # same file the daemons are handed as `HIVE_RESERVED_NAMES`, because agent # names and hive names are ONE namespace going forward — a locally-owned @@ -63,7 +67,8 @@ let # # The assertion below still checks the string this module emits: the file # is asserted to CONTAIN `swarmTierName`, so a rename that dropped it from - # the file would be an eval error rather than a silently missing guard. + # the file (or a `producerName` override the file was never updated for) + # would be an eval error rather than a silently missing guard. reservedOwners = import ../reserved-names.nix; # A published target is declared as ONE url, because that url is also the @@ -253,14 +258,25 @@ in type = lib.types.port; default = 4390; description = '' - Port the swarm tier's own unauthenticated OTLP/HTTP receiver - listens on, at `127.0.0.1` — for a **swarm-level** producer - (`swarm-controller`'s vcs/jobq counters today) to push to, on the - same host this collector runs on. No authenticator, deliberately: - unlike the per-hive receivers, the label this receiver's samples - get (`swarm`) is the one the receiver's own *existence* - establishes — there is no hive identity to forge or attribute, - so the per-hive attribution mechanism does not apply here. + Port the swarm tier's own OTLP/HTTP receiver for a + **swarm-level** producer (`swarm-controller`'s vcs/jobq counters + today) listens on, at `127.0.0.1`. Reached through the gateway at + `https://''${domain}/''${producerName}/`, same shape as a hive's + own receiver — a swarm-level producer may not share a host with + this collector (mara, on the swarm-controller topology: "the + swarm services dont have to run on the same host as the swarm + controller"), so loopback-only reachability is not a supported + shape here any more than it is for a hive. + + Authenticated by `oidc/''${producerName}`, checking for the + audience `services.hyperhive.swarm.controller.queueClientId` + requests — not the per-hive `hiveClientPrefix` namespace, because + a swarm-level producer is explicitly not a hive (see that + option's own doc comment). The `swarm` label this receiver's + samples get is still the one the receiver's own *existence* + establishes, same as before — there is no hive identity being + forged or attributed here, just a caller proving it is the one + principal allowed to push into this pipeline. ⚠️ Deliberately NOT derived from `port + (number of hives)`: that range grows every time a hive is added, and a fixed offset from @@ -272,6 +288,23 @@ in ''; }; + producerName = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm"; + description = '' + The swarm-tier producer's own component/path/authenticator name + — the same reserved literal this module's collector components + (`otlp/swarm`, `oidc/swarm`, `resource/swarm`, `metrics/swarm`) + are already named after internally. Published as an option so a + sibling module (`swarm-controller.nix`) can address the receiver + by name — `https://''${domain}/''${producerName}/` — instead of + repeating the string. Read-only for the same reason + `clientId`/`machine`/`domain` are: two spellings of a name that + has to match on both ends is a mismatch waiting to happen. + ''; + }; + domain = lib.mkOption { type = lib.types.str; default = "otel.${domainBase}"; @@ -446,9 +479,15 @@ in h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } ) hivePorts // { + # The swarm-tier producer's own route — same trailing-slash + # shape as the per-hive locations above, and load-bearing for + # the same reason: it strips `/${producerName}` before the + # request reaches the receiver, which knows nothing about the + # path it was found at. + "/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/"; # There is no swarm-wide inbox, and a closed door is the honest - # description of that. Every route into this collector belongs to - # exactly one hive. + # description of that. Every other route into this collector + # belongs to exactly one hive or the swarm-tier producer above. "/".return = "404"; }; }; @@ -870,11 +909,16 @@ in }; } ) hivePorts - # The swarm tier's OWN receiver, unauthenticated on - # purpose — see `producerPort`'s description for why no - # per-hive-shaped attribution applies here. + # The swarm tier's OWN receiver — authenticated exactly + # like a hive's, just against a different identity (the + # swarm-level producer is not a hive; see + # `producerPort`'s description for what its audience + # checks and why). // { - "otlp/${swarmTierName}".protocols.http.endpoint = "127.0.0.1:${toString cfg.producerPort}"; + "otlp/${swarmTierName}".protocols.http = { + endpoint = "127.0.0.1:${toString cfg.producerPort}"; + auth.authenticator = "oidc/${swarmTierName}"; + }; } # MERGED with the per-hive receivers, never assigned over # them. A plain assignment here would drop every hive's @@ -1050,7 +1094,13 @@ in # INERT — the collector starts clean and the receiver # naming it authenticates nothing. Derived from the same # attrset as the receivers so the two cannot disagree. - service.extensions = map (h: "oidc/${h}") (lib.attrNames hivePorts); + service.extensions = + map (h: "oidc/${h}") (lib.attrNames hivePorts) + # The swarm-tier producer's own authenticator — unconditional, + # same reasoning as its receiver above: `otlp/${swarmTierName}` + # always exists, so the authenticator it names must too, or an + # extension-not-listed startup failure follows every deploy. + ++ [ "oidc/${swarmTierName}" ]; # Fan-out, not a choice: with both configured the same # samples go upstream AND into the swarm's store. The store @@ -1107,45 +1157,64 @@ in }; } // { - extensions = lib.mapAttrs' ( - h: _: - lib.nameValuePair "oidc/${h}" { - issuer_url = autheliaCfg.url; - # The audience this hive's client is registered to - # request, and the reason one hive's token is refused by - # another hive's receiver. Same expression authelia - # registers it under — a second spelling here would deny - # every hive, as a 401 that blames the token. - audience = "${autheliaCfg.hiveClientPrefix}${h}"; - # ⛔ DO NOT ADD `issuer_ca_path` HERE. It took this - # collector down for forty minutes once, and the failure - # is invisible to every check we have. - # - # It loads only the FIRST certificate in the file it - # names. The bundle assembled for this container is - # `system CAs ++ hive trust bundle`, so the anchor sits - # ~123rd and is never in the pool: the extension then - # cannot verify authelia and the whole collector exits - # `x509: certificate signed by unknown authority`, on - # every start, with 125 valid certificates in the file. - # - # Leaving it unset makes the extension use the process - # trust store, which `trustBundle` already populates via - # `SSL_CERT_FILE` — and *that* consumer reads every - # certificate regardless of order. One file, two - # consumers, opposite parsing: the fix is to stop naming - # it twice, not to reorder the bundle. - # - # ⚠️ Nor is pointing it at the hive trust bundle a fix: - # `hive-tls.nix` writes that leading with the *hive* CA - # (`nameConstraints` = this hive's domain), which cannot - # issue a swarm-level name at all. - # - # (Kept from the original note, still true and still - # worth not re-deriving: `issuer_ca_file`, `ca_file` and - # `tls.ca_file` are INVALID KEYS for this extension.) - } - ) hivePorts; + extensions = + lib.mapAttrs' ( + h: _: + lib.nameValuePair "oidc/${h}" { + issuer_url = autheliaCfg.url; + # The audience this hive's client is registered to + # request, and the reason one hive's token is refused by + # another hive's receiver. Same expression authelia + # registers it under — a second spelling here would deny + # every hive, as a 401 that blames the token. + audience = "${autheliaCfg.hiveClientPrefix}${h}"; + # ⛔ DO NOT ADD `issuer_ca_path` HERE. It took this + # collector down for forty minutes once, and the failure + # is invisible to every check we have. + # + # It loads only the FIRST certificate in the file it + # names. The bundle assembled for this container is + # `system CAs ++ hive trust bundle`, so the anchor sits + # ~123rd and is never in the pool: the extension then + # cannot verify authelia and the whole collector exits + # `x509: certificate signed by unknown authority`, on + # every start, with 125 valid certificates in the file. + # + # Leaving it unset makes the extension use the process + # trust store, which `trustBundle` already populates via + # `SSL_CERT_FILE` — and *that* consumer reads every + # certificate regardless of order. One file, two + # consumers, opposite parsing: the fix is to stop naming + # it twice, not to reorder the bundle. + # + # ⚠️ Nor is pointing it at the hive trust bundle a fix: + # `hive-tls.nix` writes that leading with the *hive* CA + # (`nameConstraints` = this hive's domain), which cannot + # issue a swarm-level name at all. + # + # (Kept from the original note, still true and still + # worth not re-deriving: `issuer_ca_file`, `ca_file` and + # `tls.ca_file` are INVALID KEYS for this extension.) + } + ) hivePorts + # The swarm-tier producer's own authenticator. Same extension + # type and same `issuer_url`/no-`issuer_ca_path` shape as every + # `oidc/${h}` above — only the audience differs, and + # deliberately does not reuse `hiveClientPrefix`: + # `swarm-controller` is explicitly not a hive (see its own + # module's `queueClientId` doc comment), so its audience is + # its own client id rather than a value from the per-hive + # namespace. Read via the option rather than a literal so a + # rename of `queueClientId`'s default cannot silently + # desync the two ends — this authenticator and the client + # requesting the audience it checks both have to agree, or + # a real swarm-controller push gets a healthy-looking 401. + // { + "oidc/${swarmTierName}" = { + issuer_url = autheliaCfg.url; + audience = config.services.hyperhive.swarm.controller.queueClientId; + }; + }; # `upsert`, not `insert`: a sender that stamps its own # `hive` must be OVERWRITTEN, not deferred to. This diff --git a/swarm-queue-client/src/lib.rs b/swarm-queue-client/src/lib.rs index 1165142e..c7a9143f 100644 --- a/swarm-queue-client/src/lib.rs +++ b/swarm-queue-client/src/lib.rs @@ -452,8 +452,9 @@ fn build_http_client(cfg: &QueueConfig) -> Result { /// per call, same as this crate did before the reconnect-storm fix added the /// cache. /// -/// `audience` is passed straight to [`token_request`] — see its doc for why -/// it is optional and when a caller needs it. `None` reproduces this +/// `audience` is passed straight to the private `token_request` helper — +/// see its doc for why it is optional and when a caller needs it. `None` +/// reproduces this /// function's behaviour before the parameter existed, so every caller from /// before that added it (the queue connect path, `auth.rs`'s bridge client) /// is unaffected.