From 0f801c645ffa3660f4437a336efb0cb62ee92935 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 19 Aug 2026 13:59:04 +0200 Subject: [PATCH 1/7] otel: build the hive tier's collector from contrib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hive tier has to *present* a credential to the swarm tier, and the upstream collector build ships no auth extensions at all — `oauth2client` exists only in contrib. So this is what makes authenticated egress expressible, not a preference between two equivalent packages. It also matches the swarm tier, which has been on contrib since it was written. Two tiers of the same pipeline built from different component sets is a difference nobody would predict from reading either module. Not a build-farm cost: contrib is fetched, not compiled. No behaviour change on its own — nothing names an extension yet. The config that does lands separately, because that is the commit whose failure mode needs a running collector to detect: `otelcol validate` accepts a receiver naming an absent extension and the process then dies at startup, so a green build proves nothing about it. --- nix/host-modules/otel.nix | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 53d43329..a3024641 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -224,6 +224,22 @@ in services.opentelemetry-collector = { enable = true; + # Contrib, matching the swarm tier (./swarm-otel.nix). The upstream + # default build has no auth extensions at all, and this tier has to + # *present* a credential to the swarm tier — `oauth2client` lives + # only in contrib, so the package choice is what makes authenticated + # egress expressible rather than a preference. + # + # Not a build-farm cost: contrib is fetched, not compiled. + # + # ⚠️ Read the note directly below before adding any extension here. + # It describes precisely the trap this package unlocks: naming an + # extension the build lacks passes `validate` and then kills the + # collector at startup. With contrib the extensions exist — but the + # gap it warns about (a green build proving nothing about whether + # the process starts) is exactly why this module's auth wiring is + # gated by a probe that runs both collectors, not by eval. + package = pkgs.opentelemetry-collector-contrib; # `validateConfigFile` defaults to `isStorePath configFile`, # and `configFile` is null on the `settings` path — so the # upstream default is OFF for exactly the way this module From 0b98f0ac8fccc308632609a0fcff7b55f605d294 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 19 Aug 2026 14:18:51 +0200 Subject: [PATCH 2/7] swarm-authelia: give each hive client an audience and JWT access tokens A swarm service that has to tell hives apart needs the token itself to say which hive presented it. Two client-registration fields were missing for that, and both are properties of the client rather than of any one consumer: - `audience` registers which `aud` values a client may request. It is a boundary, not a label: asking for one that is not registered is refused with `invalid_target`, so a hive cannot mint a token for another hive's slot. - `accessTokenSignedResponseAlg` switches the access token from authelia's default opaque handle to an RFC 9068 JWT, which is what lets a resource server verify a token against `/jwks.json` instead of asking authelia about every request. Each hive's audience is its own client id rather than a new per-hive string. The identity is the directory's, not the consumer's, and the id is already published as `hiveClientPrefix` precisely so a second consumer does not carry its own copy of the naming rule. Both fields default to authelia's own behaviour, so no existing client changes. --- nix/host-modules/swarm-authelia.nix | 81 +++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index c25cf13a..6a419cf4 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -102,11 +102,26 @@ let # and a field later added to the submodule then existed on the # declared entries and not on these, which is an eval error reachable # only once hive identities are on. + # + # `audience` is the hive's own client id rather than a second per-hive + # string invented here. A swarm service that has to tell hives apart + # needs one name per hive that both sides already agree on, and the + # client id is that name — published as `hiveClientPrefix` for exactly + # this reason. Minting a parallel naming scheme would be a second thing + # to keep in step, and the one that drifts is the one nobody tests. + # + # `RS256` because a resource server that cannot call this provider back + # is a real case here: the swarm's telemetry collector verifies tokens + # offline against `/jwks.json`, and an opaque token gives it nothing to + # verify. The queue's auth-callout responder introspects instead, which + # is a different question asked of the same token. hiveClients = lib.mapAttrsToList (name: _: { id = "${cfg.hiveClientPrefix}${name}"; description = "HyperHive hive ${name}"; kind = "machine"; redirectUris = [ ]; + audience = [ "${cfg.hiveClientPrefix}${name}" ]; + accessTokenSignedResponseAlg = "RS256"; }) hyperhiveCfg.swarm.hives; # `swarm-authelia-bridge`'s own identity — distinct from @@ -188,6 +203,15 @@ let + lib.optionalString (c.tokenEndpointAuthMethod != null) '' printf -- ' token_endpoint_auth_method: %s\n' ${lib.escapeShellArg c.tokenEndpointAuthMethod} '' + # Flow-style YAML, matching `scopes` below. The values are client ids + # and hive names, which `Ident` already constrains to `[a-z0-9-]` — no + # character in that set needs quoting in a YAML flow sequence. + + lib.optionalString (c.audience != [ ]) '' + printf -- ' audience: [%s]\n' ${lib.escapeShellArg (lib.concatStringsSep ", " c.audience)} + '' + + lib.optionalString (c.accessTokenSignedResponseAlg != null) '' + printf -- ' access_token_signed_response_alg: %s\n' ${lib.escapeShellArg c.accessTokenSignedResponseAlg} + '' + ( if c.kind == "machine" then '' @@ -493,6 +517,63 @@ in the secret is at fault. ''; }; + + audience = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "hive-alpha" ]; + description = '' + Audiences (`aud`) this client is permitted to request a + token for. Empty means it asks for none, which is the + right answer for a client whose resource server does not + distinguish callers. + + ⚠️ Registering an audience only *permits* it — the value + lands in a token when the client **asks** for it at the + token endpoint, and a client that does not send + `audience=` receives a token with `aud: []` however + complete this list looks. Measured against authelia + 4.39.20: the config reads exactly right and the resource + server rejects every token, because a config that grants + and a request that claims are two separate acts. + + Requesting an audience that is *not* listed here is + refused with `invalid_target`, which is what makes this + usable as a boundary rather than a label: a client cannot + mint a token for a resource slot that is not its own. + ''; + }; + + accessTokenSignedResponseAlg = lib.mkOption { + type = lib.types.nullOr ( + lib.types.enum [ + "none" + "RS256" + ] + ); + default = null; + example = "RS256"; + description = '' + Signing algorithm for this client's **access** tokens. + `null` leaves authelia on its default, which issues an + opaque token (`authelia_at_…`) — a database handle that + carries no claims and means nothing to anyone but this + provider. + + Set `RS256` when the resource server verifies the token + *itself* rather than asking this provider about it: that + yields an RFC 9068 JWT (`at+jwt`) carrying `aud`, `iss` + and `client_id`, verifiable against `/jwks.json` with no + round trip. + + ⚠️ This is what makes a token readable by an + OIDC-verifying consumer at all. A resource server given + an opaque token is not *misconfigured* — it is + structurally unable to verify it, and says so in terms + that point at the verifier rather than at the token's + format. + ''; + }; }; } ); From 08faa0970e6ed3f11b2dbdb77f988a4b23570a65 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 19 Aug 2026 14:30:23 +0200 Subject: [PATCH 3/7] swarm-otel: authenticate ingest per hive, and stamp the hive from the receiver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The swarm collector accepted OTLP from anyone who could reach it, and took the `hive` resource attribute from the payload. So any writer on the swarm network could attribute metrics to any hive, and nothing downstream could tell. The label now comes from which receiver accepted the sample: one receiver per hive, each behind an `oidc` extension verifying a token minted for that hive's audience, each feeding a pipeline whose `resource` processor upserts a constant. A sender cannot influence it, because the only input is which authenticated port the bytes arrived on. That multiplicity is forced rather than preferred. A processor cannot read the token's claims — `from_context` reads request metadata, and asking it for an auth claim yields nothing, silently, with a healthy startup — and one receiver holding several credentials never reveals which one matched. The per-hive ports are internal: a hive reaches its receiver as a path under this collector's existing gateway name, so nginx (rendered from this same evaluation) is the only thing that names a port. Fronting each hive with its own vhost would need a certificate, a DNS name and a gateway entry per hive to express routing the gateway already does. Turning this on removes the unauthenticated receiver. While an open port still accepts samples the per-hive receivers are decoration, so this is the switch itself rather than a hardening layer beside it; a swarm that wants the open receiver says so. `hive-ca-trust.nix` grows `bundlePathFor`, because a consumer taking its own CA argument has to name the bundle rather than just have `SSL_CERT_FILE` exported at it. --- nix/host-modules/lib/hive-ca-trust.nix | 20 +- nix/host-modules/swarm-otel.nix | 305 +++++++++++++++++++++++-- 2 files changed, 302 insertions(+), 23 deletions(-) diff --git a/nix/host-modules/lib/hive-ca-trust.nix b/nix/host-modules/lib/hive-ca-trust.nix index 509730c3..0ad563ea 100644 --- a/nix/host-modules/lib/hive-ca-trust.nix +++ b/nix/host-modules/lib/hive-ca-trust.nix @@ -39,10 +39,26 @@ let # the bundle next to the CA and explains the split. caHostPath = "${tlsCfg.stateDir}/trust-bundle.pem"; caContainerPath = "/run/hive-ca/trust-bundle.pem"; + + # One definition, used by `trustBundle` to WRITE the bundle and published + # below so a caller can NAME it. Two copies of this path would be two + # things to keep in step, and the one that drifts is the reader. + bundleDirFor = name: "/run/${name}-ca"; + bundlePathFor = name: "${bundleDirFor name}/trust-bundle.pem"; in { inherit useSelfSigned caContainerPath; + # Where `trustBundle` below puts the assembled bundle, for the callers + # that must NAME it rather than just have it exported. `SSL_CERT_FILE` is + # set for you and needs no path here; a consumer that takes its own CA + # argument (an OIDC verifier's `issuer_ca_path`, a client's `--cacert`) + # does, and the alternative is copying `/run/-ca/…` to the call + # site. That copy breaks silently: the bundle keeps being written, the + # consumer keeps reading a path that no longer exists, and the failure + # surfaces as a TLS error naming the peer rather than the file. + inherit bundlePathFor; + # Fold into the container's `bindMounts` via `//`. Binds ONLY the public # CA cert (never the `hive-tls` state dir — it holds the CA + leaf private # keys), read-only. Empty when not self-signed, so the whole trust path @@ -106,8 +122,8 @@ in enable ? true, }: let - dir = "/run/${name}-ca"; - bundlePath = "${dir}/trust-bundle.pem"; + dir = bundleDirFor name; + bundlePath = bundlePathFor name; unit = "${name}-ca-bundle"; source = if hostUnit then caHostPath else caContainerPath; in diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 0eacc0de..7b39308c 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -40,6 +40,50 @@ let # the required-domain assertion in hive-network.nix should be what an # operator sees, not a coercion error from here. domainBase = if swarmDomain == null then "invalid" else swarmDomain; + + autheliaCfg = hyperhiveCfg.swarm.authelia; + + # Whether ingest is authenticated per hive. Derived rather than declared + # so the common case needs no attribute, but an option (below) because + # this decides whether the unauthenticated receiver exists at all. + hiveAuth = cfg.requireHiveIdentity; + + # `attrNames` is sorted, so this is a function of the hive SET and not of + # the order anyone wrote it in. + # + # These ports are internal and appear in no URL: a hive addresses its own + # receiver as a PATH on this collector's single gateway name, and nginx — + # rendered from this same evaluation — is the only thing that ever names + # the port. That is what makes deriving them safe here and unsafe in the + # obvious other place: were a hive told a port, inserting a hive would + # renumber the ones after it and silently move a port a running hive was + # already sending to. + hivePorts = lib.listToAttrs ( + lib.imap0 (i: h: lib.nameValuePair h (cfg.hivePortBase + i)) ( + lib.attrNames hyperhiveCfg.swarm.hives + ) + ); + + # The swarm's authelia is reached by its gateway name, whose leaf is + # issued by the swarm services sub-CA — so this container needs the same + # runtime CA trust every other consumer of a swarm-service name needs. + # The CA is generated at runtime and cannot be baked into a derivation, + # which is why it arrives as a bind mount rather than + # `security.pki.certificateFiles`. + caTrust = import ./lib/hive-ca-trust.nix { + inherit lib; + tlsCfg = hyperhiveCfg.tls; + inherit gatewayCfg; + }; + caBundle = caTrust.bundlePathFor cfg.machine; + + # One list, read by every pipeline: the per-hive pipelines fan out to + # exactly the same destinations as the single pipeline they replace. + # Written once because "which exporters" is a property of this tier, not + # of which hive a sample came from. + exporterNames = + lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") + ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; in { options.services.hyperhive.swarm.otel = { @@ -107,6 +151,53 @@ in ''; }; + requireHiveIdentity = lib.mkOption { + type = lib.types.bool; + default = autheliaCfg.enable && autheliaCfg.url != null; + defaultText = lib.literalExpression "swarm.authelia.enable && swarm.authelia.url != null"; + description = '' + Authenticate ingest per hive: each hive gets its own receiver, + verifying an OIDC token minted for that hive's audience, and the + `hive` label is stamped from **which receiver accepted the + sample** rather than from anything the sender wrote. + + ⚠️ Turning this on **removes the unauthenticated receiver**. That + is the point rather than a side effect: while an unauthenticated + port still accepts samples, any writer that can reach this + collector can still attribute metrics to any hive, and the + per-hive receivers are decoration. + + Defaults to whether this swarm has an authelia to mint against. + Set it false to keep the open receiver on a swarm where every + writer is already trusted — an explicit choice, which is what it + should be. + ''; + }; + + hivePortBase = lib.mkOption { + type = lib.types.port; + default = 4330; + description = '' + First port of the per-hive receiver range; each hive in + {option}`services.hyperhive.swarm.hives` takes the next one, in + sorted-name order. + + ⚠️ Internal. No client is told a port — a hive reaches its own + receiver as `https://''${domain}/`, and the gateway routes + the path. So inserting a hive renumbering the ones after it is + harmless here: nginx is rendered from this same evaluation and + moves with it. + + The range still matters because every swarm container shares the + host's network namespace, so a derived port can land on one + another service already holds — with no bind error and nothing in + any log. The assertions below check this range against every port + this module and the hive tier declare — which is as far as a + module can see, since a port another module picks is not + knowable from here without reading its config. + ''; + }; + domain = lib.mkOption { type = lib.types.str; default = "otel.${domainBase}"; @@ -143,11 +234,46 @@ in services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; - locations."/" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}"; - }; + locations = + # One name for the whole collector, and the hive is a path under + # it. The alternative — a vhost per hive — needs a certificate, + # a DNS name and a `localNames` entry per hive to express the + # same routing the gateway already does for free. + # + # ⚠️ The trailing slash on both sides is load-bearing: it is what + # strips `/` before the request reaches the receiver, which + # serves `/v1/metrics` and knows nothing about hives. Without it + # the receiver sees `//v1/metrics` and answers 404 to a + # request that authenticated perfectly. + lib.optionalAttrs hiveAuth ( + lib.mapAttrs' ( + h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } + ) hivePorts + ) + // { + "/" = + if hiveAuth then + # Not a proxy to a receiver that no longer exists. A closed + # door answering 404 is the honest description of this + # collector once ingest is per-hive: there is no + # swarm-wide inbox any more. + { return = "404"; } + else + { proxyPass = "http://127.0.0.1:${toString cfg.port}"; }; + }; }; + # Turning on the identities this tier authenticates against, which the + # option exists to allow: its own description names this module as the + # second consumer, so the queue is not a prerequisite for authenticated + # telemetry. + services.hyperhive.swarm.authelia.oidc.hiveIdentities = lib.mkIf hiveAuth true; + + # The CA bind source is written at runtime by a host unit, so the + # container has to start after it — otherwise nspawn sets up a mount + # over a file that does not exist yet. + systemd.services."container@${cfg.machine}" = lib.mkIf hiveAuth caTrust.containerOrdering; + assertions = [ { # The tier exists to hold the upstream credential and to write the @@ -164,6 +290,49 @@ in metrics store. ''; } + { + # Without a roster there are no receivers at all, so this + # collector would listen on nothing while looking configured. + assertion = !hiveAuth || hyperhiveCfg.swarm.hives != { }; + message = '' + services.hyperhive.swarm.otel.requireHiveIdentity is true but + services.hyperhive.swarm.hives is empty: ingest is authenticated + per hive, so an empty roster means this collector accepts + nothing from anyone. + + List the swarm's hives, or set requireHiveIdentity = false to + keep an unauthenticated receiver. + ''; + } + { + # A port collision between two listeners on one host is a runtime + # coin toss with nothing in any log — the failure this whole + # comment budget exists to prevent. Checked against every port + # reachable from here; a port some other module picks is not. + assertion = + let + derived = lib.attrValues hivePorts; + others = [ + cfg.port + cfg.telemetryPort + otelCfg.collector.port + ] + ++ lib.optional vmCfg.enable vmCfg.port; + all = derived ++ others; + in + !hiveAuth || lib.length (lib.unique all) == lib.length all; + message = '' + services.hyperhive.swarm.otel: the per-hive receiver range + starting at hivePortBase (${toString cfg.hivePortBase}, one port + per hive in services.hyperhive.swarm.hives) overlaps another + port on this host. + + Every swarm container shares the host's network namespace, so + two listeners claiming one port is not a build failure — it is + whichever process started first, silently. Move hivePortBase to + a free range. + ''; + } ]; containers.${cfg.machine} = { @@ -179,12 +348,16 @@ in # Read-only, and only when one is configured — binding a path that # does not exist makes nixos-container refuse to start the container, # which is a stall several layers from its cause. - bindMounts = lib.optionalAttrs (otelCfg.headersCredential != null) { - ${otelCfg.headersCredential} = { - hostPath = otelCfg.headersCredential; - isReadOnly = true; - }; - }; + bindMounts = + lib.optionalAttrs (otelCfg.headersCredential != null) { + ${otelCfg.headersCredential} = { + hostPath = otelCfg.headersCredential; + isReadOnly = true; + }; + } + # The public hive CA, read-only — only when something in here + # actually verifies a swarm-service name. + // lib.optionalAttrs hiveAuth caTrust.bindMount; config = { ... }: @@ -200,7 +373,19 @@ in inherit (config.services.hyperhive.network) bridgeIp; dnsConsumers = [ "opentelemetry-collector.service" ]; }) - ]; + ] + # `SSL_CERT_FILE` REPLACES the trust store rather than adding to + # it, so a failed assembly yields an empty pool and every TLS + # call fails while the unit looks healthy. That is why this is + # the shared helper — it carries the `Requires` and the + # non-empty check — and not a local `cat`. + ++ lib.optional hiveAuth ( + caTrust.trustBundle { + inherit pkgs; + name = cfg.machine; + consumers = [ "opentelemetry-collector" ]; + } + ); system.stateVersion = config.system.stateVersion; networking.firewall.enable = false; @@ -219,7 +404,28 @@ in # real sample through both tiers into the store. validateConfigFile = true; settings = { - receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; + # One receiver per hive when ingest is authenticated, and + # that multiplicity is forced rather than chosen. The `hive` + # label has to come from something the sender cannot write, + # and the only such thing here is WHICH RECEIVER accepted + # the sample: a processor cannot read the token's claims + # (`from_context` reads request metadata, and asking it for + # an auth claim yields nothing — silently, with a healthy + # startup), and one receiver holding many credentials never + # reveals which one matched. + receivers = + if hiveAuth then + lib.mapAttrs' ( + h: p: + lib.nameValuePair "otlp/${h}" { + protocols.http = { + endpoint = "127.0.0.1:${toString p}"; + auth.authenticator = "oidc/${h}"; + }; + } + ) hivePorts + else + { otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; }; exporters = lib.optionalAttrs vmCfg.enable { @@ -263,16 +469,73 @@ in } ]; - service.pipelines.metrics = { - receivers = [ "otlp" ]; - # Fan-out, not a choice: with both configured the same - # samples go upstream AND into the swarm's store. The store - # is for looking at this swarm; the upstream is for whoever - # aggregates across swarms, and neither replaces the other. - exporters = - lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") - ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; - }; + # ⚠️ An extension that is configured but not listed here is + # INERT — the collector starts clean and the receiver + # naming it authenticates nothing. Derived from the same + # attrset as the receivers so the two cannot disagree. + service.extensions = lib.optionals hiveAuth (map (h: "oidc/${h}") (lib.attrNames hivePorts)); + + # Fan-out, not a choice: with both configured the same + # samples go upstream AND into the swarm's store. The store + # is for looking at this swarm; the upstream is for whoever + # aggregates across swarms, and neither replaces the other. + # `exporterNames` is shared by every pipeline — where a + # sample goes is a property of this tier, not of the hive + # that sent it. + service.pipelines = + if hiveAuth then + lib.mapAttrs' ( + h: _: + lib.nameValuePair "metrics/${h}" { + receivers = [ "otlp/${h}" ]; + processors = [ "resource/${h}" ]; + exporters = exporterNames; + } + ) hivePorts + else + { + metrics = { + receivers = [ "otlp" ]; + exporters = exporterNames; + }; + }; + } + // lib.optionalAttrs hiveAuth { + extensions = lib.mapAttrs' ( + h: _: + lib.nameValuePair "oidc/${h}" { + issuer_url = autheliaCfg.url; + # The audience this hive's client is registered to + # request, and the reason one hive's token is refused by + # another hive's receiver. Same expression authelia + # registers it under — a second spelling here would deny + # every hive, as a 401 that blames the token. + audience = "${autheliaCfg.hiveClientPrefix}${h}"; + # ⚠️ `issuer_ca_path`. `issuer_ca_file`, `ca_file` and + # `tls.ca_file` are all INVALID KEYS for this extension + # — measured, and the failure is a startup error naming + # the key rather than anything about certificates. + issuer_ca_path = caBundle; + } + ) hivePorts; + + # `upsert`, not `insert`: a sender that stamps its own + # `hive` must be OVERWRITTEN, not deferred to. This + # processor is the whole attribution boundary — the value + # is a constant per receiver, so it says which hive + # authenticated, not which hive claimed to be sending. + processors = lib.mapAttrs' ( + h: _: + lib.nameValuePair "resource/${h}" { + attributes = [ + { + key = "hive"; + value = h; + action = "upsert"; + } + ]; + } + ) hivePorts; }; }; From cb787997bddf773f5181aa1937b1d78cbbeb6949 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 19 Aug 2026 14:53:10 +0200 Subject: [PATCH 4/7] otel: the hive tier presents its own identity to the swarm collector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The receiving half authenticates per hive, so this half has to prove which hive it is. It mints a token against the swarm's authelia with this hive's client and posts to that hive's path on the collector's gateway name. Holding a credential is what decides whether this tier authenticates — `clientSecretFile` non-null — rather than a second switch that could disagree with it. The default is the secret this host's own authelia minted, which is right exactly when the IdP runs here; a hive that is not that host names wherever the file landed, the same manual-copy shape the identities option already documents as unsolved. Two things that a diff will not explain: `endpoint_params.audience` is not redundant with the client's registered audience. Registering only makes an audience permissible; a token minted without asking for one carries `aud: []` and every receiver refuses it, with a config that reads correctly at both ends. `client_secret_file` keeps the secret out of nix altogether — the collector opens the file itself. It is a real key of this extension, checked against the shipped binary with a deliberate typo rejected in the same run, so "accepted" is distinguishable from "ignores everything". The path comes from systemd's `CREDENTIALS_DIRECTORY`, so nothing hardcodes a `/run/credentials` layout. An assertion covers the one deployment where this can go wrong silently: a host running both tiers with ingest authenticated and no credential to present would 401 against a collector on the same machine. --- nix/host-modules/otel.nix | 120 +++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index a3024641..ebeefc9b 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -193,6 +193,40 @@ in ''; }; + clientSecretFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = + if + config.services.hyperhive.swarm.authelia.enable && config.services.hyperhive.hiveName != null + then + "${config.services.hyperhive.swarm.authelia.hostClientSecretDir}/" + + "${config.services.hyperhive.swarm.authelia.hiveClientPrefix}${config.services.hyperhive.hiveName}.secret" + else + null; + defaultText = lib.literalExpression ''"''${swarm.authelia.hostClientSecretDir}/''${swarm.authelia.hiveClientPrefix}''${hiveName}.secret" when this host runs the swarm's IdP, else null''; + example = "/var/lib/secrets/hive-telemetry.secret"; + description = '' + Absolute path to this hive's OAuth2 client secret, used to + authenticate to the swarm's collector as this hive. + + **Whether this is set is what decides whether this tier + authenticates at all.** A credential is the only thing that makes + authenticated egress possible, so its presence is the condition + rather than a second switch that could disagree with it. + + Defaults to the secret this host's own authelia minted, which is + correct exactly when the swarm's IdP runs here. On a hive that is + not that host, the file has to arrive some other way and this + option names wherever it landed — the same manual-copy shape + `services.hyperhive.swarm.authelia.oidc.hiveIdentities` documents, + where delivering a secret to a hive that is not this host is + deliberately not solved. + + Read by `LoadCredential`, so it is never evaluated by nix, never + copied into the store and never passed in argv. + ''; + }; + metricIntervalMs = lib.mkOption { type = lib.types.nullOr lib.types.ints.positive; default = null; @@ -214,8 +248,28 @@ in config = lib.mkIf config.services.hyperhive.otel.enable ( let otel = config.services.hyperhive.otel; + autheliaCfg = config.services.hyperhive.swarm.authelia; + swarmOtelCfg = config.services.hyperhive.swarm.otel; + hiveName = config.services.hyperhive.hiveName; listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; swarmName = "otlphttp/swarm"; + authName = "oauth2client/swarm"; + + # Holding a credential IS the condition — see `clientSecretFile`. + senderAuth = otel.clientSecretFile != null && hiveName != null; + + # This hive's client id, and also the audience it must ASK for. Both + # are `hiveClientPrefix` + the hive's name because that is the one + # name the swarm already agrees on; the receiver one tier up derives + # the same string. + hiveClient = "${autheliaCfg.hiveClientPrefix}${toString hiveName}"; + + # systemd exports `CREDENTIALS_DIRECTORY` to any unit with + # `LoadCredential`, and the collector expands `${env:…}` at load. So + # the secret reaches the process as a PATH resolved at runtime — nix + # renders neither the value nor the directory, and nothing has to + # hardcode `/run/credentials/`. + credName = "swarm-client.secret"; in { # Reachable from agent containers and nowhere else: this opens @@ -279,15 +333,77 @@ in # change. `https://` because that name resolves through the # gateway even on a co-located host — see `caTrust` above for # the trust half that makes this verify. - endpoint = "https://${config.services.hyperhive.swarm.otel.domain}"; - }; + # The hive's own path under the collector's single name. The + # swarm tier gives each hive its own authenticated receiver and + # routes to it by this prefix, so the path is not decoration — + # it selects WHICH receiver, and therefore which hive the + # samples get labelled as. + endpoint = "https://${swarmOtelCfg.domain}" + lib.optionalString senderAuth "/${toString hiveName}"; + } + // lib.optionalAttrs senderAuth { auth.authenticator = authName; }; + + # ⚠️ An extension not listed here is INERT: the collector starts + # clean and the exporter naming it sends nothing authenticated. + service.extensions = lib.optional senderAuth authName; service.pipelines.metrics = { receivers = [ "otlp" ]; exporters = [ swarmName ]; }; + } + // lib.optionalAttrs senderAuth { + extensions.${authName} = { + client_id = hiveClient; + # A real key, measured against this collector version rather + # than assumed — with a deliberate typo rejected in the same + # run, so "accepted" is distinguishable from "ignores + # everything". Keeps the secret out of nix entirely: the + # collector opens the file itself. + client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credName}"; + token_url = "${toString autheliaCfg.url}/api/oidc/token"; + # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. + # Registering it on the client only makes it permissible; a + # token minted without asking carries `aud: []` and every + # receiver refuses it — with a config that reads perfectly at + # both ends. Measured against authelia 4.39.20. + endpoint_params.audience = hiveClient; + }; }; }; + + # `LoadCredential` and not a copy-oneshot: this collector is a HOST + # unit, so there is no container boundary to cross and therefore no + # reason for a second on-disk copy of the secret. systemd hands it to + # the process in a private tmpfs and exports the directory, which is + # what the config above names. + systemd.services.opentelemetry-collector.serviceConfig = lib.optionalAttrs senderAuth { + LoadCredential = [ "${credName}:${otel.clientSecretFile}" ]; + }; + + assertions = [ + { + # Only checkable on a host that runs BOTH tiers — which is the + # deployment where it can actually go wrong silently. A remote + # hive cannot see the swarm tier's config at all, so its operator + # sets the path explicitly and this says nothing. + assertion = !(swarmOtelCfg.enable && swarmOtelCfg.requireHiveIdentity) || senderAuth; + message = '' + This host runs the swarm's telemetry collector with + services.hyperhive.swarm.otel.requireHiveIdentity = true, so + ingest is authenticated per hive — but this hive's own + collector has no credential to present: + + services.hyperhive.otel.clientSecretFile = ${ + if otel.clientSecretFile == null then "null" else otel.clientSecretFile + } + services.hyperhive.hiveName = ${if hiveName == null then "null" else hiveName} + + Its samples would be refused with a 401 by the collector + running beside it. Set both, or set requireHiveIdentity = + false to accept unauthenticated ingest. + ''; + } + ]; } ); } From 9dc60061e7ce234c60b263a21b5949c606a93e6c Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 19 Aug 2026 15:06:15 +0200 Subject: [PATCH 5/7] docs: authenticated telemetry ingest, and where the hive's secret comes from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three readers, three different questions: `observability.md` gets the operator's version — what to set (nothing, on an all-local swarm), what to set elsewhere, what a misconfiguration looks like (401s and a missing hive), and what turning it off actually costs. `swarm/secrets.md` extends the existing client-secret row rather than adding one, because this is a new *reader* of a secret already described. It notes the one thing that makes this reader unusual: a host unit reads the file in place, so unlike every other consumer there is no delivery step to get wrong. `setup.md` gets one bullet under security notes, since a first-run hive gets the property without doing anything and should know it has it. --- docs/observability.md | 32 ++++++++++++++++++++++++++++++++ docs/setup.md | 5 +++++ docs/swarm/secrets.md | 9 ++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/observability.md b/docs/observability.md index 4b23a4bb..20a51a47 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -186,6 +186,38 @@ redefines it — the agent-facing value is *derived* keeps working unchanged. The bridge port is contributed to `exposeHostPorts` automatically; there is nothing to open by hand. +### Authenticated ingest + +The swarm tier gives **each hive its own receiver**, and stamps the `hive` label +from whichever receiver accepted a sample. A hive therefore cannot report +metrics as another hive, and cannot relabel its own by editing what it sends — +the label is not taken from the payload at all. + +**On an all-local swarm there is nothing to set.** Each hive already has an +identity, and its collector reads the secret that host's own authelia minted. + +**On a hive that does not host the swarm's services**, the secret has to arrive +somehow — copy it across and name it: + +```nix +services.hyperhive.otel.clientSecretFile = "/run/secrets/hive-telemetry.secret"; +``` + +Getting that wrong shows up as the hive's collector logging 401s from the swarm +tier and no metrics appearing for that hive. When a single host runs both tiers +the build catches it instead, because it can see both sides. + +To accept unauthenticated ingest — every writer on the swarm network already +trusted, or a swarm with no authelia: + +```nix +services.hyperhive.swarm.otel.requireHiveIdentity = false; +``` + +⚠️ That reopens the original hole rather than merely skipping a check: while an +unauthenticated port is listening, anything that can reach the collector can +attribute metrics to any hive. + ### `services.hyperhive.otel.collector.port` — port, default `4318` The OTLP/HTTP port the hive tier listens on, bound to the bridge IP only. The diff --git a/docs/setup.md b/docs/setup.md index d5ec447c..cdebe400 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -156,6 +156,11 @@ See [`tools/hivectl.md`](tools/hivectl.md) for every `hivectl` verb. - All config changes (forge PRs on `agent-configs/`) go through operator approval — agents can't unilaterally rebuild containers, by design. See [`boundary.md`](boundary.md) and [`security.md`](security.md). +- **Telemetry ingest is authenticated per hive**, and the `hive` label comes + from which hive authenticated rather than from the payload — so no hive can + report metrics as another. A first-run all-local hive gets this with nothing + to configure; joining a swarm you don't host needs one secret copied across. + See [`observability.md`](observability.md#authenticated-ingest). Once the hive is running, ruth records anything it needs to remember across restarts in `/agents/ruth/state/notes.md`. diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 23547b7c..4d324944 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -45,7 +45,7 @@ Every row below is read against one of these. | authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | | authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | | authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | -| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in the service's `sso.clientSecretFile` | +| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service, `otel.clientSecretFile` for the hive's telemetry collector | | OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | | authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | | wireguard private key | **the operator** — `wg genkey` | whatever `swarm.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | @@ -56,6 +56,13 @@ because nothing outside that container ever reads them. **That is the test worth applying to any secret added here** — and the client secret's plaintext half is the one row that fails it, which is the entire reason a delivery step exists. +One reader needs no delivery step: the **hive's telemetry collector**, which +authenticates to the swarm's collector as its own hive. It is a host unit rather +than a container, so on an all-local swarm it reads authelia's file where it +lies (through `LoadCredential`) and no second copy is made. On any other +topology it is an ordinary "operator provides the file" case — see +`services.hyperhive.otel.clientSecretFile`. + ### Minting the queue's callout nkeys `nats.autoGenerateCallout` mints both keypairs on the host before the queue From 7da791515038cedbb9b6305f1963a241417072f9 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 19 Aug 2026 15:11:37 +0200 Subject: [PATCH 6/7] otel: refuse a half-configured escape hatch instead of 404ing silently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turning ingest auth off without clearing a hive's credential leaves that hive's collector authenticating and addressing its own path, while an unauthenticated swarm tier serves one catch-all and forwards the URI unchanged. The receiver is asked for a path it does not serve, so telemetry stops with 404s and retries — no 401, no assertion, nothing in any log naming auth. Only reachable by overriding one side without the other, since both defaults derive from the same flag. That is what makes it worth a build error rather than a caveat: an operator who flips the documented escape hatch has no reason to suspect the sending half. Found in review by argus. --- docs/observability.md | 6 ++++++ nix/host-modules/otel.nix | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/docs/observability.md b/docs/observability.md index 20a51a47..7384ba0c 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -212,8 +212,14 @@ trusted, or a swarm with no authelia: ```nix services.hyperhive.swarm.otel.requireHiveIdentity = false; +services.hyperhive.otel.clientSecretFile = null; # on each hive that had one ``` +Both halves, because a collector that authenticates also addresses its hive's +own path, and an unauthenticated swarm tier serves no per-hive paths. Set only +the first and that hive's samples 404 instead of arriving. On a host running +both tiers the build says so; on a split host it is yours to keep in step. + ⚠️ That reopens the original hole rather than merely skipping a check: while an unauthenticated port is listening, anything that can reach the collector can attribute metrics to any hive. diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index ebeefc9b..0c1ef648 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -403,6 +403,41 @@ in false to accept unauthenticated ingest. ''; } + { + # The mirror of the assertion above, and the reason it exists is + # that the failure is SILENT rather than loud. With ingest + # unauthenticated the swarm tier serves one catch-all location + # and passes the URI through unchanged — but this tier still + # appends `/` whenever it holds a credential, so the + # receiver is asked for `//v1/metrics`, a path it does not + # serve. The result is 404s and retries: no 401, no assertion, no + # log anywhere saying telemetry stopped. + # + # Only reachable by overriding one side without the other, since + # both defaults derive from `swarm.authelia.enable` and move + # together. That is exactly why it is worth a build error — an + # operator who flips the escape hatch has no reason to suspect + # the sending half. + assertion = !(swarmOtelCfg.enable && !swarmOtelCfg.requireHiveIdentity && senderAuth); + message = '' + This host accepts unauthenticated telemetry ingest + (services.hyperhive.swarm.otel.requireHiveIdentity = false), + but its own collector still holds a credential: + + services.hyperhive.otel.clientSecretFile = ${ + if otel.clientSecretFile == null then "null" else otel.clientSecretFile + } + + A collector that authenticates also addresses its hive's own + path, and an unauthenticated swarm tier serves no per-hive + paths — so this hive's samples would 404 rather than be + refused, which no log names as an auth problem. + + Set services.hyperhive.otel.clientSecretFile = null to send + unauthenticated too, or drop the requireHiveIdentity + override. + ''; + } ]; } ); From 9bd2b9e9e66dd611535d0a8b1fd0d83012cbd83e Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 19 Aug 2026 15:22:29 +0200 Subject: [PATCH 7/7] =?UTF-8?q?otel:=20a=20hive=20always=20authenticates?= =?UTF-8?q?=20=E2=80=94=20drop=20the=20unauthenticated=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mara, reviewing this PR: "hives always require an identity, swarm controller and auth is not optional." So `requireHiveIdentity` is gone rather than defaulted, and with it every branch that had to describe an unauthenticated collector. The swarm tier now serves per-hive receivers only, and `/` answers 404 because there is no swarm-wide inbox to route to. A hive with no credential is a build error, not a quieter mode. `hivePortBase` goes too: with per-hive receivers unconditional, `port` IS the base of the range. That keeps one documented knob instead of adding a second, and its advice ("move it if something else claims that range") still holds. Two assertions replace the toggle — an empty hive roster, and a null `authelia.url`. The second matters because a guessed issuer URL evaluates cleanly, deploys cleanly, and then refuses every hive at runtime. ⚠️ `cfg.port` is deliberately no longer compared against the derived range in the collision assertion: it is now the range's first element, so listing it would make that assertion fire on every config. This also retires the asymmetry guard added earlier in review — the state it protected against (auth off on one side, credential still set on the other) is no longer representable. --- docs/observability.md | 25 +--- nix/host-modules/otel.nix | 66 +++------- nix/host-modules/swarm-otel.nix | 208 +++++++++++++------------------- 3 files changed, 107 insertions(+), 192 deletions(-) diff --git a/docs/observability.md b/docs/observability.md index 7384ba0c..a1f16a7c 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -203,26 +203,13 @@ somehow — copy it across and name it: services.hyperhive.otel.clientSecretFile = "/run/secrets/hive-telemetry.secret"; ``` -Getting that wrong shows up as the hive's collector logging 401s from the swarm -tier and no metrics appearing for that hive. When a single host runs both tiers -the build catches it instead, because it can see both sides. +**There is no unauthenticated mode.** A hive always presents an identity, so a +missing credential is a build error rather than a quieter fallback — the +collector has no anonymous route to accept samples on, and every path it serves +belongs to exactly one hive. -To accept unauthenticated ingest — every writer on the swarm network already -trusted, or a swarm with no authelia: - -```nix -services.hyperhive.swarm.otel.requireHiveIdentity = false; -services.hyperhive.otel.clientSecretFile = null; # on each hive that had one -``` - -Both halves, because a collector that authenticates also addresses its hive's -own path, and an unauthenticated swarm tier serves no per-hive paths. Set only -the first and that hive's samples 404 instead of arriving. On a host running -both tiers the build says so; on a split host it is yours to keep in step. - -⚠️ That reopens the original hole rather than merely skipping a check: while an -unauthenticated port is listening, anything that can reach the collector can -attribute metrics to any hive. +Getting the secret wrong shows up as the hive's collector logging 401s from the +swarm tier and no metrics appearing for that hive. ### `services.hyperhive.otel.collector.port` — port, default `4318` diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 0c1ef648..2800717c 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -255,7 +255,11 @@ in swarmName = "otlphttp/swarm"; authName = "oauth2client/swarm"; - # Holding a credential IS the condition — see `clientSecretFile`. + # A hive always authenticates to the swarm's collector as itself, so + # this is not a mode — it is a precondition, and the assertion below + # is what enforces it. Kept as a name because several places have to + # read "do we have what it takes", and an eval error from a null path + # names this file rather than the option an operator has to set. senderAuth = otel.clientSecretFile != null && hiveName != null; # This hive's client id, and also the audience it must ASK for. Both @@ -382,60 +386,28 @@ in assertions = [ { - # Only checkable on a host that runs BOTH tiers — which is the - # deployment where it can actually go wrong silently. A remote - # hive cannot see the swarm tier's config at all, so its operator - # sets the path explicitly and this says nothing. - assertion = !(swarmOtelCfg.enable && swarmOtelCfg.requireHiveIdentity) || senderAuth; + # A hive authenticates to the swarm's collector as itself — there + # is no unauthenticated path to fall back to, so a missing + # credential is a broken deployment rather than a quieter mode. + # Caught here because the alternative is a collector that starts + # cleanly, retries forever, and reports nothing to anyone. + assertion = senderAuth; message = '' - This host runs the swarm's telemetry collector with - services.hyperhive.swarm.otel.requireHiveIdentity = true, so - ingest is authenticated per hive — but this hive's own - collector has no credential to present: + services.hyperhive.otel.enable is true but this hive has no + identity to present to the swarm's collector: services.hyperhive.otel.clientSecretFile = ${ if otel.clientSecretFile == null then "null" else otel.clientSecretFile } services.hyperhive.hiveName = ${if hiveName == null then "null" else hiveName} - Its samples would be refused with a 401 by the collector - running beside it. Set both, or set requireHiveIdentity = - false to accept unauthenticated ingest. - ''; - } - { - # The mirror of the assertion above, and the reason it exists is - # that the failure is SILENT rather than loud. With ingest - # unauthenticated the swarm tier serves one catch-all location - # and passes the URI through unchanged — but this tier still - # appends `/` whenever it holds a credential, so the - # receiver is asked for `//v1/metrics`, a path it does not - # serve. The result is 404s and retries: no 401, no assertion, no - # log anywhere saying telemetry stopped. - # - # Only reachable by overriding one side without the other, since - # both defaults derive from `swarm.authelia.enable` and move - # together. That is exactly why it is worth a build error — an - # operator who flips the escape hatch has no reason to suspect - # the sending half. - assertion = !(swarmOtelCfg.enable && !swarmOtelCfg.requireHiveIdentity && senderAuth); - message = '' - This host accepts unauthenticated telemetry ingest - (services.hyperhive.swarm.otel.requireHiveIdentity = false), - but its own collector still holds a credential: + Every hive authenticates as itself — that is what makes the + `hive` label on its metrics mean anything — so both are + required. - services.hyperhive.otel.clientSecretFile = ${ - if otel.clientSecretFile == null then "null" else otel.clientSecretFile - } - - A collector that authenticates also addresses its hive's own - path, and an unauthenticated swarm tier serves no per-hive - paths — so this hive's samples would 404 rather than be - refused, which no log names as an auth problem. - - Set services.hyperhive.otel.clientSecretFile = null to send - unauthenticated too, or drop the requireHiveIdentity - override. + On a host that runs the swarm's identity provider, the default + already points at the secret authelia minted. On a hive that + does not, copy that hive's secret across and name it here. ''; } ]; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 7b39308c..525bac27 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -43,11 +43,6 @@ let autheliaCfg = hyperhiveCfg.swarm.authelia; - # Whether ingest is authenticated per hive. Derived rather than declared - # so the common case needs no attribute, but an option (below) because - # this decides whether the unauthenticated receiver exists at all. - hiveAuth = cfg.requireHiveIdentity; - # `attrNames` is sorted, so this is a function of the hive SET and not of # the order anyone wrote it in. # @@ -59,9 +54,7 @@ let # renumber the ones after it and silently move a port a running hive was # already sending to. hivePorts = lib.listToAttrs ( - lib.imap0 (i: h: lib.nameValuePair h (cfg.hivePortBase + i)) ( - lib.attrNames hyperhiveCfg.swarm.hives - ) + lib.imap0 (i: h: lib.nameValuePair h (cfg.port + i)) (lib.attrNames hyperhiveCfg.swarm.hives) ); # The swarm's authelia is reached by its gateway name, whose leaf is @@ -119,7 +112,17 @@ in type = lib.types.port; default = 4319; description = '' - Port this collector's OTLP/HTTP receiver listens on. + First port of this collector's receiver range. Every hive in + {option}`services.hyperhive.swarm.hives` gets its **own** + authenticated receiver — that is what makes the `hive` label + unforgeable — so the range is one port per hive, starting here, in + sorted-name order. + + ⚠️ Internal. No client is ever told a port: a hive reaches its own + receiver as `https://''${domain}/`, and the gateway routes on + that path. So adding a hive, which renumbers the ones after it, is + harmless — nginx is rendered from this same evaluation and moves + with it. ⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the hive tier already uses it (`services.hyperhive.otel.collector.port`) @@ -127,7 +130,9 @@ in listeners claiming one port on one host is not a build failure — it is a runtime coin toss over which one gets it, with nothing in any log saying so. The same collision cost a release when grafana - and the forge both defaulted to 3000. + and the forge both defaulted to 3000. The assertions below check + the whole derived range against every port this module and the hive + tier declare, which is as far as a module can see. ''; }; @@ -151,53 +156,6 @@ in ''; }; - requireHiveIdentity = lib.mkOption { - type = lib.types.bool; - default = autheliaCfg.enable && autheliaCfg.url != null; - defaultText = lib.literalExpression "swarm.authelia.enable && swarm.authelia.url != null"; - description = '' - Authenticate ingest per hive: each hive gets its own receiver, - verifying an OIDC token minted for that hive's audience, and the - `hive` label is stamped from **which receiver accepted the - sample** rather than from anything the sender wrote. - - ⚠️ Turning this on **removes the unauthenticated receiver**. That - is the point rather than a side effect: while an unauthenticated - port still accepts samples, any writer that can reach this - collector can still attribute metrics to any hive, and the - per-hive receivers are decoration. - - Defaults to whether this swarm has an authelia to mint against. - Set it false to keep the open receiver on a swarm where every - writer is already trusted — an explicit choice, which is what it - should be. - ''; - }; - - hivePortBase = lib.mkOption { - type = lib.types.port; - default = 4330; - description = '' - First port of the per-hive receiver range; each hive in - {option}`services.hyperhive.swarm.hives` takes the next one, in - sorted-name order. - - ⚠️ Internal. No client is told a port — a hive reaches its own - receiver as `https://''${domain}/`, and the gateway routes - the path. So inserting a hive renumbering the ones after it is - harmless here: nginx is rendered from this same evaluation and - moves with it. - - The range still matters because every swarm container shares the - host's network namespace, so a derived port can land on one - another service already holds — with no bind error and nothing in - any log. The assertions below check this range against every port - this module and the hive tier declare — which is as far as a - module can see, since a port another module picks is not - knowable from here without reading its config. - ''; - }; - domain = lib.mkOption { type = lib.types.str; default = "otel.${domainBase}"; @@ -245,21 +203,14 @@ in # serves `/v1/metrics` and knows nothing about hives. Without it # the receiver sees `//v1/metrics` and answers 404 to a # request that authenticated perfectly. - lib.optionalAttrs hiveAuth ( - lib.mapAttrs' ( - h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } - ) hivePorts - ) + lib.mapAttrs' ( + h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } + ) hivePorts // { - "/" = - if hiveAuth then - # Not a proxy to a receiver that no longer exists. A closed - # door answering 404 is the honest description of this - # collector once ingest is per-hive: there is no - # swarm-wide inbox any more. - { return = "404"; } - else - { proxyPass = "http://127.0.0.1:${toString cfg.port}"; }; + # There is no swarm-wide inbox, and a closed door is the honest + # description of that. Every route into this collector belongs to + # exactly one hive. + "/".return = "404"; }; }; @@ -267,12 +218,12 @@ in # option exists to allow: its own description names this module as the # second consumer, so the queue is not a prerequisite for authenticated # telemetry. - services.hyperhive.swarm.authelia.oidc.hiveIdentities = lib.mkIf hiveAuth true; + services.hyperhive.swarm.authelia.oidc.hiveIdentities = true; # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. - systemd.services."container@${cfg.machine}" = lib.mkIf hiveAuth caTrust.containerOrdering; + systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; assertions = [ { @@ -293,15 +244,30 @@ in { # Without a roster there are no receivers at all, so this # collector would listen on nothing while looking configured. - assertion = !hiveAuth || hyperhiveCfg.swarm.hives != { }; + assertion = hyperhiveCfg.swarm.hives != { }; message = '' - services.hyperhive.swarm.otel.requireHiveIdentity is true but + services.hyperhive.swarm.otel.enable is true but services.hyperhive.swarm.hives is empty: ingest is authenticated - per hive, so an empty roster means this collector accepts - nothing from anyone. + per hive, so an empty roster means this collector accepts nothing + from anyone. - List the swarm's hives, or set requireHiveIdentity = false to - keep an unauthenticated receiver. + List the swarm's hives. + ''; + } + { + # A hive proves who it is with a token this provider mints, so + # there is no version of this collector that runs without one. + # Stated as an assertion rather than a fallback because a guessed + # issuer URL evaluates cleanly and refuses every hive at runtime. + assertion = autheliaCfg.url != null; + message = '' + services.hyperhive.swarm.otel.enable is true but + services.hyperhive.swarm.authelia.url is null: every hive + authenticates to this collector as itself, and the token comes + from the swarm's identity provider. + + Point authelia.url at the swarm's provider, or enable + services.hyperhive.swarm.authelia on the host that runs it. ''; } { @@ -309,28 +275,31 @@ in # coin toss with nothing in any log — the failure this whole # comment budget exists to prevent. Checked against every port # reachable from here; a port some other module picks is not. + # + # ⚠️ `cfg.port` is deliberately absent from `others`: it is the + # FIRST element of the derived range, so listing it would make this + # assertion fire on every config. assertion = let derived = lib.attrValues hivePorts; others = [ - cfg.port cfg.telemetryPort otelCfg.collector.port ] ++ lib.optional vmCfg.enable vmCfg.port; all = derived ++ others; in - !hiveAuth || lib.length (lib.unique all) == lib.length all; + lib.length (lib.unique all) == lib.length all; message = '' - services.hyperhive.swarm.otel: the per-hive receiver range - starting at hivePortBase (${toString cfg.hivePortBase}, one port - per hive in services.hyperhive.swarm.hives) overlaps another - port on this host. + services.hyperhive.swarm.otel: the receiver range starting at + port (${toString cfg.port}, one port per hive in + services.hyperhive.swarm.hives) overlaps another port on this + host. Every swarm container shares the host's network namespace, so two listeners claiming one port is not a build failure — it is - whichever process started first, silently. Move hivePortBase to - a free range. + whichever process started first, silently. Move + services.hyperhive.swarm.otel.port to a free range. ''; } ]; @@ -357,7 +326,7 @@ in } # The public hive CA, read-only — only when something in here # actually verifies a swarm-service name. - // lib.optionalAttrs hiveAuth caTrust.bindMount; + // caTrust.bindMount; config = { ... }: @@ -379,13 +348,13 @@ in # call fails while the unit looks healthy. That is why this is # the shared helper — it carries the `Requires` and the # non-empty check — and not a local `cat`. - ++ lib.optional hiveAuth ( - caTrust.trustBundle { + ++ [ + (caTrust.trustBundle { inherit pkgs; name = cfg.machine; consumers = [ "opentelemetry-collector" ]; - } - ); + }) + ]; system.stateVersion = config.system.stateVersion; networking.firewall.enable = false; @@ -404,8 +373,8 @@ in # real sample through both tiers into the store. validateConfigFile = true; settings = { - # One receiver per hive when ingest is authenticated, and - # that multiplicity is forced rather than chosen. The `hive` + # One receiver per hive, and that multiplicity is forced + # rather than chosen. The `hive` # label has to come from something the sender cannot write, # and the only such thing here is WHICH RECEIVER accepted # the sample: a processor cannot read the token's claims @@ -413,19 +382,15 @@ in # an auth claim yields nothing — silently, with a healthy # startup), and one receiver holding many credentials never # reveals which one matched. - receivers = - if hiveAuth then - lib.mapAttrs' ( - h: p: - lib.nameValuePair "otlp/${h}" { - protocols.http = { - endpoint = "127.0.0.1:${toString p}"; - auth.authenticator = "oidc/${h}"; - }; - } - ) hivePorts - else - { otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; }; + receivers = lib.mapAttrs' ( + h: p: + lib.nameValuePair "otlp/${h}" { + protocols.http = { + endpoint = "127.0.0.1:${toString p}"; + auth.authenticator = "oidc/${h}"; + }; + } + ) hivePorts; exporters = lib.optionalAttrs vmCfg.enable { @@ -473,7 +438,7 @@ in # INERT — the collector starts clean and the receiver # naming it authenticates nothing. Derived from the same # attrset as the receivers so the two cannot disagree. - service.extensions = lib.optionals hiveAuth (map (h: "oidc/${h}") (lib.attrNames hivePorts)); + service.extensions = map (h: "oidc/${h}") (lib.attrNames hivePorts); # Fan-out, not a choice: with both configured the same # samples go upstream AND into the swarm's store. The store @@ -482,25 +447,16 @@ in # `exporterNames` is shared by every pipeline — where a # sample goes is a property of this tier, not of the hive # that sent it. - service.pipelines = - if hiveAuth then - lib.mapAttrs' ( - h: _: - lib.nameValuePair "metrics/${h}" { - receivers = [ "otlp/${h}" ]; - processors = [ "resource/${h}" ]; - exporters = exporterNames; - } - ) hivePorts - else - { - metrics = { - receivers = [ "otlp" ]; - exporters = exporterNames; - }; - }; + service.pipelines = lib.mapAttrs' ( + h: _: + lib.nameValuePair "metrics/${h}" { + receivers = [ "otlp/${h}" ]; + processors = [ "resource/${h}" ]; + exporters = exporterNames; + } + ) hivePorts; } - // lib.optionalAttrs hiveAuth { + // { extensions = lib.mapAttrs' ( h: _: lib.nameValuePair "oidc/${h}" {