From 08faa0970e6ed3f11b2dbdb77f988a4b23570a65 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 19 Aug 2026 14:30:23 +0200 Subject: [PATCH] swarm-otel: authenticate ingest per hive, and stamp the hive from the receiver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The swarm collector accepted OTLP from anyone who could reach it, and took the `hive` resource attribute from the payload. So any writer on the swarm network could attribute metrics to any hive, and nothing downstream could tell. The label now comes from which receiver accepted the sample: one receiver per hive, each behind an `oidc` extension verifying a token minted for that hive's audience, each feeding a pipeline whose `resource` processor upserts a constant. A sender cannot influence it, because the only input is which authenticated port the bytes arrived on. That multiplicity is forced rather than preferred. A processor cannot read the token's claims — `from_context` reads request metadata, and asking it for an auth claim yields nothing, silently, with a healthy startup — and one receiver holding several credentials never reveals which one matched. The per-hive ports are internal: a hive reaches its receiver as a path under this collector's existing gateway name, so nginx (rendered from this same evaluation) is the only thing that names a port. Fronting each hive with its own vhost would need a certificate, a DNS name and a gateway entry per hive to express routing the gateway already does. Turning this on removes the unauthenticated receiver. While an open port still accepts samples the per-hive receivers are decoration, so this is the switch itself rather than a hardening layer beside it; a swarm that wants the open receiver says so. `hive-ca-trust.nix` grows `bundlePathFor`, because a consumer taking its own CA argument has to name the bundle rather than just have `SSL_CERT_FILE` exported at it. --- nix/host-modules/lib/hive-ca-trust.nix | 20 +- nix/host-modules/swarm-otel.nix | 305 +++++++++++++++++++++++-- 2 files changed, 302 insertions(+), 23 deletions(-) diff --git a/nix/host-modules/lib/hive-ca-trust.nix b/nix/host-modules/lib/hive-ca-trust.nix index 509730c3..0ad563ea 100644 --- a/nix/host-modules/lib/hive-ca-trust.nix +++ b/nix/host-modules/lib/hive-ca-trust.nix @@ -39,10 +39,26 @@ let # the bundle next to the CA and explains the split. caHostPath = "${tlsCfg.stateDir}/trust-bundle.pem"; caContainerPath = "/run/hive-ca/trust-bundle.pem"; + + # One definition, used by `trustBundle` to WRITE the bundle and published + # below so a caller can NAME it. Two copies of this path would be two + # things to keep in step, and the one that drifts is the reader. + bundleDirFor = name: "/run/${name}-ca"; + bundlePathFor = name: "${bundleDirFor name}/trust-bundle.pem"; in { inherit useSelfSigned caContainerPath; + # Where `trustBundle` below puts the assembled bundle, for the callers + # that must NAME it rather than just have it exported. `SSL_CERT_FILE` is + # set for you and needs no path here; a consumer that takes its own CA + # argument (an OIDC verifier's `issuer_ca_path`, a client's `--cacert`) + # does, and the alternative is copying `/run/-ca/…` to the call + # site. That copy breaks silently: the bundle keeps being written, the + # consumer keeps reading a path that no longer exists, and the failure + # surfaces as a TLS error naming the peer rather than the file. + inherit bundlePathFor; + # Fold into the container's `bindMounts` via `//`. Binds ONLY the public # CA cert (never the `hive-tls` state dir — it holds the CA + leaf private # keys), read-only. Empty when not self-signed, so the whole trust path @@ -106,8 +122,8 @@ in enable ? true, }: let - dir = "/run/${name}-ca"; - bundlePath = "${dir}/trust-bundle.pem"; + dir = bundleDirFor name; + bundlePath = bundlePathFor name; unit = "${name}-ca-bundle"; source = if hostUnit then caHostPath else caContainerPath; in diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 0eacc0de..7b39308c 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -40,6 +40,50 @@ let # the required-domain assertion in hive-network.nix should be what an # operator sees, not a coercion error from here. domainBase = if swarmDomain == null then "invalid" else swarmDomain; + + autheliaCfg = hyperhiveCfg.swarm.authelia; + + # Whether ingest is authenticated per hive. Derived rather than declared + # so the common case needs no attribute, but an option (below) because + # this decides whether the unauthenticated receiver exists at all. + hiveAuth = cfg.requireHiveIdentity; + + # `attrNames` is sorted, so this is a function of the hive SET and not of + # the order anyone wrote it in. + # + # These ports are internal and appear in no URL: a hive addresses its own + # receiver as a PATH on this collector's single gateway name, and nginx — + # rendered from this same evaluation — is the only thing that ever names + # the port. That is what makes deriving them safe here and unsafe in the + # obvious other place: were a hive told a port, inserting a hive would + # renumber the ones after it and silently move a port a running hive was + # already sending to. + hivePorts = lib.listToAttrs ( + lib.imap0 (i: h: lib.nameValuePair h (cfg.hivePortBase + i)) ( + lib.attrNames hyperhiveCfg.swarm.hives + ) + ); + + # The swarm's authelia is reached by its gateway name, whose leaf is + # issued by the swarm services sub-CA — so this container needs the same + # runtime CA trust every other consumer of a swarm-service name needs. + # The CA is generated at runtime and cannot be baked into a derivation, + # which is why it arrives as a bind mount rather than + # `security.pki.certificateFiles`. + caTrust = import ./lib/hive-ca-trust.nix { + inherit lib; + tlsCfg = hyperhiveCfg.tls; + inherit gatewayCfg; + }; + caBundle = caTrust.bundlePathFor cfg.machine; + + # One list, read by every pipeline: the per-hive pipelines fan out to + # exactly the same destinations as the single pipeline they replace. + # Written once because "which exporters" is a property of this tier, not + # of which hive a sample came from. + exporterNames = + lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") + ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; in { options.services.hyperhive.swarm.otel = { @@ -107,6 +151,53 @@ in ''; }; + requireHiveIdentity = lib.mkOption { + type = lib.types.bool; + default = autheliaCfg.enable && autheliaCfg.url != null; + defaultText = lib.literalExpression "swarm.authelia.enable && swarm.authelia.url != null"; + description = '' + Authenticate ingest per hive: each hive gets its own receiver, + verifying an OIDC token minted for that hive's audience, and the + `hive` label is stamped from **which receiver accepted the + sample** rather than from anything the sender wrote. + + ⚠️ Turning this on **removes the unauthenticated receiver**. That + is the point rather than a side effect: while an unauthenticated + port still accepts samples, any writer that can reach this + collector can still attribute metrics to any hive, and the + per-hive receivers are decoration. + + Defaults to whether this swarm has an authelia to mint against. + Set it false to keep the open receiver on a swarm where every + writer is already trusted — an explicit choice, which is what it + should be. + ''; + }; + + hivePortBase = lib.mkOption { + type = lib.types.port; + default = 4330; + description = '' + First port of the per-hive receiver range; each hive in + {option}`services.hyperhive.swarm.hives` takes the next one, in + sorted-name order. + + ⚠️ Internal. No client is told a port — a hive reaches its own + receiver as `https://''${domain}/`, and the gateway routes + the path. So inserting a hive renumbering the ones after it is + harmless here: nginx is rendered from this same evaluation and + moves with it. + + The range still matters because every swarm container shares the + host's network namespace, so a derived port can land on one + another service already holds — with no bind error and nothing in + any log. The assertions below check this range against every port + this module and the hive tier declare — which is as far as a + module can see, since a port another module picks is not + knowable from here without reading its config. + ''; + }; + domain = lib.mkOption { type = lib.types.str; default = "otel.${domainBase}"; @@ -143,11 +234,46 @@ in services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; - locations."/" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}"; - }; + locations = + # One name for the whole collector, and the hive is a path under + # it. The alternative — a vhost per hive — needs a certificate, + # a DNS name and a `localNames` entry per hive to express the + # same routing the gateway already does for free. + # + # ⚠️ The trailing slash on both sides is load-bearing: it is what + # strips `/` before the request reaches the receiver, which + # serves `/v1/metrics` and knows nothing about hives. Without it + # the receiver sees `//v1/metrics` and answers 404 to a + # request that authenticated perfectly. + lib.optionalAttrs hiveAuth ( + lib.mapAttrs' ( + h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } + ) hivePorts + ) + // { + "/" = + if hiveAuth then + # Not a proxy to a receiver that no longer exists. A closed + # door answering 404 is the honest description of this + # collector once ingest is per-hive: there is no + # swarm-wide inbox any more. + { return = "404"; } + else + { proxyPass = "http://127.0.0.1:${toString cfg.port}"; }; + }; }; + # Turning on the identities this tier authenticates against, which the + # option exists to allow: its own description names this module as the + # second consumer, so the queue is not a prerequisite for authenticated + # telemetry. + services.hyperhive.swarm.authelia.oidc.hiveIdentities = lib.mkIf hiveAuth true; + + # The CA bind source is written at runtime by a host unit, so the + # container has to start after it — otherwise nspawn sets up a mount + # over a file that does not exist yet. + systemd.services."container@${cfg.machine}" = lib.mkIf hiveAuth caTrust.containerOrdering; + assertions = [ { # The tier exists to hold the upstream credential and to write the @@ -164,6 +290,49 @@ in metrics store. ''; } + { + # Without a roster there are no receivers at all, so this + # collector would listen on nothing while looking configured. + assertion = !hiveAuth || hyperhiveCfg.swarm.hives != { }; + message = '' + services.hyperhive.swarm.otel.requireHiveIdentity is true but + services.hyperhive.swarm.hives is empty: ingest is authenticated + per hive, so an empty roster means this collector accepts + nothing from anyone. + + List the swarm's hives, or set requireHiveIdentity = false to + keep an unauthenticated receiver. + ''; + } + { + # A port collision between two listeners on one host is a runtime + # coin toss with nothing in any log — the failure this whole + # comment budget exists to prevent. Checked against every port + # reachable from here; a port some other module picks is not. + assertion = + let + derived = lib.attrValues hivePorts; + others = [ + cfg.port + cfg.telemetryPort + otelCfg.collector.port + ] + ++ lib.optional vmCfg.enable vmCfg.port; + all = derived ++ others; + in + !hiveAuth || lib.length (lib.unique all) == lib.length all; + message = '' + services.hyperhive.swarm.otel: the per-hive receiver range + starting at hivePortBase (${toString cfg.hivePortBase}, one port + per hive in services.hyperhive.swarm.hives) overlaps another + port on this host. + + Every swarm container shares the host's network namespace, so + two listeners claiming one port is not a build failure — it is + whichever process started first, silently. Move hivePortBase to + a free range. + ''; + } ]; containers.${cfg.machine} = { @@ -179,12 +348,16 @@ in # Read-only, and only when one is configured — binding a path that # does not exist makes nixos-container refuse to start the container, # which is a stall several layers from its cause. - bindMounts = lib.optionalAttrs (otelCfg.headersCredential != null) { - ${otelCfg.headersCredential} = { - hostPath = otelCfg.headersCredential; - isReadOnly = true; - }; - }; + bindMounts = + lib.optionalAttrs (otelCfg.headersCredential != null) { + ${otelCfg.headersCredential} = { + hostPath = otelCfg.headersCredential; + isReadOnly = true; + }; + } + # The public hive CA, read-only — only when something in here + # actually verifies a swarm-service name. + // lib.optionalAttrs hiveAuth caTrust.bindMount; config = { ... }: @@ -200,7 +373,19 @@ in inherit (config.services.hyperhive.network) bridgeIp; dnsConsumers = [ "opentelemetry-collector.service" ]; }) - ]; + ] + # `SSL_CERT_FILE` REPLACES the trust store rather than adding to + # it, so a failed assembly yields an empty pool and every TLS + # call fails while the unit looks healthy. That is why this is + # the shared helper — it carries the `Requires` and the + # non-empty check — and not a local `cat`. + ++ lib.optional hiveAuth ( + caTrust.trustBundle { + inherit pkgs; + name = cfg.machine; + consumers = [ "opentelemetry-collector" ]; + } + ); system.stateVersion = config.system.stateVersion; networking.firewall.enable = false; @@ -219,7 +404,28 @@ in # real sample through both tiers into the store. validateConfigFile = true; settings = { - receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; + # One receiver per hive when ingest is authenticated, and + # that multiplicity is forced rather than chosen. The `hive` + # label has to come from something the sender cannot write, + # and the only such thing here is WHICH RECEIVER accepted + # the sample: a processor cannot read the token's claims + # (`from_context` reads request metadata, and asking it for + # an auth claim yields nothing — silently, with a healthy + # startup), and one receiver holding many credentials never + # reveals which one matched. + receivers = + if hiveAuth then + lib.mapAttrs' ( + h: p: + lib.nameValuePair "otlp/${h}" { + protocols.http = { + endpoint = "127.0.0.1:${toString p}"; + auth.authenticator = "oidc/${h}"; + }; + } + ) hivePorts + else + { otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; }; exporters = lib.optionalAttrs vmCfg.enable { @@ -263,16 +469,73 @@ in } ]; - service.pipelines.metrics = { - receivers = [ "otlp" ]; - # Fan-out, not a choice: with both configured the same - # samples go upstream AND into the swarm's store. The store - # is for looking at this swarm; the upstream is for whoever - # aggregates across swarms, and neither replaces the other. - exporters = - lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") - ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; - }; + # ⚠️ An extension that is configured but not listed here is + # INERT — the collector starts clean and the receiver + # naming it authenticates nothing. Derived from the same + # attrset as the receivers so the two cannot disagree. + service.extensions = lib.optionals hiveAuth (map (h: "oidc/${h}") (lib.attrNames hivePorts)); + + # Fan-out, not a choice: with both configured the same + # samples go upstream AND into the swarm's store. The store + # is for looking at this swarm; the upstream is for whoever + # aggregates across swarms, and neither replaces the other. + # `exporterNames` is shared by every pipeline — where a + # sample goes is a property of this tier, not of the hive + # that sent it. + service.pipelines = + if hiveAuth then + lib.mapAttrs' ( + h: _: + lib.nameValuePair "metrics/${h}" { + receivers = [ "otlp/${h}" ]; + processors = [ "resource/${h}" ]; + exporters = exporterNames; + } + ) hivePorts + else + { + metrics = { + receivers = [ "otlp" ]; + exporters = exporterNames; + }; + }; + } + // lib.optionalAttrs hiveAuth { + extensions = lib.mapAttrs' ( + h: _: + lib.nameValuePair "oidc/${h}" { + issuer_url = autheliaCfg.url; + # The audience this hive's client is registered to + # request, and the reason one hive's token is refused by + # another hive's receiver. Same expression authelia + # registers it under — a second spelling here would deny + # every hive, as a 401 that blames the token. + audience = "${autheliaCfg.hiveClientPrefix}${h}"; + # ⚠️ `issuer_ca_path`. `issuer_ca_file`, `ca_file` and + # `tls.ca_file` are all INVALID KEYS for this extension + # — measured, and the failure is a startup error naming + # the key rather than anything about certificates. + issuer_ca_path = caBundle; + } + ) hivePorts; + + # `upsert`, not `insert`: a sender that stamps its own + # `hive` must be OVERWRITTEN, not deferred to. This + # processor is the whole attribution boundary — the value + # is a constant per receiver, so it says which hive + # authenticated, not which hive claimed to be sending. + processors = lib.mapAttrs' ( + h: _: + lib.nameValuePair "resource/${h}" { + attributes = [ + { + key = "hive"; + value = h; + action = "upsert"; + } + ]; + } + ) hivePorts; }; };