diff --git a/nix/host-modules/lib/hive-ca-trust.nix b/nix/host-modules/lib/hive-ca-trust.nix index 509730c3..0ad563ea 100644 --- a/nix/host-modules/lib/hive-ca-trust.nix +++ b/nix/host-modules/lib/hive-ca-trust.nix @@ -39,10 +39,26 @@ let # the bundle next to the CA and explains the split. caHostPath = "${tlsCfg.stateDir}/trust-bundle.pem"; caContainerPath = "/run/hive-ca/trust-bundle.pem"; + + # One definition, used by `trustBundle` to WRITE the bundle and published + # below so a caller can NAME it. Two copies of this path would be two + # things to keep in step, and the one that drifts is the reader. + bundleDirFor = name: "/run/${name}-ca"; + bundlePathFor = name: "${bundleDirFor name}/trust-bundle.pem"; in { inherit useSelfSigned caContainerPath; + # Where `trustBundle` below puts the assembled bundle, for the callers + # that must NAME it rather than just have it exported. `SSL_CERT_FILE` is + # set for you and needs no path here; a consumer that takes its own CA + # argument (an OIDC verifier's `issuer_ca_path`, a client's `--cacert`) + # does, and the alternative is copying `/run/-ca/…` to the call + # site. That copy breaks silently: the bundle keeps being written, the + # consumer keeps reading a path that no longer exists, and the failure + # surfaces as a TLS error naming the peer rather than the file. + inherit bundlePathFor; + # Fold into the container's `bindMounts` via `//`. Binds ONLY the public # CA cert (never the `hive-tls` state dir — it holds the CA + leaf private # keys), read-only. Empty when not self-signed, so the whole trust path @@ -106,8 +122,8 @@ in enable ? true, }: let - dir = "/run/${name}-ca"; - bundlePath = "${dir}/trust-bundle.pem"; + dir = bundleDirFor name; + bundlePath = bundlePathFor name; unit = "${name}-ca-bundle"; source = if hostUnit then caHostPath else caContainerPath; in diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 0eacc0de..7b39308c 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -40,6 +40,50 @@ let # the required-domain assertion in hive-network.nix should be what an # operator sees, not a coercion error from here. domainBase = if swarmDomain == null then "invalid" else swarmDomain; + + autheliaCfg = hyperhiveCfg.swarm.authelia; + + # Whether ingest is authenticated per hive. Derived rather than declared + # so the common case needs no attribute, but an option (below) because + # this decides whether the unauthenticated receiver exists at all. + hiveAuth = cfg.requireHiveIdentity; + + # `attrNames` is sorted, so this is a function of the hive SET and not of + # the order anyone wrote it in. + # + # These ports are internal and appear in no URL: a hive addresses its own + # receiver as a PATH on this collector's single gateway name, and nginx — + # rendered from this same evaluation — is the only thing that ever names + # the port. That is what makes deriving them safe here and unsafe in the + # obvious other place: were a hive told a port, inserting a hive would + # renumber the ones after it and silently move a port a running hive was + # already sending to. + hivePorts = lib.listToAttrs ( + lib.imap0 (i: h: lib.nameValuePair h (cfg.hivePortBase + i)) ( + lib.attrNames hyperhiveCfg.swarm.hives + ) + ); + + # The swarm's authelia is reached by its gateway name, whose leaf is + # issued by the swarm services sub-CA — so this container needs the same + # runtime CA trust every other consumer of a swarm-service name needs. + # The CA is generated at runtime and cannot be baked into a derivation, + # which is why it arrives as a bind mount rather than + # `security.pki.certificateFiles`. + caTrust = import ./lib/hive-ca-trust.nix { + inherit lib; + tlsCfg = hyperhiveCfg.tls; + inherit gatewayCfg; + }; + caBundle = caTrust.bundlePathFor cfg.machine; + + # One list, read by every pipeline: the per-hive pipelines fan out to + # exactly the same destinations as the single pipeline they replace. + # Written once because "which exporters" is a property of this tier, not + # of which hive a sample came from. + exporterNames = + lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") + ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; in { options.services.hyperhive.swarm.otel = { @@ -107,6 +151,53 @@ in ''; }; + requireHiveIdentity = lib.mkOption { + type = lib.types.bool; + default = autheliaCfg.enable && autheliaCfg.url != null; + defaultText = lib.literalExpression "swarm.authelia.enable && swarm.authelia.url != null"; + description = '' + Authenticate ingest per hive: each hive gets its own receiver, + verifying an OIDC token minted for that hive's audience, and the + `hive` label is stamped from **which receiver accepted the + sample** rather than from anything the sender wrote. + + ⚠️ Turning this on **removes the unauthenticated receiver**. That + is the point rather than a side effect: while an unauthenticated + port still accepts samples, any writer that can reach this + collector can still attribute metrics to any hive, and the + per-hive receivers are decoration. + + Defaults to whether this swarm has an authelia to mint against. + Set it false to keep the open receiver on a swarm where every + writer is already trusted — an explicit choice, which is what it + should be. + ''; + }; + + hivePortBase = lib.mkOption { + type = lib.types.port; + default = 4330; + description = '' + First port of the per-hive receiver range; each hive in + {option}`services.hyperhive.swarm.hives` takes the next one, in + sorted-name order. + + ⚠️ Internal. No client is told a port — a hive reaches its own + receiver as `https://''${domain}/`, and the gateway routes + the path. So inserting a hive renumbering the ones after it is + harmless here: nginx is rendered from this same evaluation and + moves with it. + + The range still matters because every swarm container shares the + host's network namespace, so a derived port can land on one + another service already holds — with no bind error and nothing in + any log. The assertions below check this range against every port + this module and the hive tier declare — which is as far as a + module can see, since a port another module picks is not + knowable from here without reading its config. + ''; + }; + domain = lib.mkOption { type = lib.types.str; default = "otel.${domainBase}"; @@ -143,11 +234,46 @@ in services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; - locations."/" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}"; - }; + locations = + # One name for the whole collector, and the hive is a path under + # it. The alternative — a vhost per hive — needs a certificate, + # a DNS name and a `localNames` entry per hive to express the + # same routing the gateway already does for free. + # + # ⚠️ The trailing slash on both sides is load-bearing: it is what + # strips `/` before the request reaches the receiver, which + # serves `/v1/metrics` and knows nothing about hives. Without it + # the receiver sees `//v1/metrics` and answers 404 to a + # request that authenticated perfectly. + lib.optionalAttrs hiveAuth ( + lib.mapAttrs' ( + h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } + ) hivePorts + ) + // { + "/" = + if hiveAuth then + # Not a proxy to a receiver that no longer exists. A closed + # door answering 404 is the honest description of this + # collector once ingest is per-hive: there is no + # swarm-wide inbox any more. + { return = "404"; } + else + { proxyPass = "http://127.0.0.1:${toString cfg.port}"; }; + }; }; + # Turning on the identities this tier authenticates against, which the + # option exists to allow: its own description names this module as the + # second consumer, so the queue is not a prerequisite for authenticated + # telemetry. + services.hyperhive.swarm.authelia.oidc.hiveIdentities = lib.mkIf hiveAuth true; + + # The CA bind source is written at runtime by a host unit, so the + # container has to start after it — otherwise nspawn sets up a mount + # over a file that does not exist yet. + systemd.services."container@${cfg.machine}" = lib.mkIf hiveAuth caTrust.containerOrdering; + assertions = [ { # The tier exists to hold the upstream credential and to write the @@ -164,6 +290,49 @@ in metrics store. ''; } + { + # Without a roster there are no receivers at all, so this + # collector would listen on nothing while looking configured. + assertion = !hiveAuth || hyperhiveCfg.swarm.hives != { }; + message = '' + services.hyperhive.swarm.otel.requireHiveIdentity is true but + services.hyperhive.swarm.hives is empty: ingest is authenticated + per hive, so an empty roster means this collector accepts + nothing from anyone. + + List the swarm's hives, or set requireHiveIdentity = false to + keep an unauthenticated receiver. + ''; + } + { + # A port collision between two listeners on one host is a runtime + # coin toss with nothing in any log — the failure this whole + # comment budget exists to prevent. Checked against every port + # reachable from here; a port some other module picks is not. + assertion = + let + derived = lib.attrValues hivePorts; + others = [ + cfg.port + cfg.telemetryPort + otelCfg.collector.port + ] + ++ lib.optional vmCfg.enable vmCfg.port; + all = derived ++ others; + in + !hiveAuth || lib.length (lib.unique all) == lib.length all; + message = '' + services.hyperhive.swarm.otel: the per-hive receiver range + starting at hivePortBase (${toString cfg.hivePortBase}, one port + per hive in services.hyperhive.swarm.hives) overlaps another + port on this host. + + Every swarm container shares the host's network namespace, so + two listeners claiming one port is not a build failure — it is + whichever process started first, silently. Move hivePortBase to + a free range. + ''; + } ]; containers.${cfg.machine} = { @@ -179,12 +348,16 @@ in # Read-only, and only when one is configured — binding a path that # does not exist makes nixos-container refuse to start the container, # which is a stall several layers from its cause. - bindMounts = lib.optionalAttrs (otelCfg.headersCredential != null) { - ${otelCfg.headersCredential} = { - hostPath = otelCfg.headersCredential; - isReadOnly = true; - }; - }; + bindMounts = + lib.optionalAttrs (otelCfg.headersCredential != null) { + ${otelCfg.headersCredential} = { + hostPath = otelCfg.headersCredential; + isReadOnly = true; + }; + } + # The public hive CA, read-only — only when something in here + # actually verifies a swarm-service name. + // lib.optionalAttrs hiveAuth caTrust.bindMount; config = { ... }: @@ -200,7 +373,19 @@ in inherit (config.services.hyperhive.network) bridgeIp; dnsConsumers = [ "opentelemetry-collector.service" ]; }) - ]; + ] + # `SSL_CERT_FILE` REPLACES the trust store rather than adding to + # it, so a failed assembly yields an empty pool and every TLS + # call fails while the unit looks healthy. That is why this is + # the shared helper — it carries the `Requires` and the + # non-empty check — and not a local `cat`. + ++ lib.optional hiveAuth ( + caTrust.trustBundle { + inherit pkgs; + name = cfg.machine; + consumers = [ "opentelemetry-collector" ]; + } + ); system.stateVersion = config.system.stateVersion; networking.firewall.enable = false; @@ -219,7 +404,28 @@ in # real sample through both tiers into the store. validateConfigFile = true; settings = { - receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; + # One receiver per hive when ingest is authenticated, and + # that multiplicity is forced rather than chosen. The `hive` + # label has to come from something the sender cannot write, + # and the only such thing here is WHICH RECEIVER accepted + # the sample: a processor cannot read the token's claims + # (`from_context` reads request metadata, and asking it for + # an auth claim yields nothing — silently, with a healthy + # startup), and one receiver holding many credentials never + # reveals which one matched. + receivers = + if hiveAuth then + lib.mapAttrs' ( + h: p: + lib.nameValuePair "otlp/${h}" { + protocols.http = { + endpoint = "127.0.0.1:${toString p}"; + auth.authenticator = "oidc/${h}"; + }; + } + ) hivePorts + else + { otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; }; exporters = lib.optionalAttrs vmCfg.enable { @@ -263,16 +469,73 @@ in } ]; - service.pipelines.metrics = { - receivers = [ "otlp" ]; - # Fan-out, not a choice: with both configured the same - # samples go upstream AND into the swarm's store. The store - # is for looking at this swarm; the upstream is for whoever - # aggregates across swarms, and neither replaces the other. - exporters = - lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") - ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; - }; + # ⚠️ An extension that is configured but not listed here is + # INERT — the collector starts clean and the receiver + # naming it authenticates nothing. Derived from the same + # attrset as the receivers so the two cannot disagree. + service.extensions = lib.optionals hiveAuth (map (h: "oidc/${h}") (lib.attrNames hivePorts)); + + # Fan-out, not a choice: with both configured the same + # samples go upstream AND into the swarm's store. The store + # is for looking at this swarm; the upstream is for whoever + # aggregates across swarms, and neither replaces the other. + # `exporterNames` is shared by every pipeline — where a + # sample goes is a property of this tier, not of the hive + # that sent it. + service.pipelines = + if hiveAuth then + lib.mapAttrs' ( + h: _: + lib.nameValuePair "metrics/${h}" { + receivers = [ "otlp/${h}" ]; + processors = [ "resource/${h}" ]; + exporters = exporterNames; + } + ) hivePorts + else + { + metrics = { + receivers = [ "otlp" ]; + exporters = exporterNames; + }; + }; + } + // lib.optionalAttrs hiveAuth { + extensions = lib.mapAttrs' ( + h: _: + lib.nameValuePair "oidc/${h}" { + issuer_url = autheliaCfg.url; + # The audience this hive's client is registered to + # request, and the reason one hive's token is refused by + # another hive's receiver. Same expression authelia + # registers it under — a second spelling here would deny + # every hive, as a 401 that blames the token. + audience = "${autheliaCfg.hiveClientPrefix}${h}"; + # ⚠️ `issuer_ca_path`. `issuer_ca_file`, `ca_file` and + # `tls.ca_file` are all INVALID KEYS for this extension + # — measured, and the failure is a startup error naming + # the key rather than anything about certificates. + issuer_ca_path = caBundle; + } + ) hivePorts; + + # `upsert`, not `insert`: a sender that stamps its own + # `hive` must be OVERWRITTEN, not deferred to. This + # processor is the whole attribution boundary — the value + # is a constant per receiver, so it says which hive + # authenticated, not which hive claimed to be sending. + processors = lib.mapAttrs' ( + h: _: + lib.nameValuePair "resource/${h}" { + attributes = [ + { + key = "hive"; + value = h; + action = "upsert"; + } + ]; + } + ) hivePorts; }; };