diff --git a/docs/observability.md b/docs/observability.md index 7384ba0c..a1f16a7c 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -203,26 +203,13 @@ somehow — copy it across and name it: services.hyperhive.otel.clientSecretFile = "/run/secrets/hive-telemetry.secret"; ``` -Getting that wrong shows up as the hive's collector logging 401s from the swarm -tier and no metrics appearing for that hive. When a single host runs both tiers -the build catches it instead, because it can see both sides. +**There is no unauthenticated mode.** A hive always presents an identity, so a +missing credential is a build error rather than a quieter fallback — the +collector has no anonymous route to accept samples on, and every path it serves +belongs to exactly one hive. -To accept unauthenticated ingest — every writer on the swarm network already -trusted, or a swarm with no authelia: - -```nix -services.hyperhive.swarm.otel.requireHiveIdentity = false; -services.hyperhive.otel.clientSecretFile = null; # on each hive that had one -``` - -Both halves, because a collector that authenticates also addresses its hive's -own path, and an unauthenticated swarm tier serves no per-hive paths. Set only -the first and that hive's samples 404 instead of arriving. On a host running -both tiers the build says so; on a split host it is yours to keep in step. - -⚠️ That reopens the original hole rather than merely skipping a check: while an -unauthenticated port is listening, anything that can reach the collector can -attribute metrics to any hive. +Getting the secret wrong shows up as the hive's collector logging 401s from the +swarm tier and no metrics appearing for that hive. ### `services.hyperhive.otel.collector.port` — port, default `4318` diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 0c1ef648..2800717c 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -255,7 +255,11 @@ in swarmName = "otlphttp/swarm"; authName = "oauth2client/swarm"; - # Holding a credential IS the condition — see `clientSecretFile`. + # A hive always authenticates to the swarm's collector as itself, so + # this is not a mode — it is a precondition, and the assertion below + # is what enforces it. Kept as a name because several places have to + # read "do we have what it takes", and an eval error from a null path + # names this file rather than the option an operator has to set. senderAuth = otel.clientSecretFile != null && hiveName != null; # This hive's client id, and also the audience it must ASK for. Both @@ -382,60 +386,28 @@ in assertions = [ { - # Only checkable on a host that runs BOTH tiers — which is the - # deployment where it can actually go wrong silently. A remote - # hive cannot see the swarm tier's config at all, so its operator - # sets the path explicitly and this says nothing. - assertion = !(swarmOtelCfg.enable && swarmOtelCfg.requireHiveIdentity) || senderAuth; + # A hive authenticates to the swarm's collector as itself — there + # is no unauthenticated path to fall back to, so a missing + # credential is a broken deployment rather than a quieter mode. + # Caught here because the alternative is a collector that starts + # cleanly, retries forever, and reports nothing to anyone. + assertion = senderAuth; message = '' - This host runs the swarm's telemetry collector with - services.hyperhive.swarm.otel.requireHiveIdentity = true, so - ingest is authenticated per hive — but this hive's own - collector has no credential to present: + services.hyperhive.otel.enable is true but this hive has no + identity to present to the swarm's collector: services.hyperhive.otel.clientSecretFile = ${ if otel.clientSecretFile == null then "null" else otel.clientSecretFile } services.hyperhive.hiveName = ${if hiveName == null then "null" else hiveName} - Its samples would be refused with a 401 by the collector - running beside it. Set both, or set requireHiveIdentity = - false to accept unauthenticated ingest. - ''; - } - { - # The mirror of the assertion above, and the reason it exists is - # that the failure is SILENT rather than loud. With ingest - # unauthenticated the swarm tier serves one catch-all location - # and passes the URI through unchanged — but this tier still - # appends `/` whenever it holds a credential, so the - # receiver is asked for `//v1/metrics`, a path it does not - # serve. The result is 404s and retries: no 401, no assertion, no - # log anywhere saying telemetry stopped. - # - # Only reachable by overriding one side without the other, since - # both defaults derive from `swarm.authelia.enable` and move - # together. That is exactly why it is worth a build error — an - # operator who flips the escape hatch has no reason to suspect - # the sending half. - assertion = !(swarmOtelCfg.enable && !swarmOtelCfg.requireHiveIdentity && senderAuth); - message = '' - This host accepts unauthenticated telemetry ingest - (services.hyperhive.swarm.otel.requireHiveIdentity = false), - but its own collector still holds a credential: + Every hive authenticates as itself — that is what makes the + `hive` label on its metrics mean anything — so both are + required. - services.hyperhive.otel.clientSecretFile = ${ - if otel.clientSecretFile == null then "null" else otel.clientSecretFile - } - - A collector that authenticates also addresses its hive's own - path, and an unauthenticated swarm tier serves no per-hive - paths — so this hive's samples would 404 rather than be - refused, which no log names as an auth problem. - - Set services.hyperhive.otel.clientSecretFile = null to send - unauthenticated too, or drop the requireHiveIdentity - override. + On a host that runs the swarm's identity provider, the default + already points at the secret authelia minted. On a hive that + does not, copy that hive's secret across and name it here. ''; } ]; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 7b39308c..525bac27 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -43,11 +43,6 @@ let autheliaCfg = hyperhiveCfg.swarm.authelia; - # Whether ingest is authenticated per hive. Derived rather than declared - # so the common case needs no attribute, but an option (below) because - # this decides whether the unauthenticated receiver exists at all. - hiveAuth = cfg.requireHiveIdentity; - # `attrNames` is sorted, so this is a function of the hive SET and not of # the order anyone wrote it in. # @@ -59,9 +54,7 @@ let # renumber the ones after it and silently move a port a running hive was # already sending to. hivePorts = lib.listToAttrs ( - lib.imap0 (i: h: lib.nameValuePair h (cfg.hivePortBase + i)) ( - lib.attrNames hyperhiveCfg.swarm.hives - ) + lib.imap0 (i: h: lib.nameValuePair h (cfg.port + i)) (lib.attrNames hyperhiveCfg.swarm.hives) ); # The swarm's authelia is reached by its gateway name, whose leaf is @@ -119,7 +112,17 @@ in type = lib.types.port; default = 4319; description = '' - Port this collector's OTLP/HTTP receiver listens on. + First port of this collector's receiver range. Every hive in + {option}`services.hyperhive.swarm.hives` gets its **own** + authenticated receiver — that is what makes the `hive` label + unforgeable — so the range is one port per hive, starting here, in + sorted-name order. + + ⚠️ Internal. No client is ever told a port: a hive reaches its own + receiver as `https://''${domain}/`, and the gateway routes on + that path. So adding a hive, which renumbers the ones after it, is + harmless — nginx is rendered from this same evaluation and moves + with it. ⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the hive tier already uses it (`services.hyperhive.otel.collector.port`) @@ -127,7 +130,9 @@ in listeners claiming one port on one host is not a build failure — it is a runtime coin toss over which one gets it, with nothing in any log saying so. The same collision cost a release when grafana - and the forge both defaulted to 3000. + and the forge both defaulted to 3000. The assertions below check + the whole derived range against every port this module and the hive + tier declare, which is as far as a module can see. ''; }; @@ -151,53 +156,6 @@ in ''; }; - requireHiveIdentity = lib.mkOption { - type = lib.types.bool; - default = autheliaCfg.enable && autheliaCfg.url != null; - defaultText = lib.literalExpression "swarm.authelia.enable && swarm.authelia.url != null"; - description = '' - Authenticate ingest per hive: each hive gets its own receiver, - verifying an OIDC token minted for that hive's audience, and the - `hive` label is stamped from **which receiver accepted the - sample** rather than from anything the sender wrote. - - ⚠️ Turning this on **removes the unauthenticated receiver**. That - is the point rather than a side effect: while an unauthenticated - port still accepts samples, any writer that can reach this - collector can still attribute metrics to any hive, and the - per-hive receivers are decoration. - - Defaults to whether this swarm has an authelia to mint against. - Set it false to keep the open receiver on a swarm where every - writer is already trusted — an explicit choice, which is what it - should be. - ''; - }; - - hivePortBase = lib.mkOption { - type = lib.types.port; - default = 4330; - description = '' - First port of the per-hive receiver range; each hive in - {option}`services.hyperhive.swarm.hives` takes the next one, in - sorted-name order. - - ⚠️ Internal. No client is told a port — a hive reaches its own - receiver as `https://''${domain}/`, and the gateway routes - the path. So inserting a hive renumbering the ones after it is - harmless here: nginx is rendered from this same evaluation and - moves with it. - - The range still matters because every swarm container shares the - host's network namespace, so a derived port can land on one - another service already holds — with no bind error and nothing in - any log. The assertions below check this range against every port - this module and the hive tier declare — which is as far as a - module can see, since a port another module picks is not - knowable from here without reading its config. - ''; - }; - domain = lib.mkOption { type = lib.types.str; default = "otel.${domainBase}"; @@ -245,21 +203,14 @@ in # serves `/v1/metrics` and knows nothing about hives. Without it # the receiver sees `//v1/metrics` and answers 404 to a # request that authenticated perfectly. - lib.optionalAttrs hiveAuth ( - lib.mapAttrs' ( - h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } - ) hivePorts - ) + lib.mapAttrs' ( + h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } + ) hivePorts // { - "/" = - if hiveAuth then - # Not a proxy to a receiver that no longer exists. A closed - # door answering 404 is the honest description of this - # collector once ingest is per-hive: there is no - # swarm-wide inbox any more. - { return = "404"; } - else - { proxyPass = "http://127.0.0.1:${toString cfg.port}"; }; + # There is no swarm-wide inbox, and a closed door is the honest + # description of that. Every route into this collector belongs to + # exactly one hive. + "/".return = "404"; }; }; @@ -267,12 +218,12 @@ in # option exists to allow: its own description names this module as the # second consumer, so the queue is not a prerequisite for authenticated # telemetry. - services.hyperhive.swarm.authelia.oidc.hiveIdentities = lib.mkIf hiveAuth true; + services.hyperhive.swarm.authelia.oidc.hiveIdentities = true; # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. - systemd.services."container@${cfg.machine}" = lib.mkIf hiveAuth caTrust.containerOrdering; + systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; assertions = [ { @@ -293,15 +244,30 @@ in { # Without a roster there are no receivers at all, so this # collector would listen on nothing while looking configured. - assertion = !hiveAuth || hyperhiveCfg.swarm.hives != { }; + assertion = hyperhiveCfg.swarm.hives != { }; message = '' - services.hyperhive.swarm.otel.requireHiveIdentity is true but + services.hyperhive.swarm.otel.enable is true but services.hyperhive.swarm.hives is empty: ingest is authenticated - per hive, so an empty roster means this collector accepts - nothing from anyone. + per hive, so an empty roster means this collector accepts nothing + from anyone. - List the swarm's hives, or set requireHiveIdentity = false to - keep an unauthenticated receiver. + List the swarm's hives. + ''; + } + { + # A hive proves who it is with a token this provider mints, so + # there is no version of this collector that runs without one. + # Stated as an assertion rather than a fallback because a guessed + # issuer URL evaluates cleanly and refuses every hive at runtime. + assertion = autheliaCfg.url != null; + message = '' + services.hyperhive.swarm.otel.enable is true but + services.hyperhive.swarm.authelia.url is null: every hive + authenticates to this collector as itself, and the token comes + from the swarm's identity provider. + + Point authelia.url at the swarm's provider, or enable + services.hyperhive.swarm.authelia on the host that runs it. ''; } { @@ -309,28 +275,31 @@ in # coin toss with nothing in any log — the failure this whole # comment budget exists to prevent. Checked against every port # reachable from here; a port some other module picks is not. + # + # ⚠️ `cfg.port` is deliberately absent from `others`: it is the + # FIRST element of the derived range, so listing it would make this + # assertion fire on every config. assertion = let derived = lib.attrValues hivePorts; others = [ - cfg.port cfg.telemetryPort otelCfg.collector.port ] ++ lib.optional vmCfg.enable vmCfg.port; all = derived ++ others; in - !hiveAuth || lib.length (lib.unique all) == lib.length all; + lib.length (lib.unique all) == lib.length all; message = '' - services.hyperhive.swarm.otel: the per-hive receiver range - starting at hivePortBase (${toString cfg.hivePortBase}, one port - per hive in services.hyperhive.swarm.hives) overlaps another - port on this host. + services.hyperhive.swarm.otel: the receiver range starting at + port (${toString cfg.port}, one port per hive in + services.hyperhive.swarm.hives) overlaps another port on this + host. Every swarm container shares the host's network namespace, so two listeners claiming one port is not a build failure — it is - whichever process started first, silently. Move hivePortBase to - a free range. + whichever process started first, silently. Move + services.hyperhive.swarm.otel.port to a free range. ''; } ]; @@ -357,7 +326,7 @@ in } # The public hive CA, read-only — only when something in here # actually verifies a swarm-service name. - // lib.optionalAttrs hiveAuth caTrust.bindMount; + // caTrust.bindMount; config = { ... }: @@ -379,13 +348,13 @@ in # call fails while the unit looks healthy. That is why this is # the shared helper — it carries the `Requires` and the # non-empty check — and not a local `cat`. - ++ lib.optional hiveAuth ( - caTrust.trustBundle { + ++ [ + (caTrust.trustBundle { inherit pkgs; name = cfg.machine; consumers = [ "opentelemetry-collector" ]; - } - ); + }) + ]; system.stateVersion = config.system.stateVersion; networking.firewall.enable = false; @@ -404,8 +373,8 @@ in # real sample through both tiers into the store. validateConfigFile = true; settings = { - # One receiver per hive when ingest is authenticated, and - # that multiplicity is forced rather than chosen. The `hive` + # One receiver per hive, and that multiplicity is forced + # rather than chosen. The `hive` # label has to come from something the sender cannot write, # and the only such thing here is WHICH RECEIVER accepted # the sample: a processor cannot read the token's claims @@ -413,19 +382,15 @@ in # an auth claim yields nothing — silently, with a healthy # startup), and one receiver holding many credentials never # reveals which one matched. - receivers = - if hiveAuth then - lib.mapAttrs' ( - h: p: - lib.nameValuePair "otlp/${h}" { - protocols.http = { - endpoint = "127.0.0.1:${toString p}"; - auth.authenticator = "oidc/${h}"; - }; - } - ) hivePorts - else - { otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; }; + receivers = lib.mapAttrs' ( + h: p: + lib.nameValuePair "otlp/${h}" { + protocols.http = { + endpoint = "127.0.0.1:${toString p}"; + auth.authenticator = "oidc/${h}"; + }; + } + ) hivePorts; exporters = lib.optionalAttrs vmCfg.enable { @@ -473,7 +438,7 @@ in # INERT — the collector starts clean and the receiver # naming it authenticates nothing. Derived from the same # attrset as the receivers so the two cannot disagree. - service.extensions = lib.optionals hiveAuth (map (h: "oidc/${h}") (lib.attrNames hivePorts)); + service.extensions = map (h: "oidc/${h}") (lib.attrNames hivePorts); # Fan-out, not a choice: with both configured the same # samples go upstream AND into the swarm's store. The store @@ -482,25 +447,16 @@ in # `exporterNames` is shared by every pipeline — where a # sample goes is a property of this tier, not of the hive # that sent it. - service.pipelines = - if hiveAuth then - lib.mapAttrs' ( - h: _: - lib.nameValuePair "metrics/${h}" { - receivers = [ "otlp/${h}" ]; - processors = [ "resource/${h}" ]; - exporters = exporterNames; - } - ) hivePorts - else - { - metrics = { - receivers = [ "otlp" ]; - exporters = exporterNames; - }; - }; + service.pipelines = lib.mapAttrs' ( + h: _: + lib.nameValuePair "metrics/${h}" { + receivers = [ "otlp/${h}" ]; + processors = [ "resource/${h}" ]; + exporters = exporterNames; + } + ) hivePorts; } - // lib.optionalAttrs hiveAuth { + // { extensions = lib.mapAttrs' ( h: _: lib.nameValuePair "oidc/${h}" {