From 5478e0bf672e59bea5e3ef779ac733411462d270 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 31 Aug 2026 23:14:41 +0200 Subject: [PATCH] fix(#3554): push to the swarm's stores by domain, authenticated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collector's store exporters were gated on deploy.victoriametrics.enable / deploy.victorialogs.enable — options that say "this host RUNS the store", not "the swarm has one". A collector that did not share a host with the stores rendered no exporter at all and dropped everything it received, from every hive, silently: an absent exporter is not an error. mara ruled the premise rather than the mechanism ("the swarm always has a store"), so there is no gate and no new option for that. Both exporters are unconditional and address the stores by domain, per the cross-host rule from the OIDC client and secret-delivery unit #3517 already built. The logs exporter had the identical bug and is fixed here too. Both stores gained a machine ingest route, required in the same change: the exporter now targets https://logs./insert/..., and that vhost is browser-shaped, so shipping the collector half alone would have regressed logs ingestion that works today. Neither ingest location carries `error_page 401 =302` — a pusher handed a redirect follows it and POSTs at a login page, which answers 200. Whether the collector authenticates follows the CREDENTIAL, never another service's placement: `clientSecretFile` is a nullable option, and the delivery unit — the one thing here that may know where authelia runs, since it copies out of its container — sets it by mkDefault. An earlier revision gated this on deploy.authelia.enable directly, which put a different service's co-location in the collector's own config. Also removed rather than relaxed: the assertion that this collector has "somewhere to send". It read the store's per-host enable, so it rejected at eval exactly the deployment reaching the stores by domain exists for. Deliberately not replaced with an authentication assertion — a collector on a host of its own is a supported shape, and refusing to build it would make this fix illegal where the bug bites hardest. Knock-on worth review: collectLogs is now always satisfied, so journald collection is unconditional. Config shape validated against otelcol-contrib 0.151.0 `validate`, with a bogus-key control confirming the validator checks the extension schema. module-eval: 31 properties. --- nix/host-modules/swarm-otel.nix | 395 ++++++++++++--------- nix/host-modules/swarm-victorialogs.nix | 36 +- nix/host-modules/swarm-victoriametrics.nix | 59 ++- nix/module-eval.nix | 129 +++++++ 4 files changed, 442 insertions(+), 177 deletions(-) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 2f5cdbea..48b5a4c4 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -200,6 +200,28 @@ let # those subdirectories, so one reader covers the host and every container. hostJournalDir = "/var/log/journal"; + # Each store's OTLP route, by domain. One binding because the same string is + # both the address requested and the audience the token is minted for — two + # spellings present as a valid token refused at the store. + metricsPushUrl = "https://${vmCfg.domain}/opentelemetry/api/v1/push"; + logsPushUrl = "https://${vlCfg.domain}/insert/opentelemetry/v1/logs"; + + # Read by the extensions block, `service.extensions`, each exporter's + # authenticator and the OIDC client's audience list. Derived rather than + # repeated: an authenticator naming an unlisted extension starts clean and + # authenticates nothing. + pushAudiences = { + victoriametrics = metricsPushUrl; + victorialogs = logsPushUrl; + }; + pushAuthenticator = name: "oauth2client/${name}"; + + # Whether this collector holds a credential — a property of the credential, + # not of where any other service runs. Not an assertion: a collector on a + # host of its own is a supported shape, and refusing to build it would make + # this fix illegal where the bug bites hardest. + haveCollectorSecret = cfg.clientSecretFile != null; + # The operator-configured upstream, named once: the same exporter carries # every signal, so metrics and logs both reach it without a second # definition. @@ -210,20 +232,16 @@ let # exactly the same destinations as the single pipeline they replace. # Written once because "which exporters" is a property of this tier, not # of which hive a sample came from. - exporterNames = - upstreamExporters ++ lib.optional deployCfg.victoriametrics.enable "otlphttp/victoriametrics"; + # ⚠️ Unconditional: `deploy.victoriametrics.enable` says "this host RUNS the + # store", and a swarm has one either way. Gating on it left a collector + # elsewhere with no exporter at all, dropping everything silently. + exporterNames = upstreamExporters ++ [ "otlphttp/victoriametrics" ]; - # The same fan-out for logs, and the local store is only ONE of its - # destinations. A deployment that turns the swarm's log store off and keeps - # an upstream endpoint still collects — the store is where logs may be kept, - # not the reason to read the journal at all. - logExporterNames = - upstreamExporters ++ lib.optional deployCfg.victorialogs.enable "otlphttp/victorialogs"; + # Same fan-out for logs, unconditional for the same reason. + logExporterNames = upstreamExporters ++ [ "otlphttp/victorialogs" ]; - # Collect when there is anywhere to send it, and only then. A pipeline with - # an empty exporter list is not a quiet no-op — the collector rejects it — - # and reading the journal to drop it on the floor would be worse than not - # reading it. + # An empty exporter list is not a quiet no-op — the collector rejects it. + # Now always satisfied, as a consequence of the log store always existing. collectLogs = logExporterNames != [ ]; in { @@ -486,6 +504,23 @@ in after the change that caused it evaluated cleanly. ''; }; + + clientSecretFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-otel-oidc/swarm-collector.secret"; + description = '' + Path, inside this collector's container, to its OIDC client secret. + + `null` means it holds no credential: it still exports, and the + authenticated destinations refuse it. Set automatically where + authelia is co-located; a deployment that places authelia elsewhere + points this at a file it delivers itself. + + ⚠️ A path, never a value — a secret interpolated into a nix + expression renders world-readable into the store. + ''; + }; }; config = lib.mkIf (config.services.hyperhive.enable && deployCfg.swarm-otel.enable) { @@ -552,38 +587,40 @@ in # render nothing while reading as done; a swarm whose authelia lives # elsewhere registers it there. # - # ⚠️ Also conditional on there being a published target, and that is - # the shipped case rather than an edge — nothing declares one until - # some service publishes behind the gateway. Authelia refuses a - # bearer-authz client with no audience, so an unconditional - # declaration would break every hive that runs a collector and - # publishes nothing. - services.hyperhive.swarm.authelia.oidc.clients = - lib.mkIf (deployCfg.authelia.enable && cfg.publishedScrapeTargets != { }) - [ - { - id = cfg.clientId; - description = "HyperHive swarm collector"; - kind = "machine"; - # Grants `authelia.bearer.authz`, without which the authz - # endpoint refuses an otherwise valid token and blames the - # token rather than the missing grant. - bearerAuthz = true; - # DERIVED from the targets rather than contributed alongside - # them. A service declares a URL once and this is the - # permission to reach it; two lists that had to agree would be - # a drift to maintain, and the failure mode is the quiet one — - # a target whose audience was forgotten authenticates against - # nothing and looks like a broken scrape. - audience = lib.attrValues cfg.publishedScrapeTargets; - # Stated rather than left on authelia's default, because the - # two agreeing today is not the same as this being the - # required value: authelia permits only basic / JWT methods - # for a confidential client holding that scope, and enforces - # it in the startup validator. - tokenEndpointAuthMethod = "client_secret_basic"; - } - ]; + # ⚠️ NO LONGER conditional on a published scrape target. Authelia refuses + # a bearer-authz client with no audience, which is what that guard was + # for — and the push audiences below are unconditional, so there is now + # always at least one. Keeping the old guard would have left a collector + # that scrapes nothing pushing to the stores with no client to get a + # token from. + services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf deployCfg.authelia.enable [ + { + id = cfg.clientId; + description = "HyperHive swarm collector"; + kind = "machine"; + # Grants `authelia.bearer.authz`, without which the authz + # endpoint refuses an otherwise valid token and blames the + # token rather than the missing grant. + bearerAuthz = true; + # DERIVED from the targets rather than contributed alongside + # them. A service declares a URL once and this is the + # permission to reach it; two lists that had to agree would be + # a drift to maintain, and the failure mode is the quiet one — + # a target whose audience was forgotten authenticates against + # nothing and looks like a broken scrape. + # + # Both directions land in one list because authelia has one: what + # this collector may SCRAPE and what it may PUSH TO are the same + # kind of permission, differing only in who initiates. + audience = lib.attrValues cfg.publishedScrapeTargets ++ lib.attrValues pushAudiences; + # Stated rather than left on authelia's default, because the + # two agreeing today is not the same as this being the + # required value: authelia permits only basic / JWT methods + # for a confidential client holding that scope, and enforces + # it in the startup validator. + tokenEndpointAuthMethod = "client_secret_basic"; + } + ]; # Deliver the collector's client secret from authelia's container into # this one. On the HOST because that is the only place both container @@ -596,51 +633,57 @@ in # make the collector wait on a file that waits on a container that starts # after it. On a fresh swarm that is a permanent stall presenting as # "metrics are broken", several layers from its cause. - systemd.services.swarm-otel-oidc-secret = - lib.mkIf (deployCfg.authelia.enable && cfg.publishedScrapeTargets != { }) - { - description = "deliver the swarm collector's OIDC client secret from authelia"; - after = [ "container@${autheliaCfg.machine}.service" ]; - requires = [ "container@${autheliaCfg.machine}.service" ]; - before = [ "container@${cfg.machine}.service" ]; - wantedBy = [ "container@${cfg.machine}.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "swarm-otel-oidc-secret"; - # Longer than the bounded wait below, and that is the point: - # `DefaultTimeoutStartSec` is 90s, so without this systemd kills - # the unit before it can emit the message naming the file it was - # waiting for — the failure then reads as a timeout with no cause. - TimeoutStartSec = "180s"; - }; - path = [ pkgs.coreutils ]; - script = '' - set -euo pipefail + # The delivery unit below is the one thing here that may know where + # authelia runs — it copies out of its container — so it is also what + # names the file. `mkDefault`, so a deployment that delivers the secret + # some other way just sets the option. + services.hyperhive.swarm.otel.clientSecretFile = lib.mkIf deployCfg.authelia.enable ( + lib.mkDefault collectorSecretInContainer + ); - src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"} - dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"} + systemd.services.swarm-otel-oidc-secret = lib.mkIf deployCfg.authelia.enable { + description = "deliver the swarm collector's OIDC client secret from authelia"; + after = [ "container@${autheliaCfg.machine}.service" ]; + requires = [ "container@${autheliaCfg.machine}.service" ]; + before = [ "container@${cfg.machine}.service" ]; + wantedBy = [ "container@${cfg.machine}.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-otel-oidc-secret"; + # Longer than the bounded wait below, and that is the point: + # `DefaultTimeoutStartSec` is 90s, so without this systemd kills + # the unit before it can emit the message naming the file it was + # waiting for — the failure then reads as a timeout with no cause. + TimeoutStartSec = "180s"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail - # authelia's container is up, but its first-boot generator may - # still be minting. Bounded wait, then fail: skipping silently - # produces a collector whose scrape gets a 401 forever, which is - # the failure this whole design exists to make impossible. - for _ in $(seq 1 60); do - [ -s "$src" ] && break - sleep 2 - done - if [ ! -s "$src" ]; then - echo "authelia has not minted $src after 120s" >&2 - exit 1 - fi + src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"} + dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"} - # root-owned 0400. The collector runs under `DynamicUser`, so - # there is no uid to give it to — `LoadCredential` reads this as - # root before the sandbox exists and re-exposes it to whichever - # uid the unit got. - install -D -m 0400 -o root -g root "$src" "$dst" - ''; - }; + # authelia's container is up, but its first-boot generator may + # still be minting. Bounded wait, then fail: skipping silently + # produces a collector whose scrape gets a 401 forever, which is + # the failure this whole design exists to make impossible. + for _ in $(seq 1 60); do + [ -s "$src" ] && break + sleep 2 + done + if [ ! -s "$src" ]; then + echo "authelia has not minted $src after 120s" >&2 + exit 1 + fi + + # root-owned 0400. The collector runs under `DynamicUser`, so + # there is no uid to give it to — `LoadCredential` reads this as + # root before the sandbox exists and re-exposes it to whichever + # uid the unit got. + install -D -m 0400 -o root -g root "$src" "$dst" + ''; + }; # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount @@ -648,21 +691,10 @@ in systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; assertions = [ - { - # The tier exists to hold the upstream credential and to write the - # swarm's store. With neither, it is a process that receives - # samples and drops them — which looks healthy and loses data. - assertion = otelCfg.endpoint != "" || deployCfg.victoriametrics.enable; - message = '' - services.hyperhive.deploy.swarm-otel.enable is true but this collector - has nowhere to send what it receives: - services.hyperhive.otel.endpoint is empty and - services.hyperhive.deploy.victoriametrics.enable is false. - - Set the endpoint to export upstream, or enable the swarm's - metrics store. - ''; - } + # ⚠️ An assertion that this collector has "somewhere to send" was REMOVED + # rather than relaxed: it read the store's *per-host* enable, so it + # rejected at eval the very deployment the stores are reached by domain + # for — a collector on its own host never built. { # An empty allowlist is not an empty collection: the receiver # renders no unit filter at all and reads the host's entire @@ -1069,64 +1101,70 @@ in }; }; - exporters = - lib.optionalAttrs deployCfg.victoriametrics.enable { - # `metrics_endpoint`, NOT `endpoint`: the latter is a - # base that otlphttp appends `/v1/metrics` to, while - # VictoriaMetrics serves OTLP at - # `/opentelemetry/api/v1/push`. With `endpoint` the - # collector answers 200 to its own clients and posts the - # samples to a path that does not exist. Measured - # end-to-end, not read — `state/probe-3265-collector-to-vm.sh`. - "otlphttp/victoriametrics".metrics_endpoint = - "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push"; + exporters = { + # `metrics_endpoint`, NOT `endpoint`: the latter is a + # base that otlphttp appends `/v1/metrics` to, while + # VictoriaMetrics serves OTLP at + # `/opentelemetry/api/v1/push`. With `endpoint` the + # collector answers 200 to its own clients and posts the + # samples to a path that does not exist. Measured + # end-to-end, not read — `state/probe-3265-collector-to-vm.sh`. + "otlphttp/victoriametrics" = { + metrics_endpoint = metricsPushUrl; } - // lib.optionalAttrs (otelCfg.endpoint != "") { - ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = { - endpoint = otelCfg.endpoint; - } - // lib.optionalAttrs (otelCfg.headersCredential != null) { - # Interpolated by the collector from its environment at - # runtime, never by nix: `EnvironmentFile` below is what - # puts it there, so the value is not read into the store. - headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; - } - // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; - } - // lib.optionalAttrs deployCfg.victorialogs.enable { - # `logs_endpoint`, NOT `endpoint`, for exactly the reason the - # metrics exporter above spells out — and the trap is worse - # here, because the two stores' OTLP routes differ. `endpoint` - # is a base the exporter appends `/v1/logs` to; VictoriaLogs - # serves `/insert/opentelemetry/v1/logs`, which is also not - # the metrics store's `/opentelemetry/api/v1/push`. Both - # spellings pass `otelcol validate`. - # - # ⚠️ And the store cannot tell you either: a right path, a - # wrong path and a nonsense path all answer 400. Only its log - # distinguishes them — the real route complains about the - # encoding, everything else says "unsupported path requested". - # 🔑 THE TWO QUERY PARAMETERS ARE NOT TUNING — without the - # first one this pipeline fills the store with records that - # cannot be searched by message text, which is the only - # reason to collect logs at all. - # - # The journald receiver leaves the OTLP *body* empty and - # carries the entry as a map of journal fields, so the store - # has no message to index and writes the literal placeholder - # `missing _msg field` into `_msg` on EVERY record. Nothing - # errors: ingest returns 200, the data is all present, and a - # plain search for a line that is sitting right there returns - # nothing. `_msg_field` tells the store which field carries - # the message. Measured, both with and without. - # - # `_stream_fields` is the difference between one enormous - # stream for the whole host and one per unit per machine — - # both low-cardinality, and both fields journald sets itself. - "otlphttp/victorialogs".logs_endpoint = - "http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs" - + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT"; + // lib.optionalAttrs haveCollectorSecret { + auth.authenticator = pushAuthenticator "victoriametrics"; }; + } + // lib.optionalAttrs (otelCfg.endpoint != "") { + ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = { + endpoint = otelCfg.endpoint; + } + // lib.optionalAttrs (otelCfg.headersCredential != null) { + # Interpolated by the collector from its environment at + # runtime, never by nix: `EnvironmentFile` below is what + # puts it there, so the value is not read into the store. + headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; + } + // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; + } + // { + # `logs_endpoint`, NOT `endpoint`, for exactly the reason the + # metrics exporter above spells out — and the trap is worse + # here, because the two stores' OTLP routes differ. `endpoint` + # is a base the exporter appends `/v1/logs` to; VictoriaLogs + # serves `/insert/opentelemetry/v1/logs`, which is also not + # the metrics store's `/opentelemetry/api/v1/push`. Both + # spellings pass `otelcol validate`. + # + # ⚠️ And the store cannot tell you either: a right path, a + # wrong path and a nonsense path all answer 400. Only its log + # distinguishes them — the real route complains about the + # encoding, everything else says "unsupported path requested". + # 🔑 THE TWO QUERY PARAMETERS ARE NOT TUNING — without the + # first one this pipeline fills the store with records that + # cannot be searched by message text, which is the only + # reason to collect logs at all. + # + # The journald receiver leaves the OTLP *body* empty and + # carries the entry as a map of journal fields, so the store + # has no message to index and writes the literal placeholder + # `missing _msg field` into `_msg` on EVERY record. Nothing + # errors: ingest returns 200, the data is all present, and a + # plain search for a line that is sitting right there returns + # nothing. `_msg_field` tells the store which field carries + # the message. Measured, both with and without. + # + # `_stream_fields` is the difference between one enormous + # stream for the whole host and one per unit per machine — + # both low-cardinality, and both fields journald sets itself. + "otlphttp/victorialogs" = { + logs_endpoint = logsPushUrl + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT"; + } + // lib.optionalAttrs haveCollectorSecret { + auth.authenticator = pushAuthenticator "victorialogs"; + }; + }; # Moves this collector's self-metrics off the built-in # default of `localhost:8888`, which the hive tier holds. @@ -1155,7 +1193,11 @@ in # same reasoning as its receiver above: `otlp/${swarmTierName}` # always exists, so the authenticator it names must too, or an # extension-not-listed startup failure follows every deploy. - ++ [ "oidc/${swarmTierName}" ]; + ++ [ "oidc/${swarmTierName}" ] + # The push side's authenticators. Same rule as above: one an + # exporter names but this list omits is INERT — the collector + # starts clean and pushes unauthenticated. + ++ lib.optionals haveCollectorSecret (map pushAuthenticator (lib.attrNames pushAudiences)); # Fan-out, not a choice: with both configured the same # samples go upstream AND into the swarm's store. The store @@ -1269,7 +1311,27 @@ in issuer_url = autheliaCfg.url; audience = config.services.hyperhive.swarm.controller.queueClientId; }; - }; + } + # The push side. Opposite direction to every `oidc/*` above — + # those VALIDATE a token arriving; these OBTAIN one to send. + # Hence `oauth2client` rather than `oidc`, and hence a client + # id and secret rather than an issuer and an audience to check. + // lib.optionalAttrs haveCollectorSecret ( + lib.mapAttrs' ( + name: url: + lib.nameValuePair (pushAuthenticator name) { + client_id = cfg.clientId; + # A path, never a value. Same file the scrape side reads. + client_secret_file = collectorSecretPath; + token_url = "${autheliaCfg.url}/api/oidc/token"; + # Registered is not requested: a client that does not ASK + # for the scope gets a token carrying none, and one that + # does not ask for an audience gets `aud: []`. + scopes = [ "authelia.bearer.authz" ]; + endpoint_params.audience = url; + } + ) pushAudiences + ); # `upsert`, not `insert`: a sender that stamps its own # `hive` must be OVERWRITTEN, not deferred to. This @@ -1346,12 +1408,15 @@ in # authenticates this collector's export onward, this one # authenticates it to a service it scrapes. # - # ⚠️ Only where a published target exists. `LoadCredential` on a - # missing source is a unit that refuses to start, so declaring it - # unconditionally would take the collector down on every hive that - # scrapes nothing published — the empty case is the shipped one. - // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) { - LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ]; + # ⚠️ Gated on the secret existing, not on what it is used for. + # `LoadCredential` on a missing source is a unit that refuses to + # start, and the option that names this file is only set where + # something delivers it — so this follows that condition + # exactly rather than restating a narrower one. Where it is false + # no authenticator is rendered either, so the collector starts and + # is refused by the stores rather than failing to start. + // lib.optionalAttrs haveCollectorSecret { + LoadCredential = [ "${collectorCredentialId}:${cfg.clientSecretFile}" ]; }; }; }; diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index bdadcbcb..b9d8c334 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -18,9 +18,12 @@ # rationale (forceSSL is load-bearing there too: authelia answers a plain-http # auth subrequest with 400, which `auth_request` cannot interpret as anything # but a broken check). The store's own listener stays loopback-only and -# unauthenticated exactly as before — the collector still writes to it -# directly, never through this vhost — so this adds a new authenticated front -# door without touching the existing write path at all. +# unauthenticated, and the vhost is now the ONLY way in from outside: the +# collector pushes through it too, at its own `= /insert/...` location, because +# a swarm has one log store and the collector need not share a host with it. +# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a +# pusher handed its `error_page 401 =302` follows the redirect and POSTs at a +# login page, which answers 200. See the location itself. { pkgs, lib, @@ -177,6 +180,27 @@ in proxyPass = "http://127.0.0.1:${toString cfg.port}/"; extraConfig = swarmAuthRequest; }; + + # The swarm collector's ingest route. `=` so it outranks the `/` + # prefix above — without a more specific location it would ride that + # catch-all, and that is the whole hazard here. + # + # ⚠️ Deliberately NOT `swarmAuthRequest`. That block ends in + # `error_page 401 =302`, which is right for a browser and wrong for a + # pusher: handed a redirect it follows the redirect and POSTs its + # batch at a login page, which answers 200. Ingest then reports + # healthy while storing nothing. A machine route lets the 401 reach + # the client unchanged. + # + # `auth_request` does not inherit across sibling locations (see the + # note on `swarmAuthRequest` above), so naming it here is required + # rather than redundant. + "= /insert/opentelemetry/v1/logs" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/insert/opentelemetry/v1/logs"; + extraConfig = '' + auth_request /__hive_authelia; + ''; + }; # The subrequest itself — same target, same header set, same # reasoning as `swarm-ui.nix`'s own copy (measured against the # pinned authelia binary, not copied from an example). @@ -261,6 +285,8 @@ in # code is not evidence. # # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is - # why `listenAddress` above is loopback, why the collector is the only - # writer, and why this module declares no gateway vhost. + # why `listenAddress` above is loopback and why nothing reaches it except + # through the gateway, where the ingest route is authenticated. (This module + # does declare a vhost; the line that used to say otherwise was already + # wrong before the collector started pushing through it.) } diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index e9b7dd29..31f24ade 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -21,6 +21,7 @@ let networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; + autheliaCfg = hyperhiveCfg.swarm.authelia; swarmDomain = hyperhiveCfg.swarm.domain; # Total on a null swarm domain for the same reason every sibling module is: @@ -123,17 +124,58 @@ in # named it runs, which is what keeps scraper and target on one host by # construction instead of by luck. # - # The loopback literal introduces no new assumption — it is the address - # this module already pins the listener to, and the same one the - # collector's `otlphttp/victoriametrics` exporter already writes to. If - # that reach is ever wrong, it is wrong for the write path first. + # The loopback literal is safe for this option and NOT for the write path, + # which is the distinction that matters now that the two differ. A scrape + # target is only ever read by a collector on this host, so loopback states + # a fact. The collector's push goes to the swarm name through the gateway, + # because the collector need not be here at all. services.hyperhive.swarm.otel.scrapeTargets.victoriametrics = "127.0.0.1:${toString cfg.port}"; services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; - locations."/" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}/"; + locations = { + "/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/"; + }; + + # The swarm collector's ingest route, and the only authenticated thing + # on this vhost. `=` so it outranks the `/` prefix above, which would + # otherwise carry these writes with no check at all. + # + # ⚠️ No `error_page 401 =302` here, and its absence is the point: a + # redirect is right for a browser and wrong for a pusher, which would + # follow it and POST its batch at a login page that answers 200 — + # ingest reporting healthy while storing nothing. + "= /opentelemetry/api/v1/push" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/opentelemetry/api/v1/push"; + extraConfig = '' + auth_request /__metrics_push_authz; + ''; + }; + + # The subrequest. Same target and header set as the sibling log + # store's, which took them from `swarm-ui.nix` — `X-Original-URL` and + # `X-Original-Method` are what authelia's auth-request implementation + # reads, and the address it compares the token's audience against. + "= /__metrics_push_authz" = { + proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request"; + # nixpkgs appends its OWN `Host $host` after extraConfig, which + # would override verifiedProxyTo's — see the comment on + # verifiedProxyTo in hive-gateway/vhost-lib.nix. + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; }; }; @@ -192,5 +234,8 @@ in # deployment ever needs them. # # ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is - # loopback and why the collector — not agents — is the writer. + # loopback: the only route to it from off-host is the vhost's ingest + # location, which authenticates. The collector is still the only writer, but + # it now arrives by the swarm name rather than over loopback, because it need + # not share a host with this store. } diff --git a/nix/module-eval.nix b/nix/module-eval.nix index b3cb38ab..3cd84707 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -145,6 +145,28 @@ let lib.findFirst (c: c.job_name == job) null machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; + # A swarm collector on a host that runs NEITHER store — the fully-spread + # shape from docs/swarm/services.md, and the one the old per-host gates made + # inexpressible. It is the whole point of the cases below that this hive is + # not a degenerate configuration but a supported one. + otelNoStores = hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + deploy.victoriametrics.enable = false; + deploy.victorialogs.enable = false; + }; + otelSettings = + machine: machine.containers.swarm-otel.config.services.opentelemetry-collector.settings; + + # authelia somewhere else, the credential delivered by hand. Whether this + # collector authenticates must follow the credential, never another + # service's placement. + otelRemoteAuthelia = hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = false; + swarm.otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret"; + }; + # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is @@ -208,6 +230,113 @@ let name = "the CI container's unit definitions merge without a priority collision"; ok = forceCiServiceConfigs; } + { + # The defect itself. These exporters used to be gated on the stores' + # PER-HOST enables, so a collector that did not share a host with them + # rendered none at all and dropped everything it received, from every + # hive — silently, because an absent exporter is not an error. + name = "a collector that hosts neither store still exports to both"; + ok = + let + e = (otelSettings otelNoStores).exporters; + in + (e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs"); + } + { + # A swarm has one of each store, so the address is a swarm-level name. + # A loopback literal here is the co-location assumption written back in, + # and it renders, deploys and reports healthy while reaching nothing. + name = "the store exporters address the stores by name, never by loopback"; + ok = + let + e = (otelSettings otelNoStores).exporters; + m = e."otlphttp/victoriametrics".metrics_endpoint; + l = e."otlphttp/victorialogs".logs_endpoint; + in + !(lib.hasInfix "127.0.0.1" m) + && !(lib.hasInfix "127.0.0.1" l) + && lib.hasInfix "metrics.t.local" m + && lib.hasInfix "logs.t.local" l; + } + { + # The collector reaches these routes through the gateway now, so each + # store needs an ingest location of its own. Without one the write rides + # the `/` catch-all: unauthenticated on the metrics store, and into a + # browser redirect on the log store. + name = "each store's vhost has an authenticated ingest location"; + ok = + let + v = allLocal.services.nginx.virtualHosts; + m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null; + l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null; + in + m != null + && l != null + && lib.hasInfix "auth_request" m.extraConfig + && lib.hasInfix "auth_request" l.extraConfig; + } + { + # The arm that actually protects something. A pusher handed + # `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page, + # which answers 200 — ingest reporting healthy while storing nothing. + # The third clause is the positive control: the log store's browser + # location really does redirect, so this says the machine routes differ + # rather than that the string is absent from the whole file. + name = "the ingest locations answer 401 instead of redirecting a pusher"; + ok = + let + v = allLocal.services.nginx.virtualHosts; + m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig; + l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig; + browser = v."logs.t.local".locations."/".extraConfig; + in + !(lib.hasInfix "error_page" m) + && !(lib.hasInfix "error_page" l) + && lib.hasInfix "error_page" browser; + } + { + # Defining an exporter and REFERENCING it are two separate lists, and + # the second is where the original gate also lived. An exporter no + # pipeline names is as silent as one that does not exist — this case + # exists because a mutation that restored only the reference-side gate + # left every other case here green. + name = "every pipeline that has a store exporter defined actually sends to it"; + ok = + let + s = otelSettings otelNoStores; + used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines)); + in + builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used; + } + { + # The collector authenticates because it HOLDS a credential, not because + # authelia happens to share its host. Gating on the other service's + # placement renders a collector that pushes unauthenticated wherever + # authelia lives elsewhere — one of the supported shapes. + name = "a collector with a hand-delivered secret authenticates without authelia beside it"; + ok = + let + s = otelSettings otelRemoteAuthelia; + in + (s.exporters."otlphttp/victoriametrics" ? auth) + && builtins.elem "oauth2client/victoriametrics" s.service.extensions; + } + { + # An authenticator an exporter names but `service.extensions` omits is + # INERT — the collector starts clean and pushes unauthenticated until + # something at the far end refuses it. Checked as a set relation rather + # than by naming the two, so it keeps holding for exporters not written + # yet. + name = "every exporter authenticator is listed in service.extensions"; + ok = + let + s = otelSettings otelNoStores; + named = lib.filter (v: v != null) ( + lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters + ); + in + named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; + } { # The store's seal is spread over six gates — the stanza, the # provisioning unit, two bind mounts, a device and an EnvironmentFile.