diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 34935a42..3ce77cf6 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -51,7 +51,7 @@ neither is a renaming of the other. | authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | | OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service, `otel.clientSecretFile` for the hive's telemetry collector | | OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | -| the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `swarm.otel.clientSecretFile` — the collector need not share a host with authelia | +| the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | n/a — this collector runs on the swarm's service host, beside authelia | | authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | | wireguard private key | **the operator** — `wg genkey` | whatever `swarm.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | | queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `nats.calloutUserSeedFile` / `nats.calloutIssuerSeedFile` | @@ -77,16 +77,11 @@ placed by `swarm-otel-oidc-secret.service` and lands at homeserver rows below, and for the same reason: the container that mints the secret is not the container that reads it. -The copy is only made when authelia is enabled **on this host** and something -published is being scraped; otherwise no secret is needed and none is placed. - -⚠️ **Do not read that delivery unit as the only way this collector is fed.** -Whether it authenticates follows the *credential*, never another service's -placement: a swarm collector may run on a host that holds neither store and no -authelia, and then the secret is an ordinary operator-provided file named in -`services.hyperhive.swarm.otel.clientSecretFile` — the same shape as the hive -collector's row above. The copy unit is the convenience for the co-located case, -not the definition of the case. +There is no operator-provided variant of that one, and that is a property of +where it runs rather than an omission: the swarm's collector lives on the host +that runs the swarm's services, which is the host that runs authelia. The copy +is only made when authelia is enabled here and something published is being +scraped; otherwise no secret is needed and none is placed. ### Minting the queue's callout nkeys diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 48b5a4c4..2f5cdbea 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -200,28 +200,6 @@ let # those subdirectories, so one reader covers the host and every container. hostJournalDir = "/var/log/journal"; - # Each store's OTLP route, by domain. One binding because the same string is - # both the address requested and the audience the token is minted for — two - # spellings present as a valid token refused at the store. - metricsPushUrl = "https://${vmCfg.domain}/opentelemetry/api/v1/push"; - logsPushUrl = "https://${vlCfg.domain}/insert/opentelemetry/v1/logs"; - - # Read by the extensions block, `service.extensions`, each exporter's - # authenticator and the OIDC client's audience list. Derived rather than - # repeated: an authenticator naming an unlisted extension starts clean and - # authenticates nothing. - pushAudiences = { - victoriametrics = metricsPushUrl; - victorialogs = logsPushUrl; - }; - pushAuthenticator = name: "oauth2client/${name}"; - - # Whether this collector holds a credential — a property of the credential, - # not of where any other service runs. Not an assertion: a collector on a - # host of its own is a supported shape, and refusing to build it would make - # this fix illegal where the bug bites hardest. - haveCollectorSecret = cfg.clientSecretFile != null; - # The operator-configured upstream, named once: the same exporter carries # every signal, so metrics and logs both reach it without a second # definition. @@ -232,16 +210,20 @@ let # exactly the same destinations as the single pipeline they replace. # Written once because "which exporters" is a property of this tier, not # of which hive a sample came from. - # ⚠️ Unconditional: `deploy.victoriametrics.enable` says "this host RUNS the - # store", and a swarm has one either way. Gating on it left a collector - # elsewhere with no exporter at all, dropping everything silently. - exporterNames = upstreamExporters ++ [ "otlphttp/victoriametrics" ]; + exporterNames = + upstreamExporters ++ lib.optional deployCfg.victoriametrics.enable "otlphttp/victoriametrics"; - # Same fan-out for logs, unconditional for the same reason. - logExporterNames = upstreamExporters ++ [ "otlphttp/victorialogs" ]; + # The same fan-out for logs, and the local store is only ONE of its + # destinations. A deployment that turns the swarm's log store off and keeps + # an upstream endpoint still collects — the store is where logs may be kept, + # not the reason to read the journal at all. + logExporterNames = + upstreamExporters ++ lib.optional deployCfg.victorialogs.enable "otlphttp/victorialogs"; - # An empty exporter list is not a quiet no-op — the collector rejects it. - # Now always satisfied, as a consequence of the log store always existing. + # Collect when there is anywhere to send it, and only then. A pipeline with + # an empty exporter list is not a quiet no-op — the collector rejects it — + # and reading the journal to drop it on the floor would be worse than not + # reading it. collectLogs = logExporterNames != [ ]; in { @@ -504,23 +486,6 @@ in after the change that caused it evaluated cleanly. ''; }; - - clientSecretFile = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - example = "/var/lib/swarm-otel-oidc/swarm-collector.secret"; - description = '' - Path, inside this collector's container, to its OIDC client secret. - - `null` means it holds no credential: it still exports, and the - authenticated destinations refuse it. Set automatically where - authelia is co-located; a deployment that places authelia elsewhere - points this at a file it delivers itself. - - ⚠️ A path, never a value — a secret interpolated into a nix - expression renders world-readable into the store. - ''; - }; }; config = lib.mkIf (config.services.hyperhive.enable && deployCfg.swarm-otel.enable) { @@ -587,40 +552,38 @@ in # render nothing while reading as done; a swarm whose authelia lives # elsewhere registers it there. # - # ⚠️ NO LONGER conditional on a published scrape target. Authelia refuses - # a bearer-authz client with no audience, which is what that guard was - # for — and the push audiences below are unconditional, so there is now - # always at least one. Keeping the old guard would have left a collector - # that scrapes nothing pushing to the stores with no client to get a - # token from. - services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf deployCfg.authelia.enable [ - { - id = cfg.clientId; - description = "HyperHive swarm collector"; - kind = "machine"; - # Grants `authelia.bearer.authz`, without which the authz - # endpoint refuses an otherwise valid token and blames the - # token rather than the missing grant. - bearerAuthz = true; - # DERIVED from the targets rather than contributed alongside - # them. A service declares a URL once and this is the - # permission to reach it; two lists that had to agree would be - # a drift to maintain, and the failure mode is the quiet one — - # a target whose audience was forgotten authenticates against - # nothing and looks like a broken scrape. - # - # Both directions land in one list because authelia has one: what - # this collector may SCRAPE and what it may PUSH TO are the same - # kind of permission, differing only in who initiates. - audience = lib.attrValues cfg.publishedScrapeTargets ++ lib.attrValues pushAudiences; - # Stated rather than left on authelia's default, because the - # two agreeing today is not the same as this being the - # required value: authelia permits only basic / JWT methods - # for a confidential client holding that scope, and enforces - # it in the startup validator. - tokenEndpointAuthMethod = "client_secret_basic"; - } - ]; + # ⚠️ Also conditional on there being a published target, and that is + # the shipped case rather than an edge — nothing declares one until + # some service publishes behind the gateway. Authelia refuses a + # bearer-authz client with no audience, so an unconditional + # declaration would break every hive that runs a collector and + # publishes nothing. + services.hyperhive.swarm.authelia.oidc.clients = + lib.mkIf (deployCfg.authelia.enable && cfg.publishedScrapeTargets != { }) + [ + { + id = cfg.clientId; + description = "HyperHive swarm collector"; + kind = "machine"; + # Grants `authelia.bearer.authz`, without which the authz + # endpoint refuses an otherwise valid token and blames the + # token rather than the missing grant. + bearerAuthz = true; + # DERIVED from the targets rather than contributed alongside + # them. A service declares a URL once and this is the + # permission to reach it; two lists that had to agree would be + # a drift to maintain, and the failure mode is the quiet one — + # a target whose audience was forgotten authenticates against + # nothing and looks like a broken scrape. + audience = lib.attrValues cfg.publishedScrapeTargets; + # Stated rather than left on authelia's default, because the + # two agreeing today is not the same as this being the + # required value: authelia permits only basic / JWT methods + # for a confidential client holding that scope, and enforces + # it in the startup validator. + tokenEndpointAuthMethod = "client_secret_basic"; + } + ]; # Deliver the collector's client secret from authelia's container into # this one. On the HOST because that is the only place both container @@ -633,57 +596,51 @@ in # make the collector wait on a file that waits on a container that starts # after it. On a fresh swarm that is a permanent stall presenting as # "metrics are broken", several layers from its cause. - # The delivery unit below is the one thing here that may know where - # authelia runs — it copies out of its container — so it is also what - # names the file. `mkDefault`, so a deployment that delivers the secret - # some other way just sets the option. - services.hyperhive.swarm.otel.clientSecretFile = lib.mkIf deployCfg.authelia.enable ( - lib.mkDefault collectorSecretInContainer - ); + systemd.services.swarm-otel-oidc-secret = + lib.mkIf (deployCfg.authelia.enable && cfg.publishedScrapeTargets != { }) + { + description = "deliver the swarm collector's OIDC client secret from authelia"; + after = [ "container@${autheliaCfg.machine}.service" ]; + requires = [ "container@${autheliaCfg.machine}.service" ]; + before = [ "container@${cfg.machine}.service" ]; + wantedBy = [ "container@${cfg.machine}.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-otel-oidc-secret"; + # Longer than the bounded wait below, and that is the point: + # `DefaultTimeoutStartSec` is 90s, so without this systemd kills + # the unit before it can emit the message naming the file it was + # waiting for — the failure then reads as a timeout with no cause. + TimeoutStartSec = "180s"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail - systemd.services.swarm-otel-oidc-secret = lib.mkIf deployCfg.authelia.enable { - description = "deliver the swarm collector's OIDC client secret from authelia"; - after = [ "container@${autheliaCfg.machine}.service" ]; - requires = [ "container@${autheliaCfg.machine}.service" ]; - before = [ "container@${cfg.machine}.service" ]; - wantedBy = [ "container@${cfg.machine}.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "swarm-otel-oidc-secret"; - # Longer than the bounded wait below, and that is the point: - # `DefaultTimeoutStartSec` is 90s, so without this systemd kills - # the unit before it can emit the message naming the file it was - # waiting for — the failure then reads as a timeout with no cause. - TimeoutStartSec = "180s"; - }; - path = [ pkgs.coreutils ]; - script = '' - set -euo pipefail + src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"} + dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"} - src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"} - dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"} + # authelia's container is up, but its first-boot generator may + # still be minting. Bounded wait, then fail: skipping silently + # produces a collector whose scrape gets a 401 forever, which is + # the failure this whole design exists to make impossible. + for _ in $(seq 1 60); do + [ -s "$src" ] && break + sleep 2 + done + if [ ! -s "$src" ]; then + echo "authelia has not minted $src after 120s" >&2 + exit 1 + fi - # authelia's container is up, but its first-boot generator may - # still be minting. Bounded wait, then fail: skipping silently - # produces a collector whose scrape gets a 401 forever, which is - # the failure this whole design exists to make impossible. - for _ in $(seq 1 60); do - [ -s "$src" ] && break - sleep 2 - done - if [ ! -s "$src" ]; then - echo "authelia has not minted $src after 120s" >&2 - exit 1 - fi - - # root-owned 0400. The collector runs under `DynamicUser`, so - # there is no uid to give it to — `LoadCredential` reads this as - # root before the sandbox exists and re-exposes it to whichever - # uid the unit got. - install -D -m 0400 -o root -g root "$src" "$dst" - ''; - }; + # root-owned 0400. The collector runs under `DynamicUser`, so + # there is no uid to give it to — `LoadCredential` reads this as + # root before the sandbox exists and re-exposes it to whichever + # uid the unit got. + install -D -m 0400 -o root -g root "$src" "$dst" + ''; + }; # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount @@ -691,10 +648,21 @@ in systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; assertions = [ - # ⚠️ An assertion that this collector has "somewhere to send" was REMOVED - # rather than relaxed: it read the store's *per-host* enable, so it - # rejected at eval the very deployment the stores are reached by domain - # for — a collector on its own host never built. + { + # The tier exists to hold the upstream credential and to write the + # swarm's store. With neither, it is a process that receives + # samples and drops them — which looks healthy and loses data. + assertion = otelCfg.endpoint != "" || deployCfg.victoriametrics.enable; + message = '' + services.hyperhive.deploy.swarm-otel.enable is true but this collector + has nowhere to send what it receives: + services.hyperhive.otel.endpoint is empty and + services.hyperhive.deploy.victoriametrics.enable is false. + + Set the endpoint to export upstream, or enable the swarm's + metrics store. + ''; + } { # An empty allowlist is not an empty collection: the receiver # renders no unit filter at all and reads the host's entire @@ -1101,70 +1069,64 @@ in }; }; - exporters = { - # `metrics_endpoint`, NOT `endpoint`: the latter is a - # base that otlphttp appends `/v1/metrics` to, while - # VictoriaMetrics serves OTLP at - # `/opentelemetry/api/v1/push`. With `endpoint` the - # collector answers 200 to its own clients and posts the - # samples to a path that does not exist. Measured - # end-to-end, not read — `state/probe-3265-collector-to-vm.sh`. - "otlphttp/victoriametrics" = { - metrics_endpoint = metricsPushUrl; + exporters = + lib.optionalAttrs deployCfg.victoriametrics.enable { + # `metrics_endpoint`, NOT `endpoint`: the latter is a + # base that otlphttp appends `/v1/metrics` to, while + # VictoriaMetrics serves OTLP at + # `/opentelemetry/api/v1/push`. With `endpoint` the + # collector answers 200 to its own clients and posts the + # samples to a path that does not exist. Measured + # end-to-end, not read — `state/probe-3265-collector-to-vm.sh`. + "otlphttp/victoriametrics".metrics_endpoint = + "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push"; } - // lib.optionalAttrs haveCollectorSecret { - auth.authenticator = pushAuthenticator "victoriametrics"; + // lib.optionalAttrs (otelCfg.endpoint != "") { + ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = { + endpoint = otelCfg.endpoint; + } + // lib.optionalAttrs (otelCfg.headersCredential != null) { + # Interpolated by the collector from its environment at + # runtime, never by nix: `EnvironmentFile` below is what + # puts it there, so the value is not read into the store. + headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; + } + // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; + } + // lib.optionalAttrs deployCfg.victorialogs.enable { + # `logs_endpoint`, NOT `endpoint`, for exactly the reason the + # metrics exporter above spells out — and the trap is worse + # here, because the two stores' OTLP routes differ. `endpoint` + # is a base the exporter appends `/v1/logs` to; VictoriaLogs + # serves `/insert/opentelemetry/v1/logs`, which is also not + # the metrics store's `/opentelemetry/api/v1/push`. Both + # spellings pass `otelcol validate`. + # + # ⚠️ And the store cannot tell you either: a right path, a + # wrong path and a nonsense path all answer 400. Only its log + # distinguishes them — the real route complains about the + # encoding, everything else says "unsupported path requested". + # 🔑 THE TWO QUERY PARAMETERS ARE NOT TUNING — without the + # first one this pipeline fills the store with records that + # cannot be searched by message text, which is the only + # reason to collect logs at all. + # + # The journald receiver leaves the OTLP *body* empty and + # carries the entry as a map of journal fields, so the store + # has no message to index and writes the literal placeholder + # `missing _msg field` into `_msg` on EVERY record. Nothing + # errors: ingest returns 200, the data is all present, and a + # plain search for a line that is sitting right there returns + # nothing. `_msg_field` tells the store which field carries + # the message. Measured, both with and without. + # + # `_stream_fields` is the difference between one enormous + # stream for the whole host and one per unit per machine — + # both low-cardinality, and both fields journald sets itself. + "otlphttp/victorialogs".logs_endpoint = + "http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs" + + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT"; }; - } - // lib.optionalAttrs (otelCfg.endpoint != "") { - ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = { - endpoint = otelCfg.endpoint; - } - // lib.optionalAttrs (otelCfg.headersCredential != null) { - # Interpolated by the collector from its environment at - # runtime, never by nix: `EnvironmentFile` below is what - # puts it there, so the value is not read into the store. - headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; - } - // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; - } - // { - # `logs_endpoint`, NOT `endpoint`, for exactly the reason the - # metrics exporter above spells out — and the trap is worse - # here, because the two stores' OTLP routes differ. `endpoint` - # is a base the exporter appends `/v1/logs` to; VictoriaLogs - # serves `/insert/opentelemetry/v1/logs`, which is also not - # the metrics store's `/opentelemetry/api/v1/push`. Both - # spellings pass `otelcol validate`. - # - # ⚠️ And the store cannot tell you either: a right path, a - # wrong path and a nonsense path all answer 400. Only its log - # distinguishes them — the real route complains about the - # encoding, everything else says "unsupported path requested". - # 🔑 THE TWO QUERY PARAMETERS ARE NOT TUNING — without the - # first one this pipeline fills the store with records that - # cannot be searched by message text, which is the only - # reason to collect logs at all. - # - # The journald receiver leaves the OTLP *body* empty and - # carries the entry as a map of journal fields, so the store - # has no message to index and writes the literal placeholder - # `missing _msg field` into `_msg` on EVERY record. Nothing - # errors: ingest returns 200, the data is all present, and a - # plain search for a line that is sitting right there returns - # nothing. `_msg_field` tells the store which field carries - # the message. Measured, both with and without. - # - # `_stream_fields` is the difference between one enormous - # stream for the whole host and one per unit per machine — - # both low-cardinality, and both fields journald sets itself. - "otlphttp/victorialogs" = { - logs_endpoint = logsPushUrl + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT"; - } - // lib.optionalAttrs haveCollectorSecret { - auth.authenticator = pushAuthenticator "victorialogs"; - }; - }; # Moves this collector's self-metrics off the built-in # default of `localhost:8888`, which the hive tier holds. @@ -1193,11 +1155,7 @@ in # same reasoning as its receiver above: `otlp/${swarmTierName}` # always exists, so the authenticator it names must too, or an # extension-not-listed startup failure follows every deploy. - ++ [ "oidc/${swarmTierName}" ] - # The push side's authenticators. Same rule as above: one an - # exporter names but this list omits is INERT — the collector - # starts clean and pushes unauthenticated. - ++ lib.optionals haveCollectorSecret (map pushAuthenticator (lib.attrNames pushAudiences)); + ++ [ "oidc/${swarmTierName}" ]; # Fan-out, not a choice: with both configured the same # samples go upstream AND into the swarm's store. The store @@ -1311,27 +1269,7 @@ in issuer_url = autheliaCfg.url; audience = config.services.hyperhive.swarm.controller.queueClientId; }; - } - # The push side. Opposite direction to every `oidc/*` above — - # those VALIDATE a token arriving; these OBTAIN one to send. - # Hence `oauth2client` rather than `oidc`, and hence a client - # id and secret rather than an issuer and an audience to check. - // lib.optionalAttrs haveCollectorSecret ( - lib.mapAttrs' ( - name: url: - lib.nameValuePair (pushAuthenticator name) { - client_id = cfg.clientId; - # A path, never a value. Same file the scrape side reads. - client_secret_file = collectorSecretPath; - token_url = "${autheliaCfg.url}/api/oidc/token"; - # Registered is not requested: a client that does not ASK - # for the scope gets a token carrying none, and one that - # does not ask for an audience gets `aud: []`. - scopes = [ "authelia.bearer.authz" ]; - endpoint_params.audience = url; - } - ) pushAudiences - ); + }; # `upsert`, not `insert`: a sender that stamps its own # `hive` must be OVERWRITTEN, not deferred to. This @@ -1408,15 +1346,12 @@ in # authenticates this collector's export onward, this one # authenticates it to a service it scrapes. # - # ⚠️ Gated on the secret existing, not on what it is used for. - # `LoadCredential` on a missing source is a unit that refuses to - # start, and the option that names this file is only set where - # something delivers it — so this follows that condition - # exactly rather than restating a narrower one. Where it is false - # no authenticator is rendered either, so the collector starts and - # is refused by the stores rather than failing to start. - // lib.optionalAttrs haveCollectorSecret { - LoadCredential = [ "${collectorCredentialId}:${cfg.clientSecretFile}" ]; + # ⚠️ Only where a published target exists. `LoadCredential` on a + # missing source is a unit that refuses to start, so declaring it + # unconditionally would take the collector down on every hive that + # scrapes nothing published — the empty case is the shipped one. + // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) { + LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ]; }; }; }; diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index b9d8c334..bdadcbcb 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -18,12 +18,9 @@ # rationale (forceSSL is load-bearing there too: authelia answers a plain-http # auth subrequest with 400, which `auth_request` cannot interpret as anything # but a broken check). The store's own listener stays loopback-only and -# unauthenticated, and the vhost is now the ONLY way in from outside: the -# collector pushes through it too, at its own `= /insert/...` location, because -# a swarm has one log store and the collector need not share a host with it. -# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a -# pusher handed its `error_page 401 =302` follows the redirect and POSTs at a -# login page, which answers 200. See the location itself. +# unauthenticated exactly as before — the collector still writes to it +# directly, never through this vhost — so this adds a new authenticated front +# door without touching the existing write path at all. { pkgs, lib, @@ -180,27 +177,6 @@ in proxyPass = "http://127.0.0.1:${toString cfg.port}/"; extraConfig = swarmAuthRequest; }; - - # The swarm collector's ingest route. `=` so it outranks the `/` - # prefix above — without a more specific location it would ride that - # catch-all, and that is the whole hazard here. - # - # ⚠️ Deliberately NOT `swarmAuthRequest`. That block ends in - # `error_page 401 =302`, which is right for a browser and wrong for a - # pusher: handed a redirect it follows the redirect and POSTs its - # batch at a login page, which answers 200. Ingest then reports - # healthy while storing nothing. A machine route lets the 401 reach - # the client unchanged. - # - # `auth_request` does not inherit across sibling locations (see the - # note on `swarmAuthRequest` above), so naming it here is required - # rather than redundant. - "= /insert/opentelemetry/v1/logs" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}/insert/opentelemetry/v1/logs"; - extraConfig = '' - auth_request /__hive_authelia; - ''; - }; # The subrequest itself — same target, same header set, same # reasoning as `swarm-ui.nix`'s own copy (measured against the # pinned authelia binary, not copied from an example). @@ -285,8 +261,6 @@ in # code is not evidence. # # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is - # why `listenAddress` above is loopback and why nothing reaches it except - # through the gateway, where the ingest route is authenticated. (This module - # does declare a vhost; the line that used to say otherwise was already - # wrong before the collector started pushing through it.) + # why `listenAddress` above is loopback, why the collector is the only + # writer, and why this module declares no gateway vhost. } diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 31f24ade..e9b7dd29 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -21,7 +21,6 @@ let networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; - autheliaCfg = hyperhiveCfg.swarm.authelia; swarmDomain = hyperhiveCfg.swarm.domain; # Total on a null swarm domain for the same reason every sibling module is: @@ -124,58 +123,17 @@ in # named it runs, which is what keeps scraper and target on one host by # construction instead of by luck. # - # The loopback literal is safe for this option and NOT for the write path, - # which is the distinction that matters now that the two differ. A scrape - # target is only ever read by a collector on this host, so loopback states - # a fact. The collector's push goes to the swarm name through the gateway, - # because the collector need not be here at all. + # The loopback literal introduces no new assumption — it is the address + # this module already pins the listener to, and the same one the + # collector's `otlphttp/victoriametrics` exporter already writes to. If + # that reach is ever wrong, it is wrong for the write path first. services.hyperhive.swarm.otel.scrapeTargets.victoriametrics = "127.0.0.1:${toString cfg.port}"; services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; - locations = { - "/" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}/"; - }; - - # The swarm collector's ingest route, and the only authenticated thing - # on this vhost. `=` so it outranks the `/` prefix above, which would - # otherwise carry these writes with no check at all. - # - # ⚠️ No `error_page 401 =302` here, and its absence is the point: a - # redirect is right for a browser and wrong for a pusher, which would - # follow it and POST its batch at a login page that answers 200 — - # ingest reporting healthy while storing nothing. - "= /opentelemetry/api/v1/push" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}/opentelemetry/api/v1/push"; - extraConfig = '' - auth_request /__metrics_push_authz; - ''; - }; - - # The subrequest. Same target and header set as the sibling log - # store's, which took them from `swarm-ui.nix` — `X-Original-URL` and - # `X-Original-Method` are what authelia's auth-request implementation - # reads, and the address it compares the token's audience against. - "= /__metrics_push_authz" = { - proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request"; - # nixpkgs appends its OWN `Host $host` after extraConfig, which - # would override verifiedProxyTo's — see the comment on - # verifiedProxyTo in hive-gateway/vhost-lib.nix. - recommendedProxySettings = false; - extraConfig = '' - internal; - ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain} - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-Method $request_method; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - ''; - }; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/"; }; }; @@ -234,8 +192,5 @@ in # deployment ever needs them. # # ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is - # loopback: the only route to it from off-host is the vhost's ingest - # location, which authenticates. The collector is still the only writer, but - # it now arrives by the swarm name rather than over loopback, because it need - # not share a host with this store. + # loopback and why the collector — not agents — is the writer. } diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 3cd84707..b3cb38ab 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -145,28 +145,6 @@ let lib.findFirst (c: c.job_name == job) null machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; - # A swarm collector on a host that runs NEITHER store — the fully-spread - # shape from docs/swarm/services.md, and the one the old per-host gates made - # inexpressible. It is the whole point of the cases below that this hive is - # not a degenerate configuration but a supported one. - otelNoStores = hive { - deploy.swarm-otel.enable = true; - deploy.authelia.enable = true; - deploy.victoriametrics.enable = false; - deploy.victorialogs.enable = false; - }; - otelSettings = - machine: machine.containers.swarm-otel.config.services.opentelemetry-collector.settings; - - # authelia somewhere else, the credential delivered by hand. Whether this - # collector authenticates must follow the credential, never another - # service's placement. - otelRemoteAuthelia = hive { - deploy.swarm-otel.enable = true; - deploy.authelia.enable = false; - swarm.otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret"; - }; - # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is @@ -230,113 +208,6 @@ let name = "the CI container's unit definitions merge without a priority collision"; ok = forceCiServiceConfigs; } - { - # The defect itself. These exporters used to be gated on the stores' - # PER-HOST enables, so a collector that did not share a host with them - # rendered none at all and dropped everything it received, from every - # hive — silently, because an absent exporter is not an error. - name = "a collector that hosts neither store still exports to both"; - ok = - let - e = (otelSettings otelNoStores).exporters; - in - (e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs"); - } - { - # A swarm has one of each store, so the address is a swarm-level name. - # A loopback literal here is the co-location assumption written back in, - # and it renders, deploys and reports healthy while reaching nothing. - name = "the store exporters address the stores by name, never by loopback"; - ok = - let - e = (otelSettings otelNoStores).exporters; - m = e."otlphttp/victoriametrics".metrics_endpoint; - l = e."otlphttp/victorialogs".logs_endpoint; - in - !(lib.hasInfix "127.0.0.1" m) - && !(lib.hasInfix "127.0.0.1" l) - && lib.hasInfix "metrics.t.local" m - && lib.hasInfix "logs.t.local" l; - } - { - # The collector reaches these routes through the gateway now, so each - # store needs an ingest location of its own. Without one the write rides - # the `/` catch-all: unauthenticated on the metrics store, and into a - # browser redirect on the log store. - name = "each store's vhost has an authenticated ingest location"; - ok = - let - v = allLocal.services.nginx.virtualHosts; - m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null; - l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null; - in - m != null - && l != null - && lib.hasInfix "auth_request" m.extraConfig - && lib.hasInfix "auth_request" l.extraConfig; - } - { - # The arm that actually protects something. A pusher handed - # `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page, - # which answers 200 — ingest reporting healthy while storing nothing. - # The third clause is the positive control: the log store's browser - # location really does redirect, so this says the machine routes differ - # rather than that the string is absent from the whole file. - name = "the ingest locations answer 401 instead of redirecting a pusher"; - ok = - let - v = allLocal.services.nginx.virtualHosts; - m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig; - l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig; - browser = v."logs.t.local".locations."/".extraConfig; - in - !(lib.hasInfix "error_page" m) - && !(lib.hasInfix "error_page" l) - && lib.hasInfix "error_page" browser; - } - { - # Defining an exporter and REFERENCING it are two separate lists, and - # the second is where the original gate also lived. An exporter no - # pipeline names is as silent as one that does not exist — this case - # exists because a mutation that restored only the reference-side gate - # left every other case here green. - name = "every pipeline that has a store exporter defined actually sends to it"; - ok = - let - s = otelSettings otelNoStores; - used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines)); - in - builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used; - } - { - # The collector authenticates because it HOLDS a credential, not because - # authelia happens to share its host. Gating on the other service's - # placement renders a collector that pushes unauthenticated wherever - # authelia lives elsewhere — one of the supported shapes. - name = "a collector with a hand-delivered secret authenticates without authelia beside it"; - ok = - let - s = otelSettings otelRemoteAuthelia; - in - (s.exporters."otlphttp/victoriametrics" ? auth) - && builtins.elem "oauth2client/victoriametrics" s.service.extensions; - } - { - # An authenticator an exporter names but `service.extensions` omits is - # INERT — the collector starts clean and pushes unauthenticated until - # something at the far end refuses it. Checked as a set relation rather - # than by naming the two, so it keeps holding for exporters not written - # yet. - name = "every exporter authenticator is listed in service.extensions"; - ok = - let - s = otelSettings otelNoStores; - named = lib.filter (v: v != null) ( - lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters - ); - in - named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; - } { # The store's seal is spread over six gates — the stanza, the # provisioning unit, two bind mounts, a device and an EnvironmentFile.