diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 4367fd28..6775ee23 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -442,21 +442,6 @@ in url = "https://${cfg.domain}/"; }; - # The metrics endpoint, declared once: the collector both scrapes this - # URL and derives from it the audience its token is minted for. Same - # `behindGateway` guard, and for a stronger reason than the two above: - # with it off there is no `= /metrics` location and no `auth_request` - # in front of it, so the URL this names does not exist to be scraped - # or authorised. - # - # ⚠️ Written as the exact URL a collector requests, because that is - # what authelia compares against — this string agreeing with the - # `location` block above it is the whole mechanism. A near miss is a - # correctly minted token refused at the target. - services.hyperhive.swarm.otel.publishedScrapeTargets = lib.optionalAttrs cfg.behindGateway { - forgejo = "https://${cfg.domain}/metrics"; - }; - # `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for # git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB # clones). SSH stays direct on `forge.sshPort`. See diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 5fff0a71..7ed2e398 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -57,47 +57,6 @@ let # failing the check it was supposed to pass. operatorGroup = "admins"; - # Whether the swarm collector's client is actually registered here. - # - # Asked of the client list rather than re-derived from the conditions - # that produce it (`otel.enable`, a non-empty audience set, authelia - # being co-located). A second copy of that predicate is a second thing - # to keep in step, and the two drifting is not a build failure: a rule - # naming an unregistered client is refused by authelia's *startup* - # validator, so the whole SSO service fails to restart. - # - # Reading the registration itself also makes the rule correct for a - # registrar this module has never heard of — an operator registering - # the collector by hand against a provider that is not co-located with - # it gets the same rule, from the same expression. - # ⚠️ `swarm.otel` named in full, not through a let binding: there are two - # otel options one word apart, and only this one is the swarm's collector. - collectorClientId = config.services.hyperhive.swarm.otel.clientId; - collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients; - - # The forge's metrics endpoint: deny until a collector exists to allow. - # - # `deny` is not a placeholder, it is the protection. The endpoint is - # always served (a swarm-integrated forge always has metrics) and - # `default_policy` is `one_factor`, which means *any* authenticated - # subject — every operator today, every agent once they hold authelia - # accounts. Nor does the audience stand between a browser session and - # this data: `authn_strategies` on the authz endpoint also accepts - # `CookieSession`, and a cookie carries no audience at all. - metricsRule = { - domain = forgeCfg.domain; - resources = [ "^/metrics$" ]; - } - // ( - if collectorRegistered then - { - policy = "one_factor"; - subject = [ "oauth2:client:${collectorClientId}" ]; - } - else - { policy = "deny"; } - ); - # Upstream's `services.authelia.instances.` derives the unit, # user, group and StateDirectory from the instance name # (`authelia` + `-`). Naming them here rather than repeating the @@ -258,7 +217,7 @@ let if c.kind == "machine" then '' printf -- ' grant_types: ["client_credentials"]\n' - printf -- ' scopes: [${lib.optionalString c.bearerAuthz "authelia.bearer.authz"}]\n' + printf -- ' scopes: []\n' '' else '' @@ -587,15 +546,6 @@ in a successful consent** — the login looks like it worked right up to the last hop, and neither the redirect nor the secret is at fault. - - ⚠️ `null` is NOT accepted on a client with `bearerAuthz`. - Measured against authelia 4.39.20: under that scope the - method must be *stated*: omitting it is refused with - `must be configured as 'client_secret_basic', … but it's - configured as` an empty string. The sentence above is - true of an ordinary client and false of that one, which - is exactly how a reviewer reads this option and concludes - the assertion below is wrong. ''; }; @@ -625,34 +575,6 @@ in ''; }; - bearerAuthz = lib.mkOption { - type = lib.types.bool; - default = false; - example = true; - description = '' - Grant this client the `authelia.bearer.authz` scope, so it - may present its access token to authelia's authz endpoint - and be authorised by an `access_control` rule — how a - scraper reaches a service published behind the gateway. - - A named capability rather than a free-form `scopes` list, - for the same reason `kind` derives the rest: authelia - refuses some scope/grant combinations outright (`openid` - with `client_credentials` among them), and a list would - make those combinations expressible again. This admits the - one value that is legal here and nothing else. - - ::: {.note} - Setting this obliges two other options, and the assertions - below enforce it. Authelia checks the same thing, but only - in its `preStart` validator — which means a violation - builds and deploys cleanly and then fails to restart, - taking swarm SSO down. The assertions move that to - evaluation, where a wrong value costs nothing. - ::: - ''; - }; - accessTokenSignedResponseAlg = lib.mkOption { type = lib.types.nullOr ( lib.types.enum [ @@ -898,77 +820,6 @@ in + "services.hyperhive.swarm.hives; rename the hive or the " + "colliding client."; } - # The two obligations `authelia.bearer.authz` carries. Authelia - # enforces both itself — but in its `preStart` validator, so a - # violation produces a green `nixos-rebuild switch` and an authelia - # that then refuses to come back up, taking swarm SSO with it. - # Asserting here moves the same failure to evaluation, where a - # wrong value costs a build and nothing else. - { - assertion = lib.all (c: !c.bearerAuthz || c.audience != [ ]) cfg.oidc.clients; - message = - "services.hyperhive.swarm.authelia.oidc.clients: " - + lib.concatStringsSep ", " ( - map (c: "client '${c.id}'") (lib.filter (c: c.bearerAuthz && c.audience == [ ]) cfg.oidc.clients) - ) - + " sets bearerAuthz but declares no audience. The audience is " - + "what authorises a bearer token at a given URL, so without " - + "one the scope grants access to nothing and authelia refuses " - + "the configuration outright."; - } - { - assertion = lib.all ( - c: - !c.bearerAuthz - || lib.elem c.tokenEndpointAuthMethod [ - "client_secret_basic" - "client_secret_jwt" - "private_key_jwt" - ] - ) cfg.oidc.clients; - message = - "services.hyperhive.swarm.authelia.oidc.clients: " - + lib.concatStringsSep ", " ( - map (c: "client '${c.id}' (tokenEndpointAuthMethod = ${toString c.tokenEndpointAuthMethod})") ( - lib.filter ( - c: - c.bearerAuthz - && !lib.elem c.tokenEndpointAuthMethod [ - "client_secret_basic" - "client_secret_jwt" - "private_key_jwt" - ] - ) cfg.oidc.clients - ) - ) - + ". A confidential client carrying authelia.bearer.authz must " - + "authenticate with client_secret_basic, client_secret_jwt or " - + "private_key_jwt. Notably client_secret_post is refused, and " - + "it is what an OAuth2 client library may reach for first. " - + "Leaving it null is refused too: authelia requires the method " - + "to be STATED under this scope rather than defaulted."; - } - # Nothing else stops `bearerAuthz` on an interactive client, and it - # would silently do nothing: `renderClient` reads the flag only in - # the `machine` branch, so the scope is simply never emitted and the - # client authenticates fine while being authorised for nothing. - # - # That is this module's own failure mode one level up — a green - # build and a grant that does not exist — and `kind` defaults to - # `interactive`, so it is reached by FORGETTING a field rather than - # by writing a wrong one. - { - assertion = lib.all (c: !c.bearerAuthz || c.kind == "machine") cfg.oidc.clients; - message = - "services.hyperhive.swarm.authelia.oidc.clients: " - + lib.concatStringsSep ", " ( - map (c: "client '${c.id}'") (lib.filter (c: c.bearerAuthz && c.kind != "machine") cfg.oidc.clients) - ) - + " sets bearerAuthz with kind != \"machine\". The scope is only " - + "emitted for machine clients, so this grants nothing while " - + "evaluating and deploying cleanly. A browser client has a user " - + "to authorise and does not need it."; - } ]; # Authelia's own gateway surface: the vhost that fronts it and the @@ -1354,10 +1205,32 @@ in # The metrics rule is listed first so it cannot be shadowed # by a broader domain rule added later. rules = - # Denied or client-scoped depending on whether a - # collector is registered — see `metricsRule` above, - # which is where the reasoning for both halves lives. - lib.optional forgeCfg.behindGateway metricsRule + # The forge's metrics endpoint. `deny` is deliberate and + # is the whole protection right now: the endpoint is + # always served (a swarm-integrated forge always has + # metrics), and `default_policy` is `one_factor`, which + # means *any* authenticated subject — every operator + # today, every agent once they hold authelia accounts. + # + # Being reachable by a Bearer token is not sufficient on + # its own: `authn_strategies` on this endpoint also + # accepts `CookieSession`, and a cookie carries no + # audience, so the audience is not what stands between a + # browser session and this data. + # + # The collector gets in by REPLACING this with a + # client-scoped allow (`subject = ["oauth2:client:"]`) + # once such a client is registered. Denying until then is + # what makes publishing the endpoint safe on its own — + # authelia refuses a subject naming a client that is not + # registered, and it does so in a `preStart` validator, + # so naming one early takes the whole SSO service down on + # the next restart rather than failing the build. + lib.optional forgeCfg.behindGateway { + domain = forgeCfg.domain; + resources = [ "^/metrics$" ]; + policy = "deny"; + } ++ lib.optional uiCfg.enable { domain = uiCfg.domain; subject = [ "group:${operatorGroup}" ]; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 06e9ec9d..cb67b155 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -58,43 +58,6 @@ let # swarm-tier pipeline would be added here and inherit the check for free. reservedOwners = [ swarmTierName ]; - # A published target is declared as ONE url, because that url is also the - # audience its token is minted for — but prometheus wants the same fact in - # three fields. Split it here rather than asking a service to state it - # twice: two spellings of one address is a mismatch waiting to happen, and - # the failure is a valid token refused at the target. - # - # `null` when the shape is wrong, which the assertion below reports by name. - # ⚠️ The pattern requires `https`. A published endpoint reached over plain - # http would ship a bearer token in clear text, and that is worth an eval - # error rather than a warning nobody reads. - parsePublished = url: builtins.match "https://([^/]+)(/.*)" url; - - # The client secret takes three names, and the reason is `DynamicUser`. - # - # Upstream's collector unit runs with `DynamicUser = true`, and the - # prometheus receiver opens `client_secret_file` ITSELF, at runtime, as that - # user — so there is no stable uid to hand a file to, and the root-owned - # 0400 shape `hive-matrix-oidc-secret` delivers to would be unreadable. - # (Matrix gets away with it because `LoadCredential` reads the file as root - # before the sandbox exists, and tuwunel never opens that path itself.) - # - # `LoadCredential` solves both halves: systemd reads the file as root and - # re-exposes it to the dynamic user under a path that does not depend on - # which uid it turned out to be. - # - # At rest in the container's tree — written by the host oneshot below. - # Under /var/lib and not /run because the collector may start before the - # delivery unit on a later boot, and a secret that evaporates on reboot - # turns a working scrape into an intermittent one. - collectorSecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret"; - collectorCredentialId = "oidc-client-secret"; - # What the scrape config points at. ⚠️ This path and the `LoadCredential` - # id below are one fact spelled twice by systemd's design — both derive from - # `collectorCredentialId` so they cannot drift; a mismatch is a file the - # collector cannot open, discovered at runtime and nowhere else. - collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}"; - # `attrNames` is sorted, so this is a function of the hive SET and not of # the order anyone wrote it in. # @@ -260,56 +223,6 @@ in scrape is the inert configuration this option exists to avoid. ''; }; - - publishedScrapeTargets = lib.mkOption { - type = lib.types.attrsOf lib.types.str; - default = { }; - example = lib.literalExpression ''{ forge = "https://forge.example.com/metrics"; }''; - description = '' - Prometheus endpoints this collector scrapes **by name, with a - credential**, as ` = ""`. - - A service module declares its own entry, from its own module, the - same way it does for `scrapeTargets` — and the collector's OAuth2 - client derives its permitted audiences from these URLs, so a - target and the authorisation to reach it are **one declaration**. - Two lists that must agree would be a drift to maintain, and its - failure mode is the bad one: a target whose audience was forgotten - authenticates against nothing and reads as a scrape failure rather - than a config mistake. - - ⚠️ A **full URL**, not `host:port`. Authelia validates a bearer - token against the address being requested, so the string here is - also the audience the token is minted for; a near miss (a trailing - slash, `http` for `https`) presents as a valid token rejected at - the target, several layers from its cause. - - ⚠️ Deliberately separate from `scrapeTargets`. An entry there is - trusted because the scraper and the target share a host — that - option is loopback-and-unauthenticated by contract. An entry here - is trusted because it presents a credential. One shape for both - would leave a reader unable to tell which of those a given target - relies on. - ''; - }; - - clientId = lib.mkOption { - type = lib.types.str; - readOnly = true; - default = "swarm-collector"; - description = '' - OAuth2 client id this collector authenticates as. Published so - authelia's `access_control` rules can name it without carrying a - second copy of the string, exactly as - `services.hyperhive.swarm.authelia.hiveClientPrefix` is published - for the queue's responder. - - Two spellings drifting apart is not a build failure: authelia - refuses a rule naming an unregistered client in its startup - validator, so the swarm's SSO service fails to *restart* — long - after the change that caused it evaluated cleanly. - ''; - }; }; config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { @@ -354,106 +267,6 @@ in # telemetry. services.hyperhive.swarm.authelia.oidc.hiveIdentities = true; - # The collector's own identity, for the other direction: the hive - # identities above are how this collector authenticates its *callers*, - # this is how it authenticates *itself* to a service published behind - # the gateway. - # - # Only where authelia is co-located. A client is a row in this host's - # provider config, so declaring one against a remote provider would - # render nothing while reading as done; a swarm whose authelia lives - # elsewhere registers it there. - # - # ⚠️ Also conditional on there being a published target, and that is - # the shipped case rather than an edge — nothing declares one until - # some service publishes behind the gateway. Authelia refuses a - # bearer-authz client with no audience, so an unconditional - # declaration would break every hive that runs a collector and - # publishes nothing. - services.hyperhive.swarm.authelia.oidc.clients = - lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { }) - [ - { - id = cfg.clientId; - description = "HyperHive swarm collector"; - kind = "machine"; - # Grants `authelia.bearer.authz`, without which the authz - # endpoint refuses an otherwise valid token and blames the - # token rather than the missing grant. - bearerAuthz = true; - # DERIVED from the targets rather than contributed alongside - # them. A service declares a URL once and this is the - # permission to reach it; two lists that had to agree would be - # a drift to maintain, and the failure mode is the quiet one — - # a target whose audience was forgotten authenticates against - # nothing and looks like a broken scrape. - audience = lib.attrValues cfg.publishedScrapeTargets; - # Stated rather than left on authelia's default, because the - # two agreeing today is not the same as this being the - # required value: authelia permits only basic / JWT methods - # for a confidential client holding that scope, and enforces - # it in the startup validator. - tokenEndpointAuthMethod = "client_secret_basic"; - } - ]; - - # Deliver the collector's client secret from authelia's container into - # this one. On the HOST because that is the only place both container - # trees are addressable: they share this host's network namespace, which - # makes them feel co-located, but their filesystem roots are separate. - # - # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container - # refuses to start when a bind source is missing, and this secret does not - # exist until authelia's first boot has minted it — so binding it would - # make the collector wait on a file that waits on a container that starts - # after it. On a fresh swarm that is a permanent stall presenting as - # "metrics are broken", several layers from its cause. - systemd.services.swarm-otel-oidc-secret = - lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { }) - { - description = "deliver the swarm collector's OIDC client secret from authelia"; - after = [ "container@${autheliaCfg.machine}.service" ]; - requires = [ "container@${autheliaCfg.machine}.service" ]; - before = [ "container@${cfg.machine}.service" ]; - wantedBy = [ "container@${cfg.machine}.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "swarm-otel-oidc-secret"; - # Longer than the bounded wait below, and that is the point: - # `DefaultTimeoutStartSec` is 90s, so without this systemd kills - # the unit before it can emit the message naming the file it was - # waiting for — the failure then reads as a timeout with no cause. - TimeoutStartSec = "180s"; - }; - path = [ pkgs.coreutils ]; - script = '' - set -euo pipefail - - src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"} - dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"} - - # authelia's container is up, but its first-boot generator may - # still be minting. Bounded wait, then fail: skipping silently - # produces a collector whose scrape gets a 401 forever, which is - # the failure this whole design exists to make impossible. - for _ in $(seq 1 60); do - [ -s "$src" ] && break - sleep 2 - done - if [ ! -s "$src" ]; then - echo "authelia has not minted $src after 120s" >&2 - exit 1 - fi - - # root-owned 0400. The collector runs under `DynamicUser`, so - # there is no uid to give it to — `LoadCredential` reads this as - # root before the sandbox exists and re-exposes it to whichever - # uid the unit got. - install -D -m 0400 -o root -g root "$src" "$dst" - ''; - }; - # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. @@ -515,59 +328,6 @@ in Rename the hive. Reserved: ${lib.concatMapStringsSep ", " (n: "'${n}'") reservedOwners}. ''; } - { - # A published target that is not an `https://host/path` url. Without - # this the split returns null and the failure surfaces as - # `attempt to call elemAt on null` from inside the renderer, naming - # neither the option nor the offending value. - # - # It also enforces the scheme: a published endpoint scraped over plain - # http would put a bearer token on the wire in clear text. - assertion = lib.all (u: parsePublished u != null) (lib.attrValues cfg.publishedScrapeTargets); - message = '' - services.hyperhive.swarm.otel.publishedScrapeTargets has ${ - lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") ( - lib.filter (kv: parsePublished kv.value == null) (lib.attrsToList cfg.publishedScrapeTargets) - ) - }, which is not of the form https:///. - - The value is both the address scraped and the audience the - collector's token is minted for, so it has to be the exact url a - request goes to. https is required: this hop carries a bearer - token, and plain http would put it on the wire in clear text. - ''; - } - { - # A job name used by BOTH scrape options. Within one option this - # cannot happen — the module system refuses two definitions of the - # same key with different values, measured — but the two options - # are separate, so nothing arbitrates between them, and they - # render into a single `scrape_configs` LIST where nothing - # overwrites anything: both entries ship under one `job_name`. - # - # The message names which side each collision came from, which is - # the part a rendered-config error could not tell an operator. - assertion = - lib.intersectLists (lib.attrNames cfg.scrapeTargets) (lib.attrNames cfg.publishedScrapeTargets) - == [ ]; - message = '' - services.hyperhive.swarm.otel: ${ - lib.concatMapStringsSep ", " (j: "'${j}'") ( - lib.intersectLists (lib.attrNames cfg.scrapeTargets) (lib.attrNames cfg.publishedScrapeTargets) - ) - } is declared as both a loopback - scrapeTargets job and a publishedScrapeTargets job. - - They render into one prometheus scrape_configs list, so both - entries would ship under the same job_name — nothing overwrites - anything, and the two are scraped by different rules with - different trust. - - Rename one. A target is either reachable on loopback because it - shares this host, or published and reached with a credential; it - should not be described as both. - ''; - } { # A hive proves who it is with a token this provider mints, so # there is no version of this collector that runs without one. @@ -716,48 +476,11 @@ in # target: a `prometheus` receiver with nothing to scrape is # the shape this whole issue is about, a config that renders # and deploys perfectly while adding no data. - // lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) { - prometheus.config.scrape_configs = - lib.mapAttrsToList (job: target: { - job_name = job; - static_configs = [ { targets = [ target ]; } ]; - }) cfg.scrapeTargets - # ⚠️ `++`, so the two kinds of target land in ONE list — - # which is exactly why a job name may not appear in both - # options. A list concatenation does not resolve a - # collision the way an attrset would: both entries ship - # under the same `job_name`. The assertion above is what - # stands between that and a deploy. - ++ lib.mapAttrsToList ( - job: url: - let - parts = parsePublished url; - in - { - job_name = job; - # Split from the declared url — see `parsePublished`. - scheme = "https"; - static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ]; - metrics_path = lib.elemAt parts 1; - # Prometheus-native oauth2, not the collector's - # `oauth2client` extension: the prometheus receiver - # takes upstream's scrape config verbatim and does not - # accept an `auth` block naming a collector extension. - oauth2 = { - client_id = cfg.clientId; - # A path, never a value — nothing here may read the - # secret, or it lands in the store world-readable. - client_secret_file = collectorSecretPath; - token_url = "${autheliaCfg.url}/api/oidc/token"; - # The audience is the target's own url, and authelia - # checks it against the address being requested. - # Registered ≠ requested: a client that does not ASK - # for an audience gets a token with `aud: []` however - # complete its registration looks. - endpoint_params.audience = url; - }; - } - ) cfg.publishedScrapeTargets; + // lib.optionalAttrs (cfg.scrapeTargets != { }) { + prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: { + job_name = job; + static_configs = [ { targets = [ target ]; } ]; + }) cfg.scrapeTargets; }; exporters = @@ -939,20 +662,10 @@ in # environment variable without being read by nix, written to the # store, or passed in argv. systemd.services.opentelemetry-collector.serviceConfig = - lib.optionalAttrs (otelCfg.headersCredential != null) { - EnvironmentFile = otelCfg.headersCredential; - } - # The other credential, and the other direction: the one above - # authenticates this collector's export onward, this one - # authenticates it to a service it scrapes. - # - # ⚠️ Only where a published target exists. `LoadCredential` on a - # missing source is a unit that refuses to start, so declaring it - # unconditionally would take the collector down on every hive that - # scrapes nothing published — the empty case is the shipped one. - // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) { - LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ]; - }; + lib.optionalAttrs (otelCfg.headersCredential != null) + { + EnvironmentFile = otelCfg.headersCredential; + }; }; }; };