diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 6775ee23..4367fd28 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -442,6 +442,21 @@ in url = "https://${cfg.domain}/"; }; + # The metrics endpoint, declared once: the collector both scrapes this + # URL and derives from it the audience its token is minted for. Same + # `behindGateway` guard, and for a stronger reason than the two above: + # with it off there is no `= /metrics` location and no `auth_request` + # in front of it, so the URL this names does not exist to be scraped + # or authorised. + # + # ⚠️ Written as the exact URL a collector requests, because that is + # what authelia compares against — this string agreeing with the + # `location` block above it is the whole mechanism. A near miss is a + # correctly minted token refused at the target. + services.hyperhive.swarm.otel.publishedScrapeTargets = lib.optionalAttrs cfg.behindGateway { + forgejo = "https://${cfg.domain}/metrics"; + }; + # `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for # git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB # clones). SSH stays direct on `forge.sshPort`. See diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 7ed2e398..5fff0a71 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -57,6 +57,47 @@ let # failing the check it was supposed to pass. operatorGroup = "admins"; + # Whether the swarm collector's client is actually registered here. + # + # Asked of the client list rather than re-derived from the conditions + # that produce it (`otel.enable`, a non-empty audience set, authelia + # being co-located). A second copy of that predicate is a second thing + # to keep in step, and the two drifting is not a build failure: a rule + # naming an unregistered client is refused by authelia's *startup* + # validator, so the whole SSO service fails to restart. + # + # Reading the registration itself also makes the rule correct for a + # registrar this module has never heard of — an operator registering + # the collector by hand against a provider that is not co-located with + # it gets the same rule, from the same expression. + # ⚠️ `swarm.otel` named in full, not through a let binding: there are two + # otel options one word apart, and only this one is the swarm's collector. + collectorClientId = config.services.hyperhive.swarm.otel.clientId; + collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients; + + # The forge's metrics endpoint: deny until a collector exists to allow. + # + # `deny` is not a placeholder, it is the protection. The endpoint is + # always served (a swarm-integrated forge always has metrics) and + # `default_policy` is `one_factor`, which means *any* authenticated + # subject — every operator today, every agent once they hold authelia + # accounts. Nor does the audience stand between a browser session and + # this data: `authn_strategies` on the authz endpoint also accepts + # `CookieSession`, and a cookie carries no audience at all. + metricsRule = { + domain = forgeCfg.domain; + resources = [ "^/metrics$" ]; + } + // ( + if collectorRegistered then + { + policy = "one_factor"; + subject = [ "oauth2:client:${collectorClientId}" ]; + } + else + { policy = "deny"; } + ); + # Upstream's `services.authelia.instances.` derives the unit, # user, group and StateDirectory from the instance name # (`authelia` + `-`). Naming them here rather than repeating the @@ -217,7 +258,7 @@ let if c.kind == "machine" then '' printf -- ' grant_types: ["client_credentials"]\n' - printf -- ' scopes: []\n' + printf -- ' scopes: [${lib.optionalString c.bearerAuthz "authelia.bearer.authz"}]\n' '' else '' @@ -546,6 +587,15 @@ in a successful consent** — the login looks like it worked right up to the last hop, and neither the redirect nor the secret is at fault. + + ⚠️ `null` is NOT accepted on a client with `bearerAuthz`. + Measured against authelia 4.39.20: under that scope the + method must be *stated*: omitting it is refused with + `must be configured as 'client_secret_basic', … but it's + configured as` an empty string. The sentence above is + true of an ordinary client and false of that one, which + is exactly how a reviewer reads this option and concludes + the assertion below is wrong. ''; }; @@ -575,6 +625,34 @@ in ''; }; + bearerAuthz = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Grant this client the `authelia.bearer.authz` scope, so it + may present its access token to authelia's authz endpoint + and be authorised by an `access_control` rule — how a + scraper reaches a service published behind the gateway. + + A named capability rather than a free-form `scopes` list, + for the same reason `kind` derives the rest: authelia + refuses some scope/grant combinations outright (`openid` + with `client_credentials` among them), and a list would + make those combinations expressible again. This admits the + one value that is legal here and nothing else. + + ::: {.note} + Setting this obliges two other options, and the assertions + below enforce it. Authelia checks the same thing, but only + in its `preStart` validator — which means a violation + builds and deploys cleanly and then fails to restart, + taking swarm SSO down. The assertions move that to + evaluation, where a wrong value costs nothing. + ::: + ''; + }; + accessTokenSignedResponseAlg = lib.mkOption { type = lib.types.nullOr ( lib.types.enum [ @@ -820,6 +898,77 @@ in + "services.hyperhive.swarm.hives; rename the hive or the " + "colliding client."; } + # The two obligations `authelia.bearer.authz` carries. Authelia + # enforces both itself — but in its `preStart` validator, so a + # violation produces a green `nixos-rebuild switch` and an authelia + # that then refuses to come back up, taking swarm SSO with it. + # Asserting here moves the same failure to evaluation, where a + # wrong value costs a build and nothing else. + { + assertion = lib.all (c: !c.bearerAuthz || c.audience != [ ]) cfg.oidc.clients; + message = + "services.hyperhive.swarm.authelia.oidc.clients: " + + lib.concatStringsSep ", " ( + map (c: "client '${c.id}'") (lib.filter (c: c.bearerAuthz && c.audience == [ ]) cfg.oidc.clients) + ) + + " sets bearerAuthz but declares no audience. The audience is " + + "what authorises a bearer token at a given URL, so without " + + "one the scope grants access to nothing and authelia refuses " + + "the configuration outright."; + } + { + assertion = lib.all ( + c: + !c.bearerAuthz + || lib.elem c.tokenEndpointAuthMethod [ + "client_secret_basic" + "client_secret_jwt" + "private_key_jwt" + ] + ) cfg.oidc.clients; + message = + "services.hyperhive.swarm.authelia.oidc.clients: " + + lib.concatStringsSep ", " ( + map (c: "client '${c.id}' (tokenEndpointAuthMethod = ${toString c.tokenEndpointAuthMethod})") ( + lib.filter ( + c: + c.bearerAuthz + && !lib.elem c.tokenEndpointAuthMethod [ + "client_secret_basic" + "client_secret_jwt" + "private_key_jwt" + ] + ) cfg.oidc.clients + ) + ) + + ". A confidential client carrying authelia.bearer.authz must " + + "authenticate with client_secret_basic, client_secret_jwt or " + + "private_key_jwt. Notably client_secret_post is refused, and " + + "it is what an OAuth2 client library may reach for first. " + + "Leaving it null is refused too: authelia requires the method " + + "to be STATED under this scope rather than defaulted."; + } + # Nothing else stops `bearerAuthz` on an interactive client, and it + # would silently do nothing: `renderClient` reads the flag only in + # the `machine` branch, so the scope is simply never emitted and the + # client authenticates fine while being authorised for nothing. + # + # That is this module's own failure mode one level up — a green + # build and a grant that does not exist — and `kind` defaults to + # `interactive`, so it is reached by FORGETTING a field rather than + # by writing a wrong one. + { + assertion = lib.all (c: !c.bearerAuthz || c.kind == "machine") cfg.oidc.clients; + message = + "services.hyperhive.swarm.authelia.oidc.clients: " + + lib.concatStringsSep ", " ( + map (c: "client '${c.id}'") (lib.filter (c: c.bearerAuthz && c.kind != "machine") cfg.oidc.clients) + ) + + " sets bearerAuthz with kind != \"machine\". The scope is only " + + "emitted for machine clients, so this grants nothing while " + + "evaluating and deploying cleanly. A browser client has a user " + + "to authorise and does not need it."; + } ]; # Authelia's own gateway surface: the vhost that fronts it and the @@ -1205,32 +1354,10 @@ in # The metrics rule is listed first so it cannot be shadowed # by a broader domain rule added later. rules = - # The forge's metrics endpoint. `deny` is deliberate and - # is the whole protection right now: the endpoint is - # always served (a swarm-integrated forge always has - # metrics), and `default_policy` is `one_factor`, which - # means *any* authenticated subject — every operator - # today, every agent once they hold authelia accounts. - # - # Being reachable by a Bearer token is not sufficient on - # its own: `authn_strategies` on this endpoint also - # accepts `CookieSession`, and a cookie carries no - # audience, so the audience is not what stands between a - # browser session and this data. - # - # The collector gets in by REPLACING this with a - # client-scoped allow (`subject = ["oauth2:client:"]`) - # once such a client is registered. Denying until then is - # what makes publishing the endpoint safe on its own — - # authelia refuses a subject naming a client that is not - # registered, and it does so in a `preStart` validator, - # so naming one early takes the whole SSO service down on - # the next restart rather than failing the build. - lib.optional forgeCfg.behindGateway { - domain = forgeCfg.domain; - resources = [ "^/metrics$" ]; - policy = "deny"; - } + # Denied or client-scoped depending on whether a + # collector is registered — see `metricsRule` above, + # which is where the reasoning for both halves lives. + lib.optional forgeCfg.behindGateway metricsRule ++ lib.optional uiCfg.enable { domain = uiCfg.domain; subject = [ "group:${operatorGroup}" ]; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index cb67b155..06e9ec9d 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -58,6 +58,43 @@ let # swarm-tier pipeline would be added here and inherit the check for free. reservedOwners = [ swarmTierName ]; + # A published target is declared as ONE url, because that url is also the + # audience its token is minted for — but prometheus wants the same fact in + # three fields. Split it here rather than asking a service to state it + # twice: two spellings of one address is a mismatch waiting to happen, and + # the failure is a valid token refused at the target. + # + # `null` when the shape is wrong, which the assertion below reports by name. + # ⚠️ The pattern requires `https`. A published endpoint reached over plain + # http would ship a bearer token in clear text, and that is worth an eval + # error rather than a warning nobody reads. + parsePublished = url: builtins.match "https://([^/]+)(/.*)" url; + + # The client secret takes three names, and the reason is `DynamicUser`. + # + # Upstream's collector unit runs with `DynamicUser = true`, and the + # prometheus receiver opens `client_secret_file` ITSELF, at runtime, as that + # user — so there is no stable uid to hand a file to, and the root-owned + # 0400 shape `hive-matrix-oidc-secret` delivers to would be unreadable. + # (Matrix gets away with it because `LoadCredential` reads the file as root + # before the sandbox exists, and tuwunel never opens that path itself.) + # + # `LoadCredential` solves both halves: systemd reads the file as root and + # re-exposes it to the dynamic user under a path that does not depend on + # which uid it turned out to be. + # + # At rest in the container's tree — written by the host oneshot below. + # Under /var/lib and not /run because the collector may start before the + # delivery unit on a later boot, and a secret that evaporates on reboot + # turns a working scrape into an intermittent one. + collectorSecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret"; + collectorCredentialId = "oidc-client-secret"; + # What the scrape config points at. ⚠️ This path and the `LoadCredential` + # id below are one fact spelled twice by systemd's design — both derive from + # `collectorCredentialId` so they cannot drift; a mismatch is a file the + # collector cannot open, discovered at runtime and nowhere else. + collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}"; + # `attrNames` is sorted, so this is a function of the hive SET and not of # the order anyone wrote it in. # @@ -223,6 +260,56 @@ in scrape is the inert configuration this option exists to avoid. ''; }; + + publishedScrapeTargets = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + example = lib.literalExpression ''{ forge = "https://forge.example.com/metrics"; }''; + description = '' + Prometheus endpoints this collector scrapes **by name, with a + credential**, as ` = ""`. + + A service module declares its own entry, from its own module, the + same way it does for `scrapeTargets` — and the collector's OAuth2 + client derives its permitted audiences from these URLs, so a + target and the authorisation to reach it are **one declaration**. + Two lists that must agree would be a drift to maintain, and its + failure mode is the bad one: a target whose audience was forgotten + authenticates against nothing and reads as a scrape failure rather + than a config mistake. + + ⚠️ A **full URL**, not `host:port`. Authelia validates a bearer + token against the address being requested, so the string here is + also the audience the token is minted for; a near miss (a trailing + slash, `http` for `https`) presents as a valid token rejected at + the target, several layers from its cause. + + ⚠️ Deliberately separate from `scrapeTargets`. An entry there is + trusted because the scraper and the target share a host — that + option is loopback-and-unauthenticated by contract. An entry here + is trusted because it presents a credential. One shape for both + would leave a reader unable to tell which of those a given target + relies on. + ''; + }; + + clientId = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-collector"; + description = '' + OAuth2 client id this collector authenticates as. Published so + authelia's `access_control` rules can name it without carrying a + second copy of the string, exactly as + `services.hyperhive.swarm.authelia.hiveClientPrefix` is published + for the queue's responder. + + Two spellings drifting apart is not a build failure: authelia + refuses a rule naming an unregistered client in its startup + validator, so the swarm's SSO service fails to *restart* — long + after the change that caused it evaluated cleanly. + ''; + }; }; config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { @@ -267,6 +354,106 @@ in # telemetry. services.hyperhive.swarm.authelia.oidc.hiveIdentities = true; + # The collector's own identity, for the other direction: the hive + # identities above are how this collector authenticates its *callers*, + # this is how it authenticates *itself* to a service published behind + # the gateway. + # + # Only where authelia is co-located. A client is a row in this host's + # provider config, so declaring one against a remote provider would + # render nothing while reading as done; a swarm whose authelia lives + # elsewhere registers it there. + # + # ⚠️ Also conditional on there being a published target, and that is + # the shipped case rather than an edge — nothing declares one until + # some service publishes behind the gateway. Authelia refuses a + # bearer-authz client with no audience, so an unconditional + # declaration would break every hive that runs a collector and + # publishes nothing. + services.hyperhive.swarm.authelia.oidc.clients = + lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { }) + [ + { + id = cfg.clientId; + description = "HyperHive swarm collector"; + kind = "machine"; + # Grants `authelia.bearer.authz`, without which the authz + # endpoint refuses an otherwise valid token and blames the + # token rather than the missing grant. + bearerAuthz = true; + # DERIVED from the targets rather than contributed alongside + # them. A service declares a URL once and this is the + # permission to reach it; two lists that had to agree would be + # a drift to maintain, and the failure mode is the quiet one — + # a target whose audience was forgotten authenticates against + # nothing and looks like a broken scrape. + audience = lib.attrValues cfg.publishedScrapeTargets; + # Stated rather than left on authelia's default, because the + # two agreeing today is not the same as this being the + # required value: authelia permits only basic / JWT methods + # for a confidential client holding that scope, and enforces + # it in the startup validator. + tokenEndpointAuthMethod = "client_secret_basic"; + } + ]; + + # Deliver the collector's client secret from authelia's container into + # this one. On the HOST because that is the only place both container + # trees are addressable: they share this host's network namespace, which + # makes them feel co-located, but their filesystem roots are separate. + # + # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container + # refuses to start when a bind source is missing, and this secret does not + # exist until authelia's first boot has minted it — so binding it would + # make the collector wait on a file that waits on a container that starts + # after it. On a fresh swarm that is a permanent stall presenting as + # "metrics are broken", several layers from its cause. + systemd.services.swarm-otel-oidc-secret = + lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { }) + { + description = "deliver the swarm collector's OIDC client secret from authelia"; + after = [ "container@${autheliaCfg.machine}.service" ]; + requires = [ "container@${autheliaCfg.machine}.service" ]; + before = [ "container@${cfg.machine}.service" ]; + wantedBy = [ "container@${cfg.machine}.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-otel-oidc-secret"; + # Longer than the bounded wait below, and that is the point: + # `DefaultTimeoutStartSec` is 90s, so without this systemd kills + # the unit before it can emit the message naming the file it was + # waiting for — the failure then reads as a timeout with no cause. + TimeoutStartSec = "180s"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail + + src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"} + dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"} + + # authelia's container is up, but its first-boot generator may + # still be minting. Bounded wait, then fail: skipping silently + # produces a collector whose scrape gets a 401 forever, which is + # the failure this whole design exists to make impossible. + for _ in $(seq 1 60); do + [ -s "$src" ] && break + sleep 2 + done + if [ ! -s "$src" ]; then + echo "authelia has not minted $src after 120s" >&2 + exit 1 + fi + + # root-owned 0400. The collector runs under `DynamicUser`, so + # there is no uid to give it to — `LoadCredential` reads this as + # root before the sandbox exists and re-exposes it to whichever + # uid the unit got. + install -D -m 0400 -o root -g root "$src" "$dst" + ''; + }; + # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. @@ -328,6 +515,59 @@ in Rename the hive. Reserved: ${lib.concatMapStringsSep ", " (n: "'${n}'") reservedOwners}. ''; } + { + # A published target that is not an `https://host/path` url. Without + # this the split returns null and the failure surfaces as + # `attempt to call elemAt on null` from inside the renderer, naming + # neither the option nor the offending value. + # + # It also enforces the scheme: a published endpoint scraped over plain + # http would put a bearer token on the wire in clear text. + assertion = lib.all (u: parsePublished u != null) (lib.attrValues cfg.publishedScrapeTargets); + message = '' + services.hyperhive.swarm.otel.publishedScrapeTargets has ${ + lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") ( + lib.filter (kv: parsePublished kv.value == null) (lib.attrsToList cfg.publishedScrapeTargets) + ) + }, which is not of the form https:///. + + The value is both the address scraped and the audience the + collector's token is minted for, so it has to be the exact url a + request goes to. https is required: this hop carries a bearer + token, and plain http would put it on the wire in clear text. + ''; + } + { + # A job name used by BOTH scrape options. Within one option this + # cannot happen — the module system refuses two definitions of the + # same key with different values, measured — but the two options + # are separate, so nothing arbitrates between them, and they + # render into a single `scrape_configs` LIST where nothing + # overwrites anything: both entries ship under one `job_name`. + # + # The message names which side each collision came from, which is + # the part a rendered-config error could not tell an operator. + assertion = + lib.intersectLists (lib.attrNames cfg.scrapeTargets) (lib.attrNames cfg.publishedScrapeTargets) + == [ ]; + message = '' + services.hyperhive.swarm.otel: ${ + lib.concatMapStringsSep ", " (j: "'${j}'") ( + lib.intersectLists (lib.attrNames cfg.scrapeTargets) (lib.attrNames cfg.publishedScrapeTargets) + ) + } is declared as both a loopback + scrapeTargets job and a publishedScrapeTargets job. + + They render into one prometheus scrape_configs list, so both + entries would ship under the same job_name — nothing overwrites + anything, and the two are scraped by different rules with + different trust. + + Rename one. A target is either reachable on loopback because it + shares this host, or published and reached with a credential; it + should not be described as both. + ''; + } { # A hive proves who it is with a token this provider mints, so # there is no version of this collector that runs without one. @@ -476,11 +716,48 @@ in # target: a `prometheus` receiver with nothing to scrape is # the shape this whole issue is about, a config that renders # and deploys perfectly while adding no data. - // lib.optionalAttrs (cfg.scrapeTargets != { }) { - prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: { - job_name = job; - static_configs = [ { targets = [ target ]; } ]; - }) cfg.scrapeTargets; + // lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) { + prometheus.config.scrape_configs = + lib.mapAttrsToList (job: target: { + job_name = job; + static_configs = [ { targets = [ target ]; } ]; + }) cfg.scrapeTargets + # ⚠️ `++`, so the two kinds of target land in ONE list — + # which is exactly why a job name may not appear in both + # options. A list concatenation does not resolve a + # collision the way an attrset would: both entries ship + # under the same `job_name`. The assertion above is what + # stands between that and a deploy. + ++ lib.mapAttrsToList ( + job: url: + let + parts = parsePublished url; + in + { + job_name = job; + # Split from the declared url — see `parsePublished`. + scheme = "https"; + static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ]; + metrics_path = lib.elemAt parts 1; + # Prometheus-native oauth2, not the collector's + # `oauth2client` extension: the prometheus receiver + # takes upstream's scrape config verbatim and does not + # accept an `auth` block naming a collector extension. + oauth2 = { + client_id = cfg.clientId; + # A path, never a value — nothing here may read the + # secret, or it lands in the store world-readable. + client_secret_file = collectorSecretPath; + token_url = "${autheliaCfg.url}/api/oidc/token"; + # The audience is the target's own url, and authelia + # checks it against the address being requested. + # Registered ≠ requested: a client that does not ASK + # for an audience gets a token with `aud: []` however + # complete its registration looks. + endpoint_params.audience = url; + }; + } + ) cfg.publishedScrapeTargets; }; exporters = @@ -662,10 +939,20 @@ in # environment variable without being read by nix, written to the # store, or passed in argv. systemd.services.opentelemetry-collector.serviceConfig = - lib.optionalAttrs (otelCfg.headersCredential != null) - { - EnvironmentFile = otelCfg.headersCredential; - }; + lib.optionalAttrs (otelCfg.headersCredential != null) { + EnvironmentFile = otelCfg.headersCredential; + } + # The other credential, and the other direction: the one above + # authenticates this collector's export onward, this one + # authenticates it to a service it scrapes. + # + # ⚠️ Only where a published target exists. `LoadCredential` on a + # missing source is a unit that refuses to start, so declaring it + # unconditionally would take the collector down on every hive that + # scrapes nothing published — the empty case is the shipped one. + // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) { + LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ]; + }; }; }; };