diff --git a/nix/host-modules/local-defaults.nix b/nix/host-modules/local-defaults.nix index acda256e..767de6aa 100644 --- a/nix/host-modules/local-defaults.nix +++ b/nix/host-modules/local-defaults.nix @@ -74,6 +74,12 @@ in config.services.hyperhive.swarm = { enableRequiredServices = lib.mkDefault cfg.enableAllLocalDefaults; ca.autoConfigure = lib.mkDefault cfg.enableAllLocalDefaults; + # The queue's auth-callout nkeys. Generating them is safe exactly + # when one operator owns both the queue and its responder, which is + # what this mode asserts. On any other topology the seeds have to + # reach whoever runs the responder, and minting them here would move + # that hand-off somewhere less visible rather than removing it. + nats.autoGenerateCallout = lib.mkDefault cfg.enableAllLocalDefaults; # The controller is asserted by the MODE and by nothing else. Its own # option stays `default = false` precisely because running it is a # statement about swarm topology — but "this box is the whole diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index ada04d1f..470d097e 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -29,9 +29,175 @@ let # The responder needs all three credentials. Gating on them rather than # on `cfg.enable` keeps a half-configured hive at "queue up, denying # everyone" instead of "unit crash-looping on a missing file". - responderConfigured = cfg.calloutUserSeedFile != "" && cfg.calloutIssuerSeedFile != ""; + # + # In auto mode the seeds are minted on this host before the container + # starts, so they are configured by construction. + responderConfigured = + cfg.autoGenerateCallout || (cfg.calloutUserSeedFile != "" && cfg.calloutIssuerSeedFile != ""); clientSecretSource = "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"; introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection"; + + # Where the responder's seeds actually come from. One name for two + # origins, so everything downstream stops caring which mode it is in. + userSeedFile = if cfg.autoGenerateCallout then autoUserSeed else cfg.calloutUserSeedFile; + issuerSeedFile = if cfg.autoGenerateCallout then autoIssuerSeed else cfg.calloutIssuerSeedFile; + + # Seeds stay on the host at 0600 and never enter the container or the + # store: only the responder needs them, and it reads them by + # `LoadCredential` from here. + autoSeedDir = "/var/lib/swarm-nats-callout"; + autoUserSeed = "${autoSeedDir}/callout-user.seed"; + autoIssuerSeed = "${autoSeedDir}/issuer.seed"; + + # The runtime config directory. World-readable is correct — everything in + # it is public (the server publishes these keys to every client that + # connects) — and it deliberately does not live in the 0700 responder + # secret dir, which the `nats` user cannot traverse. + # + # ⚠️ ALL THREE FILES LIVE IN ONE DIRECTORY, AND THAT IS FORCED, NOT + # TIDINESS. `include` paths are resolved with Go's `filepath.Join` against + # the config file's own directory, which **strips a leading slash** — so an + # absolute include silently becomes a relative one: + # + # include "/nix/store/…-nats.conf" + # → open agents/atlas/…/nix/store/…-nats.conf: no such file + # + # Measured, and it is invisible to every eval-level check: the wrapper + # renders perfectly and the server refuses to start. ⇒ the includes must be + # bare filenames, which means the wrapper, the settings and the fragment + # have to be siblings. (Also measured: resolution is relative to the CONFIG + # FILE, not the process CWD, so an absolute `-c` path is fine.) + runtimeDir = "/var/lib/nats-callout"; + runtimeWrapper = "${runtimeDir}/nats.conf"; + hostRuntimeDir = "/var/lib/nixos-containers/${machine}${runtimeDir}"; + + natsFormat = pkgs.formats.json { }; + + # Upstream's rendered settings, read back out of the container's own + # evaluated config and re-rendered with the same generator — so this is the + # artifact `services.nats` would have used, not a transcription of it. + # Referencing it here is also what keeps it alive: with `ExecStart` pointed + # at a runtime path, nothing else in the closure names it, and the string + # context of this reference is what stops it being garbage-collected out + # from under a running server. + # + # ⚠️ It should be the very same store path as upstream's (same function, + # same value), but nothing depends on that being true: if it ever diverged, + # this is still a correct rendering of the same settings. + renderedSettings = natsFormat.generate "nats.conf" ( + config.containers.${machine}.config.services.nats.settings + ); + + # The two blocks whose contents depend on the callout keys, defined ONCE + # and rendered TWICE: into `services.nats.settings` with the operator's + # values, and into the runtime fragment with placeholders the generator + # substitutes. + # + # 🪤 Do NOT inline these into `settings` and hand-write the fragment. Both + # files are loaded, and the fragment is the LATER definition, so it WINS + # (measured — state/probe-3112-override.sh). A hand-written copy would + # mean a future edit to `settings` alone is silently ignored on exactly + # the hives that use auto mode: the config looks changed and behaves + # unchanged. One definition is what makes that impossible rather than + # merely unlikely. + calloutBlocks = + { + userKey, + issuerKey, + }: + { + # Two accounts, and the callout user lives in neither of the accounts + # it authorizes into. + accounts = { + # ⚠️ The nkey is not decoration and its absence was a real hole: a + # `users` entry carrying only a `user` name has no credential, and + # `CONNECT {"user":"auth"}` is then accepted with no password at + # all. Since the name is a literal in this public module, that made + # the callout-exempt identity walk-in-able from every container on + # the shared netns — the same class of hole this module exists to + # close, moved rather than fixed. Caught in review on the first + # version of this file. An nkey and NOTHING else, both halves + # measured against a running server rather than reasoned about: + # + # { user = "auth"; } → `CONNECT {"user":"auth"}` + # is accepted with no + # credential at all + # { user = "auth"; nkey = "U…"; } → refuses to START: + # "Nkey users do not take + # usernames or passwords" + # { nkey = "U…"; } → what this is + # + # A malformed key is fail-closed too: the server exits with + # "Not a valid public nkey for a user" rather than starting with a + # hole. So the only way to get a live server here is a real key + # whose seed nobody but the responder holds. + # + # 🔒 That last property is what makes auto mode safe. In auto mode + # this renders as "" — which the server refuses to start on — and + # the fragment overrides it at runtime. Every field the merge + # fails to reach therefore keeps a value that FAILS CLOSED LOUDLY, + # so an incomplete merge can never leave a walk-in-able server. + ${calloutAccount}.users = [ { nkey = userKey; } ]; + # ⚠️ `services.nats.jetstream = true` gives the SERVER JetStream; + # an account gets it only from its own grant. Measured against a + # running 2.14.1 with this exact two-account shape, because the + # failure is invisible to any config-rendering check: + # + # global jetstream only → `nats kv add` from this account fails + # `code=503 err_code=10039 jetstream not enabled for account`, + # while the server starts cleanly and logs "Starting JetStream" + # + this line → the same command succeeds + # + # The grant is per-account by design, and that is worth keeping: + # the callout account above deliberately does NOT get it. The + # responder mints credentials; it has no business holding stream + # state. + # + # ⚠️ This is also why the fragment renders the COMPLETE accounts + # block rather than only the AUTH entry: if a later definition + # replaced the block instead of merging into it, a partial fragment + # would drop this grant, and the result is a healthy-looking server + # whose every KV op fails at runtime. + ${clientAccount} = { + jetstream = "enabled"; + }; + }; + + authorization = { + timeout = "2s"; + # 🔒 THIS BLOCK IS THE FAIL-CLOSED STATE, and it is the measured + # one rather than the obvious one. + # + # Measured on the pinned nats-server 2.14.1: both + # `authorization { }` and `authorization { users: [] }` accept an + # anonymous client and answer PONG — they read like "authorize + # nobody" and are wide open. An auth_callout block sets + # `auth_required` and refuses every client whose credential no + # responder has approved, so a config whose responder does not + # exist yet denies everyone. + # + # `nats-server -t` calls all three valid; it parses, it does not + # authenticate. Only running them tells the difference. + # + # ⇒ this is both the safe interim state and the final shape. + # Nothing here has to be swapped out when the responder lands + # beside it — it only starts being able to say yes. + auth_callout = { + issuer = issuerKey; + auth_users = [ userKey ]; + account = calloutAccount; + }; + }; + }; + + # The fragment as nix renders it, with placeholders where the runtime + # values go. Rendered by the same JSON generator upstream uses, so the + # fragment is generated rather than transcribed — NATS' config parser + # accepts JSON, and an `include` of it merges (measured). + calloutTemplate = natsFormat.generate "swarm-nats-callout-template.conf" (calloutBlocks { + userKey = "@USER_PUBKEY@"; + issuerKey = "@ISSUER_PUBKEY@"; + }); in { # The swarm's message queue: one NATS server, reached by every hive. @@ -85,6 +251,32 @@ in ''; }; + autoGenerateCallout = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Generate the auth-callout nkeys on this host instead of taking + them from `calloutUserPublicKey` / `calloutIssuerPublicKey`. + + A first-boot unit mints both keypairs if they are absent, keeps + the seeds host-side at `0600`, and writes only the public halves + into a fragment the server reads. Nothing secret is evaluated, + so nothing secret reaches the nix store. + + Off by default: an option says what it is, not what a deployment + shape implies. The all-local mode turns this on — see + `services.hyperhive.enableAllLocalDefaults` in + ./local-defaults.nix, which is the one place that decides what + "the whole swarm is this box" means. + + Leave it off wherever the queue and its clients are not the same + operator's problem: the seeds have to be distributed to whoever + runs the responder, and generating them here would just move + that distribution somewhere less visible. + ''; + }; + calloutUserPublicKey = lib.mkOption { type = lib.types.str; default = ""; @@ -189,7 +381,7 @@ in # Fail at EVAL, not at boot: a queue that comes up unable to # authenticate anyone presents as every client hanging, which is # several layers from "the operator never set the issuer". - assertion = cfg.calloutIssuerPublicKey != ""; + assertion = cfg.autoGenerateCallout || cfg.calloutIssuerPublicKey != ""; message = '' services.hyperhive.swarm.nats.enable requires nats.calloutIssuerPublicKey — the public half of the account @@ -206,7 +398,7 @@ in # failure is the only place to catch that: the rendered config is # valid, the server starts, and the hole is invisible until # somebody connects. - assertion = cfg.calloutUserPublicKey != ""; + assertion = cfg.autoGenerateCallout || cfg.calloutUserPublicKey != ""; message = '' services.hyperhive.swarm.nats.enable requires nats.calloutUserPublicKey — the public half of the user nkey @@ -300,87 +492,56 @@ in serverName = "swarm-nats"; port = cfg.port; - settings = { - # Two accounts, and the callout user lives in neither of - # the accounts it authorizes into. - accounts = { - # ⚠️ The nkey is not decoration and its absence was a real - # hole: a `users` entry carrying only a `user` name has no - # credential, and `CONNECT {"user":"auth"}` is then - # accepted with no password at all. Since the name is a - # literal in this public module, that made the - # callout-exempt identity walk-in-able from every - # container on the shared netns — the same class of hole - # this module exists to close, moved rather than fixed. - # Caught in review on the first version of this file. - # An nkey and NOTHING else, both halves measured against a - # running server rather than reasoned about: - # - # { user = "auth"; } → `CONNECT {"user":"auth"}` - # is accepted with no - # credential at all - # { user = "auth"; nkey = "U…"; } → refuses to START: - # "Nkey users do not take - # usernames or passwords" - # { nkey = "U…"; } → what this is - # - # A malformed key is fail-closed too: the server exits with - # "Not a valid public nkey for a user" rather than starting - # with a hole. So the only way to get a live server here is - # a real key whose seed nobody but the responder holds. - ${calloutAccount}.users = [ { nkey = cfg.calloutUserPublicKey; } ]; - # ⚠️ `services.nats.jetstream = true` gives the SERVER - # JetStream; an account gets it only from its own grant. - # Measured against a running 2.14.1 with this exact - # two-account shape, because the failure is invisible to - # any config-rendering check: - # - # global jetstream only → `nats kv add` from this - # account fails `code=503 err_code=10039 jetstream - # not enabled for account`, while the server starts - # cleanly and logs "Starting JetStream" - # + this line → the same command succeeds - # - # The grant is per-account by design, and that is worth - # keeping: the callout account above deliberately does - # NOT get it. The responder mints credentials; it has no - # business holding stream state. - ${clientAccount} = { - jetstream = "enabled"; - }; - }; + # ⚠️ Build-time validation cannot survive auto mode, and this + # is measured rather than assumed. In auto mode the keys do + # not exist until the generator runs, so this renders with + # empty strings, and `nats-server -t` rejects exactly that: + # + # User entry requires a user + # Expected callout user to be a valid public account nkey, + # got "" + # + # ⇒ leaving it on would fail the BUILD of every all-local + # hive. Upstream's own description of this option names the + # case: disable it "when the config includes other files". + # + # The check is not lost, it MOVES: the server parses both + # halves at start, with the fragment present, and refuses to + # come up if either is malformed. Later rather than at build + # time — say so in the docs; do not paper over it with an + # assertion of our own, which would be a second system + # deciding what counts as a valid config. + validateConfig = !cfg.autoGenerateCallout; - authorization = { - timeout = "2s"; - # 🔒 THIS BLOCK IS THE FAIL-CLOSED STATE, and it is the - # measured one rather than the obvious one. - # - # Measured on the pinned nats-server 2.14.1: both - # `authorization { }` and `authorization { users: [] }` - # accept an anonymous client and answer PONG — they read - # like "authorize nobody" and are wide open. An - # auth_callout block sets `auth_required` and refuses - # every client whose credential no responder has - # approved, so a config whose responder does not exist - # yet denies everyone. - # - # `nats-server -t` calls all three valid; it parses, it - # does not authenticate. Only running them tells the - # difference. - # - # ⇒ this is both the safe interim state and the final - # shape. Nothing here has to be swapped out when the - # responder lands beside it — it only starts being able - # to say yes. - auth_callout = { - issuer = cfg.calloutIssuerPublicKey; - auth_users = [ cfg.calloutUserPublicKey ]; - account = calloutAccount; - }; - }; + # One definition, rendered here with the operator's values and + # into the runtime fragment with placeholders. See + # `calloutBlocks` for why it is not written out twice. + settings = calloutBlocks { + userKey = cfg.calloutUserPublicKey; + issuerKey = cfg.calloutIssuerPublicKey; }; }; + # Auto mode only: point the server at a wrapper that includes + # upstream's rendered settings *verbatim* plus the runtime + # fragment. The alternative — teaching this module to render the + # config itself — would throw away upstream's `settings` + # rendering, which is where all the reviewed reasoning lives. + # + # The wrapper is written by the generator rather than being a + # store file, because its includes must be siblings of the + # fragment (see `runtimeDir`). A server started before the + # generator has run therefore finds no config at all and refuses, + # which is the same fail-closed direction as every other failure + # mode here. + # + # `mkForce` because `services.nats` defines ExecStart inside an + # `mkMerge`; a plain override conflicts rather than winning + # (measured — state/probe-3112-mkforce.sh). + systemd.services.nats.serviceConfig.ExecStart = lib.mkIf cfg.autoGenerateCallout ( + lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}" + ); + # The auth-callout responder: the half that lets the server # above say *yes*. Without it the `auth_callout` block is a # door nobody can open, which is the deliberate interim state. @@ -444,6 +605,12 @@ in description = "deliver the swarm queue responder's credentials"; before = [ "container@swarm-nats.service" ]; wantedBy = [ "container@swarm-nats.service" ]; + # In auto mode the seeds this copies do not exist until the generator + # has run. `requires` as well as `after`: if minting fails there is + # nothing to deliver, and a copy that silently succeeds with a stale + # or absent seed is worse than not running. + after = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service"; + requires = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service"; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -453,13 +620,92 @@ in script = '' set -euo pipefail install -d -m 0700 ${lib.escapeShellArg secretDir} - install -m 0400 ${lib.escapeShellArg cfg.calloutUserSeedFile} \ + install -m 0400 ${lib.escapeShellArg userSeedFile} \ ${lib.escapeShellArg (hostPath "callout-user.seed")} - install -m 0400 ${lib.escapeShellArg cfg.calloutIssuerSeedFile} \ + install -m 0400 ${lib.escapeShellArg issuerSeedFile} \ ${lib.escapeShellArg (hostPath "issuer.seed")} install -m 0400 ${lib.escapeShellArg clientSecretSource} \ ${lib.escapeShellArg (hostPath "oidc-client.secret")} ''; }; + + # Auto mode: mint the callout keypairs on this host before anything + # that needs them exists. + # + # ⚠️ ON THE HOST, not inside the container, and that is the decision + # the container boundary forces. The responder is a separate unit that + # needs the user SEED; generating inside the container would trap the + # seed there and require a way back out — a new secret-export path, + # which is precisely the mechanism this issue exists to avoid + # inventing. Here, the seeds never leave a 0700 host directory and only + # the public halves cross the boundary. + systemd.services.swarm-nats-callout-keys = lib.mkIf cfg.autoGenerateCallout { + description = "mint the swarm queue's auth-callout nkeys"; + before = [ "container@swarm-nats.service" ]; + wantedBy = [ "container@swarm-nats.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-nats-callout-keys"; + }; + path = [ + pkgs.coreutils + pkgs.nkeys + ]; + script = '' + set -euo pipefail + umask 077 + install -d -m 0700 ${lib.escapeShellArg autoSeedDir} + + # Mint iff absent. Idempotence is the whole contract: this runs on + # every boot, and regenerating would silently invalidate every + # credential the responder has already issued against the old + # issuer. + if [ ! -s ${lib.escapeShellArg autoUserSeed} ]; then + nk -gen user > ${lib.escapeShellArg autoUserSeed}.tmp + mv ${lib.escapeShellArg autoUserSeed}.tmp ${lib.escapeShellArg autoUserSeed} + fi + if [ ! -s ${lib.escapeShellArg autoIssuerSeed} ]; then + nk -gen account > ${lib.escapeShellArg autoIssuerSeed}.tmp + mv ${lib.escapeShellArg autoIssuerSeed}.tmp ${lib.escapeShellArg autoIssuerSeed} + fi + chmod 0600 ${lib.escapeShellArg autoUserSeed} ${lib.escapeShellArg autoIssuerSeed} + + user_pub="$(nk -inkey ${lib.escapeShellArg autoUserSeed} -pubout)" + issuer_pub="$(nk -inkey ${lib.escapeShellArg autoIssuerSeed} -pubout)" + + # Public halves only — world-readable on purpose, since the server + # publishes them to every client that connects. + install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir} + sed -e "s|@USER_PUBKEY@|$user_pub|g" \ + -e "s|@ISSUER_PUBKEY@|$issuer_pub|g" \ + ${calloutTemplate} > ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp + # Mode BEFORE the rename: a rename publishes whatever the file + # already is, so setting it afterwards leaves a window where the + # live path has the wrong mode. Same trap as the gateway's + # atomic-publish path. + chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp + mv ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp \ + ${lib.escapeShellArg hostRuntimeDir}/callout.conf + + # Upstream's rendered settings, as a sibling the wrapper can name + # without a leading slash. Refreshed unconditionally — unlike the + # seeds, this one MUST track the current system, and a stale copy + # would silently run yesterday's config. + ln -sfn ${renderedSettings} ${lib.escapeShellArg hostRuntimeDir}/settings.conf + + # The wrapper. Bare filenames on purpose: NATS resolves an include + # against the config file's own directory and strips a leading + # slash, so an absolute path here would silently fail to open. + # printf, not a heredoc: this lives inside a nix indented string, so + # a heredoc's terminator depends on nix's indentation stripping and + # would break the moment `nix fmt` re-indented this block. + printf 'include "settings.conf"\ninclude "callout.conf"\n' \ + > ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp + chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp + mv ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp \ + ${lib.escapeShellArg hostRuntimeDir}/nats.conf + ''; + }; }; }