diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 8c670efc..a4727fed 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -6,16 +6,11 @@ This page is that answer, one row per secret. Two rules run through all of it. -**Private key material and access tokens are paths, never values.** Every option -carrying one takes a file path (`*File`), because a literal written into a nix -expression is rendered into the nix store — world-readable and permanent. No -option in this tree accepts one inline, and adding one would be a leak rather -than a convenience. - -The rule is about what must stay secret, not about credentials generally. -**Public material is a value**: a certificate, or a public nkey like -`swarm.nats.calloutUserPublicKey`, is published to every client that connects, -so the store is a perfectly good place for it. +**A secret is a path, never a value.** Every option that carries a credential +takes a file path (`*File`), because a literal written into a nix expression is +rendered into the nix store — which is world-readable and permanent. There is no +option anywhere in this tree that accepts a secret inline, and adding one would +be a leak rather than a convenience. **The generator and the reader are usually in different containers.** They share the host's network namespace, which makes them feel co-located, but their @@ -49,42 +44,12 @@ Every row below is read against one of these. | OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | | authelia subject store | `swarmctl` | `users.json` (canonical) → `users.yml` (rendered) | `swarmctl`, on the host that runs authelia | | wireguard private key | **the operator** — `wg genkey` | whatever `swarm.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `nats.calloutUserSeedFile` / `nats.calloutIssuerSeedFile` | The three keys authelia mints for itself are generated in-container precisely because nothing outside that container ever reads them. **That is the test worth applying to any secret added here** — and the client secret's plaintext half is the one row that fails it, which is the entire reason a delivery step exists. -### Minting the queue's callout nkeys - -`nats.autoGenerateCallout` mints both keypairs on the host before the queue -starts. It is on by default only under `enableAllLocalDefaults` — the one -topology where the queue, its responder and the operator are the same person. On -every other topology, mint them yourself: - -``` -nk -gen user > callout-user.seed # the responder's own identity -nk -gen account > issuer.seed # signs the user JWTs it hands out -nk -inkey callout-user.seed -pubout # → calloutUserPublicKey -nk -inkey issuer.seed -pubout # → calloutIssuerPublicKey -``` - -Keep both seeds at `0600` and name them in `calloutUserSeedFile` / -`calloutIssuerSeedFile`. Possession of the **issuer** seed is the authority to -admit anyone to the queue, so it belongs wherever the responder runs and nowhere -else. - -A hive that sets neither the public keys nor `autoGenerateCallout` fails at -eval, naming the option it wants. That is deliberate: a queue that started -without them would accept `CONNECT {"user":"auth"}` from anyone sharing the -host's network namespace, and nothing would look wrong until somebody connected. - -One consequence of the generated path worth knowing before you debug it: with -`autoGenerateCallout` set, the queue's config is assembled at boot rather than at -build time, so a malformed one surfaces when the container starts instead of -when the system builds. The server names the offending file and refuses to run. - ## Hive-level — one of each per hive | secret | generated by | lives at | diff --git a/nix/host-modules/local-defaults.nix b/nix/host-modules/local-defaults.nix index 767de6aa..acda256e 100644 --- a/nix/host-modules/local-defaults.nix +++ b/nix/host-modules/local-defaults.nix @@ -74,12 +74,6 @@ in config.services.hyperhive.swarm = { enableRequiredServices = lib.mkDefault cfg.enableAllLocalDefaults; ca.autoConfigure = lib.mkDefault cfg.enableAllLocalDefaults; - # The queue's auth-callout nkeys. Generating them is safe exactly - # when one operator owns both the queue and its responder, which is - # what this mode asserts. On any other topology the seeds have to - # reach whoever runs the responder, and minting them here would move - # that hand-off somewhere less visible rather than removing it. - nats.autoGenerateCallout = lib.mkDefault cfg.enableAllLocalDefaults; # The controller is asserted by the MODE and by nothing else. Its own # option stays `default = false` precisely because running it is a # statement about swarm topology — but "this box is the whole diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 7895e78c..ada04d1f 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -29,153 +29,9 @@ let # The responder needs all three credentials. Gating on them rather than # on `cfg.enable` keeps a half-configured hive at "queue up, denying # everyone" instead of "unit crash-looping on a missing file". - # - # In auto mode the seeds are minted on this host before the container - # starts, so they are configured by construction. - responderConfigured = - cfg.autoGenerateCallout || (cfg.calloutUserSeedFile != "" && cfg.calloutIssuerSeedFile != ""); + responderConfigured = cfg.calloutUserSeedFile != "" && cfg.calloutIssuerSeedFile != ""; clientSecretSource = "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"; introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection"; - - # Where the responder's seeds actually come from. One name for two - # origins, so everything downstream stops caring which mode it is in. - userSeedFile = if cfg.autoGenerateCallout then autoUserSeed else cfg.calloutUserSeedFile; - issuerSeedFile = if cfg.autoGenerateCallout then autoIssuerSeed else cfg.calloutIssuerSeedFile; - - # Seeds stay on the host at 0600 and never enter the container or the - # store: only the responder needs them, and it reads them by - # `LoadCredential` from here. - autoSeedDir = "/var/lib/swarm-nats-callout"; - autoUserSeed = "${autoSeedDir}/callout-user.seed"; - autoIssuerSeed = "${autoSeedDir}/issuer.seed"; - - # The runtime config directory: wrapper, settings symlink and fragment. - # World-readable is correct (everything in it is public) and it is not in - # the 0700 responder secret dir, which the `nats` user cannot traverse. - # - # ⚠️ ONE DIRECTORY IS FORCED, NOT TIDINESS. NATS resolves an `include` with - # `filepath.Join(configDir, path)`, which strips a leading slash, so an - # absolute include silently becomes relative and is never found. The - # includes must therefore be bare filenames, i.e. siblings. Invisible to - # eval: the wrapper renders perfectly and the server refuses to start. - runtimeDir = "/var/lib/nats-callout"; - runtimeWrapper = "${runtimeDir}/nats.conf"; - hostRuntimeDir = "/var/lib/nixos-containers/${machine}${runtimeDir}"; - - natsFormat = pkgs.formats.json { }; - - # Upstream's rendered settings, re-rendered with the same generator on the - # same value — the artifact `services.nats` would have used, not a - # transcription. ⚠️ This reference is also the only thing keeping it alive: - # `ExecStart` names a runtime path, so nothing else in the closure names - # the store file, and its string context is what stops it being - # garbage-collected out from under a running server. - renderedSettings = natsFormat.generate "nats.conf" ( - config.containers.${machine}.config.services.nats.settings - ); - - # Defined once, rendered twice: into `settings` with the operator's values, - # and into the runtime fragment with placeholders the generator fills in. - # - # 🪤 Do not inline these and hand-write the fragment instead. Both files are - # loaded and the fragment is the LATER definition, so it wins — a - # hand-written copy would make a future edit to `settings` silently - # ineffective on exactly the hives that use auto mode. - calloutBlocks = - { - userKey, - issuerKey, - }: - { - # Two accounts, and the callout user lives in neither of the accounts - # it authorizes into. - accounts = { - # ⚠️ The nkey is not decoration and its absence was a real hole: a - # `users` entry carrying only a `user` name has no credential, and - # `CONNECT {"user":"auth"}` is then accepted with no password at - # all. Since the name is a literal in this public module, that made - # the callout-exempt identity walk-in-able from every container on - # the shared netns — the same class of hole this module exists to - # close, moved rather than fixed. Caught in review on the first - # version of this file. An nkey and NOTHING else, both halves - # measured against a running server rather than reasoned about: - # - # { user = "auth"; } → `CONNECT {"user":"auth"}` - # is accepted with no - # credential at all - # { user = "auth"; nkey = "U…"; } → refuses to START: - # "Nkey users do not take - # usernames or passwords" - # { nkey = "U…"; } → what this is - # - # A malformed key is fail-closed too: the server exits with - # "Not a valid public nkey for a user" rather than starting with a - # hole. So the only way to get a live server here is a real key - # whose seed nobody but the responder holds. - # - # 🔒 That property is what makes auto mode safe: it renders as "" - # there, so any field the fragment fails to override keeps a value - # the server refuses to start on. An incomplete merge cannot leave - # a walk-in-able server. - ${calloutAccount}.users = [ { nkey = userKey; } ]; - # ⚠️ `services.nats.jetstream = true` gives the SERVER JetStream; - # an account gets it only from its own grant. Measured against a - # running 2.14.1 with this exact two-account shape, because the - # failure is invisible to any config-rendering check: - # - # global jetstream only → `nats kv add` from this account fails - # `code=503 err_code=10039 jetstream not enabled for account`, - # while the server starts cleanly and logs "Starting JetStream" - # + this line → the same command succeeds - # - # The grant is per-account by design, and that is worth keeping: - # the callout account above deliberately does NOT get it. The - # responder mints credentials; it has no business holding stream - # state. - # - # ⚠️ Also why the fragment renders the COMPLETE accounts block: if a - # later definition replaced rather than merged, a partial one would - # drop this grant and every KV op would fail on a healthy server. - ${clientAccount} = { - jetstream = "enabled"; - }; - }; - - authorization = { - timeout = "2s"; - # 🔒 THIS BLOCK IS THE FAIL-CLOSED STATE, and it is the measured - # one rather than the obvious one. - # - # Measured on the pinned nats-server 2.14.1: both - # `authorization { }` and `authorization { users: [] }` accept an - # anonymous client and answer PONG — they read like "authorize - # nobody" and are wide open. An auth_callout block sets - # `auth_required` and refuses every client whose credential no - # responder has approved, so a config whose responder does not - # exist yet denies everyone. - # - # `nats-server -t` calls all three valid; it parses, it does not - # authenticate. Only running them tells the difference. - # - # ⇒ this is both the safe interim state and the final shape. - # Nothing here has to be swapped out when the responder lands - # beside it — it only starts being able to say yes. - auth_callout = { - issuer = issuerKey; - auth_users = [ userKey ]; - account = calloutAccount; - }; - }; - }; - - # The fragment as nix renders it, with placeholders where the runtime - # values go. Rendered by the same JSON generator upstream uses, so the - # fragment is generated rather than transcribed — NATS' config parser - # accepts JSON, and an `include` of it merges (measured). - calloutTemplate = natsFormat.generate "swarm-nats-callout-template.conf" (calloutBlocks { - userKey = "@USER_PUBKEY@"; - issuerKey = "@ISSUER_PUBKEY@"; - }); in { # The swarm's message queue: one NATS server, reached by every hive. @@ -229,26 +85,6 @@ in ''; }; - autoGenerateCallout = lib.mkOption { - type = lib.types.bool; - default = false; - example = true; - description = '' - Generate the auth-callout nkeys on this host instead of taking - them from `calloutUserPublicKey` / `calloutIssuerPublicKey`. - - A first-boot unit mints both keypairs if absent, keeps the seeds - host-side at `0600`, and writes only the public halves into a - fragment the server reads. Nothing secret is evaluated, so - nothing secret reaches the nix store. - - Leave it off wherever the queue and its clients are not the same - operator's problem: the seeds must reach whoever runs the - responder, and minting them here only moves that distribution - somewhere less visible. `enableAllLocalDefaults` turns it on. - ''; - }; - calloutUserPublicKey = lib.mkOption { type = lib.types.str; default = ""; @@ -353,7 +189,7 @@ in # Fail at EVAL, not at boot: a queue that comes up unable to # authenticate anyone presents as every client hanging, which is # several layers from "the operator never set the issuer". - assertion = cfg.autoGenerateCallout || cfg.calloutIssuerPublicKey != ""; + assertion = cfg.calloutIssuerPublicKey != ""; message = '' services.hyperhive.swarm.nats.enable requires nats.calloutIssuerPublicKey — the public half of the account @@ -370,7 +206,7 @@ in # failure is the only place to catch that: the rendered config is # valid, the server starts, and the hole is invisible until # somebody connects. - assertion = cfg.autoGenerateCallout || cfg.calloutUserPublicKey != ""; + assertion = cfg.calloutUserPublicKey != ""; message = '' services.hyperhive.swarm.nats.enable requires nats.calloutUserPublicKey — the public half of the user nkey @@ -464,32 +300,87 @@ in serverName = "swarm-nats"; port = cfg.port; - # In auto mode the keys are empty until the generator runs, - # and `nats-server -t` rejects that ("Expected callout user to - # be a valid public account nkey, got \"\""), so leaving this - # on fails the BUILD of every all-local hive. Upstream's own - # description names the case: disable it when the config - # includes other files. The check moves to server start. - validateConfig = !cfg.autoGenerateCallout; + settings = { + # Two accounts, and the callout user lives in neither of + # the accounts it authorizes into. + accounts = { + # ⚠️ The nkey is not decoration and its absence was a real + # hole: a `users` entry carrying only a `user` name has no + # credential, and `CONNECT {"user":"auth"}` is then + # accepted with no password at all. Since the name is a + # literal in this public module, that made the + # callout-exempt identity walk-in-able from every + # container on the shared netns — the same class of hole + # this module exists to close, moved rather than fixed. + # Caught in review on the first version of this file. + # An nkey and NOTHING else, both halves measured against a + # running server rather than reasoned about: + # + # { user = "auth"; } → `CONNECT {"user":"auth"}` + # is accepted with no + # credential at all + # { user = "auth"; nkey = "U…"; } → refuses to START: + # "Nkey users do not take + # usernames or passwords" + # { nkey = "U…"; } → what this is + # + # A malformed key is fail-closed too: the server exits with + # "Not a valid public nkey for a user" rather than starting + # with a hole. So the only way to get a live server here is + # a real key whose seed nobody but the responder holds. + ${calloutAccount}.users = [ { nkey = cfg.calloutUserPublicKey; } ]; + # ⚠️ `services.nats.jetstream = true` gives the SERVER + # JetStream; an account gets it only from its own grant. + # Measured against a running 2.14.1 with this exact + # two-account shape, because the failure is invisible to + # any config-rendering check: + # + # global jetstream only → `nats kv add` from this + # account fails `code=503 err_code=10039 jetstream + # not enabled for account`, while the server starts + # cleanly and logs "Starting JetStream" + # + this line → the same command succeeds + # + # The grant is per-account by design, and that is worth + # keeping: the callout account above deliberately does + # NOT get it. The responder mints credentials; it has no + # business holding stream state. + ${clientAccount} = { + jetstream = "enabled"; + }; + }; - settings = calloutBlocks { - userKey = cfg.calloutUserPublicKey; - issuerKey = cfg.calloutIssuerPublicKey; + authorization = { + timeout = "2s"; + # 🔒 THIS BLOCK IS THE FAIL-CLOSED STATE, and it is the + # measured one rather than the obvious one. + # + # Measured on the pinned nats-server 2.14.1: both + # `authorization { }` and `authorization { users: [] }` + # accept an anonymous client and answer PONG — they read + # like "authorize nobody" and are wide open. An + # auth_callout block sets `auth_required` and refuses + # every client whose credential no responder has + # approved, so a config whose responder does not exist + # yet denies everyone. + # + # `nats-server -t` calls all three valid; it parses, it + # does not authenticate. Only running them tells the + # difference. + # + # ⇒ this is both the safe interim state and the final + # shape. Nothing here has to be swapped out when the + # responder lands beside it — it only starts being able + # to say yes. + auth_callout = { + issuer = cfg.calloutIssuerPublicKey; + auth_users = [ cfg.calloutUserPublicKey ]; + account = calloutAccount; + }; + }; }; }; - # A wrapper that includes upstream's rendered settings verbatim - # plus the runtime fragment; rendering the config ourselves - # instead would throw away upstream's `settings`, where the - # reviewed reasoning lives. The generator writes it, because the - # includes must be siblings of the fragment (see `runtimeDir`). - # - # `mkForce`: upstream defines ExecStart inside an `mkMerge`, so a - # plain override conflicts rather than wins. - systemd.services.nats.serviceConfig.ExecStart = lib.mkIf cfg.autoGenerateCallout ( - lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}" - ); - # The auth-callout responder: the half that lets the server # above say *yes*. Without it the `auth_callout` block is a # door nobody can open, which is the deliberate interim state. @@ -553,12 +444,6 @@ in description = "deliver the swarm queue responder's credentials"; before = [ "container@swarm-nats.service" ]; wantedBy = [ "container@swarm-nats.service" ]; - # In auto mode the seeds this copies do not exist until the generator - # has run. `requires` as well as `after`: if minting fails there is - # nothing to deliver, and a copy that silently succeeds with a stale - # or absent seed is worse than not running. - after = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service"; - requires = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service"; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -568,83 +453,13 @@ in script = '' set -euo pipefail install -d -m 0700 ${lib.escapeShellArg secretDir} - install -m 0400 ${lib.escapeShellArg userSeedFile} \ + install -m 0400 ${lib.escapeShellArg cfg.calloutUserSeedFile} \ ${lib.escapeShellArg (hostPath "callout-user.seed")} - install -m 0400 ${lib.escapeShellArg issuerSeedFile} \ + install -m 0400 ${lib.escapeShellArg cfg.calloutIssuerSeedFile} \ ${lib.escapeShellArg (hostPath "issuer.seed")} install -m 0400 ${lib.escapeShellArg clientSecretSource} \ ${lib.escapeShellArg (hostPath "oidc-client.secret")} ''; }; - - # ⚠️ Minted on the HOST, not in the container, because the responder is - # a separate unit that needs the user seed: generating it inside would - # trap it there and require a secret-export path back out — the exact - # mechanism this is meant to avoid inventing. Only public halves cross. - systemd.services.swarm-nats-callout-keys = lib.mkIf cfg.autoGenerateCallout { - description = "mint the swarm queue's auth-callout nkeys"; - before = [ "container@swarm-nats.service" ]; - wantedBy = [ "container@swarm-nats.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "swarm-nats-callout-keys"; - }; - path = [ - pkgs.coreutils - pkgs.nkeys - ]; - script = '' - set -euo pipefail - umask 077 - install -d -m 0700 ${lib.escapeShellArg autoSeedDir} - - # Mint iff absent. Idempotence is the whole contract: this runs on - # every boot, and regenerating would silently invalidate every - # credential the responder has already issued against the old - # issuer. - if [ ! -s ${lib.escapeShellArg autoUserSeed} ]; then - nk -gen user > ${lib.escapeShellArg autoUserSeed}.tmp - mv ${lib.escapeShellArg autoUserSeed}.tmp ${lib.escapeShellArg autoUserSeed} - fi - if [ ! -s ${lib.escapeShellArg autoIssuerSeed} ]; then - nk -gen account > ${lib.escapeShellArg autoIssuerSeed}.tmp - mv ${lib.escapeShellArg autoIssuerSeed}.tmp ${lib.escapeShellArg autoIssuerSeed} - fi - chmod 0600 ${lib.escapeShellArg autoUserSeed} ${lib.escapeShellArg autoIssuerSeed} - - user_pub="$(nk -inkey ${lib.escapeShellArg autoUserSeed} -pubout)" - issuer_pub="$(nk -inkey ${lib.escapeShellArg autoIssuerSeed} -pubout)" - - # Public halves only — world-readable on purpose, since the server - # publishes them to every client that connects. - install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir} - sed -e "s|@USER_PUBKEY@|$user_pub|g" \ - -e "s|@ISSUER_PUBKEY@|$issuer_pub|g" \ - ${calloutTemplate} > ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp - # Mode BEFORE the rename: a rename publishes whatever the file - # already is, so setting it afterwards leaves a window where the - # live path has the wrong mode. Same trap as the gateway's - # atomic-publish path. - chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp - mv ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp \ - ${lib.escapeShellArg hostRuntimeDir}/callout.conf - - # Upstream's rendered settings, as a sibling the wrapper can name - # without a leading slash. Refreshed unconditionally — unlike the - # seeds, this one MUST track the current system, and a stale copy - # would silently run yesterday's config. - ln -sfn ${renderedSettings} ${lib.escapeShellArg hostRuntimeDir}/settings.conf - - # The wrapper. Bare filenames — see `runtimeDir`. printf rather than - # a heredoc, whose terminator would depend on nix's indentation - # stripping and break the next time `nix fmt` touched this block. - printf 'include "settings.conf"\ninclude "callout.conf"\n' \ - > ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp - chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp - mv ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp \ - ${lib.escapeShellArg hostRuntimeDir}/nats.conf - ''; - }; }; }