diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 4c1cdd4a..d02e51b8 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -14,7 +14,7 @@ than a convenience. The rule is about what must stay secret, not about credentials generally. **Public material is a value**: a certificate, or a public nkey like -`swarm.nats.calloutUserPublicKey`, is published to every client that connects, +`deploy.nats.calloutUserPublicKey`, is published to every client that connects, so the store is a perfectly good place for it. **The generator and the reader are usually in different containers.** They share @@ -54,7 +54,7 @@ neither is a renaming of the other. | the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `swarm.otel.clientSecretFile` — the collector need not share a host with authelia | | authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | | wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `nats.calloutUserSeedFile` / `nats.calloutIssuerSeedFile` | +| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `deploy.nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `deploy.nats.calloutUserSeedFile` / `deploy.nats.calloutIssuerSeedFile` | | the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | | the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it is stated rather than inferred | n/a — only the host running the store seals anything | @@ -90,7 +90,7 @@ not the definition of the case. ### Minting the queue's callout nkeys -`nats.autoGenerateCallout` mints both keypairs on the host before the queue +`deploy.nats.autoGenerateCallout` mints both keypairs on the host before the queue starts. It is on by default only under `singleHostSwarm` — the one topology where the queue, its responder and the operator are the same person. On every other topology, mint them yourself: @@ -102,8 +102,8 @@ nk -inkey callout-user.seed -pubout # → calloutUserPublicKey nk -inkey issuer.seed -pubout # → calloutIssuerPublicKey ``` -Keep both seeds at `0600` and name them in `calloutUserSeedFile` / -`calloutIssuerSeedFile`. Possession of the **issuer** seed is the authority to +Keep both seeds at `0600` and name them in `deploy.nats.calloutUserSeedFile` / +`deploy.nats.calloutIssuerSeedFile`. Possession of the **issuer** seed is the authority to admit anyone to the queue, so it belongs wherever the responder runs and nowhere else. diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 2ad3738f..757fd42c 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -98,6 +98,37 @@ in [ "services" "hyperhive" "deploy" "forgejo" "ci" "package" ] ) + # The queue's callout identity. What stays under `swarm.nats` is what the + # queue IS to every hive — the ports it answers on, the client id it is + # registered under, the responder package; these five are what the host + # running it decides. + # + # ⚠️ The two PUBLIC keys move with their seeds. A public key looks + # swarm-wide, and the user key genuinely is published to every client that + # connects — but peers RECEIVE it over the wire, they never configure it, + # and splitting a keypair across two namespaces is worse than either + # placement. `enable` moved earlier; its shim is the one above. + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "autoGenerateCallout" ] + [ "services" "hyperhive" "deploy" "nats" "autoGenerateCallout" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "calloutUserPublicKey" ] + [ "services" "hyperhive" "deploy" "nats" "calloutUserPublicKey" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "calloutIssuerPublicKey" ] + [ "services" "hyperhive" "deploy" "nats" "calloutIssuerPublicKey" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "calloutUserSeedFile" ] + [ "services" "hyperhive" "deploy" "nats" "calloutUserSeedFile" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "calloutIssuerSeedFile" ] + [ "services" "hyperhive" "deploy" "nats" "calloutIssuerSeedFile" ] + ) + # The rest of the forge split. What stays under `swarm.forge` is what the # forge IS from any hive's point of view — its package, the names and ports # it answers on, the URLs it advertises, the client id it is registered diff --git a/nix/host-modules/local-defaults.nix b/nix/host-modules/local-defaults.nix index 528f86de..8ef4b51d 100644 --- a/nix/host-modules/local-defaults.nix +++ b/nix/host-modules/local-defaults.nix @@ -78,14 +78,16 @@ in # the ⚠️ below gives about `swarm`. config.services.hyperhive.deploy.allSwarmServices = lib.mkDefault cfg.deploy.singleHostSwarm; + # The queue's auth-callout nkeys. Generating them is safe exactly + # when one operator owns both the queue and its responder, which is + # what this mode asserts. On any other topology the seeds have to + # reach whoever runs the responder, and minting them here would move + # that hand-off somewhere less visible rather than removing it. + config.services.hyperhive.deploy.nats.autoGenerateCallout = + lib.mkDefault cfg.deploy.singleHostSwarm; + config.services.hyperhive.swarm = { ca.autoConfigure = lib.mkDefault cfg.deploy.singleHostSwarm; - # The queue's auth-callout nkeys. Generating them is safe exactly - # when one operator owns both the queue and its responder, which is - # what this mode asserts. On any other topology the seeds have to - # reach whoever runs the responder, and minting them here would move - # that hand-off somewhere less visible rather than removing it. - nats.autoGenerateCallout = lib.mkDefault cfg.deploy.singleHostSwarm; # The controller's queue coordinates. Co-location is what makes these # derivable at all — loopback only reaches the queue when the queue is # here, and the minted client secret only exists on the host authelia diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index bb7d8ddd..b1f92ff4 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -32,7 +32,6 @@ let SWARMCTL_AUTHELIA_USERS_FILE = autheliaCfg.hostUsersFile; }; - natsCfg = config.services.hyperhive.swarm.nats; forgeCfg = config.services.hyperhive.swarm.forge; swarmDomain = config.services.hyperhive.swarm.domain; diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 989522a6..605c5c4a 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -55,14 +55,17 @@ let # In auto mode the seeds are minted on this host before the container # starts, so they are configured by construction. responderConfigured = - cfg.autoGenerateCallout || (cfg.calloutUserSeedFile != "" && cfg.calloutIssuerSeedFile != ""); + deployCfg.nats.autoGenerateCallout + || (deployCfg.nats.calloutUserSeedFile != "" && deployCfg.nats.calloutIssuerSeedFile != ""); clientSecretSource = "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"; introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection"; # Where the responder's seeds actually come from. One name for two # origins, so everything downstream stops caring which mode it is in. - userSeedFile = if cfg.autoGenerateCallout then autoUserSeed else cfg.calloutUserSeedFile; - issuerSeedFile = if cfg.autoGenerateCallout then autoIssuerSeed else cfg.calloutIssuerSeedFile; + userSeedFile = + if deployCfg.nats.autoGenerateCallout then autoUserSeed else deployCfg.nats.calloutUserSeedFile; + issuerSeedFile = + if deployCfg.nats.autoGenerateCallout then autoIssuerSeed else deployCfg.nats.calloutIssuerSeedFile; # Seeds stay on the host at 0600 and never enter the container or the # store: only the responder needs them, and it reads them by @@ -281,6 +284,32 @@ in ''; }; + authPackage = lib.mkOption { + type = lib.types.package; + defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth"; + description = '' + The auth-callout responder package. + + ⚠️ Named `authPackage`, not `package`, on purpose: this module + deliberately has **no** `package` option for the server itself + (see the note above — upstream's `services.nats` resolves + `pkgs.nats-server` on its own), so a bare `package` here would + read as "the NATS package" and mean something else entirely. + ''; + }; + }; + + # What stays above is what the queue IS to every hive: the ports it answers + # on, the client id it is registered under, the responder package. The + # callout identity below is what the host running it decides — whether it + # mints its own keypairs, and where the seeds sit. `enable` already lives in + # ./deploy.nix, which also carries the renames. + # + # ⚠️ The two PUBLIC keys move with their seeds rather than staying: peers + # receive the user key over the wire when they connect (docs/swarm/secrets.md), + # they never configure it, and splitting a keypair across two namespaces is + # worse than either placement. + options.services.hyperhive.deploy.nats = { autoGenerateCallout = lib.mkOption { type = lib.types.bool; default = false; @@ -324,7 +353,7 @@ in — which is what makes a hive with no responder genuinely closed rather than merely gated. - Required when `enable` is set. + Required when `deploy.nats.enable` is set. ''; }; @@ -342,27 +371,13 @@ in and its whole job is to be widely known; the matching *seed* is the secret, is never named here, and reaches only the responder. - Required when `enable` is set. Without it the server has no - issuer to trust and no client can be authorized — which is the - fail-closed state described below, but arrived at by accident + Required when `deploy.nats.enable` is set. Without it the server + has no issuer to trust and no client can be authorized — which is + the fail-closed state described below, but arrived at by accident rather than on purpose, so it fails at eval instead. ''; }; - authPackage = lib.mkOption { - type = lib.types.package; - defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth"; - description = '' - The auth-callout responder package. - - ⚠️ Named `authPackage`, not `package`, on purpose: this module - deliberately has **no** `package` option for the server itself - (see the note above — upstream's `services.nats` resolves - `pkgs.nats-server` on its own), so a bare `package` here would - read as "the NATS package" and mean something else entirely. - ''; - }; - calloutUserSeedFile = lib.mkOption { type = lib.types.str; default = ""; @@ -412,11 +427,12 @@ in # Fail at EVAL, not at boot: a queue that comes up unable to # authenticate anyone presents as every client hanging, which is # several layers from "the operator never set the issuer". - assertion = cfg.autoGenerateCallout || cfg.calloutIssuerPublicKey != ""; + assertion = deployCfg.nats.autoGenerateCallout || deployCfg.nats.calloutIssuerPublicKey != ""; message = '' services.hyperhive.deploy.nats.enable requires - nats.calloutIssuerPublicKey — the public half of the account - nkey that signs user JWTs for this queue. + services.hyperhive.deploy.nats.calloutIssuerPublicKey — the + public half of the account nkey that signs user JWTs for this + queue. It is public and belongs in config; the matching seed is a secret and is delivered to the callout responder instead. See @@ -429,11 +445,12 @@ in # failure is the only place to catch that: the rendered config is # valid, the server starts, and the hole is invisible until # somebody connects. - assertion = cfg.autoGenerateCallout || cfg.calloutUserPublicKey != ""; + assertion = deployCfg.nats.autoGenerateCallout || deployCfg.nats.calloutUserPublicKey != ""; message = '' services.hyperhive.deploy.nats.enable requires - nats.calloutUserPublicKey — the public half of the user nkey - the auth-callout responder authenticates as. + services.hyperhive.deploy.nats.calloutUserPublicKey — the + public half of the user nkey the auth-callout responder + authenticates as. It is exempt from callout approval by design, which is exactly why it needs its own credential: a `users` entry with a name @@ -457,11 +474,11 @@ in # boolean is two places for a future edit to land in only one. assertion = responderConfigured; message = '' - services.hyperhive.swarm.nats has callout public keys but no + services.hyperhive.deploy.nats has callout public keys but no seed files: - nats.calloutUserSeedFile = "${cfg.calloutUserSeedFile}" - nats.calloutIssuerSeedFile = "${cfg.calloutIssuerSeedFile}" + deploy.nats.calloutUserSeedFile = "${deployCfg.nats.calloutUserSeedFile}" + deploy.nats.calloutIssuerSeedFile = "${deployCfg.nats.calloutIssuerSeedFile}" Each seed is the private half of the public key already set here — the server verifies with the public half, the responder @@ -469,8 +486,8 @@ in this queue with an auth-callout nobody answers, which refuses every client rather than degrading. - Set both seed paths, or set nats.autoGenerateCallout = true to - have this host mint all four. + Set both seed paths, or set deploy.nats.autoGenerateCallout = true + to have this host mint all four. ''; } { @@ -608,12 +625,12 @@ in # on fails the BUILD of every all-local hive. Upstream's own # description names the case: disable it when the config # includes other files. The check moves to server start. - validateConfig = !cfg.autoGenerateCallout; + validateConfig = !deployCfg.nats.autoGenerateCallout; settings = calloutBlocks { - userKey = cfg.calloutUserPublicKey; - issuerKey = cfg.calloutIssuerPublicKey; + userKey = deployCfg.nats.calloutUserPublicKey; + issuerKey = deployCfg.nats.calloutIssuerPublicKey; } // { # The monitoring endpoint, which is what the exporter below @@ -671,7 +688,7 @@ in # # `mkForce`: upstream defines ExecStart inside an `mkMerge`, so a # plain override conflicts rather than wins. - systemd.services.nats.serviceConfig.ExecStart = lib.mkIf cfg.autoGenerateCallout ( + systemd.services.nats.serviceConfig.ExecStart = lib.mkIf deployCfg.nats.autoGenerateCallout ( lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}" ); @@ -757,7 +774,7 @@ in # nothing to deliver, and a copy that silently succeeds with a stale # or absent seed is worse than not running. after = - lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service" + lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service" # The third credential does not come from the generator above — it is # minted by authelia's FIRST BOOT, inside its own container. Ordering # after that container is necessary and NOT sufficient: the container @@ -765,7 +782,7 @@ in # has finished. The wait in the script is what actually closes it; # this only stops us spinning for the full timeout on every boot. ++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service"; - requires = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service"; + requires = lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service"; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -820,7 +837,7 @@ in # a separate unit that needs the user seed: generating it inside would # trap it there and require a secret-export path back out — the exact # mechanism this is meant to avoid inventing. Only public halves cross. - systemd.services.swarm-nats-callout-keys = lib.mkIf cfg.autoGenerateCallout { + systemd.services.swarm-nats-callout-keys = lib.mkIf deployCfg.nats.autoGenerateCallout { description = "mint the swarm queue's auth-callout nkeys"; before = [ "container@swarm-nats.service" ]; wantedBy = [ "container@swarm-nats.service" ]; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 301ce64c..82326028 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -104,6 +104,20 @@ let swarm.matrix.sso.clientSecretFile = "/etc/matrix/oidc.secret"; }; + # The queue's callout identity, fourth split slice. `autoGenerateCallout` is + # left FALSE on purpose: that is what makes the seed paths the thing deciding + # `responderConfigured`, so the assertion below is about the seeds rather + # than about the auto-mint branch. All five old paths are defined, so + # dropping any single shim entry fails the eval, not just the arms read. + natsOldPath = hive { + deploy.nats.enable = true; + swarm.nats.autoGenerateCallout = false; + swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed"; + swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; + }; + baoPkcs11 = hive { deploy.bao.enable = true; deploy.bao.seal = "pkcs11"; @@ -266,6 +280,19 @@ let builtins.elem httpPort ports && matrixOldPath.containers.hive-matrix.bindMounts ? "/etc/matrix/register.token"; } + { + # Reads the DELIVERY UNIT, not the options: `responderConfigured` gates + # whether it exists at all, and the seed path is interpolated into its + # script. A rename that resolved but stopped reaching the module would + # leave the responder with no credentials and this case would catch it. + name = "a config written against the pre-rename nats paths still delivers the responder's seeds"; + ok = + let + units = natsOldPath.systemd.services; + in + units ? swarm-nats-auth-secrets + && lib.hasInfix "/run/secrets/nats-user.seed" units.swarm-nats-auth-secrets.script; + } { # The gateway's per-name issuer choice. If this ever collapses to a # constant, every swarm-service vhost serves a certificate its CA diff --git a/swarm-nats-auth/src/main.rs b/swarm-nats-auth/src/main.rs index 103ef8d0..a849419c 100644 --- a/swarm-nats-auth/src/main.rs +++ b/swarm-nats-auth/src/main.rs @@ -46,12 +46,12 @@ struct Args { nats_url: String, /// Path to the seed of the callout-exempt user this responder connects as. - /// Its public half is `services.hyperhive.swarm.nats.calloutUserPublicKey`. + /// Its public half is `services.hyperhive.deploy.nats.calloutUserPublicKey`. #[arg(long)] user_seed_file: PathBuf, /// Path to the account signing seed used to sign issued user JWTs. Its - /// public half is `services.hyperhive.swarm.nats.calloutIssuerPublicKey`. + /// public half is `services.hyperhive.deploy.nats.calloutIssuerPublicKey`. #[arg(long)] issuer_seed_file: PathBuf,