diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index d9ed8795..d046f5b9 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -55,7 +55,7 @@ neither is a renaming of the other. | authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | | wireguard private key | **the operator** — `wg genkey` | whatever `swarm.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | | queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `nats.calloutUserSeedFile` / `nats.calloutIssuerSeedFile` | -| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a *client* of it and holds none of this | +| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | `/var/lib/swarm-bao` on the host of whoever runs the store, bind-mounted into the `swarm-bao` container | n/a — there is one store; a hive elsewhere is a *client* of it and holds none of this | | the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it is stated rather than inferred | n/a — only the host running the store seals anything | The three keys authelia mints for itself are generated in-container precisely diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 501569f8..cdad7b38 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -39,34 +39,13 @@ let networkCfg = hyperhiveCfg.network; swarmDomain = hyperhiveCfg.swarm.domain; - # Upstream's own default, kept so its documentation matches. The raft data - # lives INSIDE the container on `ephemeral = false`, the same way - # ./swarm-grafana.nix keeps its sqlite database — no sibling service binds - # its state out to the host. - # - # ⚠️ Do not bind-mount this path. Upstream pairs `StateDirectory=` with - # `DynamicUser=`, which makes systemd hold the state at - # `/var/lib/private/openbao` and symlink this to it; that relocation is a - # rename, and a rename of an active mount point fails `EBUSY` at - # `STATE_DIRECTORY` — the unit then dies before `bao` runs at all. + # Two names for one location. `stateDir` is where openbao writes inside the + # container — upstream's own default, kept so its documentation matches. The + # host path is a sibling of the other swarm services' state rather than a + # path inside the container's tree, so `nixos-container destroy` cannot take + # the swarm's secrets with it. stateDir = "/var/lib/openbao"; - - # The TLS material is its own small bind, on both sides of the boundary at - # the same path. Separate from `stateDir` because a HOST unit writes it and - # the container only reads it, and because systemd owns `stateDir` and moves - # it around; this directory is ours. - tlsDir = "/var/lib/swarm-bao-tls"; - - # Where the key surfaces for openbao to open. NOT `${tlsDir}/server-key.pem`: - # that file is root-owned 0600 on the host, and the service runs as a - # `DynamicUser`, so it cannot read the bind-mounted original. `LoadCredential` - # is systemd's answer to exactly this — PID 1 reads the source as root and - # re-exposes it inside the unit, owned by the service's own account. - # `$CREDENTIALS_DIRECTORY` is `/run/credentials/`, and the config file - # is rendered ahead of time, so the path is spelled out rather than read from - # the environment. - serverKeyCredential = "server-key"; - serverKeyCredentialPath = "/run/credentials/openbao.service/${serverKeyCredential}"; + hostStateDir = "/var/lib/swarm-bao"; # The PIN is deliberately absent here. It arrives as `BAO_HSM_PIN` from an # EnvironmentFile the provisioning unit writes, because a value interpolated @@ -76,9 +55,8 @@ let # empty is the difference between a store that unseals itself and one that # needs a human after every restart. # The PKCS11 token store and its PINs live on the HOST and are bind-mounted - # in. Losing them loses the sealed store — the raft data is worth nothing - # without the key that unseals it — so unlike the state directory these are - # deliberately a host-level fact an operator can back up. + # in. Losing them loses the sealed store, so they are a host-level fact an + # operator can back up — the same reasoning as the raft data below. tokenStoreDir = "/var/lib/swarm-bao-token"; pinEnvFile = "${tokenStoreDir}/pin.env"; @@ -95,15 +73,12 @@ let # sees, not a coercion error from here. domainBase = if swarmDomain == null then "invalid" else swarmDomain; - # Where the leaf lands for openbao to read. `tlsDir` is bind-mounted at the - # same path on both sides, so the delivery below needs no second mount, and - # nothing has to bind `deploy.hive-controller.tls.stateDir`, which holds the - # hive CA's private key. - # - # The certificate and the client CA are public material and are read straight - # off the mount; only the key takes the credential path above. - serverCertPath = "${tlsDir}/server.pem"; - serverKeyPath = "${tlsDir}/server-key.pem"; + # Where the leaf lands for openbao to read. Inside the container, because + # `hostStateDir` is already bind-mounted at `stateDir` — so the delivery + # below needs no second mount, and nothing has to bind + # `deploy.hive-controller.tls.stateDir`, which holds the hive CA's private key. + serverCertPath = "${stateDir}/server.pem"; + serverKeyPath = "${stateDir}/server-key.pem"; # The host-side sources, verbatim from the options — no fallback, because a # fallback is exactly the CA opinion this module must not hold. The units @@ -124,14 +99,14 @@ let # has said what to trust yet". listenerTls = { tls_cert_file = serverCertPath; - tls_key_file = serverKeyCredentialPath; + tls_key_file = serverKeyPath; } // lib.optionalAttrs (baoDeploy.clientCaFile != null) { tls_client_ca_file = clientCaPath; tls_require_and_verify_client_cert = true; }; - clientCaPath = "${tlsDir}/client-ca.pem"; + clientCaPath = "${stateDir}/client-ca.pem"; extraListeners = lib.listToAttrs ( lib.imap1 ( @@ -390,8 +365,8 @@ in # the container is ordered. Same trap `hostClientSecretDir` # documents in ./swarm-authelia.nix. Ordering against whatever # mints it belongs with whatever named the path, not here. - # 2. `tlsDir` is already mounted at the same path inside, so a copy - # needs no second mount. + # 2. `hostStateDir` is already mounted at `stateDir`, so a copy needs + # no second mount. # 3. A directory holding a leaf usually holds the CA's private key # beside it. Binding that directory to reach one file inside it # would hand the container authority to mint any name that CA can — @@ -407,11 +382,7 @@ in }; script = '' set -euo pipefail - # 0755, not 0700: the container reads the certificate and the client - # CA straight off this mount as a non-root user, so it has to be able - # to traverse the directory. The key inside stays 0600 and reaches - # the service through `LoadCredential` instead. - install -d -m 0755 ${tlsDir} + install -d -m 0700 ${hostStateDir} # Fail loudly rather than start a store that cannot serve. The path # is configured, so a missing file means whatever was supposed to @@ -426,8 +397,8 @@ in fi done - install -m 0644 ${lib.escapeShellArg serverCertSrc} ${tlsDir}/server.pem - install -m 0600 ${lib.escapeShellArg serverKeySrc} ${tlsDir}/server-key.pem + install -m 0644 ${lib.escapeShellArg serverCertSrc} ${hostStateDir}/server.pem + install -m 0600 ${lib.escapeShellArg serverKeySrc} ${hostStateDir}/server-key.pem '' + lib.optionalString (baoDeploy.clientCaFile != null) '' @@ -435,7 +406,7 @@ in echo "deploy.bao.clientCaFile names ${baoDeploy.clientCaFile}, which is missing or empty." >&2 exit 1 fi - install -m 0644 ${lib.escapeShellArg baoDeploy.clientCaFile} ${tlsDir}/client-ca.pem + install -m 0644 ${lib.escapeShellArg baoDeploy.clientCaFile} ${hostStateDir}/client-ca.pem ''; }; @@ -498,16 +469,15 @@ in # rather than a convenience for nginx. privateNetwork = false; - # Only what a HOST unit writes and this container reads crosses the - # boundary. The raft state deliberately does not: `ephemeral = false` - # keeps the container's own /var, systemd owns `${stateDir}` through - # `StateDirectory=`, and binding over it is what breaks the unit. + # Raft state outlives the container. `ephemeral = false` keeps the + # container's own /var, but a bind makes the store's data a host-level + # fact an operator can back up and a `nixos-container destroy` cannot + # take with it — which for the swarm's secrets is the difference between + # a rebuild and an outage. bindMounts = { - # Read-only: `swarm-bao-certs` on the host is the only writer, and - # the store has no reason to modify its own identity. - ${tlsDir} = { - hostPath = tlsDir; - isReadOnly = true; + ${stateDir} = { + hostPath = hostStateDir; + isReadOnly = false; }; } // lib.optionalAttrs (baoDeploy.seal == "pkcs11") { @@ -558,30 +528,15 @@ in // sealSettings; }; - # The private key crosses the user boundary here, not on the mount. - # `swarm-bao-certs` installs it 0600 root-owned, and the unit runs - # as a `DynamicUser`, so the bind-mounted file is unreadable to it — - # PID 1 opens the source as root and re-exposes it under - # `${serverKeyCredentialPath}`, owned by the service's own account. - # - # One assignment, not two: `serviceConfig.X = …` beside a - # `serviceConfig = …` is a duplicate attribute inside a single - # attrset literal and does not parse. Module merging happens across - # `config` blocks, not within a literal — so the seal's half joins - # with `optionalAttrs`. - systemd.services.openbao.serviceConfig = - lib.optionalAttrs haveServerTls { - LoadCredential = [ "${serverKeyCredential}:${serverKeyPath}" ]; - } - # The PIN reaches openbao as an environment variable read from a - # 0400 file the provisioning unit wrote — never as a value in this - # expression, which would render it world-readable into the store. - # `TPM2_PKCS11_STORE` is required because the store is not at the - # library's default location. - // lib.optionalAttrs (baoDeploy.seal == "pkcs11") { - EnvironmentFile = pinEnvFile; - Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ]; - }; + # The PIN reaches openbao as an environment variable read from a + # 0400 file the provisioning unit wrote — never as a value in this + # expression, which would render it world-readable into the store. + # `TPM2_PKCS11_STORE` is required because the store is not at the + # library's default location. + systemd.services.openbao.serviceConfig = lib.mkIf (baoDeploy.seal == "pkcs11") { + EnvironmentFile = pinEnvFile; + Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ]; + }; # ⚠️ Upstream sets `restartIfChanged = false` on this unit, on # purpose: a restart SEALS the store and disconnects every client.