diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 5921ecc4..490bd360 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -82,6 +82,13 @@ let tokenStoreDir = "/var/lib/swarm-bao-token"; pinEnvFile = "${tokenStoreDir}/pin.env"; + # openbao runs as a `DynamicUser`: its uid is allocated at start by the + # container's PID 1, so nothing can name it ahead of time — not this + # expression, and not a unit on the host. A group is the handle that + # survives that, which is why provisioning happens inside the container and + # hands the store over by group rather than by owner. + tokenGroup = "swarm-bao-token"; + sealSettings = lib.optionalAttrs (baoDeploy.seal == "pkcs11") { seal.pkcs11 = { lib = "${pkgs.tpm2-pkcs11}/lib/libtpm2_pkcs11.so"; @@ -263,8 +270,9 @@ in ⚠️ This is a **declaration**, and nothing at evaluation time can check it: nix runs on the build machine and cannot see the target's TPM. - Saying `pkcs11` on a host without one fails at activation, when the - provisioning unit cannot create the token. That is deliberate — a + Saying `pkcs11` on a host without one fails when the store's container + starts and its provisioning unit cannot create the token — later than + activation, and on the container's journal. That is deliberate — a store that comes up sealed by software while the config says hardware is weaker than it reads, and silently so. ''; @@ -599,53 +607,24 @@ in ''; }; - systemd.services.swarm-bao-token = lib.mkIf (baoDeploy.seal == "pkcs11") { - description = "provision the swarm secret store's TPM-backed PKCS11 token"; + # The token store is a HOST path (see the bind mount below), so the + # directory has to exist before the container starts — `bindMounts` only + # adds a `RequiresMountsFor`, and nothing in nixos-containers creates a + # `hostPath`. Everything else about it is the container's: the identity + # that must own the contents is allocated there. + systemd.services.swarm-bao-token-dir = lib.mkIf (baoDeploy.seal == "pkcs11") { + description = "create the swarm secret store's PKCS11 token directory"; before = [ "container@${cfg.machine}.service" ]; requiredBy = [ "container@${cfg.machine}.service" ]; - path = [ - pkgs.openssl - pkgs.tpm2-pkcs11 - pkgs.tpm2-tools - ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; + # Create-only on purpose. A plain `install -d` re-imposes 0700 root on + # every boot, which locks the store's own service back out of the + # directory the moment the host reboots. script = '' - set -euo pipefail - install -d -m 0700 ${tokenStoreDir} - # Required whenever the store is not at its default location, or the - # library cannot find the token the seal asks for. - export TPM2_PKCS11_STORE=${tokenStoreDir} - - # Absence is the only trigger. `openssl rand` is the same generator - # the grafana admin key uses; the value never passes through a nix - # expression, which would render it world-readable into the store. - for p in so-pin user-pin; do - if [ ! -e ${tokenStoreDir}/$p ]; then - ( umask 077; openssl rand -hex 16 > ${tokenStoreDir}/$p ) - chmod 0400 ${tokenStoreDir}/$p - fi - done - - if [ ! -e ${tokenStoreDir}/tpm2_pkcs11.sqlite3 ]; then - pid=$(tpm2_ptool init --path ${tokenStoreDir} | sed -n 's/.*id: //p') - tpm2_ptool addtoken --path ${tokenStoreDir} --pid="$pid" \ - --label=swarm-bao \ - --sopin="$(cat ${tokenStoreDir}/so-pin)" \ - --userpin="$(cat ${tokenStoreDir}/user-pin)" - # AES rather than RSA on purpose: openbao discussion 1826 - # reports an RSA keypair here yielding duplicate labels, so - # `bao operator init` fails with "got more than 1 key for the - # label" and then CKR_GENERAL_ERROR. The seal supports AES-GCM. - tpm2_ptool addkey --path ${tokenStoreDir} --label=swarm-bao \ - --userpin="$(cat ${tokenStoreDir}/user-pin)" \ - --algorithm=aes256 --key-label=swarm-bao-seal - fi - - ( umask 077; printf 'BAO_HSM_PIN=%s\n' "$(cat ${tokenStoreDir}/user-pin)" > ${pinEnvFile} ) - chmod 0400 ${pinEnvFile} + test -d ${tokenStoreDir} || install -d -m 0700 ${tokenStoreDir} ''; }; @@ -658,10 +637,11 @@ in # rather than a convenience for nginx. privateNetwork = false; - # Only what a HOST unit writes and this container reads crosses the - # boundary. The raft state deliberately does not: `ephemeral = false` - # keeps the container's own /var, systemd owns `${stateDir}` through - # `StateDirectory=`, and binding over it is what breaks the unit. + # Only material an operator has to be able to back up crosses the + # boundary — the store's identity and its seal. The raft state + # deliberately does not: `ephemeral = false` keeps the container's own + # /var, systemd owns `${stateDir}` through `StateDirectory=`, and + # binding over it is what breaks the unit. bindMounts = { # Read-only: `swarm-bao-certs` on the host is the only writer, and # the store has no reason to modify its own identity. @@ -677,6 +657,15 @@ in hostPath = tokenStoreDir; isReadOnly = false; }; + + # The node itself, not just permission to use it: `allowedDevices` + # renders `DeviceAllow=`, the cgroup gate and nothing more, while + # nspawn builds its own /dev and cannot create device nodes. Whether + # the seal's own user may open it is a third question, tracked apart. + "/dev/tpmrm0" = { + hostPath = "/dev/tpmrm0"; + isReadOnly = false; + }; }; # The seal talks to the TPM through the kernel's resource manager, so @@ -708,6 +697,70 @@ in # resolvconf on would let host-tracking regenerate it empty. networking.resolvconf.enable = lib.mkForce false; + users.groups = lib.mkIf (baoDeploy.seal == "pkcs11") { ${tokenGroup} = { }; }; + + # Provisioned here rather than on the host because the store has to + # end up reachable by openbao's `DynamicUser`, and that identity + # only exists inside this container — a host unit can create the + # same files and has no name to hand them to. The store itself + # stays on the host mount: this changes who writes it, not where it + # lives. + systemd.services.swarm-bao-token = lib.mkIf (baoDeploy.seal == "pkcs11") { + description = "provision the swarm secret store's TPM-backed PKCS11 token"; + before = [ "openbao.service" ]; + requiredBy = [ "openbao.service" ]; + path = [ + pkgs.openssl + pkgs.tpm2-pkcs11 + pkgs.tpm2-tools + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + set -euo pipefail + install -d -m 0770 -g ${tokenGroup} ${tokenStoreDir} + # Required whenever the store is not at its default location, or the + # library cannot find the token the seal asks for. + export TPM2_PKCS11_STORE=${tokenStoreDir} + + # Absence is the only trigger. `openssl rand` is the same generator + # the grafana admin key uses; the value never passes through a nix + # expression, which would render it world-readable into the store. + for p in so-pin user-pin; do + if [ ! -e ${tokenStoreDir}/$p ]; then + ( umask 077; openssl rand -hex 16 > ${tokenStoreDir}/$p ) + chmod 0400 ${tokenStoreDir}/$p + fi + done + + if [ ! -e ${tokenStoreDir}/tpm2_pkcs11.sqlite3 ]; then + pid=$(tpm2_ptool init --path ${tokenStoreDir} | sed -n 's/.*id: //p') + tpm2_ptool addtoken --path ${tokenStoreDir} --pid="$pid" \ + --label=swarm-bao \ + --sopin="$(cat ${tokenStoreDir}/so-pin)" \ + --userpin="$(cat ${tokenStoreDir}/user-pin)" + # AES rather than RSA on purpose: openbao discussion 1826 + # reports an RSA keypair here yielding duplicate labels, so + # `bao operator init` fails with "got more than 1 key for the + # label" and then CKR_GENERAL_ERROR. The seal supports AES-GCM. + tpm2_ptool addkey --path ${tokenStoreDir} --label=swarm-bao \ + --userpin="$(cat ${tokenStoreDir}/user-pin)" \ + --algorithm=aes256 --key-label=swarm-bao-seal + fi + + # Outside the branch above on purpose: a store provisioned by an + # earlier version of this module is root-owned and unreadable to + # the seal, and only the pins stay private to root. + chgrp ${tokenGroup} ${tokenStoreDir}/tpm2_pkcs11.sqlite3 + chmod 0660 ${tokenStoreDir}/tpm2_pkcs11.sqlite3 + + ( umask 077; printf 'BAO_HSM_PIN=%s\n' "$(cat ${tokenStoreDir}/user-pin)" > ${pinEnvFile} ) + chmod 0400 ${pinEnvFile} + ''; + }; + services.openbao = { enable = true; package = baoDeploy.package; @@ -743,6 +796,11 @@ in // lib.optionalAttrs (baoDeploy.seal == "pkcs11") { EnvironmentFile = pinEnvFile; Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ]; + # `DynamicUser` implies `ProtectSystem=strict`, which leaves the + # bind mount read-only to this unit however it is owned, and the + # pkcs11 library opens its sqlite store read-write. + ReadWritePaths = [ tokenStoreDir ]; + SupplementaryGroups = [ tokenGroup ]; }; # ⚠️ Upstream sets `restartIfChanged = false` on this unit, on diff --git a/nix/module-eval.nix b/nix/module-eval.nix index a1a27059..cc911b5c 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -129,6 +129,10 @@ let # the daemon's own startup is otherwise the first reader. baoSettings = machine: machine.containers.swarm-bao.config.services.openbao.settings; + # The store's units live inside its container, so the gates below have to + # look there rather than at the host's service set. + baoUnits = machine: machine.containers.swarm-bao.config.systemd.services; + # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is @@ -193,19 +197,50 @@ let ok = forceCiServiceConfigs; } { - # The store's seal is spread over five gates — the stanza, the - # provisioning unit, a bind mount, a device and an EnvironmentFile. + # The store's seal is spread over six gates — the stanza, the + # provisioning unit, two bind mounts, a device and an EnvironmentFile. # Rendering only some of them is the dangerous state: a store that # says hardware-backed and seals with a software key, which no # assertion can catch because every value is individually valid. name = "a shamir store renders no TPM provisioning unit"; - ok = !(baoShamir.systemd.services ? swarm-bao-token); + ok = !(baoUnits baoShamir ? swarm-bao-token); } { # Presence control for the case above. Without it, a typo in the - # option name would satisfy the absence arm forever. - name = "a pkcs11 store renders the TPM provisioning unit"; - ok = baoPkcs11.systemd.services ? swarm-bao-token; + # option name would satisfy the absence arm forever. The second half is + # the fix itself: the unit has to run where openbao's `DynamicUser` is + # allocated, and a host unit writing the same bytes has no name to hand + # them to. + name = "a pkcs11 store provisions the token in the container, not on the host"; + ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token); + } + { + # `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn + # mounts its own /dev and cannot create device nodes, so permission to + # use a device that was never bound in opens nothing. Neither half + # fails on its own, which is why they are asserted as a pair. + name = "a pkcs11 store gets the TPM device bound in, not merely allowed"; + ok = + let + c = baoPkcs11.containers.swarm-bao; + in + (c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices; + } + { + # `DynamicUser` implies `ProtectSystem=strict`, so the token directory is + # read-only to the seal however it is owned, and the group is the only + # handle on a uid allocated at start. Dropping either surfaces as a + # pkcs11 error deep in a library, naming neither the mount nor the user. + name = "the store's seal may write the token directory, and is in its group"; + ok = + let + sc = (baoUnits baoPkcs11).openbao.serviceConfig; + in + # `or [ ]` rather than a bare select: the interesting mutation is the + # key being gone, and a select would abort the whole run with a nix + # trace instead of failing this case by name. + builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ]) + && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ]); } { # The store's mTLS identity is a separate trust domain from both CAs in