From a58c7af3bd9ab66ad5e1a686fa6c770b57fb3026 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 31 Aug 2026 22:33:18 +0200 Subject: [PATCH] fix(#3860): provision the bao pkcs11 token inside its container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit openbao runs as a DynamicUser, so the uid that has to open the token store is allocated by the container's PID 1 and cannot be named by a host unit — the store was created root-owned 0700 and the seal could never read it. The provisioning unit moves inside the container and hands the sqlite store over by group; the host keeps only the mkdir the bind mount needs, create-only so a reboot does not re-impose 0700. Two further layers blocked the same start, both measured while fixing this one: - DynamicUser implies ProtectSystem=strict (systemd.exec(5)), so the bind mount was read-only to openbao however it was owned, and the pkcs11 library opens its store read-write. ReadWritePaths= is required and was absent. - allowedDevices renders DeviceAllow= and nothing else, and nspawn builds its own /dev as a fixed tmpfs and cannot create device nodes — verified against a live container, whose /dev holds no host devices at all. /dev/tpmrm0 was therefore absent inside swarm-bao, not merely unpermitted. It is now bound in. Whether openbao's dynamic uid may *open* that node is a third question: the tss gid is dynamically allocated, so no name or number means the same thing on both sides of the boundary. Filed separately rather than guessed at here. The two module-eval cases that asserted the unit on the host now assert it in the container and absent from the host, and two new cases pin the device bind and the write access — each was individually valid and collectively required, which is the state no assertion catches. Both new cases select with `or [ ]`: mutation-testing them showed that a bare select aborts the run with a nix trace instead of failing the case by name, which also hid the second failure behind the first. --- nix/host-modules/swarm-bao.nix | 150 +++++++++++++++++++++++---------- nix/module-eval.nix | 47 +++++++++-- 2 files changed, 145 insertions(+), 52 deletions(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 5921ecc4..490bd360 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -82,6 +82,13 @@ let tokenStoreDir = "/var/lib/swarm-bao-token"; pinEnvFile = "${tokenStoreDir}/pin.env"; + # openbao runs as a `DynamicUser`: its uid is allocated at start by the + # container's PID 1, so nothing can name it ahead of time — not this + # expression, and not a unit on the host. A group is the handle that + # survives that, which is why provisioning happens inside the container and + # hands the store over by group rather than by owner. + tokenGroup = "swarm-bao-token"; + sealSettings = lib.optionalAttrs (baoDeploy.seal == "pkcs11") { seal.pkcs11 = { lib = "${pkgs.tpm2-pkcs11}/lib/libtpm2_pkcs11.so"; @@ -263,8 +270,9 @@ in ⚠️ This is a **declaration**, and nothing at evaluation time can check it: nix runs on the build machine and cannot see the target's TPM. - Saying `pkcs11` on a host without one fails at activation, when the - provisioning unit cannot create the token. That is deliberate — a + Saying `pkcs11` on a host without one fails when the store's container + starts and its provisioning unit cannot create the token — later than + activation, and on the container's journal. That is deliberate — a store that comes up sealed by software while the config says hardware is weaker than it reads, and silently so. ''; @@ -599,53 +607,24 @@ in ''; }; - systemd.services.swarm-bao-token = lib.mkIf (baoDeploy.seal == "pkcs11") { - description = "provision the swarm secret store's TPM-backed PKCS11 token"; + # The token store is a HOST path (see the bind mount below), so the + # directory has to exist before the container starts — `bindMounts` only + # adds a `RequiresMountsFor`, and nothing in nixos-containers creates a + # `hostPath`. Everything else about it is the container's: the identity + # that must own the contents is allocated there. + systemd.services.swarm-bao-token-dir = lib.mkIf (baoDeploy.seal == "pkcs11") { + description = "create the swarm secret store's PKCS11 token directory"; before = [ "container@${cfg.machine}.service" ]; requiredBy = [ "container@${cfg.machine}.service" ]; - path = [ - pkgs.openssl - pkgs.tpm2-pkcs11 - pkgs.tpm2-tools - ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; + # Create-only on purpose. A plain `install -d` re-imposes 0700 root on + # every boot, which locks the store's own service back out of the + # directory the moment the host reboots. script = '' - set -euo pipefail - install -d -m 0700 ${tokenStoreDir} - # Required whenever the store is not at its default location, or the - # library cannot find the token the seal asks for. - export TPM2_PKCS11_STORE=${tokenStoreDir} - - # Absence is the only trigger. `openssl rand` is the same generator - # the grafana admin key uses; the value never passes through a nix - # expression, which would render it world-readable into the store. - for p in so-pin user-pin; do - if [ ! -e ${tokenStoreDir}/$p ]; then - ( umask 077; openssl rand -hex 16 > ${tokenStoreDir}/$p ) - chmod 0400 ${tokenStoreDir}/$p - fi - done - - if [ ! -e ${tokenStoreDir}/tpm2_pkcs11.sqlite3 ]; then - pid=$(tpm2_ptool init --path ${tokenStoreDir} | sed -n 's/.*id: //p') - tpm2_ptool addtoken --path ${tokenStoreDir} --pid="$pid" \ - --label=swarm-bao \ - --sopin="$(cat ${tokenStoreDir}/so-pin)" \ - --userpin="$(cat ${tokenStoreDir}/user-pin)" - # AES rather than RSA on purpose: openbao discussion 1826 - # reports an RSA keypair here yielding duplicate labels, so - # `bao operator init` fails with "got more than 1 key for the - # label" and then CKR_GENERAL_ERROR. The seal supports AES-GCM. - tpm2_ptool addkey --path ${tokenStoreDir} --label=swarm-bao \ - --userpin="$(cat ${tokenStoreDir}/user-pin)" \ - --algorithm=aes256 --key-label=swarm-bao-seal - fi - - ( umask 077; printf 'BAO_HSM_PIN=%s\n' "$(cat ${tokenStoreDir}/user-pin)" > ${pinEnvFile} ) - chmod 0400 ${pinEnvFile} + test -d ${tokenStoreDir} || install -d -m 0700 ${tokenStoreDir} ''; }; @@ -658,10 +637,11 @@ in # rather than a convenience for nginx. privateNetwork = false; - # Only what a HOST unit writes and this container reads crosses the - # boundary. The raft state deliberately does not: `ephemeral = false` - # keeps the container's own /var, systemd owns `${stateDir}` through - # `StateDirectory=`, and binding over it is what breaks the unit. + # Only material an operator has to be able to back up crosses the + # boundary — the store's identity and its seal. The raft state + # deliberately does not: `ephemeral = false` keeps the container's own + # /var, systemd owns `${stateDir}` through `StateDirectory=`, and + # binding over it is what breaks the unit. bindMounts = { # Read-only: `swarm-bao-certs` on the host is the only writer, and # the store has no reason to modify its own identity. @@ -677,6 +657,15 @@ in hostPath = tokenStoreDir; isReadOnly = false; }; + + # The node itself, not just permission to use it: `allowedDevices` + # renders `DeviceAllow=`, the cgroup gate and nothing more, while + # nspawn builds its own /dev and cannot create device nodes. Whether + # the seal's own user may open it is a third question, tracked apart. + "/dev/tpmrm0" = { + hostPath = "/dev/tpmrm0"; + isReadOnly = false; + }; }; # The seal talks to the TPM through the kernel's resource manager, so @@ -708,6 +697,70 @@ in # resolvconf on would let host-tracking regenerate it empty. networking.resolvconf.enable = lib.mkForce false; + users.groups = lib.mkIf (baoDeploy.seal == "pkcs11") { ${tokenGroup} = { }; }; + + # Provisioned here rather than on the host because the store has to + # end up reachable by openbao's `DynamicUser`, and that identity + # only exists inside this container — a host unit can create the + # same files and has no name to hand them to. The store itself + # stays on the host mount: this changes who writes it, not where it + # lives. + systemd.services.swarm-bao-token = lib.mkIf (baoDeploy.seal == "pkcs11") { + description = "provision the swarm secret store's TPM-backed PKCS11 token"; + before = [ "openbao.service" ]; + requiredBy = [ "openbao.service" ]; + path = [ + pkgs.openssl + pkgs.tpm2-pkcs11 + pkgs.tpm2-tools + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + set -euo pipefail + install -d -m 0770 -g ${tokenGroup} ${tokenStoreDir} + # Required whenever the store is not at its default location, or the + # library cannot find the token the seal asks for. + export TPM2_PKCS11_STORE=${tokenStoreDir} + + # Absence is the only trigger. `openssl rand` is the same generator + # the grafana admin key uses; the value never passes through a nix + # expression, which would render it world-readable into the store. + for p in so-pin user-pin; do + if [ ! -e ${tokenStoreDir}/$p ]; then + ( umask 077; openssl rand -hex 16 > ${tokenStoreDir}/$p ) + chmod 0400 ${tokenStoreDir}/$p + fi + done + + if [ ! -e ${tokenStoreDir}/tpm2_pkcs11.sqlite3 ]; then + pid=$(tpm2_ptool init --path ${tokenStoreDir} | sed -n 's/.*id: //p') + tpm2_ptool addtoken --path ${tokenStoreDir} --pid="$pid" \ + --label=swarm-bao \ + --sopin="$(cat ${tokenStoreDir}/so-pin)" \ + --userpin="$(cat ${tokenStoreDir}/user-pin)" + # AES rather than RSA on purpose: openbao discussion 1826 + # reports an RSA keypair here yielding duplicate labels, so + # `bao operator init` fails with "got more than 1 key for the + # label" and then CKR_GENERAL_ERROR. The seal supports AES-GCM. + tpm2_ptool addkey --path ${tokenStoreDir} --label=swarm-bao \ + --userpin="$(cat ${tokenStoreDir}/user-pin)" \ + --algorithm=aes256 --key-label=swarm-bao-seal + fi + + # Outside the branch above on purpose: a store provisioned by an + # earlier version of this module is root-owned and unreadable to + # the seal, and only the pins stay private to root. + chgrp ${tokenGroup} ${tokenStoreDir}/tpm2_pkcs11.sqlite3 + chmod 0660 ${tokenStoreDir}/tpm2_pkcs11.sqlite3 + + ( umask 077; printf 'BAO_HSM_PIN=%s\n' "$(cat ${tokenStoreDir}/user-pin)" > ${pinEnvFile} ) + chmod 0400 ${pinEnvFile} + ''; + }; + services.openbao = { enable = true; package = baoDeploy.package; @@ -743,6 +796,11 @@ in // lib.optionalAttrs (baoDeploy.seal == "pkcs11") { EnvironmentFile = pinEnvFile; Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ]; + # `DynamicUser` implies `ProtectSystem=strict`, which leaves the + # bind mount read-only to this unit however it is owned, and the + # pkcs11 library opens its sqlite store read-write. + ReadWritePaths = [ tokenStoreDir ]; + SupplementaryGroups = [ tokenGroup ]; }; # ⚠️ Upstream sets `restartIfChanged = false` on this unit, on diff --git a/nix/module-eval.nix b/nix/module-eval.nix index a1a27059..cc911b5c 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -129,6 +129,10 @@ let # the daemon's own startup is otherwise the first reader. baoSettings = machine: machine.containers.swarm-bao.config.services.openbao.settings; + # The store's units live inside its container, so the gates below have to + # look there rather than at the host's service set. + baoUnits = machine: machine.containers.swarm-bao.config.systemd.services; + # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is @@ -193,19 +197,50 @@ let ok = forceCiServiceConfigs; } { - # The store's seal is spread over five gates — the stanza, the - # provisioning unit, a bind mount, a device and an EnvironmentFile. + # The store's seal is spread over six gates — the stanza, the + # provisioning unit, two bind mounts, a device and an EnvironmentFile. # Rendering only some of them is the dangerous state: a store that # says hardware-backed and seals with a software key, which no # assertion can catch because every value is individually valid. name = "a shamir store renders no TPM provisioning unit"; - ok = !(baoShamir.systemd.services ? swarm-bao-token); + ok = !(baoUnits baoShamir ? swarm-bao-token); } { # Presence control for the case above. Without it, a typo in the - # option name would satisfy the absence arm forever. - name = "a pkcs11 store renders the TPM provisioning unit"; - ok = baoPkcs11.systemd.services ? swarm-bao-token; + # option name would satisfy the absence arm forever. The second half is + # the fix itself: the unit has to run where openbao's `DynamicUser` is + # allocated, and a host unit writing the same bytes has no name to hand + # them to. + name = "a pkcs11 store provisions the token in the container, not on the host"; + ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token); + } + { + # `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn + # mounts its own /dev and cannot create device nodes, so permission to + # use a device that was never bound in opens nothing. Neither half + # fails on its own, which is why they are asserted as a pair. + name = "a pkcs11 store gets the TPM device bound in, not merely allowed"; + ok = + let + c = baoPkcs11.containers.swarm-bao; + in + (c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices; + } + { + # `DynamicUser` implies `ProtectSystem=strict`, so the token directory is + # read-only to the seal however it is owned, and the group is the only + # handle on a uid allocated at start. Dropping either surfaces as a + # pkcs11 error deep in a library, naming neither the mount nor the user. + name = "the store's seal may write the token directory, and is in its group"; + ok = + let + sc = (baoUnits baoPkcs11).openbao.serviceConfig; + in + # `or [ ]` rather than a bare select: the interesting mutation is the + # key being gone, and a select would abort the whole run with a nix + # trace instead of failing this case by name. + builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ]) + && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ]); } { # The store's mTLS identity is a separate trust domain from both CAs in