From c044a33be904931ef2b5398973bb153077fbc0c6 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 31 Aug 2026 23:03:19 +0200 Subject: [PATCH] fix(#3882): pin the gid that owns the TPM node, on both sides of the boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #3880 bound /dev/tpmrm0 into the store's container, which was necessary and not sufficient: a bind mount preserves the host's ownership, openbao runs as a DynamicUser with an empty CapabilityBoundingSet (so no CAP_DAC_OVERRIDE), and the host applies no tpm udev rule at all — measured, 0 hits across all 41 host closures against 62 rule files as a control. The node therefore keeps the kernel default and the seal cannot open it. A name cannot fix this. NixOS allocates system groups at activation, per machine, so `tss` — or any group declared on both sides — gets two different ids, and the device node carries the number. mara picked pinning a gid with an overridable default (deploy.bao.tpmGid). The default sits above the range NixOS auto-assigns system groups from (400-999, measured in update-users-groups.pl) and above the normal-user range, and below systemd's DynamicUser range (61184-65519), so it collides with nothing any of those allocate. The module-eval case compares the two sides rather than checking each against a literal: the property is that they AGREE, not what they agree on. Its absence arm is a shamir store, which never opens a TPM and must not claim a device node's group — without it, pinning unconditionally would look identical. --- nix/host-modules/swarm-bao.nix | 65 ++++++++++++++++++++++++++++++++-- nix/module-eval.nix | 27 ++++++++++++-- 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 490bd360..46347a86 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -89,6 +89,12 @@ let # hands the store over by group rather than by owner. tokenGroup = "swarm-bao-token"; + # The store's half of the TPM device's ownership. Declared on BOTH sides of + # the container boundary with the same pinned gid: the node is the host's and + # carries a number, while the unit that opens it lives in the container, so a + # name alone resolves to two different groups. See `deploy.bao.tpmGid`. + tpmGroup = "swarm-bao-tpm"; + sealSettings = lib.optionalAttrs (baoDeploy.seal == "pkcs11") { seal.pkcs11 = { lib = "${pkgs.tpm2-pkcs11}/lib/libtpm2_pkcs11.so"; @@ -278,6 +284,32 @@ in ''; }; + tpmGid = lib.mkOption { + type = lib.types.int; + default = 31337; + example = 4242; + description = '' + Numeric group id owning `/dev/tpmrm0`, so the store's seal can open it. + + ⚠️ A **number**, and it has a default, because a name cannot do this + job. The store runs as a `DynamicUser` whose uid is allocated inside + its container, and NixOS allocates `tss` — and every other system + group — at *activation*, per machine. A group declared on the host and + a group of the same name declared in the container therefore get + different ids, and the device node carries the number. Pinning one + value is what makes the two sides agree. + + The default sits above the range NixOS auto-assigns system groups from + (400–999) and above the normal-user range (1000–29999), and below the + range systemd allocates `DynamicUser` ids from (61184–65519), so it + collides with nothing those allocate. Override it if it collides with + something this module cannot see. + + Only read when {option}`services.hyperhive.deploy.bao.seal` is + `pkcs11`; a shamir store never touches the TPM. + ''; + }; + serverCertFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -607,6 +639,21 @@ in ''; }; + # The device node is the host's, so its ownership is set here — the + # container can only be given a group that already matches. + # + # ⚠️ If the host also sets `security.tpm2.tssGroup`, both rules match + # `tpmrm*` and the later one wins. That is survivable (either group + # reaches the device) but worth knowing before debugging a 0660 node the + # store still cannot open. + users.groups.${tpmGroup} = lib.mkIf (baoDeploy.seal == "pkcs11") { + gid = baoDeploy.tpmGid; + }; + + services.udev.extraRules = lib.mkIf (baoDeploy.seal == "pkcs11") '' + KERNEL=="tpmrm[0-9]*", MODE="0660", GROUP="${tpmGroup}" + ''; + # The token store is a HOST path (see the bind mount below), so the # directory has to exist before the container starts — `bindMounts` only # adds a `RequiresMountsFor`, and nothing in nixos-containers creates a @@ -697,7 +744,14 @@ in # resolvconf on would let host-tracking regenerate it empty. networking.resolvconf.enable = lib.mkForce false; - users.groups = lib.mkIf (baoDeploy.seal == "pkcs11") { ${tokenGroup} = { }; }; + # `${tokenGroup}` takes whatever gid this container allocates — + # nothing outside reads it. `${tpmGroup}` must take the PINNED one, + # because the device node it names is the host's and matching is by + # number; letting this side auto-allocate is the whole bug. + users.groups = lib.mkIf (baoDeploy.seal == "pkcs11") { + ${tokenGroup} = { }; + ${tpmGroup}.gid = baoDeploy.tpmGid; + }; # Provisioned here rather than on the host because the store has to # end up reachable by openbao's `DynamicUser`, and that identity @@ -800,7 +854,14 @@ in # bind mount read-only to this unit however it is owned, and the # pkcs11 library opens its sqlite store read-write. ReadWritePaths = [ tokenStoreDir ]; - SupplementaryGroups = [ tokenGroup ]; + # Two groups, two different jobs: the store's sqlite file, and + # the TPM device the seal opens through it. `CapabilityBounding + # Set=` is empty upstream, so there is no `CAP_DAC_OVERRIDE` to + # fall back on — group membership is the only way in. + SupplementaryGroups = [ + tokenGroup + tpmGroup + ]; }; # ⚠️ Upstream sets `restartIfChanged = false` on this unit, on diff --git a/nix/module-eval.nix b/nix/module-eval.nix index cc911b5c..5ee51e0c 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -231,7 +231,7 @@ let # read-only to the seal however it is owned, and the group is the only # handle on a uid allocated at start. Dropping either surfaces as a # pkcs11 error deep in a library, naming neither the mount nor the user. - name = "the store's seal may write the token directory, and is in its group"; + name = "the store's seal may write the token directory, and is in both its groups"; ok = let sc = (baoUnits baoPkcs11).openbao.serviceConfig; @@ -240,7 +240,30 @@ let # key being gone, and a select would abort the whole run with a nix # trace instead of failing this case by name. builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ]) - && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ]); + && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ]) + && builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]); + } + { + # The device node belongs to the HOST and is matched by NUMBER, while the + # unit that opens it lives in the container — so the two sides holding + # the same gid is the entire mechanism. Letting either side auto-allocate + # renders cleanly, deploys cleanly, and leaves a 0660 node the seal + # cannot open. Compared rather than each checked against a literal: the + # property is that they AGREE, not what they agree on. + name = "the TPM group has the same gid on the host and inside the container"; + ok = + let + host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null; + inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null; + in + host != null && host == inner; + } + { + # Absence arm for the case above — a shamir store never opens a TPM, so + # it must not claim a device node's group. Without this, pinning the gid + # unconditionally would look identical. + name = "a shamir store claims no TPM device group"; + ok = !(baoShamir.users.groups ? swarm-bao-tpm); } { # The store's mTLS identity is a separate trust domain from both CAs in