fix(#3882): pin the gid that owns the TPM node, on both sides of the boundary

#3880 bound /dev/tpmrm0 into the store's container, which was necessary and
not sufficient: a bind mount preserves the host's ownership, openbao runs as a
DynamicUser with an empty CapabilityBoundingSet (so no CAP_DAC_OVERRIDE), and
the host applies no tpm udev rule at all — measured, 0 hits across all 41 host
closures against 62 rule files as a control. The node therefore keeps the
kernel default and the seal cannot open it.

A name cannot fix this. NixOS allocates system groups at activation, per
machine, so `tss` — or any group declared on both sides — gets two different
ids, and the device node carries the number. mara picked pinning a gid with an
overridable default (deploy.bao.tpmGid).

The default sits above the range NixOS auto-assigns system groups from
(400-999, measured in update-users-groups.pl) and above the normal-user range,
and below systemd's DynamicUser range (61184-65519), so it collides with
nothing any of those allocate.

The module-eval case compares the two sides rather than checking each against
a literal: the property is that they AGREE, not what they agree on. Its
absence arm is a shamir store, which never opens a TPM and must not claim a
device node's group — without it, pinning unconditionally would look identical.
This commit is contained in:
atlas 2026-08-31 23:03:19 +02:00
commit c044a33be9
2 changed files with 88 additions and 4 deletions

View file

@ -89,6 +89,12 @@ let
# hands the store over by group rather than by owner. # hands the store over by group rather than by owner.
tokenGroup = "swarm-bao-token"; tokenGroup = "swarm-bao-token";
# The store's half of the TPM device's ownership. Declared on BOTH sides of
# the container boundary with the same pinned gid: the node is the host's and
# carries a number, while the unit that opens it lives in the container, so a
# name alone resolves to two different groups. See `deploy.bao.tpmGid`.
tpmGroup = "swarm-bao-tpm";
sealSettings = lib.optionalAttrs (baoDeploy.seal == "pkcs11") { sealSettings = lib.optionalAttrs (baoDeploy.seal == "pkcs11") {
seal.pkcs11 = { seal.pkcs11 = {
lib = "${pkgs.tpm2-pkcs11}/lib/libtpm2_pkcs11.so"; lib = "${pkgs.tpm2-pkcs11}/lib/libtpm2_pkcs11.so";
@ -278,6 +284,32 @@ in
''; '';
}; };
tpmGid = lib.mkOption {
type = lib.types.int;
default = 31337;
example = 4242;
description = ''
Numeric group id owning `/dev/tpmrm0`, so the store's seal can open it.
A **number**, and it has a default, because a name cannot do this
job. The store runs as a `DynamicUser` whose uid is allocated inside
its container, and NixOS allocates `tss` and every other system
group at *activation*, per machine. A group declared on the host and
a group of the same name declared in the container therefore get
different ids, and the device node carries the number. Pinning one
value is what makes the two sides agree.
The default sits above the range NixOS auto-assigns system groups from
(400999) and above the normal-user range (100029999), and below the
range systemd allocates `DynamicUser` ids from (6118465519), so it
collides with nothing those allocate. Override it if it collides with
something this module cannot see.
Only read when {option}`services.hyperhive.deploy.bao.seal` is
`pkcs11`; a shamir store never touches the TPM.
'';
};
serverCertFile = lib.mkOption { serverCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str; type = lib.types.nullOr lib.types.str;
default = null; default = null;
@ -607,6 +639,21 @@ in
''; '';
}; };
# The device node is the host's, so its ownership is set here — the
# container can only be given a group that already matches.
#
# ⚠️ If the host also sets `security.tpm2.tssGroup`, both rules match
# `tpmrm*` and the later one wins. That is survivable (either group
# reaches the device) but worth knowing before debugging a 0660 node the
# store still cannot open.
users.groups.${tpmGroup} = lib.mkIf (baoDeploy.seal == "pkcs11") {
gid = baoDeploy.tpmGid;
};
services.udev.extraRules = lib.mkIf (baoDeploy.seal == "pkcs11") ''
KERNEL=="tpmrm[0-9]*", MODE="0660", GROUP="${tpmGroup}"
'';
# The token store is a HOST path (see the bind mount below), so the # The token store is a HOST path (see the bind mount below), so the
# directory has to exist before the container starts — `bindMounts` only # directory has to exist before the container starts — `bindMounts` only
# adds a `RequiresMountsFor`, and nothing in nixos-containers creates a # adds a `RequiresMountsFor`, and nothing in nixos-containers creates a
@ -697,7 +744,14 @@ in
# resolvconf on would let host-tracking regenerate it empty. # resolvconf on would let host-tracking regenerate it empty.
networking.resolvconf.enable = lib.mkForce false; networking.resolvconf.enable = lib.mkForce false;
users.groups = lib.mkIf (baoDeploy.seal == "pkcs11") { ${tokenGroup} = { }; }; # `${tokenGroup}` takes whatever gid this container allocates —
# nothing outside reads it. `${tpmGroup}` must take the PINNED one,
# because the device node it names is the host's and matching is by
# number; letting this side auto-allocate is the whole bug.
users.groups = lib.mkIf (baoDeploy.seal == "pkcs11") {
${tokenGroup} = { };
${tpmGroup}.gid = baoDeploy.tpmGid;
};
# Provisioned here rather than on the host because the store has to # Provisioned here rather than on the host because the store has to
# end up reachable by openbao's `DynamicUser`, and that identity # end up reachable by openbao's `DynamicUser`, and that identity
@ -800,7 +854,14 @@ in
# bind mount read-only to this unit however it is owned, and the # bind mount read-only to this unit however it is owned, and the
# pkcs11 library opens its sqlite store read-write. # pkcs11 library opens its sqlite store read-write.
ReadWritePaths = [ tokenStoreDir ]; ReadWritePaths = [ tokenStoreDir ];
SupplementaryGroups = [ tokenGroup ]; # Two groups, two different jobs: the store's sqlite file, and
# the TPM device the seal opens through it. `CapabilityBounding
# Set=` is empty upstream, so there is no `CAP_DAC_OVERRIDE` to
# fall back on — group membership is the only way in.
SupplementaryGroups = [
tokenGroup
tpmGroup
];
}; };
# ⚠️ Upstream sets `restartIfChanged = false` on this unit, on # ⚠️ Upstream sets `restartIfChanged = false` on this unit, on

View file

@ -231,7 +231,7 @@ let
# read-only to the seal however it is owned, and the group is the only # read-only to the seal however it is owned, and the group is the only
# handle on a uid allocated at start. Dropping either surfaces as a # handle on a uid allocated at start. Dropping either surfaces as a
# pkcs11 error deep in a library, naming neither the mount nor the user. # pkcs11 error deep in a library, naming neither the mount nor the user.
name = "the store's seal may write the token directory, and is in its group"; name = "the store's seal may write the token directory, and is in both its groups";
ok = ok =
let let
sc = (baoUnits baoPkcs11).openbao.serviceConfig; sc = (baoUnits baoPkcs11).openbao.serviceConfig;
@ -240,7 +240,30 @@ let
# key being gone, and a select would abort the whole run with a nix # key being gone, and a select would abort the whole run with a nix
# trace instead of failing this case by name. # trace instead of failing this case by name.
builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ]) builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ])
&& builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ]); && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ])
&& builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]);
}
{
# The device node belongs to the HOST and is matched by NUMBER, while the
# unit that opens it lives in the container — so the two sides holding
# the same gid is the entire mechanism. Letting either side auto-allocate
# renders cleanly, deploys cleanly, and leaves a 0660 node the seal
# cannot open. Compared rather than each checked against a literal: the
# property is that they AGREE, not what they agree on.
name = "the TPM group has the same gid on the host and inside the container";
ok =
let
host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null;
inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null;
in
host != null && host == inner;
}
{
# Absence arm for the case above — a shamir store never opens a TPM, so
# it must not claim a device node's group. Without this, pinning the gid
# unconditionally would look identical.
name = "a shamir store claims no TPM device group";
ok = !(baoShamir.users.groups ? swarm-bao-tpm);
} }
{ {
# The store's mTLS identity is a separate trust domain from both CAs in # The store's mTLS identity is a separate trust domain from both CAs in