8d554477ca
`swarm-bao-agent-pki` carried its own copy of the mtime-vs-start compare, with the same `stat -c %.6Y | tr -d .` read: on a comma-locale host the `-le` errored and, under `seal = "pkcs11"`, openbao was restarted on every run of the unit. The compare moves out of `refresh_unit` into `written_after_start` (0 newer, 1 not newer, 2 unreadable and logged at `err`), and both `refresh_unit` and the agent-pki unit call it. The agent-pki unit cannot call `refresh_unit` itself: it restarts with `restart`, not `try-restart`, runs a fail-closed check of the bundle between the compare and the restart, and under shamir prints the step instead of restarting. It acts only on 0, so an unreadable time skips the restart as `refresh_unit` does. `script-test-refresh-consumer` covers the three return codes in the agent-pki call shape; `module-eval-bao-basics` asserts both seal arms decide on that call and that no `%.6Y` read is left in the unit. Refs #5230
386 lines
16 KiB
Nix
386 lines
16 KiB
Nix
# `checks.module-eval-bao-basics` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
baoNames
|
|
baoSettings
|
|
baoStream
|
|
bridgePorts
|
|
;
|
|
|
|
baoPkcs11 = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.seal = "pkcs11";
|
|
};
|
|
|
|
baoShamir = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.seal = "shamir";
|
|
};
|
|
|
|
baoExplicitCerts = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.serverCertFile = "/etc/pki/bao.pem";
|
|
deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem";
|
|
};
|
|
|
|
inherit (shared) autheliaOnly;
|
|
|
|
baoWithForge = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
};
|
|
|
|
# The store's units live inside its container, so the gates below have to
|
|
# look there rather than at the host's service set.
|
|
baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;
|
|
|
|
tlsDir = "/var/lib/swarm-bao-tls";
|
|
cases = [
|
|
{
|
|
# The store's seal is spread over six gates — the stanza, the
|
|
# provisioning unit, two bind mounts, a device and an EnvironmentFile.
|
|
# Rendering only some of them is the dangerous state: a store that
|
|
# says hardware-backed and seals with a software key, which no
|
|
# assertion can catch because every value is individually valid.
|
|
name = "a shamir store renders no TPM provisioning unit";
|
|
ok = !(baoUnits baoShamir ? swarm-bao-token);
|
|
}
|
|
{
|
|
# Presence control for the case above. Without it, a typo in the
|
|
# option name would satisfy the absence arm forever. The second half is
|
|
# the fix itself: the unit has to run where openbao's `DynamicUser` is
|
|
# allocated, and a host unit writing the same bytes has no name to hand
|
|
# them to.
|
|
name = "a pkcs11 store provisions the token in the container, not on the host";
|
|
ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token);
|
|
}
|
|
{
|
|
# `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn
|
|
# mounts its own /dev and cannot create device nodes, so permission to
|
|
# use a device that was never bound in opens nothing. Neither half
|
|
# fails on its own, which is why they are asserted as a pair.
|
|
name = "a pkcs11 store gets the TPM device bound in, not merely allowed";
|
|
ok =
|
|
let
|
|
c = baoPkcs11.containers.swarm-bao;
|
|
in
|
|
(c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices;
|
|
}
|
|
{
|
|
# The stanza's label and the label the unit creates are two literals that
|
|
# have to name one object, and the mechanism is asserted with them
|
|
# because it is valid only for an RSA key: openbao takes AES-GCM or
|
|
# RSA-OAEP and a TPM 2.0 has neither GCM nor an opinion about which the
|
|
# seal asked for. Every wrong combination renders and deploys, and
|
|
# surfaces as a pkcs11 error at `operator init`.
|
|
name = "the seal asks for the RSA key the provisioning unit creates";
|
|
ok =
|
|
let
|
|
p = (baoSettings baoPkcs11).seal.pkcs11;
|
|
in
|
|
(p.mechanism or "") == "CKM_RSA_PKCS_OAEP"
|
|
&& lib.hasInfix "--algorithm=rsa2048 --key-label=${p.key_label or ""}" (
|
|
(baoUnits baoPkcs11).swarm-bao-token.script or ""
|
|
);
|
|
}
|
|
{
|
|
# `DynamicUser` implies `ProtectSystem=strict`, so the token directory is
|
|
# read-only to the seal however it is owned, and the group is the only
|
|
# handle on a uid allocated at start. Dropping either surfaces as a
|
|
# pkcs11 error deep in a library, naming neither the mount nor the user.
|
|
name = "the store's seal may write the token directory, and is in both its groups";
|
|
ok =
|
|
let
|
|
sc = (baoUnits baoPkcs11).openbao.serviceConfig;
|
|
in
|
|
# `or [ ]` rather than a bare select: the interesting mutation is the
|
|
# key being gone, and a select would abort the whole run with a nix
|
|
# trace instead of failing this case by name.
|
|
builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ])
|
|
&& builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ])
|
|
&& builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]);
|
|
}
|
|
{
|
|
# The device node belongs to the HOST and is matched by NUMBER, while the
|
|
# unit that opens it lives in the container — so the two sides holding
|
|
# the same gid is the entire mechanism. Letting either side auto-allocate
|
|
# renders cleanly, deploys cleanly, and leaves a 0660 node the seal
|
|
# cannot open. Compared rather than each checked against a literal: the
|
|
# property is that they AGREE, not what they agree on.
|
|
name = "the TPM group has the same gid on the host and inside the container";
|
|
ok =
|
|
let
|
|
host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null;
|
|
inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null;
|
|
in
|
|
host != null && host == inner;
|
|
}
|
|
{
|
|
# Absence arm for the case above — a shamir store never opens a TPM, so
|
|
# it must not claim a device node's group. Without this, pinning the gid
|
|
# unconditionally would look identical.
|
|
name = "a shamir store claims no TPM device group";
|
|
ok = !(baoShamir.users.groups ? swarm-bao-tpm);
|
|
}
|
|
{
|
|
# The store's mTLS identity is a separate trust domain from both CAs in
|
|
# this tree, because it must not come from an authority the store will
|
|
# itself distribute. What supplies it is the glue, which mints a CA of
|
|
# the store's own — so an enabled store has all three paths, and if this
|
|
# ever reads null again the store stops coming up on its own.
|
|
name = "a deployed store is given its own certificate, key and client CA";
|
|
ok =
|
|
let
|
|
b = baoPkcs11.services.hyperhive.deploy.bao;
|
|
in
|
|
b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null;
|
|
}
|
|
{
|
|
# Everything the glue sets is `mkDefault`, and this is the case that
|
|
# says so: a deployment whose certificates come from somewhere the glue
|
|
# has never heard of must win. Also the presence control for the case
|
|
# above — a renamed option would read `null` on both and satisfy
|
|
# neither, but only this one names a value.
|
|
name = "an operator's own certificate path beats the glue's default";
|
|
ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem";
|
|
}
|
|
{
|
|
# The name a reader dials has to resolve where the store runs; a
|
|
# multi-host swarm resolves it upstream instead.
|
|
name = "the store's host answers for the store's name";
|
|
ok = builtins.elem "bao.t.local" (baoNames baoPkcs11);
|
|
}
|
|
{
|
|
# What makes that name reachable from inside an agent container, and the
|
|
# single reason it is a stream server rather than a vhost: `ssl_preread`
|
|
# routes on the SNI without decrypting, so the handshake bao completes is
|
|
# still the client's own and the certificate it authenticates by arrives
|
|
# intact. Terminating here would hand the store one identity for the
|
|
# whole swarm.
|
|
name = "the store's host passes connections through without terminating TLS";
|
|
ok =
|
|
let
|
|
s = baoStream baoPkcs11;
|
|
in
|
|
lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s;
|
|
}
|
|
{
|
|
# The address half, and it is bridge-only for a reason a wildcard would
|
|
# hide until deploy: the store already holds `127.0.0.1:8200` in this
|
|
# same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start
|
|
# — taking every hive domain behind the gateway down with it.
|
|
name = "the passthrough listens on the bridge, not on every address";
|
|
ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11);
|
|
}
|
|
{
|
|
# The routing half: the SNI picks the backend and the only name that
|
|
# resolves to one is the store's own. A `default` that pointed anywhere
|
|
# would make this host a relay for whatever name a client invented.
|
|
name = "the passthrough routes only the store's name, to its loopback listener";
|
|
ok =
|
|
let
|
|
s = baoStream baoPkcs11;
|
|
in
|
|
lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s
|
|
&& lib.hasInfix "bao.t.local 127.0.0.1:8200;" s
|
|
&& lib.hasInfix ''default "";'' s;
|
|
}
|
|
{
|
|
# The listener is only half of reachable: the bridge firewall drops
|
|
# everything not named here, and a silent drop is the failure that reads
|
|
# as "the store is down" from inside a container.
|
|
name = "the store's port is open on the bridge where the store runs";
|
|
ok = builtins.elem 8200 (bridgePorts baoPkcs11);
|
|
}
|
|
{
|
|
# Raft refuses to start without it, and says so in a message that names
|
|
# neither the setting nor the stanza.
|
|
name = "the store advertises a cluster address";
|
|
ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or "");
|
|
}
|
|
{
|
|
# The listener trusts the agent CA through a file no cert-auth role
|
|
# names. At least one listener verifies clients, so an empty set cannot
|
|
# pass.
|
|
name = "every client-verifying listener reads the listener-only bundle";
|
|
ok =
|
|
let
|
|
verifying = lib.filter (l: l ? tls_client_ca_file) (
|
|
lib.attrValues (baoSettings baoPkcs11).listener
|
|
);
|
|
in
|
|
verifying != [ ]
|
|
&& lib.all (l: l.tls_client_ca_file == "${tlsDir}/listener-client-ca.pem") verifying;
|
|
}
|
|
{
|
|
# The other half of keeping agents out of host roles: those roles pin
|
|
# the store CA alone. The positive count is the control that the text
|
|
# searched is the one the roles are written in.
|
|
name = "host cert-auth roles pin client-ca.pem and never the listener bundle or the agent CA";
|
|
ok =
|
|
let
|
|
scripts = lib.concatStrings (
|
|
lib.mapAttrsToList (
|
|
n: u: lib.optionalString (lib.hasPrefix "swarm-bao-" n) (u.script or "")
|
|
) baoPkcs11.systemd.services
|
|
);
|
|
in
|
|
lib.hasInfix "certificate=@${tlsDir}/client-ca.pem" scripts
|
|
&& !(lib.hasInfix "certificate=@${tlsDir}/listener-client-ca.pem" scripts)
|
|
&& !(lib.hasInfix "certificate=@${tlsDir}/agent-ca.pem" scripts);
|
|
}
|
|
{
|
|
# Both writers compose the bundle through one function, which refuses
|
|
# a result that does not begin with the store CA and replaces the file
|
|
# only when its bytes change.
|
|
name = "both bundle writers use the one composer, which keeps the store CA first";
|
|
ok =
|
|
let
|
|
s = baoPkcs11.systemd.services;
|
|
composes =
|
|
u:
|
|
lib.hasInfix "compose_listener_bundle() {" u.script
|
|
&& lib.hasInfix "\ncompose_listener_bundle\n" u.script
|
|
&& lib.hasInfix ''cmp -s -n "$(stat -c %s ${tlsDir}/client-ca.pem)" ${tlsDir}/client-ca.pem "$tmp"'' u.script
|
|
&& lib.hasInfix ''cmp -s "$tmp" ${tlsDir}/listener-client-ca.pem'' u.script
|
|
&& lib.hasInfix "mktemp -p ${tlsDir} " u.script;
|
|
in
|
|
s ? swarm-bao-agent-pki && composes s.swarm-bao-certs && composes s.swarm-bao-agent-pki;
|
|
}
|
|
{
|
|
# openbao reads its client-CA file only at start, so picking up the
|
|
# agent CA is a restart: automatic where the store unseals itself,
|
|
# printed where a human has to. The shamir arm is the control. Both
|
|
# decide on the shared locale-independent compare
|
|
# (../host-modules/lib/refresh-consumer.nix, run by
|
|
# `script-test-refresh-consumer`) and act only on its 0.
|
|
name = "the agent PKI unit restarts openbao under pkcs11 and only prints the step under shamir";
|
|
ok =
|
|
let
|
|
restart = ''systemctl --machine="$machine" restart openbao.service'';
|
|
compare = ''
|
|
written_after_start openbao.service ${tlsDir}/listener-client-ca.pem --machine="$machine" || newer=$?
|
|
'';
|
|
p = baoPkcs11.systemd.services.swarm-bao-agent-pki.script;
|
|
sh = baoShamir.systemd.services.swarm-bao-agent-pki.script;
|
|
in
|
|
lib.hasInfix restart p
|
|
&& lib.all (s: lib.hasInfix compare s && lib.hasInfix ''if [ "$newer" != 0 ]; then'' s) [
|
|
p
|
|
sh
|
|
]
|
|
&& !(lib.hasInfix "%.6Y" p)
|
|
&& !(lib.hasInfix restart sh)
|
|
&& lib.hasInfix "# then unseal" sh;
|
|
}
|
|
{
|
|
# The browser UI's listener must not loosen the one every reader dials.
|
|
# `loopback` is named so an empty filter cannot pass.
|
|
name = "every listener but metrics and the UI still requires a client certificate";
|
|
ok =
|
|
let
|
|
l = (baoSettings baoPkcs11).listener;
|
|
api = builtins.removeAttrs l [
|
|
"metrics"
|
|
"ui"
|
|
];
|
|
in
|
|
api ? loopback
|
|
&& lib.all (x: (x.tls_require_and_verify_client_cert or false) == true) (lib.attrValues api);
|
|
}
|
|
{
|
|
# No client certificate on this one, so loopback is all that keeps it
|
|
# away from everything but this netns.
|
|
name = "the UI listener is loopback-only and the UI is served";
|
|
ok =
|
|
let
|
|
s = baoSettings baoPkcs11;
|
|
in
|
|
(s.ui or false) == true
|
|
&& lib.hasPrefix "127.0.0.1:" (s.listener.ui.address or "")
|
|
&& !(s.listener.ui ? tls_require_and_verify_client_cert);
|
|
}
|
|
{
|
|
# The nginx in front of it is the listener's only client and must bind
|
|
# loopback too, and it refuses the endpoints a browser never needs.
|
|
name = "the UI's nginx binds loopback and refuses the unseal and root-generation paths";
|
|
ok =
|
|
let
|
|
v = baoPkcs11.containers.swarm-bao.config.services.nginx.virtualHosts.bao-ui;
|
|
in
|
|
lib.all (x: x.addr == "127.0.0.1") v.listen
|
|
&& (v.locations."~* ^/v1/sys/(unseal|seal|step-down)/?$".return or null) == "403"
|
|
&& (v.locations."~* ^/v1/sys/(rekey|generate-root)".return or null) == "403";
|
|
}
|
|
{
|
|
# A new vhost gets no group gate from `auth_request`; authelia's
|
|
# default policy admits any session. Evaluated on a host that runs
|
|
# authelia and no store, because that is where the rule has to render.
|
|
name = "authelia restricts the store's UI name to admins wherever authelia runs";
|
|
ok =
|
|
let
|
|
rules =
|
|
autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
|
|
in
|
|
!autheliaOnly.services.hyperhive.deploy.bao.enable
|
|
&& builtins.elem {
|
|
domain = "bao-ui.t.local";
|
|
subject = [ "group:admins" ];
|
|
policy = "one_factor";
|
|
} rules;
|
|
}
|
|
{
|
|
# The other half of that rule: the vhost exists, and asks authelia.
|
|
name = "the store's host serves the UI name behind the authelia subrequest";
|
|
ok =
|
|
let
|
|
v = baoPkcs11.services.nginx.virtualHosts."bao-ui.t.local";
|
|
in
|
|
lib.hasInfix "auth_request /__hive_authelia;" (v.locations."/".extraConfig or "")
|
|
&& (v.locations."/".proxyPass or "") == "http://127.0.0.1:8206";
|
|
}
|
|
{
|
|
# The stream passthrough every reader dials rides the same nginx as the
|
|
# UI's vhost. On a host whose only swarm name is the UI, that nginx must
|
|
# not wait on the services leaf, which needs a login to this very store.
|
|
# The control is a store host that also fronts forge: its gateway waits,
|
|
# exactly as it did before the UI existed.
|
|
name = "a store host fronting only the UI does not hold nginx on a store login";
|
|
ok =
|
|
let
|
|
waits =
|
|
m:
|
|
builtins.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy;
|
|
orders =
|
|
m:
|
|
builtins.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before;
|
|
in
|
|
baoPkcs11.services.hyperhive.swarm.localServiceDomains == [ "bao-ui.t.local" ]
|
|
&& baoPkcs11.systemd.services ? hive-gateway-self-signed-cert
|
|
&& !(waits baoPkcs11)
|
|
&& !(orders baoPkcs11)
|
|
&& waits baoWithForge
|
|
&& orders baoWithForge;
|
|
}
|
|
];
|
|
in
|
|
runGroup "bao-basics" cases
|