18ed3f5040
Thirty-three module-eval fixtures were a second (or later) full `nixosSystem` evaluation of a config another fixture already evaluated: 16 shapes defined identically in two to seven cluster files, plus duplicates inside agent-queue-bao, core-toggle and swarm-services-switch. Each cluster imports ./lib.nix itself, so nothing was shared between them. nix/module-eval/shared.nix now defines each of those 16 shapes once. nix/checks.nix instantiates it a single time and passes it as `shared` to the 23 clusters that read one, and each cluster rebinds the fixture under its existing local name. Assertions are unchanged. Case text changes only in core-toggle, where two cases built the forge-host fixture inline and now read the file's `forgeHere`, and in two inline `let` bindings (bao-otel-collector, grafana) that now take the shared fixture. Six folds are value-equal rather than textually equal: they drop an explicit `deploy.hive-controller.enable = false` or `deploy.swarm-otel.enable = false` that equals what the option already resolves to for that config. Nothing under nix/host-modules sets either option apart from the `mkDefault allSwarmServices` on swarm-otel, and the only option-priority read there is `domain.highestPrio`. Part of #5171.
376 lines
16 KiB
Nix
376 lines
16 KiB
Nix
# `checks.module-eval-bao-basics` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
baoNames
|
|
baoSettings
|
|
baoStream
|
|
bridgePorts
|
|
;
|
|
|
|
baoPkcs11 = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.seal = "pkcs11";
|
|
};
|
|
|
|
baoShamir = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.seal = "shamir";
|
|
};
|
|
|
|
baoExplicitCerts = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.serverCertFile = "/etc/pki/bao.pem";
|
|
deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem";
|
|
};
|
|
|
|
inherit (shared) autheliaOnly;
|
|
|
|
baoWithForge = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
};
|
|
|
|
# The store's units live inside its container, so the gates below have to
|
|
# look there rather than at the host's service set.
|
|
baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;
|
|
|
|
tlsDir = "/var/lib/swarm-bao-tls";
|
|
cases = [
|
|
{
|
|
# The store's seal is spread over six gates — the stanza, the
|
|
# provisioning unit, two bind mounts, a device and an EnvironmentFile.
|
|
# Rendering only some of them is the dangerous state: a store that
|
|
# says hardware-backed and seals with a software key, which no
|
|
# assertion can catch because every value is individually valid.
|
|
name = "a shamir store renders no TPM provisioning unit";
|
|
ok = !(baoUnits baoShamir ? swarm-bao-token);
|
|
}
|
|
{
|
|
# Presence control for the case above. Without it, a typo in the
|
|
# option name would satisfy the absence arm forever. The second half is
|
|
# the fix itself: the unit has to run where openbao's `DynamicUser` is
|
|
# allocated, and a host unit writing the same bytes has no name to hand
|
|
# them to.
|
|
name = "a pkcs11 store provisions the token in the container, not on the host";
|
|
ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token);
|
|
}
|
|
{
|
|
# `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn
|
|
# mounts its own /dev and cannot create device nodes, so permission to
|
|
# use a device that was never bound in opens nothing. Neither half
|
|
# fails on its own, which is why they are asserted as a pair.
|
|
name = "a pkcs11 store gets the TPM device bound in, not merely allowed";
|
|
ok =
|
|
let
|
|
c = baoPkcs11.containers.swarm-bao;
|
|
in
|
|
(c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices;
|
|
}
|
|
{
|
|
# The stanza's label and the label the unit creates are two literals that
|
|
# have to name one object, and the mechanism is asserted with them
|
|
# because it is valid only for an RSA key: openbao takes AES-GCM or
|
|
# RSA-OAEP and a TPM 2.0 has neither GCM nor an opinion about which the
|
|
# seal asked for. Every wrong combination renders and deploys, and
|
|
# surfaces as a pkcs11 error at `operator init`.
|
|
name = "the seal asks for the RSA key the provisioning unit creates";
|
|
ok =
|
|
let
|
|
p = (baoSettings baoPkcs11).seal.pkcs11;
|
|
in
|
|
(p.mechanism or "") == "CKM_RSA_PKCS_OAEP"
|
|
&& lib.hasInfix "--algorithm=rsa2048 --key-label=${p.key_label or ""}" (
|
|
(baoUnits baoPkcs11).swarm-bao-token.script or ""
|
|
);
|
|
}
|
|
{
|
|
# `DynamicUser` implies `ProtectSystem=strict`, so the token directory is
|
|
# read-only to the seal however it is owned, and the group is the only
|
|
# handle on a uid allocated at start. Dropping either surfaces as a
|
|
# pkcs11 error deep in a library, naming neither the mount nor the user.
|
|
name = "the store's seal may write the token directory, and is in both its groups";
|
|
ok =
|
|
let
|
|
sc = (baoUnits baoPkcs11).openbao.serviceConfig;
|
|
in
|
|
# `or [ ]` rather than a bare select: the interesting mutation is the
|
|
# key being gone, and a select would abort the whole run with a nix
|
|
# trace instead of failing this case by name.
|
|
builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ])
|
|
&& builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ])
|
|
&& builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]);
|
|
}
|
|
{
|
|
# The device node belongs to the HOST and is matched by NUMBER, while the
|
|
# unit that opens it lives in the container — so the two sides holding
|
|
# the same gid is the entire mechanism. Letting either side auto-allocate
|
|
# renders cleanly, deploys cleanly, and leaves a 0660 node the seal
|
|
# cannot open. Compared rather than each checked against a literal: the
|
|
# property is that they AGREE, not what they agree on.
|
|
name = "the TPM group has the same gid on the host and inside the container";
|
|
ok =
|
|
let
|
|
host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null;
|
|
inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null;
|
|
in
|
|
host != null && host == inner;
|
|
}
|
|
{
|
|
# Absence arm for the case above — a shamir store never opens a TPM, so
|
|
# it must not claim a device node's group. Without this, pinning the gid
|
|
# unconditionally would look identical.
|
|
name = "a shamir store claims no TPM device group";
|
|
ok = !(baoShamir.users.groups ? swarm-bao-tpm);
|
|
}
|
|
{
|
|
# The store's mTLS identity is a separate trust domain from both CAs in
|
|
# this tree, because it must not come from an authority the store will
|
|
# itself distribute. What supplies it is the glue, which mints a CA of
|
|
# the store's own — so an enabled store has all three paths, and if this
|
|
# ever reads null again the store stops coming up on its own.
|
|
name = "a deployed store is given its own certificate, key and client CA";
|
|
ok =
|
|
let
|
|
b = baoPkcs11.services.hyperhive.deploy.bao;
|
|
in
|
|
b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null;
|
|
}
|
|
{
|
|
# Everything the glue sets is `mkDefault`, and this is the case that
|
|
# says so: a deployment whose certificates come from somewhere the glue
|
|
# has never heard of must win. Also the presence control for the case
|
|
# above — a renamed option would read `null` on both and satisfy
|
|
# neither, but only this one names a value.
|
|
name = "an operator's own certificate path beats the glue's default";
|
|
ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem";
|
|
}
|
|
{
|
|
# The name a reader dials has to resolve where the store runs; a
|
|
# multi-host swarm resolves it upstream instead.
|
|
name = "the store's host answers for the store's name";
|
|
ok = builtins.elem "bao.t.local" (baoNames baoPkcs11);
|
|
}
|
|
{
|
|
# What makes that name reachable from inside an agent container, and the
|
|
# single reason it is a stream server rather than a vhost: `ssl_preread`
|
|
# routes on the SNI without decrypting, so the handshake bao completes is
|
|
# still the client's own and the certificate it authenticates by arrives
|
|
# intact. Terminating here would hand the store one identity for the
|
|
# whole swarm.
|
|
name = "the store's host passes connections through without terminating TLS";
|
|
ok =
|
|
let
|
|
s = baoStream baoPkcs11;
|
|
in
|
|
lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s;
|
|
}
|
|
{
|
|
# The address half, and it is bridge-only for a reason a wildcard would
|
|
# hide until deploy: the store already holds `127.0.0.1:8200` in this
|
|
# same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start
|
|
# — taking every hive domain behind the gateway down with it.
|
|
name = "the passthrough listens on the bridge, not on every address";
|
|
ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11);
|
|
}
|
|
{
|
|
# The routing half: the SNI picks the backend and the only name that
|
|
# resolves to one is the store's own. A `default` that pointed anywhere
|
|
# would make this host a relay for whatever name a client invented.
|
|
name = "the passthrough routes only the store's name, to its loopback listener";
|
|
ok =
|
|
let
|
|
s = baoStream baoPkcs11;
|
|
in
|
|
lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s
|
|
&& lib.hasInfix "bao.t.local 127.0.0.1:8200;" s
|
|
&& lib.hasInfix ''default "";'' s;
|
|
}
|
|
{
|
|
# The listener is only half of reachable: the bridge firewall drops
|
|
# everything not named here, and a silent drop is the failure that reads
|
|
# as "the store is down" from inside a container.
|
|
name = "the store's port is open on the bridge where the store runs";
|
|
ok = builtins.elem 8200 (bridgePorts baoPkcs11);
|
|
}
|
|
{
|
|
# Raft refuses to start without it, and says so in a message that names
|
|
# neither the setting nor the stanza.
|
|
name = "the store advertises a cluster address";
|
|
ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or "");
|
|
}
|
|
{
|
|
# The listener trusts the agent CA through a file no cert-auth role
|
|
# names. At least one listener verifies clients, so an empty set cannot
|
|
# pass.
|
|
name = "every client-verifying listener reads the listener-only bundle";
|
|
ok =
|
|
let
|
|
verifying = lib.filter (l: l ? tls_client_ca_file) (
|
|
lib.attrValues (baoSettings baoPkcs11).listener
|
|
);
|
|
in
|
|
verifying != [ ]
|
|
&& lib.all (l: l.tls_client_ca_file == "${tlsDir}/listener-client-ca.pem") verifying;
|
|
}
|
|
{
|
|
# The other half of keeping agents out of host roles: those roles pin
|
|
# the store CA alone. The positive count is the control that the text
|
|
# searched is the one the roles are written in.
|
|
name = "host cert-auth roles pin client-ca.pem and never the listener bundle or the agent CA";
|
|
ok =
|
|
let
|
|
scripts = lib.concatStrings (
|
|
lib.mapAttrsToList (
|
|
n: u: lib.optionalString (lib.hasPrefix "swarm-bao-" n) (u.script or "")
|
|
) baoPkcs11.systemd.services
|
|
);
|
|
in
|
|
lib.hasInfix "certificate=@${tlsDir}/client-ca.pem" scripts
|
|
&& !(lib.hasInfix "certificate=@${tlsDir}/listener-client-ca.pem" scripts)
|
|
&& !(lib.hasInfix "certificate=@${tlsDir}/agent-ca.pem" scripts);
|
|
}
|
|
{
|
|
# Both writers compose the bundle through one function, which refuses
|
|
# a result that does not begin with the store CA and replaces the file
|
|
# only when its bytes change.
|
|
name = "both bundle writers use the one composer, which keeps the store CA first";
|
|
ok =
|
|
let
|
|
s = baoPkcs11.systemd.services;
|
|
composes =
|
|
u:
|
|
lib.hasInfix "compose_listener_bundle() {" u.script
|
|
&& lib.hasInfix "\ncompose_listener_bundle\n" u.script
|
|
&& lib.hasInfix ''cmp -s -n "$(stat -c %s ${tlsDir}/client-ca.pem)" ${tlsDir}/client-ca.pem "$tmp"'' u.script
|
|
&& lib.hasInfix ''cmp -s "$tmp" ${tlsDir}/listener-client-ca.pem'' u.script
|
|
&& lib.hasInfix "mktemp -p ${tlsDir} " u.script;
|
|
in
|
|
s ? swarm-bao-agent-pki && composes s.swarm-bao-certs && composes s.swarm-bao-agent-pki;
|
|
}
|
|
{
|
|
# openbao reads its client-CA file only at start, so picking up the
|
|
# agent CA is a restart: automatic where the store unseals itself,
|
|
# printed where a human has to. The shamir arm is the control.
|
|
name = "the agent PKI unit restarts openbao under pkcs11 and only prints the step under shamir";
|
|
ok =
|
|
let
|
|
restart = ''systemctl --machine="$machine" restart openbao.service'';
|
|
p = baoPkcs11.systemd.services.swarm-bao-agent-pki.script;
|
|
sh = baoShamir.systemd.services.swarm-bao-agent-pki.script;
|
|
in
|
|
lib.hasInfix restart p
|
|
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' p
|
|
&& !(lib.hasInfix restart sh)
|
|
&& lib.hasInfix "# then unseal" sh;
|
|
}
|
|
{
|
|
# The browser UI's listener must not loosen the one every reader dials.
|
|
# `loopback` is named so an empty filter cannot pass.
|
|
name = "every listener but metrics and the UI still requires a client certificate";
|
|
ok =
|
|
let
|
|
l = (baoSettings baoPkcs11).listener;
|
|
api = builtins.removeAttrs l [
|
|
"metrics"
|
|
"ui"
|
|
];
|
|
in
|
|
api ? loopback
|
|
&& lib.all (x: (x.tls_require_and_verify_client_cert or false) == true) (lib.attrValues api);
|
|
}
|
|
{
|
|
# No client certificate on this one, so loopback is all that keeps it
|
|
# away from everything but this netns.
|
|
name = "the UI listener is loopback-only and the UI is served";
|
|
ok =
|
|
let
|
|
s = baoSettings baoPkcs11;
|
|
in
|
|
(s.ui or false) == true
|
|
&& lib.hasPrefix "127.0.0.1:" (s.listener.ui.address or "")
|
|
&& !(s.listener.ui ? tls_require_and_verify_client_cert);
|
|
}
|
|
{
|
|
# The nginx in front of it is the listener's only client and must bind
|
|
# loopback too, and it refuses the endpoints a browser never needs.
|
|
name = "the UI's nginx binds loopback and refuses the unseal and root-generation paths";
|
|
ok =
|
|
let
|
|
v = baoPkcs11.containers.swarm-bao.config.services.nginx.virtualHosts.bao-ui;
|
|
in
|
|
lib.all (x: x.addr == "127.0.0.1") v.listen
|
|
&& (v.locations."~* ^/v1/sys/(unseal|seal|step-down)/?$".return or null) == "403"
|
|
&& (v.locations."~* ^/v1/sys/(rekey|generate-root)".return or null) == "403";
|
|
}
|
|
{
|
|
# A new vhost gets no group gate from `auth_request`; authelia's
|
|
# default policy admits any session. Evaluated on a host that runs
|
|
# authelia and no store, because that is where the rule has to render.
|
|
name = "authelia restricts the store's UI name to admins wherever authelia runs";
|
|
ok =
|
|
let
|
|
rules =
|
|
autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
|
|
in
|
|
!autheliaOnly.services.hyperhive.deploy.bao.enable
|
|
&& builtins.elem {
|
|
domain = "bao-ui.t.local";
|
|
subject = [ "group:admins" ];
|
|
policy = "one_factor";
|
|
} rules;
|
|
}
|
|
{
|
|
# The other half of that rule: the vhost exists, and asks authelia.
|
|
name = "the store's host serves the UI name behind the authelia subrequest";
|
|
ok =
|
|
let
|
|
v = baoPkcs11.services.nginx.virtualHosts."bao-ui.t.local";
|
|
in
|
|
lib.hasInfix "auth_request /__hive_authelia;" (v.locations."/".extraConfig or "")
|
|
&& (v.locations."/".proxyPass or "") == "http://127.0.0.1:8206";
|
|
}
|
|
{
|
|
# The stream passthrough every reader dials rides the same nginx as the
|
|
# UI's vhost. On a host whose only swarm name is the UI, that nginx must
|
|
# not wait on the services leaf, which needs a login to this very store.
|
|
# The control is a store host that also fronts forge: its gateway waits,
|
|
# exactly as it did before the UI existed.
|
|
name = "a store host fronting only the UI does not hold nginx on a store login";
|
|
ok =
|
|
let
|
|
waits =
|
|
m:
|
|
builtins.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy;
|
|
orders =
|
|
m:
|
|
builtins.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before;
|
|
in
|
|
baoPkcs11.services.hyperhive.swarm.localServiceDomains == [ "bao-ui.t.local" ]
|
|
&& baoPkcs11.systemd.services ? hive-gateway-self-signed-cert
|
|
&& !(waits baoPkcs11)
|
|
&& !(orders baoPkcs11)
|
|
&& waits baoWithForge
|
|
&& orders baoWithForge;
|
|
}
|
|
];
|
|
in
|
|
runGroup "bao-basics" cases
|